virgo-modules 0.2.10__py3-none-any.whl → 0.3.0__py3-none-any.whl

virgo_modules/src/edge_utils/conformal_utils.py

@@ -0,0 +1,71 @@
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from mapie.classification import MapieClassifier
+from sklearn.pipeline import Pipeline
+import mlflow
+import numpy as np
+
+
+def get_conformal_classifiers(model, data, targets):
+    classfiers = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        seg_model = Pipeline([ 
+            ('pipe',model['pipe_transform']),
+            ('model',model['model'].estimators_[i])
+        ])
+        mapie_class = MapieClassifier(seg_model, cv='prefit', random_state=123, method="lac")
+        mapie_class.fit(data, data[targets[i]].values)
+        classfiers.append(mapie_class)
+    return classfiers
+
+def log_confmodels(runid, classifiers):
+    with mlflow.start_run(run_id=runid) as run:
+        for i,classifier in enumerate(classifiers):
+            mlflow.sklearn.log_model(classifier,f"conformal/conformal_model-{i}")
+        print('models were logged')
+
+def load_confmodel(runid, target_variables):
+    classifiers = list()
+    for i in range(len(target_variables)):
+        folder = f"conformal/conformal_model-{i}"
+        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
+        classifiers.append(model)
+    return classifiers
+
+
+def get_conformal_prediction(classifier, alphas, data, prefix='conf'):
+    _, y_pis = classifier.predict(data, alpha=alphas)
+    for i,alpha in enumerate(alphas):
+        data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
+        data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
+    return data
+
+def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
+    ### corect labels ####
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(color = 'blue'),showlegend=True,legendgroup='go up', name='go up'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),showlegend=True,legendgroup='go down',name='go down'),secondary_y=True)
+    for i,alpha in enumerate(alphas, start=1):
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_up' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_up + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'blue')
+                                     ,showlegend=False,legendgroup='go up',name='go up', text=df_[col_alpha],textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_down' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_down + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'coral')
+                                     ,showlegend=False,legendgroup='go down', name='go down',text=df_[col_alpha].astype(str),textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"))
+    fig.update_layout(title_text="sirius - edge probabilities conformal",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

virgo_modules/src/{edge_utils.py → edge_utils/edge_utils.py}

@@ -9,7 +9,10 @@ from feature_engine.imputation import  MeanMedianImputer
 from feature_engine.discretisation import EqualWidthDiscretiser
 from feature_engine.datetime import DatetimeFeatures
 
-from .transformer_utils import VirgoWinsorizerFeature, InverseHyperbolicSine, FeaturesEntropy, FeatureSelector
+from ..transformer_utils import VirgoWinsorizerFeature, InverseHyperbolicSine, FeaturesEntropy, FeatureSelector
+
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
 
 class produce_model_wrapper:
     """
@@ -386,4 +389,29 @@ class ExpandingMultipleTimeSeriesKFold:
         -------
         number_window (int): number of splits
         """
-        return self.number_window
+        return self.number_window
+    
+def edge_probas_lines(data, threshold, plot = False, look_back = 750):
+    """
+    produce a plotly plot of edges and closing prices
+
+            Parameters:
+                    data (pd.DataFrame): asset data with edge probabilities
+                    plot (boolean): if true, display plot
+                    threshold (float): edge threshold
+                    look_back (int): number of rows back to display
+
+            Returns:
+                    fig (obj): plotly go object
+    """
+    df = data[['Date','Close','proba_target_down','proba_target_up']].iloc[-look_back:]
+    
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),name='go down'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(opacity=0.1,size=80), name='go up'),secondary_y=True)
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"),)
+    fig.update_layout(title_text="sirius - edge probabilities",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

virgo_modules/src/edge_utils/shap_utils.py

@@ -0,0 +1,81 @@
+import shap
+import mlflow
+import pandas as pd
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+
+def get_explainers(model, data):
+    explainers = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        transf_data = model['pipe_transform'].transform(data)
+        predictor = model['model'].estimators_[i]
+        explainer= shap.Explainer(predictor, transf_data)
+        explainers.append(explainer)
+    return explainers
+
+def log_explainer(runid, classifiers):
+    with mlflow.start_run(run_id=runid) as run:
+        for i,classifier in enumerate(classifiers):
+            mlflow.sklearn.log_model(classifier,f"explainer/explainer-{i}")
+        print('models were logged')
+
+def load_explainer(runid, target_variables):
+    explainers = list()
+    for i in range(len(target_variables)):
+        folder = f"explainer/explainer-{i}"
+        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
+        explainers.append(model)
+    return explainers
+
+def get_shapvalues(explainers, data):
+    shap_values = {}
+    for i,explainer in enumerate(explainers):
+        shap_value_i = explainer(data)
+        shap_values[i] = shap_value_i
+    return shap_values
+
+def get_explainerclusters(model, data, targets):
+    clustermodels = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        transf_data = model['pipe_transform'].transform(data)
+        Y = data[targets[i]]
+        cluster_model = shap.utils.hclust(transf_data, Y)
+        clustermodels.append(cluster_model)
+    return clustermodels
+
+def mean_shap(data, explainers, pipe_transform):
+    t_data = pipe_transform.transform(data)
+    input_features = t_data.columns
+    shap_results = get_shapvalues(explainers,t_data)
+    arrays_ = list()
+    for k,_ in shap_results.items():
+        arrays_.append(shap_results.get(k).values)
+    shap_results_mean = np.mean(np.array(arrays_), axis = 0)
+    df_shap = pd.DataFrame(shap_results_mean, columns=input_features, index=data.index)
+    df_shap['Close'] = data['Close']
+    df_shap['Date'] = data['Date']
+    return df_shap
+
+def edge_shap_lines(data, dict_shap_values, plot = False, look_back = 750):
+    ### corect labels ####
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    top_features_shap = dict_shap_values.keys()
+    for col in top_features_shap:
+        fig.add_trace(go.Scatter(x=df.Date, y=df[col],mode='lines+markers',name=dict_shap_values.get(col).get('tag')),secondary_y=True)
+    fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig
+
+def log_top_shap(runid, top_shap):
+    with mlflow.start_run(run_id=runid) as run:
+        mlflow.log_dict(top_shap,f"explainer/top_shap.json")
+        print('artifact was logged')
+
+def load_top_shap(runid):
+    folder = f"explainer/top_shap.json"
+    top_shap = mlflow.artifacts.load_dict(f"runs:/{runid}/{folder}")
+    return top_shap

virgo_modules/src/re_utils.py

@@ -1574,8 +1574,9 @@ def produce_probas(model,data, target_variables):
     """
     label_prediction = ['proba_'+x for x in target_variables]
     predictions = model.predict_proba(data)
+    if isinstance(predictions, list):
+        predictions = np.array([ x[:,1].T for x in predictions]).T
     predictions = pd.DataFrame(predictions, columns = label_prediction, index = data.index)
-    
     result_df = pd.concat([data, predictions], axis=1)
     result_df = result_df[['Date'] + target_variables + label_prediction]
 
@@ -1604,27 +1605,9 @@ def produce_signals(result_df, feature_name, threshold, label_prediction):
 
     return result_df
 
-def edge_probas_lines(data, threshold, plot = False, look_back = 750):
-    """
-    produce a plotly plot of edges and closing prices
-
-            Parameters:
-                    data (pd.DataFrame): asset data with edge probabilities
-                    plot (boolean): if true, display plot
-                    threshold (float): edge threshold
-                    look_back (int): number of rows back to display
-
-            Returns:
-                    fig (obj): plotly go object
-    """
-    df = data[['Date','Close','proba_target_down','proba_target_up']].iloc[-look_back:]
-    
-    fig = make_subplots(specs=[[{"secondary_y": True}]])
-    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',name='Close price'))
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),name='go down'),secondary_y=True)
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(opacity=0.1,size=80), name='go up'),secondary_y=True)
-    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"),)
-    fig.update_layout(title_text="sirius - edge probabilities",width=1200,height = 500)
-    if plot:
-        fig.show()
-    return fig
+def clean_cols(data, patterns):
+    drop_cols = list()
+    for pattern in patterns:
+        drop_cols = drop_cols + [ x for x in data.columns if pattern in x]
+    data = data.drop(columns = drop_cols)
+    return data

{virgo_modules-0.2.10.dist-info → virgo_modules-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo-modules
-Version: 0.2.10
+Version: 0.3.0
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire

virgo_modules-0.3.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+virgo_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+virgo_modules/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+virgo_modules/src/aws_utils.py,sha256=q0l7D7ofo09Lu1QQjv-esheQ06uiSy1Pdq3xMul8zvk,2571
+virgo_modules/src/backtester.py,sha256=OhiWyzDX0PthXGuhChyWUmDN3cLkzVYe95zS4nGtia8,22106
+virgo_modules/src/hmm_utils.py,sha256=fFWxmh9q3rjiKRHnxNk9k7O4fDrxVxkmp3pbpLvktjc,21116
+virgo_modules/src/pull_artifacts.py,sha256=5OPrgR7pcMSdpbevDRhf0ebk7g7ZRjff4NpTIIWAKjE,1989
+virgo_modules/src/re_utils.py,sha256=n5LD2IKURNsBsWeQ_xw98lh1YwLdl3efNLNT7qwFGzM,73863
+virgo_modules/src/ticketer_source.py,sha256=4vT8YweRtS28Zs66DBUG2rZ5EcFk9S0hiqNS_fz_I1c,100017
+virgo_modules/src/transformer_utils.py,sha256=LLwKYZRq5hrPVimnq3taD0Lh-q3Bq21fy1I4Icbnxi8,7677
+virgo_modules/src/edge_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+virgo_modules/src/edge_utils/conformal_utils.py,sha256=4gAOh2vxeWGR7iMF3TKnfQcLHlaH9M5B2lcmzo7V8AY,3755
+virgo_modules/src/edge_utils/edge_utils.py,sha256=U3GVyLgz0gEaR7AGjWseFcWt-IOusZvXJjwpBFknxNs,15643
+virgo_modules/src/edge_utils/shap_utils.py,sha256=877gfucW-iLbf-aStgIcAng1XxQi_AgJsVjvdAzzWc8,3233
+virgo_modules-0.3.0.dist-info/LICENSE,sha256=pNgFyCYgmimaw0o6V20JupZLROycAnOA_HDDh1tX2V4,1097
+virgo_modules-0.3.0.dist-info/METADATA,sha256=FuPBQsfBwgD6NtAeCYtNjkdTyEsVSaEk31zlhhBMv2M,883
+virgo_modules-0.3.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+virgo_modules-0.3.0.dist-info/top_level.txt,sha256=ZjI-qEkDtT-8mFwGAWnXfqPOKEGlIhWRW1es1VyXc60,14
+virgo_modules-0.3.0.dist-info/RECORD,,

virgo_modules-0.2.10.dist-info/RECORD

@@ -1,15 +0,0 @@
-virgo_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-virgo_modules/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-virgo_modules/src/aws_utils.py,sha256=q0l7D7ofo09Lu1QQjv-esheQ06uiSy1Pdq3xMul8zvk,2571
-virgo_modules/src/backtester.py,sha256=OhiWyzDX0PthXGuhChyWUmDN3cLkzVYe95zS4nGtia8,22106
-virgo_modules/src/edge_utils.py,sha256=XN2oEOwADXF9IGNUGx0Ai8B1yDAiU2WDateLnEJh5FE,14243
-virgo_modules/src/hmm_utils.py,sha256=fFWxmh9q3rjiKRHnxNk9k7O4fDrxVxkmp3pbpLvktjc,21116
-virgo_modules/src/pull_artifacts.py,sha256=5OPrgR7pcMSdpbevDRhf0ebk7g7ZRjff4NpTIIWAKjE,1989
-virgo_modules/src/re_utils.py,sha256=PoHFqYiccUrKZJ2wvDbisrK1Uo9ekX8laAeGc2XxLxY,74852
-virgo_modules/src/ticketer_source.py,sha256=4vT8YweRtS28Zs66DBUG2rZ5EcFk9S0hiqNS_fz_I1c,100017
-virgo_modules/src/transformer_utils.py,sha256=LLwKYZRq5hrPVimnq3taD0Lh-q3Bq21fy1I4Icbnxi8,7677
-virgo_modules-0.2.10.dist-info/LICENSE,sha256=pNgFyCYgmimaw0o6V20JupZLROycAnOA_HDDh1tX2V4,1097
-virgo_modules-0.2.10.dist-info/METADATA,sha256=-9U5ff9siZLh3T97ZAieJooEJKYfTVMpIM3nw7PiZeA,884
-virgo_modules-0.2.10.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-virgo_modules-0.2.10.dist-info/top_level.txt,sha256=ZjI-qEkDtT-8mFwGAWnXfqPOKEGlIhWRW1es1VyXc60,14
-virgo_modules-0.2.10.dist-info/RECORD,,