virgo-modules 0.2.9__tar.gz → 0.3.0__tar.gz

{virgo_modules-0.2.9 → virgo_modules-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo_modules
-Version: 0.2.9
+Version: 0.3.0
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire
@@ -12,23 +12,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9, <3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: feature-engine==1.6.1
-Requires-Dist: matplotlib==3.6.3
-Requires-Dist: mlflow==2.1.1
-Requires-Dist: numpy==1.23.5
-Requires-Dist: optuna==3.1.0
-Requires-Dist: pandas==1.5.3
-Requires-Dist: plotly==5.15.0
-Requires-Dist: rsa==4.9
-Requires-Dist: scikit-learn==1.2.1
-Requires-Dist: scipy==1.10.0
-Requires-Dist: seaborn==0.12.2
-Requires-Dist: starlette==0.22.0
-Requires-Dist: statsmodels==0.13.5
-Requires-Dist: ta==0.10.2
-Requires-Dist: yfinance==0.2.9
-Requires-Dist: hmmlearn==0.3.0
-Requires-Dist: boto3
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 

{virgo_modules-0.2.9 → virgo_modules-0.3.0}/setup.py

@@ -5,7 +5,7 @@ with open("virgo_app/README.md", "r") as f:
 
 setup(
     name="virgo_modules",
-    version="0.2.9",
+    version="0.3.0",
     description="data processing and statistical modeling using stock market data",
     package_dir={"": "virgo_app"},
     packages=find_packages(where="virgo_app"),
@@ -20,9 +20,9 @@ setup(
         "Programming Language :: Python :: 3.9",
         "Operating System :: OS Independent",
     ],
-    install_requires=["feature-engine==1.6.1","matplotlib==3.6.3","mlflow==2.1.1","numpy==1.23.5","optuna==3.1.0","pandas==1.5.3",
-                "plotly==5.15.0","rsa==4.9","scikit-learn==1.2.1","scipy==1.10.0","seaborn==0.12.2","starlette==0.22.0","statsmodels==0.13.5",
-                "ta==0.10.2","yfinance==0.2.9","hmmlearn==0.3.0","boto3"],
+    # install_requires=["feature-engine==1.6.1","matplotlib==3.6.3","mlflow==2.1.1","numpy==1.23.5","optuna==3.1.0","pandas==1.5.3",
+    #             "plotly==5.15.0","rsa==4.9","scikit-learn==1.2.1","scipy==1.10.0","seaborn==0.12.2","starlette==0.22.0","statsmodels==0.13.5",
+    #             "ta==0.10.2","yfinance==0.2.9","hmmlearn==0.3.0","boto3"],
     extras_require={
         "dev": ["pytest>=7.0"],
     },

virgo_modules-0.3.0/virgo_app/virgo_modules/src/edge_utils/conformal_utils.py

@@ -0,0 +1,71 @@
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from mapie.classification import MapieClassifier
+from sklearn.pipeline import Pipeline
+import mlflow
+import numpy as np
+
+
+def get_conformal_classifiers(model, data, targets):
+    classfiers = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        seg_model = Pipeline([ 
+            ('pipe',model['pipe_transform']),
+            ('model',model['model'].estimators_[i])
+        ])
+        mapie_class = MapieClassifier(seg_model, cv='prefit', random_state=123, method="lac")
+        mapie_class.fit(data, data[targets[i]].values)
+        classfiers.append(mapie_class)
+    return classfiers
+
+def log_confmodels(runid, classifiers):
+    with mlflow.start_run(run_id=runid) as run:
+        for i,classifier in enumerate(classifiers):
+            mlflow.sklearn.log_model(classifier,f"conformal/conformal_model-{i}")
+        print('models were logged')
+
+def load_confmodel(runid, target_variables):
+    classifiers = list()
+    for i in range(len(target_variables)):
+        folder = f"conformal/conformal_model-{i}"
+        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
+        classifiers.append(model)
+    return classifiers
+
+
+def get_conformal_prediction(classifier, alphas, data, prefix='conf'):
+    _, y_pis = classifier.predict(data, alpha=alphas)
+    for i,alpha in enumerate(alphas):
+        data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
+        data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
+    return data
+
+def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
+    ### corect labels ####
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(color = 'blue'),showlegend=True,legendgroup='go up', name='go up'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),showlegend=True,legendgroup='go down',name='go down'),secondary_y=True)
+    for i,alpha in enumerate(alphas, start=1):
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_up' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_up + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'blue')
+                                     ,showlegend=False,legendgroup='go up',name='go up', text=df_[col_alpha],textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_down' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_down + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'coral')
+                                     ,showlegend=False,legendgroup='go down', name='go down',text=df_[col_alpha].astype(str),textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"))
+    fig.update_layout(title_text="sirius - edge probabilities conformal",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

{virgo_modules-0.2.9/virgo_app/virgo_modules/src → virgo_modules-0.3.0/virgo_app/virgo_modules/src/edge_utils}/edge_utils.py

@@ -9,7 +9,10 @@ from feature_engine.imputation import  MeanMedianImputer
 from feature_engine.discretisation import EqualWidthDiscretiser
 from feature_engine.datetime import DatetimeFeatures
 
-from .transformer_utils import VirgoWinsorizerFeature, InverseHyperbolicSine, FeaturesEntropy, FeatureSelector
+from ..transformer_utils import VirgoWinsorizerFeature, InverseHyperbolicSine, FeaturesEntropy, FeatureSelector
+
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
 
 class produce_model_wrapper:
     """
@@ -386,4 +389,29 @@ class ExpandingMultipleTimeSeriesKFold:
         -------
         number_window (int): number of splits
         """
-        return self.number_window
+        return self.number_window
+    
+def edge_probas_lines(data, threshold, plot = False, look_back = 750):
+    """
+    produce a plotly plot of edges and closing prices
+
+            Parameters:
+                    data (pd.DataFrame): asset data with edge probabilities
+                    plot (boolean): if true, display plot
+                    threshold (float): edge threshold
+                    look_back (int): number of rows back to display
+
+            Returns:
+                    fig (obj): plotly go object
+    """
+    df = data[['Date','Close','proba_target_down','proba_target_up']].iloc[-look_back:]
+    
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),name='go down'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(opacity=0.1,size=80), name='go up'),secondary_y=True)
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"),)
+    fig.update_layout(title_text="sirius - edge probabilities",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

virgo_modules-0.3.0/virgo_app/virgo_modules/src/edge_utils/shap_utils.py

@@ -0,0 +1,81 @@
+import shap
+import mlflow
+import pandas as pd
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+
+def get_explainers(model, data):
+    explainers = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        transf_data = model['pipe_transform'].transform(data)
+        predictor = model['model'].estimators_[i]
+        explainer= shap.Explainer(predictor, transf_data)
+        explainers.append(explainer)
+    return explainers
+
+def log_explainer(runid, classifiers):
+    with mlflow.start_run(run_id=runid) as run:
+        for i,classifier in enumerate(classifiers):
+            mlflow.sklearn.log_model(classifier,f"explainer/explainer-{i}")
+        print('models were logged')
+
+def load_explainer(runid, target_variables):
+    explainers = list()
+    for i in range(len(target_variables)):
+        folder = f"explainer/explainer-{i}"
+        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
+        explainers.append(model)
+    return explainers
+
+def get_shapvalues(explainers, data):
+    shap_values = {}
+    for i,explainer in enumerate(explainers):
+        shap_value_i = explainer(data)
+        shap_values[i] = shap_value_i
+    return shap_values
+
+def get_explainerclusters(model, data, targets):
+    clustermodels = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        transf_data = model['pipe_transform'].transform(data)
+        Y = data[targets[i]]
+        cluster_model = shap.utils.hclust(transf_data, Y)
+        clustermodels.append(cluster_model)
+    return clustermodels
+
+def mean_shap(data, explainers, pipe_transform):
+    t_data = pipe_transform.transform(data)
+    input_features = t_data.columns
+    shap_results = get_shapvalues(explainers,t_data)
+    arrays_ = list()
+    for k,_ in shap_results.items():
+        arrays_.append(shap_results.get(k).values)
+    shap_results_mean = np.mean(np.array(arrays_), axis = 0)
+    df_shap = pd.DataFrame(shap_results_mean, columns=input_features, index=data.index)
+    df_shap['Close'] = data['Close']
+    df_shap['Date'] = data['Date']
+    return df_shap
+
+def edge_shap_lines(data, dict_shap_values, plot = False, look_back = 750):
+    ### corect labels ####
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    top_features_shap = dict_shap_values.keys()
+    for col in top_features_shap:
+        fig.add_trace(go.Scatter(x=df.Date, y=df[col],mode='lines+markers',name=dict_shap_values.get(col).get('tag')),secondary_y=True)
+    fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig
+
+def log_top_shap(runid, top_shap):
+    with mlflow.start_run(run_id=runid) as run:
+        mlflow.log_dict(top_shap,f"explainer/top_shap.json")
+        print('artifact was logged')
+
+def load_top_shap(runid):
+    folder = f"explainer/top_shap.json"
+    top_shap = mlflow.artifacts.load_dict(f"runs:/{runid}/{folder}")
+    return top_shap

{virgo_modules-0.2.9 → virgo_modules-0.3.0}/virgo_app/virgo_modules/src/re_utils.py

@@ -1574,8 +1574,9 @@ def produce_probas(model,data, target_variables):
     """
     label_prediction = ['proba_'+x for x in target_variables]
     predictions = model.predict_proba(data)
+    if isinstance(predictions, list):
+        predictions = np.array([ x[:,1].T for x in predictions]).T
     predictions = pd.DataFrame(predictions, columns = label_prediction, index = data.index)
-    
     result_df = pd.concat([data, predictions], axis=1)
     result_df = result_df[['Date'] + target_variables + label_prediction]
 
@@ -1604,27 +1605,9 @@ def produce_signals(result_df, feature_name, threshold, label_prediction):
 
     return result_df
 
-def edge_probas_lines(data, threshold, plot = False, look_back = 750):
-    """
-    produce a plotly plot of edges and closing prices
-
-            Parameters:
-                    data (pd.DataFrame): asset data with edge probabilities
-                    plot (boolean): if true, display plot
-                    threshold (float): edge threshold
-                    look_back (int): number of rows back to display
-
-            Returns:
-                    fig (obj): plotly go object
-    """
-    df = data[['Date','Close','proba_target_down','proba_target_up']].iloc[-look_back:]
-    
-    fig = make_subplots(specs=[[{"secondary_y": True}]])
-    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',name='Close price'))
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),name='go down'),secondary_y=True)
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(opacity=0.1,size=80), name='go up'),secondary_y=True)
-    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"),)
-    fig.update_layout(title_text="sirius - edge probabilities",width=1200,height = 500)
-    if plot:
-        fig.show()
-    return fig
+def clean_cols(data, patterns):
+    drop_cols = list()
+    for pattern in patterns:
+        drop_cols = drop_cols + [ x for x in data.columns if pattern in x]
+    data = data.drop(columns = drop_cols)
+    return data

{virgo_modules-0.2.9 → virgo_modules-0.3.0}/virgo_app/virgo_modules/src/ticketer_source.py

@@ -1287,7 +1287,7 @@ class stock_eda_panel(object):
                 
             setattr(self,f'settings_{name_attr}_pricefeature' , {'type_func': type_func, 'window': window, 'distance': distance})
 
-    def pair_index_feature(self, pair_symbol, feature_label, window, threshold, plot = False, save_features = False):
+    def pair_index_feature(self, pair_symbol, feature_label,threshold, window = None,ta_method='ROC',param_set=False,plot = False, save_features = False):
         """
         perform additional asset ROC feature, then a new feature is created in the main dataframe
 
@@ -1295,8 +1295,10 @@ class stock_eda_panel(object):
         ----------
         pair_symbol (str): symbol of the asset to extract the data
         feature_label (str): name of the resulting feature
-        window (int): window to apply to the feature
+        window (int): window to apply to the feature as default (this parameter is going to be deprecated)
         threshold (float): alpha or z thrsholds for the normalized feature
+        param_set (dict): parameter set in case ta_method is other than ROC
+        ta_method (str): method to use, available RSI, ROC, VORTEX, STOCH 
         plot (boolean): True to display plot
         save_features (boolean): True to save feature configuration and feature names
         
@@ -1326,7 +1328,20 @@ class stock_eda_panel(object):
         self.pair_index_df = self.pair_index_df.fillna(method = 'bfill')
         self.pair_index_df = self.pair_index_df.fillna(method = 'ffill')
 
-        self.pair_index_df[feature_label] = ROCIndicator(close = self.pair_index_df['Close'], window = window).roc()
+        if ta_method == 'ROC':
+            window = window if window else param_set.get('window')
+            roc = ROCIndicator(close = self.pair_index_df['Close'], window = window).roc()
+            self.pair_index_df[feature_label] = roc.replace([np.inf, -np.inf], 0).fillna(method = 'ffill')
+        elif ta_method == 'RSI':
+            rsi = RSIIndicator(close = self.pair_index_df['Close'], **param_set).rsi()
+            self.pair_index_df[feature_label] = rsi.replace([np.inf, -np.inf], 0).fillna(method = 'ffill')
+        elif ta_method == 'VORTEX':
+            vortex = VortexIndicator(close = self.pair_index_df['Close'], high = self.pair_index_df['High'], low = self.pair_index_df['Low'], **param_set).vortex_indicator_diff()
+            self.pair_index_df[feature_label] = vortex.replace([np.inf, -np.inf], 0).fillna(method = 'ffill')
+        elif ta_method == 'STOCH':
+            stoch = StochRSIIndicator(close = self.pair_index_df['Close'], **param_set).stochrsi()
+            self.pair_index_df[feature_label] = stoch.replace([np.inf, -np.inf], 0).fillna(method = 'ffill')
+
         df_to_merge = self.pair_index_df[['Date',feature_label]]
         self.df = self.df.merge(df_to_merge, on ='Date',how = 'left')
 

{virgo_modules-0.2.9 → virgo_modules-0.3.0}/virgo_app/virgo_modules.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo-modules
-Version: 0.2.9
+Version: 0.3.0
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire
@@ -12,23 +12,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9, <3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: feature-engine==1.6.1
-Requires-Dist: matplotlib==3.6.3
-Requires-Dist: mlflow==2.1.1
-Requires-Dist: numpy==1.23.5
-Requires-Dist: optuna==3.1.0
-Requires-Dist: pandas==1.5.3
-Requires-Dist: plotly==5.15.0
-Requires-Dist: rsa==4.9
-Requires-Dist: scikit-learn==1.2.1
-Requires-Dist: scipy==1.10.0
-Requires-Dist: seaborn==0.12.2
-Requires-Dist: starlette==0.22.0
-Requires-Dist: statsmodels==0.13.5
-Requires-Dist: ta==0.10.2
-Requires-Dist: yfinance==0.2.9
-Requires-Dist: hmmlearn==0.3.0
-Requires-Dist: boto3
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 

{virgo_modules-0.2.9 → virgo_modules-0.3.0}/virgo_app/virgo_modules.egg-info/SOURCES.txt

@@ -10,9 +10,12 @@ virgo_app/virgo_modules.egg-info/top_level.txt
 virgo_app/virgo_modules/src/__init__.py
 virgo_app/virgo_modules/src/aws_utils.py
 virgo_app/virgo_modules/src/backtester.py
-virgo_app/virgo_modules/src/edge_utils.py
 virgo_app/virgo_modules/src/hmm_utils.py
 virgo_app/virgo_modules/src/pull_artifacts.py
 virgo_app/virgo_modules/src/re_utils.py
 virgo_app/virgo_modules/src/ticketer_source.py
-virgo_app/virgo_modules/src/transformer_utils.py
+virgo_app/virgo_modules/src/transformer_utils.py
+virgo_app/virgo_modules/src/edge_utils/__init__.py
+virgo_app/virgo_modules/src/edge_utils/conformal_utils.py
+virgo_app/virgo_modules/src/edge_utils/edge_utils.py
+virgo_app/virgo_modules/src/edge_utils/shap_utils.py

virgo_modules-0.3.0/virgo_app/virgo_modules.egg-info/requires.txt

@@ -0,0 +1,3 @@
+
+[dev]
+pytest>=7.0

virgo_modules-0.2.9/virgo_app/virgo_modules.egg-info/requires.txt

@@ -1,20 +0,0 @@
-feature-engine==1.6.1
-matplotlib==3.6.3
-mlflow==2.1.1
-numpy==1.23.5
-optuna==3.1.0
-pandas==1.5.3
-plotly==5.15.0
-rsa==4.9
-scikit-learn==1.2.1
-scipy==1.10.0
-seaborn==0.12.2
-starlette==0.22.0
-statsmodels==0.13.5
-ta==0.10.2
-yfinance==0.2.9
-hmmlearn==0.3.0
-boto3
-
-[dev]
-pytest>=7.0