virgo-modules 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

virgo_modules/src/edge_utils/conformal_utils.py

@@ -1,44 +1,79 @@
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
-from mapie.classification import MapieClassifier
 from sklearn.pipeline import Pipeline
 import mlflow
+import pandas as pd
 import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from mapie.classification import SplitConformalClassifier
 
+class ConformalStack(mlflow.pyfunc.PythonModel):
+    def __init__(self, model,targets, alphas):
+        self.model = model
+        self.targets = targets
+        self.alphas = alphas
+    def fit(self, data):
+        self.classifiers = dict()
+        for i,target in enumerate(self.targets):
+            st = SingleStack(self.model["model"],i)
+            st.fit()
+            seg_model = Pipeline([ 
+                ('pipe',self.model['pipe_transform']),
+                ('modelbase',st)
+            ])
+            mapie_class = SplitConformalClassifier(seg_model, prefit=True, random_state=123, conformity_score="lac", confidence_level=1-np.array(self.alphas))
+            mapie_class.conformalize(data, data[self.targets[i]].values)
+            self.classifiers[target] = mapie_class
+    def predict_conformal(self, data, ):
+        for target in self.targets:
+            prefix = target+"_conf"
+            _, y_pis = self.classifiers[target].predict_set(data)
+            for i,alpha in enumerate(self.alphas):
+                data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
+                data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
+        return data
+    
 
-def get_conformal_classifiers(model, data, targets):
-    classfiers = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        seg_model = Pipeline([ 
-            ('pipe',model['pipe_transform']),
-            ('model',model['model'].estimators_[i])
-        ])
-        mapie_class = MapieClassifier(seg_model, cv='prefit', random_state=123, method="lac")
-        mapie_class.fit(data, data[targets[i]].values)
-        classfiers.append(mapie_class)
-    return classfiers
+class SingleStack(ClassifierMixin, BaseEstimator):
+    def __init__(self, model, estimator_index):
+        self.model = model
+        self.estimator_index = estimator_index
+        
+    def fit(self):
+        self._is_fitted = True
+        self.classes_ = [0,1]
 
-def log_confmodels(runid, classifiers):
-    with mlflow.start_run(run_id=runid) as run:
-        for i,classifier in enumerate(classifiers):
-            mlflow.sklearn.log_model(classifier,name = f"conformal_model-{i}")
-        print('models were logged')
+    def predict_proba(self, X):
+        metas_pred = dict()
+        for i,cont in enumerate(self.model.estimators, start=1):
+            _,estimator = cont
+            meta_pred = estimator.predict_proba(X)
+            metas_pred[f"meta{i}0"] = meta_pred[0][:,1]
+            metas_pred[f"meta{i}1"] = meta_pred[1][:,1]
+        self.meta_preds_df__ = pd.DataFrame(metas_pred)
 
-def load_confmodel(runid, target_variables):
-    classifiers = list()
-    for i in range(len(target_variables)):
-        folder = f"conformal_model-{i}"
-        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}",)
-        classifiers.append(model)
-    return classifiers
-
-
-def get_conformal_prediction(classifier, alphas, data, prefix='conf'):
-    _, y_pis = classifier.predict(data, alpha=alphas)
-    for i,alpha in enumerate(alphas):
-        data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
-        data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
-    return data
+        prediction_vector = list()
+        for i,cont in enumerate(self.model.meta_estimators, start=0):
+            _,estimator = cont
+            metacols = [f"meta{j}{i}" for j in range(1,len(self.model.estimators)+1)]
+            preds = estimator.predict_proba(self.meta_preds_df__[metacols].values)
+            prediction_vector.append(preds)
+        return prediction_vector[self.estimator_index]
+        
+    def predict(self, X):
+        prediction_vector = list()
+        _ = self.predict_proba(X)
+        for i,cont in enumerate(self.model.meta_estimators, start=0):
+            _,estimator = cont
+            metacols = [f"meta{j}{i}" for j in range(1,len(self.model.estimators)+1)]
+            preds = estimator.predict(self.meta_preds_df__[metacols].values)
+            prediction_vector.append(preds) 
+        
+        p = np.array(tuple(prediction_vector))
+        return p.reshape((p.shape[1],p.shape[0]))[:,self.estimator_index]
+    
+    def __sklearn_is_fitted__(self):
+        return hasattr(self, "_is_fitted") and self._is_fitted
 
 def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
     ### corect labels ####

virgo_modules/src/edge_utils/shap_utils.py

@@ -5,49 +5,30 @@ import numpy as np
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
 
-def get_explainers(model, data):
-    explainers = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        transf_data = model['pipe_transform'].transform(data)
-        predictor = model['model'].estimators_[i]
-        explainer= shap.Explainer(predictor, transf_data)
-        explainers.append(explainer)
-    return explainers
-
-def log_explainer(runid, classifiers):
-    with mlflow.start_run(run_id=runid) as run:
-        for i,classifier in enumerate(classifiers):
-            mlflow.sklearn.log_model(classifier,f"explainer/explainer-{i}")
-        print('models were logged')
-
-def load_explainer(runid, target_variables):
-    explainers = list()
-    for i in range(len(target_variables)):
-        folder = f"explainer/explainer-{i}"
-        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
-        explainers.append(model)
-    return explainers
-
-def get_shapvalues(explainers, data):
-    shap_values = {}
-    for i,explainer in enumerate(explainers):
-        shap_value_i = explainer(data)
-        shap_values[i] = shap_value_i
-    return shap_values
-
-def get_explainerclusters(model, data, targets):
-    clustermodels = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        transf_data = model['pipe_transform'].transform(data)
-        Y = data[targets[i]]
-        cluster_model = shap.utils.hclust(transf_data, Y)
-        clustermodels.append(cluster_model)
-    return clustermodels
-
-def mean_shap(data, explainers, pipe_transform, dict_shap_values):
+class StackInterpretor(mlflow.pyfunc.PythonModel):
+    def __init__(self, model, targets):
+        self.base_estimators = model.estimators_
+        self.targets = targets
+    def fit_interpretor(self, data):
+        interpretors = {}
+        for label, predictor in zip(self.targets,self.base_estimators):
+            explainer = shap.Explainer(predictor, data)
+            interpretors[label] = explainer
+        self.interpretors = interpretors
+    def get_shap_values(self, data):
+        shap_values = dict()
+        for label, interpretor in self.interpretors.items():
+            shap_value = interpretor(data)
+            shap_values[label] = shap_value
+        return shap_values
+    def register_map(self, mapping):
+        self.mapping = mapping
+
+def mean_shap(data, explainers, pipe_transform):
     t_data = pipe_transform.transform(data)
     input_features = t_data.columns
-    shap_results = get_shapvalues(explainers,t_data)
+    shap_results = explainers.get_shap_values(t_data)
+    dict_shap_values = explainers.mapping
     arrays_ = list()
     for k,_ in shap_results.items():
         arrays_.append(shap_results.get(k).values)
@@ -70,14 +51,4 @@ def edge_shap_lines(data, plot = False, look_back = 750):
     fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
     if plot:
         fig.show()
-    return fig
-
-def log_top_shap(runid, top_shap):
-    with mlflow.start_run(run_id=runid) as run:
-        mlflow.log_dict(top_shap,f"explainer/top_shap.json")
-        print('artifact was logged')
-
-def load_top_shap(runid):
-    folder = f"explainer/top_shap.json"
-    top_shap = mlflow.artifacts.load_dict(f"runs:/{runid}/{folder}")
-    return top_shap
+    return fig

{virgo_modules-0.5.1.dist-info → virgo_modules-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo-modules
-Version: 0.5.1
+Version: 0.6.1
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire

{virgo_modules-0.5.1.dist-info → virgo_modules-0.6.1.dist-info}/RECORD

@@ -8,12 +8,12 @@ virgo_modules/src/re_utils.py,sha256=DBY_VBB1wKm5D7znutpF_66CTLZhJfx54h8Ws0YzdN4
 virgo_modules/src/ticketer_source.py,sha256=jxP-OOeoyN2JxRQg-mX6t6WNJXiIrhWKDywDxpYANxU,101977
 virgo_modules/src/transformer_utils.py,sha256=ysCUp3cB3_7Jr9OHDqhg2_6Vu0k1YVjfqbvQNbxpbhI,8990
 virgo_modules/src/edge_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-virgo_modules/src/edge_utils/conformal_utils.py,sha256=WK54ctvdnFrCAv3_slFBC71Ojy9xgC1wODR7xmvXGgg,3743
+virgo_modules/src/edge_utils/conformal_utils.py,sha256=cKm4KSM261Eu1FJn4oowKYiKIesW81VbqITIvopGSVk,5410
 virgo_modules/src/edge_utils/edge_utils.py,sha256=7nYPLDNyKqeKIuOOwQi4wsBibzs9gP1HgYMISXJX1Y8,19522
-virgo_modules/src/edge_utils/shap_utils.py,sha256=OMKVO-4gtOng9GeSuhEgAEQe4FF3AtqXjm-GUBLqYFc,3349
+virgo_modules/src/edge_utils/shap_utils.py,sha256=FgcHkfddvdFSeUqEubYa2ExRGVAWSthqK4b-eKagEmo,2333
 virgo_modules/src/edge_utils/stack_model.py,sha256=QqE91uLo2KauGEj91AVNANB1xE7J4Fa49YOX7k5mFng,4257
-virgo_modules-0.5.1.dist-info/LICENSE,sha256=pNgFyCYgmimaw0o6V20JupZLROycAnOA_HDDh1tX2V4,1097
-virgo_modules-0.5.1.dist-info/METADATA,sha256=cDHMIPEJe-YznvEfgdToAL6GsMIrcrB0ti9sTjPlhcw,876
-virgo_modules-0.5.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-virgo_modules-0.5.1.dist-info/top_level.txt,sha256=ZjI-qEkDtT-8mFwGAWnXfqPOKEGlIhWRW1es1VyXc60,14
-virgo_modules-0.5.1.dist-info/RECORD,,
+virgo_modules-0.6.1.dist-info/LICENSE,sha256=pNgFyCYgmimaw0o6V20JupZLROycAnOA_HDDh1tX2V4,1097
+virgo_modules-0.6.1.dist-info/METADATA,sha256=9EtSQrm2xy6-S4wGgWwWbL5V7yz-8BV6TlK3G18LyoM,876
+virgo_modules-0.6.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+virgo_modules-0.6.1.dist-info/top_level.txt,sha256=ZjI-qEkDtT-8mFwGAWnXfqPOKEGlIhWRW1es1VyXc60,14
+virgo_modules-0.6.1.dist-info/RECORD,,