virgo-modules 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

virgo_modules/src/edge_utils/conformal_utils.py

@@ -1,44 +1,79 @@
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
-from mapie.classification import MapieClassifier
 from sklearn.pipeline import Pipeline
 import mlflow
+import pandas as pd
 import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from mapie.classification import SplitConformalClassifier
 
+class ConformalStack(mlflow.pyfunc.PythonModel):
+    def __init__(self, model,targets, alphas):
+        self.model = model
+        self.targets = targets
+        self.alphas = alphas
+    def fit(self, data):
+        self.classifiers = dict()
+        for i,target in enumerate(self.targets):
+            st = SingleStack(self.model["model"],i)
+            st.fit()
+            seg_model = Pipeline([ 
+                ('pipe',self.model['pipe_transform']),
+                ('modelbase',st)
+            ])
+            mapie_class = SplitConformalClassifier(seg_model, prefit=True, random_state=123, conformity_score="lac", confidence_level=1-np.array(self.alphas))
+            mapie_class.conformalize(data, data[self.targets[i]].values)
+            self.classifiers[target] = mapie_class
+    def predict_conformal(self, data, ):
+        for target in self.targets:
+            prefix = target+"_conf"
+            _, y_pis = self.classifiers[target].predict_set(data)
+            for i,alpha in enumerate(self.alphas):
+                data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
+                data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
+        return data
+    
 
-def get_conformal_classifiers(model, data, targets):
-    classfiers = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        seg_model = Pipeline([ 
-            ('pipe',model['pipe_transform']),
-            ('model',model['model'].estimators_[i])
-        ])
-        mapie_class = MapieClassifier(seg_model, cv='prefit', random_state=123, method="lac")
-        mapie_class.fit(data, data[targets[i]].values)
-        classfiers.append(mapie_class)
-    return classfiers
+class SingleStack(ClassifierMixin, BaseEstimator):
+    def __init__(self, model, estimator_index):
+        self.model = model
+        self.estimator_index = estimator_index
+        
+    def fit(self):
+        self._is_fitted = True
+        self.classes_ = [0,1]
 
-def log_confmodels(runid, classifiers):
-    with mlflow.start_run(run_id=runid) as run:
-        for i,classifier in enumerate(classifiers):
-            mlflow.sklearn.log_model(classifier,name = f"conformal_model-{i}")
-        print('models were logged')
+    def predict_proba(self, X):
+        metas_pred = dict()
+        for i,cont in enumerate(self.model.estimators, start=1):
+            _,estimator = cont
+            meta_pred = estimator.predict_proba(X)
+            metas_pred[f"meta{i}0"] = meta_pred[0][:,1]
+            metas_pred[f"meta{i}1"] = meta_pred[1][:,1]
+        self.meta_preds_df__ = pd.DataFrame(metas_pred)
 
-def load_confmodel(runid, target_variables):
-    classifiers = list()
-    for i in range(len(target_variables)):
-        folder = f"conformal_model-{i}"
-        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}",)
-        classifiers.append(model)
-    return classifiers
-
-
-def get_conformal_prediction(classifier, alphas, data, prefix='conf'):
-    _, y_pis = classifier.predict(data, alpha=alphas)
-    for i,alpha in enumerate(alphas):
-        data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
-        data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
-    return data
+        prediction_vector = list()
+        for i,cont in enumerate(self.model.meta_estimators, start=0):
+            _,estimator = cont
+            metacols = [f"meta{j}{i}" for j in range(1,len(self.model.estimators)+1)]
+            preds = estimator.predict_proba(self.meta_preds_df__[metacols].values)
+            prediction_vector.append(preds)
+        return prediction_vector[self.estimator_index]
+        
+    def predict(self, X):
+        prediction_vector = list()
+        _ = self.predict_proba(X)
+        for i,cont in enumerate(self.model.meta_estimators, start=0):
+            _,estimator = cont
+            metacols = [f"meta{j}{i}" for j in range(1,len(self.model.estimators)+1)]
+            preds = estimator.predict(self.meta_preds_df__[metacols].values)
+            prediction_vector.append(preds) 
+        
+        p = np.array(tuple(prediction_vector))
+        return p.reshape((p.shape[1],p.shape[0]))[:,self.estimator_index]
+    
+    def __sklearn_is_fitted__(self):
+        return hasattr(self, "_is_fitted") and self._is_fitted
 
 def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
     ### corect labels ####

virgo_modules/src/edge_utils/shap_utils.py

@@ -5,49 +5,30 @@ import numpy as np
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
 
-def get_explainers(model, data):
-    explainers = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        transf_data = model['pipe_transform'].transform(data)
-        predictor = model['model'].estimators_[i]
-        explainer= shap.Explainer(predictor, transf_data)
-        explainers.append(explainer)
-    return explainers
-
-def log_explainer(runid, classifiers):
-    with mlflow.start_run(run_id=runid) as run:
-        for i,classifier in enumerate(classifiers):
-            mlflow.sklearn.log_model(classifier,f"explainer/explainer-{i}")
-        print('models were logged')
-
-def load_explainer(runid, target_variables):
-    explainers = list()
-    for i in range(len(target_variables)):
-        folder = f"explainer/explainer-{i}"
-        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
-        explainers.append(model)
-    return explainers
-
-def get_shapvalues(explainers, data):
-    shap_values = {}
-    for i,explainer in enumerate(explainers):
-        shap_value_i = explainer(data)
-        shap_values[i] = shap_value_i
-    return shap_values
-
-def get_explainerclusters(model, data, targets):
-    clustermodels = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        transf_data = model['pipe_transform'].transform(data)
-        Y = data[targets[i]]
-        cluster_model = shap.utils.hclust(transf_data, Y)
-        clustermodels.append(cluster_model)
-    return clustermodels
-
-def mean_shap(data, explainers, pipe_transform, dict_shap_values):
+class StackInterpretor(mlflow.pyfunc.PythonModel):
+    def __init__(self, model, targets):
+        self.base_estimators = model.estimators_
+        self.targets = targets
+    def fit_interpretor(self, data):
+        interpretors = {}
+        for label, predictor in zip(self.targets,self.base_estimators):
+            explainer = shap.Explainer(predictor, data)
+            interpretors[label] = explainer
+        self.interpretors = interpretors
+    def get_shap_values(self, data):
+        shap_values = dict()
+        for label, interpretor in self.interpretors.items():
+            shap_value = interpretor(data)
+            shap_values[label] = shap_value
+        return shap_values
+    def register_map(self, mapping):
+        self.mapping = mapping
+
+def mean_shap(data, explainers, pipe_transform):
     t_data = pipe_transform.transform(data)
     input_features = t_data.columns
-    shap_results = get_shapvalues(explainers,t_data)
+    shap_results = explainers.get_shap_values(t_data)
+    dict_shap_values = explainers.mapping
     arrays_ = list()
     for k,_ in shap_results.items():
         arrays_.append(shap_results.get(k).values)
@@ -70,14 +51,4 @@ def edge_shap_lines(data, plot = False, look_back = 750):
     fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
     if plot:
         fig.show()
-    return fig
-
-def log_top_shap(runid, top_shap):
-    with mlflow.start_run(run_id=runid) as run:
-        mlflow.log_dict(top_shap,f"explainer/top_shap.json")
-        print('artifact was logged')
-
-def load_top_shap(runid):
-    folder = f"explainer/top_shap.json"
-    top_shap = mlflow.artifacts.load_dict(f"runs:/{runid}/{folder}")
-    return top_shap
+    return fig

virgo_modules/src/market/market_tools.py

@@ -0,0 +1,189 @@
+import gc
+
+import pandas as pd
+import numpy as np
+
+from sklearn.linear_model import HuberRegressor
+from scipy import stats
+
+import matplotlib.pyplot as plt
+import seaborn as sns; sns.set()
+
+from matplotlib import cm
+import matplotlib.colors as mcolors
+
+class MarketAnalysis:
+    """
+    Class that perform market analysis using robust linear regression
+
+    Attributes
+    ----------
+    data : pd.DataFrame
+        input data
+    market_features : list
+        list of market feature (log returns) to apply analysis
+    return_cols: str
+        main log return feature
+    col_map: dict
+        dictionary containing rename of market features
+
+    Methods
+    -------
+    compute_beta(data=pd.DataFrame, feature_x=str, feature_y=str):
+        compute betas given x and y using robust linear regression
+    get_correlation(data=pd.DataFrame, feature_x=str, feature_y=str):
+        compute correlation given x and y
+    produce_beta_report(data=pd.DataFrame):
+        produce beta report
+    compute_general_report(sample_size=int, offset=int, index=str, subsample_ts=int, show_plot=bool):
+        compute full report, global and latest window
+    """
+
+    def __init__(self, data, market_features, return_col, col_map=None):
+        self.data = data.dropna()
+        self.market_features = market_features
+        self.return_cols = return_col
+        self.col_map=col_map
+        
+    def compute_beta(self, data, feature_x, feature_y):
+        """
+        compute betas given x and y using robust linear regression
+
+        Parameters
+        ----------
+        data (pd.DataFrame): input data containing analysis features
+        feature_x (str): name of the feature x
+        feature_y (str): name of the feature y
+
+        Returns
+        -------
+        (beta(str), alpha(str))
+        """
+        x = data[feature_x].values.reshape(-1,1)
+        y = data[feature_y].values.reshape(-1,1)
+        huber_regr = HuberRegressor(fit_intercept = True)
+        huber_regr.fit(x, y)
+        beta, alpha = huber_regr.coef_[0], huber_regr.intercept_
+        return beta, alpha
+
+    def get_correlation(self, data, feature_x, feature_y):
+        """
+        compute correlation given x and y
+
+        Parameters
+        ----------
+        data (pd.DataFrame): input data containing analysis features
+        feature_x (str): name of the feature x
+        feature_y (str): name of the feature y
+
+        Returns
+        -------
+        r (float)
+        """
+        x = data[feature_x]
+        y = data[feature_y]
+        r = stats.mstats.pearsonr(x, y)[0]
+        return r
+
+    def produce_beta_report(self, data):
+        """
+        produce beta report
+
+        Parameters
+        ----------
+        data (pd.DataFrame): input data containing analysis features
+
+        Returns
+        -------
+        report (pd.DataFrame)
+        """
+        result = {
+            "market_index": list(),
+            "beta": list(),
+            "alpha": list(),
+            "r": list()
+        }
+        for index in self.market_features:
+            beta, alpha = self.compute_beta( data, self.return_cols, index)
+            r = self.get_correlation( data, self.return_cols, index)
+            result["market_index"].append(index)
+            result["beta"].append(beta)
+            result["alpha"].append(alpha)
+            result["r"].append(r)
+        pd_result = pd.DataFrame(result)
+        pd_result = pd_result.sort_values("r", ascending=False)
+        if self.col_map:
+            pd_result["map_market_index"] = pd_result.market_index.map(self.col_map)
+        return pd_result
+        
+    def compute_general_report(self, sample_size, offset, index=False, subsample_ts=False, show_plot=True):
+        """
+        compute full report, global and latest window
+
+        Parameters
+        ----------
+        sample_size (int): sample size for every beta computation
+        offset (int): offset or overlap between samples
+        index (str): if provided, bet fit index is taken
+        subsample_ts (int): subsample for iterative beta calculation
+        show_plot (bool): whether to show plot
+
+        Returns
+        -------
+        (report (pd.DataFrame), latest_report (pd.DataFrame), figure (mtpl.plt))
+        """
+        general_report = self.produce_beta_report(self.data)
+        current_report = self.produce_beta_report(self.data.iloc[sample_size:,:])
+        if not index:
+            index = general_report.head(1).market_index.values[0]
+        b = general_report[general_report.market_index == index].beta.values
+        a = general_report[general_report.market_index == index].alpha.values
+
+        figure, ax = plt.subplot_mosaic(
+            [["scatter_total", "scatter_sample",'ts','ts']],
+            layout="constrained",
+            figsize=(18, 5)
+        )
+        x = self.data[self.return_cols]
+        y = self.data[index]
+        ax['scatter_total'].scatter(x, y)
+        ax['scatter_total'].plot(x, b*x+a, color='red')
+
+        if subsample_ts:
+            merger_df = self.data.iloc[-subsample_ts:,:].copy()
+        else:
+            merger_df = self.data.copy()
+        ax['ts'].plot(merger_df.Date, merger_df.Close, color = 'grey', alpha = 0.3)
+        b_array = list()
+        for i in range(0,len(merger_df)-sample_size,offset):
+            merger_ = merger_df.sort_values('Date', ascending = False).iloc[i:i+sample_size,:]
+            b, a = self.compute_beta(merger_, self.return_cols, index)
+            x = merger_[self.return_cols]
+            y = merger_[index]
+            normalize_ = mcolors.Normalize(vmin=-2.0, vmax=2.0)
+            colormap_ = cm.jet
+            ax['scatter_sample'].plot(x, y,'o', color = 'blue', alpha = 0.1)
+            ax['scatter_sample'].plot(x, b*x+a, color=colormap_(normalize_(b)))
+            ax['scatter_sample'].set_xlim(-0.08, 0.08)
+            ax['scatter_sample'].set_ylim(-0.08, 0.08)
+            plot = ax['ts'].scatter(merger_.Date, merger_.Close, color=colormap_(normalize_(b)), s = 10)
+            b_array.append(b)
+        normalize_ = mcolors.Normalize(vmin=np.min(b_array), vmax=np.max(b_array))
+        colormap_ = cm.jet
+        x_global = self.data[self.return_cols]
+        scalarmappaple = cm.ScalarMappable(norm=normalize_, cmap=colormap_)
+        scalarmappaple.set_array(x_global)
+        if self.col_map:
+            map_index = self.col_map.get(index)
+            title = f'market analysis of {map_index}'
+        else:
+            title = f'market analysis'
+        plt.title(title)
+        plt.colorbar(scalarmappaple)
+        del merger_df
+        gc.collect()
+        if show_plot:
+            plt.show()
+        else:
+            plt.close() 
+        return general_report, current_report, figure

virgo_modules/src/ticketer_source.py

@@ -1,7 +1,7 @@
 import yfinance as yf
 import pandas as pd
 import numpy as np
-import json
+import gc
 
 import matplotlib.pyplot as plt
 import matplotlib.gridspec as gridspec
@@ -138,6 +138,10 @@ class stock_eda_panel(object):
         perform analysis of lags of the mean rolling log return
     compute_clip_bands(feature_name=str,threshold=float):
         compute outlier detection for a given signal, Note that this follows mean reversion procedure and feature has to be stationary. Also botton and roof resulting signals is attached to the dataframe
+    extract_sec_data(symbol=str, base_columns=list(str), rename_columns=dict):
+        extract new asset data and merge it to the main asset data
+    lag_log_return(lags=int, feature=str, feature_name=str):
+        compute log return given some lags
     signal_plotter(feature_name=str):
         display analysis plot of a feature with high and low signals
     log_features_standard(feature_name=str):
@@ -667,6 +671,63 @@ class stock_eda_panel(object):
         self.df[f'signal_low_{feature_name}'] = np.where( (self.df[f'norm_{feature_name}'] < self.df[f'lower_{feature_name}'] ), 1, 0)
         self.df[f'signal_up_{feature_name}'] = np.where( (self.df[f'norm_{feature_name}'] > self.df[f'upper_{feature_name}'] ), 1, 0)
 
+    def extract_sec_data(self, symbol, base_columns, rename_columns=None):
+        """
+        extract new asset data and merge it to the main asset data
+
+        Parameters
+        ----------
+        symbol (str): symbol to extract data
+        base_columns (list): list of columns to persist
+        rename_columns (dict): map of the new column names using pd.DataFrame.rename()
+
+        Returns
+        -------
+        None
+        """
+        begin_date = self.today - relativedelta(days = self.n_days)
+        begin_date_str = begin_date.strftime('%Y-%m-%d')
+        
+        stock = yf.Ticker(symbol)
+        df = stock.history(period=self.data_window)
+        df = df.sort_values('Date')
+        df.reset_index(inplace=True)
+        df['Date'] = pd.to_datetime(df['Date'], format='mixed',utc=True).dt.date
+        df['Date'] = pd.to_datetime(df['Date'])
+        df = df[df.Date >= begin_date_str ]
+        df = df[base_columns]
+        if rename_columns:
+            df = df.rename(columns=rename_columns)
+        right_df = df.copy()
+        
+        dates_vector = self.df.Date.to_frame()
+        right_df = dates_vector.merge(right_df, on ='Date',how = 'left')
+        right_df = right_df.fillna(method = 'bfill')
+        right_df = right_df.fillna(method = 'ffill')
+
+        self.df = self.df.merge(right_df, on ='Date',how = 'left')
+        self.df = self.df.sort_values("Date")
+        del right_df
+        gc.collect()
+
+    def lag_log_return(self, lags, feature, feature_name=False):
+        """
+        compute log return given some lags
+
+        Parameters
+        ----------
+        lags (int): lag to apply log return
+        feature (str): feature to apply log return
+        feature_name (str): rename resuling name
+
+        Returns
+        -------
+        None
+        """
+
+        feature_name = feature_name if feature_name else f"{feature}_log_return"
+        self.df[feature_name] = np.log(self.df[feature]/self.df[feature].shift(lags))
+
     def signal_plotter(self, feature_name):
 
         """
@@ -2304,33 +2365,3 @@ class analyse_index(stock_eda_panel):
     
         self.states_result = result
 
-def get_relevant_beta(data_market, ticket_name,  show_plot = True, save_path = False, save_aws = False, aws_credentials = False):
-    '''
-    select relevant beta result data of a given asset
-
-            Parameters:
-                    data_market (pd.DataFrame): dataframe of the market results 
-                    ticket_name (str): name of the asset
-                    show_plot (bool): If tru, plot results
-                    save_path (str): local path for saving e.g r'C:/path/to/the/file/'
-                    save_aws (str):  remote key in s3 bucket path e.g. 'path/to/file/'
-                    aws_credentials (dict): dict of the aws credentials
-
-            Returns:
-                    selection (pd.DataFrame): dataframe of the most relevant beta
-    '''
-    all_betas = data_market[data_market.asset == ticket_name].sort_values('general_r', ascending = False)
-    all_betas['gen_r2'] = all_betas.general_r ** 2
-    all_betas['sampl_r2'] = all_betas.sample_r ** 2
-    selection = all_betas.sort_values('gen_r2',ascending =False).head(2).sort_values('sampl_r2',ascending =False).head(1).drop(columns = ['gen_r2','sampl_r2'])
-
-    if show_plot:
-        print(selection)
-    if save_path:
-        result_plot_name = f'market_best_fit.csv'
-        selection.to_csv(save_path+result_plot_name)
-
-    if save_path and save_aws:
-        # upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{ticket_name}/'+result_plot_name,input_path = save_path+result_plot_name)
-        upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = save_aws + result_plot_name, input_path = save_path + result_plot_name, aws_credentials = aws_credentials)
-    return selection

{virgo_modules-0.5.1.dist-info → virgo_modules-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo-modules
-Version: 0.5.1
+Version: 0.7.0
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire

{virgo_modules-0.5.1.dist-info → virgo_modules-0.7.0.dist-info}/RECORD

@@ -5,15 +5,17 @@ virgo_modules/src/backtester.py,sha256=OhiWyzDX0PthXGuhChyWUmDN3cLkzVYe95zS4nGti
 virgo_modules/src/hmm_utils.py,sha256=D7axAnCdSe1_1EgRyli2PAnM2f6699hTY9GcxjPXG-o,21221
 virgo_modules/src/pull_artifacts.py,sha256=5OPrgR7pcMSdpbevDRhf0ebk7g7ZRjff4NpTIIWAKjE,1989
 virgo_modules/src/re_utils.py,sha256=DBY_VBB1wKm5D7znutpF_66CTLZhJfx54h8Ws0YzdN4,74641
-virgo_modules/src/ticketer_source.py,sha256=jxP-OOeoyN2JxRQg-mX6t6WNJXiIrhWKDywDxpYANxU,101977
+virgo_modules/src/ticketer_source.py,sha256=aJZNB_YK0JrSrUBUBkAfolIMxcTozNFrZeoNxkhpqK8,102547
 virgo_modules/src/transformer_utils.py,sha256=ysCUp3cB3_7Jr9OHDqhg2_6Vu0k1YVjfqbvQNbxpbhI,8990
 virgo_modules/src/edge_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-virgo_modules/src/edge_utils/conformal_utils.py,sha256=WK54ctvdnFrCAv3_slFBC71Ojy9xgC1wODR7xmvXGgg,3743
+virgo_modules/src/edge_utils/conformal_utils.py,sha256=cKm4KSM261Eu1FJn4oowKYiKIesW81VbqITIvopGSVk,5410
 virgo_modules/src/edge_utils/edge_utils.py,sha256=7nYPLDNyKqeKIuOOwQi4wsBibzs9gP1HgYMISXJX1Y8,19522
-virgo_modules/src/edge_utils/shap_utils.py,sha256=OMKVO-4gtOng9GeSuhEgAEQe4FF3AtqXjm-GUBLqYFc,3349
+virgo_modules/src/edge_utils/shap_utils.py,sha256=FgcHkfddvdFSeUqEubYa2ExRGVAWSthqK4b-eKagEmo,2333
 virgo_modules/src/edge_utils/stack_model.py,sha256=QqE91uLo2KauGEj91AVNANB1xE7J4Fa49YOX7k5mFng,4257
-virgo_modules-0.5.1.dist-info/LICENSE,sha256=pNgFyCYgmimaw0o6V20JupZLROycAnOA_HDDh1tX2V4,1097
-virgo_modules-0.5.1.dist-info/METADATA,sha256=cDHMIPEJe-YznvEfgdToAL6GsMIrcrB0ti9sTjPlhcw,876
-virgo_modules-0.5.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-virgo_modules-0.5.1.dist-info/top_level.txt,sha256=ZjI-qEkDtT-8mFwGAWnXfqPOKEGlIhWRW1es1VyXc60,14
-virgo_modules-0.5.1.dist-info/RECORD,,
+virgo_modules/src/market/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+virgo_modules/src/market/market_tools.py,sha256=vBt66_7E3ANz7avzfeNw_RHMGvG9lh5PRhxmcf_Oyjc,6880
+virgo_modules-0.7.0.dist-info/LICENSE,sha256=pNgFyCYgmimaw0o6V20JupZLROycAnOA_HDDh1tX2V4,1097
+virgo_modules-0.7.0.dist-info/METADATA,sha256=tart49AI1D8oLYtFI7mxY43ReNUxWpsX34PuByszh3Q,876
+virgo_modules-0.7.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+virgo_modules-0.7.0.dist-info/top_level.txt,sha256=ZjI-qEkDtT-8mFwGAWnXfqPOKEGlIhWRW1es1VyXc60,14
+virgo_modules-0.7.0.dist-info/RECORD,,