cryptodatapy 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

cryptodatapy/transform/clean.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 from typing import Optional, Union
 import pandas as pd
-
 from cryptodatapy.transform.od import OutlierDetection
 from cryptodatapy.transform.impute import Impute
 from cryptodatapy.transform.filter import Filter
@@ -66,9 +65,9 @@ class CleanData:
         self.excluded_cols = None
         self.outliers = None
         self.yhat = None
+        self.repaired_df = None
         self.filtered_df = None
         self.filtered_tickers = None
-        self.repaired_df = None
         self.summary = pd.DataFrame()
         self.initialize_summary()
         self.check_types()
@@ -127,7 +126,7 @@ class CleanData:
 
         # add to summary
         self.summary.loc["%_outliers", self.outliers.unstack().columns] = (
-            self.outliers.unstack().notna().sum() / self.df.unstack().shape[0]
+            self.outliers.unstack().notna().sum() / od.df.unstack().notna().sum()
         ).values * 100
 
         # filtered df
@@ -157,7 +156,7 @@ class CleanData:
 
         # add repaired % to summary
         rep_vals = self.repaired_df.unstack().notna().sum() - self.df.unstack().notna().sum()
-        self.summary.loc["%_imputed", self.df.unstack().columns] = rep_vals / self.df.unstack().shape[0] * 100
+        self.summary.loc["%_imputed", self.df.unstack().columns] = rep_vals / self.df.unstack().notna().sum() * 100
 
         # repaired df
         if self.excluded_cols is not None:
@@ -192,9 +191,8 @@ class CleanData:
 
         # add to summary
         filtered_vals = self.df.unstack().notna().sum() - self.filtered_df.unstack().notna().sum()
-        self.summary.loc["%_below_avg_trading_val", self.df.unstack().columns] = (
-            filtered_vals / self.df.unstack().shape[0]
-        ).values * 100
+        self.summary.loc["%_below_avg_trading_val", self.df.unstack().columns] = \
+            (filtered_vals / self.df.unstack().notna().sum()).values * 100
 
         # filtered df
         self.df = self.filtered_df.sort_index()
@@ -223,7 +221,7 @@ class CleanData:
             self.df.unstack().notna().sum() - self.filtered_df.unstack().notna().sum()
         )
         self.summary.loc["%_missing_vals_gaps", self.df.unstack().columns] = (
-            missing_vals_gap / self.df.unstack().shape[0]
+            missing_vals_gap / self.df.unstack().notna().sum()
         ).values * 100
 
         # filtered df
@@ -258,23 +256,21 @@ class CleanData:
         )
 
         # add to summary
-        self.summary.loc["n_tickers_below_min_obs", self.df.unstack().columns] = len(self.filtered_tickers)
+        self.summary.loc["n_filtered_tickers", self.df.unstack().columns] = len(self.filtered_tickers)
 
         # filtered df
         self.df = self.filtered_df.sort_index()
 
         return self
 
-    def filter_delisted_tickers(self, field: str = 'close', n_unch_vals: int = 30) -> CleanData:
+    def filter_delisted_tickers(self, method: str = 'replace') -> CleanData:
         """
         Removes delisted tickers from dataframe.
 
         Parameters
         ----------
-        field: str, default 'close'
-            Field/column to use for detecting delisted tickers.
-        n_unch_vals: int, default 30
-            Number of consecutive unchanged values to consider a ticker as delisted.
+        method: str, {'replace', 'remove'}, default 'replace'
+            Method to use for handling delisted tickers.
 
         Returns
         -------
@@ -282,7 +278,7 @@ class CleanData:
             CleanData object
         """
         # filter tickers
-        self.filtered_df = Filter(self.df).remove_delisted(field=field, n_unch_vals=n_unch_vals)
+        self.filtered_df = Filter(self.df).delisted_tickers(method=method)
 
         # tickers < min obs
         self.filtered_tickers = list(
@@ -292,6 +288,12 @@ class CleanData:
         )
 
         # add to summary
+        filtered_vals = (
+            self.df.unstack().notna().sum() - self.filtered_df.unstack().notna().sum()
+        )
+        self.summary.loc["%_delisted_ticker_vals", self.df.unstack().columns] = (
+            filtered_vals / self.df.unstack().notna().sum()
+        ).values * 100
         self.summary.loc["n_filtered_tickers", self.df.unstack().columns] = len(self.filtered_tickers)
 
         # filtered df