AeroViz 0.1.9.0__tar.gz → 0.1.9.2__tar.gz

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/dataProcess/Chemistry/_ocec.py

@@ -114,8 +114,7 @@ def _basic(_lcres, _mass, _ocec_ratio, _ocec_ratio_month, _hr_lim, _range, _wiso
     _out = {}
 
     # OC1, OC2, OC3, OC4, PC
-    _df_bsc = _lcres[['OC1_raw', 'OC2_raw', 'OC3_raw', 'OC4_raw']] / _lcres['Sample_Volume'].to_frame().values.copy()
-    _df_bsc.rename(columns={'OC1_raw': 'OC1', 'OC2_raw': 'OC2', 'OC3_raw': 'OC3', 'OC4_raw': 'OC4'}, inplace=True)
+    _df_bsc = _lcres[['OC1', 'OC2', 'OC3', 'OC4', 'PC']].copy()
 
     # SOC, POC, OC/EC
     if _ocec_ratio is not None:
@@ -144,7 +143,8 @@ def _basic(_lcres, _mass, _ocec_ratio, _ocec_ratio_month, _hr_lim, _range, _wiso
     _df_ratio = DataFrame(index=_df_bsc.index)
 
     for _ky, _val in _df_bsc.items():
-        if 'OC/EC' in _ky: continue
+        if 'OC/EC' in _ky:
+            continue
         _df_ratio[f'{_ky}/Thermal_OC'] = _val / _lcres['Thermal_OC']
         _df_ratio[f'{_ky}/Optical_OC'] = _val / _lcres['Optical_OC']
 
@@ -159,14 +159,14 @@ def _basic(_lcres, _mass, _ocec_ratio, _ocec_ratio_month, _hr_lim, _range, _wiso
         _df_ratio[f'Optical_EC/PM'] = _lcres['Optical_EC'] / _mass
 
     # ratio status
-    _df_bsc = concat((_lcres, _df_bsc.copy()), axis=1)
+    _df_bsc = concat((_lcres.loc[:, :'Sample_Volume'], _df_bsc.copy()), axis=1)
 
     for _ky, _df in _df_ratio.items():
         _df_bsc[f'{_ky}_status'] = 'Normal'
         _df_bsc[f'{_ky}_status'] = _df_bsc[f'{_ky}_status'].mask(_df > 1, 'Warning')
 
     # out
-    _out['ratio'] = _df_ratio
     _out['basic'] = _df_bsc
+    _out['ratio'] = _df_ratio
 
     return _out

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/dataProcess/SizeDistr/_size_distr.py

@@ -69,7 +69,11 @@ def _basic(df, hybrid, unit, bin_rg, input_type):
 
             df_oth[f'total_{_tp_nam}_{_md_nam}'], df_oth[f'GMD_{_tp_nam}_{_md_nam}'], df_oth[
                 f'GSD_{_tp_nam}_{_md_nam}'] = _geometric_prop(_dia, _dt)
-            df_oth[f'mode_{_tp_nam}_{_md_nam}'] = _dt.idxmax(axis=1)
+
+            mask = _dt.notna().any(axis=1)
+
+            df_oth.loc[mask, f'mode_{_tp_nam}_{_md_nam}'] = _dt.loc[mask].idxmax(axis=1)
+            df_oth.loc[~mask, f'mode_{_tp_nam}_{_md_nam}'] = n.nan
 
     ## out
     out_dic['other'] = df_oth

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/plot/hysplit/hysplit.py

@@ -7,21 +7,29 @@ import pandas as pd
 
 from AeroViz.plot.utils import set_figure
 
-# TODO: Hybrid Single-Particle Lagrangian Integrated Trajectory (HYSPLIT) model
+# Hybrid Single-Particle Lagrangian Integrated Trajectory (HYSPLIT) model
 
 
 __all__ = ['hysplit']
 
 # 設置默認文件路徑
-DEFAULT_FILE = Path(__file__).parent.parent.parent / 'data' / '240228_00.txt'
+DEFAULT_FILE = Path(__file__).parent.parent.parent / 'data' / 'hysplit_example_data.txt'
 
 
 def read_hysplit_data(file: Path):
     data = pd.read_csv(file, skiprows=8, sep=r'\s+', names=range(0, 12), engine='python')
     data = data.reset_index(drop=False)
-    data.columns = ['category', 'name', 'abc', 'year', 'month', 'hour', 'min', 'cont', 'backward', 'lat', 'lon',
+    data.columns = ['category', 'name', 'year', 'month', 'day', 'hour', 'minute', 'count', 'backward', 'lat', 'lon',
                     'height', 'pressure']
 
+    time_cols = ['year', 'month', 'day', 'hour', 'minute']
+
+    data['time'] = pd.to_datetime(data[time_cols].astype(str).agg(''.join, axis=1), format='%y%m%d%H%M')
+
+    data = data.drop(columns=time_cols)
+
+    data = data[['time'] + [col for col in data.columns if col != 'time']]
+
     return data
 
 

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/__init__.py

@@ -74,7 +74,7 @@ def RawDataReader(instrument_name: str,
     if not isinstance(path, Path):
         path = Path(path)
     if not path.exists() or not path.is_dir():
-        raise ValueError(f"The specified path '{path}' does not exist or is not a directory.")
+        raise FileNotFoundError(f"The specified path '{path}' does not exist or is not a directory.")
 
     # Validate the QC frequency
     if qc_freq is not None:

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/config/supported_instruments.py

@@ -75,6 +75,8 @@ meta = {
             "Thermal EC": ["Thermal_EC"],
             "Optical OC": ["Optical_OC"],
             "Optical EC": ["Optical_EC"],
+            "Thermal OC & EC": ["Thermal_OC", "Thermal_EC"],
+            "Optical OC & EC": ["Optical_OC", "Optical_EC"],
         },
     },
 
@@ -93,53 +95,41 @@ meta = {
             "SO42-": ["SO42-"],
             "Main Salt (NH4+, NO3-, SO42-)": ["NO3-", "SO42-", "NH4+"],
         },
+        # https://www.yangyao-env.com/web/product/product_in2.jsp?pd_id=PD1640151884502
+
+        # HF: 0.08, F-: 0.08, PO43-: None is not measured
+        "MDL": {
+            'HF': None, 'HCl': 0.05, 'HNO2': 0.01, 'HNO3': 0.05, 'G-SO2': 0.05, 'NH3': 0.1,
+            'Na+': 0.05, 'NH4+': 0.08, 'K+': 0.08, 'Mg2+': 0.05, 'Ca2+': 0.05,
+            'F-': None, 'Cl-': 0.05, 'NO2-': 0.05, 'NO3-': 0.01, 'PO43-': None, 'SO42-': 0.05,
+        },
+
+        "MR": {
+            'HF': 200, 'HCl': 200, 'HNO2': 200, 'HNO3': 200, 'G-SO2': 200, 'NH3': 300,
+            'Na+': 300, 'NH4+': 300, 'K+': 300, 'Mg2+': 300, 'Ca2+': 300,
+            'F-': 300, 'Cl-': 300, 'NO2-': 300, 'NO3-': 300, 'PO43-': None, 'SO42-': 300,
+        }
     },
 
     "XRF": {
         "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": {
-            "Al": ["Al"],
-            "Si": ["Si"],
-            "P": ["P"],
-            "S": ["S"],
-            "Cl": ["Cl"],
-            "K": ["K"],
-            "Ca": ["Ca"],
-            "Ti": ["Ti"],
-            "V": ["V"],
-            "Cr": ["Cr"],
-            "Mn": ["Mn"],
-            "Fe": ["Fe"],
-            "Ni": ["Ni"],
-            "Cu": ["Cu"],
-            "Zn": ["Zn"],
-            "As": ["As"],
-            "Se": ["Se"],
-            "Br": ["Br"],
-            "Rb": ["Rb"],
-            "Sr": ["Sr"],
-            "Y": ["Y"],
-            "Zr": ["Zr"],
-            "Mo": ["Mo"],
-            "Ag": ["Ag"],
-            "Cd": ["Cd"],
-            "In": ["In"],
-            "Sn": ["Sn"],
-            "Sb": ["Sb"],
-            "Te": ["Te"],
-            "Cs": ["Cs"],
-            "Ba": ["Ba"],
-            "La": ["La"],
-            "Ce": ["Ce"],
-            "W": ["W"],
-            "Pt": ["Pt"],
-            "Au": ["Au"],
-            "Hg": ["Hg"],
-            "Tl": ["Tl"],
-            "Pb": ["Pb"],
-            "Bi": ["Bi"],
+            "Several trace element (Al, Si, Ti, V, Cr, Mn, Fe)": ["Al", "Si", "Ti", "V", "Cr", "Mn", "Fe"],
+
         },
+        # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
+        "MDL": {
+            'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2, 'Cl': 1.7,
+            'K': 1.2, 'Ca': 0.3, 'Ti': 1.6, 'V': 0.12, 'Cr': 0.12,
+            'Mn': 0.14, 'Fe': 0.17, 'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079,
+            'Zn': 0.067, 'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
+            'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28, 'Zr': 0.33,
+            'Nb': 0.41, 'Mo': 0.48, 'Pd': 2.2, 'Ag': 1.9, 'Cd': 2.5,
+            'In': 3.1, 'Sn': 4.1, 'Sb': 5.2, 'Te': 0.6, 'Cs': 0.37,
+            'Ba': 0.39, 'La': 0.36, 'Ce': 0.3, 'W': 0.0001, 'Pt': 0.12,
+            'Au': 0.1, 'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13, 'Bi': 0.13
+        }
     },
 
     "VOC": {
@@ -174,7 +164,7 @@ meta = {
         "freq": "1h",
         "deter_key": {
             "Main Salt (Na+, NH4+, Cl-, NO3-, SO42-)": ["Na+", "NH4+", "Cl-", "NO3-", "SO42-"],
-            "XRF (Al, Ti, V, Cr, Mn, Fe)": ["Al", "Ti", "V", "Cr", "Mn", "Fe"],
+            "Several trace element (Al, Ti, V, Cr, Mn, Fe)": ["Al", "Ti", "V", "Cr", "Mn", "Fe"],
         },
     },
 }

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/core/__init__.py

@@ -7,11 +7,12 @@ from typing import Optional
 
 import numpy as np
 import pandas as pd
-from pandas import DataFrame, concat, read_pickle
+from pandas import DataFrame, concat, read_pickle, to_numeric
 from rich.console import Console
 from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
 from AeroViz.rawDataReader.config.supported_instruments import meta
+from AeroViz.rawDataReader.core.qc import DataQualityControl
 
 __all__ = ['AbstractReader']
 
@@ -75,18 +76,20 @@ class AbstractReader(ABC):
 
     @abstractmethod
     def _QC(self, df: DataFrame) -> DataFrame:
-        return self.n_sigma_QC(df)
+        return df
 
     def _setup_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.nam)
         logger.setLevel(logging.INFO)
 
         for handler in logger.handlers[:]:
+            handler.close()
             logger.removeHandler(handler)
 
         handler = logging.FileHandler(self.path / f'{self.nam}.log')
         handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
         logger.addHandler(handler)
+
         return logger
 
     def _rate_calculate(self, raw_data, qc_data) -> None:
@@ -94,18 +97,25 @@ class AbstractReader(ABC):
             period_size = len(raw_data.resample('1h').mean().index)
 
             for _nam, _key in self.meta['deter_key'].items():
-                _key, _drop_how = (qc_data.keys(), 'all') if _key is ['all'] else (_key, 'any')
+                _columns_key, _drop_how = (qc_data.keys(), 'all') if _key == ['all'] else (_key, 'any')
 
-                sample_size = len(raw_data[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
-                qc_size = len(qc_data[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                sample_size = len(raw_data[_columns_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                qc_size = len(qc_data[_columns_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
 
                 # validate rate calculation
-                if period_size < sample_size or sample_size < qc_size or period_size == 0 or sample_size == 0:
-                    raise ValueError(f"Invalid sample sizes: period={period_size}, sample={sample_size}, QC={qc_size}")
+                if period_size == 0 or sample_size == 0 or qc_size == 0:
+                    print(f'\t\t\033[91m No data for this period... skipping\033[0m')
+                    continue
+
+                if period_size < sample_size or sample_size < qc_size:
+                    print(
+                        f'\t\tInvalid size relationship: period={period_size}, sample={sample_size}, QC={qc_size}... skipping')
+                    continue
 
-                _acq_rate = round((sample_size / period_size) * 100, 1)
-                _yid_rate = round((qc_size / sample_size) * 100, 1)
-                _OEE_rate = round((qc_size / period_size) * 100, 1)
+                else:
+                    _acq_rate = round((sample_size / period_size) * 100, 1)
+                    _yid_rate = round((qc_size / sample_size) * 100, 1)
+                    _OEE_rate = round((qc_size / period_size) * 100, 1)
 
                 self.logger.info(f'{_nam}:')
                 self.logger.info(f"\tAcquisition rate: {_acq_rate}%")
@@ -114,8 +124,8 @@ class AbstractReader(ABC):
                 self.logger.info(f"{'=' * 60}")
 
                 print(f'\n\t{_nam} : ')
-                print(f'\t\tacquisition rate | yield rate | OEE rate :'
-                      f' \033[91m{_acq_rate}% | {_yid_rate}% -> {_OEE_rate}%\033[0m')
+                print(f'\t\tacquisition rate | yield rate -> OEE rate : '
+                      f'\033[91m{_acq_rate}% | {_yid_rate}% -> {_OEE_rate}%\033[0m')
 
         if self.meta['deter_key'] is not None:
             # use qc_freq to calculate each period rate
@@ -163,9 +173,7 @@ class AbstractReader(ABC):
         new_index = pd.date_range(user_start or df_start, user_end or df_end, freq=freq, name='time')
 
         # Process data: convert to numeric, resample, and reindex
-        return (_df.apply(pd.to_numeric, errors='coerce')
-                .resample(freq).mean()
-                .reindex(new_index))
+        return _df.reindex(new_index)
 
     def _outlier_process(self, _df):
         outlier_file = self.path / 'outlier.json'
@@ -235,8 +243,8 @@ class AbstractReader(ABC):
 
         raw_data = concat(df_list, axis=0).groupby(level=0).first()
 
-        raw_data = self._timeIndex_process(raw_data)
-        qc_data = self._QC(raw_data)
+        raw_data = self._timeIndex_process(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
+        qc_data = self._QC(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
 
         return raw_data, qc_data
 
@@ -279,6 +287,8 @@ class AbstractReader(ABC):
         self.logger.info(f"{'-' * 60}")
 
         if self.rate:
+            _f_raw = _f_raw.apply(to_numeric, errors='coerce')
+            _f_qc = _f_qc.apply(to_numeric, errors='coerce')
             self._rate_calculate(_f_raw, _f_qc)
 
         return _f_qc if self.qc else _f_raw
@@ -297,84 +307,6 @@ class AbstractReader(ABC):
 
         return df[new_order]
 
-    @staticmethod
-    def n_sigma_QC(df: pd.DataFrame, std_range: int = 5) -> pd.DataFrame:
-        # 確保輸入是DataFrame
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-
-        df_ave = df.mean()
-        df_std = df.std()
-
-        lower_bound = df < (df_ave - df_std * std_range)
-        upper_bound = df > (df_ave + df_std * std_range)
-
-        return df.mask(lower_bound | upper_bound)
-
-    @staticmethod
-    def IQR_QC(df: pd.DataFrame, log_dist=False) -> pd.DataFrame:
-        # 確保輸入是DataFrame
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-
-        df_transformed = np.log10(df) if log_dist else df
-
-        _df_q1 = df_transformed.quantile(0.25)
-        _df_q3 = df_transformed.quantile(0.75)
-
-        _df_iqr = _df_q3 - _df_q1
-
-        # Calculate lower and upper bounds
-        lower_bound = df_transformed < (_df_q1 - 1.5 * _df_iqr)
-        upper_bound = df_transformed > (_df_q3 + 1.5 * _df_iqr)
-
-        # Apply the filter to the original dataframe
-        return df.mask(lower_bound | upper_bound)
-
-    @staticmethod
-    def rolling_IQR_QC(df: pd.DataFrame, window_size=24, log_dist=False) -> pd.DataFrame:
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-        df_transformed = np.log10(df) if log_dist else df
-
-        def iqr_filter(x):
-            q1, q3 = x.quantile(0.25), x.quantile(0.75)
-            iqr = q3 - q1
-            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
-            return (x >= lower) & (x <= upper)
-
-        mask = df_transformed.rolling(window=window_size, center=True, min_periods=1).apply(iqr_filter)
-        return df.where(mask, np.nan)
-
     @staticmethod
     def time_aware_IQR_QC(df: pd.DataFrame, time_window='1D', log_dist=False) -> pd.DataFrame:
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-        df_transformed = np.log10(df) if log_dist else df
-
-        def iqr_filter(group):
-            q1, q3 = group.quantile(0.25), group.quantile(0.75)
-            iqr = q3 - q1
-            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
-            return (group >= lower) & (group <= upper)
-
-        mask = df_transformed.groupby(pd.Grouper(freq=time_window)).transform(iqr_filter)
-        return df.where(mask, np.nan)
-
-    @staticmethod
-    def mad_iqr_hybrid_QC(df: pd.DataFrame, mad_threshold=3.5, log_dist=False) -> pd.DataFrame:
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-        df_transformed = np.log10(df) if log_dist else df
-
-        # IQR 方法
-        q1, q3 = df_transformed.quantile(0.25), df_transformed.quantile(0.75)
-        iqr = q3 - q1
-        iqr_lower, iqr_upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
-
-        # MAD 方法
-        median = df_transformed.median()
-        mad = (df_transformed - median).abs().median()
-        mad_lower, mad_upper = median - mad_threshold * mad, median + mad_threshold * mad
-
-        # 结合两种方法
-        lower = np.maximum(iqr_lower, mad_lower)
-        upper = np.minimum(iqr_upper, mad_upper)
-
-        mask = (df_transformed >= lower) & (df_transformed <= upper)
-        return df.where(mask, np.nan)
+        return DataQualityControl().time_aware_iqr(df, time_window=time_window, log_dist=log_dist)

aeroviz-0.1.9.2/AeroViz/rawDataReader/core/qc.py

@@ -0,0 +1,184 @@
+import numpy as np
+import pandas as pd
+
+
+class DataQualityControl:
+    """A class providing various methods for data quality control and outlier detection"""
+
+    @staticmethod
+    def _ensure_dataframe(df: pd.DataFrame | pd.Series) -> pd.DataFrame:
+        """Ensure input data is in DataFrame format"""
+        return df.to_frame() if isinstance(df, pd.Series) else df
+
+    @staticmethod
+    def _transform_if_log(df: pd.DataFrame, log_dist: bool) -> pd.DataFrame:
+        """Transform data to log scale if required"""
+        return np.log10(df) if log_dist else df
+
+    @classmethod
+    def n_sigma(cls, df: pd.DataFrame, std_range: int = 5) -> pd.DataFrame:
+        """
+        Detect outliers using n-sigma method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        std_range : int, default=5
+            Number of standard deviations to use as threshold
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_ave = df.mean()
+        df_std = df.std()
+
+        lower_bound = df < (df_ave - df_std * std_range)
+        upper_bound = df > (df_ave + df_std * std_range)
+
+        return df.mask(lower_bound | upper_bound)
+
+    @classmethod
+    def iqr(cls, df: pd.DataFrame, log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using Interquartile Range (IQR) method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        q1 = df_transformed.quantile(0.25)
+        q3 = df_transformed.quantile(0.75)
+        iqr = q3 - q1
+
+        lower_bound = df_transformed < (q1 - 1.5 * iqr)
+        upper_bound = df_transformed > (q3 + 1.5 * iqr)
+
+        return df.mask(lower_bound | upper_bound)
+
+    @classmethod
+    def rolling_iqr(cls, df: pd.DataFrame, window_size: int = 24,
+                    log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using rolling window IQR method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        window_size : int, default=24
+            Size of the rolling window
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        def iqr_filter(x):
+            q1, q3 = x.quantile(0.25), x.quantile(0.75)
+            iqr = q3 - q1
+            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+            return (x >= lower) & (x <= upper)
+
+        mask = df_transformed.rolling(
+            window=window_size,
+            center=True,
+            min_periods=1
+        ).apply(iqr_filter)
+
+        return df.where(mask, np.nan)
+
+    @classmethod
+    def time_aware_iqr(cls, df: pd.DataFrame, time_window: str = '1D',
+                       log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using time-aware IQR method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        time_window : str, default='1D'
+            Time window size (e.g., '1D' for one day)
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        def iqr_filter(group):
+            q1, q3 = group.quantile(0.25), group.quantile(0.75)
+            iqr = q3 - q1
+            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+            return (group >= lower) & (group <= upper)
+
+        mask = df_transformed.groupby(
+            pd.Grouper(freq=time_window)
+        ).transform(iqr_filter)
+
+        return df.where(mask, np.nan)
+
+    @classmethod
+    def mad_iqr_hybrid(cls, df: pd.DataFrame, mad_threshold: float = 3.5,
+                       log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using a hybrid of MAD and IQR methods
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        mad_threshold : float, default=3.5
+            Threshold for MAD method
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        # IQR method
+        q1, q3 = df_transformed.quantile(0.25), df_transformed.quantile(0.75)
+        iqr = q3 - q1
+        iqr_lower, iqr_upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+
+        # MAD method
+        median = df_transformed.median()
+        mad = (df_transformed - median).abs().median()
+        mad_lower = median - mad_threshold * mad
+        mad_upper = median + mad_threshold * mad
+
+        # Combine both methods
+        lower = np.maximum(iqr_lower, mad_lower)
+        upper = np.minimum(iqr_upper, mad_upper)
+
+        mask = (df_transformed >= lower) & (df_transformed <= upper)
+        return df.where(mask, np.nan)

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/script/AE33.py

@@ -11,14 +11,14 @@ class Reader(AbstractReader):
             self.logger.info(f'\t {file} may not be a whole daily data. Make sure the file is correct.')
 
         _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
-                         delimiter=r'\s+', skiprows=5, usecols=range(67)).apply(to_numeric, errors='coerce')
+                         delimiter=r'\s+', skiprows=5, usecols=range(67))
         _df.columns = _df.columns.str.strip(';')
 
         # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
         if self.meta.get('error_state', False):
             _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].apply(to_numeric, errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/script/AE43.py

@@ -7,7 +7,7 @@ class Reader(AbstractReader):
     nam = 'AE43'
 
     def _raw_reader(self, file):
-        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time').apply(to_numeric, errors='coerce')
+        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time')
         _df_id = _df['SetupID'].iloc[-1]
 
         # get last SetupID data
@@ -18,7 +18,7 @@ class Reader(AbstractReader):
         if self.meta.get('error_state', False):
             _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].apply(to_numeric, errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/script/Aurora.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, low_memory=False, index_col=0).apply(to_numeric, errors='coerce')
+            _df = read_csv(f, low_memory=False, index_col=0)
 
             _df.index = to_datetime(_df.index, errors='coerce')
             _df.index.name = 'time'
@@ -24,7 +24,7 @@ class Reader(AbstractReader):
                 'RH': 'RH'
             })
 
-            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR']]
+            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR']].apply(to_numeric, errors='coerce')
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

{aeroviz-0.1.9.0 → aeroviz-0.1.9.2}/AeroViz/rawDataReader/script/BC1054.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, parse_dates=True, index_col=0).apply(to_numeric, errors='coerce')
+            _df = read_csv(f, parse_dates=True, index_col=0)
 
             _df.columns = _df.columns.str.replace(' ', '')
 
@@ -29,7 +29,8 @@ class Reader(AbstractReader):
             if self.meta.get('error_state', False):
                 _df = _df[~_df['Status'].isin(self.meta.get('error_state'))]
 
-            _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']]
+            _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].apply(to_numeric,
+                                                                                                     errors='coerce')
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]