AeroViz 0.1.9.0__py3-none-any.whl → 0.1.9.2__py3-none-any.whl

AeroViz/dataProcess/Chemistry/_ocec.py

@@ -114,8 +114,7 @@ def _basic(_lcres, _mass, _ocec_ratio, _ocec_ratio_month, _hr_lim, _range, _wiso
     _out = {}
 
     # OC1, OC2, OC3, OC4, PC
-    _df_bsc = _lcres[['OC1_raw', 'OC2_raw', 'OC3_raw', 'OC4_raw']] / _lcres['Sample_Volume'].to_frame().values.copy()
-    _df_bsc.rename(columns={'OC1_raw': 'OC1', 'OC2_raw': 'OC2', 'OC3_raw': 'OC3', 'OC4_raw': 'OC4'}, inplace=True)
+    _df_bsc = _lcres[['OC1', 'OC2', 'OC3', 'OC4', 'PC']].copy()
 
     # SOC, POC, OC/EC
     if _ocec_ratio is not None:
@@ -144,7 +143,8 @@ def _basic(_lcres, _mass, _ocec_ratio, _ocec_ratio_month, _hr_lim, _range, _wiso
     _df_ratio = DataFrame(index=_df_bsc.index)
 
     for _ky, _val in _df_bsc.items():
-        if 'OC/EC' in _ky: continue
+        if 'OC/EC' in _ky:
+            continue
         _df_ratio[f'{_ky}/Thermal_OC'] = _val / _lcres['Thermal_OC']
         _df_ratio[f'{_ky}/Optical_OC'] = _val / _lcres['Optical_OC']
 
@@ -159,14 +159,14 @@ def _basic(_lcres, _mass, _ocec_ratio, _ocec_ratio_month, _hr_lim, _range, _wiso
         _df_ratio[f'Optical_EC/PM'] = _lcres['Optical_EC'] / _mass
 
     # ratio status
-    _df_bsc = concat((_lcres, _df_bsc.copy()), axis=1)
+    _df_bsc = concat((_lcres.loc[:, :'Sample_Volume'], _df_bsc.copy()), axis=1)
 
     for _ky, _df in _df_ratio.items():
         _df_bsc[f'{_ky}_status'] = 'Normal'
         _df_bsc[f'{_ky}_status'] = _df_bsc[f'{_ky}_status'].mask(_df > 1, 'Warning')
 
     # out
-    _out['ratio'] = _df_ratio
     _out['basic'] = _df_bsc
+    _out['ratio'] = _df_ratio
 
     return _out

AeroViz/dataProcess/SizeDistr/_size_distr.py

@@ -69,7 +69,11 @@ def _basic(df, hybrid, unit, bin_rg, input_type):
 
             df_oth[f'total_{_tp_nam}_{_md_nam}'], df_oth[f'GMD_{_tp_nam}_{_md_nam}'], df_oth[
                 f'GSD_{_tp_nam}_{_md_nam}'] = _geometric_prop(_dia, _dt)
-            df_oth[f'mode_{_tp_nam}_{_md_nam}'] = _dt.idxmax(axis=1)
+
+            mask = _dt.notna().any(axis=1)
+
+            df_oth.loc[mask, f'mode_{_tp_nam}_{_md_nam}'] = _dt.loc[mask].idxmax(axis=1)
+            df_oth.loc[~mask, f'mode_{_tp_nam}_{_md_nam}'] = n.nan
 
     ## out
     out_dic['other'] = df_oth

AeroViz/plot/hysplit/hysplit.py

@@ -7,21 +7,29 @@ import pandas as pd
 
 from AeroViz.plot.utils import set_figure
 
-# TODO: Hybrid Single-Particle Lagrangian Integrated Trajectory (HYSPLIT) model
+# Hybrid Single-Particle Lagrangian Integrated Trajectory (HYSPLIT) model
 
 
 __all__ = ['hysplit']
 
 # 設置默認文件路徑
-DEFAULT_FILE = Path(__file__).parent.parent.parent / 'data' / '240228_00.txt'
+DEFAULT_FILE = Path(__file__).parent.parent.parent / 'data' / 'hysplit_example_data.txt'
 
 
 def read_hysplit_data(file: Path):
     data = pd.read_csv(file, skiprows=8, sep=r'\s+', names=range(0, 12), engine='python')
     data = data.reset_index(drop=False)
-    data.columns = ['category', 'name', 'abc', 'year', 'month', 'hour', 'min', 'cont', 'backward', 'lat', 'lon',
+    data.columns = ['category', 'name', 'year', 'month', 'day', 'hour', 'minute', 'count', 'backward', 'lat', 'lon',
                     'height', 'pressure']
 
+    time_cols = ['year', 'month', 'day', 'hour', 'minute']
+
+    data['time'] = pd.to_datetime(data[time_cols].astype(str).agg(''.join, axis=1), format='%y%m%d%H%M')
+
+    data = data.drop(columns=time_cols)
+
+    data = data[['time'] + [col for col in data.columns if col != 'time']]
+
     return data
 
 

AeroViz/rawDataReader/__init__.py

@@ -74,7 +74,7 @@ def RawDataReader(instrument_name: str,
     if not isinstance(path, Path):
         path = Path(path)
     if not path.exists() or not path.is_dir():
-        raise ValueError(f"The specified path '{path}' does not exist or is not a directory.")
+        raise FileNotFoundError(f"The specified path '{path}' does not exist or is not a directory.")
 
     # Validate the QC frequency
     if qc_freq is not None:

AeroViz/rawDataReader/config/supported_instruments.py

@@ -75,6 +75,8 @@ meta = {
             "Thermal EC": ["Thermal_EC"],
             "Optical OC": ["Optical_OC"],
             "Optical EC": ["Optical_EC"],
+            "Thermal OC & EC": ["Thermal_OC", "Thermal_EC"],
+            "Optical OC & EC": ["Optical_OC", "Optical_EC"],
         },
     },
 
@@ -93,53 +95,41 @@ meta = {
             "SO42-": ["SO42-"],
             "Main Salt (NH4+, NO3-, SO42-)": ["NO3-", "SO42-", "NH4+"],
         },
+        # https://www.yangyao-env.com/web/product/product_in2.jsp?pd_id=PD1640151884502
+
+        # HF: 0.08, F-: 0.08, PO43-: None is not measured
+        "MDL": {
+            'HF': None, 'HCl': 0.05, 'HNO2': 0.01, 'HNO3': 0.05, 'G-SO2': 0.05, 'NH3': 0.1,
+            'Na+': 0.05, 'NH4+': 0.08, 'K+': 0.08, 'Mg2+': 0.05, 'Ca2+': 0.05,
+            'F-': None, 'Cl-': 0.05, 'NO2-': 0.05, 'NO3-': 0.01, 'PO43-': None, 'SO42-': 0.05,
+        },
+
+        "MR": {
+            'HF': 200, 'HCl': 200, 'HNO2': 200, 'HNO3': 200, 'G-SO2': 200, 'NH3': 300,
+            'Na+': 300, 'NH4+': 300, 'K+': 300, 'Mg2+': 300, 'Ca2+': 300,
+            'F-': 300, 'Cl-': 300, 'NO2-': 300, 'NO3-': 300, 'PO43-': None, 'SO42-': 300,
+        }
     },
 
     "XRF": {
         "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": {
-            "Al": ["Al"],
-            "Si": ["Si"],
-            "P": ["P"],
-            "S": ["S"],
-            "Cl": ["Cl"],
-            "K": ["K"],
-            "Ca": ["Ca"],
-            "Ti": ["Ti"],
-            "V": ["V"],
-            "Cr": ["Cr"],
-            "Mn": ["Mn"],
-            "Fe": ["Fe"],
-            "Ni": ["Ni"],
-            "Cu": ["Cu"],
-            "Zn": ["Zn"],
-            "As": ["As"],
-            "Se": ["Se"],
-            "Br": ["Br"],
-            "Rb": ["Rb"],
-            "Sr": ["Sr"],
-            "Y": ["Y"],
-            "Zr": ["Zr"],
-            "Mo": ["Mo"],
-            "Ag": ["Ag"],
-            "Cd": ["Cd"],
-            "In": ["In"],
-            "Sn": ["Sn"],
-            "Sb": ["Sb"],
-            "Te": ["Te"],
-            "Cs": ["Cs"],
-            "Ba": ["Ba"],
-            "La": ["La"],
-            "Ce": ["Ce"],
-            "W": ["W"],
-            "Pt": ["Pt"],
-            "Au": ["Au"],
-            "Hg": ["Hg"],
-            "Tl": ["Tl"],
-            "Pb": ["Pb"],
-            "Bi": ["Bi"],
+            "Several trace element (Al, Si, Ti, V, Cr, Mn, Fe)": ["Al", "Si", "Ti", "V", "Cr", "Mn", "Fe"],
+
         },
+        # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
+        "MDL": {
+            'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2, 'Cl': 1.7,
+            'K': 1.2, 'Ca': 0.3, 'Ti': 1.6, 'V': 0.12, 'Cr': 0.12,
+            'Mn': 0.14, 'Fe': 0.17, 'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079,
+            'Zn': 0.067, 'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
+            'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28, 'Zr': 0.33,
+            'Nb': 0.41, 'Mo': 0.48, 'Pd': 2.2, 'Ag': 1.9, 'Cd': 2.5,
+            'In': 3.1, 'Sn': 4.1, 'Sb': 5.2, 'Te': 0.6, 'Cs': 0.37,
+            'Ba': 0.39, 'La': 0.36, 'Ce': 0.3, 'W': 0.0001, 'Pt': 0.12,
+            'Au': 0.1, 'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13, 'Bi': 0.13
+        }
     },
 
     "VOC": {
@@ -174,7 +164,7 @@ meta = {
         "freq": "1h",
         "deter_key": {
             "Main Salt (Na+, NH4+, Cl-, NO3-, SO42-)": ["Na+", "NH4+", "Cl-", "NO3-", "SO42-"],
-            "XRF (Al, Ti, V, Cr, Mn, Fe)": ["Al", "Ti", "V", "Cr", "Mn", "Fe"],
+            "Several trace element (Al, Ti, V, Cr, Mn, Fe)": ["Al", "Ti", "V", "Cr", "Mn", "Fe"],
         },
     },
 }

AeroViz/rawDataReader/core/__init__.py

@@ -7,11 +7,12 @@ from typing import Optional
 
 import numpy as np
 import pandas as pd
-from pandas import DataFrame, concat, read_pickle
+from pandas import DataFrame, concat, read_pickle, to_numeric
 from rich.console import Console
 from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
 from AeroViz.rawDataReader.config.supported_instruments import meta
+from AeroViz.rawDataReader.core.qc import DataQualityControl
 
 __all__ = ['AbstractReader']
 
@@ -75,18 +76,20 @@ class AbstractReader(ABC):
 
     @abstractmethod
     def _QC(self, df: DataFrame) -> DataFrame:
-        return self.n_sigma_QC(df)
+        return df
 
     def _setup_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.nam)
         logger.setLevel(logging.INFO)
 
         for handler in logger.handlers[:]:
+            handler.close()
             logger.removeHandler(handler)
 
         handler = logging.FileHandler(self.path / f'{self.nam}.log')
         handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
         logger.addHandler(handler)
+
         return logger
 
     def _rate_calculate(self, raw_data, qc_data) -> None:
@@ -94,18 +97,25 @@ class AbstractReader(ABC):
             period_size = len(raw_data.resample('1h').mean().index)
 
             for _nam, _key in self.meta['deter_key'].items():
-                _key, _drop_how = (qc_data.keys(), 'all') if _key is ['all'] else (_key, 'any')
+                _columns_key, _drop_how = (qc_data.keys(), 'all') if _key == ['all'] else (_key, 'any')
 
-                sample_size = len(raw_data[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
-                qc_size = len(qc_data[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                sample_size = len(raw_data[_columns_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                qc_size = len(qc_data[_columns_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
 
                 # validate rate calculation
-                if period_size < sample_size or sample_size < qc_size or period_size == 0 or sample_size == 0:
-                    raise ValueError(f"Invalid sample sizes: period={period_size}, sample={sample_size}, QC={qc_size}")
+                if period_size == 0 or sample_size == 0 or qc_size == 0:
+                    print(f'\t\t\033[91m No data for this period... skipping\033[0m')
+                    continue
+
+                if period_size < sample_size or sample_size < qc_size:
+                    print(
+                        f'\t\tInvalid size relationship: period={period_size}, sample={sample_size}, QC={qc_size}... skipping')
+                    continue
 
-                _acq_rate = round((sample_size / period_size) * 100, 1)
-                _yid_rate = round((qc_size / sample_size) * 100, 1)
-                _OEE_rate = round((qc_size / period_size) * 100, 1)
+                else:
+                    _acq_rate = round((sample_size / period_size) * 100, 1)
+                    _yid_rate = round((qc_size / sample_size) * 100, 1)
+                    _OEE_rate = round((qc_size / period_size) * 100, 1)
 
                 self.logger.info(f'{_nam}:')
                 self.logger.info(f"\tAcquisition rate: {_acq_rate}%")
@@ -114,8 +124,8 @@ class AbstractReader(ABC):
                 self.logger.info(f"{'=' * 60}")
 
                 print(f'\n\t{_nam} : ')
-                print(f'\t\tacquisition rate | yield rate | OEE rate :'
-                      f' \033[91m{_acq_rate}% | {_yid_rate}% -> {_OEE_rate}%\033[0m')
+                print(f'\t\tacquisition rate | yield rate -> OEE rate : '
+                      f'\033[91m{_acq_rate}% | {_yid_rate}% -> {_OEE_rate}%\033[0m')
 
         if self.meta['deter_key'] is not None:
             # use qc_freq to calculate each period rate
@@ -163,9 +173,7 @@ class AbstractReader(ABC):
         new_index = pd.date_range(user_start or df_start, user_end or df_end, freq=freq, name='time')
 
         # Process data: convert to numeric, resample, and reindex
-        return (_df.apply(pd.to_numeric, errors='coerce')
-                .resample(freq).mean()
-                .reindex(new_index))
+        return _df.reindex(new_index)
 
     def _outlier_process(self, _df):
         outlier_file = self.path / 'outlier.json'
@@ -235,8 +243,8 @@ class AbstractReader(ABC):
 
         raw_data = concat(df_list, axis=0).groupby(level=0).first()
 
-        raw_data = self._timeIndex_process(raw_data)
-        qc_data = self._QC(raw_data)
+        raw_data = self._timeIndex_process(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
+        qc_data = self._QC(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
 
         return raw_data, qc_data
 
@@ -279,6 +287,8 @@ class AbstractReader(ABC):
         self.logger.info(f"{'-' * 60}")
 
         if self.rate:
+            _f_raw = _f_raw.apply(to_numeric, errors='coerce')
+            _f_qc = _f_qc.apply(to_numeric, errors='coerce')
             self._rate_calculate(_f_raw, _f_qc)
 
         return _f_qc if self.qc else _f_raw
@@ -297,84 +307,6 @@ class AbstractReader(ABC):
 
         return df[new_order]
 
-    @staticmethod
-    def n_sigma_QC(df: pd.DataFrame, std_range: int = 5) -> pd.DataFrame:
-        # 確保輸入是DataFrame
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-
-        df_ave = df.mean()
-        df_std = df.std()
-
-        lower_bound = df < (df_ave - df_std * std_range)
-        upper_bound = df > (df_ave + df_std * std_range)
-
-        return df.mask(lower_bound | upper_bound)
-
-    @staticmethod
-    def IQR_QC(df: pd.DataFrame, log_dist=False) -> pd.DataFrame:
-        # 確保輸入是DataFrame
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-
-        df_transformed = np.log10(df) if log_dist else df
-
-        _df_q1 = df_transformed.quantile(0.25)
-        _df_q3 = df_transformed.quantile(0.75)
-
-        _df_iqr = _df_q3 - _df_q1
-
-        # Calculate lower and upper bounds
-        lower_bound = df_transformed < (_df_q1 - 1.5 * _df_iqr)
-        upper_bound = df_transformed > (_df_q3 + 1.5 * _df_iqr)
-
-        # Apply the filter to the original dataframe
-        return df.mask(lower_bound | upper_bound)
-
-    @staticmethod
-    def rolling_IQR_QC(df: pd.DataFrame, window_size=24, log_dist=False) -> pd.DataFrame:
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-        df_transformed = np.log10(df) if log_dist else df
-
-        def iqr_filter(x):
-            q1, q3 = x.quantile(0.25), x.quantile(0.75)
-            iqr = q3 - q1
-            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
-            return (x >= lower) & (x <= upper)
-
-        mask = df_transformed.rolling(window=window_size, center=True, min_periods=1).apply(iqr_filter)
-        return df.where(mask, np.nan)
-
     @staticmethod
     def time_aware_IQR_QC(df: pd.DataFrame, time_window='1D', log_dist=False) -> pd.DataFrame:
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-        df_transformed = np.log10(df) if log_dist else df
-
-        def iqr_filter(group):
-            q1, q3 = group.quantile(0.25), group.quantile(0.75)
-            iqr = q3 - q1
-            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
-            return (group >= lower) & (group <= upper)
-
-        mask = df_transformed.groupby(pd.Grouper(freq=time_window)).transform(iqr_filter)
-        return df.where(mask, np.nan)
-
-    @staticmethod
-    def mad_iqr_hybrid_QC(df: pd.DataFrame, mad_threshold=3.5, log_dist=False) -> pd.DataFrame:
-        df = df.to_frame() if isinstance(df, pd.Series) else df
-        df_transformed = np.log10(df) if log_dist else df
-
-        # IQR 方法
-        q1, q3 = df_transformed.quantile(0.25), df_transformed.quantile(0.75)
-        iqr = q3 - q1
-        iqr_lower, iqr_upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
-
-        # MAD 方法
-        median = df_transformed.median()
-        mad = (df_transformed - median).abs().median()
-        mad_lower, mad_upper = median - mad_threshold * mad, median + mad_threshold * mad
-
-        # 结合两种方法
-        lower = np.maximum(iqr_lower, mad_lower)
-        upper = np.minimum(iqr_upper, mad_upper)
-
-        mask = (df_transformed >= lower) & (df_transformed <= upper)
-        return df.where(mask, np.nan)
+        return DataQualityControl().time_aware_iqr(df, time_window=time_window, log_dist=log_dist)

AeroViz/rawDataReader/core/qc.py

@@ -0,0 +1,184 @@
+import numpy as np
+import pandas as pd
+
+
+class DataQualityControl:
+    """A class providing various methods for data quality control and outlier detection"""
+
+    @staticmethod
+    def _ensure_dataframe(df: pd.DataFrame | pd.Series) -> pd.DataFrame:
+        """Ensure input data is in DataFrame format"""
+        return df.to_frame() if isinstance(df, pd.Series) else df
+
+    @staticmethod
+    def _transform_if_log(df: pd.DataFrame, log_dist: bool) -> pd.DataFrame:
+        """Transform data to log scale if required"""
+        return np.log10(df) if log_dist else df
+
+    @classmethod
+    def n_sigma(cls, df: pd.DataFrame, std_range: int = 5) -> pd.DataFrame:
+        """
+        Detect outliers using n-sigma method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        std_range : int, default=5
+            Number of standard deviations to use as threshold
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_ave = df.mean()
+        df_std = df.std()
+
+        lower_bound = df < (df_ave - df_std * std_range)
+        upper_bound = df > (df_ave + df_std * std_range)
+
+        return df.mask(lower_bound | upper_bound)
+
+    @classmethod
+    def iqr(cls, df: pd.DataFrame, log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using Interquartile Range (IQR) method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        q1 = df_transformed.quantile(0.25)
+        q3 = df_transformed.quantile(0.75)
+        iqr = q3 - q1
+
+        lower_bound = df_transformed < (q1 - 1.5 * iqr)
+        upper_bound = df_transformed > (q3 + 1.5 * iqr)
+
+        return df.mask(lower_bound | upper_bound)
+
+    @classmethod
+    def rolling_iqr(cls, df: pd.DataFrame, window_size: int = 24,
+                    log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using rolling window IQR method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        window_size : int, default=24
+            Size of the rolling window
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        def iqr_filter(x):
+            q1, q3 = x.quantile(0.25), x.quantile(0.75)
+            iqr = q3 - q1
+            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+            return (x >= lower) & (x <= upper)
+
+        mask = df_transformed.rolling(
+            window=window_size,
+            center=True,
+            min_periods=1
+        ).apply(iqr_filter)
+
+        return df.where(mask, np.nan)
+
+    @classmethod
+    def time_aware_iqr(cls, df: pd.DataFrame, time_window: str = '1D',
+                       log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using time-aware IQR method
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        time_window : str, default='1D'
+            Time window size (e.g., '1D' for one day)
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        def iqr_filter(group):
+            q1, q3 = group.quantile(0.25), group.quantile(0.75)
+            iqr = q3 - q1
+            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+            return (group >= lower) & (group <= upper)
+
+        mask = df_transformed.groupby(
+            pd.Grouper(freq=time_window)
+        ).transform(iqr_filter)
+
+        return df.where(mask, np.nan)
+
+    @classmethod
+    def mad_iqr_hybrid(cls, df: pd.DataFrame, mad_threshold: float = 3.5,
+                       log_dist: bool = False) -> pd.DataFrame:
+        """
+        Detect outliers using a hybrid of MAD and IQR methods
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input data
+        mad_threshold : float, default=3.5
+            Threshold for MAD method
+        log_dist : bool, default=False
+            Whether to apply log transformation to data
+
+        Returns
+        -------
+        pd.DataFrame
+            Cleaned DataFrame with outliers masked as NaN
+        """
+        df = cls._ensure_dataframe(df)
+        df_transformed = cls._transform_if_log(df, log_dist)
+
+        # IQR method
+        q1, q3 = df_transformed.quantile(0.25), df_transformed.quantile(0.75)
+        iqr = q3 - q1
+        iqr_lower, iqr_upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+
+        # MAD method
+        median = df_transformed.median()
+        mad = (df_transformed - median).abs().median()
+        mad_lower = median - mad_threshold * mad
+        mad_upper = median + mad_threshold * mad
+
+        # Combine both methods
+        lower = np.maximum(iqr_lower, mad_lower)
+        upper = np.minimum(iqr_upper, mad_upper)
+
+        mask = (df_transformed >= lower) & (df_transformed <= upper)
+        return df.where(mask, np.nan)

AeroViz/rawDataReader/script/AE33.py

@@ -11,14 +11,14 @@ class Reader(AbstractReader):
             self.logger.info(f'\t {file} may not be a whole daily data. Make sure the file is correct.')
 
         _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
-                         delimiter=r'\s+', skiprows=5, usecols=range(67)).apply(to_numeric, errors='coerce')
+                         delimiter=r'\s+', skiprows=5, usecols=range(67))
         _df.columns = _df.columns.str.strip(';')
 
         # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
         if self.meta.get('error_state', False):
             _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].apply(to_numeric, errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/AE43.py

@@ -7,7 +7,7 @@ class Reader(AbstractReader):
     nam = 'AE43'
 
     def _raw_reader(self, file):
-        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time').apply(to_numeric, errors='coerce')
+        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time')
         _df_id = _df['SetupID'].iloc[-1]
 
         # get last SetupID data
@@ -18,7 +18,7 @@ class Reader(AbstractReader):
         if self.meta.get('error_state', False):
             _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].apply(to_numeric, errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/Aurora.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, low_memory=False, index_col=0).apply(to_numeric, errors='coerce')
+            _df = read_csv(f, low_memory=False, index_col=0)
 
             _df.index = to_datetime(_df.index, errors='coerce')
             _df.index.name = 'time'
@@ -24,7 +24,7 @@ class Reader(AbstractReader):
                 'RH': 'RH'
             })
 
-            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR']]
+            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR']].apply(to_numeric, errors='coerce')
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/BC1054.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, parse_dates=True, index_col=0).apply(to_numeric, errors='coerce')
+            _df = read_csv(f, parse_dates=True, index_col=0)
 
             _df.columns = _df.columns.str.replace(' ', '')
 
@@ -29,7 +29,8 @@ class Reader(AbstractReader):
             if self.meta.get('error_state', False):
                 _df = _df[~_df['Status'].isin(self.meta.get('error_state'))]
 
-            _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']]
+            _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].apply(to_numeric,
+                                                                                                     errors='coerce')
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/EPA.py

@@ -1,3 +1,4 @@
+import numpy as np
 from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -14,7 +15,7 @@ class Reader(AbstractReader):
     def _raw_reader(self, file):
         # 查詢小時值(測項).csv & 查詢小時值(直式).csv (有、無輸出有效值都可以)
         df = read_csv(file, encoding='big5', encoding_errors='ignore', index_col=0, parse_dates=True,
-                      on_bad_lines='skip').apply(to_numeric, errors='coerce')
+                      on_bad_lines='skip')
 
         if len(df.groupby('測站')) > 1:
             raise ValueError(f'Multiple stations found in the file: {df['測站'].unique()}')
@@ -29,11 +30,12 @@ class Reader(AbstractReader):
             df.index.name = 'Time'
 
             # 如果沒有將無效值拿掉就輸出 請將包含 #、L 的字串替換成 # 或 _
-            df = df.replace(to_replace=r'\d*[#]\b', value='#', regex=True)
-            df = df.replace(to_replace=r'\d*[L]\b', value='_', regex=True)
+            df = df.replace(to_replace=r'\d*\.?\d*[#]\b', value='#', regex=True)
+            df = df.replace(to_replace=r'\d*\.?\d*[L]\b', value='_', regex=True)
 
             # 欄位排序
-            return self.reorder_dataframe_columns(df, [desired_order1])
+            return self.reorder_dataframe_columns(df, [desired_order1]).apply(to_numeric, errors='coerce')
 
     def _QC(self, _df):
+        _df = _df.mask(_df < 0, np.nan)
         return _df

AeroViz/rawDataReader/script/GRIMM.py

@@ -7,7 +7,6 @@ class Reader(AbstractReader):
     nam = 'GRIMM'
 
     def _raw_reader(self, file):
-
         _df = read_csv(file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
                        dayfirst=True).rename_axis("Time")
         _df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)

AeroViz/rawDataReader/script/IGAC.py

@@ -12,11 +12,13 @@ class Reader(AbstractReader):
     def _raw_reader(self, file):
 
         with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, parse_dates=True, index_col=0, na_values='-').apply(to_numeric, errors='coerce')
+            _df = read_csv(f, parse_dates=True, index_col=0, na_values='-')
 
             _df.columns = _df.keys().str.strip(' ')
             _df.index.name = 'time'
 
+            _df = _df.apply(to_numeric, errors='coerce')
+
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     def _QC(self, _df):

AeroViz/rawDataReader/script/MA350.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         _df = read_csv(file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis(
-            "Time").apply(to_numeric, errors='coerce')
+            "Time")
 
         _df = _df.rename(columns={
             'UV BCc': 'BC1',
@@ -26,7 +26,8 @@ class Reader(AbstractReader):
         # if self.meta.get('error_state', False):
         #     _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE', 'BB']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE', 'BB']].apply(to_numeric,
+                                                                                                           errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/Minion.py

@@ -2,8 +2,9 @@ from typing import Literal
 
 import numpy as np
 import pandas
-from pandas import read_excel, to_numeric
+from pandas import DataFrame, read_excel
 
+from AeroViz.rawDataReader.config.supported_instruments import meta
 from AeroViz.rawDataReader.core import AbstractReader
 
 pandas.set_option("future.no_silent_downcasting", True)
@@ -13,143 +14,181 @@ desired_order1 = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC',
 
 desired_order2 = ['Benzene', 'Toluene', 'EthylBenzene', 'm/p-Xylene', 'o-Xylene']
 
-desired_order3 = ['Al', 'Si', 'P', 'S', 'Cl', 'K', 'Ca', 'Ti', 'V', 'Cr', 'Mn', 'Fe',
-                  'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Rb', 'Sr',
-                  'Y', 'Zr', 'Nb', 'Mo', 'Pd', 'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te',
-                  'Cs', 'Ba', 'La', 'Ce', 'W', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi']
-
-desired_order4 = ['NH3', 'HF', 'HCl', 'HNO2', 'HNO3', 'G-SO2',
-                  'Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+',
-                  'F-', 'Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-']
+MDL_NUMBER = -999
 
 
 class Reader(AbstractReader):
     nam = 'Minion'
 
+    # 楠梓8月數據(環境部)(空品、重金屬和氣膠可用率) -> 楠梓8月數據_level1 -> NZ_minion_XXXX
     def _raw_reader(self, file):
-        # 讀取 Excel 文件
         df = read_excel(file, index_col=0, parse_dates=True)
+        df.index.name = 'Time'
 
         # 重命名列，去除空白
         df = df.rename(columns=lambda x: x.strip())
 
-        # 保存單位行並給它一個名稱
-        units = df.iloc[0].copy()
+        # 保存單位
+        self.units = df.iloc[0].copy()
 
         # 刪除原始數據中的單位行
         df = df.iloc[1:]
 
         # 替換特定值
-        df = df.replace({'維護校正': '*', np.nan: '-', '0L': '_', 'Nodata': '-'}, inplace=False)
-        df = df.replace(to_replace=r'\d*[#]\b', value='#', regex=True)
-        df = df.replace(to_replace=r'\d*[L]\b', value='_', regex=True)
+        df = df.replace({'維護校正': '*', np.nan: '-', 'Nodata': '-', '0L': MDL_NUMBER})
+        # df = df.replace(to_replace=r'\d*\.?\d*[#]\b', value='_', regex=True)
+        df = df.replace(to_replace=r'\d*\.?\d*[L]\b', value=MDL_NUMBER, regex=True)
+
+        # 處理除了'WD'列的 0 值 替換為 '_'
+        for col in [col for col in df.columns if col != 'WD']:
+            df[col] = df[col].replace({0: MDL_NUMBER})
+
+        # replace to numeric for estimating qc rate
+        df = df.replace({'_': MDL_NUMBER})
 
-        # 處理除了'WD'列的 0 值
-        non_wd_columns = [col for col in df.columns if col != 'WD']
-        df.loc[:, non_wd_columns] = df.loc[:, non_wd_columns].replace({0: '_'})
+        XRF_col = list(meta.get('XRF').get('MDL').keys())
+        IGAC_col = list(meta.get('IGAC').get('MDL').keys())
 
         # 重新排序列
-        df = self.reorder_dataframe_columns(df, [desired_order1, desired_order2, desired_order3, desired_order4])
+        df = self.reorder_dataframe_columns(df, [desired_order1, desired_order2, XRF_col, IGAC_col])
 
         # 將單位行添加回 DataFrame
         # df = concat([units.to_frame().T, df])
 
-        df.index.name = 'Time'
+        # save Level1 data
+        output_folder = file.parent / 'Level1'
+        output_folder.mkdir(parents=True, exist_ok=True)
+        df.to_csv(output_folder / f'{file.stem}_Level1.csv')
 
         return df.loc[~df.index.duplicated() & df.index.notna()]
 
     def _QC(self, _df):
+        IGAC_col = list(meta.get('IGAC').get('MDL'))
+        XRF_col = list(meta.get('XRF').get('MDL'))
+
+        # IGAC MDL QC
+        _df[IGAC_col] = self.IGAC_QAQC(_df[IGAC_col])
+
+        # XRF MDL QC
+        _df[XRF_col] = self.XRF_QAQC(_df[XRF_col])
+
         # remove negative value
-        _df = _df.mask((_df < 0))
+        # _df = _df.mask((_df < 0))
+        _df = _df.mask(_df == MDL_NUMBER, np.nan)
 
-        # XRF QAQC
-        _df = self.XRF_QAQC(_df)
+        col = [col for col in desired_order1 if col != 'WD']
+        _df[col] = self.time_aware_IQR_QC(_df[col])
 
-        # ions balance
-        _df = self.IGAC_QAQC(_df)
+        # Calculate the mass and ion balance
+        # mass tolerance = ± 1, ions balance tolerance = ± 1
 
-        # QC data in 6h
-        _df = self.time_aware_IQR_QC(_df)
+        # # conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
+        # _df_salt = df.mask(df.sum(axis=1, min_count=1) > df.PM25).dropna(subset=_main).copy()
+
+        ions_mass = _df[['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'Cl-', 'NO3-', 'SO42-']].sum(axis=1)
+        element_mass = _df[XRF_col].sum(axis=1)
+
+        estimated_mass = ions_mass + element_mass
+
+        valid_mask = 2 * _df['PM2.5'] > estimated_mass
+
+        _df.loc[~valid_mask, IGAC_col + XRF_col] = np.nan
 
         return _df
 
-    # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
-    def XRF_QAQC(self, df, MDL_replace: Literal['nan', '0.5 * MDL'] = 'nan'):
-        MDL = {
-            'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2,
-            'Cl': 1.7, 'K': 1.2, 'Ca': 0.3, 'Ti': 1.6,
-            'V': 0.12, 'Cr': 0.12, 'Mn': 0.14, 'Fe': 0.17,
-            'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079, 'Zn': 0.067,
-            'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
-            'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28,
-            'Zr': 0.33, 'Nb': 0.41, 'Mo': 0.48, 'Pd': 2.2,
-            'Ag': 1.9, 'Cd': 2.5, 'In': 3.1, 'Sn': 4.1,
-            'Sb': 5.2, 'Te': 0.6, 'Cs': 0.37, 'Ba': 0.39,
-            'La': 0.36, 'Ce': 0.3, 'W': 0.0001, 'Pt': 0.12,
-            'Au': 0.1, 'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13,
-            'Bi': 0.13
-        }
-
-        # Br Li internal standard
-
-        # 將小於 MDL 值的數據替換為 nan or 1/2 MDL
-        for element, threshold in MDL.items():
-            if element in df.columns:
-                rep = np.nan if MDL_replace == 'nan' else 0.5 * threshold
-                df[element] = df[element].where(df[element] >= threshold, rep)
-
-        self.logger.info(f"{'=' * 60}")
-        self.logger.info(f"XRF QAQC summary:")
-        self.logger.info("\t\ttransform values below MDL to NaN")
-        self.logger.info(f"{'=' * 60}")
+    def mdlReplace_timeAware_qc(self, df: DataFrame, MDL: dict, MDL_replace) -> DataFrame:
+        # Step 1: Track MDL positions and values below threshold
+        mdl_mask = (df.eq(MDL_NUMBER) |
+                    df.apply(lambda x: x < MDL.get(x.name, float('-inf'))))
+
+        # Step 2: Convert all values below MDL to MDL_NUMBER (-999)
+        df_mdl = df.mask(mdl_mask, MDL_NUMBER)
+
+        # Step 3: Apply time_aware_IQR_QC (excluding MDL_NUMBER values)
+        df_qc = self.time_aware_IQR_QC(df_mdl.mask(df_mdl == MDL_NUMBER))
+
+        # Step 4: Handle values below MDL according to specified method
+        if MDL_replace == '0.5 * MDL':
+            for column, threshold in MDL.items():
+                if column in df.columns and threshold is not None:
+                    df_qc.loc[df_mdl[column] == MDL_NUMBER, column] = 0.5 * threshold
+                else:
+                    df_qc.loc[df_mdl[column] == MDL_NUMBER, column] = np.nan
+        else:  # 'nan'
+            df_qc = df_qc.mask(df_mdl == MDL_NUMBER, np.nan)
+
+        return df_qc
+
+    def XRF_QAQC(self,
+                 df: DataFrame,
+                 MDL_replace: Literal['nan', '0.5 * MDL'] = '0.5 * MDL'
+                 ) -> DataFrame:
+        """
+        Perform Quality Assurance and Quality Control for XRF data
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input dataframe with XRF data
+        MDL_replace : {'nan', '0.5 * MDL'}, default='nan'
+            Method to handle values below MDL:
+            - 'nan': Replace with NaN
+            - '0.5 * MDL': Replace with half of MDL value
+
+        Returns
+        -------
+        pd.DataFrame
+            Processed dataframe with QC applied and MDL values handled
+        """
+        MDL = meta.get('XRF').get('MDL')
+
+        df = self.mdlReplace_timeAware_qc(df, MDL, MDL_replace)
 
         # 轉換單位 ng/m3 -> ug/m3
         if df.Al.max() > 10 and df.Fe.max() > 10:
-            # 確保 MDL.keys() 中的所有列都存在於 _df 中
             columns_to_convert = [col for col in MDL.keys() if col in df.columns]
-
             df[columns_to_convert] = df[columns_to_convert].div(1000)
 
+        self.logger.info(f"XRF QAQC summary: transform values below MDL to {MDL_replace}")
+
         return df
 
-    def IGAC_QAQC(self, df, tolerance=1):
+    def IGAC_QAQC(self,
+                  df: DataFrame,
+                  MDL_replace: Literal['nan', '0.5 * MDL'] = '0.5 * MDL',
+                  tolerance: float = 1
+                  ) -> DataFrame:
         """
-        Calculate the balance of ions in the system
+        Perform Quality Assurance and Quality Control for IGAC data
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input dataframe with IGAC data
+        MDL_replace : {'nan', '0.5 * MDL'}, default='nan'
+            Method to handle values below MDL:
+            - 'nan': Replace with NaN
+            - '0.5 * MDL': Replace with half of MDL value
+        tolerance : float, default=1
+            Tolerance value for QC checks
+
+        Returns
+        -------
+        pd.DataFrame
+            Processed dataframe with QC applied and MDL values handled
         """
-        # https://www.yangyao-env.com/web/product/product_in2.jsp?pd_id=PD1640151884502
-        MDL = {
-            'HF': 0.08, 'HCl': 0.05, 'HNO2': 0.01, 'HNO3': 0.05, 'G-SO2': 0.05, 'NH3': 0.1,
-            'Na+': 0.05, 'NH4+': 0.08, 'K+': 0.08, 'Mg2+': 0.05, 'Ca2+': 0.05,
-            'F-': 0.08, 'Cl-': 0.05, 'NO2-': 0.05, 'NO3-': 0.01, 'PO43-': None, 'SO42-': 0.05,
-        }
-
-        MR = {
-            'HF': 200, 'HCl': 200, 'HNO2': 200, 'HNO3': 200, 'G-SO2': 200, 'NH3': 300,
-            'Na+': 300, 'NH4+': 300, 'K+': 300, 'Mg2+': 300, 'Ca2+': 300,
-            'F-': 300, 'Cl-': 300, 'NO2-': 300, 'NO3-': 300, 'PO43-': None, 'SO42-': 300,
-        }
+        MDL = meta.get('IGAC').get('MDL')
 
+        df = self.mdlReplace_timeAware_qc(df, MDL, MDL_replace)
+
+        # Define the ions
+        _df = df.copy()
         _cation, _anion, _main = (['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'],
                                   ['Cl-', 'NO2-', 'NO3-', 'SO42-'],
                                   ['SO42-', 'NO3-', 'NH4+'])
-        # QC: replace values below MDL with 0.5 * MDL -> ions balance -> PM2.5 > main salt
-        # mass tolerance = 0.3, ions balance tolerance = 0.3
-
-        # # conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
-        # _df_salt = df.mask(df.sum(axis=1, min_count=1) > df.PM25).dropna(subset=_main).copy()
-
-        # Define the ions
-        item = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'Cl-', 'NO2-', 'NO3-', 'SO42-']
 
-        # Calculate the balance
-        _df = df[item].apply(to_numeric, errors='coerce')
-
-        # for (_key, _df_col) in _df.items():
-        #     _df[_key] = _df_col.mask(_df_col < MDL[_key], MDL[_key] / 2)
-
-        _df['+_mole'] = _df[['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+']].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1,
-                                                                                                             skipna=True)
-        _df['-_mole'] = _df[['Cl-', 'NO2-', 'NO3-', 'SO42-']].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
+        _df['+_mole'] = _df[_cation].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1, skipna=True)
+        _df['-_mole'] = _df[_anion].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
 
         # Avoid division by zero
         _df['ratio'] = np.where(_df['-_mole'] != 0, _df['+_mole'] / _df['-_mole'], np.nan)
@@ -157,24 +196,19 @@ class Reader(AbstractReader):
         # Calculate bounds
         lower_bound, upper_bound = 1 - tolerance, 1 + tolerance
 
-        # 根据ratio决定是否保留原始数据
+        # 根據ratio决定是否保留原始数据
         valid_mask = ((_df['ratio'] <= upper_bound) & (_df['ratio'] >= lower_bound) &
                       ~np.isnan(_df['+_mole']) & ~np.isnan(_df['-_mole']))
 
-        # 保留数据或将不符合条件的行设为NaN
-        df.loc[~valid_mask, item] = np.nan
+        # 保留数據或將不符合的條件設為NaN
+        df.loc[~valid_mask] = np.nan
 
-        # 计算保留的数据的百分比
+        # 計算保留的数據的百分比
         retained_percentage = (valid_mask.sum() / len(df)) * 100
 
-        self.logger.info(f"{'=' * 60}")
-        self.logger.info(f"Ions balance summary:")
-        self.logger.info(f"\t\tretain {retained_percentage.__round__(0)}% data within tolerance {tolerance}")
-        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Ions balance summary: {retained_percentage.__round__(0)}% within tolerance ± {tolerance}")
 
         if retained_percentage < 70:
             self.logger.warning("Warning: The percentage of retained data is less than 70%")
 
-        # print(f"\tretain {retained_percentage.__round__(0)}% data within tolerance {tolerance}")
-
         return df

AeroViz/rawDataReader/script/NEPH.py

@@ -1,4 +1,4 @@
-from pandas import to_datetime, read_csv, DataFrame
+from pandas import to_datetime, read_csv, DataFrame, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -42,7 +42,7 @@ class Reader(AbstractReader):
 
                 _df_out.mask(_df_out['status'] != 0)  # 0000 -> numeric to 0
 
-                _df = _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
+                _df = _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']].apply(to_numeric, errors='coerce')
 
                 return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/OCEC.py

@@ -9,7 +9,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3).apply(to_numeric, errors='coerce')
+            _df = read_csv(f, skiprows=3)
 
             _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
             _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
@@ -51,6 +51,8 @@ class Reader(AbstractReader):
                 'ECPk5-ug C': 'EC5_raw',
             })
 
+            _df = _df.apply(to_numeric, errors='coerce')
+
             _df['OC1'] = _df['OC1_raw'] / _df['Sample_Volume']
             _df['OC2'] = _df['OC2_raw'] / _df['Sample_Volume']
             _df['OC3'] = _df['OC3_raw'] / _df['Sample_Volume']
@@ -64,7 +66,7 @@ class Reader(AbstractReader):
             # _df['EC4'] = _df['EC4_raw'] / _df['Sample_Volume']
             # _df['EC5'] = _df['EC5_raw'] / _df['Sample_Volume']
 
-            _df = _df[['Thermal_OC', 'Optical_OC', 'Thermal_EC', 'Optical_EC', 'TC', 'Sample_Volume',
+            _df = _df[['Thermal_OC', 'Thermal_EC', 'Optical_OC', 'Optical_EC', 'TC', 'Sample_Volume',
                        'OC1', 'OC2', 'OC3', 'OC4', 'PC']]
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]

AeroViz/rawDataReader/script/TEOM.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3, index_col=False).apply(to_numeric, errors='coerce')
+            _df = read_csv(f, skiprows=3, index_col=False)
 
             _df = _df.rename(columns={'Time Stamp': 'time',
                                       'System status': 'status',
@@ -27,7 +27,7 @@ class Reader(AbstractReader):
 
             _df = _df.where(_df['status'] < 1)
 
-        _df = _df[['PM_NV', 'PM_Total', 'noise']]
+        _df = _df[['PM_NV', 'PM_Total', 'noise']].apply(to_numeric, errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

{AeroViz-0.1.9.0.dist-info → AeroViz-0.1.9.2.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.9.0
+Version: 0.1.9.2
 Summary: Aerosol science
 Home-page: https://github.com/Alex870521/AeroViz
 Author: alex
 Author-email: alex870521@gmail.com
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
@@ -21,6 +22,8 @@ Requires-Dist: windrose ==1.9.2
 Requires-Dist: cartopy ==0.24.1
 Requires-Dist: tabulate ==0.9.0
 Requires-Dist: rich ~=13.7.1
+Provides-Extra: test
+Requires-Dist: pytest >=7.0.0 ; extra == 'test'
 
 ## <div align="center">AeroViz for Aerosol Science Visualization</div>
 

{AeroViz-0.1.9.0.dist-info → AeroViz-0.1.9.2.dist-info}/RECORD

@@ -1,13 +1,13 @@
 AeroViz/__init__.py,sha256=A5W6SR71uY_eW44Sh-Yk6blJQ_G1aHrkSzeP2YTPQc4,371
-AeroViz/data/240228_00.txt,sha256=DWfY83EW3fOcv9dW-Y4pudq8-M7BJlXD-tlMSYrAk2w,8946
 AeroViz/data/DEFAULT_DATA.csv,sha256=eeeyeh8vSLKkE5tAF0TYnUNOyQIH98VA41bJaAP204Y,2248526
 AeroViz/data/DEFAULT_PNSD_DATA.csv,sha256=imLvLA80oYwo_jzXZtlQn5hZ76d47HUIlK2jp0tZPrg,2636511
+AeroViz/data/hysplit_example_data.txt,sha256=DWfY83EW3fOcv9dW-Y4pudq8-M7BJlXD-tlMSYrAk2w,8946
 AeroViz/dataProcess/__init__.py,sha256=D3rTVUiGfs_daGuaotVtbijOgLAp6HaRWchj-zoEnHw,828
 AeroViz/dataProcess/Chemistry/__init__.py,sha256=fyyomjxkQcUNWDx4R5jPrHafAftN-v2liUZii9OlaiU,2058
 AeroViz/dataProcess/Chemistry/_calculate.py,sha256=q7ojTFPok0vg8k_1PMECNdP5CPanR9NWQ4Rx5iTcHew,599
 AeroViz/dataProcess/Chemistry/_isoropia.py,sha256=3wp_FXdN230awlStMbctutwld4oot9WaAVXETGd6PSs,3255
 AeroViz/dataProcess/Chemistry/_mass_volume.py,sha256=0joH2BAx0NUwDFzyrLgG-v7WrGl46R7zWxwbajWBV8o,5378
-AeroViz/dataProcess/Chemistry/_ocec.py,sha256=FKvuh6iMhz6eFne9WJZFyaeo0FyV9a1KmaZ0nm4-67I,6031
+AeroViz/dataProcess/Chemistry/_ocec.py,sha256=1UpSwdxYVy_LpUjtoaEUoXA-r1nKEFJoyYglf_6CoXA,5899
 AeroViz/dataProcess/Chemistry/_partition.py,sha256=tKhb6BJns46UiUlEq6Zq7ahYnvUJ_whY3tWE54C3bqU,1023
 AeroViz/dataProcess/Chemistry/_teom.py,sha256=IiM-TrifWpQLTbKllG-4k4c3mvQulfcmjswWu6muCXA,486
 AeroViz/dataProcess/Chemistry/isrpia.cnf,sha256=iWXTqsOZFmNrJxAI9nYuilZ9h6ru1icdPFVim7YKc_k,566
@@ -28,7 +28,7 @@ AeroViz/dataProcess/SizeDistr/_merge_v1.py,sha256=6Anb8DszoatK66tc9ccA6ZApbqtL7p
 AeroViz/dataProcess/SizeDistr/_merge_v2.py,sha256=8OzUKw7hTg-yuQBipuFKgBS_7c7zbApN_BNr00G8q9c,9046
 AeroViz/dataProcess/SizeDistr/_merge_v3.py,sha256=HN2ARFmeWOawOWRPPv_pHEGBBZNgXVbH4dDTxcN7rdY,18749
 AeroViz/dataProcess/SizeDistr/_merge_v4.py,sha256=b8RVAievGIOLrmJHJXRsKXQ1tkMkm6rx43S7XAfeXE4,16228
-AeroViz/dataProcess/SizeDistr/_size_distr.py,sha256=o5fTwLH7j9j5129e3uciSJrOR8AGCN3tkAyGffoPENg,3127
+AeroViz/dataProcess/SizeDistr/_size_distr.py,sha256=ULhGKlxE9QmbDO_PS3HOSKzepeMfJZWabJvGXqsDEvE,3259
 AeroViz/dataProcess/VOC/__init__.py,sha256=8GNP0RMymTkJXK18pSgfLHqrKPWboN-3x1_Ke4UrI44,259
 AeroViz/dataProcess/VOC/_potential_par.py,sha256=h3rVdvtBvC6xHa_ZG4Oq5eXezeSZtHNy6T6I40maIcM,3863
 AeroViz/dataProcess/VOC/support_voc.json,sha256=tMYp_NERqhSriVRE2NavXh33CQ5CnsbJHtmMFlE5q_E,6804
@@ -44,7 +44,7 @@ AeroViz/plot/violin.py,sha256=pU2Z2yTWocEtImmCAmbtn0WvXtUOrnCGOdDOrLxjooU,2689
 AeroViz/plot/distribution/__init__.py,sha256=nhbIegWczkuEfWsE7-2jfF0dnpmPDzJJzjq8Fuh6q5k,28
 AeroViz/plot/distribution/distribution.py,sha256=sAjqtqKavFwQqI8PGPFnpvZFSU-w2UKjcTTC5L91f4E,20595
 AeroViz/plot/hysplit/__init__.py,sha256=VrEkha2OEFp_00Xj9R98C96niZ7fYqJzGPeYsbojtzA,23
-AeroViz/plot/hysplit/hysplit.py,sha256=yDIQuhlP3IPXRy0BCRkzqyJ_PfYPqIF-S1lpksW1dFk,2504
+AeroViz/plot/hysplit/hysplit.py,sha256=gSCkemFLRvsk4m8zYbxbsjrdU14NkN9ZNfVRvdq69aM,2796
 AeroViz/plot/meteorology/__init__.py,sha256=hhGfQE3IUzS3Eaju_nO7LomPPHJnd-zAAZZweXOXs2M,27
 AeroViz/plot/meteorology/meteorology.py,sha256=6hk-5olgQTw2SB-GhEizLN19vRVBztgiXoruh8Q6Zns,11282
 AeroViz/plot/optical/PyMieScatt_update.py,sha256=g3vlzATjzYSYZd3LwmwxEmdkpo4cHJ3KY4rePY4jwZk,21065
@@ -68,24 +68,25 @@ AeroViz/plot/utils/fRH.json,sha256=t-2ux4TLOYAB-4jJ72LSM4jv1jk9XkaxKYNMDepMHIg,6
 AeroViz/plot/utils/plt_utils.py,sha256=7Au3r2-7AZQmzrO2OfcyTFomJRLHgu1Npb7wxQqUvzY,3438
 AeroViz/plot/utils/sklearn_utils.py,sha256=hKFfkVkYLRxkIDKvO9COHXwhjD_UWqQigdT3mDToni4,2098
 AeroViz/plot/utils/units.json,sha256=JKcqvLA6dkc8REV_NhX85Jl9LA4iAZxzw9RJp6JTla0,2965
-AeroViz/rawDataReader/__init__.py,sha256=0JUjzD54KRWQQ5C07zpVDwHlg02QoSaQ89rRFB4RwNM,4852
+AeroViz/rawDataReader/__init__.py,sha256=FDUsJ_v6wG9DUcNOjmqaWJvVdMgatJrqGYPZdbaW7Wo,4859
 AeroViz/rawDataReader/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-AeroViz/rawDataReader/config/supported_instruments.py,sha256=CGE34wsXyq-Za8IIYu2rt3JzoANrFBgrphqvl2FYTC0,5296
-AeroViz/rawDataReader/core/__init__.py,sha256=rXEFM1gi5TjOmYh5ouov-zFvHH22gI-X26-WRGemwu4,15106
-AeroViz/rawDataReader/script/AE33.py,sha256=FGET-JGW2H4cTRoduuIS8VGFYK7jEnPIhqlBJuNAGFQ,1278
-AeroViz/rawDataReader/script/AE43.py,sha256=H8lrI6jqLTRwbslVmbkB_YxEcPl42uEVIfKk6WGTTBI,1231
+AeroViz/rawDataReader/config/supported_instruments.py,sha256=RR2TPome27O3ERNxQ043boDMfvWG9BHvhpGJ6VQC5gw,5988
+AeroViz/rawDataReader/core/__init__.py,sha256=i5JTd8IhhTYI2bpEMQ27XmxPZojACUzWnKcw4gGofnE,12691
+AeroViz/rawDataReader/core/qc.py,sha256=tFIVsfph8yZIK6NRKQxaZYHcruJclriKSvR0oC12T0Q,5698
+AeroViz/rawDataReader/script/AE33.py,sha256=FbbFJ93aLVjA8k2QZ_fKcI9uXoux2k0AL3O73iY879I,1278
+AeroViz/rawDataReader/script/AE43.py,sha256=GjcICBJ3nIANyMd4kovteBUtkyCGLTos07BczgSCuVE,1231
 AeroViz/rawDataReader/script/APS_3321.py,sha256=x75G72Xl0vElr6Njbv8SlOcosAHNozseaJzAxVmfXyI,1697
-AeroViz/rawDataReader/script/Aurora.py,sha256=HDLyHOw62tgfLjjwYCWVAJKsc4SB7aLnIjI6HI_WTRM,1491
-AeroViz/rawDataReader/script/BC1054.py,sha256=pvHnUA_gJIRf9jEUi8vzNyErXMHRQWLnOsExIZR0_OA,1574
-AeroViz/rawDataReader/script/EPA.py,sha256=lIdWx9roM1unyqSjTtd5aAOGoITcU5e-P0XYt0k8Mjg,1578
-AeroViz/rawDataReader/script/GRIMM.py,sha256=UyWeqZfOcbIVCmLk_0P8xSh6eQiq_U2Gse84O2mTnlQ,847
-AeroViz/rawDataReader/script/IGAC.py,sha256=ZdskNc65wVx2znmbjJp2J_rxVg5vuqxB1HWRoqxb7Ho,2364
-AeroViz/rawDataReader/script/MA350.py,sha256=w0QCoJxMIMwaLOLWLE65FM7MY9kcvpkRMAowRA5TaYk,1490
-AeroViz/rawDataReader/script/Minion.py,sha256=BjuJe2KWLJKgvVnV0WfilGh2DvaWqJDDNj8i0z3oeuU,7306
-AeroViz/rawDataReader/script/NEPH.py,sha256=6qs2oiS6zDOFkTNEu9T-8hrCuoZHPfjd5UoMacobAno,3168
-AeroViz/rawDataReader/script/OCEC.py,sha256=45jaEp1cjmPPjbf44eErhE2wO6GXpDVsJIQ9jPj1XcE,3390
+AeroViz/rawDataReader/script/Aurora.py,sha256=2duNsK2WCWk21Rd2d4EugAA_yN27p2AjRFd9ClJ2aUA,1491
+AeroViz/rawDataReader/script/BC1054.py,sha256=tuDyq8M5BPbmu1yJr9zXYS2piMGz08yTQXGT6tK9jxA,1675
+AeroViz/rawDataReader/script/EPA.py,sha256=1ZXEcCnIMOhEXu0JwzeCgmhRtPzBNo2CfLhfhstOT4k,1649
+AeroViz/rawDataReader/script/GRIMM.py,sha256=-D4U83ihjAqcvOAnk7NET59IZfV1JzPYKRQjrIQyBDM,846
+AeroViz/rawDataReader/script/IGAC.py,sha256=i6WT3rX0n0e4hq7NfWN6tVwCuKAeV9ARxPkXZSbQj74,2387
+AeroViz/rawDataReader/script/MA350.py,sha256=EfPTFhgDAjI7r0G6kW7pjog-4MBOnvW0cyFqIkCxEP8,1597
+AeroViz/rawDataReader/script/Minion.py,sha256=9G_q-EhE3nfJoxWFwAnMYdY0teSYqcYxTkk0JW5lmY0,7793
+AeroViz/rawDataReader/script/NEPH.py,sha256=x6HgnvpmmhOOvB4-nL-jTfoSo0x8FUxVBXPqAyfhZVk,3215
+AeroViz/rawDataReader/script/OCEC.py,sha256=jWWaNbCjP5MJDYrdWUhjrQLClaWqC8SGDVPIFJ9xljU,3413
 AeroViz/rawDataReader/script/SMPS.py,sha256=EtXmeukOIwqfMwMJqv99_STfVg0uPdVr96r-tfD95gk,2774
-AeroViz/rawDataReader/script/TEOM.py,sha256=Ew4JqDf_qpGFvvBLwX824kKIRXMM6QZLwEB4t4xkTSk,2103
+AeroViz/rawDataReader/script/TEOM.py,sha256=jsxU4W46FmLjiIthmPOHo6CAYFZiPENhW80WjMCiIPA,2103
 AeroViz/rawDataReader/script/VOC.py,sha256=GUme72ZyjSzREsFNUgOV_OCESIVJBXY9KrKP1c9Av7I,1248
 AeroViz/rawDataReader/script/XRF.py,sha256=SU1-D94GkwdkjlNXcyXbwQG1tOYCpeL6GTVkaLBHc-s,187
 AeroViz/rawDataReader/script/__init__.py,sha256=s3c797Q8EAGcJCxVRTA-KdHie-vHLNYbMxwa5c0qz-I,214
@@ -94,8 +95,8 @@ AeroViz/tools/database.py,sha256=05VzjJyhlRrhsZdhfFQ__7CxGm4MdFekLjz3_Is5h9U,343
 AeroViz/tools/dataclassifier.py,sha256=_wpv0PlZ5EGkcNqHxfFtdEsYvHP5FVE8sMZXikhm_YE,4492
 AeroViz/tools/dataprinter.py,sha256=Jq2Yztpa9YCOeLDVTrRs7PhSdNIPhEAexVj1YSuJ7hY,2249
 AeroViz/tools/datareader.py,sha256=iTQ0U8hdNMjCdbiH7EiKW10UEoxzxXRHc4s5_1IikJo,1933
-AeroViz-0.1.9.0.dist-info/LICENSE,sha256=E-679GpGGkp3irmtuJXiT7R4cNUA4cmsH6Q7QUgPf5U,1069
-AeroViz-0.1.9.0.dist-info/METADATA,sha256=Zl55acxCrzwi4z_s5W3PwjY4LEzrbHv3O-gtZZae0T4,6253
-AeroViz-0.1.9.0.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-AeroViz-0.1.9.0.dist-info/top_level.txt,sha256=BYsmTst_o4FZOKRP1XIvIMlN6mMTTXNfnSToL2_nVbQ,8
-AeroViz-0.1.9.0.dist-info/RECORD,,
+AeroViz-0.1.9.2.dist-info/LICENSE,sha256=E-679GpGGkp3irmtuJXiT7R4cNUA4cmsH6Q7QUgPf5U,1069
+AeroViz-0.1.9.2.dist-info/METADATA,sha256=qTnQ4ONlpadkTfYa5lhvSQ3DxuI4p_geNcv16f_bOjI,6373
+AeroViz-0.1.9.2.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+AeroViz-0.1.9.2.dist-info/top_level.txt,sha256=BYsmTst_o4FZOKRP1XIvIMlN6mMTTXNfnSToL2_nVbQ,8
+AeroViz-0.1.9.2.dist-info/RECORD,,