AeroViz 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl

AeroViz/rawDataReader/core/__init__.py

@@ -105,15 +105,17 @@ class AbstractReader(ABC):
 
                 _acq_rate = round((sample_size / period_size) * 100, 1)
                 _yid_rate = round((qc_size / sample_size) * 100, 1)
+                _OEE_rate = round((qc_size / period_size) * 100, 1)
 
                 self.logger.info(f'{_nam}:')
                 self.logger.info(f"\tAcquisition rate: {_acq_rate}%")
                 self.logger.info(f'\tYield       rate: {_yid_rate}%')
+                self.logger.info(f'\tOEE         rate: {_OEE_rate}%')
                 self.logger.info(f"{'=' * 60}")
 
                 print(f'\n\t{_nam} : ')
-                print(f'\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-                print(f'\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+                print(f'\t\tacquisition rate | yield rate | OEE rate :'
+                      f' \033[91m{_acq_rate}% | {_yid_rate}% -> {_OEE_rate}%\033[0m')
 
         if self.meta['deter_key'] is not None:
             # use qc_freq to calculate each period rate
@@ -296,22 +298,83 @@ class AbstractReader(ABC):
         return df[new_order]
 
     @staticmethod
-    def n_sigma_QC(df: DataFrame, std_range: int = 5) -> DataFrame:
-        df_ave, df_std = df.mean(), df.std()
-        df_lowb, df_highb = df < (df_ave - df_std * std_range), df > (df_ave + df_std * std_range)
+    def n_sigma_QC(df: pd.DataFrame, std_range: int = 5) -> pd.DataFrame:
+        # 確保輸入是DataFrame
+        df = df.to_frame() if isinstance(df, pd.Series) else df
 
-        return df.mask(df_lowb | df_highb).copy()
+        df_ave = df.mean()
+        df_std = df.std()
+
+        lower_bound = df < (df_ave - df_std * std_range)
+        upper_bound = df > (df_ave + df_std * std_range)
+
+        return df.mask(lower_bound | upper_bound)
 
-    # "四分位數範圍法"（Inter-quartile Range Method）
     @staticmethod
-    def IQR_QC(df: DataFrame, log_dist=False) -> tuple[DataFrame, DataFrame]:
-        df = np.log10(df) if log_dist else df
+    def IQR_QC(df: pd.DataFrame, log_dist=False) -> pd.DataFrame:
+        # 確保輸入是DataFrame
+        df = df.to_frame() if isinstance(df, pd.Series) else df
+
+        df_transformed = np.log10(df) if log_dist else df
+
+        _df_q1 = df_transformed.quantile(0.25)
+        _df_q3 = df_transformed.quantile(0.75)
 
-        _df_qua = df.quantile([.25, .75])
-        _df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
         _df_iqr = _df_q3 - _df_q1
 
-        _se = concat([_df_q1 - 1.5 * _df_iqr] * len(df), axis=1).T.set_index(df.index)
-        _le = concat([_df_q3 + 1.5 * _df_iqr] * len(df), axis=1).T.set_index(df.index)
+        # Calculate lower and upper bounds
+        lower_bound = df_transformed < (_df_q1 - 1.5 * _df_iqr)
+        upper_bound = df_transformed > (_df_q3 + 1.5 * _df_iqr)
+
+        # Apply the filter to the original dataframe
+        return df.mask(lower_bound | upper_bound)
+
+    @staticmethod
+    def rolling_IQR_QC(df: pd.DataFrame, window_size=24, log_dist=False) -> pd.DataFrame:
+        df = df.to_frame() if isinstance(df, pd.Series) else df
+        df_transformed = np.log10(df) if log_dist else df
+
+        def iqr_filter(x):
+            q1, q3 = x.quantile(0.25), x.quantile(0.75)
+            iqr = q3 - q1
+            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+            return (x >= lower) & (x <= upper)
+
+        mask = df_transformed.rolling(window=window_size, center=True, min_periods=1).apply(iqr_filter)
+        return df.where(mask, np.nan)
+
+    @staticmethod
+    def time_aware_IQR_QC(df: pd.DataFrame, time_window='1D', log_dist=False) -> pd.DataFrame:
+        df = df.to_frame() if isinstance(df, pd.Series) else df
+        df_transformed = np.log10(df) if log_dist else df
 
-        return (10 ** _se, 10 ** _le) if log_dist else (_se, _le)
+        def iqr_filter(group):
+            q1, q3 = group.quantile(0.25), group.quantile(0.75)
+            iqr = q3 - q1
+            lower, upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+            return (group >= lower) & (group <= upper)
+
+        mask = df_transformed.groupby(pd.Grouper(freq=time_window)).transform(iqr_filter)
+        return df.where(mask, np.nan)
+
+    @staticmethod
+    def mad_iqr_hybrid_QC(df: pd.DataFrame, mad_threshold=3.5, log_dist=False) -> pd.DataFrame:
+        df = df.to_frame() if isinstance(df, pd.Series) else df
+        df_transformed = np.log10(df) if log_dist else df
+
+        # IQR 方法
+        q1, q3 = df_transformed.quantile(0.25), df_transformed.quantile(0.75)
+        iqr = q3 - q1
+        iqr_lower, iqr_upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+
+        # MAD 方法
+        median = df_transformed.median()
+        mad = (df_transformed - median).abs().median()
+        mad_lower, mad_upper = median - mad_threshold * mad, median + mad_threshold * mad
+
+        # 结合两种方法
+        lower = np.maximum(iqr_lower, mad_lower)
+        upper = np.minimum(iqr_upper, mad_upper)
+
+        mask = (df_transformed >= lower) & (df_transformed <= upper)
+        return df.where(mask, np.nan)

AeroViz/rawDataReader/script/AE33.py

@@ -1,4 +1,4 @@
-from pandas import read_table
+from pandas import read_table, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -8,10 +8,10 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         if file.stat().st_size / 1024 < 550:
-            print('\t It may not be a whole daily data.')
+            self.logger.info(f'\t {file} may not be a whole daily data. Make sure the file is correct.')
 
         _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
-                         delimiter=r'\s+', skiprows=5, usecols=range(67))
+                         delimiter=r'\s+', skiprows=5, usecols=range(67)).apply(to_numeric, errors='coerce')
         _df.columns = _df.columns.str.strip(';')
 
         # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
@@ -23,8 +23,13 @@ class Reader(AbstractReader):
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     def _QC(self, _df):
+        _index = _df.index.copy()
+
         # remove negative value
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
+        _df = _df.mask((_df <= 0) | (_df > 20000))
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/AE43.py

@@ -1,4 +1,4 @@
-from pandas import read_csv
+from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -7,7 +7,7 @@ class Reader(AbstractReader):
     nam = 'AE43'
 
     def _raw_reader(self, file):
-        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time')
+        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time').apply(to_numeric, errors='coerce')
         _df_id = _df['SetupID'].iloc[-1]
 
         # get last SetupID data
@@ -24,8 +24,13 @@ class Reader(AbstractReader):
 
     # QC data
     def _QC(self, _df):
+        _index = _df.index.copy()
+
         # remove negative value
-        _df = _df.mask((_df < 0).copy())
+        _df = _df.mask((_df <= 0) | (_df > 20000))
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/Aurora.py

@@ -1,4 +1,4 @@
-from pandas import to_datetime, read_csv
+from pandas import to_datetime, read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, low_memory=False, index_col=0)
+            _df = read_csv(f, low_memory=False, index_col=0).apply(to_numeric, errors='coerce')
 
             _df.index = to_datetime(_df.index, errors='coerce')
             _df.index.name = 'time'
@@ -24,17 +24,21 @@ class Reader(AbstractReader):
                 'RH': 'RH'
             })
 
-            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
+            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR']]
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-    # QC data
     def _QC(self, _df):
-        # remove negative value
-        _df = _df.mask((_df <= 0) | (_df > 2000)).copy()
+        _index = _df.index.copy()
 
-        # total scattering is larger than back scattering
-        _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
+        _df = _df.mask((_df <= 0) | (_df > 2000))
 
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        _df = _df.loc[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
+
+        _df = _df.loc[(_df['B'] > _df['G']) & (_df['G'] > _df['R'])]
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df)
+
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/BC1054.py

@@ -1,4 +1,4 @@
-from pandas import read_csv
+from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, parse_dates=True, index_col=0)
+            _df = read_csv(f, parse_dates=True, index_col=0).apply(to_numeric, errors='coerce')
 
             _df.columns = _df.columns.str.replace(' ', '')
 
@@ -33,10 +33,14 @@ class Reader(AbstractReader):
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-    # QC data
     def _QC(self, _df):
+        _index = _df.index.copy()
+
         # remove negative value
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
+        _df = _df.mask((_df <= 0) | (_df > 20000))
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/EPA.py

@@ -1,4 +1,4 @@
-from pandas import read_csv
+from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -14,7 +14,7 @@ class Reader(AbstractReader):
     def _raw_reader(self, file):
         # 查詢小時值(測項).csv & 查詢小時值(直式).csv (有、無輸出有效值都可以)
         df = read_csv(file, encoding='big5', encoding_errors='ignore', index_col=0, parse_dates=True,
-                      on_bad_lines='skip')
+                      on_bad_lines='skip').apply(to_numeric, errors='coerce')
 
         if len(df.groupby('測站')) > 1:
             raise ValueError(f'Multiple stations found in the file: {df['測站'].unique()}')
@@ -36,4 +36,4 @@ class Reader(AbstractReader):
             return self.reorder_dataframe_columns(df, [desired_order1])
 
     def _QC(self, _df):
-        return _df.resample('6h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        return _df

AeroViz/rawDataReader/script/GRIMM.py

@@ -24,5 +24,4 @@ class Reader(AbstractReader):
         return _df / 0.035
 
     def _QC(self, _df):
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        return _df

AeroViz/rawDataReader/script/MA350.py

@@ -1,4 +1,4 @@
-from pandas import read_csv
+from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -7,7 +7,8 @@ class Reader(AbstractReader):
     nam = 'MA350'
 
     def _raw_reader(self, file):
-        _df = read_csv(file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis("Time")
+        _df = read_csv(file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis(
+            "Time").apply(to_numeric, errors='coerce')
 
         _df = _df.rename(columns={
             'UV BCc': 'BC1',
@@ -31,8 +32,14 @@ class Reader(AbstractReader):
 
     # QC data
     def _QC(self, _df):
+        _index = _df.index.copy()
+
         # remove negative value
-        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'AAE', 'BB']].mask((_df < 0).copy())
+        _df = _df.mask(
+            (_df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5']] <= 0) | (_df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5']] > 20000))
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/Minion.py

@@ -60,7 +60,7 @@ class Reader(AbstractReader):
 
     def _QC(self, _df):
         # remove negative value
-        _df = _df.mask((_df < 0).copy())
+        _df = _df.mask((_df < 0))
 
         # XRF QAQC
         _df = self.XRF_QAQC(_df)
@@ -69,7 +69,9 @@ class Reader(AbstractReader):
         _df = self.IGAC_QAQC(_df)
 
         # QC data in 6h
-        return _df.resample('6h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        _df = self.time_aware_IQR_QC(_df)
+
+        return _df
 
     # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
     def XRF_QAQC(self, df, MDL_replace: Literal['nan', '0.5 * MDL'] = 'nan'):
@@ -87,7 +89,10 @@ class Reader(AbstractReader):
             'Au': 0.1, 'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13,
             'Bi': 0.13
         }
-        # 將小於 MDL 值的數據替換為 nan or 5/6 MDL
+
+        # Br Li internal standard
+
+        # 將小於 MDL 值的數據替換為 nan or 1/2 MDL
         for element, threshold in MDL.items():
             if element in df.columns:
                 rep = np.nan if MDL_replace == 'nan' else 0.5 * threshold
@@ -137,7 +142,7 @@ class Reader(AbstractReader):
         item = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'Cl-', 'NO2-', 'NO3-', 'SO42-']
 
         # Calculate the balance
-        _df = df[item].apply(lambda x: to_numeric(x, errors='coerce'))
+        _df = df[item].apply(to_numeric, errors='coerce')
 
         # for (_key, _df_col) in _df.items():
         #     _df[_key] = _df_col.mask(_df_col < MDL[_key], MDL[_key] / 2)

AeroViz/rawDataReader/script/NEPH.py

@@ -58,13 +58,23 @@ class Reader(AbstractReader):
                 print(f'\n\t\t\t Length mismatch in {file} data. Returning an empty DataFrame.')
                 return _df_out
 
-    # QC data
     def _QC(self, _df):
+        MDL_sensitivity = {'B': .1, 'G': .1, 'R': .3}
+
+        _index = _df.index.copy()
+
         # remove negative value
-        _df = _df.mask((_df <= 5).copy())
+        _df = _df.mask((_df <= 0) | (_df > 2000))
 
         # total scattering is larger than back scattering
-        _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
+        _df = _df.loc[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
+
+        # blue scattering is larger than green scattering, green scattering is larger than red scattering
+        # due to the nephelometer's Green PMT in FS is already aged, this QC may delete too many data
+        # _df = _df.loc[(_df['B'] > _df['G']) & (_df['G'] > _df['R'])]
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df)
 
-        # QC data in 1h
-        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/OCEC.py

@@ -1,4 +1,5 @@
-from pandas import to_datetime, read_csv
+import numpy as np
+from pandas import to_datetime, read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -8,11 +9,18 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3, nrows=25)
+            _df = read_csv(f, skiprows=3).apply(to_numeric, errors='coerce')
 
             _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
             _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
+
+            if _df['time'].isna().any():
+                _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %H:%M:%S', errors='coerce')
+
             _df = _df.set_index('time')
+
+            _df = _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
             _df.index = _df.index.round('1h')
 
             _df = _df.rename(columns={
@@ -34,6 +42,8 @@ class Reader(AbstractReader):
                 'OCPk2-ug C': 'OC2_raw',
                 'OCPk3-ug C': 'OC3_raw',
                 'OCPk4-ug C': 'OC4_raw',
+                'Pyrolized C ug': 'PC_raw',
+
                 'ECPk1-ug C': 'EC1_raw',
                 'ECPk2-ug C': 'EC2_raw',
                 'ECPk3-ug C': 'EC3_raw',
@@ -41,26 +51,40 @@ class Reader(AbstractReader):
                 'ECPk5-ug C': 'EC5_raw',
             })
 
+            _df['OC1'] = _df['OC1_raw'] / _df['Sample_Volume']
+            _df['OC2'] = _df['OC2_raw'] / _df['Sample_Volume']
+            _df['OC3'] = _df['OC3_raw'] / _df['Sample_Volume']
+            _df['OC4'] = _df['OC4_raw'] / _df['Sample_Volume']
+
+            _df['PC'] = _df['Thermal_OC'] - _df['OC1'] - _df['OC2'] - _df['OC3'] - _df['OC4']
+
+            # _df['EC1'] = _df['EC1_raw'] / _df['Sample_Volume']
+            # _df['EC2'] = _df['EC2_raw'] / _df['Sample_Volume']
+            # _df['EC3'] = _df['EC3_raw'] / _df['Sample_Volume']
+            # _df['EC4'] = _df['EC4_raw'] / _df['Sample_Volume']
+            # _df['EC5'] = _df['EC5_raw'] / _df['Sample_Volume']
+
             _df = _df[['Thermal_OC', 'Optical_OC', 'Thermal_EC', 'Optical_EC', 'TC', 'Sample_Volume',
-                       'OC1_raw', 'OC2_raw', 'OC3_raw', 'OC4_raw', 'EC1_raw', 'EC2_raw', 'EC3_raw', 'EC4_raw',
-                       'EC5_raw']]
+                       'OC1', 'OC2', 'OC3', 'OC4', 'PC']]
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     # QC data
     def _QC(self, _df):
-        import numpy as np
+        MDL = {'Thermal_OC': 0.3,
+               'Optical_OC': 0.3,
+               'Thermal_EC': 0.015,
+               'Optical_EC': 0.015
+               }
+
+        _index = _df.index.copy()
 
-        _df = _df.mask((_df <= 0) | (_df > 100)).copy()
+        _df = _df.mask((_df <= -5) | (_df > 100))
 
-        thresholds = {
-            'Thermal_OC': 0.3,
-            'Optical_OC': 0.3,
-            'Thermal_EC': 0.015,
-            'Optical_EC': 0.015
-        }
+        for col, threshold in MDL.items():
+            _df.loc[_df[col] <= threshold, col] = np.nan
 
-        for col, thresh in thresholds.items():
-            _df.loc[_df[col] <= thresh, col] = np.nan
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
-        return _df
+        return _df.dropna(subset=['Thermal_OC', 'Optical_OC']).reindex(_index)

AeroViz/rawDataReader/script/TEOM.py

@@ -1,4 +1,4 @@
-from pandas import to_datetime, read_csv
+from pandas import to_datetime, read_csv, Timedelta, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3, index_col=False)
+            _df = read_csv(f, skiprows=3, index_col=False).apply(to_numeric, errors='coerce')
 
             _df = _df.rename(columns={'Time Stamp': 'time',
                                       'System status': 'status',
@@ -33,16 +33,20 @@ class Reader(AbstractReader):
 
     # QC data
     def _QC(self, _df):
-
-        _df_idx = _df.index.copy()
+        _index = _df.index.copy()
 
         # remove negative value
-        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df <= 0).copy())
+        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df <= 0))
 
         # QC data in 1 hr
-        # remove data where size < 8 in 1-hr
+        # use time_aware_IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
+
+        # remove data where size < 50% in 1-hr
+        points_per_hour = Timedelta('1h') / Timedelta(self.meta['freq'])
         for _key in ['PM_Total', 'PM_NV']:
-            _size = _df[_key].dropna().resample('1h').size().reindex(_df_idx).ffill().copy()
-            _df[_key] = _df[_key].mask(_size < 8)
+            _size = _df[_key].dropna().resample('1h').size().reindex(_index).ffill()
+            _df[_key] = _df[_key].mask(_size < points_per_hour * 0.5)
 
-        return _df.reindex(_df_idx)
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/VOC.py

@@ -26,7 +26,7 @@ class Reader(AbstractReader):
             if valid_keys:
                 return _df[valid_keys].loc[~_df.index.duplicated() & _df.index.notna()]
             else:
-                self.logger.warning("沒有找到匹配的鍵。返回原始DataFrame並移除含NaN的行。")
+                self.logger.warning("沒有找到匹配的鍵。返回原始DataFrame。")
                 return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     def _QC(self, _df):

{AeroViz-0.1.7.dist-info → AeroViz-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.7
+Version: 0.1.8
 Summary: Aerosol science
 Home-page: https://github.com/Alex870521/AeroViz
 Author: alex
@@ -56,14 +56,19 @@ Requires-Dist: rich ~=13.7.1
 pip install AeroViz
 ```
 
+For Windows users: Run `install_windows.bat`
+
+For Linux and Mac users: Run `install_unix.bat`
+
 ## <div align="center">Quick Start</div>
 
 ```python
-import AeroViz
+from datetime import datetime
+from pathlib import Path
 from AeroViz import RawDataReader, DataProcess, plot
 
 # Read data from a supported instrument
-data = RawDataReader('NEPH', '/path/to/data', start='2024-01-01', end='2024-01-31')
+data = RawDataReader('NEPH', Path('/path/to/data'), start=datetime(2024, 2, 1), end=datetime(2024, 4, 30))
 
 # Create a visualization
 plot.timeseries(data, y='scattering_coefficient')
@@ -71,13 +76,13 @@ plot.timeseries(data, y='scattering_coefficient')
 
 For more detailed usage instructions, please refer to our [User Guide]().
 
-## RawDataReader
+## <div align="center"> RawDataReader
 
 RawDataReader supports a wide range of aerosol instruments, including NEPH, SMPS, AE33, and many more. It handles
 various file types and time resolutions, making data processing efficient and standardized.
 
 For a detailed list of supported instruments, file types, and data columns, please refer to
-our [RawDataReader Usage Guide](docs/RawDataReader_Usage_Guide.md) in the `docs` folder.
+our [RawDataReader Usage Guide](docs/user_guide/RawDataReader) in the `docs` folder.
 
 ### Key Features:
 
@@ -108,7 +113,7 @@ The AeroViz project currently supports data from the following instruments:
 > **Note:** We are continuously working to support more instruments. Please check back for updates or contribute to our
 > project on GitHub.
 
-## <div align="center">DataProcess Supported Method</div>
+## <div align="center">DataProcess</div>
 
 The AeroViz project currently supports the following processing methods:
 
@@ -126,9 +131,6 @@ For detailed documentation, please refer to the `docs` folder, which includes:
 | Documentation                              | Description                |
 |--------------------------------------------|----------------------------|
 | [User Guide](docs/user_guide)              | Basic usage instructions   |
-| [Developer Guide](docs/developer_guide.md) | Developer guidelines       |
-| [API Reference](docs/api_reference.md)     | API documentation          |
-| [FAQ](docs/faq.md)                         | Frequently Asked Questions |
 | [Changelog](docs/changelog.md)             | List of changes            |
 
 </div>