AeroViz 0.1.3b0__py3-none-any.whl → 0.1.4__py3-none-any.whl

AeroViz/rawDataReader/script/EPA_vertical.py

@@ -1,3 +1,4 @@
+import numpy as np
 from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -7,12 +8,39 @@ class Reader(AbstractReader):
     nam = 'EPA_vertical'
 
     def _raw_reader(self, _file):
-        with _file.open('r', encoding='big5', errors='ignore') as f:
-            _df = read_csv(f, names=['time', 'station', 'comp', 'data', None], skiprows=1, na_values=['-'],
-                           parse_dates=['time'], index_col='time')
-            _df['data'] = to_numeric(_df['data'], errors='coerce')
+        with _file.open('r', encoding='ascii', errors='ignore') as f:
+            # 有、無輸出有效值都可以
+            # read 查詢小時值(測項).csv
+            df = read_csv(f, encoding='ascii', encoding_errors='ignore', index_col=0, parse_dates=True,
+                          usecols=lambda col: col != 'Unnamed: 1')
 
-            _df_piv = _df.pivot_table(values='data', columns='comp', index='time')
-            _df_piv.index.name = 'time'
+            df.index.name = 'Time'
+            df.rename(columns={'AMB_TEMP': 'AT', 'WIND_SPEED': 'WS', 'WIND_DIREC': 'WD'}, inplace=True)
 
-        return _df_piv
+            # 欄位排序
+            desired_order = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC', 'CH4', 'PM10', 'PM2.5', 'WS', 'WD',
+                             'AT', 'RH']
+
+            missing_columns = []
+
+            for col in desired_order:
+                if col not in df.columns:
+                    df[col] = np.nan
+                    missing_columns.append(col)
+
+            if missing_columns:
+                self.logger.info(f"{'=' * 60}")
+                self.logger.info(f"Missing columns: {missing_columns}")
+                self.logger.info(f"{'=' * 60}")
+                print(f"Missing columns: {missing_columns}")
+
+            df = df[desired_order]
+
+            # 如果沒有將無效值拿掉就輸出 請將包含 #、L、O 的字串替換成 *
+            df.replace(to_replace=r'\d*[#LO]\b', value='*', regex=True, inplace=True)
+            df = df.apply(to_numeric, errors='coerce')
+
+        return df
+
+    def _QC(self, _df):
+        return _df

AeroViz/rawDataReader/script/GRIMM.py

@@ -24,12 +24,5 @@ class Reader(AbstractReader):
         return _df / 0.035
 
     def _QC(self, _df):
-        # QC data in 1 hr
-        def _QC_func(_df_1hr):
-            _df_ave = _df_1hr.mean()
-            _df_std = _df_1hr.std()
-            _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-            return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-        return _df.resample('5min').apply(_QC_func).resample('1h').mean()
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/{IGAC_ZM.py → IGAC.py}

@@ -8,22 +8,21 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-    nam = 'IGAC_ZM'
+    nam = 'IGAC'
 
     def _raw_reader(self, _file):
 
-        with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, parse_dates=[0], index_col=[0], na_values=['-']).apply(to_numeric, errors='coerce')
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=True, index_col=0, na_values='-').apply(to_numeric, errors='coerce')
 
             _df.columns = _df.keys().str.strip(' ')
             _df.index.name = 'time'
 
-        return _df.loc[_df.index.dropna()].loc[~_df.index.duplicated()]
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-    ## QC data
     def _QC(self, _df):
 
-        ## QC parameter, function (MDL SE LE)
+        # QC parameter, function (MDL SE LE)
         _mdl = {
             'Na+': 0.06,
             'NH4+': 0.05,
@@ -35,7 +34,8 @@ class Reader(AbstractReader):
             'NO3-': 0.11,
             'SO42-': 0.08,
         }
-        _mdl.update(self._oth_set.get('mdl', {}))
+
+        # _mdl.update(self._oth_set.get('mdl', {}))
 
         def _se_le(_df_, _log=False):
             _df_ = np.log10(_df_) if _log else _df_
@@ -51,27 +51,27 @@ class Reader(AbstractReader):
                 return 10 ** _se, 10 ** _le
             return _se, _le
 
-        _cation, _anion, _main = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'], ['Cl-', 'NO2-', 'NO3-', 'SO42-', ], ['SO42-',
-                                                                                                             'NO3-',
-                                                                                                             'NH4+']
+        _cation, _anion, _main = (['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'],
+                                  ['Cl-', 'NO2-', 'NO3-', 'SO42-', ],
+                                  ['SO42-', 'NO3-', 'NH4+'])
 
         _df_salt = _df[_mdl.keys()].copy()
         _df_pm = _df['PM2.5'].copy()
 
-        ## lower than PM2.5
-        ## conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
+        # lower than PM2.5
+        # conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
         _df_salt = _df_salt.mask(_df_salt.sum(axis=1, min_count=1) > _df_pm).dropna(subset=_main).copy()
 
-        ## mdl
+        # mdl
         for (_key, _df_col), _mdl_val in zip(_df_salt.items(), _mdl.values()):
             _df_salt[_key] = _df_col.mask(_df_col < _mdl_val, _mdl_val / 2)
 
-        ## calculate SE LE
-        ## salt < LE
+        # calculate SE LE
+        # salt < LE
         _se, _le = _se_le(_df_salt, _log=True)
         _df_salt = _df_salt.mask(_df_salt > _le).copy()
 
-        ## C/A, A/C
+        # C/A, A/C
         _rat_CA = (_df_salt[_cation].sum(axis=1) / _df_salt[_anion].sum(axis=1)).to_frame()
         _rat_AC = (1 / _rat_CA).copy()
 
@@ -83,7 +83,7 @@ class Reader(AbstractReader):
 
         _df_salt = _df_salt.where((_cond_CA * _cond_AC)[0]).copy()
 
-        ## conc. of main salt > SE
+        # conc. of main salt > SE
         _se, _le = _se_le(_df_salt[_main], _log=True)
         _df_salt[_main] = _df_salt[_main].mask(_df_salt[_main] < _se).copy()
 

AeroViz/rawDataReader/script/MA350.py

@@ -22,24 +22,17 @@ class Reader(AbstractReader):
             'BB (%)': 'BB',
         })
 
-        # remove data without Status=32 (Automatic Tape Advance), 65536 (Tape Move)
-        # if not self._oth_set.get('ignore_err', False):
-        #     _df = _df.where((_df['Status'] != 32) | (_df['Status'] != 65536)).copy()
+        # if self.meta.get('error_state', False):
+        #     _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE', 'BB']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE', 'BB']]
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     # QC data
     def _QC(self, _df):
         # remove negative value
         _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'AAE', 'BB']].mask((_df < 0).copy())
 
-        # call by _QC function
-        # QC data in 1 hr
-        def _QC_func(_df_1hr):
-            _df_ave = _df_1hr.mean()
-            _df_std = _df_1hr.std()
-            _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-            return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-        return _df.resample('1h', group_keys=False).apply(_QC_func).resample('5min').mean()
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/Minion.py

@@ -0,0 +1,103 @@
+import numpy as np
+from pandas import read_csv, to_datetime, to_numeric
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'Minion'
+
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, low_memory=False, index_col=0)
+
+            _df.index = to_datetime(_df.index, errors='coerce')
+            _df.index.name = 'time'
+
+            _df.columns = _df.keys().str.strip(' ')
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        # XRF QAQC
+        _df = self.XRF_QAQC(_df)
+
+        # ions balance
+        _df = self.ions_balance(_df)
+
+        # remove negative value
+        _df = _df.mask((_df < 0).copy())
+
+        # QC data in 6h
+        return _df.resample('6h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+
+    # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
+    def XRF_QAQC(self, df):
+        MDL = {
+            'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2,
+            'Cl': 1.7, 'K': 1.2, 'Ca': 0.3, 'Ti': 1.6,
+            'V': 0.12, 'Cr': 0.12, 'Mn': 0.14, 'Fe': 0.17,
+            'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079, 'Zn': 0.067,
+            'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
+            'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28,
+            'Zr': 0.33, 'Nb': 0.41, 'Mo': 0.48, 'Ag': 1.9,
+            'Cd': 2.5, 'In': 3.1, 'Sn': 4.1, 'Sb': 5.2,
+            'Te': 0.6, 'I': 0.49, 'Cs': 0.37, 'Ba': 0.39,
+            'La': 0.36, 'Ce': 0.3, 'Pt': 0.12, 'Au': 0.1,
+            'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13, 'Bi': 0.13
+        }
+        # 將小於 MDL 值的數據替換為 NaN
+        for element, threshold in MDL.items():
+            if element in df.columns:
+                df[element] = df[element].where(df[element] >= threshold, np.nan)
+
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"XRF QAQC summary:")
+        self.logger.info("\t\ttransform values below MDL to NaN")
+        self.logger.info(f"{'=' * 60}")
+
+        return df
+
+    def ions_balance(self, df, tolerance=0.3):
+        """
+        Calculate the balance of ions in the system
+        """
+        # Define the ions
+        item = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'F-', 'Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-']
+
+        # Calculate the balance
+        _df = df[item].copy()
+        _df = _df.apply(lambda x: to_numeric(x, errors='coerce'))
+        _df['+_mole'] = _df[['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+']].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1,
+                                                                                                             skipna=True)
+        _df['-_mole'] = _df[['Cl-', 'NO2-', 'NO3-', 'SO42-']].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
+
+        # Avoid division by zero
+        _df['ratio'] = np.where(_df['-_mole'] != 0, _df['+_mole'] / _df['-_mole'], np.nan)
+
+        # Calculate bounds
+        lower_bound, upper_bound = 1 - tolerance, 1 + tolerance
+
+        # 根据ratio决定是否保留原始数据
+        valid_mask = (
+                (_df['ratio'] <= upper_bound) &
+                (_df['ratio'] >= lower_bound) &
+                ~np.isnan(_df['+_mole']) &
+                ~np.isnan(_df['-_mole'])
+        )
+
+        # 保留数据或将不符合条件的行设为NaN
+        df.loc[~valid_mask, item] = np.nan
+
+        # 计算保留的数据的百分比
+        retained_percentage = (valid_mask.sum() / len(df)) * 100
+
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Ions balance summary:")
+        self.logger.info(f"\t\tretain {retained_percentage.__round__(0)}% data within tolerance {tolerance}")
+        self.logger.info(f"{'=' * 60}")
+
+        if retained_percentage < 70:
+            self.logger.warning("Warning: The percentage of retained data is less than 70%")
+
+        return df

AeroViz/rawDataReader/script/NEPH.py

@@ -13,21 +13,24 @@ class Reader(AbstractReader):
             _df_grp = _df.groupby(0)
 
             # T : time
-            _df_tm = _df_grp.get_group('T')[[1, 2, 3, 4, 5, 6]].astype(int)
-
-            for _k in [2, 3, 4, 5, 6]:
-                _df_tm[_k] = _df_tm[_k].astype(int).map('{:02d}'.format).copy()
-            _df_tm = _df_tm.astype(str)
-
-            _idx_tm = to_datetime((_df_tm[1] + _df_tm[2] + _df_tm[3] + _df_tm[4] + _df_tm[5] + _df_tm[6]),
-                                  format='%Y%m%d%H%M%S')
+            _idx_tm = to_datetime(
+                _df_grp.get_group('T')[[1, 2, 3, 4, 5, 6]]
+                .map(lambda x: f"{int(x):02d}")
+                .agg(''.join, axis=1),
+                format='%Y%m%d%H%M%S'
+            )
 
             # D : data
             # col : 3~8 B G R BB BG BR
             # 1e6
             try:
                 _df_dt = _df_grp.get_group('D')[[1, 2, 3, 4, 5, 6, 7, 8]].set_index(_idx_tm)
-                _df_out = (_df_dt.groupby(1).get_group('NBXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+
+                try:
+                    _df_out = (_df_dt.groupby(1).get_group('NBXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+                except KeyError:
+                    _df_out = (_df_dt.groupby(1).get_group('NTXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+
                 _df_out.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR']
                 _df_out.index.name = 'Time'
 
@@ -39,24 +42,24 @@ class Reader(AbstractReader):
 
                 _df_out.mask(_df_out['status'] != 0)  # 0000 -> numeric to 0
 
-                return _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
+                _df = _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
+
+                return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
             except ValueError:
                 group_sizes = _df_grp.size()
                 print(group_sizes)
-                # Define the valid groups
-                valid_groups = {'B', 'G', 'R', 'D', 'T', 'Y', 'Z'}
 
-                # Find the rows where the value in the first column is not in valid_groups
+                # Define valid groups and find invalid indices
+                valid_groups = {'B', 'G', 'R', 'D', 'T', 'Y', 'Z'}
                 invalid_indices = _df[~_df[0].isin(valid_groups)].index
 
-                # Print the invalid indices and their corresponding values
-                invalid_values = _df.loc[invalid_indices, 0]
+                # Print invalid indices and values
                 print("Invalid values and their indices:")
-                for idx, value in zip(invalid_indices, invalid_values):
-                    print(f"Index: {idx}, Value: {value}")
+                for idx in invalid_indices:
+                    print(f"Index: {idx}, Value: {_df.at[idx, 0]}")
 
-                # If there's a length mismatch, return an empty DataFrame with the same index and column names
+                # Return an empty DataFrame with specified columns if there's a length mismatch
                 columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
                 _df_out = DataFrame(index=_idx_tm, columns=columns)
                 _df_out.index.name = 'Time'
@@ -66,15 +69,7 @@ class Reader(AbstractReader):
     # QC data
     def _QC(self, _df):
         # remove negative value
-        _df = _df.mask((_df <= 0).copy())
-
-        # call by _QC function
-        # QC data in 1 hr
-        def _QC_func(_df_1hr):
-            _df_ave = _df_1hr.mean()
-            _df_std = _df_1hr.std()
-            _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-            return _df_1hr.mask(_df_lowb | _df_highb).copy()
+        _df = _df.mask((_df <= 5).copy())
 
-        return _df.resample('1h', group_keys=False).apply(_QC_func)
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/SMPS_TH.py

@@ -21,19 +21,19 @@ class Reader(AbstractReader):
             _df_idx = to_datetime(_df.index, errors='coerce')
         return _df[_newkey.keys()].rename(_newkey, axis=1).set_index(_df_idx).loc[_df_idx.dropna()]
 
-    ## QC data
+    # QC data
     def _QC(self, _df):
         import numpy as n
 
-        ## mask out the data size lower than 7
+        # mask out the data size lower than 7
         _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 
-        ## remove total conc. lower than 2000
+        # remove total conc. lower than 2000
         _df = _df.mask(_df['total'] < 2000)
 
-        ## remove the bin over 400 nm which num. conc. larger than 4000
+        # remove the bin over 400 nm which num. conc. larger than 4000
         _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
 
         _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)

AeroViz/rawDataReader/script/SMPS_aim11.py

@@ -21,29 +21,29 @@ class Reader(AbstractReader):
             _df = read_csv(f, skiprows=skiprows)
             _tm_idx = to_datetime(_df['DateTime Sample Start'], format='%d/%m/%Y %X', errors='coerce')
 
-            ## index
+            # index
             _df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
 
-            ## keys
+            # keys
             _key = to_numeric(_df.keys(), errors='coerce')
             _df.columns = _key
             _df = _df.loc[:, ~_key.isna()]
 
         return _df.apply(to_numeric, errors='coerce')
 
-    ## QC data
+    # QC data
     def _QC(self, _df):
         import numpy as n
 
-        ## mask out the data size lower than 7
+        # mask out the data size lower than 7
         _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 
-        ## remove total conc. lower than 2000
+        # remove total conc. lower than 2000
         _df = _df.mask(_df['total'] < 2000)
 
-        ## remove the bin over 400 nm which num. conc. larger than 4000
+        # remove the bin over 400 nm which num. conc. larger than 4000
         _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
 
         _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)

AeroViz/rawDataReader/script/SMPS_genr.py

@@ -21,29 +21,29 @@ class Reader(AbstractReader):
             _df = read_table(f, skiprows=skiprows)
             _tm_idx = to_datetime(_df['Date'] + _df['Start Time'], format='%m/%d/%y%X', errors='coerce')
 
-            ## index
+            # index
             _df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
 
-            ## keys
+            # keys
             _key = to_numeric(_df.keys(), errors='coerce')
             _df.columns = _key
             _df = _df.loc[:, ~_key.isna()]
 
         return _df.apply(to_numeric, errors='coerce')
 
-    ## QC data
+    # QC data
     def _QC(self, _df):
         import numpy as n
 
-        ## mask out the data size lower than 7
+        # mask out the data size lower than 7
         _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 
-        ## remove total conc. lower than 2000
+        # remove total conc. lower than 2000
         _df = _df.mask(_df['total'] < 2000)
 
-        ## remove the bin over 400 nm which num. conc. larger than 4000
+        # remove the bin over 400 nm which num. conc. larger than 4000
         _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
 
         _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)

AeroViz/rawDataReader/script/Sunset_OCEC.py

@@ -0,0 +1,60 @@
+from pandas import to_datetime, read_csv
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'Sunset_OCEC'
+
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, skiprows=3)
+
+            _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
+            _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
+            _df = _df.set_index('time')
+
+            _df = _df.rename(columns={
+                'Thermal/Optical OC (ugC/LCm^3)': 'Thermal_OC',
+                'OC ugC/m^3 (Thermal/Optical)': 'Thermal_OC',
+
+                'Thermal/Optical EC (ugC/LCm^3)': 'Thermal_EC',
+                'EC ugC/m^3 (Thermal/Optical)': 'Thermal_EC',
+
+                'OC=TC-BC (ugC/LCm^3)': 'Optical_OC',
+                'OC by diff ugC (TC-OptEC)': 'Optical_OC',
+
+                'BC (ugC/LCm^3)': 'Optical_EC',
+                'OptEC ugC/m^3': 'Optical_EC',
+
+                'Sample Volume Local Condition Actual m^3': 'Sample_Volume',
+                'TC (ugC/LCm^3)': 'TC',
+                'TC ugC/m^3': 'TC',
+                'OCPk1-ug C': 'OC1',
+                'OCPk2-ug C': 'OC2',
+                'OCPk3-ug C': 'OC3',
+                'OCPk4-ug C': 'OC4',
+                'Pyrolized C ug': 'PC'
+            })
+
+            _df = _df[['Thermal_OC', 'Optical_OC', 'Thermal_EC', 'Optical_EC', 'TC', 'OC1', 'OC2', 'OC3', 'OC4']]
+
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    # QC data
+    def _QC(self, _df):
+        import numpy as np
+
+        _df = _df.where(_df > 0)
+
+        thresholds = {
+            'Thermal_OC': 0.3,
+            'Optical_OC': 0.3,
+            'Thermal_EC': 0.015,
+            'Optical_EC': 0.015
+        }
+
+        for col, thresh in thresholds.items():
+            _df.loc[_df[col] <= thresh, col] = np.nan
+
+        return _df

AeroViz/rawDataReader/script/TEOM.py

@@ -27,18 +27,20 @@ class Reader(AbstractReader):
 
             _df = _df.where(_df['status'] < 1e-7)
 
-        return _df[['PM_NV', 'PM_Total', 'noise', ]]
+        _df = _df[['PM_NV', 'PM_Total', 'noise', ]]
 
-    ## QC data
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    # QC data
     def _QC(self, _df):
 
         _df_idx = _df.index.copy()
 
-        ## remove negative value
-        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df < 0).copy())
+        # remove negative value
+        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df <= 0).copy())
 
-        ## QC data in 1 hr
-        ## remove data where size < 8 in 1-hr
+        # QC data in 1 hr
+        # remove data where size < 8 in 1-hr
         for _key in ['PM_Total', 'PM_NV']:
             _size = _df[_key].dropna().resample('1h').size().reindex(_df_idx).ffill().copy()
             _df[_key] = _df[_key].mask(_size < 8)

AeroViz/rawDataReader/script/Table.py

@@ -1,6 +1,5 @@
 # read meteorological data from google sheet
 
-
 from pandas import read_csv, to_datetime
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -10,9 +9,7 @@ class Reader(AbstractReader):
     nam = 'Table'
 
     def _raw_reader(self, _file):
-        self.meta['freq'] = self._oth_set.get('data_freq') or self.meta['freq']
-
-        with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
             _df = read_csv(f, low_memory=False, index_col=0)
 
             _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
@@ -20,9 +17,11 @@ class Reader(AbstractReader):
 
             _df.columns = _df.keys().str.strip(' ')
 
-            _df = _df.loc[_df.index.dropna()].copy()
-
-        return _df.loc[~_df.index.duplicated()]
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     def _QC(self, _df):
-        return _df
+        # remove negative value
+        _df = _df.mask((_df < 0).copy())
+
+        # QC data in 6h
+        return _df.resample('6h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/VOC.py

@@ -0,0 +1,26 @@
+
+from pandas import read_csv
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'VOC'
+
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=[0], index_col=[0], na_values=('-', 'N.D.'))
+
+            _df.columns = _df.keys().str.strip(' ')
+            _df.index.name = 'time'
+
+            try:
+                _df = _df[self.meta["key"]].loc[_df.index.dropna()]
+
+            except KeyError:
+                _df = _df[self.meta["key_2"]].loc[_df.index.dropna()]
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        return _df

AeroViz/rawDataReader/script/__init__.py

@@ -1,22 +1,20 @@
 __all__ = [
     'NEPH',
     'Aurora',
-    'Table',
-    'EPA_vertical',
-    'APS_3321',
     'SMPS_TH',
+    'SMPS_genr',
+    'SMPS_aim11',
+    'APS_3321',
+    'GRIMM',
     'AE33',
     'AE43',
     'BC1054',
     'MA350',
     'TEOM',
-    'OCEC_RES',
-    'OCEC_LCRES',
-    'IGAC_TH',
-    'IGAC_ZM',
-    'VOC_TH',
-    'VOC_ZM',
-    'SMPS_genr',
-    'SMPS_aim11',
-    'GRIMM'
+    'Sunset_OCEC',
+    'IGAC',
+    'VOC',
+    'Table',
+    'EPA_vertical',
+    'Minion'
 ]