AeroViz 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

AeroViz/rawDataReader/script/NEPH.py

@@ -4,77 +4,72 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'NEPH'
-
-	def _raw_reader(self, _file):
-		with _file.open('r', encoding='utf-8', errors='ignore') as f:
-			_df = read_csv(f, header=None, names=range(11))
-
-			_df_grp = _df.groupby(0)
-
-			# T : time
-			_df_tm = _df_grp.get_group('T')[[1, 2, 3, 4, 5, 6]].astype(int)
-
-			for _k in [2, 3, 4, 5, 6]:
-				_df_tm[_k] = _df_tm[_k].astype(int).map('{:02d}'.format).copy()
-			_df_tm = _df_tm.astype(str)
-
-			_idx_tm = to_datetime((_df_tm[1] + _df_tm[2] + _df_tm[3] + _df_tm[4] + _df_tm[5] + _df_tm[6]),
-								  format='%Y%m%d%H%M%S')
-
-			# D : data
-			# col : 3~8 B G R BB BG BR
-			# 1e6
-			try:
-				_df_dt = _df_grp.get_group('D')[[1, 2, 3, 4, 5, 6, 7, 8]].set_index(_idx_tm)
-				_df_out = (_df_dt.groupby(1).get_group('NBXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
-				_df_out.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR']
-				_df_out.index.name = 'Time'
-
-				# Y : state
-				# col : 5 RH
-				_df_st = _df_grp.get_group('Y')
-				_df_out['RH'] = _df_st[5].values
-				_df_out['status'] = _df_st[9].values
-
-				_df_out.mask(_df_out['status'] != 0)  # 0000 -> numeric to 0
-
-				return _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
-
-			except ValueError:
-				group_sizes = _df_grp.size()
-				print(group_sizes)
-				# Define the valid groups
-				valid_groups = {'B', 'G', 'R', 'D', 'T', 'Y', 'Z'}
-
-				# Find the rows where the value in the first column is not in valid_groups
-				invalid_indices = _df[~_df[0].isin(valid_groups)].index
-
-				# Print the invalid indices and their corresponding values
-				invalid_values = _df.loc[invalid_indices, 0]
-				print("Invalid values and their indices:")
-				for idx, value in zip(invalid_indices, invalid_values):
-					print(f"Index: {idx}, Value: {value}")
-
-				# If there's a length mismatch, return an empty DataFrame with the same index and column names
-				columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
-				_df_out = DataFrame(index=_idx_tm, columns=columns)
-				_df_out.index.name = 'Time'
-				print(f'\n\t\t\t Length mismatch in {_file} data. Returning an empty DataFrame.')
-				return _df_out
-
-	# QC data
-	def _QC(self, _df):
-		# remove negative value
-		_df = _df.mask((_df <= 0).copy())
-
-		# call by _QC function
-		# QC data in 1 hr
-		def _QC_func(_df_1hr):
-			_df_ave = _df_1hr.mean()
-			_df_std = _df_1hr.std()
-			_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-			return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-		return _df.resample('1h', group_keys=False).apply(_QC_func)
+    nam = 'NEPH'
+
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, header=None, names=range(11))
+
+            _df_grp = _df.groupby(0)
+
+            # T : time
+            _idx_tm = to_datetime(
+                _df_grp.get_group('T')[[1, 2, 3, 4, 5, 6]]
+                .map(lambda x: f"{int(x):02d}")
+                .agg(''.join, axis=1),
+                format='%Y%m%d%H%M%S'
+            )
+
+            # D : data
+            # col : 3~8 B G R BB BG BR
+            # 1e6
+            try:
+                _df_dt = _df_grp.get_group('D')[[1, 2, 3, 4, 5, 6, 7, 8]].set_index(_idx_tm)
+
+                try:
+                    _df_out = (_df_dt.groupby(1).get_group('NBXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+                except KeyError:
+                    _df_out = (_df_dt.groupby(1).get_group('NTXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+
+                _df_out.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR']
+                _df_out.index.name = 'Time'
+
+                # Y : state
+                # col : 5 RH
+                _df_st = _df_grp.get_group('Y')
+                _df_out['RH'] = _df_st[5].values
+                _df_out['status'] = _df_st[9].values
+
+                _df_out.mask(_df_out['status'] != 0)  # 0000 -> numeric to 0
+
+                _df = _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
+
+                return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+            except ValueError:
+                group_sizes = _df_grp.size()
+                print(group_sizes)
+
+                # Define valid groups and find invalid indices
+                valid_groups = {'B', 'G', 'R', 'D', 'T', 'Y', 'Z'}
+                invalid_indices = _df[~_df[0].isin(valid_groups)].index
+
+                # Print invalid indices and values
+                print("Invalid values and their indices:")
+                for idx in invalid_indices:
+                    print(f"Index: {idx}, Value: {_df.at[idx, 0]}")
+
+                # Return an empty DataFrame with specified columns if there's a length mismatch
+                columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
+                _df_out = DataFrame(index=_idx_tm, columns=columns)
+                _df_out.index.name = 'Time'
+                print(f'\n\t\t\t Length mismatch in {_file} data. Returning an empty DataFrame.')
+                return _df_out
+
+    # QC data
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df.mask((_df <= 5).copy())
+
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/SMPS_TH.py

@@ -4,38 +4,38 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'SMPS_TH'
+    nam = 'SMPS_TH'
 
-	def _raw_reader(self, _file):
-		with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
-			_df = read_table(f, skiprows=18, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
-			_key = list(_df.keys()[6:-26])
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_table(f, skiprows=18, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
+            _key = list(_df.keys()[6:-26])
 
-			_newkey = {}
-			for _k in _key:
-				_newkey[_k] = float(_k).__round__(4)
+            _newkey = {}
+            for _k in _key:
+                _newkey[_k] = float(_k).__round__(4)
 
-			# _newkey['Total Conc.(#/cm)'] = 'total'
-			# _newkey['Mode(nm)']	= 'mode'
+            # _newkey['Total Conc.(#/cm)'] = 'total'
+            # _newkey['Mode(nm)']	= 'mode'
 
-			_df_idx = to_datetime(_df.index, errors='coerce')
-		return _df[_newkey.keys()].rename(_newkey, axis=1).set_index(_df_idx).loc[_df_idx.dropna()]
+            _df_idx = to_datetime(_df.index, errors='coerce')
+        return _df[_newkey.keys()].rename(_newkey, axis=1).set_index(_df_idx).loc[_df_idx.dropna()]
 
-	## QC data
-	def _QC(self, _df):
-		import numpy as n
+    # QC data
+    def _QC(self, _df):
+        import numpy as n
 
-		## mask out the data size lower than 7
-		_df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-		_df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-		_df = _df.mask(_df_size < 7)
+        # mask out the data size lower than 7
+        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
+        _df = _df.mask(_df_size < 7)
 
-		## remove total conc. lower than 2000
-		_df = _df.mask(_df['total'] < 2000)
+        # remove total conc. lower than 2000
+        _df = _df.mask(_df['total'] < 2000)
 
-		## remove the bin over 400 nm which num. conc. larger than 4000
-		_df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
+        # remove the bin over 400 nm which num. conc. larger than 4000
+        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
 
-		_df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
+        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
 
-		return _df[_df.keys()[:-1]]
+        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/SMPS_aim11.py

@@ -4,48 +4,48 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'SMPS_aim11'
+    nam = 'SMPS_aim11'
 
-	def _raw_reader(self, _file):
-		with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
 
-			skiprows = 0
-			for _line in f:
+            skiprows = 0
+            for _line in f:
 
-				if _line.split(',')[0] == 'Scan Number':
-					f.seek(0)
-					break
+                if _line.split(',')[0] == 'Scan Number':
+                    f.seek(0)
+                    break
 
-				skiprows += 1
-			# breakpoint()
-			_df = read_csv(f, skiprows=skiprows)
-			_tm_idx = to_datetime(_df['DateTime Sample Start'], format='%d/%m/%Y %X', errors='coerce')
+                skiprows += 1
+            # breakpoint()
+            _df = read_csv(f, skiprows=skiprows)
+            _tm_idx = to_datetime(_df['DateTime Sample Start'], format='%d/%m/%Y %X', errors='coerce')
 
-			## index
-			_df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
+            # index
+            _df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
 
-			## keys
-			_key = to_numeric(_df.keys(), errors='coerce')
-			_df.columns = _key
-			_df = _df.loc[:, ~_key.isna()]
+            # keys
+            _key = to_numeric(_df.keys(), errors='coerce')
+            _df.columns = _key
+            _df = _df.loc[:, ~_key.isna()]
 
-		return _df.apply(to_numeric, errors='coerce')
+        return _df.apply(to_numeric, errors='coerce')
 
-	## QC data
-	def _QC(self, _df):
-		import numpy as n
+    # QC data
+    def _QC(self, _df):
+        import numpy as n
 
-		## mask out the data size lower than 7
-		_df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-		_df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-		_df = _df.mask(_df_size < 7)
+        # mask out the data size lower than 7
+        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
+        _df = _df.mask(_df_size < 7)
 
-		## remove total conc. lower than 2000
-		_df = _df.mask(_df['total'] < 2000)
+        # remove total conc. lower than 2000
+        _df = _df.mask(_df['total'] < 2000)
 
-		## remove the bin over 400 nm which num. conc. larger than 4000
-		_df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
+        # remove the bin over 400 nm which num. conc. larger than 4000
+        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
 
-		_df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
+        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
 
-		return _df[_df.keys()[:-1]]
+        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/SMPS_genr.py

@@ -4,48 +4,48 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'SMPS_genr'
+    nam = 'SMPS_genr'
 
-	def _raw_reader(self, _file):
-		with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
 
-			skiprows = 0
-			for _line in f:
+            skiprows = 0
+            for _line in f:
 
-				if _line.split('\t')[0] == 'Sample #':
-					f.seek(0)
-					break
+                if _line.split('\t')[0] == 'Sample #':
+                    f.seek(0)
+                    break
 
-				skiprows += 1
+                skiprows += 1
 
-			_df = read_table(f, skiprows=skiprows)
-			_tm_idx = to_datetime(_df['Date'] + _df['Start Time'], format='%m/%d/%y%X', errors='coerce')
+            _df = read_table(f, skiprows=skiprows)
+            _tm_idx = to_datetime(_df['Date'] + _df['Start Time'], format='%m/%d/%y%X', errors='coerce')
 
-			## index
-			_df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
+            # index
+            _df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
 
-			## keys
-			_key = to_numeric(_df.keys(), errors='coerce')
-			_df.columns = _key
-			_df = _df.loc[:, ~_key.isna()]
+            # keys
+            _key = to_numeric(_df.keys(), errors='coerce')
+            _df.columns = _key
+            _df = _df.loc[:, ~_key.isna()]
 
-		return _df.apply(to_numeric, errors='coerce')
+        return _df.apply(to_numeric, errors='coerce')
 
-	## QC data
-	def _QC(self, _df):
-		import numpy as n
+    # QC data
+    def _QC(self, _df):
+        import numpy as n
 
-		## mask out the data size lower than 7
-		_df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-		_df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-		_df = _df.mask(_df_size < 7)
+        # mask out the data size lower than 7
+        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
+        _df = _df.mask(_df_size < 7)
 
-		## remove total conc. lower than 2000
-		_df = _df.mask(_df['total'] < 2000)
+        # remove total conc. lower than 2000
+        _df = _df.mask(_df['total'] < 2000)
 
-		## remove the bin over 400 nm which num. conc. larger than 4000
-		_df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
+        # remove the bin over 400 nm which num. conc. larger than 4000
+        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
 
-		_df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
+        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
 
-		return _df[_df.keys()[:-1]]
+        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/Sunset_OCEC.py

@@ -0,0 +1,60 @@
+from pandas import to_datetime, read_csv
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'Sunset_OCEC'
+
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, skiprows=3)
+
+            _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
+            _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
+            _df = _df.set_index('time')
+
+            _df = _df.rename(columns={
+                'Thermal/Optical OC (ugC/LCm^3)': 'Thermal_OC',
+                'OC ugC/m^3 (Thermal/Optical)': 'Thermal_OC',
+
+                'Thermal/Optical EC (ugC/LCm^3)': 'Thermal_EC',
+                'EC ugC/m^3 (Thermal/Optical)': 'Thermal_EC',
+
+                'OC=TC-BC (ugC/LCm^3)': 'Optical_OC',
+                'OC by diff ugC (TC-OptEC)': 'Optical_OC',
+
+                'BC (ugC/LCm^3)': 'Optical_EC',
+                'OptEC ugC/m^3': 'Optical_EC',
+
+                'Sample Volume Local Condition Actual m^3': 'Sample_Volume',
+                'TC (ugC/LCm^3)': 'TC',
+                'TC ugC/m^3': 'TC',
+                'OCPk1-ug C': 'OC1',
+                'OCPk2-ug C': 'OC2',
+                'OCPk3-ug C': 'OC3',
+                'OCPk4-ug C': 'OC4',
+                'Pyrolized C ug': 'PC'
+            })
+
+            _df = _df[['Thermal_OC', 'Optical_OC', 'Thermal_EC', 'Optical_EC', 'TC', 'OC1', 'OC2', 'OC3', 'OC4']]
+
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    # QC data
+    def _QC(self, _df):
+        import numpy as np
+
+        _df = _df.where(_df > 0)
+
+        thresholds = {
+            'Thermal_OC': 0.3,
+            'Optical_OC': 0.3,
+            'Thermal_EC': 0.015,
+            'Optical_EC': 0.015
+        }
+
+        for col, thresh in thresholds.items():
+            _df.loc[_df[col] <= thresh, col] = np.nan
+
+        return _df

AeroViz/rawDataReader/script/TEOM.py

@@ -4,43 +4,45 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'TEOM'
+    nam = 'TEOM'
 
-	def _raw_reader(self, _file):
-		with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
-			_df = read_csv(f, skiprows=3, index_col=False)
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, skiprows=3, index_col=False)
 
-			_df = _df.rename(columns={'Time Stamp': 'time',
-									  'System status': 'status',
-									  'PM-2.5 base MC': 'PM_NV',
-									  'PM-2.5 MC': 'PM_Total',
-									  'PM-2.5 TEOM noise': 'noise', })
+            _df = _df.rename(columns={'Time Stamp': 'time',
+                                      'System status': 'status',
+                                      'PM-2.5 base MC': 'PM_NV',
+                                      'PM-2.5 MC': 'PM_Total',
+                                      'PM-2.5 TEOM noise': 'noise', })
 
-			_time_replace = {'十一月': '11', '十二月': '12', '一月': '01', '二月': '02', '三月': '03', '四月': '04',
-							 '五月': '05', '六月': '06', '七月': '07', '八月': '08', '九月': '09', '十月': '10'}
+            _time_replace = {'十一月': '11', '十二月': '12', '一月': '01', '二月': '02', '三月': '03', '四月': '04',
+                             '五月': '05', '六月': '06', '七月': '07', '八月': '08', '九月': '09', '十月': '10'}
 
-			_tm_idx = _df.time
-			for _ori, _rpl in _time_replace.items():
-				_tm_idx = _tm_idx.str.replace(_ori, _rpl)
+            _tm_idx = _df.time
+            for _ori, _rpl in _time_replace.items():
+                _tm_idx = _tm_idx.str.replace(_ori, _rpl)
 
-			_df = _df.set_index(to_datetime(_tm_idx, errors='coerce', format='%d - %m - %Y %X'))
+            _df = _df.set_index(to_datetime(_tm_idx, errors='coerce', format='%d - %m - %Y %X'))
 
-			_df = _df.where(_df['status'] < 1e-7)
+            _df = _df.where(_df['status'] < 1e-7)
 
-		return _df[['PM_NV', 'PM_Total', 'noise', ]]
+        _df = _df[['PM_NV', 'PM_Total', 'noise', ]]
 
-	## QC data
-	def _QC(self, _df):
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-		_df_idx = _df.index.copy()
+    # QC data
+    def _QC(self, _df):
 
-		## remove negative value
-		_df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df < 0).copy())
+        _df_idx = _df.index.copy()
 
-		## QC data in 1 hr
-		## remove data where size < 8 in 1-hr
-		for _key in ['PM_Total', 'PM_NV']:
-			_size = _df[_key].dropna().resample('1h').size().reindex(_df_idx).ffill().copy()
-			_df[_key] = _df[_key].mask(_size < 8)
+        # remove negative value
+        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df <= 0).copy())
 
-		return _df.reindex(_df_idx)
+        # QC data in 1 hr
+        # remove data where size < 8 in 1-hr
+        for _key in ['PM_Total', 'PM_NV']:
+            _size = _df[_key].dropna().resample('1h').size().reindex(_df_idx).ffill().copy()
+            _df[_key] = _df[_key].mask(_size < 8)
+
+        return _df.reindex(_df_idx)

AeroViz/rawDataReader/script/Table.py

@@ -1,28 +1,27 @@
 # read meteorological data from google sheet
 
-
 from pandas import read_csv, to_datetime
 
 from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'Table'
-
-	def _raw_reader(self, _file):
-		self.meta['freq'] = self._oth_set.get('data_freq') or self.meta['freq']
+    nam = 'Table'
 
-		with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
-			_df = read_csv(f, low_memory=False, index_col=0)
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, low_memory=False, index_col=0)
 
-			_df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
-			_df.index.name = 'time'
+            _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
+            _df.index.name = 'time'
 
-			_df.columns = _df.keys().str.strip(' ')
+            _df.columns = _df.keys().str.strip(' ')
 
-			_df = _df.loc[_df.index.dropna()].copy()
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-		return _df.loc[~_df.index.duplicated()]
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df.mask((_df < 0).copy())
 
-	def _QC(self, _df):
-		return _df
+        # QC data in 6h
+        return _df.resample('6h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/VOC.py

@@ -0,0 +1,26 @@
+
+from pandas import read_csv
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'VOC'
+
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=[0], index_col=[0], na_values=('-', 'N.D.'))
+
+            _df.columns = _df.keys().str.strip(' ')
+            _df.index.name = 'time'
+
+            try:
+                _df = _df[self.meta["key"]].loc[_df.index.dropna()]
+
+            except KeyError:
+                _df = _df[self.meta["key_2"]].loc[_df.index.dropna()]
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        return _df

AeroViz/rawDataReader/script/__init__.py

@@ -1,22 +1,20 @@
 __all__ = [
-	'NEPH',
-	'Aurora',
-	'Table',
-	'EPA_vertical',
-	'APS_3321',
-	'SMPS_TH',
-	'AE33',
-	'AE43',
-	'BC1054',
-	'MA350',
-	'TEOM',
-	'OCEC_RES',
-	'OCEC_LCRES',
-	'IGAC_TH',
-	'IGAC_ZM',
-	'VOC_TH',
-	'VOC_ZM',
-	'SMPS_genr',
-	'SMPS_aim11',
-	'GRIMM'
+    'NEPH',
+    'Aurora',
+    'SMPS_TH',
+    'SMPS_genr',
+    'SMPS_aim11',
+    'APS_3321',
+    'GRIMM',
+    'AE33',
+    'AE43',
+    'BC1054',
+    'MA350',
+    'TEOM',
+    'Sunset_OCEC',
+    'IGAC',
+    'VOC',
+    'Table',
+    'EPA_vertical',
+    'Minion'
 ]