AeroViz 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

AeroViz/rawDataReader/script/AE43.py

@@ -4,31 +4,28 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'AE43'
+    nam = 'AE43'
 
-	def _raw_reader(self, _file):
-		_df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
-		_df_id = _df['SetupID'].iloc[-1]
+    def _raw_reader(self, _file):
+        _df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
+        _df_id = _df['SetupID'].iloc[-1]
 
-		# get last SetupID data
-		_df = _df.groupby('SetupID').get_group(_df_id)[
-			['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']].copy()
+        # get last SetupID data
+        _df = _df.groupby('SetupID').get_group(_df_id)[
+            ['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']].copy()
 
-		# remove data without Status=0
-		_df = _df.where(_df['Status'] == 0).copy()
+        # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
+        if self.meta.get('error_state', False):
+            _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-		return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
 
-	# QC data
-	def _QC(self, _df):
-		# remove negative value
-		_df = _df.mask((_df < 0).copy())
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-		# QC data in 5 min
-		def _QC_func(df):
-			_df_ave, _df_std = df.mean(), df.std()
-			_df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
+    # QC data
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df.mask((_df < 0).copy())
 
-			return df.mask(_df_lowb | _df_highb).copy()
-
-		return _df.resample('5min').apply(_QC_func).resample('1h').mean()
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/APS_3321.py

@@ -5,43 +5,43 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'APS_3321'
+    nam = 'APS_3321'
 
-	def _raw_reader(self, _file):
-		with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
-			_df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
-			_key = list(_df.keys()[3:54])  ## 542 ~ 1981
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
+            _key = list(_df.keys()[3:54])  ## 542 ~ 1981
 
-			## create new keys
-			_newkey = {}
-			for _k in _key:
-				_newkey[_k] = float(_k).__round__(4)
-			# _newkey['Mode(m)'] = 'mode'
+            # create new keys
+            _newkey = {}
+            for _k in _key:
+                _newkey[_k] = float(_k).__round__(4)
+            # _newkey['Mode(m)'] = 'mode'
 
-			## get new dataframe
-			_df = _df[_newkey.keys()].rename(_newkey, axis=1)
-			# _df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
+            # get new dataframe
+            _df = _df[_newkey.keys()].rename(_newkey, axis=1)
+            # df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
 
-			_df_idx = to_datetime(_df.index, errors='coerce')
+            _df_idx = to_datetime(_df.index, errors='coerce')
 
-		return _df.set_index(_df_idx).loc[_df_idx.dropna()]
+        return _df.set_index(_df_idx).loc[_df_idx.dropna()]
 
-	## QC data
-	def _QC(self, _df):
-		## mask out the data size lower than 7
-		_df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-		_df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-		_df = _df.mask(_df_size < 7)
+    # QC data
+    def _QC(self, _df):
+        # mask out the data size lower than 7
+        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
+        _df = _df.mask(_df_size < 7)
 
-		## remove total conc. lower than 700
-		_df = _df.mask(_df['total'] > 700)
+        # remove total conc. lower than 700
+        _df = _df.mask(_df['total'] > 700)
 
-		# not confirmed
-		"""
-		## remove the bin over 4000 nm which num. conc. larger than 1
-		# _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2]>=4.]
+        # not confirmed
+        """
+        ## remove the bin over 4000 nm which num. conc. larger than 1
+        # _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2]>=4.]
 
-		# _df_1hr[_df_remv_ky] = _df_1hr[_df_remv_ky].copy().mask(_df_1hr[_df_remv_ky]>1.)
-		# """
+        # _df_1hr[_df_remv_ky] = _df_1hr[_df_remv_ky].copy().mask(_df_1hr[_df_remv_ky]>1.)
+        # """
 
-		return _df[_df.keys()[:-1]]
+        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/Aurora.py

@@ -4,35 +4,34 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'Aurora'
+    nam = 'Aurora'
 
-	def _raw_reader(self, _file):
-		with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
-			_df = read_csv(f, low_memory=False, index_col=0)
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, low_memory=False, index_col=0)
 
-			_df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
-			_df.index.name = 'time'
+            _df.index = to_datetime(_df.index, errors='coerce')
+            _df.index.name = 'time'
 
-			_df.columns = _df.keys().str.strip(' ')
+            _df.columns = _df.keys().str.strip(' ')
 
-			_df = _df.loc[
-				_df.index.dropna(), ['0°σspB', '0°σspG', '0°σspR', '90°σspB', '90°σspG', '90°σspR', 'RH']].copy()
-			_df.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
+            # consider another csv format
+            _df = _df.rename(columns={
+                '0°σspB': 'B', '0°σspG': 'G', '0°σspR': 'R',
+                '90°σspB': 'BB', '90°σspG': 'BG', '90°σspR': 'BR',
+                'Blue': 'B', 'Green': 'G', 'Red': 'R',
+                'B_Blue': 'BB', 'B_Green': 'BG', 'B_Red': 'BR',
+                'RH': 'RH'
+            })
 
-		return _df
+            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
 
-	## QC data
-	def _QC(self, _df):
-		## remove negative value
-		_df = _df.mask((_df <= 0).copy())
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-		## call by _QC function
-		## QC data in 1 hr
-		def _QC_func(_df_1hr):
-			_df_ave = _df_1hr.mean()
-			_df_std = _df_1hr.std()
-			_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
+    # QC data
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df.mask((_df <= 0) | (_df > 2000)).copy()
 
-			return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-		return _df.resample('1h', group_keys=False).apply(_QC_func)
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/BC1054.py

@@ -4,43 +4,39 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'BC1054'
-
-	def _raw_reader(self, _file):
-		with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
-			_df = read_csv(f, parse_dates=['Time'], index_col='Time')
-
-			_df = _df.rename(columns={
-				'BC1(ng/m3)': 'BC1',
-				'BC2(ng/m3)': 'BC2',
-				'BC3(ng/m3)': 'BC3',
-				'BC4(ng/m3)': 'BC4',
-				'BC5(ng/m3)': 'BC5',
-				'BC6(ng/m3)': 'BC6',
-				'BC7(ng/m3)': 'BC7',
-				'BC8(ng/m3)': 'BC8',
-				'BC9(ng/m3)': 'BC9',
-				'BC10(ng/m3)': 'BC10'
-			})
-
-			# remove data without Status=32 (Automatic Tape Advance), 65536 (Tape Move)
-			# if not self._oth_set.get('ignore_err', False):
-			#     _df = _df.where((_df['Status'] != 32) | (_df['Status'] != 65536)).copy()
-
-			return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10', 'Status']]
-
-	# QC data
-	def _QC(self, _df):
-		# remove negative value
-		_df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
-
-		# call by _QC function
-		# QC data in 1 hr
-		def _QC_func(_df_1hr):
-			_df_ave = _df_1hr.mean()
-			_df_std = _df_1hr.std()
-			_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-			return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-		return _df.resample('1h', group_keys=False).apply(_QC_func).resample('5min').mean()
+    nam = 'BC1054'
+
+    def _raw_reader(self, _file):
+        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=True, index_col=0)
+
+            _df.columns = _df.columns.str.replace(' ', '')
+
+            _df = _df.rename(columns={
+                'BC1(ng/m3)': 'BC1',
+                'BC2(ng/m3)': 'BC2',
+                'BC3(ng/m3)': 'BC3',
+                'BC4(ng/m3)': 'BC4',
+                'BC5(ng/m3)': 'BC5',
+                'BC6(ng/m3)': 'BC6',
+                'BC7(ng/m3)': 'BC7',
+                'BC8(ng/m3)': 'BC8',
+                'BC9(ng/m3)': 'BC9',
+                'BC10(ng/m3)': 'BC10'
+            })
+
+            # remove data without Status=1, 8, 16, 32 (Automatic Tape Advance), 65536 (Tape Move)
+            if self.meta.get('error_state', False):
+                _df = _df[~_df['Status'].isin(self.meta.get('error_state'))]
+
+            _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']]
+
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    # QC data
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
+
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/EPA_vertical.py

@@ -1,18 +1,46 @@
+import numpy as np
 from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'EPA_vertical'
+    nam = 'EPA_vertical'
 
-	def _raw_reader(self, _file):
-		with _file.open('r', encoding='big5', errors='ignore') as f:
-			_df = read_csv(f, names=['time', 'station', 'comp', 'data', None], skiprows=1, na_values=['-'],
-						   parse_dates=['time'], index_col='time')
-			_df['data'] = to_numeric(_df['data'], errors='coerce')
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='ascii', errors='ignore') as f:
+            # 有、無輸出有效值都可以
+            # read 查詢小時值(測項).csv
+            df = read_csv(f, encoding='ascii', encoding_errors='ignore', index_col=0, parse_dates=True,
+                          usecols=lambda col: col != 'Unnamed: 1')
 
-			_df_piv = _df.pivot_table(values='data', columns='comp', index='time')
-			_df_piv.index.name = 'time'
+            df.index.name = 'Time'
+            df.rename(columns={'AMB_TEMP': 'AT', 'WIND_SPEED': 'WS', 'WIND_DIREC': 'WD'}, inplace=True)
 
-		return _df_piv
+            # 欄位排序
+            desired_order = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC', 'CH4', 'PM10', 'PM2.5', 'WS', 'WD',
+                             'AT', 'RH']
+
+            missing_columns = []
+
+            for col in desired_order:
+                if col not in df.columns:
+                    df[col] = np.nan
+                    missing_columns.append(col)
+
+            if missing_columns:
+                self.logger.info(f"{'=' * 60}")
+                self.logger.info(f"Missing columns: {missing_columns}")
+                self.logger.info(f"{'=' * 60}")
+                print(f"Missing columns: {missing_columns}")
+
+            df = df[desired_order]
+
+            # 如果沒有將無效值拿掉就輸出 請將包含 #、L、O 的字串替換成 *
+            df.replace(to_replace=r'\d*[#LO]\b', value='*', regex=True, inplace=True)
+            df = df.apply(to_numeric, errors='coerce')
+
+        return df
+
+    def _QC(self, _df):
+        return _df

AeroViz/rawDataReader/script/GRIMM.py

@@ -4,32 +4,25 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'GRIMM'
+    nam = 'GRIMM'
 
-	def _raw_reader(self, _file):
+    def _raw_reader(self, _file):
 
-		_df = read_csv(_file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
-					   dayfirst=True).rename_axis("Time")
-		_df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
+        _df = read_csv(_file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
+                       dayfirst=True).rename_axis("Time")
+        _df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
 
-		if _file.name.startswith("A407ST"):
-			_df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
-		else:
-			_df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
+        if _file.name.startswith("A407ST"):
+            _df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
+        else:
+            _df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
 
-		if _df.empty:
-			print(_file, "is empty")
-			return None
+        if _df.empty:
+            print(_file, "is empty")
+            return None
 
-		return _df / 0.035
+        return _df / 0.035
 
-	def _QC(self, _df):
-		# QC data in 1 hr
-		def _QC_func(_df_1hr):
-			_df_ave = _df_1hr.mean()
-			_df_std = _df_1hr.std()
-			_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-			return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-		return _df.resample('5min').apply(_QC_func).resample('1h').mean()
+    def _QC(self, _df):
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/IGAC.py

@@ -0,0 +1,90 @@
+# read meteorological data from google sheet
+
+
+import numpy as np
+from pandas import read_csv, concat, to_numeric
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'IGAC'
+
+    def _raw_reader(self, _file):
+
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=True, index_col=0, na_values='-').apply(to_numeric, errors='coerce')
+
+            _df.columns = _df.keys().str.strip(' ')
+            _df.index.name = 'time'
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+
+        # QC parameter, function (MDL SE LE)
+        _mdl = {
+            'Na+': 0.06,
+            'NH4+': 0.05,
+            'K+': 0.05,
+            'Mg2+': 0.12,
+            'Ca2+': 0.07,
+            'Cl-': 0.07,
+            'NO2-': 0.05,
+            'NO3-': 0.11,
+            'SO42-': 0.08,
+        }
+
+        # _mdl.update(self._oth_set.get('mdl', {}))
+
+        def _se_le(_df_, _log=False):
+            _df_ = np.log10(_df_) if _log else _df_
+
+            _df_qua = _df_.quantile([.25, .75])
+            _df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
+            _df_iqr = _df_q3 - _df_q1
+
+            _se = concat([_df_q1 - 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
+            _le = concat([_df_q3 + 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
+
+            if _log:
+                return 10 ** _se, 10 ** _le
+            return _se, _le
+
+        _cation, _anion, _main = (['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'],
+                                  ['Cl-', 'NO2-', 'NO3-', 'SO42-', ],
+                                  ['SO42-', 'NO3-', 'NH4+'])
+
+        _df_salt = _df[_mdl.keys()].copy()
+        _df_pm = _df['PM2.5'].copy()
+
+        # lower than PM2.5
+        # conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
+        _df_salt = _df_salt.mask(_df_salt.sum(axis=1, min_count=1) > _df_pm).dropna(subset=_main).copy()
+
+        # mdl
+        for (_key, _df_col), _mdl_val in zip(_df_salt.items(), _mdl.values()):
+            _df_salt[_key] = _df_col.mask(_df_col < _mdl_val, _mdl_val / 2)
+
+        # calculate SE LE
+        # salt < LE
+        _se, _le = _se_le(_df_salt, _log=True)
+        _df_salt = _df_salt.mask(_df_salt > _le).copy()
+
+        # C/A, A/C
+        _rat_CA = (_df_salt[_cation].sum(axis=1) / _df_salt[_anion].sum(axis=1)).to_frame()
+        _rat_AC = (1 / _rat_CA).copy()
+
+        _se, _le = _se_le(_rat_CA, )
+        _cond_CA = (_rat_CA < _le) & (_rat_CA > 0)
+
+        _se, _le = _se_le(_rat_AC, )
+        _cond_AC = (_rat_AC < _le) & (_rat_AC > 0)
+
+        _df_salt = _df_salt.where((_cond_CA * _cond_AC)[0]).copy()
+
+        # conc. of main salt > SE
+        _se, _le = _se_le(_df_salt[_main], _log=True)
+        _df_salt[_main] = _df_salt[_main].mask(_df_salt[_main] < _se).copy()
+
+        return _df_salt.reindex(_df.index)

AeroViz/rawDataReader/script/MA350.py

@@ -4,42 +4,35 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'MA350'
-
-	def _raw_reader(self, _file):
-		_df = read_csv(_file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis("Time")
-
-		_df = _df.rename(columns={
-			'UV BCc': 'BC1',
-			'Blue BCc': 'BC2',
-			'Green BCc': 'BC3',
-			'Red BCc': 'BC4',
-			'IR BCc': 'BC5',
-			'Biomass BCc  (ng/m^3)': 'BB mass',
-			'Fossil fuel BCc  (ng/m^3)': 'FF mass',
-			'Delta-C  (ng/m^3)': 'Delta-C',
-			'AAE': 'AAE',
-			'BB (%)': 'BB',
-		})
-
-		# remove data without Status=32 (Automatic Tape Advance), 65536 (Tape Move)
-		# if not self._oth_set.get('ignore_err', False):
-		#     _df = _df.where((_df['Status'] != 32) | (_df['Status'] != 65536)).copy()
-
-		return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE', 'BB']]
-
-	# QC data
-	def _QC(self, _df):
-		# remove negative value
-		_df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'AAE', 'BB']].mask((_df < 0).copy())
-
-		# call by _QC function
-		# QC data in 1 hr
-		def _QC_func(_df_1hr):
-			_df_ave = _df_1hr.mean()
-			_df_std = _df_1hr.std()
-			_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
-
-			return _df_1hr.mask(_df_lowb | _df_highb).copy()
-
-		return _df.resample('1h', group_keys=False).apply(_QC_func).resample('5min').mean()
+    nam = 'MA350'
+
+    def _raw_reader(self, _file):
+        _df = read_csv(_file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis("Time")
+
+        _df = _df.rename(columns={
+            'UV BCc': 'BC1',
+            'Blue BCc': 'BC2',
+            'Green BCc': 'BC3',
+            'Red BCc': 'BC4',
+            'IR BCc': 'BC5',
+            'Biomass BCc  (ng/m^3)': 'BB mass',
+            'Fossil fuel BCc  (ng/m^3)': 'FF mass',
+            'Delta-C  (ng/m^3)': 'Delta-C',
+            'AAE': 'AAE',
+            'BB (%)': 'BB',
+        })
+
+        # if self.meta.get('error_state', False):
+        #     _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
+
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE', 'BB']]
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    # QC data
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'AAE', 'BB']].mask((_df < 0).copy())
+
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/Minion.py

@@ -0,0 +1,103 @@
+import numpy as np
+from pandas import read_csv, to_datetime, to_numeric
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'Minion'
+
+    def _raw_reader(self, _file):
+        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, low_memory=False, index_col=0)
+
+            _df.index = to_datetime(_df.index, errors='coerce')
+            _df.index.name = 'time'
+
+            _df.columns = _df.keys().str.strip(' ')
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        # XRF QAQC
+        _df = self.XRF_QAQC(_df)
+
+        # ions balance
+        _df = self.ions_balance(_df)
+
+        # remove negative value
+        _df = _df.mask((_df < 0).copy())
+
+        # QC data in 6h
+        return _df.resample('6h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+
+    # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
+    def XRF_QAQC(self, df):
+        MDL = {
+            'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2,
+            'Cl': 1.7, 'K': 1.2, 'Ca': 0.3, 'Ti': 1.6,
+            'V': 0.12, 'Cr': 0.12, 'Mn': 0.14, 'Fe': 0.17,
+            'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079, 'Zn': 0.067,
+            'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
+            'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28,
+            'Zr': 0.33, 'Nb': 0.41, 'Mo': 0.48, 'Ag': 1.9,
+            'Cd': 2.5, 'In': 3.1, 'Sn': 4.1, 'Sb': 5.2,
+            'Te': 0.6, 'I': 0.49, 'Cs': 0.37, 'Ba': 0.39,
+            'La': 0.36, 'Ce': 0.3, 'Pt': 0.12, 'Au': 0.1,
+            'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13, 'Bi': 0.13
+        }
+        # 將小於 MDL 值的數據替換為 NaN
+        for element, threshold in MDL.items():
+            if element in df.columns:
+                df[element] = df[element].where(df[element] >= threshold, np.nan)
+
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"XRF QAQC summary:")
+        self.logger.info("\t\ttransform values below MDL to NaN")
+        self.logger.info(f"{'=' * 60}")
+
+        return df
+
+    def ions_balance(self, df, tolerance=0.3):
+        """
+        Calculate the balance of ions in the system
+        """
+        # Define the ions
+        item = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'F-', 'Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-']
+
+        # Calculate the balance
+        _df = df[item].copy()
+        _df = _df.apply(lambda x: to_numeric(x, errors='coerce'))
+        _df['+_mole'] = _df[['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+']].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1,
+                                                                                                             skipna=True)
+        _df['-_mole'] = _df[['Cl-', 'NO2-', 'NO3-', 'SO42-']].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
+
+        # Avoid division by zero
+        _df['ratio'] = np.where(_df['-_mole'] != 0, _df['+_mole'] / _df['-_mole'], np.nan)
+
+        # Calculate bounds
+        lower_bound, upper_bound = 1 - tolerance, 1 + tolerance
+
+        # 根据ratio决定是否保留原始数据
+        valid_mask = (
+                (_df['ratio'] <= upper_bound) &
+                (_df['ratio'] >= lower_bound) &
+                ~np.isnan(_df['+_mole']) &
+                ~np.isnan(_df['-_mole'])
+        )
+
+        # 保留数据或将不符合条件的行设为NaN
+        df.loc[~valid_mask, item] = np.nan
+
+        # 计算保留的数据的百分比
+        retained_percentage = (valid_mask.sum() / len(df)) * 100
+
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Ions balance summary:")
+        self.logger.info(f"\t\tretain {retained_percentage.__round__(0)}% data within tolerance {tolerance}")
+        self.logger.info(f"{'=' * 60}")
+
+        if retained_percentage < 70:
+            self.logger.warning("Warning: The percentage of retained data is less than 70%")
+
+        return df