AeroViz 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

AeroViz/rawDataReader/core/__init__.py

@@ -1,400 +1,277 @@
 import json as jsn
+import logging
 import pickle as pkl
 from abc import ABC, abstractmethod
-from datetime import datetime as dtm, timedelta as dtmdt
-from itertools import chain
+from datetime import datetime as dtm
 from pathlib import Path
+from typing import Any
 
 import numpy as np
 from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
 
-from ..utils.config import meta
+from ..config.supported_instruments import meta
 
 __all__ = ['AbstractReader']
 
 
 class AbstractReader(ABC):
-	nam = 'AbstractReader'
+    nam = 'AbstractReader'
+
+    # initial data
+    # input : file path, reset switch
+
+    # list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
+    # pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
+    # 'reset=True'
+
+    def __init__(self,
+                 path: Path | str,
+                 qc: bool = True,
+                 csv_raw: bool = True,
+                 reset: bool = False,
+                 rate: bool = False,
+                 append_data: bool = False):
+
+        self.path = Path(path)
+        self.meta = meta[self.nam]
+        self.logger = self._setup_logger()
+
+        self.reset = reset
+        self.rate = rate
+        self.qc = qc
+        self.csv = csv_raw
+        self.apnd = append_data & reset
+
+        self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
+        self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
+        self.pkl_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.pkl'
+        self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
+        self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
+
+    # dependency injection function, customize each instrument
+    @abstractmethod
+    def _raw_reader(self, _file):
+        pass
+
+    @abstractmethod
+    def _QC(self, df: DataFrame):
+        return df
+
+    def __call__(self,
+                 start: dtm | None = None,
+                 end: dtm | None = None,
+                 mean_freq: str = '1h',
+                 csv_out: bool = True,
+                 ) -> DataFrame | None:
+
+        if start and end and end <= start:
+            raise ValueError(f"Invalid time range: start {start} is after end {end}")
+
+        data = self._run(start, end)
+
+        if data is not None:
+            if mean_freq:
+                data = data.resample(mean_freq).mean()
+            if csv_out:
+                data.to_csv(self.csv_out)
+
+        return data
+
+    @staticmethod
+    def basic_QC(df: DataFrame):
+        df_ave, df_std = df.mean(), df.std()
+        df_lowb, df_highb = df < (df_ave - df_std * 1.5), df > (df_ave + df_std * 1.5)
+
+        return df.mask(df_lowb | df_highb).copy()
+
+    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
+    def _raw_process(self, _df):
+        # get time from df and set time to whole time to create time index
+        _st, _ed = _df.index.sort_values()[[0, -1]]
+        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
+                               freq=self.meta['freq'])
+        _tm_index.name = 'time'
+
+        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
+
+    def _setup_logger(self) -> logging.Logger:
+        logger = logging.getLogger(self.nam)
+        logger.setLevel(logging.INFO)
+        handler = logging.FileHandler(self.path / f'{self.nam}.log')
+        handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
+        logger.addHandler(handler)
+        return logger
+
+    # acquisition rate and yield rate
+    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
+        if self.meta['deter_key'] is not None:
+            _start, _end = _fout_qc.index[[0, -1]]
+
+            _drop_how = 'any'
+            _the_size = len(_fout_raw.resample('1h').mean().index)
+
+            self.logger.info(f"{'=' * 60}")
+            self.logger.info(
+                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} ~ {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
+            self.logger.info(
+                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} ~ {_end.strftime('%Y-%m-%d %H:%M:%S')}")
+            self.logger.info(f"{'-' * 60}")
+            print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
+
+            for _nam, _key in self.meta['deter_key'].items():
+                if _key == ['all']:
+                    _key, _drop_how = _fout_qc.keys(), 'all'
+
+                _real_size = len(_fout_raw[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                _QC_size = len(_fout_qc[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+
+                try:
+                    _acq_rate = round((_real_size / _the_size) * 100, 1)
+                    _yid_rate = round((_QC_size / _real_size) * 100, 1)
+                except ZeroDivisionError:
+                    _acq_rate, _yid_rate = 0, 0
+
+                self.logger.info(f'{_nam}:')
+                self.logger.info(f"\tAcquisition rate: {_acq_rate}%")
+                self.logger.info(f'\tYield       rate: {_yid_rate}%')
+                self.logger.info(f"{'=' * 60}")
+
+                print(f'\t\t{_nam} : ')
+                print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
+                print(f'\t\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+
+    # process time index
+    @staticmethod
+    def _tmidx_process(_start, _end, _df):
+        _st, _ed = _df.index.sort_values()[[0, -1]]
+        _start, _end = to_datetime(_start) or _st, to_datetime(_end) or _ed
+        _idx = date_range(_start, _end, freq=_df.index.freq.copy())
+        _idx.name = 'time'
+
+        return _df.reindex(_idx), _st, _ed
+
+    # append new data to exist pkl
+    @staticmethod
+    def _apnd_prcs(_df_done, _df_apnd):
+
+        if _df_apnd is not None:
+            _df = concat([_df_apnd.dropna(how='all').copy(), _df_done.dropna(how='all').copy()])
 
-	# initial config
-	# input : file path, reset switch
+            _idx = date_range(*_df.index.sort_values()[[0, -1]], freq=_df_done.index.freq.copy())
+            _idx.name = 'time'
+
+            return _df.loc[~_df.index.duplicated()].copy().reindex(_idx)
 
-	# list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
-	# pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
-	# 'reset=True'
+        return _df_done
 
-	def __init__(self, _path, QC=True, csv_raw=True, reset=False, rate=False, append_data=False, update_meta=None):
-		# logging.info(f'\n{self.nam}')
-		# print('='*65)
-		# logger.info(f"Reading file and process data")
+    # remove outlier
+    def _outlier_prcs(self, _df):
 
-		# class parameter
-		# self.index = lambda _freq: date_range(_sta, _fin, freq=_freq)
-		self.path = Path(_path)
-		self.meta = meta[self.nam]
+        if (self.path / 'outlier.json') not in self.path.glob('*.json'):
+            return _df
+
+        with (self.path / 'outlier.json').open('r', encoding='utf-8', errors='ignore') as f:
+            self.outlier = jsn.load(f)
 
-		if update_meta is not None:
-			self.meta.update(update_meta)
+        for _st, _ed in self.outlier.values():
+            _df.loc[_st:_ed] = np.nan
 
-		self.reset = reset
-		self.rate = rate
-		self.qc = QC
-		self.csv = csv_raw
-		self.apnd = append_data & reset
+        return _df
 
-		self.pkl_nam = f'_read_{self.nam.lower()}.pkl'
-		self.csv_nam = f'_read_{self.nam.lower()}.csv'
+    # save pickle file
+    def _save_data(self, raw_data: DataFrame, qc_data: DataFrame) -> None:
+        self._safe_pickle_dump(self.pkl_nam, qc_data)
+        if self.csv:
+            qc_data.to_csv(self.csv_nam)
 
-		self.pkl_nam_raw = f'_read_{self.nam.lower()}_raw.pkl'
-		self.csv_nam_raw = f'_read_{self.nam.lower()}_raw.csv'
+        if self.meta['deter_key'] is not None:
+            self._safe_pickle_dump(self.pkl_nam_raw, raw_data)
+            if self.csv:
+                raw_data.to_csv(self.csv_nam_raw)
 
-		self.csv_out = f'output_{self.nam.lower()}.csv'
+    @staticmethod
+    def _safe_pickle_dump(file_path: Path, data: Any) -> None:
+        while True:
+            try:
+                with file_path.open('wb') as f:
+                    pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
+                break
+            except PermissionError as err:
+                print('\n', err)
+                input('\t\t\33[41m Please close the file and press "Enter" \33[0m\n')
 
-	# print(f" from {_sta.strftime('%Y-%m-%d %X')} to {_fin.strftime('%Y-%m-%d %X')}")
-	# print('='*65)
-	# print(f"{dtm.now().strftime('%m/%d %X')}")
+    # read pickle file
+    def _read_pkl(self):
+        with self.pkl_nam.open('rb') as qc_data, self.pkl_nam_raw.open('rb') as raw_data:
+            return pkl.load(raw_data), pkl.load(qc_data)
 
-	# get data
-	def __call__(self,
-				 start: dtm | None = None,
-				 end: dtm | None = None,
-				 mean_freq='1h',
-				 csv_out=True,
-				 **kwarg):
+    def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
+        patterns = {self.meta['pattern'].lower(), self.meta['pattern'].upper(), self.meta['pattern']}
+        files = [f for pattern in patterns for f in self.path.glob(pattern)
+                 if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
-		self._oth_set = kwarg
+        if not files:
+            print(f"\t\t\033[31mNo files in '{self.path}' could be read. Please check the current path.\033[0m")
+            return None, None
 
-		if start and end and end <= start:
-			raise ValueError(
-				f'\nPlease check out input time : \n\tstart : {start.strftime("%Y-%m-%d %X")}\n\tend   : {end.strftime("%Y-%m-%d %X")}')
+        df_list = []
+        for file in files:
+            print(f"\r\t\treading {file.name}", end='')
+            df = self._raw_reader(file)
+            if df is not None:
+                df_list.append(df)
 
-		fout = self._run(start, end)
+        if not df_list:
+            return None, None
 
-		if fout is not None:
-			if mean_freq is not None:
-				fout = fout.resample(mean_freq).mean()
+        raw_data = self._raw_process(concat(df_list))
+        qc_data = self._QC(raw_data)
 
-			if csv_out:
-				fout.to_csv(self.path / self.csv_out)
+        return raw_data, qc_data
 
-		return fout
+    # main flow
+    def _run(self, _start, _end):
+        _f_raw_done, _f_qc_done = None, None
 
-	# dependency injection function
-	@abstractmethod
-	def _raw_reader(self, _file):
-		# customize each instrument
-		pass
+        # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.apnd):
+            print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
 
-	@abstractmethod
-	def _QC(self, df: DataFrame):
-		# customize each instrument
-		return df
+            _f_raw_done, _f_qc_done = self._read_pkl()
 
-	# set each to true datetime(18:30:01 -> 18:30:00) and rindex data
-	def _raw_process(self, _df):
-		# get time from df and set time to whole time to create time index
-		_st, _ed = _df.index.sort_values()[[0, -1]]
-		_tm_index = date_range(_st.strftime('%Y%m%d %H00'),
-							   (_ed + dtmdt(hours=1)).strftime('%Y%m%d %H00'),
-							   freq=self.meta['freq'])
-		_tm_index.name = 'time'
+            if not self.apnd:
+                _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
+                _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
 
-		return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
+                _f_qc_done = self._outlier_prcs(_f_qc_done)
 
-	# acquisition rate and yield rate
-	def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
+                if self.rate:
+                    self._rate_calculate(_f_raw_done, _f_qc_done, _start_raw, _end_raw)
 
-		if self.meta['deter_key'] is not None:
-			_start, _end = _fout_qc.index[[0, -1]]
+                return _f_qc_done if self.qc else _f_raw_done
 
-			_drop_how = 'any'
-			_the_size = len(_fout_raw.resample('1h').mean().index)
+        # read raw data
+        print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
 
-			_f_pth = (self.path / f'{self.nam}.log')
-			_f = _f_pth.open('r+' if _f_pth.exists() else 'w+')
+        _f_raw, _f_qc = self._read_raw_files()
+        if _f_raw is None:
+            return None
 
-			_cont = _f.read()
-			_f.seek(0)
+        # append new data and pickle data
+        if self.apnd and self.pkl_nam.exists():
+            _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
+            _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
 
-			_f.write(f"\n{dtm.now().strftime('%Y/%m/%d %X')}\n")
-			_f.write(f"{'-' * 60}\n")
-			_f.write(f"rawdata time : \n\t{_st_raw.strftime('%Y-%m-%d %X')} ~ {_ed_raw.strftime('%Y-%m-%d %X')}\n")
-			_f.write(f"output time : \n\t{_start.strftime('%Y-%m-%d %X')} ~ {_end.strftime('%Y-%m-%d %X')}\n")
-			_f.write(f"{'-' * 60}\n")
-			print(f"\n\t\tfrom {_start.strftime('%Y-%m-%d %X')} to {_end.strftime('%Y-%m-%d %X')}\n")
+        _f_qc = self._outlier_prcs(_f_qc)
 
-			for _nam, _key in self.meta['deter_key'].items():
+        # save
+        self._save_data(_f_raw, _f_qc)
 
-				if _key == ['all']:
-					_key, _drop_how = _fout_qc.keys(), 'all'
+        # process time index
+        # if (_start is not None)|(_end is not None):
+        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
+        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
 
-				_real_size = len(_fout_raw[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
-				_QC_size = len(_fout_qc[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+        self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
 
-				try:
-					_acq_rate = round((_real_size / _the_size) * 100, 1)
-					_yid_rate = round((_QC_size / _real_size) * 100, 1)
-				except ZeroDivisionError:
-					_acq_rate, _yid_rate = 0, 0
-
-				_f.write(f'{_nam} : \n')
-				_f.write(f"\tacquisition rate : {_acq_rate}%\n")
-				_f.write(f'\tyield rate : {_yid_rate}%\n')
-
-				print(f'\t\t{_nam} : ')
-				print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-				print(f'\t\t\tyield rate : \033[91m{_yid_rate}%\033[0m')
-
-			_f.write(f"{'=' * 40}\n")
-			_f.write(_cont)
-
-			_f.close()
-
-	# process time index
-	@staticmethod
-	def _tmidx_process(_start, _end, _df):
-		_st, _ed = _df.index.sort_values()[[0, -1]]
-		_start, _end = to_datetime(_start) or _st, to_datetime(_end) or _ed
-		_idx = date_range(_start, _end, freq=_df.index.freq.copy())
-		_idx.name = 'time'
-
-		return _df.reindex(_idx), _st, _ed
-
-	# append new data to exist pkl
-	@staticmethod
-	def _apnd_prcs(_df_done, _df_apnd):
-
-		if _df_apnd is not None:
-			_df = concat([_df_apnd.dropna(how='all').copy(), _df_done.dropna(how='all').copy()])
-
-			_idx = date_range(*_df.index.sort_values()[[0, -1]], freq=_df_done.index.freq.copy())
-			_idx.name = 'time'
-
-			return _df.loc[~_df.index.duplicated()].copy().reindex(_idx)
-
-		return _df_done
-
-	# remove outlier
-	def _outlier_prcs(self, _df):
-
-		if (self.path / 'outlier.json') not in self.path.glob('*.json'):
-			return _df
-
-		with (self.path / 'outlier.json').open('r', encoding='utf-8', errors='ignore') as f:
-			self.outlier = jsn.load(f)
-
-		for _st, _ed in self.outlier.values():
-			_df.loc[_st:_ed] = np.nan
-
-		return _df
-
-	# save pickle file
-	def _save_dt(self, _save_raw, _save_qc):
-		# dump pickle file
-		_check = True
-		while _check:
-			try:
-				with (self.path / self.pkl_nam).open('wb') as f:
-					pkl.dump(_save_qc, f, protocol=pkl.HIGHEST_PROTOCOL)
-
-				# dump csv file
-				if self.csv:
-					_save_qc.to_csv(self.path / self.csv_nam)
-
-				# output raw data if qc file
-				if self.meta['deter_key'] is not None:
-					with (self.path / self.pkl_nam_raw).open('wb') as f:
-						pkl.dump(_save_raw, f, protocol=pkl.HIGHEST_PROTOCOL)
-
-					if self.csv:
-						_save_raw.to_csv(self.path / self.csv_nam_raw)
-
-				_check = False
-
-			except PermissionError as _err:
-				print('\n', _err)
-				input('\t\t\33[41m Please Close The File And Press "Enter" \33[0m\n')
-
-	# read pickle file
-	def _read_pkl(self, ):
-		with (self.path / self.pkl_nam).open('rb') as f:
-			_fout_qc = pkl.load(f)
-
-		if (self.path / self.pkl_nam_raw).exists():
-			with (self.path / self.pkl_nam_raw).open('rb') as f:
-				_fout_raw = pkl.load(f)
-		else:
-			_fout_raw = _fout_qc
-
-		return _fout_raw, _fout_qc
-
-	# read raw data
-	def _read_raw(self, ):
-		pattern = self.meta['pattern']
-		patterns = {pattern, pattern.lower(), pattern.upper()}
-		_df_con, _f_list = None, list(chain.from_iterable(self.path.glob(p) for p in patterns))
-
-		for file in _f_list:
-			if file.name in [self.csv_out, self.csv_nam, self.csv_nam_raw, f'{self.nam}.log']:
-				continue
-
-			print(f"\r\t\treading {file.name}", end='')
-
-			_df = self._raw_reader(file)
-
-			# concat the concated list
-			if _df is not None:
-				_df_con = concat([_df_con, _df]) if _df_con is not None else _df
-
-		if _df_con is None:
-			print(f"\t\t\033[31mNo File in '{self.path}' Could Read, Please Check Out the Current Path\033[0m")
-			return None, None
-
-		# QC
-		_fout_raw = self._raw_process(_df_con)
-		_fout_qc = self._QC(_fout_raw)
-
-		return _fout_raw, _fout_qc
-
-	# main flow
-	def _run(self, _start, _end):
-
-		_f_raw_done, _f_qc_done = None, None
-
-		# read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-		_pkl_exist = self.path / self.pkl_nam in list(self.path.glob('*.pkl'))
-		if _pkl_exist & ((~self.reset) | self.apnd):
-			print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
-
-			_f_raw_done, _f_qc_done = self._read_pkl()
-
-			if not self.apnd:
-				_f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
-				_f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
-
-				_f_qc_done = self._outlier_prcs(_f_qc_done)
-
-				if self.rate:
-					self._rate_calculate(_f_raw_done, _f_qc_done, _start_raw, _end_raw)
-
-				return _f_qc_done if self.qc else _f_raw_done
-
-		# read raw data
-		print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
-
-		_f_raw, _f_qc = self._read_raw()
-		if _f_raw is None:
-			return None
-
-		# append new data and pickle data
-		if self.apnd & _pkl_exist:
-			_f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
-			_f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
-
-		_f_qc = self._outlier_prcs(_f_qc)
-
-		# save
-		self._save_dt(_f_raw, _f_qc)
-
-		# process time index
-		# if (_start is not None)|(_end is not None):
-		_f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
-		_f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
-
-		self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
-
-		return _f_qc if self.qc else _f_raw
-
-	# -------------------------------------------------------------------------------------
-	# old flow
-	# def __run(self, _start, _end):
-	#
-	#     ## read pickle if pickle file exists and 'reset=False' or process raw data
-	#     if (self.path / self.pkl_nam in list(self.path.glob('*.pkl'))) & (~self.reset):
-	#         print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
-	#
-	#         with (self.path / self.pkl_nam).open('rb') as f:
-	#             _fout_qc = pkl.load(f)
-	#
-	#         _exist = (self.path / self.pkl_nam_raw).exists()
-	#         if _exist:
-	#             with (self.path / self.pkl_nam_raw).open('rb') as f:
-	#                 _fout_raw = pkl.load(f)
-	#         else:
-	#             _fout_raw = _fout_qc
-	#
-	#         _start, _end = to_datetime(_start) or _fout_qc.index[0], to_datetime(_end) or _fout_qc.index[-1]
-	#         _idx = date_range(_start, _end, freq=_fout_qc.index.freq.copy())
-	#         _idx.name = 'time'
-	#
-	#         _fout_raw, _fout_qc = _fout_raw.reindex(_idx), _fout_qc.reindex(_idx)
-	#         if (self.rate) & (_exist):
-	#             self._rate_calculate(_fout_raw, _fout_qc)
-	#
-	#         return _fout_qc if self.qc else _fout_raw
-	#     else:
-	#         print(
-	#             f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
-	#
-	#     ##=================================================================================================================
-	#     ## read raw data
-	#     _df_con, _f_list = None, list(self.path.glob(self.meta['pattern']))
-	#
-	#     if len(_f_list) == 0:
-	#         print(f"\t\t\033[31mNo File in '{self.path}' Could Read, Please Check Out the Current Path\033[0m")
-	#         return None
-	#
-	#     for file in _f_list:
-	#         if file.name in [self.csv_out, self.csv_nam, self.csv_nam_raw, f'{self.nam}.log']: continue
-	#
-	#         print(f"\r\t\treading {file.name}", end='')
-	#
-	#         _df = self._raw_reader(file)
-	#
-	#         ## concat the concated list
-	#         if _df is not None:
-	#             _df_con = concat([_df_con, _df]) if _df_con is not None else _df
-	#     print()
-	#
-	#     ## QC
-	#     _save_raw = self._raw_process(_df_con)
-	#     _save_qc = self._QC(_save_raw)
-	#
-	#     _start, _end = to_datetime(_start) or _save_raw.index[0], to_datetime(_end) or _save_raw.index[-1]
-	#     _idx = date_range(_start, _end, freq=_save_raw.index.freq.copy())
-	#     _idx.name = 'time'
-	#
-	#     _fout_raw, _fout_qc = _save_raw.reindex(_idx).copy(), _save_qc.reindex(_idx).copy()
-	#
-	#     self._rate_calculate(_fout_raw, _fout_qc)
-	#
-	#     ##=================================================================================================================
-	#     ## dump pickle file
-	#     _check = True
-	#     while _check:
-	#
-	#         try:
-	#             with (self.path / self.pkl_nam).open('wb') as f:
-	#                 pkl.dump(_save_qc, f, protocol=pkl.HIGHEST_PROTOCOL)
-	#
-	#             ## dump csv file
-	#             if self.csv:
-	#                 _save_qc.to_csv(self.path / self.csv_nam)
-	#
-	#             ## output raw data if qc file
-	#             if self.meta['deter_key'] is not None:
-	#                 with (self.path / self.pkl_nam_raw).open('wb') as f:
-	#                     pkl.dump(_save_raw, f, protocol=pkl.HIGHEST_PROTOCOL)
-	#
-	#                 if self.csv:
-	#                     _save_raw.to_csv(self.path / self.csv_nam_raw)
-	#
-	#                 return _fout_qc if self.qc else _fout_raw
-	#
-	#             _check = False
-	#
-	#         except PermissionError as _err:
-	#             print('\n', _err)
-	#             input('\t\t\33[41m Please Close The File And Press "Enter" \33[0m\n')
-	#
-	#     return _fout_qc
+        return _f_qc if self.qc else _f_raw

AeroViz/rawDataReader/script/AE33.py

@@ -4,28 +4,27 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-	nam = 'AE33'
+    nam = 'AE33'
 
-	def _raw_reader(self, _file):
-		_df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
-						 delimiter=r'\s+', skiprows=5, usecols=range(67))
-		_df.columns = _df.columns.str.strip(';')
+    def _raw_reader(self, _file):
+        if _file.stat().st_size / 1024 < 550:
+            print('\t It may not be a whole daily data.')
 
-		# remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
-		if not self._oth_set.get('ignore_err', False):
-			_df = _df.where((_df['Status'] != 0) | (_df['Status'] != 128) | (_df['Status'] != 256)).copy()
+        _df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
+                         delimiter=r'\s+', skiprows=5, usecols=range(67))
+        _df.columns = _df.columns.str.strip(';')
 
-		return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']]
+        # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
+        if self.meta.get('error_state', False):
+            _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-	def _QC(self, _df):
-		# remove negative value
-		_df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
 
-		# QC data in 5 min
-		def _QC_func(df):
-			_df_ave, _df_std = df.mean(), df.std()
-			_df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-			return df.mask(_df_lowb | _df_highb).copy()
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
 
-		return _df.resample('5min').apply(_QC_func).resample('1h').mean()
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()