AeroViz 0.1.3b0__py3-none-any.whl → 0.1.5__py3-none-any.whl

AeroViz/rawDataReader/core/__init__.py

@@ -1,14 +1,16 @@
 import json as jsn
+import logging
 import pickle as pkl
 from abc import ABC, abstractmethod
-from datetime import datetime as dtm, timedelta as dtmdt
-from itertools import chain
+from datetime import datetime as dtm
 from pathlib import Path
+from typing import Any
 
 import numpy as np
+import pandas as pd
 from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
 
-from ..utils.config import meta
+from ..config.supported_instruments import meta
 
 __all__ = ['AbstractReader']
 
@@ -16,117 +18,108 @@ __all__ = ['AbstractReader']
 class AbstractReader(ABC):
     nam = 'AbstractReader'
 
-    # initial config
+    # initial data
     # input : file path, reset switch
 
     # list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
     # pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
     # 'reset=True'
 
-    def __init__(self, _path, QC=True, csv_raw=True, reset=False, rate=False, append_data=False, update_meta=None):
-        # logging.info(f'\n{self.nam}')
-        # print('='*65)
-        # logger.info(f"Reading file and process data")
+    def __init__(self,
+                 path: Path | str,
+                 qc: bool = True,
+                 csv_raw: bool = True,
+                 reset: bool = False,
+                 rate: bool = False,
+                 append_data: bool = False):
 
-        # class parameter
-        # self.index = lambda _freq: date_range(_sta, _fin, freq=_freq)
-        self.path = Path(_path)
+        self.path = Path(path)
         self.meta = meta[self.nam]
-
-        if update_meta is not None:
-            self.meta.update(update_meta)
+        self.logger = self._setup_logger()
 
         self.reset = reset
         self.rate = rate
-        self.qc = QC
+        self.qc = qc
         self.csv = csv_raw
-        self.apnd = append_data & reset
-
-        self.pkl_nam = f'_read_{self.nam.lower()}.pkl'
-        self.csv_nam = f'_read_{self.nam.lower()}.csv'
+        self.append = append_data & reset
 
-        self.pkl_nam_raw = f'_read_{self.nam.lower()}_raw.pkl'
-        self.csv_nam_raw = f'_read_{self.nam.lower()}_raw.csv'
+        self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
+        self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
+        self.pkl_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.pkl'
+        self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
+        self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
 
-        self.csv_out = f'output_{self.nam.lower()}.csv'
+    # dependency injection function, customize each instrument
+    @abstractmethod
+    def _raw_reader(self, file):
+        pass
 
-    # print(f" from {_sta.strftime('%Y-%m-%d %X')} to {_fin.strftime('%Y-%m-%d %X')}")
-    # print('='*65)
-    # print(f"{dtm.now().strftime('%m/%d %X')}")
+    @abstractmethod
+    def _QC(self, df: DataFrame):
+        return df
 
-    # get data
     def __call__(self,
                  start: dtm | None = None,
                  end: dtm | None = None,
-                 mean_freq='1h',
-                 csv_out=True,
-                 **kwarg):
-
-        self._oth_set = kwarg
+                 mean_freq: str = '1h',
+                 csv_out: bool = True,
+                 ) -> DataFrame | None:
 
         if start and end and end <= start:
-            raise ValueError(
-                f'\nPlease check out input time : '
-                f'\n\tstart : {start.strftime("%Y-%m-%d %X")}'
-                f'\n\tend : {end.strftime("%Y-%m-%d %X")}')
-
-        fout = self._run(start, end)
+            raise ValueError(f"Invalid time range: start {start} is after end {end}")
 
-        if fout is not None:
-            if mean_freq is not None:
-                fout = fout.resample(mean_freq).mean()
+        data = self._run(start, end)
 
+        if data is not None:
+            if mean_freq:
+                data = data.resample(mean_freq).mean()
             if csv_out:
-                fout.to_csv(self.path / self.csv_out)
+                data.to_csv(self.csv_out)
 
-        return fout
+        return data
 
-    # dependency injection function
-    @abstractmethod
-    def _raw_reader(self, _file):
-        # customize each instrument
-        pass
+    @staticmethod
+    def basic_QC(df: DataFrame):
+        df_ave, df_std = df.mean(), df.std()
+        df_lowb, df_highb = df < (df_ave - df_std * 1.5), df > (df_ave + df_std * 1.5)
 
-    @abstractmethod
-    def _QC(self, df: DataFrame):
-        # customize each instrument
-        return df
+        return df.mask(df_lowb | df_highb).copy()
 
     # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
     def _raw_process(self, _df):
         # get time from df and set time to whole time to create time index
         _st, _ed = _df.index.sort_values()[[0, -1]]
-        _tm_index = date_range(_st.strftime('%Y%m%d %H00'),
-                               (_ed + dtmdt(hours=1)).strftime('%Y%m%d %H00'),
+        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
                                freq=self.meta['freq'])
         _tm_index.name = 'time'
 
         return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
 
+    def _setup_logger(self) -> logging.Logger:
+        logger = logging.getLogger(self.nam)
+        logger.setLevel(logging.INFO)
+        handler = logging.FileHandler(self.path / f'{self.nam}.log')
+        handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
+        logger.addHandler(handler)
+        return logger
+
     # acquisition rate and yield rate
     def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
-
         if self.meta['deter_key'] is not None:
             _start, _end = _fout_qc.index[[0, -1]]
 
             _drop_how = 'any'
             _the_size = len(_fout_raw.resample('1h').mean().index)
 
-            _f_pth = (self.path / f'{self.nam}.log')
-            _f = _f_pth.open('r+' if _f_pth.exists() else 'w+')
-
-            _cont = _f.read()
-            _f.seek(0)
-
-            _f.write(f"\n{dtm.now().strftime('%Y/%m/%d %X')}\n")
-            _f.write(f"{'-' * 60}\n")
-            _f.write(f"rawdata time : \n\t{_st_raw.strftime('%Y-%m-%d %X')} ~ {_ed_raw.strftime('%Y-%m-%d %X')}\n")
-            _f.write(f"output time : \n\t{_start.strftime('%Y-%m-%d %X')} ~ {_end.strftime('%Y-%m-%d %X')}\n")
-            _f.write(f"{'-' * 60}\n")
-            print(f"\n\t\tfrom {_start.strftime('%Y-%m-%d %X')} to {_end.strftime('%Y-%m-%d %X')}\n")
+            self.logger.info(f"{'=' * 60}")
+            self.logger.info(
+                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} to {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
+            self.logger.info(
+                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}")
+            self.logger.info(f"{'-' * 60}")
+            print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
 
             for _nam, _key in self.meta['deter_key'].items():
-
                 if _key == ['all']:
                     _key, _drop_how = _fout_qc.keys(), 'all'
 
@@ -139,18 +132,14 @@ class AbstractReader(ABC):
                 except ZeroDivisionError:
                     _acq_rate, _yid_rate = 0, 0
 
-                _f.write(f'{_nam} : \n')
-                _f.write(f"\tacquisition rate : {_acq_rate}%\n")
-                _f.write(f'\tyield rate : {_yid_rate}%\n')
+                self.logger.info(f'{_nam}:')
+                self.logger.info(f"\tAcquisition rate: {_acq_rate}%")
+                self.logger.info(f'\tYield       rate: {_yid_rate}%')
+                self.logger.info(f"{'=' * 60}")
 
                 print(f'\t\t{_nam} : ')
                 print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-                print(f'\t\t\tyield rate : \033[91m{_yid_rate}%\033[0m')
-
-            _f.write(f"{'=' * 40}\n")
-            _f.write(_cont)
-
-            _f.close()
+                print(f'\t\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
 
     # process time index
     @staticmethod
@@ -191,86 +180,78 @@ class AbstractReader(ABC):
         return _df
 
     # save pickle file
-    def _save_dt(self, _save_raw, _save_qc):
-        # dump pickle file
-        _check = True
-        while _check:
-            try:
-                with (self.path / self.pkl_nam).open('wb') as f:
-                    pkl.dump(_save_qc, f, protocol=pkl.HIGHEST_PROTOCOL)
-
-                # dump csv file
-                if self.csv:
-                    _save_qc.to_csv(self.path / self.csv_nam)
-
-                # output raw data if qc file
-                if self.meta['deter_key'] is not None:
-                    with (self.path / self.pkl_nam_raw).open('wb') as f:
-                        pkl.dump(_save_raw, f, protocol=pkl.HIGHEST_PROTOCOL)
-
-                    if self.csv:
-                        _save_raw.to_csv(self.path / self.csv_nam_raw)
+    def _save_data(self, raw_data: DataFrame, qc_data: DataFrame) -> None:
+        self._safe_pickle_dump(self.pkl_nam, qc_data)
+        if self.csv:
+            qc_data.to_csv(self.csv_nam)
 
-                _check = False
+        if self.meta['deter_key'] is not None:
+            self._safe_pickle_dump(self.pkl_nam_raw, raw_data)
+            if self.csv:
+                raw_data.to_csv(self.csv_nam_raw)
 
-            except PermissionError as _err:
-                print('\n', _err)
-                input('\t\t\33[41m Please Close The File And Press "Enter" \33[0m\n')
+    @staticmethod
+    def _safe_pickle_dump(file_path: Path, data: Any) -> None:
+        while True:
+            try:
+                with file_path.open('wb') as f:
+                    pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
+                break
+            except PermissionError as err:
+                print('\n', err)
+                input('\t\t\33[41m Please close the file and press "Enter" \33[0m\n')
 
     # read pickle file
-    def _read_pkl(self, ):
-        with (self.path / self.pkl_nam).open('rb') as f:
-            _fout_qc = pkl.load(f)
+    def _read_pkl(self):
+        with self.pkl_nam.open('rb') as qc_data, self.pkl_nam_raw.open('rb') as raw_data:
+            return pkl.load(raw_data), pkl.load(qc_data)
 
-        if (self.path / self.pkl_nam_raw).exists():
-            with (self.path / self.pkl_nam_raw).open('rb') as f:
-                _fout_raw = pkl.load(f)
-        else:
-            _fout_raw = _fout_qc
+    def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
+        patterns = {self.meta['pattern'].lower(), self.meta['pattern'].upper(), self.meta['pattern']}
+        files = [f for pattern in patterns for f in self.path.glob(pattern)
+                 if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
-        return _fout_raw, _fout_qc
+        if not files:
+            raise FileNotFoundError(f"\t\t\033[31mNo files in '{self.path}' could be read."
+                                    f"Please check the current path.\033[0m")
 
-    # read raw data
-    def _read_raw(self, ):
-        pattern = self.meta['pattern']
-        patterns = {pattern, pattern.lower(), pattern.upper()}
-        _df_con, _f_list = None, list(chain.from_iterable(self.path.glob(p) for p in patterns))
+        df_list = []
+        for file in files:
+            print(f"\r\t\treading {file.name}", end='')
 
-        for file in _f_list:
-            if file.name in [self.csv_out, self.csv_nam, self.csv_nam_raw, f'{self.nam}.log']:
-                continue
+            try:
+                df = self._raw_reader(file)
 
-            print(f"\r\t\treading {file.name}", end='')
+                if df is not None and not df.empty:
+                    df_list.append(df)
+                else:
+                    self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
 
-            _df = self._raw_reader(file)
+            except pd.errors.ParserError as e:
+                self.logger.error(f"Error tokenizing data: {e}")
 
-            # concat the concated list
-            if _df is not None:
-                _df_con = concat([_df_con, _df]) if _df_con is not None else _df
+            except Exception as e:
+                self.logger.error(f"Error reading {file.name}: {e}")
 
-        if _df_con is None:
-            print(f"\t\t\033[31mNo File in '{self.path}' Could Read, Please Check Out the Current Path\033[0m")
-            return None, None
+        if not df_list:
+            raise ValueError("All files were either empty or failed to read.")
 
-        # QC
-        _fout_raw = self._raw_process(_df_con)
-        _fout_qc = self._QC(_fout_raw)
+        raw_data = self._raw_process(concat(df_list))
+        qc_data = self._QC(raw_data)
 
-        return _fout_raw, _fout_qc
+        return raw_data, qc_data
 
     # main flow
     def _run(self, _start, _end):
-
         _f_raw_done, _f_qc_done = None, None
 
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-        _pkl_exist = self.path / self.pkl_nam in list(self.path.glob('*.pkl'))
-        if _pkl_exist & ((~self.reset) | self.apnd):
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.append):
             print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
 
             _f_raw_done, _f_qc_done = self._read_pkl()
 
-            if not self.apnd:
+            if not self.append:
                 _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
                 _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
 
@@ -284,19 +265,17 @@ class AbstractReader(ABC):
         # read raw data
         print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
 
-        _f_raw, _f_qc = self._read_raw()
-        if _f_raw is None:
-            return None
+        _f_raw, _f_qc = self._read_raw_files()
 
         # append new data and pickle data
-        if self.apnd & _pkl_exist:
+        if self.append and self.pkl_nam.exists():
             _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
             _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
 
         _f_qc = self._outlier_prcs(_f_qc)
 
         # save
-        self._save_dt(_f_raw, _f_qc)
+        self._save_data(_f_raw, _f_qc)
 
         # process time index
         # if (_start is not None)|(_end is not None):
@@ -306,97 +285,3 @@ class AbstractReader(ABC):
         self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
 
         return _f_qc if self.qc else _f_raw
-
-# -------------------------------------------------------------------------------------
-# old flow
-# def __run(self, _start, _end):
-#
-#     ## read pickle if pickle file exists and 'reset=False' or process raw data
-#     if (self.path / self.pkl_nam in list(self.path.glob('*.pkl'))) & (~self.reset):
-#         print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
-#
-#         with (self.path / self.pkl_nam).open('rb') as f:
-#             _fout_qc = pkl.load(f)
-#
-#         _exist = (self.path / self.pkl_nam_raw).exists()
-#         if _exist:
-#             with (self.path / self.pkl_nam_raw).open('rb') as f:
-#                 _fout_raw = pkl.load(f)
-#         else:
-#             _fout_raw = _fout_qc
-#
-#         _start, _end = to_datetime(_start) or _fout_qc.index[0], to_datetime(_end) or _fout_qc.index[-1]
-#         _idx = date_range(_start, _end, freq=_fout_qc.index.freq.copy())
-#         _idx.name = 'time'
-#
-#         _fout_raw, _fout_qc = _fout_raw.reindex(_idx), _fout_qc.reindex(_idx)
-#         if (self.rate) & (_exist):
-#             self._rate_calculate(_fout_raw, _fout_qc)
-#
-#         return _fout_qc if self.qc else _fout_raw
-#     else:
-#         print(
-#             f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
-#
-#     ##=================================================================================================================
-#     ## read raw data
-#     _df_con, _f_list = None, list(self.path.glob(self.meta['pattern']))
-#
-#     if len(_f_list) == 0:
-#         print(f"\t\t\033[31mNo File in '{self.path}' Could Read, Please Check Out the Current Path\033[0m")
-#         return None
-#
-#     for file in _f_list:
-#         if file.name in [self.csv_out, self.csv_nam, self.csv_nam_raw, f'{self.nam}.log']: continue
-#
-#         print(f"\r\t\treading {file.name}", end='')
-#
-#         _df = self._raw_reader(file)
-#
-#         ## concat the concated list
-#         if _df is not None:
-#             _df_con = concat([_df_con, _df]) if _df_con is not None else _df
-#     print()
-#
-#     ## QC
-#     _save_raw = self._raw_process(_df_con)
-#     _save_qc = self._QC(_save_raw)
-#
-#     _start, _end = to_datetime(_start) or _save_raw.index[0], to_datetime(_end) or _save_raw.index[-1]
-#     _idx = date_range(_start, _end, freq=_save_raw.index.freq.copy())
-#     _idx.name = 'time'
-#
-#     _fout_raw, _fout_qc = _save_raw.reindex(_idx).copy(), _save_qc.reindex(_idx).copy()
-#
-#     self._rate_calculate(_fout_raw, _fout_qc)
-#
-#     ##=================================================================================================================
-#     ## dump pickle file
-#     _check = True
-#     while _check:
-#
-#         try:
-#             with (self.path / self.pkl_nam).open('wb') as f:
-#                 pkl.dump(_save_qc, f, protocol=pkl.HIGHEST_PROTOCOL)
-#
-#             ## dump csv file
-#             if self.csv:
-#                 _save_qc.to_csv(self.path / self.csv_nam)
-#
-#             ## output raw data if qc file
-#             if self.meta['deter_key'] is not None:
-#                 with (self.path / self.pkl_nam_raw).open('wb') as f:
-#                     pkl.dump(_save_raw, f, protocol=pkl.HIGHEST_PROTOCOL)
-#
-#                 if self.csv:
-#                     _save_raw.to_csv(self.path / self.csv_nam_raw)
-#
-#                 return _fout_qc if self.qc else _fout_raw
-#
-#             _check = False
-#
-#         except PermissionError as _err:
-#             print('\n', _err)
-#             input('\t\t\33[41m Please Close The File And Press "Enter" \33[0m\n')
-#
-#     return _fout_qc

AeroViz/rawDataReader/script/AE33.py

@@ -6,26 +6,25 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'AE33'
 
-    def _raw_reader(self, _file):
-        _df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
+    def _raw_reader(self, file):
+        if file.stat().st_size / 1024 < 550:
+            print('\t It may not be a whole daily data.')
+
+        _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
                          delimiter=r'\s+', skiprows=5, usecols=range(67))
         _df.columns = _df.columns.str.strip(';')
 
         # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
-        if not self._oth_set.get('ignore_err', False):
-            _df = _df.where((_df['Status'] != 0) | (_df['Status'] != 128) | (_df['Status'] != 256)).copy()
+        if self.meta.get('error_state', False):
+            _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
+
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
 
-        return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']]
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     def _QC(self, _df):
         # remove negative value
         _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
 
-        # QC data in 5 min
-        def _QC_func(df):
-            _df_ave, _df_std = df.mean(), df.std()
-            _df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
-
-            return df.mask(_df_lowb | _df_highb).copy()
-
-        return _df.resample('5min').apply(_QC_func).resample('1h').mean()
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/AE43.py

@@ -6,29 +6,26 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'AE43'
 
-    def _raw_reader(self, _file):
-        _df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
+    def _raw_reader(self, file):
+        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time')
         _df_id = _df['SetupID'].iloc[-1]
 
         # get last SetupID data
         _df = _df.groupby('SetupID').get_group(_df_id)[
             ['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']].copy()
 
-        # remove data without Status=0
-        _df = _df.where(_df['Status'] == 0).copy()
+        # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
+        if self.meta.get('error_state', False):
+            _df = _df.where(~_df['Status'].isin(self.meta['error_state'])).copy()
 
-        return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+        _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     # QC data
     def _QC(self, _df):
         # remove negative value
         _df = _df.mask((_df < 0).copy())
 
-        # QC data in 5 min
-        def _QC_func(df):
-            _df_ave, _df_std = df.mean(), df.std()
-            _df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
-
-            return df.mask(_df_lowb | _df_highb).copy()
-
-        return _df.resample('5min').apply(_QC_func).resample('1h').mean()
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/APS_3321.py

@@ -7,33 +7,33 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'APS_3321'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
             _df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
             _key = list(_df.keys()[3:54])  ## 542 ~ 1981
 
-            ## create new keys
+            # create new keys
             _newkey = {}
             for _k in _key:
                 _newkey[_k] = float(_k).__round__(4)
             # _newkey['Mode(m)'] = 'mode'
 
-            ## get new dataframe
+            # get new dataframe
             _df = _df[_newkey.keys()].rename(_newkey, axis=1)
-            # _df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
+            # df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
 
             _df_idx = to_datetime(_df.index, errors='coerce')
 
         return _df.set_index(_df_idx).loc[_df_idx.dropna()]
 
-    ## QC data
+    # QC data
     def _QC(self, _df):
-        ## mask out the data size lower than 7
+        # mask out the data size lower than 7
         _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 
-        ## remove total conc. lower than 700
+        # remove total conc. lower than 700
         _df = _df.mask(_df['total'] > 700)
 
         # not confirmed

AeroViz/rawDataReader/script/Aurora.py

@@ -6,33 +6,35 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'Aurora'
 
-    def _raw_reader(self, _file):
-        with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
             _df = read_csv(f, low_memory=False, index_col=0)
 
-            _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
+            _df.index = to_datetime(_df.index, errors='coerce')
             _df.index.name = 'time'
 
             _df.columns = _df.keys().str.strip(' ')
 
-            _df = _df.loc[
-                _df.index.dropna(), ['0°σspB', '0°σspG', '0°σspR', '90°σspB', '90°σspG', '90°σspR', 'RH']].copy()
-            _df.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
+            # consider another csv format
+            _df = _df.rename(columns={
+                '0°σspB': 'B', '0°σspG': 'G', '0°σspR': 'R',
+                '90°σspB': 'BB', '90°σspG': 'BG', '90°σspR': 'BR',
+                'Blue': 'B', 'Green': 'G', 'Red': 'R',
+                'B_Blue': 'BB', 'B_Green': 'BG', 'B_Red': 'BR',
+                'RH': 'RH'
+            })
 
-        return _df
+            _df = _df[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']]
 
-    ## QC data
-    def _QC(self, _df):
-        ## remove negative value
-        _df = _df.mask((_df <= 0).copy())
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-        ## call by _QC function
-        ## QC data in 1 hr
-        def _QC_func(_df_1hr):
-            _df_ave = _df_1hr.mean()
-            _df_std = _df_1hr.std()
-            _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
+    # QC data
+    def _QC(self, _df):
+        # remove negative value
+        _df = _df.mask((_df <= 0) | (_df > 2000)).copy()
 
-            return _df_1hr.mask(_df_lowb | _df_highb).copy()
+        # total scattering is larger than back scattering
+        _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
 
-        return _df.resample('1h', group_keys=False).apply(_QC_func)
+        # QC data in 1h
+        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()