AeroViz 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

AeroViz/rawDataReader/core/__init__.py

@@ -1,36 +1,39 @@
-import json as jsn
+import json
 import logging
-import pickle as pkl
 from abc import ABC, abstractmethod
-from datetime import datetime as dtm
+from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Optional
 
 import numpy as np
 import pandas as pd
-from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
+from pandas import DataFrame, concat, read_pickle
+from rich.console import Console
+from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
-from ..config.supported_instruments import meta
+from AeroViz.rawDataReader.config.supported_instruments import meta
 
 __all__ = ['AbstractReader']
 
 
 class AbstractReader(ABC):
-    nam = 'AbstractReader'
+    """
+    Abstract class for reading raw data from different instruments. Each instrument should have a separate class that
+    inherits from this class and implements the abstract methods. The abstract methods are `_raw_reader` and `_QC`.
 
-    # initial data
-    # input : file path, reset switch
+    List the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
+    pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
+    'reset=True'
+    """
 
-    # list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
-    # pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
-    # 'reset=True'
+    nam = 'AbstractReader'
 
     def __init__(self,
                  path: Path | str,
-                 qc: bool = True,
-                 csv_raw: bool = True,
                  reset: bool = False,
-                 rate: bool = False,
+                 qc: bool = True,
+                 qc_freq: Optional[str] = None,
+                 rate: bool = True,
                  append_data: bool = False):
 
         self.path = Path(path)
@@ -38,10 +41,10 @@ class AbstractReader(ABC):
         self.logger = self._setup_logger()
 
         self.reset = reset
-        self.rate = rate
         self.qc = qc
-        self.csv = csv_raw
-        self.append = append_data & reset
+        self.qc_freq = qc_freq
+        self.rate = rate
+        self.append = append_data and reset
 
         self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
         self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
@@ -49,24 +52,12 @@ class AbstractReader(ABC):
         self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
         self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
 
-    # dependency injection function, customize each instrument
-    @abstractmethod
-    def _raw_reader(self, file):
-        pass
-
-    @abstractmethod
-    def _QC(self, df: DataFrame):
-        return df
-
     def __call__(self,
-                 start: dtm | None = None,
-                 end: dtm | None = None,
+                 start: datetime,
+                 end: datetime,
                  mean_freq: str = '1h',
                  csv_out: bool = True,
-                 ) -> DataFrame | None:
-
-        if start and end and end <= start:
-            raise ValueError(f"Invalid time range: start {start} is after end {end}")
+                 ) -> DataFrame:
 
         data = self._run(start, end)
 
@@ -78,210 +69,249 @@ class AbstractReader(ABC):
 
         return data
 
-    @staticmethod
-    def basic_QC(df: DataFrame):
-        df_ave, df_std = df.mean(), df.std()
-        df_lowb, df_highb = df < (df_ave - df_std * 1.5), df > (df_ave + df_std * 1.5)
-
-        return df.mask(df_lowb | df_highb).copy()
-
-    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
-    def _raw_process(self, _df):
-        # get time from df and set time to whole time to create time index
-        _st, _ed = _df.index.sort_values()[[0, -1]]
-        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
-                               freq=self.meta['freq'])
-        _tm_index.name = 'time'
+    @abstractmethod
+    def _raw_reader(self, file):
+        pass
 
-        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
+    @abstractmethod
+    def _QC(self, df: DataFrame) -> DataFrame:
+        return self.n_sigma_QC(df)
 
     def _setup_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.nam)
         logger.setLevel(logging.INFO)
+
+        for handler in logger.handlers[:]:
+            logger.removeHandler(handler)
+
         handler = logging.FileHandler(self.path / f'{self.nam}.log')
-        handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
+        handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
         logger.addHandler(handler)
         return logger
 
-    # acquisition rate and yield rate
-    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
-        if self.meta['deter_key'] is not None:
-            _start, _end = _fout_qc.index[[0, -1]]
-
-            _drop_how = 'any'
-            _the_size = len(_fout_raw.resample('1h').mean().index)
-
-            self.logger.info(f"{'=' * 60}")
-            self.logger.info(
-                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} to {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(
-                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(f"{'-' * 60}")
-            print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
+    def _rate_calculate(self, raw_data, qc_data) -> None:
+        def __base_rate(raw_data, qc_data):
+            period_size = len(raw_data.resample('1h').mean().index)
 
             for _nam, _key in self.meta['deter_key'].items():
-                if _key == ['all']:
-                    _key, _drop_how = _fout_qc.keys(), 'all'
+                _key, _drop_how = (qc_data.keys(), 'all') if _key is ['all'] else (_key, 'any')
 
-                _real_size = len(_fout_raw[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
-                _QC_size = len(_fout_qc[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                sample_size = len(raw_data[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+                qc_size = len(qc_data[_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
 
-                try:
-                    _acq_rate = round((_real_size / _the_size) * 100, 1)
-                    _yid_rate = round((_QC_size / _real_size) * 100, 1)
-                except ZeroDivisionError:
-                    _acq_rate, _yid_rate = 0, 0
+                # validate rate calculation
+                if period_size < sample_size or sample_size < qc_size or period_size == 0 or sample_size == 0:
+                    raise ValueError(f"Invalid sample sizes: period={period_size}, sample={sample_size}, QC={qc_size}")
+
+                _acq_rate = round((sample_size / period_size) * 100, 1)
+                _yid_rate = round((qc_size / sample_size) * 100, 1)
 
                 self.logger.info(f'{_nam}:')
                 self.logger.info(f"\tAcquisition rate: {_acq_rate}%")
                 self.logger.info(f'\tYield       rate: {_yid_rate}%')
                 self.logger.info(f"{'=' * 60}")
 
-                print(f'\t\t{_nam} : ')
-                print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-                print(f'\t\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
-
-    # process time index
-    @staticmethod
-    def _tmidx_process(_start, _end, _df):
-        _st, _ed = _df.index.sort_values()[[0, -1]]
-        _start, _end = to_datetime(_start) or _st, to_datetime(_end) or _ed
-        _idx = date_range(_start, _end, freq=_df.index.freq.copy())
-        _idx.name = 'time'
-
-        return _df.reindex(_idx), _st, _ed
-
-    # append new data to exist pkl
-    @staticmethod
-    def _apnd_prcs(_df_done, _df_apnd):
-
-        if _df_apnd is not None:
-            _df = concat([_df_apnd.dropna(how='all').copy(), _df_done.dropna(how='all').copy()])
-
-            _idx = date_range(*_df.index.sort_values()[[0, -1]], freq=_df_done.index.freq.copy())
-            _idx.name = 'time'
-
-            return _df.loc[~_df.index.duplicated()].copy().reindex(_idx)
-
-        return _df_done
+                print(f'\n\t{_nam} : ')
+                print(f'\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
+                print(f'\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
 
-    # remove outlier
-    def _outlier_prcs(self, _df):
-
-        if (self.path / 'outlier.json') not in self.path.glob('*.json'):
+        if self.meta['deter_key'] is not None:
+            # use qc_freq to calculate each period rate
+            if self.qc_freq is not None:
+                raw_data_grouped = raw_data.groupby(pd.Grouper(freq=self.qc_freq))
+                qc_data_grouped = qc_data.groupby(pd.Grouper(freq=self.qc_freq))
+
+                for (month, _sub_raw_data), (_, _sub_qc_data) in zip(raw_data_grouped, qc_data_grouped):
+                    self.logger.info(
+                        f"\tProcessing: {_sub_raw_data.index[0].strftime('%F')} to {_sub_raw_data.index[-1].strftime('%F')}")
+                    print(
+                        f"\n\tProcessing: {_sub_raw_data.index[0].strftime('%F')} to {_sub_raw_data.index[-1].strftime('%F')}")
+
+                    __base_rate(_sub_raw_data, _sub_qc_data)
+
+            else:
+                __base_rate(raw_data, qc_data)
+
+    def _timeIndex_process(self, _df, user_start=None, user_end=None, append_df=None):
+        """
+        Process time index, resample data, extract specified time range, and optionally append new data.
+
+        :param _df: Input DataFrame with time index
+        :param user_start: Start of user-specified time range (optional)
+        :param user_end: End of user-specified time range (optional)
+        :param append_df: DataFrame to append (optional)
+        :return: Processed DataFrame
+        """
+        # Round timestamps and remove duplicates
+        _df = _df.groupby(_df.index.round('1min')).first()
+
+        # Determine frequency
+        freq = _df.index.inferred_freq or self.meta['freq']
+
+        # Append new data if provided
+        if append_df is not None:
+            append_df.index = append_df.index.round('1min')
+            _df = pd.concat([append_df.dropna(how='all'), _df.dropna(how='all')])
+            _df = _df.loc[~_df.index.duplicated()]
+
+        # Determine time range
+        df_start, df_end = _df.index.sort_values()[[0, -1]]
+
+        # Create new time index
+        new_index = pd.date_range(user_start or df_start, user_end or df_end, freq=freq, name='time')
+
+        # Process data: convert to numeric, resample, and reindex
+        return (_df.apply(pd.to_numeric, errors='coerce')
+                .resample(freq).mean()
+                .reindex(new_index))
+
+    def _outlier_process(self, _df):
+        outlier_file = self.path / 'outlier.json'
+
+        if not outlier_file.exists():
             return _df
 
-        with (self.path / 'outlier.json').open('r', encoding='utf-8', errors='ignore') as f:
-            self.outlier = jsn.load(f)
+        with outlier_file.open('r', encoding='utf-8', errors='ignore') as f:
+            outliers = json.load(f)
 
-        for _st, _ed in self.outlier.values():
+        for _st, _ed in outliers.values():
             _df.loc[_st:_ed] = np.nan
 
         return _df
 
-    # save pickle file
     def _save_data(self, raw_data: DataFrame, qc_data: DataFrame) -> None:
-        self._safe_pickle_dump(self.pkl_nam, qc_data)
-        if self.csv:
-            qc_data.to_csv(self.csv_nam)
+        try:
+            raw_data.to_pickle(self.pkl_nam_raw)
+            raw_data.to_csv(self.csv_nam_raw)
 
-        if self.meta['deter_key'] is not None:
-            self._safe_pickle_dump(self.pkl_nam_raw, raw_data)
-            if self.csv:
-                raw_data.to_csv(self.csv_nam_raw)
+            if self.meta['deter_key'] is not None:
+                qc_data.to_pickle(self.pkl_nam)
+                qc_data.to_csv(self.csv_nam)
 
-    @staticmethod
-    def _safe_pickle_dump(file_path: Path, data: Any) -> None:
-        while True:
-            try:
-                with file_path.open('wb') as f:
-                    pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
-                break
-            except PermissionError as err:
-                print('\n', err)
-                input('\t\t\33[41m Please close the file and press "Enter" \33[0m\n')
-
-    # read pickle file
-    def _read_pkl(self):
-        with self.pkl_nam.open('rb') as qc_data, self.pkl_nam_raw.open('rb') as raw_data:
-            return pkl.load(raw_data), pkl.load(qc_data)
+        except Exception as e:
+            raise IOError(f"Error saving data. {e}")
 
     def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
-        patterns = {self.meta['pattern'].lower(), self.meta['pattern'].upper(), self.meta['pattern']}
-        files = [f for pattern in patterns for f in self.path.glob(pattern)
+        files = [f
+                 for file_pattern in self.meta['pattern']
+                 for pattern in {file_pattern.lower(), file_pattern.upper(), file_pattern}
+                 for f in self.path.glob(pattern)
                  if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
         if not files:
-            raise FileNotFoundError(f"\t\t\033[31mNo files in '{self.path}' could be read."
-                                    f"Please check the current path.\033[0m")
+            raise FileNotFoundError(f"No files in '{self.path}' could be read. Please check the current path.")
 
         df_list = []
-        for file in files:
-            print(f"\r\t\treading {file.name}", end='')
-
-            try:
-                df = self._raw_reader(file)
+        with Progress(
+                TextColumn("[bold blue]{task.description}", style="bold blue"),
+                BarColumn(bar_width=18, complete_style="green", finished_style="bright_green"),
+                TaskProgressColumn(),
+                TimeRemainingColumn(),
+                TextColumn("{task.fields[filename]}", style="yellow"),
+                console=Console(force_terminal=True, color_system="auto"),
+                expand=False
+        ) as progress:
+            task = progress.add_task(f"Reading {self.nam} files", total=len(files), filename="")
+            for file in files:
+                progress.update(task, advance=1, filename=file.name)
+                try:
+                    df = self._raw_reader(file)
 
-                if df is not None and not df.empty:
-                    df_list.append(df)
-                else:
-                    self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
+                    if df is not None and not df.empty:
+                        df_list.append(df)
+                    else:
+                        self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
 
-            except pd.errors.ParserError as e:
-                self.logger.error(f"Error tokenizing data: {e}")
+                except pd.errors.ParserError as e:
+                    self.logger.error(f"Error tokenizing data: {e}")
 
-            except Exception as e:
-                self.logger.error(f"Error reading {file.name}: {e}")
+                except Exception as e:
+                    self.logger.error(f"Error reading {file.name}: {e}")
 
         if not df_list:
             raise ValueError("All files were either empty or failed to read.")
 
-        raw_data = self._raw_process(concat(df_list))
+        raw_data = concat(df_list, axis=0).groupby(level=0).first()
+
+        raw_data = self._timeIndex_process(raw_data)
         qc_data = self._QC(raw_data)
 
         return raw_data, qc_data
 
-    # main flow
-    def _run(self, _start, _end):
-        _f_raw_done, _f_qc_done = None, None
-
+    def _run(self, user_start, user_end):
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.append):
-            print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and not self.reset:
+            print(f"\n{datetime.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mPICKLE\033[0m "
+                  f"from {user_start} to {user_end}\n")
 
-            _f_raw_done, _f_qc_done = self._read_pkl()
+            _f_raw_done, _f_qc_done = read_pickle(self.pkl_nam_raw), read_pickle(self.pkl_nam)
 
-            if not self.append:
-                _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
-                _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
+            if self.append:
+                print(f"Appending new data from {user_start} to {user_end}")
+                _f_raw_new, _f_qc_new = self._read_raw_files()
+                _f_raw = self._timeIndex_process(_f_raw_done, append_df=_f_raw_new)
+                _f_qc = self._timeIndex_process(_f_qc_done, append_df=_f_qc_new)
+            else:
+                _f_raw, _f_qc = _f_raw_done, _f_qc_done
+                return _f_qc if self.qc else _f_raw
 
-                _f_qc_done = self._outlier_prcs(_f_qc_done)
+        else:
+            print(f"\n{datetime.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mRAW DATA\033[0m "
+                  f"from {user_start} to {user_end}\n")
 
-                if self.rate:
-                    self._rate_calculate(_f_raw_done, _f_qc_done, _start_raw, _end_raw)
+            _f_raw, _f_qc = self._read_raw_files()
 
-                return _f_qc_done if self.qc else _f_raw_done
-
-        # read raw data
-        print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
-
-        _f_raw, _f_qc = self._read_raw_files()
-
-        # append new data and pickle data
-        if self.append and self.pkl_nam.exists():
-            _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
-            _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
+        # process time index
+        data_start, data_end = _f_raw.index.sort_values()[[0, -1]]
 
-        _f_qc = self._outlier_prcs(_f_qc)
+        _f_raw = self._timeIndex_process(_f_raw, user_start, user_end)
+        _f_qc = self._timeIndex_process(_f_qc, user_start, user_end)
+        _f_qc = self._outlier_process(_f_qc)
 
         # save
         self._save_data(_f_raw, _f_qc)
 
-        # process time index
-        # if (_start is not None)|(_end is not None):
-        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
-        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Raw data time : {data_start} to {data_end}")
+        self.logger.info(f"Output   time : {user_start} to {user_end}")
+        self.logger.info(f"{'-' * 60}")
 
-        self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
+        if self.rate:
+            self._rate_calculate(_f_raw, _f_qc)
 
         return _f_qc if self.qc else _f_raw
+
+    @staticmethod
+    def reorder_dataframe_columns(df, order_lists, others_col=False):
+        new_order = []
+
+        for order in order_lists:
+            # 只添加存在於DataFrame中的欄位，且不重複添加
+            new_order.extend([col for col in order if col in df.columns and col not in new_order])
+
+        if others_col:
+            # 添加所有不在新順序列表中的原始欄位，保持它們的原始順序
+            new_order.extend([col for col in df.columns if col not in new_order])
+
+        return df[new_order]
+
+    @staticmethod
+    def n_sigma_QC(df: DataFrame, std_range: int = 5) -> DataFrame:
+        df_ave, df_std = df.mean(), df.std()
+        df_lowb, df_highb = df < (df_ave - df_std * std_range), df > (df_ave + df_std * std_range)
+
+        return df.mask(df_lowb | df_highb).copy()
+
+    # "四分位數範圍法"（Inter-quartile Range Method）
+    @staticmethod
+    def IQR_QC(df: DataFrame, log_dist=False) -> tuple[DataFrame, DataFrame]:
+        df = np.log10(df) if log_dist else df
+
+        _df_qua = df.quantile([.25, .75])
+        _df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
+        _df_iqr = _df_q3 - _df_q1
+
+        _se = concat([_df_q1 - 1.5 * _df_iqr] * len(df), axis=1).T.set_index(df.index)
+        _le = concat([_df_q3 + 1.5 * _df_iqr] * len(df), axis=1).T.set_index(df.index)
+
+        return (10 ** _se, 10 ** _le) if log_dist else (_se, _le)

AeroViz/rawDataReader/script/AE33.py

@@ -27,4 +27,4 @@ class Reader(AbstractReader):
         _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
 
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/AE43.py

@@ -28,4 +28,4 @@ class Reader(AbstractReader):
         _df = _df.mask((_df < 0).copy())
 
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/APS_3321.py

@@ -1,4 +1,4 @@
-import numpy as n
+import numpy as np
 from pandas import to_datetime, read_table
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -29,7 +29,7 @@ class Reader(AbstractReader):
     # QC data
     def _QC(self, _df):
         # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 

AeroViz/rawDataReader/script/Aurora.py

@@ -37,4 +37,4 @@ class Reader(AbstractReader):
         _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
 
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/BC1054.py

@@ -39,4 +39,4 @@ class Reader(AbstractReader):
         _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
 
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/EPA.py

@@ -0,0 +1,39 @@
+from pandas import read_csv
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+desired_order1 = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC',
+                  'CH4', 'PM10', 'PM2.5', 'PM1', 'WS', 'WD', 'AT', 'RH']
+
+desired_order2 = ['Benzene', 'Toluene', 'EthylBenzene', 'm/p-Xylene', 'o-Xylene']
+
+
+class Reader(AbstractReader):
+    nam = 'EPA'
+
+    def _raw_reader(self, file):
+        # 查詢小時值(測項).csv & 查詢小時值(直式).csv (有、無輸出有效值都可以)
+        df = read_csv(file, encoding='big5', encoding_errors='ignore', index_col=0, parse_dates=True,
+                      on_bad_lines='skip')
+
+        if len(df.groupby('測站')) > 1:
+            raise ValueError(f'Multiple stations found in the file: {df['測站'].unique()}')
+        else:
+            if '測站' in df.columns:
+                df.drop(columns=['測站'], inplace=True)
+
+            if '測項' in df.columns:
+                df = df.pivot(columns='測項', values='資料')
+
+            df.rename(columns={'AMB_TEMP': 'AT', 'WIND_SPEED': 'WS', 'WIND_DIREC': 'WD'}, inplace=True)
+            df.index.name = 'Time'
+
+            # 如果沒有將無效值拿掉就輸出 請將包含 #、L 的字串替換成 # 或 _
+            df = df.replace(to_replace=r'\d*[#]\b', value='#', regex=True)
+            df = df.replace(to_replace=r'\d*[L]\b', value='_', regex=True)
+
+            # 欄位排序
+            return self.reorder_dataframe_columns(df, [desired_order1])
+
+    def _QC(self, _df):
+        return _df.resample('6h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/GRIMM.py

@@ -25,4 +25,4 @@ class Reader(AbstractReader):
 
     def _QC(self, _df):
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/IGAC.py

@@ -1,8 +1,7 @@
 # read meteorological data from google sheet
 
 
-import numpy as np
-from pandas import read_csv, concat, to_numeric
+from pandas import read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -35,24 +34,8 @@ class Reader(AbstractReader):
             'SO42-': 0.08,
         }
 
-        # _mdl.update(self._oth_set.get('mdl', {}))
-
-        def _se_le(_df_, _log=False):
-            _df_ = np.log10(_df_) if _log else _df_
-
-            _df_qua = _df_.quantile([.25, .75])
-            _df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
-            _df_iqr = _df_q3 - _df_q1
-
-            _se = concat([_df_q1 - 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
-            _le = concat([_df_q3 + 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
-
-            if _log:
-                return 10 ** _se, 10 ** _le
-            return _se, _le
-
         _cation, _anion, _main = (['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'],
-                                  ['Cl-', 'NO2-', 'NO3-', 'SO42-', ],
+                                  ['Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-', ],
                                   ['SO42-', 'NO3-', 'NH4+'])
 
         _df_salt = _df[_mdl.keys()].copy()
@@ -68,23 +51,23 @@ class Reader(AbstractReader):
 
         # calculate SE LE
         # salt < LE
-        _se, _le = _se_le(_df_salt, _log=True)
+        _se, _le = self.IQR_QC(_df_salt, log_dist=True)
         _df_salt = _df_salt.mask(_df_salt > _le).copy()
 
         # C/A, A/C
         _rat_CA = (_df_salt[_cation].sum(axis=1) / _df_salt[_anion].sum(axis=1)).to_frame()
         _rat_AC = (1 / _rat_CA).copy()
 
-        _se, _le = _se_le(_rat_CA, )
+        _se, _le = self.IQR_QC(_rat_CA, )
         _cond_CA = (_rat_CA < _le) & (_rat_CA > 0)
 
-        _se, _le = _se_le(_rat_AC, )
+        _se, _le = self.IQR_QC(_rat_AC, )
         _cond_AC = (_rat_AC < _le) & (_rat_AC > 0)
 
         _df_salt = _df_salt.where((_cond_CA * _cond_AC)[0]).copy()
 
         # conc. of main salt > SE
-        _se, _le = _se_le(_df_salt[_main], _log=True)
+        _se, _le = self.IQR_QC(_df_salt[_main], log_dist=True)
         _df_salt[_main] = _df_salt[_main].mask(_df_salt[_main] < _se).copy()
 
         return _df_salt.reindex(_df.index)

AeroViz/rawDataReader/script/MA350.py

@@ -35,4 +35,4 @@ class Reader(AbstractReader):
         _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'AAE', 'BB']].mask((_df < 0).copy())
 
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()