AeroViz 0.1.5__tar.gz → 0.1.6__tar.gz

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/dataProcess/Chemistry/_mass_volume.py

@@ -2,13 +2,14 @@ from pandas import concat, DataFrame
 
 
 def _basic(df_che, df_ref, df_water, df_density, nam_lst):
+
     df_all = concat(df_che, axis=1)
     index = df_all.index.copy()
     df_all.columns = nam_lst
 
     # parameter
     mol_A, mol_S, mol_N = df_all['NH4+'] / 18, df_all['SO42-'] / 96, df_all['NO3-'] / 62
-    df_all['status'] = (mol_A) / (2 * mol_S + mol_N)
+    df_all['status'] = mol_A / (2 * mol_S + mol_N)
 
     convert_nam = {'AS': 'SO42-',
                    'AN': 'NO3-',
@@ -92,7 +93,7 @@ def _basic(df_che, df_ref, df_water, df_density, nam_lst):
     df_mass['total'] = df_mass.sum(axis=1, min_count=6)
 
     qc_ratio = df_mass['total'] / df_ref
-    qc_cond = (qc_ratio >= 0.7) & (qc_ratio <= 1.3)
+    qc_cond = (qc_ratio >= 0.5) & (qc_ratio <= 1.5)
 
     # volume
     df_vol = DataFrame()
@@ -153,7 +154,7 @@ def _basic(df_che, df_ref, df_water, df_density, nam_lst):
     out.update(ri_dic)
 
     for _ky, _df in out.items():
-        out[_ky] = _df.reindex(index).where(qc_cond)
+        out[_ky] = _df.reindex(index)
 
     return out
 

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/dataProcess/Optical/_IMPROVE.py

@@ -1,8 +1,7 @@
-import pickle as pkl
 from pathlib import Path
 
 import numpy as np
-from pandas import DataFrame
+from pandas import DataFrame, read_pickle
 
 from AeroViz.dataProcess.core import union_index
 
@@ -12,7 +11,7 @@ def _revised(_df_mass, _df_RH):
 
     # fRH
     with (Path(__file__).parent / 'fRH.pkl').open('rb') as f:
-        _fRH = pkl.load(f)
+        _fRH = read_pickle(f)
         _fRH.loc[np.nan] = np.nan
 
     def fRH(_RH):

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/dataProcess/SizeDistr/__init__.py

@@ -1,15 +1,11 @@
 from ..core import Writer, run_process
 
-__all__ = [
-
-    'SizeDistr',
-
-]
+__all__ = ['SizeDistr']
 
 
 class SizeDistr(Writer):
 
-    ## basic
+    # basic
     @run_process('SizeDistr - basic', 'distr_basic')
     def basic(self, df, hybrid_bin_start_loc=None, unit='nm', bin_range=(0, 20000), input_type='norm'):
         from ._size_distr import _basic
@@ -18,7 +14,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v4', 'distr_merge')
     def merge_SMPS_APS_v4(self, df_smps, df_aps, df_pm25, aps_unit='um',
                           smps_overlap_lowbound=500, aps_fit_highbound=1000, dndsdv_alg=True,
@@ -30,7 +26,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v3', 'distr_merge')
     def merge_SMPS_APS_v3(self, df_smps, df_aps, aps_unit='um',
                           smps_overlap_lowbound=500, aps_fit_highbound=1000, dndsdv_alg=True):
@@ -40,7 +36,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v2', 'distr_merge')
     def merge_SMPS_APS_v2(self, df_smps, df_aps, aps_unit='um',
                           smps_overlap_lowbound=500, aps_fit_highbound=1000):
@@ -50,7 +46,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v1', 'distr_merge')
     def merge_SMPS_APS(self, df_smps, df_aps, aps_unit='um', shift_mode='mobility',
                        smps_overlap_lowbound=523, aps_fit_highbound=800):

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/rawDataReader/__init__.py

@@ -12,7 +12,7 @@ def RawDataReader(instrument_name: str,
                   qc: bool = True,
                   csv_raw: bool = True,
                   reset: bool = False,
-                  rate: bool = False,
+                  rate: bool = True,
                   append_data: bool = False,
                   start: datetime | None = None,
                   end: datetime | None = None,
@@ -72,9 +72,7 @@ def RawDataReader(instrument_name: str,
     instrument_class_map = {
         'NEPH': NEPH,
         'Aurora': Aurora,
-        'SMPS_genr': SMPS_genr,
-        'SMPS_aim11': SMPS_aim11,
-        'SMPS_TH': SMPS_TH,
+        'SMPS': SMPS,
         'GRIMM': GRIMM,
         'APS_3321': APS_3321,
         'AE33': AE33,
@@ -82,7 +80,7 @@ def RawDataReader(instrument_name: str,
         'BC1054': BC1054,
         'MA350': MA350,
         'TEOM': TEOM,
-        'Sunset_OCEC': Sunset_OCEC,
+        'OCEC': OCEC,
         'IGAC': IGAC,
         'VOC': VOC,
         'Table': Table,

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/rawDataReader/config/supported_instruments.py

@@ -2,76 +2,64 @@
 
 meta = {
     "NEPH": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "5min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
     "Aurora": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
-    "SMPS_TH": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_genr": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_aim11": {
-        "pattern": "*.csv",
+    "SMPS": {
+        "pattern": ["*.txt", "*.csv"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "GRIMM": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "APS_3321": {
-        "pattern": "*.TXT",
+        "pattern": ["*.txt"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "AE33": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE33*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE33*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "AE43": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE43*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE43*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "BC1054": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC9"]},
         "error_state": [1, 2, 4, 8, 16, 32, 65536],
     },
 
     "MA350": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC5"]},
     },
 
     "TEOM": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "6min",
         "deter_key": {
             "PM1.0 Mass Conc.": ["PM_Total"],
@@ -79,8 +67,8 @@ meta = {
         },
     },
 
-    "Sunset_OCEC": {
-        "pattern": "*LCRes.csv",
+    "OCEC": {
+        "pattern": ["*LCRes.csv"],
         "freq": "1h",
         "deter_key": {
             "Thermal OC": ["Thermal_OC"],
@@ -91,7 +79,7 @@ meta = {
     },
 
     "IGAC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": {
             "Na+": ["Na+"],
@@ -108,7 +96,7 @@ meta = {
     },
 
     "VOC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "key": [
             'Benzene', 'Toluene', 'Ethylbenzene', 'm/p-Xylene', 'o-Xylene', 'Ethane', 'Propane', 'Isobutane',
@@ -129,20 +117,31 @@ meta = {
     },
 
     "Table": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": None,
     },
 
     "EPA_vertical": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": None,
     },
 
     "Minion": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
-        "deter_key": None,
+        "deter_key": {
+            "Na+": ["Na+"],
+            "NH4+": ["NH4+"],
+            "K+": ["K+"],
+            "Mg2+": ["Mg2+"],
+            "Ca2+": ["Ca2+"],
+            "Cl-": ["Cl-"],
+            "NO2-": ["NO2-"],
+            "NO3-": ["NO3-"],
+            "SO42-": ["SO42-"],
+            "Main Salt (NH4+, NO3-, SO42-)": ["NO3-", "SO42-", "NH4+"],
+        },
     },
 }

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/rawDataReader/core/__init__.py

@@ -1,4 +1,4 @@
-import json as jsn
+import json
 import logging
 import pickle as pkl
 from abc import ABC, abstractmethod
@@ -9,28 +9,35 @@ from typing import Any
 import numpy as np
 import pandas as pd
 from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
+from rich.console import Console
+from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
 from ..config.supported_instruments import meta
 
 __all__ = ['AbstractReader']
 
 
+console = Console(force_terminal=True, color_system="auto")
+
+
 class AbstractReader(ABC):
-    nam = 'AbstractReader'
+    """
+    Abstract class for reading raw data from different instruments. Each instrument should have a separate class that
+    inherits from this class and implements the abstract methods. The abstract methods are `_raw_reader` and `_QC`.
 
-    # initial data
-    # input : file path, reset switch
+    List the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
+    pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
+    'reset=True'
+    """
 
-    # list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
-    # pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
-    # 'reset=True'
+    nam = 'AbstractReader'
 
     def __init__(self,
                  path: Path | str,
                  qc: bool = True,
                  csv_raw: bool = True,
                  reset: bool = False,
-                 rate: bool = False,
+                 rate: bool = True,
                  append_data: bool = False):
 
         self.path = Path(path)
@@ -41,7 +48,7 @@ class AbstractReader(ABC):
         self.rate = rate
         self.qc = qc
         self.csv = csv_raw
-        self.append = append_data & reset
+        self.append = append_data and reset
 
         self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
         self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
@@ -49,21 +56,12 @@ class AbstractReader(ABC):
         self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
         self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
 
-    # dependency injection function, customize each instrument
-    @abstractmethod
-    def _raw_reader(self, file):
-        pass
-
-    @abstractmethod
-    def _QC(self, df: DataFrame):
-        return df
-
     def __call__(self,
                  start: dtm | None = None,
                  end: dtm | None = None,
                  mean_freq: str = '1h',
                  csv_out: bool = True,
-                 ) -> DataFrame | None:
+                 ) -> DataFrame:
 
         if start and end and end <= start:
             raise ValueError(f"Invalid time range: start {start} is after end {end}")
@@ -78,6 +76,14 @@ class AbstractReader(ABC):
 
         return data
 
+    @abstractmethod
+    def _raw_reader(self, file):
+        pass
+
+    @abstractmethod
+    def _QC(self, df: DataFrame):
+        return df
+
     @staticmethod
     def basic_QC(df: DataFrame):
         df_ave, df_std = df.mean(), df.std()
@@ -85,40 +91,25 @@ class AbstractReader(ABC):
 
         return df.mask(df_lowb | df_highb).copy()
 
-    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
-    def _raw_process(self, _df):
-        # get time from df and set time to whole time to create time index
-        _st, _ed = _df.index.sort_values()[[0, -1]]
-        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
-                               freq=self.meta['freq'])
-        _tm_index.name = 'time'
-
-        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
-
     def _setup_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.nam)
         logger.setLevel(logging.INFO)
+
+        for handler in logger.handlers[:]:
+            logger.removeHandler(handler)
+
         handler = logging.FileHandler(self.path / f'{self.nam}.log')
         handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
         logger.addHandler(handler)
         return logger
 
-    # acquisition rate and yield rate
-    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
+    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw) -> None:
         if self.meta['deter_key'] is not None:
             _start, _end = _fout_qc.index[[0, -1]]
 
             _drop_how = 'any'
             _the_size = len(_fout_raw.resample('1h').mean().index)
 
-            self.logger.info(f"{'=' * 60}")
-            self.logger.info(
-                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} to {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(
-                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(f"{'-' * 60}")
-            print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
-
             for _nam, _key in self.meta['deter_key'].items():
                 if _key == ['all']:
                     _key, _drop_how = _fout_qc.keys(), 'all'
@@ -137,9 +128,19 @@ class AbstractReader(ABC):
                 self.logger.info(f'\tYield       rate: {_yid_rate}%')
                 self.logger.info(f"{'=' * 60}")
 
-                print(f'\t\t{_nam} : ')
-                print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-                print(f'\t\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+                print(f'\n\t{_nam} : ')
+                print(f'\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
+                print(f'\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+
+    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
+    def _raw_process(self, _df):
+        # get time from df and set time to whole time to create time index
+        _st, _ed = _df.index.sort_values()[[0, -1]]
+        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
+                               freq=self.meta['freq'])
+        _tm_index.name = 'time'
+
+        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
 
     # process time index
     @staticmethod
@@ -153,7 +154,7 @@ class AbstractReader(ABC):
 
     # append new data to exist pkl
     @staticmethod
-    def _apnd_prcs(_df_done, _df_apnd):
+    def _append_process(_df_done, _df_apnd):
 
         if _df_apnd is not None:
             _df = concat([_df_apnd.dropna(how='all').copy(), _df_done.dropna(how='all').copy()])
@@ -165,16 +166,16 @@ class AbstractReader(ABC):
 
         return _df_done
 
-    # remove outlier
-    def _outlier_prcs(self, _df):
+    def _outlier_process(self, _df):
+        outlier_file = self.path / 'outlier.json'
 
-        if (self.path / 'outlier.json') not in self.path.glob('*.json'):
+        if not outlier_file.exists():
             return _df
 
-        with (self.path / 'outlier.json').open('r', encoding='utf-8', errors='ignore') as f:
-            self.outlier = jsn.load(f)
+        with outlier_file.open('r', encoding='utf-8', errors='ignore') as f:
+            outliers = json.load(f)
 
-        for _st, _ed in self.outlier.values():
+        for _st, _ed in outliers.values():
             _df.loc[_st:_ed] = np.nan
 
         return _df
@@ -192,14 +193,13 @@ class AbstractReader(ABC):
 
     @staticmethod
     def _safe_pickle_dump(file_path: Path, data: Any) -> None:
-        while True:
-            try:
-                with file_path.open('wb') as f:
-                    pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
-                break
-            except PermissionError as err:
-                print('\n', err)
-                input('\t\t\33[41m Please close the file and press "Enter" \33[0m\n')
+        try:
+            with file_path.open('wb') as f:
+                pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
+        except PermissionError as e:
+            raise IOError(f"Unable to write to {file_path}. The file may be in use or you may not have permission: {e}")
+        except Exception as e:
+            raise IOError(f"Error writing to {file_path}: {e}")
 
     # read pickle file
     def _read_pkl(self):
@@ -207,31 +207,41 @@ class AbstractReader(ABC):
             return pkl.load(raw_data), pkl.load(qc_data)
 
     def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
-        patterns = {self.meta['pattern'].lower(), self.meta['pattern'].upper(), self.meta['pattern']}
-        files = [f for pattern in patterns for f in self.path.glob(pattern)
+        files = [f
+                 for file_pattern in self.meta['pattern']
+                 for pattern in {file_pattern.lower(), file_pattern.upper(), file_pattern}
+                 for f in self.path.glob(pattern)
                  if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
         if not files:
-            raise FileNotFoundError(f"\t\t\033[31mNo files in '{self.path}' could be read."
-                                    f"Please check the current path.\033[0m")
+            raise FileNotFoundError(f"No files in '{self.path}' could be read. Please check the current path.")
 
         df_list = []
-        for file in files:
-            print(f"\r\t\treading {file.name}", end='')
-
-            try:
-                df = self._raw_reader(file)
+        with Progress(
+                TextColumn("[bold blue]{task.description}", style="bold blue"),
+                BarColumn(bar_width=18, complete_style="green", finished_style="bright_green"),
+                TaskProgressColumn(),
+                TimeRemainingColumn(),
+                TextColumn("{task.fields[filename]}", style="yellow"),
+                console=console,
+                expand=False
+        ) as progress:
+            task = progress.add_task(f"Reading {self.nam} files", total=len(files), filename="")
+            for file in files:
+                progress.update(task, advance=1, filename=file.name)
+                try:
+                    df = self._raw_reader(file)
 
-                if df is not None and not df.empty:
-                    df_list.append(df)
-                else:
-                    self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
+                    if df is not None and not df.empty:
+                        df_list.append(df)
+                    else:
+                        self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
 
-            except pd.errors.ParserError as e:
-                self.logger.error(f"Error tokenizing data: {e}")
+                except pd.errors.ParserError as e:
+                    self.logger.error(f"Error tokenizing data: {e}")
 
-            except Exception as e:
-                self.logger.error(f"Error reading {file.name}: {e}")
+                except Exception as e:
+                    self.logger.error(f"Error reading {file.name}: {e}")
 
         if not df_list:
             raise ValueError("All files were either empty or failed to read.")
@@ -241,47 +251,42 @@ class AbstractReader(ABC):
 
         return raw_data, qc_data
 
-    # main flow
     def _run(self, _start, _end):
-        _f_raw_done, _f_qc_done = None, None
-
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.append):
-            print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and not self.reset:
+            print(f"\n{dtm.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mPICKLE\033[0m "
+                  f"from {_start} to {_end}\n")
 
             _f_raw_done, _f_qc_done = self._read_pkl()
 
-            if not self.append:
-                _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
-                _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
-
-                _f_qc_done = self._outlier_prcs(_f_qc_done)
-
-                if self.rate:
-                    self._rate_calculate(_f_raw_done, _f_qc_done, _start_raw, _end_raw)
-
-                return _f_qc_done if self.qc else _f_raw_done
+            if self.append:
+                print(f"Appending new data from {_start} to {_end}")
+                _f_raw_new, _f_qc_new = self._read_raw_files()
+                _f_raw = self._append_process(_f_raw_done, _f_raw_new)
+                _f_qc = self._append_process(_f_qc_done, _f_qc_new)
+            else:
+                _f_raw, _f_qc = _f_raw_done, _f_qc_done
 
-        # read raw data
-        print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
+        else:
+            print(f"\n{dtm.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mRAW DATA\033[0m "
+                  f"from {_start} to {_end}\n")
+            _f_raw, _f_qc = self._read_raw_files()
 
-        _f_raw, _f_qc = self._read_raw_files()
-
-        # append new data and pickle data
-        if self.append and self.pkl_nam.exists():
-            _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
-            _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
+        # process time index
+        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
+        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
 
-        _f_qc = self._outlier_prcs(_f_qc)
+        _f_qc = self._outlier_process(_f_qc)
 
         # save
         self._save_data(_f_raw, _f_qc)
 
-        # process time index
-        # if (_start is not None)|(_end is not None):
-        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
-        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Raw data time : {_start_raw} to {_end_raw}")
+        self.logger.info(f"Output   time : {_start} to {_end}")
+        self.logger.info(f"{'-' * 60}")
 
-        self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
+        if self.rate:
+            self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
 
         return _f_qc if self.qc else _f_raw

{aeroviz-0.1.5 → aeroviz-0.1.6}/AeroViz/rawDataReader/script/APS_3321.py

@@ -1,4 +1,4 @@
-import numpy as n
+import numpy as np
 from pandas import to_datetime, read_table
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -29,7 +29,7 @@ class Reader(AbstractReader):
     # QC data
     def _QC(self, _df):
         # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 

aeroviz-0.1.5/AeroViz/rawDataReader/script/Sunset_OCEC.py → aeroviz-0.1.6/AeroViz/rawDataReader/script/OCEC.py

@@ -4,7 +4,7 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-    nam = 'Sunset_OCEC'
+    nam = 'OCEC'
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f: