AeroViz 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

AeroViz/rawDataReader/__init__.py

@@ -12,20 +12,67 @@ def RawDataReader(instrument_name: str,
                   qc: bool = True,
                   csv_raw: bool = True,
                   reset: bool = False,
-                  rate: bool = False,
+                  rate: bool = True,
                   append_data: bool = False,
                   start: datetime | None = None,
                   end: datetime | None = None,
                   mean_freq='1h',
                   csv_out=True,
                   ):
+    """
+    Factory function to instantiate the appropriate reader module for a given instrument and
+    return the processed data over the specified time range.
+
+    Parameters
+    ----------
+    instrument_name : str
+        The name of the instrument for which to read data. Must be a valid key in the `meta` dictionary.
+    path : Path
+        The directory where raw data files for the instrument are stored.
+    qc : bool, optional (default=True)
+        If True, apply quality control (QC) to the raw data.
+    csv_raw : bool, optional (default=True)
+        If True, read raw data from CSV files.
+    reset : bool, optional (default=False)
+        If True, reset the state and reprocess the data from scratch.
+    rate : bool, optional (default=False)
+        If True, calculate rates from the data.
+    append_data : bool, optional (default=False)
+        If True, append new data to the existing dataset instead of overwriting it.
+    start : datetime, optional
+        Start time for filtering the data. If None, no start time filtering will be applied.
+    end : datetime, optional
+        End time for filtering the data. If None, no end time filtering will be applied.
+    mean_freq : str, optional (default='1h')
+        Resampling frequency for averaging the data. Example: '1h' for hourly mean.
+    csv_out : bool, optional (default=True)
+        If True, output the processed data as a CSV file.
+
+    Return
+    ------
+    reader_module : Reader
+        An instance of the reader module corresponding to the specified instrument, which processes
+        the data and returns it in a usable format.
+
+    Raises
+    ------
+    ValueError
+        If the `instrument_name` provided is not a valid key in the `meta` dictionary.
+
+    Example
+    -------
+    To read and process data for the BC1054 instrument:
+
+    >>> from pathlib import Path
+    >>> from datetime import datetime
+    >>> data = RawDataReader(instrument_name='BC1054', path=Path('/path/to/data'),
+    >>>                      start=datetime(2024, 1, 1), end=datetime(2024, 2, 1))
+    """
     # Mapping of instrument names to their respective classes
     instrument_class_map = {
         'NEPH': NEPH,
         'Aurora': Aurora,
-        'SMPS_genr': SMPS_genr,
-        'SMPS_aim11': SMPS_aim11,
-        'SMPS_TH': SMPS_TH,
+        'SMPS': SMPS,
         'GRIMM': GRIMM,
         'APS_3321': APS_3321,
         'AE33': AE33,
@@ -33,7 +80,7 @@ def RawDataReader(instrument_name: str,
         'BC1054': BC1054,
         'MA350': MA350,
         'TEOM': TEOM,
-        'Sunset_OCEC': Sunset_OCEC,
+        'OCEC': OCEC,
         'IGAC': IGAC,
         'VOC': VOC,
         'Table': Table,

AeroViz/rawDataReader/config/supported_instruments.py

@@ -2,76 +2,64 @@
 
 meta = {
     "NEPH": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "5min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
     "Aurora": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
-    "SMPS_TH": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_genr": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_aim11": {
-        "pattern": "*.csv",
+    "SMPS": {
+        "pattern": ["*.txt", "*.csv"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "GRIMM": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "APS_3321": {
-        "pattern": "*.TXT",
+        "pattern": ["*.txt"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "AE33": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE33*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE33*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "AE43": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE43*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE43*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "BC1054": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC9"]},
         "error_state": [1, 2, 4, 8, 16, 32, 65536],
     },
 
     "MA350": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC5"]},
     },
 
     "TEOM": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "6min",
         "deter_key": {
             "PM1.0 Mass Conc.": ["PM_Total"],
@@ -79,21 +67,19 @@ meta = {
         },
     },
 
-    "Sunset_OCEC": {
-        "pattern": "*LCRes.csv",
+    "OCEC": {
+        "pattern": ["*LCRes.csv"],
         "freq": "1h",
         "deter_key": {
-            "Thermal OC/EC": ["Thermal_EC", "Thermal_OC"],
             "Thermal OC": ["Thermal_OC"],
             "Thermal EC": ["Thermal_EC"],
-            "Optical OC/EC": ["Optical_EC", "Optical_OC"],
             "Optical OC": ["Optical_OC"],
             "Optical EC": ["Optical_EC"],
         },
     },
 
     "IGAC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": {
             "Na+": ["Na+"],
@@ -110,46 +96,52 @@ meta = {
     },
 
     "VOC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
-        "key": ['Ethane', 'Propane', 'Isobutane', 'n-Butane', 'Cyclopentane', 'Isopentane',
-                'n-Pentane', '2,2-Dimethylbutane', '2,3-Dimethylbutane', '2-Methylpentane',
-                '3-Methylpentane', 'n-Hexane', 'Methylcyclopentane', '2,4-Dimethylpentane',
-                'Cyclohexane', '2-Methylhexane', '2-Methylhexane', '3-Methylheptane',
-                '2,2,4-Trimethylpentane', 'n-Heptane', 'Methylcyclohexane',
-                '2,3,4-Trimethylpentane', '2-Methylheptane', '3-Methylhexane', 'n-Octane',
-                'n-Nonane', 'n-Decane', 'n-Undecane', 'Ethylene', 'Propylene', 't-2-Butene',
-                '1-Butene', 'cis-2-Butene', 't-2-Pentene', '1-Pentene', 'cis-2-Pentene',
-                'isoprene', 'Acetylene', 'Benzene', 'Toluene', 'Ethylbenzene', 'm,p-Xylene',
-                'Styrene', 'o-Xylene', 'Isopropylbenzene', 'n-Propylbenzene', 'm-Ethyltoluene',
-                'p-Ethyltoluene', '1,3,5-Trimethylbenzene', 'o-Ethyltoluene',
-                '1,2,4-Trimethylbenzene', '1,2,3-Trimethylbenzene', 'm-Diethylbenzene',
-                'p-Diethylbenzene'],
-
-        "key_2": ['Isopentane', 'Hexane', '2-Methylhexane', '3-Methylhexane', '2-Methylheptane', '3-Methylheptane',
-                  'Propene', '1.3-Butadiene', 'Isoprene', '1-Octene',
-                  'Benzene', 'Toluene', 'Ethylbenzene', 'm.p-Xylene', 'o-Xylene', 'Iso-Propylbenzene', 'Styrene',
-                  'n-Propylbenzene', '3.4-Ethyltoluene', '1.3.5-TMB', '2-Ethyltoluene', '1.2.4-TMB', '1.2.3-TMB',
-                  'Acetaldehyde', 'Ethanol', 'Acetone', 'IPA', 'Ethyl Acetate', 'Butyl Acetate',
-                  'VCM', 'TCE', 'PCE', '1.4-DCB', '1.2-DCB'],
+        "key": [
+            'Benzene', 'Toluene', 'Ethylbenzene', 'm/p-Xylene', 'o-Xylene', 'Ethane', 'Propane', 'Isobutane',
+            'n-Butane', 'Isopentane', 'n-Pentane', 'n-Hexane', 'n-Heptane', 'n-Octane', 'n-Nonane', 'n-Decane',
+            'n-Undecane', 'n-Dodecane', 'Ethylene', 'Propylene', '1-Butene', 't-2-Butene', 'cis-2-Butene',
+            '1-Pentene', 't-2-Pentene', 'cis-2-Pentene', '1-Hexene', 'Acetylene', 'Cyclopentane', 'Methylcyclopentane',
+            'Cyclohexane', 'Methylcyclohexane', 'Isoprene', '2,2-Dimethylbutane', '2,3-Dimethylbutane',
+            '2-Methylpentane', '3-Methylpentane', '2,4-Dimethylpentane', '2-Methylhexane', '2,3-Dimethylpentane',
+            '3-Methylheptane', '2,2,4-Trimethylpentane', '2,3,4-Trimethylpentane', '2-Methylheptane', '3-Methylhexane',
+            'Styrene', 'Isopropylbenzene', 'n-Propylbenzene', 'm-Ethyltoluene', 'p-Ethyltoluene', 'm-Diethylbenzene',
+            'p-Diethylbenzene', '1,3,5-Trimethylbenzene', 'o-Ethyltoluene', '1,2,4-Trimethylbenzene',
+            '1,2,3-Trimethylbenzene',
+            '1.2-DCB', '1.4-DCB', '1.3-Butadiene', '1-Octene', '2-Ethyltoluene', '3.4-Ethyltoluene', 'Acetaldehyde',
+            'Acetone', 'Butyl Acetate', 'Ethanol', 'Ethyl Acetate', 'Hexane', 'IPA', 'Iso-Propylbenzene',
+            'PCE', 'Propene', 'TCE', 'VCM',
+        ],
         "deter_key": None,
     },
 
     "Table": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": None,
     },
 
     "EPA_vertical": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": None,
     },
 
     "Minion": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
-        "deter_key": None,
+        "deter_key": {
+            "Na+": ["Na+"],
+            "NH4+": ["NH4+"],
+            "K+": ["K+"],
+            "Mg2+": ["Mg2+"],
+            "Ca2+": ["Ca2+"],
+            "Cl-": ["Cl-"],
+            "NO2-": ["NO2-"],
+            "NO3-": ["NO3-"],
+            "SO42-": ["SO42-"],
+            "Main Salt (NH4+, NO3-, SO42-)": ["NO3-", "SO42-", "NH4+"],
+        },
     },
 }

AeroViz/rawDataReader/core/__init__.py

@@ -1,4 +1,4 @@
-import json as jsn
+import json
 import logging
 import pickle as pkl
 from abc import ABC, abstractmethod
@@ -7,29 +7,37 @@ from pathlib import Path
 from typing import Any
 
 import numpy as np
+import pandas as pd
 from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
+from rich.console import Console
+from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
 from ..config.supported_instruments import meta
 
 __all__ = ['AbstractReader']
 
 
+console = Console(force_terminal=True, color_system="auto")
+
+
 class AbstractReader(ABC):
-    nam = 'AbstractReader'
+    """
+    Abstract class for reading raw data from different instruments. Each instrument should have a separate class that
+    inherits from this class and implements the abstract methods. The abstract methods are `_raw_reader` and `_QC`.
 
-    # initial data
-    # input : file path, reset switch
+    List the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
+    pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
+    'reset=True'
+    """
 
-    # list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
-    # pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
-    # 'reset=True'
+    nam = 'AbstractReader'
 
     def __init__(self,
                  path: Path | str,
                  qc: bool = True,
                  csv_raw: bool = True,
                  reset: bool = False,
-                 rate: bool = False,
+                 rate: bool = True,
                  append_data: bool = False):
 
         self.path = Path(path)
@@ -40,7 +48,7 @@ class AbstractReader(ABC):
         self.rate = rate
         self.qc = qc
         self.csv = csv_raw
-        self.apnd = append_data & reset
+        self.append = append_data and reset
 
         self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
         self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
@@ -48,21 +56,12 @@ class AbstractReader(ABC):
         self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
         self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
 
-    # dependency injection function, customize each instrument
-    @abstractmethod
-    def _raw_reader(self, _file):
-        pass
-
-    @abstractmethod
-    def _QC(self, df: DataFrame):
-        return df
-
     def __call__(self,
                  start: dtm | None = None,
                  end: dtm | None = None,
                  mean_freq: str = '1h',
                  csv_out: bool = True,
-                 ) -> DataFrame | None:
+                 ) -> DataFrame:
 
         if start and end and end <= start:
             raise ValueError(f"Invalid time range: start {start} is after end {end}")
@@ -77,6 +76,14 @@ class AbstractReader(ABC):
 
         return data
 
+    @abstractmethod
+    def _raw_reader(self, file):
+        pass
+
+    @abstractmethod
+    def _QC(self, df: DataFrame):
+        return df
+
     @staticmethod
     def basic_QC(df: DataFrame):
         df_ave, df_std = df.mean(), df.std()
@@ -84,40 +91,25 @@ class AbstractReader(ABC):
 
         return df.mask(df_lowb | df_highb).copy()
 
-    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
-    def _raw_process(self, _df):
-        # get time from df and set time to whole time to create time index
-        _st, _ed = _df.index.sort_values()[[0, -1]]
-        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
-                               freq=self.meta['freq'])
-        _tm_index.name = 'time'
-
-        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
-
     def _setup_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.nam)
         logger.setLevel(logging.INFO)
+
+        for handler in logger.handlers[:]:
+            logger.removeHandler(handler)
+
         handler = logging.FileHandler(self.path / f'{self.nam}.log')
         handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
         logger.addHandler(handler)
         return logger
 
-    # acquisition rate and yield rate
-    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
+    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw) -> None:
         if self.meta['deter_key'] is not None:
             _start, _end = _fout_qc.index[[0, -1]]
 
             _drop_how = 'any'
             _the_size = len(_fout_raw.resample('1h').mean().index)
 
-            self.logger.info(f"{'=' * 60}")
-            self.logger.info(
-                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} ~ {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(
-                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} ~ {_end.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(f"{'-' * 60}")
-            print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
-
             for _nam, _key in self.meta['deter_key'].items():
                 if _key == ['all']:
                     _key, _drop_how = _fout_qc.keys(), 'all'
@@ -136,9 +128,19 @@ class AbstractReader(ABC):
                 self.logger.info(f'\tYield       rate: {_yid_rate}%')
                 self.logger.info(f"{'=' * 60}")
 
-                print(f'\t\t{_nam} : ')
-                print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-                print(f'\t\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+                print(f'\n\t{_nam} : ')
+                print(f'\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
+                print(f'\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+
+    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
+    def _raw_process(self, _df):
+        # get time from df and set time to whole time to create time index
+        _st, _ed = _df.index.sort_values()[[0, -1]]
+        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
+                               freq=self.meta['freq'])
+        _tm_index.name = 'time'
+
+        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
 
     # process time index
     @staticmethod
@@ -152,7 +154,7 @@ class AbstractReader(ABC):
 
     # append new data to exist pkl
     @staticmethod
-    def _apnd_prcs(_df_done, _df_apnd):
+    def _append_process(_df_done, _df_apnd):
 
         if _df_apnd is not None:
             _df = concat([_df_apnd.dropna(how='all').copy(), _df_done.dropna(how='all').copy()])
@@ -164,16 +166,16 @@ class AbstractReader(ABC):
 
         return _df_done
 
-    # remove outlier
-    def _outlier_prcs(self, _df):
+    def _outlier_process(self, _df):
+        outlier_file = self.path / 'outlier.json'
 
-        if (self.path / 'outlier.json') not in self.path.glob('*.json'):
+        if not outlier_file.exists():
             return _df
 
-        with (self.path / 'outlier.json').open('r', encoding='utf-8', errors='ignore') as f:
-            self.outlier = jsn.load(f)
+        with outlier_file.open('r', encoding='utf-8', errors='ignore') as f:
+            outliers = json.load(f)
 
-        for _st, _ed in self.outlier.values():
+        for _st, _ed in outliers.values():
             _df.loc[_st:_ed] = np.nan
 
         return _df
@@ -191,14 +193,13 @@ class AbstractReader(ABC):
 
     @staticmethod
     def _safe_pickle_dump(file_path: Path, data: Any) -> None:
-        while True:
-            try:
-                with file_path.open('wb') as f:
-                    pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
-                break
-            except PermissionError as err:
-                print('\n', err)
-                input('\t\t\33[41m Please close the file and press "Enter" \33[0m\n')
+        try:
+            with file_path.open('wb') as f:
+                pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
+        except PermissionError as e:
+            raise IOError(f"Unable to write to {file_path}. The file may be in use or you may not have permission: {e}")
+        except Exception as e:
+            raise IOError(f"Error writing to {file_path}: {e}")
 
     # read pickle file
     def _read_pkl(self):
@@ -206,72 +207,86 @@ class AbstractReader(ABC):
             return pkl.load(raw_data), pkl.load(qc_data)
 
     def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
-        patterns = {self.meta['pattern'].lower(), self.meta['pattern'].upper(), self.meta['pattern']}
-        files = [f for pattern in patterns for f in self.path.glob(pattern)
+        files = [f
+                 for file_pattern in self.meta['pattern']
+                 for pattern in {file_pattern.lower(), file_pattern.upper(), file_pattern}
+                 for f in self.path.glob(pattern)
                  if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
         if not files:
-            print(f"\t\t\033[31mNo files in '{self.path}' could be read. Please check the current path.\033[0m")
-            return None, None
+            raise FileNotFoundError(f"No files in '{self.path}' could be read. Please check the current path.")
 
         df_list = []
-        for file in files:
-            print(f"\r\t\treading {file.name}", end='')
-            df = self._raw_reader(file)
-            if df is not None:
-                df_list.append(df)
+        with Progress(
+                TextColumn("[bold blue]{task.description}", style="bold blue"),
+                BarColumn(bar_width=18, complete_style="green", finished_style="bright_green"),
+                TaskProgressColumn(),
+                TimeRemainingColumn(),
+                TextColumn("{task.fields[filename]}", style="yellow"),
+                console=console,
+                expand=False
+        ) as progress:
+            task = progress.add_task(f"Reading {self.nam} files", total=len(files), filename="")
+            for file in files:
+                progress.update(task, advance=1, filename=file.name)
+                try:
+                    df = self._raw_reader(file)
+
+                    if df is not None and not df.empty:
+                        df_list.append(df)
+                    else:
+                        self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
+
+                except pd.errors.ParserError as e:
+                    self.logger.error(f"Error tokenizing data: {e}")
+
+                except Exception as e:
+                    self.logger.error(f"Error reading {file.name}: {e}")
 
         if not df_list:
-            return None, None
+            raise ValueError("All files were either empty or failed to read.")
 
         raw_data = self._raw_process(concat(df_list))
         qc_data = self._QC(raw_data)
 
         return raw_data, qc_data
 
-    # main flow
     def _run(self, _start, _end):
-        _f_raw_done, _f_qc_done = None, None
-
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.apnd):
-            print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and not self.reset:
+            print(f"\n{dtm.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mPICKLE\033[0m "
+                  f"from {_start} to {_end}\n")
 
             _f_raw_done, _f_qc_done = self._read_pkl()
 
-            if not self.apnd:
-                _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
-                _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
-
-                _f_qc_done = self._outlier_prcs(_f_qc_done)
-
-                if self.rate:
-                    self._rate_calculate(_f_raw_done, _f_qc_done, _start_raw, _end_raw)
-
-                return _f_qc_done if self.qc else _f_raw_done
+            if self.append:
+                print(f"Appending new data from {_start} to {_end}")
+                _f_raw_new, _f_qc_new = self._read_raw_files()
+                _f_raw = self._append_process(_f_raw_done, _f_raw_new)
+                _f_qc = self._append_process(_f_qc_done, _f_qc_new)
+            else:
+                _f_raw, _f_qc = _f_raw_done, _f_qc_done
 
-        # read raw data
-        print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
+        else:
+            print(f"\n{dtm.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mRAW DATA\033[0m "
+                  f"from {_start} to {_end}\n")
+            _f_raw, _f_qc = self._read_raw_files()
 
-        _f_raw, _f_qc = self._read_raw_files()
-        if _f_raw is None:
-            return None
-
-        # append new data and pickle data
-        if self.apnd and self.pkl_nam.exists():
-            _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
-            _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
+        # process time index
+        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
+        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
 
-        _f_qc = self._outlier_prcs(_f_qc)
+        _f_qc = self._outlier_process(_f_qc)
 
         # save
         self._save_data(_f_raw, _f_qc)
 
-        # process time index
-        # if (_start is not None)|(_end is not None):
-        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
-        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Raw data time : {_start_raw} to {_end_raw}")
+        self.logger.info(f"Output   time : {_start} to {_end}")
+        self.logger.info(f"{'-' * 60}")
 
-        self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
+        if self.rate:
+            self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
 
         return _f_qc if self.qc else _f_raw

AeroViz/rawDataReader/script/AE33.py

@@ -6,11 +6,11 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'AE33'
 
-    def _raw_reader(self, _file):
-        if _file.stat().st_size / 1024 < 550:
+    def _raw_reader(self, file):
+        if file.stat().st_size / 1024 < 550:
             print('\t It may not be a whole daily data.')
 
-        _df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
+        _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
                          delimiter=r'\s+', skiprows=5, usecols=range(67))
         _df.columns = _df.columns.str.strip(';')
 

AeroViz/rawDataReader/script/AE43.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'AE43'
 
-    def _raw_reader(self, _file):
-        _df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
+    def _raw_reader(self, file):
+        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time')
         _df_id = _df['SetupID'].iloc[-1]
 
         # get last SetupID data