AeroViz 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl

AeroViz/dataProcess/Chemistry/_mass_volume.py

@@ -2,13 +2,14 @@ from pandas import concat, DataFrame
 
 
 def _basic(df_che, df_ref, df_water, df_density, nam_lst):
+
     df_all = concat(df_che, axis=1)
     index = df_all.index.copy()
     df_all.columns = nam_lst
 
     # parameter
     mol_A, mol_S, mol_N = df_all['NH4+'] / 18, df_all['SO42-'] / 96, df_all['NO3-'] / 62
-    df_all['status'] = (mol_A) / (2 * mol_S + mol_N)
+    df_all['status'] = mol_A / (2 * mol_S + mol_N)
 
     convert_nam = {'AS': 'SO42-',
                    'AN': 'NO3-',
@@ -92,7 +93,7 @@ def _basic(df_che, df_ref, df_water, df_density, nam_lst):
     df_mass['total'] = df_mass.sum(axis=1, min_count=6)
 
     qc_ratio = df_mass['total'] / df_ref
-    qc_cond = (qc_ratio >= 0.7) & (qc_ratio <= 1.3)
+    qc_cond = (qc_ratio >= 0.5) & (qc_ratio <= 1.5)
 
     # volume
     df_vol = DataFrame()
@@ -153,7 +154,7 @@ def _basic(df_che, df_ref, df_water, df_density, nam_lst):
     out.update(ri_dic)
 
     for _ky, _df in out.items():
-        out[_ky] = _df.reindex(index).where(qc_cond)
+        out[_ky] = _df.reindex(index)
 
     return out
 

AeroViz/dataProcess/Optical/_IMPROVE.py

@@ -1,8 +1,7 @@
-import pickle as pkl
 from pathlib import Path
 
 import numpy as np
-from pandas import DataFrame
+from pandas import DataFrame, read_pickle
 
 from AeroViz.dataProcess.core import union_index
 
@@ -12,7 +11,7 @@ def _revised(_df_mass, _df_RH):
 
     # fRH
     with (Path(__file__).parent / 'fRH.pkl').open('rb') as f:
-        _fRH = pkl.load(f)
+        _fRH = read_pickle(f)
         _fRH.loc[np.nan] = np.nan
 
     def fRH(_RH):

AeroViz/dataProcess/SizeDistr/__init__.py

@@ -1,15 +1,11 @@
 from ..core import Writer, run_process
 
-__all__ = [
-
-    'SizeDistr',
-
-]
+__all__ = ['SizeDistr']
 
 
 class SizeDistr(Writer):
 
-    ## basic
+    # basic
     @run_process('SizeDistr - basic', 'distr_basic')
     def basic(self, df, hybrid_bin_start_loc=None, unit='nm', bin_range=(0, 20000), input_type='norm'):
         from ._size_distr import _basic
@@ -18,7 +14,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v4', 'distr_merge')
     def merge_SMPS_APS_v4(self, df_smps, df_aps, df_pm25, aps_unit='um',
                           smps_overlap_lowbound=500, aps_fit_highbound=1000, dndsdv_alg=True,
@@ -30,7 +26,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v3', 'distr_merge')
     def merge_SMPS_APS_v3(self, df_smps, df_aps, aps_unit='um',
                           smps_overlap_lowbound=500, aps_fit_highbound=1000, dndsdv_alg=True):
@@ -40,7 +36,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v2', 'distr_merge')
     def merge_SMPS_APS_v2(self, df_smps, df_aps, aps_unit='um',
                           smps_overlap_lowbound=500, aps_fit_highbound=1000):
@@ -50,7 +46,7 @@ class SizeDistr(Writer):
 
         return self, out
 
-    ## merge
+    # merge
     @run_process('SizeDistr - merge_SMPS_APS_v1', 'distr_merge')
     def merge_SMPS_APS(self, df_smps, df_aps, aps_unit='um', shift_mode='mobility',
                        smps_overlap_lowbound=523, aps_fit_highbound=800):

AeroViz/rawDataReader/__init__.py

@@ -12,7 +12,7 @@ def RawDataReader(instrument_name: str,
                   qc: bool = True,
                   csv_raw: bool = True,
                   reset: bool = False,
-                  rate: bool = False,
+                  rate: bool = True,
                   append_data: bool = False,
                   start: datetime | None = None,
                   end: datetime | None = None,
@@ -72,9 +72,7 @@ def RawDataReader(instrument_name: str,
     instrument_class_map = {
         'NEPH': NEPH,
         'Aurora': Aurora,
-        'SMPS_genr': SMPS_genr,
-        'SMPS_aim11': SMPS_aim11,
-        'SMPS_TH': SMPS_TH,
+        'SMPS': SMPS,
         'GRIMM': GRIMM,
         'APS_3321': APS_3321,
         'AE33': AE33,
@@ -82,7 +80,7 @@ def RawDataReader(instrument_name: str,
         'BC1054': BC1054,
         'MA350': MA350,
         'TEOM': TEOM,
-        'Sunset_OCEC': Sunset_OCEC,
+        'OCEC': OCEC,
         'IGAC': IGAC,
         'VOC': VOC,
         'Table': Table,

AeroViz/rawDataReader/config/supported_instruments.py

@@ -2,76 +2,64 @@
 
 meta = {
     "NEPH": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "5min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
     "Aurora": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
-    "SMPS_TH": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_genr": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_aim11": {
-        "pattern": "*.csv",
+    "SMPS": {
+        "pattern": ["*.txt", "*.csv"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "GRIMM": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "APS_3321": {
-        "pattern": "*.TXT",
+        "pattern": ["*.txt"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "AE33": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE33*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE33*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "AE43": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE43*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE43*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "BC1054": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC9"]},
         "error_state": [1, 2, 4, 8, 16, 32, 65536],
     },
 
     "MA350": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC5"]},
     },
 
     "TEOM": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "6min",
         "deter_key": {
             "PM1.0 Mass Conc.": ["PM_Total"],
@@ -79,8 +67,8 @@ meta = {
         },
     },
 
-    "Sunset_OCEC": {
-        "pattern": "*LCRes.csv",
+    "OCEC": {
+        "pattern": ["*LCRes.csv"],
         "freq": "1h",
         "deter_key": {
             "Thermal OC": ["Thermal_OC"],
@@ -91,7 +79,7 @@ meta = {
     },
 
     "IGAC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": {
             "Na+": ["Na+"],
@@ -108,7 +96,7 @@ meta = {
     },
 
     "VOC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "key": [
             'Benzene', 'Toluene', 'Ethylbenzene', 'm/p-Xylene', 'o-Xylene', 'Ethane', 'Propane', 'Isobutane',
@@ -129,20 +117,31 @@ meta = {
     },
 
     "Table": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": None,
     },
 
     "EPA_vertical": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": None,
     },
 
     "Minion": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
-        "deter_key": None,
+        "deter_key": {
+            "Na+": ["Na+"],
+            "NH4+": ["NH4+"],
+            "K+": ["K+"],
+            "Mg2+": ["Mg2+"],
+            "Ca2+": ["Ca2+"],
+            "Cl-": ["Cl-"],
+            "NO2-": ["NO2-"],
+            "NO3-": ["NO3-"],
+            "SO42-": ["SO42-"],
+            "Main Salt (NH4+, NO3-, SO42-)": ["NO3-", "SO42-", "NH4+"],
+        },
     },
 }

AeroViz/rawDataReader/core/__init__.py

@@ -1,4 +1,4 @@
-import json as jsn
+import json
 import logging
 import pickle as pkl
 from abc import ABC, abstractmethod
@@ -9,28 +9,35 @@ from typing import Any
 import numpy as np
 import pandas as pd
 from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
+from rich.console import Console
+from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
 from ..config.supported_instruments import meta
 
 __all__ = ['AbstractReader']
 
 
+console = Console(force_terminal=True, color_system="auto")
+
+
 class AbstractReader(ABC):
-    nam = 'AbstractReader'
+    """
+    Abstract class for reading raw data from different instruments. Each instrument should have a separate class that
+    inherits from this class and implements the abstract methods. The abstract methods are `_raw_reader` and `_QC`.
 
-    # initial data
-    # input : file path, reset switch
+    List the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
+    pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
+    'reset=True'
+    """
 
-    # list the file in the path and read pickle file if it exists, else read raw data and dump the pickle file the
-    # pickle file will be generated after read raw data first time, if you want to re-read the rawdata, please set
-    # 'reset=True'
+    nam = 'AbstractReader'
 
     def __init__(self,
                  path: Path | str,
                  qc: bool = True,
                  csv_raw: bool = True,
                  reset: bool = False,
-                 rate: bool = False,
+                 rate: bool = True,
                  append_data: bool = False):
 
         self.path = Path(path)
@@ -41,7 +48,7 @@ class AbstractReader(ABC):
         self.rate = rate
         self.qc = qc
         self.csv = csv_raw
-        self.append = append_data & reset
+        self.append = append_data and reset
 
         self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
         self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
@@ -49,21 +56,12 @@ class AbstractReader(ABC):
         self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
         self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
 
-    # dependency injection function, customize each instrument
-    @abstractmethod
-    def _raw_reader(self, file):
-        pass
-
-    @abstractmethod
-    def _QC(self, df: DataFrame):
-        return df
-
     def __call__(self,
                  start: dtm | None = None,
                  end: dtm | None = None,
                  mean_freq: str = '1h',
                  csv_out: bool = True,
-                 ) -> DataFrame | None:
+                 ) -> DataFrame:
 
         if start and end and end <= start:
             raise ValueError(f"Invalid time range: start {start} is after end {end}")
@@ -78,6 +76,14 @@ class AbstractReader(ABC):
 
         return data
 
+    @abstractmethod
+    def _raw_reader(self, file):
+        pass
+
+    @abstractmethod
+    def _QC(self, df: DataFrame):
+        return df
+
     @staticmethod
     def basic_QC(df: DataFrame):
         df_ave, df_std = df.mean(), df.std()
@@ -85,40 +91,25 @@ class AbstractReader(ABC):
 
         return df.mask(df_lowb | df_highb).copy()
 
-    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
-    def _raw_process(self, _df):
-        # get time from df and set time to whole time to create time index
-        _st, _ed = _df.index.sort_values()[[0, -1]]
-        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
-                               freq=self.meta['freq'])
-        _tm_index.name = 'time'
-
-        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
-
     def _setup_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.nam)
         logger.setLevel(logging.INFO)
+
+        for handler in logger.handlers[:]:
+            logger.removeHandler(handler)
+
         handler = logging.FileHandler(self.path / f'{self.nam}.log')
         handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
         logger.addHandler(handler)
         return logger
 
-    # acquisition rate and yield rate
-    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw):
+    def _rate_calculate(self, _fout_raw, _fout_qc, _st_raw, _ed_raw) -> None:
         if self.meta['deter_key'] is not None:
             _start, _end = _fout_qc.index[[0, -1]]
 
             _drop_how = 'any'
             _the_size = len(_fout_raw.resample('1h').mean().index)
 
-            self.logger.info(f"{'=' * 60}")
-            self.logger.info(
-                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} to {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(
-                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}")
-            self.logger.info(f"{'-' * 60}")
-            print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
-
             for _nam, _key in self.meta['deter_key'].items():
                 if _key == ['all']:
                     _key, _drop_how = _fout_qc.keys(), 'all'
@@ -137,9 +128,19 @@ class AbstractReader(ABC):
                 self.logger.info(f'\tYield       rate: {_yid_rate}%')
                 self.logger.info(f"{'=' * 60}")
 
-                print(f'\t\t{_nam} : ')
-                print(f'\t\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
-                print(f'\t\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+                print(f'\n\t{_nam} : ')
+                print(f'\t\tacquisition rate : \033[91m{_acq_rate}%\033[0m')
+                print(f'\t\tyield       rate : \033[91m{_yid_rate}%\033[0m')
+
+    # set each to true datetime(18:30:01 -> 18:30:00) and rindex data
+    def _raw_process(self, _df):
+        # get time from df and set time to whole time to create time index
+        _st, _ed = _df.index.sort_values()[[0, -1]]
+        _tm_index = date_range(_st.strftime('%Y%m%d %H00'), _ed.floor('h').strftime('%Y%m%d %H00'),
+                               freq=self.meta['freq'])
+        _tm_index.name = 'time'
+
+        return _df.apply(to_numeric, errors='coerce').resample(self.meta['freq']).mean().reindex(_tm_index)
 
     # process time index
     @staticmethod
@@ -153,7 +154,7 @@ class AbstractReader(ABC):
 
     # append new data to exist pkl
     @staticmethod
-    def _apnd_prcs(_df_done, _df_apnd):
+    def _append_process(_df_done, _df_apnd):
 
         if _df_apnd is not None:
             _df = concat([_df_apnd.dropna(how='all').copy(), _df_done.dropna(how='all').copy()])
@@ -165,16 +166,16 @@ class AbstractReader(ABC):
 
         return _df_done
 
-    # remove outlier
-    def _outlier_prcs(self, _df):
+    def _outlier_process(self, _df):
+        outlier_file = self.path / 'outlier.json'
 
-        if (self.path / 'outlier.json') not in self.path.glob('*.json'):
+        if not outlier_file.exists():
             return _df
 
-        with (self.path / 'outlier.json').open('r', encoding='utf-8', errors='ignore') as f:
-            self.outlier = jsn.load(f)
+        with outlier_file.open('r', encoding='utf-8', errors='ignore') as f:
+            outliers = json.load(f)
 
-        for _st, _ed in self.outlier.values():
+        for _st, _ed in outliers.values():
             _df.loc[_st:_ed] = np.nan
 
         return _df
@@ -192,14 +193,13 @@ class AbstractReader(ABC):
 
     @staticmethod
     def _safe_pickle_dump(file_path: Path, data: Any) -> None:
-        while True:
-            try:
-                with file_path.open('wb') as f:
-                    pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
-                break
-            except PermissionError as err:
-                print('\n', err)
-                input('\t\t\33[41m Please close the file and press "Enter" \33[0m\n')
+        try:
+            with file_path.open('wb') as f:
+                pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL)
+        except PermissionError as e:
+            raise IOError(f"Unable to write to {file_path}. The file may be in use or you may not have permission: {e}")
+        except Exception as e:
+            raise IOError(f"Error writing to {file_path}: {e}")
 
     # read pickle file
     def _read_pkl(self):
@@ -207,31 +207,41 @@ class AbstractReader(ABC):
             return pkl.load(raw_data), pkl.load(qc_data)
 
     def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
-        patterns = {self.meta['pattern'].lower(), self.meta['pattern'].upper(), self.meta['pattern']}
-        files = [f for pattern in patterns for f in self.path.glob(pattern)
+        files = [f
+                 for file_pattern in self.meta['pattern']
+                 for pattern in {file_pattern.lower(), file_pattern.upper(), file_pattern}
+                 for f in self.path.glob(pattern)
                  if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
         if not files:
-            raise FileNotFoundError(f"\t\t\033[31mNo files in '{self.path}' could be read."
-                                    f"Please check the current path.\033[0m")
+            raise FileNotFoundError(f"No files in '{self.path}' could be read. Please check the current path.")
 
         df_list = []
-        for file in files:
-            print(f"\r\t\treading {file.name}", end='')
-
-            try:
-                df = self._raw_reader(file)
+        with Progress(
+                TextColumn("[bold blue]{task.description}", style="bold blue"),
+                BarColumn(bar_width=18, complete_style="green", finished_style="bright_green"),
+                TaskProgressColumn(),
+                TimeRemainingColumn(),
+                TextColumn("{task.fields[filename]}", style="yellow"),
+                console=console,
+                expand=False
+        ) as progress:
+            task = progress.add_task(f"Reading {self.nam} files", total=len(files), filename="")
+            for file in files:
+                progress.update(task, advance=1, filename=file.name)
+                try:
+                    df = self._raw_reader(file)
 
-                if df is not None and not df.empty:
-                    df_list.append(df)
-                else:
-                    self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
+                    if df is not None and not df.empty:
+                        df_list.append(df)
+                    else:
+                        self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
 
-            except pd.errors.ParserError as e:
-                self.logger.error(f"Error tokenizing data: {e}")
+                except pd.errors.ParserError as e:
+                    self.logger.error(f"Error tokenizing data: {e}")
 
-            except Exception as e:
-                self.logger.error(f"Error reading {file.name}: {e}")
+                except Exception as e:
+                    self.logger.error(f"Error reading {file.name}: {e}")
 
         if not df_list:
             raise ValueError("All files were either empty or failed to read.")
@@ -241,47 +251,42 @@ class AbstractReader(ABC):
 
         return raw_data, qc_data
 
-    # main flow
     def _run(self, _start, _end):
-        _f_raw_done, _f_qc_done = None, None
-
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.append):
-            print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and not self.reset:
+            print(f"\n{dtm.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mPICKLE\033[0m "
+                  f"from {_start} to {_end}\n")
 
             _f_raw_done, _f_qc_done = self._read_pkl()
 
-            if not self.append:
-                _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
-                _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
-
-                _f_qc_done = self._outlier_prcs(_f_qc_done)
-
-                if self.rate:
-                    self._rate_calculate(_f_raw_done, _f_qc_done, _start_raw, _end_raw)
-
-                return _f_qc_done if self.qc else _f_raw_done
+            if self.append:
+                print(f"Appending new data from {_start} to {_end}")
+                _f_raw_new, _f_qc_new = self._read_raw_files()
+                _f_raw = self._append_process(_f_raw_done, _f_raw_new)
+                _f_qc = self._append_process(_f_qc_done, _f_qc_new)
+            else:
+                _f_raw, _f_qc = _f_raw_done, _f_qc_done
 
-        # read raw data
-        print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
+        else:
+            print(f"\n{dtm.now().strftime('%m/%d %X')} : Reading {self.nam} \033[96mRAW DATA\033[0m "
+                  f"from {_start} to {_end}\n")
+            _f_raw, _f_qc = self._read_raw_files()
 
-        _f_raw, _f_qc = self._read_raw_files()
-
-        # append new data and pickle data
-        if self.append and self.pkl_nam.exists():
-            _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
-            _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
+        # process time index
+        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
+        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
 
-        _f_qc = self._outlier_prcs(_f_qc)
+        _f_qc = self._outlier_process(_f_qc)
 
         # save
         self._save_data(_f_raw, _f_qc)
 
-        # process time index
-        # if (_start is not None)|(_end is not None):
-        _f_raw, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw)
-        _f_qc, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc)
+        self.logger.info(f"{'=' * 60}")
+        self.logger.info(f"Raw data time : {_start_raw} to {_end_raw}")
+        self.logger.info(f"Output   time : {_start} to {_end}")
+        self.logger.info(f"{'-' * 60}")
 
-        self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
+        if self.rate:
+            self._rate_calculate(_f_raw, _f_qc, _start_raw, _end_raw)
 
         return _f_qc if self.qc else _f_raw

AeroViz/rawDataReader/script/APS_3321.py

@@ -1,4 +1,4 @@
-import numpy as n
+import numpy as np
 from pandas import to_datetime, read_table
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -29,7 +29,7 @@ class Reader(AbstractReader):
     # QC data
     def _QC(self, _df):
         # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
+        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
         _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
         _df = _df.mask(_df_size < 7)
 

AeroViz/rawDataReader/script/{Sunset_OCEC.py → OCEC.py}

@@ -4,7 +4,7 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-    nam = 'Sunset_OCEC'
+    nam = 'OCEC'
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:

AeroViz/rawDataReader/script/SMPS.py

@@ -0,0 +1,76 @@
+import csv
+
+import numpy as np
+from pandas import to_datetime, to_numeric, read_csv, isna
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+def find_header_row(file_obj, delimiter):
+    csv_reader = csv.reader(file_obj, delimiter=delimiter)
+    for skip, row in enumerate(csv_reader):
+        if row and (row[0] in ['Sample #', 'Scan Number']):
+            return skip
+    raise ValueError("Header row not found")
+
+
+def parse_date(df, date_format):
+    if 'Date' in df.columns and 'Start Time' in df.columns:
+        return to_datetime(df['Date'] + ' ' + df['Start Time'], format=date_format, errors='coerce')
+    elif 'DateTime Sample Start' in df.columns:
+        return to_datetime(df['DateTime Sample Start'], format=date_format, errors='coerce')
+    else:
+        raise ValueError("Expected date columns not found")
+
+
+class Reader(AbstractReader):
+    nam = 'SMPS'
+
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            if file.suffix.lower() == '.txt':
+                delimiter, date_formats = '\t', ['%m/%d/%y %X', '%m/%d/%Y %X']
+            else:  # csv
+                delimiter, date_formats = ',', ['%d/%m/%Y %X']
+
+            skip = find_header_row(f, delimiter)
+            f.seek(0)
+
+            _df = read_csv(f, sep=delimiter, skiprows=skip)
+
+            for date_format in date_formats:
+                _time_index = parse_date(_df, date_format)
+                if not isna(_time_index).all():
+                    break
+            else:
+                raise ValueError("Unable to parse dates with given formats")
+
+            # sequence the data
+            numeric_cols = [col for col in _df.columns if col.strip().replace('.', '').isdigit()]
+            numeric_cols.sort(key=lambda x: float(x.strip()))
+
+            _df.index = _time_index
+            _df.index.name = 'time'
+
+            _df_smps = _df[numeric_cols]
+            _df_smps.columns = _df_smps.columns.astype(float)
+
+            return _df_smps.apply(to_numeric, errors='coerce')
+
+    # QC data
+    def _QC(self, _df):
+
+        # mask out the data size lower than 7
+        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
+        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
+        _df = _df.mask(_df_size < 7)
+
+        # remove total conc. lower than 2000
+        _df = _df.mask(_df['total'] < 2000)
+
+        # remove the bin over 400 nm which num. conc. larger than 4000
+        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
+
+        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
+
+        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/VOC.py

@@ -20,8 +20,8 @@ class Reader(AbstractReader):
 
             if invalid_keys:
                 self.logger.warning(f'{invalid_keys} are not supported keys.')
-                print(f'\n\t\t{invalid_keys} are not supported keys.'
-                      f'\n\t\tPlease check the\033[91m support_voc.md\033[0m file to use the correct name.')
+                print(f'\n\t{invalid_keys} are not supported keys.'
+                      f'\n\tPlease check the\033[91m support_voc.md\033[0m file to use the correct name.')
 
             if valid_keys:
                 return _df[valid_keys].loc[~_df.index.duplicated() & _df.index.notna()]

AeroViz/rawDataReader/script/__init__.py

@@ -1,9 +1,7 @@
 __all__ = [
     'NEPH',
     'Aurora',
-    'SMPS_TH',
-    'SMPS_genr',
-    'SMPS_aim11',
+    'SMPS',
     'APS_3321',
     'GRIMM',
     'AE33',
@@ -11,7 +9,7 @@ __all__ = [
     'BC1054',
     'MA350',
     'TEOM',
-    'Sunset_OCEC',
+    'OCEC',
     'IGAC',
     'VOC',
     'Table',

{AeroViz-0.1.5.dist-info → AeroViz-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.5
+Version: 0.1.6
 Summary: Aerosol science
 Home-page: https://github.com/Alex870521/AeroViz
 Author: alex
@@ -11,14 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pandas
-Requires-Dist: numpy
-Requires-Dist: matplotlib
-Requires-Dist: seaborn
-Requires-Dist: scipy
-Requires-Dist: scikit-learn
-Requires-Dist: windrose
-Requires-Dist: tabulate
+Requires-Dist: pandas ==2.2.2
+Requires-Dist: numpy ==1.26.4
+Requires-Dist: matplotlib ==3.8.4
+Requires-Dist: scipy ==1.14.0
+Requires-Dist: seaborn ==0.13.2
+Requires-Dist: scikit-learn ==1.5.1
+Requires-Dist: windrose ==1.9.2
+Requires-Dist: tabulate ==0.9.0
+Requires-Dist: rich ~=13.7.1
 
 ## <div align="center">AeroViz for Aerosol Science Visualization</div>
 
@@ -52,13 +53,15 @@ Requires-Dist: tabulate
 ## <div align="center">Installation</div>
 
 ```bash
-pip install AeroViz # ensure the python version is >= 3.12
+pip install AeroViz
 ```
 
 ## <div align="center">Usage</div>
 
 ```python
 import AeroViz
+
+from AeroViz import RawDataReader, DataProcess, plot
 ```
 
 ## <div align="center">RawDataReader Supported Instruments</div>

{AeroViz-0.1.5.dist-info → AeroViz-0.1.6.dist-info}/RECORD

@@ -5,14 +5,14 @@ AeroViz/dataProcess/__init__.py,sha256=D3rTVUiGfs_daGuaotVtbijOgLAp6HaRWchj-zoEn
 AeroViz/dataProcess/Chemistry/__init__.py,sha256=fyyomjxkQcUNWDx4R5jPrHafAftN-v2liUZii9OlaiU,2058
 AeroViz/dataProcess/Chemistry/_calculate.py,sha256=q7ojTFPok0vg8k_1PMECNdP5CPanR9NWQ4Rx5iTcHew,599
 AeroViz/dataProcess/Chemistry/_isoropia.py,sha256=3wp_FXdN230awlStMbctutwld4oot9WaAVXETGd6PSs,3255
-AeroViz/dataProcess/Chemistry/_mass_volume.py,sha256=oqFjRDYZ1om0_kG4g8XNi2QpUPcHsukpnWKydQoNEVg,5394
+AeroViz/dataProcess/Chemistry/_mass_volume.py,sha256=0joH2BAx0NUwDFzyrLgG-v7WrGl46R7zWxwbajWBV8o,5378
 AeroViz/dataProcess/Chemistry/_ocec.py,sha256=gqDTs9rOyr0RXdN1Nrxv6Vgvh04IFgHwk4gAJUgZZGc,5437
 AeroViz/dataProcess/Chemistry/_partition.py,sha256=tKhb6BJns46UiUlEq6Zq7ahYnvUJ_whY3tWE54C3bqU,1023
 AeroViz/dataProcess/Chemistry/_teom.py,sha256=IiM-TrifWpQLTbKllG-4k4c3mvQulfcmjswWu6muCXA,486
 AeroViz/dataProcess/Chemistry/isrpia.cnf,sha256=iWXTqsOZFmNrJxAI9nYuilZ9h6ru1icdPFVim7YKc_k,566
 AeroViz/dataProcess/Chemistry/isrpia2.exe,sha256=xFYPTXHSjl_-atmVcTmqqOkTZgUn1p8h-kFMcoGMIQw,1560032
 AeroViz/dataProcess/Optical/Angstrom_exponent.py,sha256=KSGM_hf2oUPsLoLuhYSRDK5zdekwce9KgqhzRnf2Y4g,706
-AeroViz/dataProcess/Optical/_IMPROVE.py,sha256=_u0uyKC9ZQpB4uLMx3x8Dm5hSUMcdGUifBcC2bGaM9c,1675
+AeroViz/dataProcess/Optical/_IMPROVE.py,sha256=n3uHUQTivckIrNlbE8rP7lPZGcMWZUAnS_qX8pXd55Q,1670
 AeroViz/dataProcess/Optical/__init__.py,sha256=h_IYhaum0DhK7Qw6lWFvjAEv7uAqLVVPEwU0V4hJnX4,1371
 AeroViz/dataProcess/Optical/_absorption.py,sha256=pYHIoj_R23wKvxR4acUqqU9JXKY40wgsFvZH78rETrY,1234
 AeroViz/dataProcess/Optical/_extinction.py,sha256=anUemSXmLvJi2CSxqTYgQCgUW4pSUqgZ0q0xZ4zs0zY,1189
@@ -20,7 +20,7 @@ AeroViz/dataProcess/Optical/_mie.py,sha256=K_I8NqRStCdExRVxq2rX5hQdvYmfKuu9JCkNt
 AeroViz/dataProcess/Optical/_mie_sd.py,sha256=cbWSorfG-l92csaXsZJ7005f8ij2H_VZopIhNPCwx70,4812
 AeroViz/dataProcess/Optical/_scattering.py,sha256=w5xXXOfafV9a_dXY60YqTyzFDZppjBJeLcKl5PeCQe4,961
 AeroViz/dataProcess/Optical/fRH.pkl,sha256=PhJ-BYTaUDjl2Lp5Wz0zeiLQCzrsNdq8nP7Jd1BXyXE,4528
-AeroViz/dataProcess/SizeDistr/__init__.py,sha256=PJVAdb-v1q8h38JXIkpVcr6q4mg38V26sk4xD_2Mjlc,2186
+AeroViz/dataProcess/SizeDistr/__init__.py,sha256=fka1IJW4wZzYWE9HjWi8i03ab0JLFXOUdaHWYs4VHEQ,2172
 AeroViz/dataProcess/SizeDistr/__merge.py,sha256=A-ZmduUA9a0nVM8szhG1AWhsfUPu2UhGIaqFLcmg7Lk,9422
 AeroViz/dataProcess/SizeDistr/_merge.py,sha256=CyZwUc2M4lL-OE74e8mXq41nC6X0LJTAV8TwpO3fS2o,9176
 AeroViz/dataProcess/SizeDistr/_merge_v1.py,sha256=6Anb8DszoatK66tc9ccA6ZApbqtL7pZRPR5TKHj-Tbs,9526
@@ -79,13 +79,13 @@ AeroViz/process/script/PSD.py,sha256=unSvyHp1RqgQecY-Lab-wRFGP2u45blSO4NCNTaxcRQ
 AeroViz/process/script/PSD_dry.py,sha256=P9rSQVA78jHXrY2bDiC1iQvG0w9Js8QiBNfooPOXzfY,3269
 AeroViz/process/script/__init__.py,sha256=QExUHHK79oiwZXvk8u00P8rLwWhGOc1fpZY2Fx8Vkxg,188
 AeroViz/process/script/retrieve_RI.py,sha256=9OQJU1w6VzVqKz00e9GdWswRAnaETe_1meAHp5N_s54,2711
-AeroViz/rawDataReader/__init__.py,sha256=kouHFAxpbl1Xj4IBHqzNaUExuxi0vF6E59l8Oryktq8,3949
+AeroViz/rawDataReader/__init__.py,sha256=jXLrf0A-Hxufw-IQp0-Nyci4PszWk5YU4SnpH4ob1RE,3862
 AeroViz/rawDataReader/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-AeroViz/rawDataReader/config/supported_instruments.py,sha256=6pkjRqe4Kz5023N_C3OQhX3u9PtfCpeUWhIA25Y9T7Q,4273
-AeroViz/rawDataReader/core/__init__.py,sha256=oBxig1HXb2QmWAdPwEKXx8L_85LPX8CPbBc0eYR9C5A,10744
+AeroViz/rawDataReader/config/supported_instruments.py,sha256=iE2dqnAhXsHUeVfho5vYYcXL97fg_xt3VvvYcyxCU48,4408
+AeroViz/rawDataReader/core/__init__.py,sha256=W_RwJZmMCTjTwWiTSxcXFhkasO3cyQWEPSdA7-ymeqU,11058
 AeroViz/rawDataReader/script/AE33.py,sha256=Nv0u0w_V50sDjsLh17ZkNT6A75TjIo2A1O6GYc4zkJg,1107
 AeroViz/rawDataReader/script/AE43.py,sha256=Qq9MdiBKzDuKIEFPanmn7wUbkN5VeSTG6Dopw8i0W90,1056
-AeroViz/rawDataReader/script/APS_3321.py,sha256=bzE-GaElXU16xmOgYr6tR62GAtgRZxYqRrifbBRuU-0,1694
+AeroViz/rawDataReader/script/APS_3321.py,sha256=x75G72Xl0vElr6Njbv8SlOcosAHNozseaJzAxVmfXyI,1697
 AeroViz/rawDataReader/script/Aurora.py,sha256=b0XPs2gc_2UJj4HnOHwW7o255zsCPp459goaDqeAJA8,1387
 AeroViz/rawDataReader/script/BC1054.py,sha256=QGU1SRDoE8e4yNXUX7RsdK5zh72fNd2Q0IpUy7iFzrg,1486
 AeroViz/rawDataReader/script/EPA_vertical.py,sha256=MpcbOzbigivsuldCxzn_B6POtQMzahGQhM_sWRDVP9s,1669
@@ -94,21 +94,19 @@ AeroViz/rawDataReader/script/IGAC.py,sha256=BDhmsNy02EvLY0sd75NM6XvLGVTEJlxyQQbp
 AeroViz/rawDataReader/script/MA350.py,sha256=YRY19R2V6nyHMpGMOLVZvK8_41OCCzdTcSnuZR-OJsU,1287
 AeroViz/rawDataReader/script/Minion.py,sha256=c128ZkkxakV6wq1k9NtEq0scdi1mh9Mn0jhQFqz_uVE,3940
 AeroViz/rawDataReader/script/NEPH.py,sha256=OSDtbJ4sujtnf3bzbENXoyJwA-RIeOkPXNCEJ6MvkPQ,2735
-AeroViz/rawDataReader/script/SMPS_TH.py,sha256=4bK8Fm4oyPZd3GegcOttjamZf2ZqcL_uaKUXdNtv8_0,1468
-AeroViz/rawDataReader/script/SMPS_aim11.py,sha256=lop0YmOOWde9vBKMXV3nqwqZOxR-OCbYZznSf4QWuGs,1632
-AeroViz/rawDataReader/script/SMPS_genr.py,sha256=rR5OO63ozDKMcGBfNFPi_f49gUbx2qYx60GMSgC9jwg,1609
-AeroViz/rawDataReader/script/Sunset_OCEC.py,sha256=6TyosCQ4N6jB9yZOfAJHwiIvkTYCwCThmMbhe4_U9a0,2322
+AeroViz/rawDataReader/script/OCEC.py,sha256=t3N_bs5RishlfzFqxTSfMlemZBDIco6cKZ4mU0j9_6g,2315
+AeroViz/rawDataReader/script/SMPS.py,sha256=RnqysTWNaFautGnyPKZgMjdtPdauv9Mqu9IpuTEGpXs,2706
 AeroViz/rawDataReader/script/TEOM.py,sha256=Hx_DDnvGSfSl9cgIVmuf4VmphaFfNCY60mqPWoLmNvw,1796
 AeroViz/rawDataReader/script/Table.py,sha256=othBoQSFVcjfWX0_Q4y8BkjCBc5CihvC6LyrhyJWywk,800
-AeroViz/rawDataReader/script/VOC.py,sha256=JaMDojHrLJL37LdcNzI7xEGy_592feokJw9kedcwL20,1273
-AeroViz/rawDataReader/script/__init__.py,sha256=UndjtWjvJHMma7WtArL5sF9UHiQsanx5sT2otQDqzng,270
+AeroViz/rawDataReader/script/VOC.py,sha256=sUn3ba_iYQ8Uy8qKNI-6qb1YvHsKN4G6BosBnLLMctc,1269
+AeroViz/rawDataReader/script/__init__.py,sha256=EXLjVwKN-wr3pMi-ILjiPH86ojwfoIif5vG2Q__DGZc,225
 AeroViz/tools/__init__.py,sha256=tPUmCI9Fi1LgE-QSVnzan5jRdYQBIENH9PRDXB_DhTw,109
 AeroViz/tools/database.py,sha256=05VzjJyhlRrhsZdhfFQ__7CxGm4MdFekLjz3_Is5h9U,3430
 AeroViz/tools/dataclassifier.py,sha256=_wpv0PlZ5EGkcNqHxfFtdEsYvHP5FVE8sMZXikhm_YE,4492
 AeroViz/tools/dataprinter.py,sha256=Jq2Yztpa9YCOeLDVTrRs7PhSdNIPhEAexVj1YSuJ7hY,2249
 AeroViz/tools/datareader.py,sha256=iTQ0U8hdNMjCdbiH7EiKW10UEoxzxXRHc4s5_1IikJo,1933
-AeroViz-0.1.5.dist-info/LICENSE,sha256=E-679GpGGkp3irmtuJXiT7R4cNUA4cmsH6Q7QUgPf5U,1069
-AeroViz-0.1.5.dist-info/METADATA,sha256=dWKrlnoRNBUgRf5R1VvNVcurimM8slO9RMiSeEw6Stk,5323
-AeroViz-0.1.5.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-AeroViz-0.1.5.dist-info/top_level.txt,sha256=BYsmTst_o4FZOKRP1XIvIMlN6mMTTXNfnSToL2_nVbQ,8
-AeroViz-0.1.5.dist-info/RECORD,,
+AeroViz-0.1.6.dist-info/LICENSE,sha256=E-679GpGGkp3irmtuJXiT7R4cNUA4cmsH6Q7QUgPf5U,1069
+AeroViz-0.1.6.dist-info/METADATA,sha256=Fp-P1AKOfLLX0c46uZms4TfEQ5ImonxFHThq5oHFs4c,5434
+AeroViz-0.1.6.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+AeroViz-0.1.6.dist-info/top_level.txt,sha256=BYsmTst_o4FZOKRP1XIvIMlN6mMTTXNfnSToL2_nVbQ,8
+AeroViz-0.1.6.dist-info/RECORD,,

AeroViz/rawDataReader/script/SMPS_TH.py

@@ -1,41 +0,0 @@
-from pandas import to_datetime, read_table
-
-from AeroViz.rawDataReader.core import AbstractReader
-
-
-class Reader(AbstractReader):
-    nam = 'SMPS_TH'
-
-    def _raw_reader(self, file):
-        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_table(f, skiprows=18, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
-            _key = list(_df.keys()[6:-26])
-
-            _newkey = {}
-            for _k in _key:
-                _newkey[_k] = float(_k).__round__(4)
-
-            # _newkey['Total Conc.(#/cm)'] = 'total'
-            # _newkey['Mode(nm)']	= 'mode'
-
-            _df_idx = to_datetime(_df.index, errors='coerce')
-        return _df[_newkey.keys()].rename(_newkey, axis=1).set_index(_df_idx).loc[_df_idx.dropna()]
-
-    # QC data
-    def _QC(self, _df):
-        import numpy as n
-
-        # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-        _df = _df.mask(_df_size < 7)
-
-        # remove total conc. lower than 2000
-        _df = _df.mask(_df['total'] < 2000)
-
-        # remove the bin over 400 nm which num. conc. larger than 4000
-        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
-
-        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
-
-        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/SMPS_aim11.py

@@ -1,51 +0,0 @@
-from pandas import to_datetime, read_csv, to_numeric
-
-from AeroViz.rawDataReader.core import AbstractReader
-
-
-class Reader(AbstractReader):
-    nam = 'SMPS_aim11'
-
-    def _raw_reader(self, file):
-        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-
-            skiprows = 0
-            for _line in f:
-
-                if _line.split(',')[0] == 'Scan Number':
-                    f.seek(0)
-                    break
-
-                skiprows += 1
-            # breakpoint()
-            _df = read_csv(f, skiprows=skiprows)
-            _tm_idx = to_datetime(_df['DateTime Sample Start'], format='%d/%m/%Y %X', errors='coerce')
-
-            # index
-            _df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
-
-            # keys
-            _key = to_numeric(_df.keys(), errors='coerce')
-            _df.columns = _key
-            _df = _df.loc[:, ~_key.isna()]
-
-        return _df.apply(to_numeric, errors='coerce')
-
-    # QC data
-    def _QC(self, _df):
-        import numpy as n
-
-        # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-        _df = _df.mask(_df_size < 7)
-
-        # remove total conc. lower than 2000
-        _df = _df.mask(_df['total'] < 2000)
-
-        # remove the bin over 400 nm which num. conc. larger than 4000
-        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
-
-        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
-
-        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/SMPS_genr.py

@@ -1,51 +0,0 @@
-from pandas import to_datetime, read_table, to_numeric
-
-from AeroViz.rawDataReader.core import AbstractReader
-
-
-class Reader(AbstractReader):
-    nam = 'SMPS_genr'
-
-    def _raw_reader(self, file):
-        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-
-            skiprows = 0
-            for _line in f:
-
-                if _line.split('\t')[0] == 'Sample #':
-                    f.seek(0)
-                    break
-
-                skiprows += 1
-
-            _df = read_table(f, skiprows=skiprows)
-            _tm_idx = to_datetime(_df['Date'] + _df['Start Time'], format='%m/%d/%y%X', errors='coerce')
-
-            # index
-            _df = _df.set_index(_tm_idx).loc[_tm_idx.dropna()]
-
-            # keys
-            _key = to_numeric(_df.keys(), errors='coerce')
-            _df.columns = _key
-            _df = _df.loc[:, ~_key.isna()]
-
-        return _df.apply(to_numeric, errors='coerce')
-
-    # QC data
-    def _QC(self, _df):
-        import numpy as n
-
-        # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
-        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-        _df = _df.mask(_df_size < 7)
-
-        # remove total conc. lower than 2000
-        _df = _df.mask(_df['total'] < 2000)
-
-        # remove the bin over 400 nm which num. conc. larger than 4000
-        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
-
-        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
-
-        return _df[_df.keys()[:-1]]