AeroViz 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

AeroViz/rawDataReader/__init__.py

@@ -19,6 +19,55 @@ def RawDataReader(instrument_name: str,
                   mean_freq='1h',
                   csv_out=True,
                   ):
+    """
+    Factory function to instantiate the appropriate reader module for a given instrument and
+    return the processed data over the specified time range.
+
+    Parameters
+    ----------
+    instrument_name : str
+        The name of the instrument for which to read data. Must be a valid key in the `meta` dictionary.
+    path : Path
+        The directory where raw data files for the instrument are stored.
+    qc : bool, optional (default=True)
+        If True, apply quality control (QC) to the raw data.
+    csv_raw : bool, optional (default=True)
+        If True, read raw data from CSV files.
+    reset : bool, optional (default=False)
+        If True, reset the state and reprocess the data from scratch.
+    rate : bool, optional (default=False)
+        If True, calculate rates from the data.
+    append_data : bool, optional (default=False)
+        If True, append new data to the existing dataset instead of overwriting it.
+    start : datetime, optional
+        Start time for filtering the data. If None, no start time filtering will be applied.
+    end : datetime, optional
+        End time for filtering the data. If None, no end time filtering will be applied.
+    mean_freq : str, optional (default='1h')
+        Resampling frequency for averaging the data. Example: '1h' for hourly mean.
+    csv_out : bool, optional (default=True)
+        If True, output the processed data as a CSV file.
+
+    Return
+    ------
+    reader_module : Reader
+        An instance of the reader module corresponding to the specified instrument, which processes
+        the data and returns it in a usable format.
+
+    Raises
+    ------
+    ValueError
+        If the `instrument_name` provided is not a valid key in the `meta` dictionary.
+
+    Example
+    -------
+    To read and process data for the BC1054 instrument:
+
+    >>> from pathlib import Path
+    >>> from datetime import datetime
+    >>> data = RawDataReader(instrument_name='BC1054', path=Path('/path/to/data'),
+    >>>                      start=datetime(2024, 1, 1), end=datetime(2024, 2, 1))
+    """
     # Mapping of instrument names to their respective classes
     instrument_class_map = {
         'NEPH': NEPH,

AeroViz/rawDataReader/config/supported_instruments.py

@@ -83,10 +83,8 @@ meta = {
         "pattern": "*LCRes.csv",
         "freq": "1h",
         "deter_key": {
-            "Thermal OC/EC": ["Thermal_EC", "Thermal_OC"],
             "Thermal OC": ["Thermal_OC"],
             "Thermal EC": ["Thermal_EC"],
-            "Optical OC/EC": ["Optical_EC", "Optical_OC"],
             "Optical OC": ["Optical_OC"],
             "Optical EC": ["Optical_EC"],
         },
@@ -112,26 +110,21 @@ meta = {
     "VOC": {
         "pattern": "*.csv",
         "freq": "1h",
-        "key": ['Ethane', 'Propane', 'Isobutane', 'n-Butane', 'Cyclopentane', 'Isopentane',
-                'n-Pentane', '2,2-Dimethylbutane', '2,3-Dimethylbutane', '2-Methylpentane',
-                '3-Methylpentane', 'n-Hexane', 'Methylcyclopentane', '2,4-Dimethylpentane',
-                'Cyclohexane', '2-Methylhexane', '2-Methylhexane', '3-Methylheptane',
-                '2,2,4-Trimethylpentane', 'n-Heptane', 'Methylcyclohexane',
-                '2,3,4-Trimethylpentane', '2-Methylheptane', '3-Methylhexane', 'n-Octane',
-                'n-Nonane', 'n-Decane', 'n-Undecane', 'Ethylene', 'Propylene', 't-2-Butene',
-                '1-Butene', 'cis-2-Butene', 't-2-Pentene', '1-Pentene', 'cis-2-Pentene',
-                'isoprene', 'Acetylene', 'Benzene', 'Toluene', 'Ethylbenzene', 'm,p-Xylene',
-                'Styrene', 'o-Xylene', 'Isopropylbenzene', 'n-Propylbenzene', 'm-Ethyltoluene',
-                'p-Ethyltoluene', '1,3,5-Trimethylbenzene', 'o-Ethyltoluene',
-                '1,2,4-Trimethylbenzene', '1,2,3-Trimethylbenzene', 'm-Diethylbenzene',
-                'p-Diethylbenzene'],
-
-        "key_2": ['Isopentane', 'Hexane', '2-Methylhexane', '3-Methylhexane', '2-Methylheptane', '3-Methylheptane',
-                  'Propene', '1.3-Butadiene', 'Isoprene', '1-Octene',
-                  'Benzene', 'Toluene', 'Ethylbenzene', 'm.p-Xylene', 'o-Xylene', 'Iso-Propylbenzene', 'Styrene',
-                  'n-Propylbenzene', '3.4-Ethyltoluene', '1.3.5-TMB', '2-Ethyltoluene', '1.2.4-TMB', '1.2.3-TMB',
-                  'Acetaldehyde', 'Ethanol', 'Acetone', 'IPA', 'Ethyl Acetate', 'Butyl Acetate',
-                  'VCM', 'TCE', 'PCE', '1.4-DCB', '1.2-DCB'],
+        "key": [
+            'Benzene', 'Toluene', 'Ethylbenzene', 'm/p-Xylene', 'o-Xylene', 'Ethane', 'Propane', 'Isobutane',
+            'n-Butane', 'Isopentane', 'n-Pentane', 'n-Hexane', 'n-Heptane', 'n-Octane', 'n-Nonane', 'n-Decane',
+            'n-Undecane', 'n-Dodecane', 'Ethylene', 'Propylene', '1-Butene', 't-2-Butene', 'cis-2-Butene',
+            '1-Pentene', 't-2-Pentene', 'cis-2-Pentene', '1-Hexene', 'Acetylene', 'Cyclopentane', 'Methylcyclopentane',
+            'Cyclohexane', 'Methylcyclohexane', 'Isoprene', '2,2-Dimethylbutane', '2,3-Dimethylbutane',
+            '2-Methylpentane', '3-Methylpentane', '2,4-Dimethylpentane', '2-Methylhexane', '2,3-Dimethylpentane',
+            '3-Methylheptane', '2,2,4-Trimethylpentane', '2,3,4-Trimethylpentane', '2-Methylheptane', '3-Methylhexane',
+            'Styrene', 'Isopropylbenzene', 'n-Propylbenzene', 'm-Ethyltoluene', 'p-Ethyltoluene', 'm-Diethylbenzene',
+            'p-Diethylbenzene', '1,3,5-Trimethylbenzene', 'o-Ethyltoluene', '1,2,4-Trimethylbenzene',
+            '1,2,3-Trimethylbenzene',
+            '1.2-DCB', '1.4-DCB', '1.3-Butadiene', '1-Octene', '2-Ethyltoluene', '3.4-Ethyltoluene', 'Acetaldehyde',
+            'Acetone', 'Butyl Acetate', 'Ethanol', 'Ethyl Acetate', 'Hexane', 'IPA', 'Iso-Propylbenzene',
+            'PCE', 'Propene', 'TCE', 'VCM',
+        ],
         "deter_key": None,
     },
 

AeroViz/rawDataReader/core/__init__.py

@@ -7,6 +7,7 @@ from pathlib import Path
 from typing import Any
 
 import numpy as np
+import pandas as pd
 from pandas import DataFrame, date_range, concat, to_numeric, to_datetime
 
 from ..config.supported_instruments import meta
@@ -40,7 +41,7 @@ class AbstractReader(ABC):
         self.rate = rate
         self.qc = qc
         self.csv = csv_raw
-        self.apnd = append_data & reset
+        self.append = append_data & reset
 
         self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
         self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
@@ -50,7 +51,7 @@ class AbstractReader(ABC):
 
     # dependency injection function, customize each instrument
     @abstractmethod
-    def _raw_reader(self, _file):
+    def _raw_reader(self, file):
         pass
 
     @abstractmethod
@@ -112,9 +113,9 @@ class AbstractReader(ABC):
 
             self.logger.info(f"{'=' * 60}")
             self.logger.info(
-                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} ~ {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
+                f"Raw data time : {_st_raw.strftime('%Y-%m-%d %H:%M:%S')} to {_ed_raw.strftime('%Y-%m-%d %H:%M:%S')}")
             self.logger.info(
-                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} ~ {_end.strftime('%Y-%m-%d %H:%M:%S')}")
+                f"Output   time : {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}")
             self.logger.info(f"{'-' * 60}")
             print(f"\n\n\t\tfrom {_start.strftime('%Y-%m-%d %H:%M:%S')} to {_end.strftime('%Y-%m-%d %H:%M:%S')}\n")
 
@@ -211,18 +212,29 @@ class AbstractReader(ABC):
                  if f.name not in [self.csv_out.name, self.csv_nam.name, self.csv_nam_raw.name, f'{self.nam}.log']]
 
         if not files:
-            print(f"\t\t\033[31mNo files in '{self.path}' could be read. Please check the current path.\033[0m")
-            return None, None
+            raise FileNotFoundError(f"\t\t\033[31mNo files in '{self.path}' could be read."
+                                    f"Please check the current path.\033[0m")
 
         df_list = []
         for file in files:
             print(f"\r\t\treading {file.name}", end='')
-            df = self._raw_reader(file)
-            if df is not None:
-                df_list.append(df)
+
+            try:
+                df = self._raw_reader(file)
+
+                if df is not None and not df.empty:
+                    df_list.append(df)
+                else:
+                    self.logger.warning(f"File {file.name} produced an empty DataFrame or None.")
+
+            except pd.errors.ParserError as e:
+                self.logger.error(f"Error tokenizing data: {e}")
+
+            except Exception as e:
+                self.logger.error(f"Error reading {file.name}: {e}")
 
         if not df_list:
-            return None, None
+            raise ValueError("All files were either empty or failed to read.")
 
         raw_data = self._raw_process(concat(df_list))
         qc_data = self._QC(raw_data)
@@ -234,12 +246,12 @@ class AbstractReader(ABC):
         _f_raw_done, _f_qc_done = None, None
 
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
-        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.apnd):
+        if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and (not self.reset or self.append):
             print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mPICKLE\033[0m file of {self.nam}")
 
             _f_raw_done, _f_qc_done = self._read_pkl()
 
-            if not self.apnd:
+            if not self.append:
                 _f_raw_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_raw_done)
                 _f_qc_done, _start_raw, _end_raw = self._tmidx_process(_start, _end, _f_qc_done)
 
@@ -254,11 +266,9 @@ class AbstractReader(ABC):
         print(f"\n\t{dtm.now().strftime('%m/%d %X')} : Reading \033[96mRAW DATA\033[0m of {self.nam} and process it")
 
         _f_raw, _f_qc = self._read_raw_files()
-        if _f_raw is None:
-            return None
 
         # append new data and pickle data
-        if self.apnd and self.pkl_nam.exists():
+        if self.append and self.pkl_nam.exists():
             _f_raw = self._apnd_prcs(_f_raw_done, _f_raw)
             _f_qc = self._apnd_prcs(_f_qc_done, _f_qc)
 

AeroViz/rawDataReader/script/AE33.py

@@ -6,11 +6,11 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'AE33'
 
-    def _raw_reader(self, _file):
-        if _file.stat().st_size / 1024 < 550:
+    def _raw_reader(self, file):
+        if file.stat().st_size / 1024 < 550:
             print('\t It may not be a whole daily data.')
 
-        _df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
+        _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
                          delimiter=r'\s+', skiprows=5, usecols=range(67))
         _df.columns = _df.columns.str.strip(';')
 

AeroViz/rawDataReader/script/AE43.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'AE43'
 
-    def _raw_reader(self, _file):
-        _df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
+    def _raw_reader(self, file):
+        _df = read_csv(file, parse_dates={'time': ['StartTime']}, index_col='time')
         _df_id = _df['SetupID'].iloc[-1]
 
         # get last SetupID data

AeroViz/rawDataReader/script/APS_3321.py

@@ -7,8 +7,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'APS_3321'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
             _df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
             _key = list(_df.keys()[3:54])  ## 542 ~ 1981
 

AeroViz/rawDataReader/script/Aurora.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'Aurora'
 
-    def _raw_reader(self, _file):
-        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
             _df = read_csv(f, low_memory=False, index_col=0)
 
             _df.index = to_datetime(_df.index, errors='coerce')
@@ -33,5 +33,8 @@ class Reader(AbstractReader):
         # remove negative value
         _df = _df.mask((_df <= 0) | (_df > 2000)).copy()
 
+        # total scattering is larger than back scattering
+        _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
+
         # QC data in 1h
         return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/BC1054.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'BC1054'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
             _df = read_csv(f, parse_dates=True, index_col=0)
 
             _df.columns = _df.columns.str.replace(' ', '')

AeroViz/rawDataReader/script/EPA_vertical.py

@@ -7,8 +7,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'EPA_vertical'
 
-    def _raw_reader(self, _file):
-        with _file.open('r', encoding='ascii', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with file.open('r', encoding='ascii', errors='ignore') as f:
             # 有、無輸出有效值都可以
             # read 查詢小時值(測項).csv
             df = read_csv(f, encoding='ascii', encoding_errors='ignore', index_col=0, parse_dates=True,

AeroViz/rawDataReader/script/GRIMM.py

@@ -6,19 +6,19 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'GRIMM'
 
-    def _raw_reader(self, _file):
+    def _raw_reader(self, file):
 
-        _df = read_csv(_file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
+        _df = read_csv(file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
                        dayfirst=True).rename_axis("Time")
         _df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
 
-        if _file.name.startswith("A407ST"):
+        if file.name.startswith("A407ST"):
             _df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
         else:
             _df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
 
         if _df.empty:
-            print(_file, "is empty")
+            print(file, "is empty")
             return None
 
         return _df / 0.035

AeroViz/rawDataReader/script/IGAC.py

@@ -10,9 +10,9 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'IGAC'
 
-    def _raw_reader(self, _file):
+    def _raw_reader(self, file):
 
-        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
             _df = read_csv(f, parse_dates=True, index_col=0, na_values='-').apply(to_numeric, errors='coerce')
 
             _df.columns = _df.keys().str.strip(' ')

AeroViz/rawDataReader/script/MA350.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'MA350'
 
-    def _raw_reader(self, _file):
-        _df = read_csv(_file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis("Time")
+    def _raw_reader(self, file):
+        _df = read_csv(file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis("Time")
 
         _df = _df.rename(columns={
             'UV BCc': 'BC1',

AeroViz/rawDataReader/script/Minion.py

@@ -7,8 +7,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'Minion'
 
-    def _raw_reader(self, _file):
-        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
             _df = read_csv(f, low_memory=False, index_col=0)
 
             _df.index = to_datetime(_df.index, errors='coerce')

AeroViz/rawDataReader/script/NEPH.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'NEPH'
 
-    def _raw_reader(self, _file):
-        with _file.open('r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with file.open('r', encoding='utf-8', errors='ignore') as f:
             _df = read_csv(f, header=None, names=range(11))
 
             _df_grp = _df.groupby(0)
@@ -47,23 +47,15 @@ class Reader(AbstractReader):
                 return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
             except ValueError:
-                group_sizes = _df_grp.size()
-                print(group_sizes)
-
                 # Define valid groups and find invalid indices
-                valid_groups = {'B', 'G', 'R', 'D', 'T', 'Y', 'Z'}
-                invalid_indices = _df[~_df[0].isin(valid_groups)].index
-
-                # Print invalid indices and values
+                invalid_indices = _df[~_df[0].isin({'B', 'G', 'R', 'D', 'T', 'Y', 'Z'})].index
                 print("Invalid values and their indices:")
-                for idx in invalid_indices:
-                    print(f"Index: {idx}, Value: {_df.at[idx, 0]}")
+                print("\n".join([f"Index: {idx}, Value: {_df.at[idx, 0]}" for idx in invalid_indices]))
 
                 # Return an empty DataFrame with specified columns if there's a length mismatch
-                columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
-                _df_out = DataFrame(index=_idx_tm, columns=columns)
+                _df_out = DataFrame(index=_idx_tm, columns=['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH'])
                 _df_out.index.name = 'Time'
-                print(f'\n\t\t\t Length mismatch in {_file} data. Returning an empty DataFrame.')
+                print(f'\n\t\t\t Length mismatch in {file} data. Returning an empty DataFrame.')
                 return _df_out
 
     # QC data
@@ -71,5 +63,8 @@ class Reader(AbstractReader):
         # remove negative value
         _df = _df.mask((_df <= 5).copy())
 
+        # total scattering is larger than back scattering
+        _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
+
         # QC data in 1h
         return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/SMPS_TH.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'SMPS_TH'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
             _df = read_table(f, skiprows=18, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
             _key = list(_df.keys()[6:-26])
 

AeroViz/rawDataReader/script/SMPS_aim11.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'SMPS_aim11'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
 
             skiprows = 0
             for _line in f:

AeroViz/rawDataReader/script/SMPS_genr.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'SMPS_genr'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
 
             skiprows = 0
             for _line in f:

AeroViz/rawDataReader/script/Sunset_OCEC.py

@@ -6,38 +6,44 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'Sunset_OCEC'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3)
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, skiprows=3, nrows=25)
 
             _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
             _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
             _df = _df.set_index('time')
+            _df.index = _df.index.round('1h')
 
             _df = _df.rename(columns={
                 'Thermal/Optical OC (ugC/LCm^3)': 'Thermal_OC',
-                'OC ugC/m^3 (Thermal/Optical)': 'Thermal_OC',
-
                 'Thermal/Optical EC (ugC/LCm^3)': 'Thermal_EC',
-                'EC ugC/m^3 (Thermal/Optical)': 'Thermal_EC',
-
                 'OC=TC-BC (ugC/LCm^3)': 'Optical_OC',
-                'OC by diff ugC (TC-OptEC)': 'Optical_OC',
-
                 'BC (ugC/LCm^3)': 'Optical_EC',
+                'TC (ugC/LCm^3)': 'TC',
+
+                'OC ugC/m^3 (Thermal/Optical)': 'Thermal_OC',
+                'EC ugC/m^3 (Thermal/Optical)': 'Thermal_EC',
+                'OC by diff ugC (TC-OptEC)': 'Optical_OC',
                 'OptEC ugC/m^3': 'Optical_EC',
+                'TC ugC/m^3': 'TC',
 
                 'Sample Volume Local Condition Actual m^3': 'Sample_Volume',
-                'TC (ugC/LCm^3)': 'TC',
-                'TC ugC/m^3': 'TC',
-                'OCPk1-ug C': 'OC1',
-                'OCPk2-ug C': 'OC2',
-                'OCPk3-ug C': 'OC3',
-                'OCPk4-ug C': 'OC4',
-                'Pyrolized C ug': 'PC'
+
+                'OCPk1-ug C': 'OC1_raw',
+                'OCPk2-ug C': 'OC2_raw',
+                'OCPk3-ug C': 'OC3_raw',
+                'OCPk4-ug C': 'OC4_raw',
+                'ECPk1-ug C': 'EC1_raw',
+                'ECPk2-ug C': 'EC2_raw',
+                'ECPk3-ug C': 'EC3_raw',
+                'ECPk4-ug C': 'EC4_raw',
+                'ECPk5-ug C': 'EC5_raw',
             })
 
-            _df = _df[['Thermal_OC', 'Optical_OC', 'Thermal_EC', 'Optical_EC', 'TC', 'OC1', 'OC2', 'OC3', 'OC4']]
+            _df = _df[['Thermal_OC', 'Optical_OC', 'Thermal_EC', 'Optical_EC', 'TC', 'Sample_Volume',
+                       'OC1_raw', 'OC2_raw', 'OC3_raw', 'OC4_raw', 'EC1_raw', 'EC2_raw', 'EC3_raw', 'EC4_raw',
+                       'EC5_raw']]
 
             return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/TEOM.py

@@ -6,8 +6,8 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'TEOM'
 
-    def _raw_reader(self, _file):
-        with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
             _df = read_csv(f, skiprows=3, index_col=False)
 
             _df = _df.rename(columns={'Time Stamp': 'time',

AeroViz/rawDataReader/script/Table.py

@@ -8,11 +8,11 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'Table'
 
-    def _raw_reader(self, _file):
-        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+    def _raw_reader(self, file):
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
             _df = read_csv(f, low_memory=False, index_col=0)
 
-            _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
+            _df.index = to_datetime(_df.index, errors='coerce')
             _df.index.name = 'time'
 
             _df.columns = _df.keys().str.strip(' ')

AeroViz/rawDataReader/script/VOC.py

@@ -1,4 +1,3 @@
-
 from pandas import read_csv
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -7,20 +6,28 @@ from AeroViz.rawDataReader.core import AbstractReader
 class Reader(AbstractReader):
     nam = 'VOC'
 
-    def _raw_reader(self, _file):
-        with _file.open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, parse_dates=[0], index_col=[0], na_values=('-', 'N.D.'))
+    def _raw_reader(self, file):
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=True, index_col=0, na_values=('-', 'N.D.'))
 
             _df.columns = _df.keys().str.strip(' ')
             _df.index.name = 'time'
 
-            try:
-                _df = _df[self.meta["key"]].loc[_df.index.dropna()]
+            support_voc = set(self.meta["key"])
+
+            valid_keys = [key for key in _df.keys() if key in support_voc]
+            invalid_keys = [key for key in _df.keys() if key not in support_voc]
 
-            except KeyError:
-                _df = _df[self.meta["key_2"]].loc[_df.index.dropna()]
+            if invalid_keys:
+                self.logger.warning(f'{invalid_keys} are not supported keys.')
+                print(f'\n\t\t{invalid_keys} are not supported keys.'
+                      f'\n\t\tPlease check the\033[91m support_voc.md\033[0m file to use the correct name.')
 
-        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+            if valid_keys:
+                return _df[valid_keys].loc[~_df.index.duplicated() & _df.index.notna()]
+            else:
+                self.logger.warning("沒有找到匹配的鍵。返回原始DataFrame並移除含NaN的行。")
+                return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
     def _QC(self, _df):
         return _df

{AeroViz-0.1.4.dist-info → AeroViz-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.4
+Version: 0.1.5
 Summary: Aerosol science
 Home-page: https://github.com/Alex870521/AeroViz
 Author: alex