AeroViz 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

AeroViz/dataProcess/Optical/_IMPROVE.py

@@ -41,7 +41,7 @@ def _revised(_df_mass, _df_RH):
 
         _df['AS'] = 2.2 * _frhs * _df_mass['S_AS'] + 4.8 * _frhl * _df_mass['L_AS']
         _df['AN'] = 2.4 * _frhs * _df_mass['S_AN'] + 5.1 * _frhl * _df_mass['L_AN']
-        _df['OM'] = 2.8 * _df_mass['S_OM'] + 6.1 * _frhl * _df_mass['L_OM']
+        _df['OM'] = 2.8 * _df_mass['S_OM'] + 6.1 * _df_mass['L_OM']
         _df['Soil'] = _df_mass['Soil']
         _df['SS'] = 1.7 * _frhss * _df_mass['SS']
         _df['EC'] = 10 * _df_mass['EC']

AeroViz/plot/templates/koschmieder.py

@@ -89,7 +89,7 @@ def koschmieder(df: pd.DataFrame,
            )
 
     plt.xticks(ticks=np.array(range(0, 31, 5)), labels=np.array(range(0, 31, 5)))
-
+    fig.savefig('koschmieder.png', dpi=600)
     plt.show()
 
     return fig, ax

AeroViz/rawDataReader/__init__.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Literal
 
 from pandas import Grouper, Timedelta
 
@@ -10,66 +10,111 @@ from AeroViz.rawDataReader.script import *
 __all__ = ['RawDataReader']
 
 SUPPORTED_INSTRUMENTS = [
-    NEPH, Aurora, SMPS, GRIMM, APS, AE33, AE43, BC1054,
-    MA350, TEOM, OCEC, IGAC, VOC, EPA, Minion
+    NEPH, Aurora, SMPS, APS, GRIMM, AE33, AE43, BC1054,
+    MA350, BAM1020, TEOM, OCEC, IGAC, VOC, EPA, Minion
 ]
 
+SIZE_RANGE_INSTRUMENTS = ['SMPS', 'APS', 'GRIMM']
 
-def RawDataReader(instrument_name: str,
+
+def RawDataReader(instrument: str,
                   path: Path | str,
                   reset: bool = False,
                   qc: bool | str = True,
-                  qc_freq: str | None = None,
-                  rate: bool = True,
-                  append_data: bool = False,
                   start: datetime = None,
                   end: datetime = None,
                   mean_freq: str = '1h',
-                  csv_out: bool = True,
-                  **kwargs: Any):
+                  size_range: tuple[float, float] | None = None,
+                  suppress_warnings: bool = False,
+                  log_level: Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] = 'INFO',
+                  **kwargs):
     """
     Factory function to instantiate the appropriate reader module for a given instrument and
     return the processed data over the specified time range.
 
-    :param instrument_name: The name of the instrument for which to read data. Must be a valid key in the `meta` dictionary.
-    :param path: The directory where raw data files for the instrument are stored.
-    :param reset: If True, reset the state and reprocess the data from scratch.
-    :param qc: If True, apply quality control (QC) to the raw data.
-    :param qc_freq: Frequency at which to perform QC. Must be one of 'W', 'M', 'Q', 'Y' for weekly, monthly, quarterly, or yearly.
-    :param rate: If True, calculate rates from the data.
-    :param append_data: If True, append new data to the existing dataset instead of overwriting it.
-    :param start: Start time for filtering the data. If None, no start time filtering will be applied.
-    :param end: End time for filtering the data. If None, no end time filtering will be applied.
-    :param mean_freq: Resampling frequency for averaging the data. Example: '1h' for hourly mean.
-    :param csv_out: If True, output the processed data as a CSV file.
-
-    :return: An instance of the reader module corresponding to the specified instrument, which processes the data and returns it in a usable format.
-
-    :raises ValueError: If the `instrument_name` provided is not a valid key in the `meta` dictionary.
-    :raises ValueError: If the specified path does not exist or is not a directory.
-    :raises ValueError: If the QC frequency is invalid.
-    :raises ValueError: If start and end times are not both provided or are invalid.
-    :raises ValueError: If the mean_freq is not a valid frequency string.
-
-    :Example:
-
-    To read and process data for the BC1054 instrument:
-
+    Parameters
+    ----------
+    instrument : str
+       The instrument name for which to read data, must be a valid key in the meta dictionary
+
+    path : Path or str
+       The directory where raw data files for the instrument are stored
+
+    reset : bool or str
+       Data processing control mode:
+       False (default) - Use existing processed data if available
+       True - Force reprocess all data from raw files
+       'append' - Add new data to existing processed data
+
+    qc : bool or str
+       Quality control and rate calculation mode:
+       True (default) - Apply QC and calculate overall rates
+       False - Skip QC and return raw data only
+       str - Calculate rates at specified intervals:
+             'W' - Weekly rates
+             'MS' - Month start rates
+             'QS' - Quarter start rates
+             'YS' - Year start rates
+             Can add number prefix (e.g., '2MS' for bi-monthly)
+
+    start : datetime
+       Start time for filtering the data
+
+    end : datetime
+       End time for filtering the data
+
+    mean_freq : str
+       Resampling frequency for averaging the data (e.g., '1h' for hourly mean)
+
+    size_range : tuple[float, float], optional
+       Size range in nanometers (min_size, max_size) for SMPS/APS data filtering
+
+    suppress_warnings : bool, optional
+       Whether to suppress warning messages (default: False)
+
+    log_level : {'DEBUG', 'INFO', 'WARNING', 'ERROR'}
+       Logging level (default: 'INFO')
+
+    **kwargs
+       Additional arguments to pass to the reader module
+
+    Returns
+    -------
+    pd.DataFrame
+       Processed data with specified QC and time range
+
+    Raises
+    ------
+    ValueError
+       If instrument name is invalid
+       If path does not exist
+       If QC frequency is invalid
+       If time range is invalid
+       If mean_freq format is invalid
+
+    Examples
+    --------
     >>> from pathlib import Path
     >>> from datetime import datetime
+    >>> from AeroViz import RawDataReader
     >>>
-    >>> data = RawDataReader(
-    ...     instrument_name='BC1054',
-    ...     path=Path('/path/to/data'),
-    ...     start=datetime(2024, 2, 1),
-    ...     end=datetime(2024, 7, 31))
+    >>> df_ae33 = RawDataReader(
+    ...     instrument='AE33',
+    ...     path=Path('/path/to/your/data/folder'),
+    ...     reset=True,
+    ...     qc='1MS',
+    ...     start=datetime(2024, 1, 1),
+    ...     end=datetime(2024, 6, 30),
+    ...     mean_freq='1h',
+    ... )
     """
+
     # Mapping of instrument names to their respective classes
     instrument_class_map = {cls.__name__.split('.')[-1]: cls for cls in SUPPORTED_INSTRUMENTS}
 
     # Check if the instrument name is in the map
-    if instrument_name not in meta.keys():
-        raise ValueError(f"Instrument name '{instrument_name}' is not valid. \nMust be one of: {list(meta.keys())}")
+    if instrument not in meta.keys():
+        raise ValueError(f"Instrument name '{instrument}' is not valid. \nMust be one of: {list(meta.keys())}")
 
     # Check if path exists and is a directory
     if not isinstance(path, Path):
@@ -78,22 +123,21 @@ def RawDataReader(instrument_name: str,
         raise FileNotFoundError(f"The specified path '{path}' does not exist or is not a directory.")
 
     # Validate the QC frequency
-    if qc_freq is not None:
+    if isinstance(qc, str):
         try:
-            Grouper(freq=qc_freq)
-        except ValueError as e:
-            raise ValueError(f"Invalid frequency: {qc_freq}. Error: {str(e)}")
-        except TypeError as e:
-            raise ValueError(f"Invalid frequency type: {qc_freq}. Frequency should be a string.")
-
-    if start and end:
-        if end.hour == 0 and end.minute == 0 and end.second == 0:
-            end = end.replace(hour=23, minute=59, second=59)
-    else:
+            Grouper(freq=qc)
+        except (ValueError, TypeError):
+            raise ValueError(f"Invalid frequency: {qc}. Must be one of: "
+                             f"W (week), MS (month start), QS (quarter start), YS (year start)")
+
+    # Verify input times
+    if not (start and end):
         raise ValueError("Both start and end times must be provided.")
     if end <= start:
         raise ValueError(f"Invalid time range: start {start} is after end {end}")
 
+    end = end.replace(hour=23, minute=59, second=59) if end.hour == 0 and end.minute == 0 else end
+
     # Verify that mean_freq format
     try:
         Timedelta(mean_freq)
@@ -101,19 +145,40 @@ def RawDataReader(instrument_name: str,
         raise ValueError(
             f"Invalid mean_freq: '{mean_freq}'. It should be a valid frequency string (e.g., '1h', '30min', '1D').")
 
+    # Validate size range
+    if size_range is not None:
+        if instrument not in SIZE_RANGE_INSTRUMENTS:
+            raise ValueError(f"Size range filtering is only supported for {SIZE_RANGE_INSTRUMENTS}")
+
+        min_size, max_size = size_range
+        if not isinstance(min_size, (int, float)) or not isinstance(max_size, (int, float)):
+            raise ValueError("Size range values must be numeric")
+        if min_size >= max_size:
+            raise ValueError("Minimum size must be less than maximum size")
+
+        if instrument == 'SMPS':
+            if not (1 <= min_size <= 1000) or not (1 <= max_size <= 1000):
+                raise ValueError("SMPS size range must be between 1 and 1000 nm")
+        elif instrument == 'APS':
+            if not (500 <= min_size <= 20000) or not (500 <= max_size <= 20000):
+                raise ValueError("APS size range must be between 500 and 20000 nm")
+
+        kwargs.update({'size_range': size_range})
+
+    kwargs.update({
+        'suppress_warnings': suppress_warnings,
+        'log_level': log_level
+    })
+
     # Instantiate the class and return the instance
-    reader_module = instrument_class_map[instrument_name].Reader(
+    reader_module = instrument_class_map[instrument].Reader(
         path=path,
         reset=reset,
         qc=qc,
-        qc_freq=qc_freq,
-        rate=rate,
-        append_data=append_data,
         **kwargs
     )
     return reader_module(
         start=start,
         end=end,
         mean_freq=mean_freq,
-        csv_out=csv_out,
     )

AeroViz/rawDataReader/config/supported_instruments.py

@@ -58,6 +58,13 @@ meta = {
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC5"]},
     },
 
+    "BAM1020": {
+        "pattern": ["*.csv"],
+        "freq": "1h",
+        "deter_key": {
+            "Mass Conc.": ["Conc"]},
+    },
+
     "TEOM": {
         "pattern": ["*.csv"],
         "freq": "6min",
@@ -71,10 +78,6 @@ meta = {
         "pattern": ["*LCRes.csv"],
         "freq": "1h",
         "deter_key": {
-            "Thermal OC": ["Thermal_OC"],
-            "Thermal EC": ["Thermal_EC"],
-            "Optical OC": ["Optical_OC"],
-            "Optical EC": ["Optical_EC"],
             "Thermal OC & EC": ["Thermal_OC", "Thermal_EC"],
             "Optical OC & EC": ["Optical_OC", "Optical_EC"],
         },

AeroViz/rawDataReader/core/__init__.py

@@ -3,17 +3,16 @@ from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
-from typing import Optional, Generator
+from typing import Generator
 
 import numpy as np
 import pandas as pd
-from pandas import DataFrame, concat, read_pickle, to_numeric
 from rich.console import Console
 from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
 
 from AeroViz.rawDataReader.config.supported_instruments import meta
 from AeroViz.rawDataReader.core.logger import ReaderLogger
-from AeroViz.rawDataReader.core.qc import DataQualityControl
+from AeroViz.rawDataReader.core.qc import QualityControl
 
 __all__ = ['AbstractReader']
 
@@ -32,45 +31,43 @@ class AbstractReader(ABC):
 
     def __init__(self,
                  path: Path | str,
-                 reset: bool = False,
-                 qc: bool = True,
-                 qc_freq: Optional[str] = None,
-                 rate: bool = True,
-                 append_data: bool = False,
+                 reset: bool | str = False,
+                 qc: bool | str = True,
                  **kwargs):
 
         self.path = Path(path)
         self.meta = meta[self.nam]
-        self.logger = ReaderLogger(self.nam, self.path)
+        output_folder = self.path / f'{self.nam.lower()}_outputs'
+        output_folder.mkdir(parents=True, exist_ok=True)
 
-        self.reset = reset
-        self.qc = qc
-        self.qc_freq = qc_freq
-        self.rate = rate
-        self.append = append_data and reset
+        self.logger = ReaderLogger(
+            self.nam, output_folder,
+            kwargs.get('log_level').upper() if not kwargs.get('suppress_warnings') else 'ERROR')
 
-        self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
-        self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
-        self.pkl_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.pkl'
-        self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
-        self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
+        self.reset = reset is True
+        self.append = reset == 'append'
+        self.qc = qc  # if qc, then calculate rate
+        self.qc_freq = qc if isinstance(qc, str) else None
+        self.kwargs = kwargs
 
-        self.size_range = kwargs.get('size_range', (11.8, 593.5))
+        self.pkl_nam = output_folder / f'_read_{self.nam.lower()}.pkl'
+        self.csv_nam = output_folder / f'_read_{self.nam.lower()}.csv'
+        self.pkl_nam_raw = output_folder / f'_read_{self.nam.lower()}_raw.pkl'
+        self.csv_nam_raw = output_folder / f'_read_{self.nam.lower()}_raw.csv'
+        self.csv_out = output_folder / f'output_{self.nam.lower()}.csv'
 
     def __call__(self,
                  start: datetime,
                  end: datetime,
                  mean_freq: str = '1h',
-                 csv_out: bool = True,
-                 ) -> DataFrame:
+                 ) -> pd.DataFrame:
 
         data = self._run(start, end)
 
         if data is not None:
-            if mean_freq:
-                data = data.resample(mean_freq).mean()
-            if csv_out:
-                data.to_csv(self.csv_out)
+            data = data.resample(mean_freq).mean()
+
+        data.to_csv(self.csv_out)
 
         return data
 
@@ -79,7 +76,7 @@ class AbstractReader(ABC):
         pass
 
     @abstractmethod
-    def _QC(self, df: DataFrame) -> DataFrame:
+    def _QC(self, df: pd.DataFrame) -> pd.DataFrame:
         return df
 
     def _rate_calculate(self, raw_data, qc_data) -> None:
@@ -182,7 +179,7 @@ class AbstractReader(ABC):
 
         return _df
 
-    def _save_data(self, raw_data: DataFrame, qc_data: DataFrame) -> None:
+    def _save_data(self, raw_data: pd.DataFrame, qc_data: pd.DataFrame) -> None:
         try:
             raw_data.to_pickle(self.pkl_nam_raw)
             raw_data.to_csv(self.csv_nam_raw)
@@ -222,7 +219,7 @@ class AbstractReader(ABC):
                 for msg in msgs:
                     original[level](msg)
 
-    def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
+    def _read_raw_files(self) -> tuple[pd.DataFrame | None, pd.DataFrame | None]:
         files = [f
                  for file_pattern in self.meta['pattern']
                  for pattern in {file_pattern.lower(), file_pattern.upper(), file_pattern}
@@ -242,7 +239,7 @@ class AbstractReader(ABC):
                     if (df := self._raw_reader(file)) is not None and not df.empty:
                         df_list.append(df)
                     else:
-                        self.logger.warning(f"\tFile {file.name} produced an empty DataFrame or None.")
+                        self.logger.debug(f"\tFile {file.name} produced an empty DataFrame or None.")
 
                 except Exception as e:
                     self.logger.error(f"Error reading {file.name}: {e}")
@@ -250,13 +247,15 @@ class AbstractReader(ABC):
         if not df_list:
             raise ValueError(f"\033[41m\033[97mAll files were either empty or failed to read.\033[0m")
 
-        raw_data = concat(df_list, axis=0).groupby(level=0).first()
+        raw_data = pd.concat(df_list, axis=0).groupby(level=0).first()
 
-        if self.nam == 'SMPS':
+        if self.nam in ['SMPS', 'APS', 'GRIMM']:
             raw_data = raw_data.sort_index(axis=1, key=lambda x: x.astype(float))
 
-        raw_data = self._timeIndex_process(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
-        qc_data = self._QC(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
+        raw_data = self._timeIndex_process(raw_data)
+
+        raw_data = raw_data.apply(pd.to_numeric, errors='coerce').copy(deep=True)
+        qc_data = self._QC(raw_data).apply(pd.to_numeric, errors='coerce').copy(deep=True)
 
         return raw_data, qc_data
 
@@ -265,7 +264,7 @@ class AbstractReader(ABC):
         if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and not self.reset:
             self.logger.info_box(f"Reading {self.nam} PICKLE from {user_start} to {user_end}", color_part="PICKLE")
 
-            _f_raw_done, _f_qc_done = read_pickle(self.pkl_nam_raw), read_pickle(self.pkl_nam)
+            _f_raw_done, _f_qc_done = pd.read_pickle(self.pkl_nam_raw), pd.read_pickle(self.pkl_nam)
 
             if self.append:
                 self.logger.info_box(f"Appending New data from {user_start} to {user_end}", color_part="New data")
@@ -292,25 +291,26 @@ class AbstractReader(ABC):
         # save
         self._save_data(_f_raw, _f_qc)
 
-        if self.rate:
-            self._rate_calculate(_f_raw.apply(to_numeric, errors='coerce'), _f_qc.apply(to_numeric, errors='coerce'))
+        if self.qc:
+            self._rate_calculate(_f_raw.apply(pd.to_numeric, errors='coerce'),
+                                 _f_qc.apply(pd.to_numeric, errors='coerce'))
 
         return _f_qc if self.qc else _f_raw
 
     @staticmethod
-    def reorder_dataframe_columns(df, order_lists, others_col=False):
+    def reorder_dataframe_columns(df, order_lists: list[list], keep_others: bool = False):
         new_order = []
 
         for order in order_lists:
-            # 只添加存在於DataFrame中的欄位，且不重複添加
+            # Only add column that exist in the DataFrame and do not add them repeatedly
             new_order.extend([col for col in order if col in df.columns and col not in new_order])
 
-        if others_col:
-            # 添加所有不在新順序列表中的原始欄位，保持它們的原始順序
+        if keep_others:
+            # Add all original fields not in the new order list, keeping their original order
             new_order.extend([col for col in df.columns if col not in new_order])
 
         return df[new_order]
 
     @staticmethod
     def time_aware_IQR_QC(df: pd.DataFrame, time_window='1D', log_dist=False) -> pd.DataFrame:
-        return DataQualityControl().time_aware_iqr(df, time_window=time_window, log_dist=log_dist)
+        return QualityControl().time_aware_iqr(df, time_window=time_window, log_dist=log_dist)

AeroViz/rawDataReader/core/logger.py

@@ -8,9 +8,10 @@ from pathlib import Path
 
 
 class ReaderLogger:
-    def __init__(self, name: str, log_path: Path):
+    def __init__(self, name: str, log_path: Path, log_level: str = 'WARNING'):
         self.name = name
         self.log_path = log_path
+        self._log_level = getattr(logging, log_level)
 
         # 檢查是否支持顏色輸出
         self.color_support = self._check_color_support()
@@ -92,7 +93,7 @@ class ReaderLogger:
     def _setup_logger(self) -> logging.Logger:
         """設置logger"""
         logger = logging.getLogger(self.name)
-        logger.setLevel(logging.INFO)
+        logger.setLevel(self._log_level)
 
         # 移除現有的 handlers
         for handler in logger.handlers[:]:
@@ -135,6 +136,9 @@ class ReaderLogger:
             text = text.encode('ascii', 'replace').decode('ascii')
         return text
 
+    def debug(self, msg: str):
+        self.logger.debug(self._safe_print(msg))
+
     def info(self, msg: str):
         self.logger.info(self._safe_print(msg))
 

AeroViz/rawDataReader/core/qc.py

@@ -2,7 +2,7 @@ import numpy as np
 import pandas as pd
 
 
-class DataQualityControl:
+class QualityControl:
     """A class providing various methods for data quality control and outlier detection"""
 
     @staticmethod

AeroViz/rawDataReader/script/AE33.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         if file.stat().st_size / 1024 < 550:
-            self.logger.info(f'\t {file.name} may not be a whole daily data. Make sure the file is correct.')
+            self.logger.warning(f'\t {file.name} may not be a whole daily data. Make sure the file is correct.')
 
         _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
                          delimiter=r'\s+', skiprows=5, usecols=range(67))