AeroViz 0.1.11__tar.gz → 0.1.13__tar.gz

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/dataProcess/Optical/_IMPROVE.py

@@ -41,7 +41,7 @@ def _revised(_df_mass, _df_RH):
 
         _df['AS'] = 2.2 * _frhs * _df_mass['S_AS'] + 4.8 * _frhl * _df_mass['L_AS']
         _df['AN'] = 2.4 * _frhs * _df_mass['S_AN'] + 5.1 * _frhl * _df_mass['L_AN']
-        _df['OM'] = 2.8 * _df_mass['S_OM'] + 6.1 * _frhl * _df_mass['L_OM']
+        _df['OM'] = 2.8 * _df_mass['S_OM'] + 6.1 * _df_mass['L_OM']
         _df['Soil'] = _df_mass['Soil']
         _df['SS'] = 1.7 * _frhss * _df_mass['SS']
         _df['EC'] = 10 * _df_mass['EC']

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/plot/meteorology/hysplit.py

@@ -40,14 +40,17 @@ def hysplit(file: Path = DEFAULT_FILE):
     # 創建地圖
     fig, ax = plt.subplots(figsize=(4, 5), subplot_kw={'projection': ccrs.PlateCarree()})
 
+    ax.set_global()
+    # ax.stock_img()
+
     # 設置地圖範圍
     ax.set_extent([116, 126, 17, 30], crs=ccrs.PlateCarree())
 
     # 添加自然地理特徵
-    ax.add_feature(cfeature.LAND)
-    ax.add_feature(cfeature.OCEAN)
-    ax.add_feature(cfeature.COASTLINE)
-    ax.add_feature(cfeature.BORDERS, linestyle=':')
+    ax.add_feature(cfeature.LAND.with_scale('10m'))
+    ax.add_feature(cfeature.OCEAN.with_scale('10m'))
+    ax.add_feature(cfeature.COASTLINE.with_scale('10m'))
+    ax.add_feature(cfeature.BORDERS.with_scale('10m'), linestyle=':')
 
     # 添加經緯度網格
     ax.gridlines(draw_labels=True, dms=True, x_inline=False, y_inline=False)

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/__init__.py

@@ -10,7 +10,7 @@ from AeroViz.rawDataReader.script import *
 __all__ = ['RawDataReader']
 
 SUPPORTED_INSTRUMENTS = [
-    NEPH, Aurora, SMPS, GRIMM, APS_3321, AE33, AE43, BC1054,
+    NEPH, Aurora, SMPS, GRIMM, APS, AE33, AE43, BC1054,
     MA350, TEOM, OCEC, IGAC, VOC, EPA, Minion
 ]
 
@@ -62,7 +62,7 @@ def RawDataReader(instrument_name: str,
     ...     instrument_name='BC1054',
     ...     path=Path('/path/to/data'),
     ...     start=datetime(2024, 2, 1),
-    ...     end=datetime(2024, 7, 31, 23))
+    ...     end=datetime(2024, 7, 31))
     """
     # Mapping of instrument names to their respective classes
     instrument_class_map = {cls.__name__.split('.')[-1]: cls for cls in SUPPORTED_INSTRUMENTS}
@@ -71,7 +71,7 @@ def RawDataReader(instrument_name: str,
     if instrument_name not in meta.keys():
         raise ValueError(f"Instrument name '{instrument_name}' is not valid. \nMust be one of: {list(meta.keys())}")
 
-    # 檢查 path 是否存在且是一個目錄
+    # Check if path exists and is a directory
     if not isinstance(path, Path):
         path = Path(path)
     if not path.exists() or not path.is_dir():
@@ -94,7 +94,7 @@ def RawDataReader(instrument_name: str,
     if end <= start:
         raise ValueError(f"Invalid time range: start {start} is after end {end}")
 
-    # 驗證 mean_freq 的格式是否正確
+    # Verify that mean_freq format
     try:
         Timedelta(mean_freq)
     except ValueError:

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/config/supported_instruments.py

@@ -25,7 +25,7 @@ meta = {
         "deter_key": {"Bins": ["all"]},
     },
 
-    "APS_3321": {
+    "APS": {
         "pattern": ["*.txt"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/core/__init__.py

@@ -143,7 +143,7 @@ class AbstractReader(ABC):
         :return: Processed DataFrame
         """
         # Round timestamps and remove duplicates
-        _df = _df.groupby(_df.index.round('1min')).first()
+        _df = _df.groupby(_df.index.floor('1min')).first()
 
         # Determine frequency
         freq = _df.index.inferred_freq or self.meta['freq']
@@ -161,7 +161,12 @@ class AbstractReader(ABC):
         new_index = pd.date_range(user_start or df_start, user_end or df_end, freq=freq, name='time')
 
         # Process data: convert to numeric, resample, and reindex
-        return _df.reindex(new_index)
+        if freq in ['1min', 'min', 'T']:
+            return _df.reindex(new_index, method='nearest', tolerance='1min')
+        elif freq in ['1h', 'h', 'H']:
+            return _df.reindex(new_index, method='nearest', tolerance='1h')
+        else:
+            return _df.reindex(new_index, method='nearest', tolerance=freq)
 
     def _outlier_process(self, _df):
         outlier_file = self.path / 'outlier.json'

aeroviz-0.1.13/AeroViz/rawDataReader/core/logger.py

@@ -0,0 +1,167 @@
+import codecs
+import logging
+import os
+import platform
+import re
+import sys
+from pathlib import Path
+
+
+class ReaderLogger:
+    def __init__(self, name: str, log_path: Path):
+        self.name = name
+        self.log_path = log_path
+
+        # 檢查是否支持顏色輸出
+        self.color_support = self._check_color_support()
+
+        # 設置顏色代碼
+        if self.color_support:
+            self.CYAN = '\033[96m'
+            self.BLUE = '\033[94m'
+            self.GREEN = '\033[92m'
+            self.YELLOW = '\033[93m'
+            self.RED = '\033[91m'
+            self.RESET = '\033[0m'
+        else:
+            self.CYAN = ''
+            self.BLUE = ''
+            self.GREEN = ''
+            self.YELLOW = ''
+            self.RED = ''
+            self.RESET = ''
+
+        # 檢查 Unicode 支持
+        self.unicode_support = self._setup_unicode()
+
+        # 設置框架字符
+        if self.unicode_support:
+            self.BOX_TOP_LEFT = "╔"
+            self.BOX_TOP_RIGHT = "╗"
+            self.BOX_BOTTOM_LEFT = "╚"
+            self.BOX_BOTTOM_RIGHT = "╝"
+            self.BOX_HORIZONTAL = "═"
+            self.BOX_VERTICAL = "║"
+            self.ARROW = "▶"
+        else:
+            self.BOX_TOP_LEFT = "+"
+            self.BOX_TOP_RIGHT = "+"
+            self.BOX_BOTTOM_LEFT = "+"
+            self.BOX_BOTTOM_RIGHT = "+"
+            self.BOX_HORIZONTAL = "-"
+            self.BOX_VERTICAL = "|"
+            self.ARROW = ">"
+
+        self.logger = self._setup_logger()
+
+    def _check_color_support(self) -> bool:
+        """檢查環境是否支持顏色輸出"""
+        # 檢查是否在 Spyder 或其他 IDE 中運行
+        if any(IDE in os.environ.get('PYTHONPATH', '') for IDE in ['spyder', 'jupyter']):
+            return False
+
+        # 檢查是否強制啟用或禁用顏色
+        if 'FORCE_COLOR' in os.environ:
+            return os.environ['FORCE_COLOR'].lower() in ('1', 'true', 'yes')
+
+        # Windows 檢查
+        if platform.system().lower() == 'windows':
+            return ('ANSICON' in os.environ or
+                    'WT_SESSION' in os.environ or  # Windows Terminal
+                    'ConEmuANSI' in os.environ or
+                    os.environ.get('TERM_PROGRAM', '').lower() == 'vscode')
+
+        # 其他系統檢查
+        return hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
+
+    def _setup_unicode(self) -> bool:
+        """設置 Unicode 支持"""
+        if platform.system().lower() == 'windows':
+            try:
+                if hasattr(sys.stdout, 'reconfigure'):
+                    sys.stdout.reconfigure(encoding='utf-8')
+                elif hasattr(sys.stdout, 'buffer'):
+                    sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer)
+                else:
+                    return False
+                return True
+            except Exception:
+                return False
+        return True
+
+    def _setup_logger(self) -> logging.Logger:
+        """設置logger"""
+        logger = logging.getLogger(self.name)
+        logger.setLevel(logging.INFO)
+
+        # 移除現有的 handlers
+        for handler in logger.handlers[:]:
+            handler.close()
+            logger.removeHandler(handler)
+
+        # 清理 ANSI 格式化器
+        class CleanFormatter(logging.Formatter):
+            def format(self, record):
+                formatted_msg = super().format(record)
+                return re.sub(r'\033\[[0-9;]*m', '', formatted_msg)
+
+        # 設置檔案處理器
+        try:
+            log_dir = Path(self.log_path)
+            log_dir.mkdir(parents=True, exist_ok=True)
+            file_handler = logging.FileHandler(
+                log_dir / f'{self.name}.log',
+                encoding='utf-8',
+                errors='replace'
+            )
+            file_handler.setFormatter(
+                CleanFormatter('%(asctime)s - %(message)s',
+                               datefmt='%Y-%m-%d %H:%M:%S')
+            )
+            logger.addHandler(file_handler)
+        except Exception as e:
+            print(f"Warning: Could not set up file logging: {e}")
+
+        # 設置控制台處理器
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(logging.Formatter('%(message)s'))
+        logger.addHandler(console_handler)
+
+        return logger
+
+    def _safe_print(self, text: str) -> str:
+        """安全打印，處理編碼問題"""
+        if not self.unicode_support:
+            text = text.encode('ascii', 'replace').decode('ascii')
+        return text
+
+    def info(self, msg: str):
+        self.logger.info(self._safe_print(msg))
+
+    def warning(self, msg: str):
+        self.logger.warning(self._safe_print(msg))
+
+    def error(self, msg: str):
+        self.logger.error(self._safe_print(msg))
+
+    def info_box(self, text: str, color_part: str = None, width: int = 80):
+        """創建帶框的消息，可選擇性地為部分文本著色"""
+        # 處理文本
+        display_text = text.replace(color_part, " " * len(color_part)) if color_part else text
+
+        # 計算padding
+        left_padding = " " * ((width - len(display_text)) // 2)
+        right_padding = " " * (width - len(display_text) - len(left_padding))
+
+        # 處理顏色
+        if color_part and self.color_support:
+            content = text.replace(color_part, f"{self.CYAN}{color_part}{self.RESET}")
+        else:
+            content = text
+
+        __content__ = f"{left_padding}{content}{right_padding}"
+
+        # 使用當前設置的框架字符
+        self.info(f"{self.BOX_TOP_LEFT}{self.BOX_HORIZONTAL * width}{self.BOX_TOP_RIGHT}")
+        self.info(f"{self.BOX_VERTICAL}{__content__}{self.BOX_VERTICAL}")
+        self.info(f"{self.BOX_BOTTOM_LEFT}{self.BOX_HORIZONTAL * width}{self.BOX_BOTTOM_RIGHT}")

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/AE33.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         if file.stat().st_size / 1024 < 550:
-            self.logger.info(f'\t {file} may not be a whole daily data. Make sure the file is correct.')
+            self.logger.info(f'\t {file.name} may not be a whole daily data. Make sure the file is correct.')
 
         _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
                          delimiter=r'\s+', skiprows=5, usecols=range(67))

aeroviz-0.1.13/AeroViz/rawDataReader/script/APS.py

@@ -0,0 +1,44 @@
+import numpy as np
+from pandas import to_datetime, read_table
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'APS'
+
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']},
+                             date_format='%m/%d/%y %H:%M:%S').set_index('Time')
+
+            # 542 nm ~ 1981 nm
+            _df = _df.iloc[:, 3:54].rename(columns=lambda x: round(float(x), 4))
+
+            _df_idx = to_datetime(_df.index, format='%m/%d/%y %H:%M:%S', errors='coerce')
+
+        return _df.set_index(_df_idx).loc[_df_idx.dropna()]
+
+    # QC data
+    def _QC(self, _df):
+        _df = _df.copy()
+        _index = _df.index.copy()
+
+        # mask out the data size lower than 7
+        _df.loc[:, 'total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
+
+        hourly_counts = (_df['total']
+                         .dropna()
+                         .resample('h')
+                         .size()
+                         .resample('6min')
+                         .ffill()
+                         .reindex(_df.index, method='ffill', tolerance='6min'))
+
+        # Remove data with less than 6 data per hour
+        _df = _df.mask(hourly_counts < 6)
+
+        # remove total conc. lower than 700 or lower than 1
+        _df = _df.mask((_df['total'] > 700) | (_df['total'] < 1))
+
+        return _df[_df.keys()[:-1]]

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/Minion.py

@@ -187,6 +187,8 @@ class Reader(AbstractReader):
                                   ['Cl-', 'NO2-', 'NO3-', 'SO42-'],
                                   ['SO42-', 'NO3-', 'NH4+'])
 
+        CA_range = ()  # CA, AC Q3=1.5 * IQR
+
         _df['+_mole'] = _df[_cation].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1, skipna=True)
         _df['-_mole'] = _df[_anion].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
 

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/OCEC.py

@@ -9,12 +9,12 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3)
+            _df = read_csv(f, skiprows=3, on_bad_lines='skip')
 
             _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
             _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
 
-            if _df['time'].isna().any():
+            if _df['time'].isna().all():
                 _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %H:%M:%S', errors='coerce')
 
             _df = _df.set_index('time')
@@ -73,8 +73,8 @@ class Reader(AbstractReader):
 
     # QC data
     def _QC(self, _df):
-        MDL = {'Thermal_OC': 0.3,
-               'Optical_OC': 0.3,
+        MDL = {'Thermal_OC': 0.3,  # 0.89
+               'Optical_OC': 0.3,  # 0.08
                'Thermal_EC': 0.015,
                'Optical_EC': 0.015
                }

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/SMPS.py

@@ -59,25 +59,42 @@ class Reader(AbstractReader):
             if _df_smps.columns[0] != self.size_range[0] or _df_smps.columns[-1] != self.size_range[1]:
                 self.logger.info(f'\tSMPS file: {file.name} is not match the default size range {self.size_range}, '
                                  f'it is ({_df_smps.columns[0]}, {_df_smps.columns[-1]})')
+                return None
 
             return _df_smps.apply(to_numeric, errors='coerce')
 
     # QC data
     def _QC(self, _df):
+        _df = _df.copy()
+        _index = _df.index.copy()
+
         size_range_mask = (_df.columns.astype(float) >= self.size_range[0]) & (
-                    _df.columns.astype(float) <= self.size_range[1])
+                _df.columns.astype(float) <= self.size_range[1])
         _df = _df.loc[:, size_range_mask]
 
         # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
-        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-        _df = _df.mask(_df_size < 7)
+        _df.loc[:, 'total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.columns[:-1].to_numpy(float)))).mean()
+
+        hourly_counts = (_df['total']
+                         .dropna()
+                         .resample('h')
+                         .size()
+                         .resample('6min')
+                         .ffill()
+                         .reindex(_df.index, method='ffill', tolerance='6min'))
 
-        # remove total conc. lower than 2000
+        # Remove data with less than 6 data per hour
+        _df = _df.mask(hourly_counts < 6)
+
+        # remove total conc. (dN) lower than 2000
         _df = _df.mask(_df['total'] < 2000)
+        _df = _df.drop('total', axis=1)
+
+        # remove single bin conc. (dN/dlogdp) larger than 1e6
+        _df = _df.mask((_df > 1e6).any(axis=1))
 
         # remove the bin over 400 nm which num. conc. larger than 4000
-        _df_remv_ky = _df.keys()[:-1][_df.keys()[:-1] >= 400.]
-        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
+        large_bins = _df.columns[_df.columns.astype(float) >= 400]
+        _df = _df.mask((_df[large_bins] > 4000).any(axis=1))
 
-        return _df[_df.keys()[:-1]]
+        return _df

{aeroviz-0.1.11 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/__init__.py

@@ -2,7 +2,7 @@ __all__ = [
     'NEPH',
     'Aurora',
     'SMPS',
-    'APS_3321',
+    'APS',
     'GRIMM',
     'AE33',
     'AE43',