AeroViz 0.1.21__py3-none-any.whl

AeroViz/rawDataReader/core/logger.py

@@ -0,0 +1,171 @@
+import codecs
+import logging
+import os
+import platform
+import re
+import sys
+from pathlib import Path
+
+
+class ReaderLogger:
+    def __init__(self, name: str, log_path: Path, log_level: str = 'INFO'):
+        self.name = name
+        self.log_path = log_path
+        self._log_level = getattr(logging, log_level)
+
+        # 檢查是否支持顏色輸出
+        self.color_support = self._check_color_support()
+
+        # 設置顏色代碼
+        if self.color_support:
+            self.CYAN = '\033[96m'
+            self.BLUE = '\033[94m'
+            self.GREEN = '\033[92m'
+            self.YELLOW = '\033[93m'
+            self.RED = '\033[91m'
+            self.RESET = '\033[0m'
+        else:
+            self.CYAN = ''
+            self.BLUE = ''
+            self.GREEN = ''
+            self.YELLOW = ''
+            self.RED = ''
+            self.RESET = ''
+
+        # 檢查 Unicode 支持
+        self.unicode_support = self._check_unicode_support()
+
+        # 設置框架字符
+        if self.unicode_support:
+            self.BOX_TOP_LEFT = "╭"
+            self.BOX_TOP_RIGHT = "╮"
+            self.BOX_BOTTOM_LEFT = "╰"
+            self.BOX_BOTTOM_RIGHT = "╯"
+            self.BOX_HORIZONTAL = "─"
+            self.BOX_VERTICAL = "│"
+            self.ARROW = "▶"
+        else:
+            self.BOX_TOP_LEFT = "+"
+            self.BOX_TOP_RIGHT = "+"
+            self.BOX_BOTTOM_LEFT = "+"
+            self.BOX_BOTTOM_RIGHT = "+"
+            self.BOX_HORIZONTAL = "-"
+            self.BOX_VERTICAL = "|"
+            self.ARROW = ">"
+
+        self.logger = self._setup_logger()
+
+    def _check_color_support(self) -> bool:
+        """檢查環境是否支持顏色輸出"""
+        # 檢查是否在 Spyder 或其他 IDE 中運行
+        if any(IDE in os.environ.get('PYTHONPATH', '') for IDE in ['spyder', 'jupyter']):
+            return False
+
+        # 檢查是否強制啟用或禁用顏色
+        if 'FORCE_COLOR' in os.environ:
+            return os.environ['FORCE_COLOR'].lower() in ('1', 'true', 'yes')
+
+        # Windows 檢查
+        if platform.system().lower() == 'windows':
+            return ('ANSICON' in os.environ or
+                    'WT_SESSION' in os.environ or  # Windows Terminal
+                    'ConEmuANSI' in os.environ or
+                    os.environ.get('TERM_PROGRAM', '').lower() == 'vscode')
+
+        # 其他系統檢查
+        return hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
+
+    def _check_unicode_support(self) -> bool:
+        """設置 Unicode 支持"""
+        if platform.system().lower() == 'windows':
+            try:
+                if hasattr(sys.stdout, 'reconfigure'):
+                    sys.stdout.reconfigure(encoding='utf-8')
+                elif hasattr(sys.stdout, 'buffer'):
+                    sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer)
+                else:
+                    return False
+                return True
+            except Exception:
+                return False
+        return True
+
+    def _setup_logger(self) -> logging.Logger:
+        """設置logger"""
+        logger = logging.getLogger(self.name)
+        logger.setLevel(self._log_level)
+
+        # 移除現有的 handlers
+        for handler in logger.handlers[:]:
+            handler.close()
+            logger.removeHandler(handler)
+
+        # 清理 ANSI 格式化器
+        class CleanFormatter(logging.Formatter):
+            def format(self, record):
+                formatted_msg = super().format(record)
+                return re.sub(r'\033\[[0-9;]*m', '', formatted_msg)
+
+        # 設置檔案處理器
+        try:
+            log_dir = Path(self.log_path)
+            log_dir.mkdir(parents=True, exist_ok=True)
+            file_handler = logging.FileHandler(
+                log_dir / f'{self.name}.log',
+                encoding='utf-8',
+                errors='replace'
+            )
+            file_handler.setFormatter(
+                CleanFormatter('%(asctime)s - %(message)s',
+                               datefmt='%Y-%m-%d %H:%M:%S')
+            )
+            logger.addHandler(file_handler)
+        except Exception as e:
+            print(f"Warning: Could not set up file logging: {e}")
+
+        # 設置控制台處理器
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(logging.Formatter('%(message)s'))
+        logger.addHandler(console_handler)
+
+        return logger
+
+    def _safe_print(self, text: str) -> str:
+        """安全打印，處理編碼問題"""
+        if not self.unicode_support:
+            text = text.encode('ascii', 'replace').decode('ascii')
+        return text
+
+    def debug(self, msg: str):
+        self.logger.debug(self._safe_print(msg))
+
+    def info(self, msg: str):
+        self.logger.info(self._safe_print(msg))
+
+    def warning(self, msg: str):
+        self.logger.warning(self._safe_print(msg))
+
+    def error(self, msg: str):
+        self.logger.error(self._safe_print(msg))
+
+    def info_box(self, text: str, color_part: str = None, width: int = 80):
+        """創建帶框的消息，可選擇性地為部分文本著色"""
+        # 處理文本
+        display_text = text.replace(color_part, " " * len(color_part)) if color_part else text
+
+        # 計算padding
+        left_padding = " " * ((width - len(display_text)) // 2)
+        right_padding = " " * (width - len(display_text) - len(left_padding))
+
+        # 處理顏色
+        if color_part and self.color_support:
+            content = text.replace(color_part, f"{self.CYAN}{color_part}{self.RESET}")
+        else:
+            content = text
+
+        __content__ = f"{left_padding}{content}{right_padding}"
+
+        # 使用當前設置的框架字符
+        self.info(f"{self.BOX_TOP_LEFT}{self.BOX_HORIZONTAL * width}{self.BOX_TOP_RIGHT}")
+        self.info(f"{self.BOX_VERTICAL}{__content__}{self.BOX_VERTICAL}")
+        self.info(f"{self.BOX_BOTTOM_LEFT}{self.BOX_HORIZONTAL * width}{self.BOX_BOTTOM_RIGHT}")

AeroViz/rawDataReader/core/pre_process.py

@@ -0,0 +1,308 @@
+import numba
+import numpy as np
+import pandas as pd
+
+
+@numba.jit(nopython=True)
+def _angstrom_fit_numba(log_wavelengths, log_values):
+    """
+    Fast implementation of linear fit for Ångström exponent calculation using numba.
+
+    Parameters
+    ----------
+    log_wavelengths : numpy.ndarray
+        Log of wavelengths
+    log_values : numpy.ndarray
+        Log of measurement values
+
+    Returns
+    -------
+    tuple
+        Slope and intercept of the linear fit
+    """
+    n = len(log_wavelengths)
+    sum_x = np.sum(log_wavelengths)
+    sum_y = np.sum(log_values)
+    sum_xy = np.sum(log_wavelengths * log_values)
+    sum_xx = np.sum(log_wavelengths * log_wavelengths)
+
+    # Calculate slope and intercept
+    slope = (n * sum_xy - sum_x * sum_y) / (n * sum_xx - sum_x * sum_x)
+    intercept = (sum_y - slope * sum_x) / n
+
+    return slope, intercept
+
+
+@numba.jit(nopython=True)
+def calculate_bulk_angstrom_numba(abs_values, log_wavelengths):
+    """
+    JIT-compiled function to calculate Ångström exponents for multiple rows.
+
+    Parameters
+    ----------
+    abs_values : numpy.ndarray
+        2D array of absorption values [n_samples, n_wavelengths]
+    log_wavelengths : numpy.ndarray
+        Log of wavelengths
+
+    Returns
+    -------
+    numpy.ndarray
+        Array of [slope, intercept] pairs for each row
+    """
+    n_samples = abs_values.shape[0]
+    results = np.empty((n_samples, 2))
+
+    for i in range(n_samples):
+        row = abs_values[i]
+
+        # Skip rows with zero or negative values
+        if np.any(row <= 0):
+            results[i, 0] = np.nan
+            results[i, 1] = np.nan
+            continue
+
+        log_values = np.log(row)
+        results[i, 0], results[i, 1] = _angstrom_fit_numba(log_wavelengths, log_values)
+
+    return results
+
+
+@numba.jit(nopython=True)
+def calculate_specific_wavelengths_numba(ref_values, ae_values, ratio_factor):
+    """
+    JIT-compiled function to calculate values at specific wavelengths using Ångström relation.
+
+    Parameters
+    ----------
+    ref_values : numpy.ndarray
+        Reference values at reference wavelength
+    ae_values : numpy.ndarray
+        Ångström exponent values (positive)
+    ratio_factor : float
+        Wavelength ratio factor (target_wl / ref_wl)
+
+    Returns
+    -------
+    numpy.ndarray
+        Calculated values at target wavelength
+
+    Notes
+    -----
+    This function implements the Ångström power law relationship:
+    X(λ₂) = X(λ₁) × (λ₂/λ₁)^(-AE)
+
+    where:
+    - X is either absorption or scattering coefficient
+    - AE is the Ångström exponent (AAE for absorption, SAE for scattering)
+    - The negative sign in the exponent reflects that both absorption and
+      scattering coefficients typically decrease with increasing wavelength
+
+    By convention, both AAE and SAE are reported as positive values in the literature,
+    with the negative sign included in the formula. This function expects positive
+    AE values and applies the negative sign internally.
+
+    Typical values:
+    - AAE: 1-2 for black carbon, higher for brown carbon and dust
+    - SAE: 0-4, with ~4 for small particles and ~0 for large particles
+
+    """
+    n_samples = len(ref_values)
+    results = np.empty(n_samples)
+
+    for i in range(n_samples):
+        if np.isnan(ae_values[i]):
+            results[i] = np.nan
+        else:
+            # Note the negative sign to follow the Ångström relation
+            results[i] = ref_values[i] * (ratio_factor ** -ae_values[i])
+
+    return results
+
+
+def _scaCoe(df, instru, specified_band: list):
+    """
+    Calculate scattering coefficients and Ångström exponent for scattering.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Data frame containing scattering measurements
+    instru : str
+        Instrument type ('Neph' or 'Aurora')
+    specified_band : list
+        List of wavelengths to calculate scattering coefficients for
+
+    Returns
+    -------
+    pandas.DataFrame
+        Data frame with scattering coefficients and Ångström exponent
+    """
+    band_Neph = np.array([450, 550, 700])
+    band_Aurora = np.array([450, 525, 635])
+
+    band = band_Neph if instru == 'Neph' else band_Aurora
+
+    # Create mask for valid rows to avoid copying data
+    mask = ~df[['B', 'G', 'R']].isna().any(axis=1)
+
+    # Pre-allocate output DataFrame
+    result_columns = [f'sca_{_band}' for _band in specified_band] + ['SAE']
+    result_df = pd.DataFrame(np.nan, index=df.index, columns=result_columns)
+
+    # Calculate only for valid rows
+    if mask.any():
+        if instru == 'Neph':
+            # For Nephelometer, directly use G column
+            if len(specified_band) == 1 and specified_band[0] == 550:
+                # Common case optimization
+                result_df.loc[mask, f'sca_550'] = df.loc[mask, 'G']
+            else:
+                # Need to extrapolate to other wavelengths
+                bgr_values = df.loc[mask, ['B', 'G', 'R']].values
+                log_band = np.log(band)
+
+                # Calculate SAE using numba function
+                sae_results = calculate_bulk_angstrom_numba(bgr_values, log_band)
+
+                # Use the calculated SAE to get scattering at specified wavelengths
+                for i, wl in enumerate(specified_band):
+                    closest_idx = np.abs(band - wl).argmin()
+                    ref_wl = band[closest_idx]
+                    ref_idx = ['B', 'G', 'R'][closest_idx]
+
+                    # Get reference measurements
+                    ref_values = df.loc[mask, ref_idx].values
+
+                    # Calculate scattering at target wavelength
+                    ratio = wl / ref_wl
+                    result_df.loc[mask, f'sca_{wl}'] = ref_values * (ratio ** -sae_results[:, 0])
+
+                # Store SAE values
+                result_df.loc[mask, 'SAE'] = sae_results[:, 0]
+        else:
+            # For Aurora, calculate using numba-optimized function instead of get_species_wavelength
+            bgr_values = df.loc[mask, ['B', 'G', 'R']].values
+            log_band = np.log(band)
+
+            # Calculate SAE using numba function
+            sae_results = calculate_bulk_angstrom_numba(bgr_values, log_band)
+
+            # Store SAE values
+            result_df.loc[mask, 'SAE'] = sae_results[:, 0]
+
+            # Calculate scattering at specified wavelengths
+            for i, wl in enumerate(specified_band):
+                closest_idx = np.abs(band - wl).argmin()
+                ref_wl = band[closest_idx]
+                ref_idx = ['B', 'G', 'R'][closest_idx]
+
+                # Get reference measurements
+                ref_values = df.loc[mask, ref_idx].values
+
+                # Calculate using the same function as for absorption, but with negative SAE
+                ratio = wl / ref_wl
+                # Note the negative sign for SAE
+                neg_sae_values = -sae_results[:, 0]  # Negative SAE for scattering
+                result_df.loc[mask, f'sca_{wl}'] = calculate_specific_wavelengths_numba(
+                    ref_values, neg_sae_values, ratio)
+
+    # Combine with original data
+    return pd.concat([df, result_df], axis=1)
+
+
+def _absCoe(df, instru, specified_band: list):
+    """
+    Calculate absorption coefficients and Ångström exponent for absorption.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Data frame containing black carbon measurements
+    instru : str
+        Instrument type ('AE33', 'BC1054', or 'MA350')
+    specified_band : list
+        List of wavelengths to calculate absorption coefficients for
+
+    Returns
+    -------
+    pandas.DataFrame
+        Data frame with original data, absorption coefficients,
+        coefficients at specified wavelengths, and Ångström exponent
+    """
+    config = {
+        'AE33': {
+            'band': np.array([370, 470, 520, 590, 660, 880, 950]),
+            'MAE': np.array([18.47, 14.54, 13.14, 11.58, 10.35, 7.77, 7.19]) * 1e-3,
+            'eBC': 'BC6'
+        },
+        'BC1054': {
+            'band': np.array([370, 430, 470, 525, 565, 590, 660, 700, 880, 950]),
+            'MAE': np.array([18.48, 15.90, 14.55, 13.02, 12.10, 11.59, 10.36, 9.77, 7.77, 7.20]) * 1e-3,
+            'eBC': 'BC9'
+        },
+        'MA350': {
+            'band': np.array([375, 470, 528, 625, 880]),
+            'MAE': np.array([24.069, 19.070, 17.028, 14.091, 10.120]) * 1e-3,
+            'eBC': 'BC5'
+        }
+    }
+
+    # Get configuration for the instrument
+    band_config = config[instru]
+
+    # Create mask for valid rows - non-zero and non-NaN
+    mask = ~((df == 0).all(axis=1) | df.isna().any(axis=1))
+
+    # Pre-allocate output columns
+    result_columns = ([f'abs_{_band}' for _band in band_config['band']] +
+                      [f'abs_{_band}' for _band in specified_band] +
+                      ['eBC', 'AAE'])
+    result_df = pd.DataFrame(np.nan, index=df.index, columns=result_columns)
+
+    # Exit early if no valid data
+    if not mask.any():
+        return pd.concat([df, result_df], axis=1)
+
+    # Get valid rows for processing
+    df_valid = df[mask]
+
+    # Calculate absorption coefficients (vectorized)
+    for i, wl in enumerate(band_config['band']):
+        col_name = f'abs_{wl}'
+        if col_name not in result_df.columns:
+            continue
+        result_df.loc[mask, col_name] = df_valid[df_valid.columns[i]] * band_config['MAE'][i]
+
+    # Extract absorption values as array for AAE calculation
+    abs_cols = [f'abs_{wl}' for wl in band_config['band']]
+    abs_values = result_df.loc[mask, abs_cols].values
+
+    # Calculate AAE with numba
+    log_wavelengths = np.log(band_config['band'])
+    aae_results = calculate_bulk_angstrom_numba(abs_values, log_wavelengths)
+
+    # Store AAE values
+    result_df.loc[mask, 'AAE'] = aae_results[:, 0]
+
+    # Calculate absorption at specified wavelengths
+    for target_wl in specified_band:
+        # Find the closest reference wavelength
+        closest_idx = np.abs(band_config['band'] - target_wl).argmin()
+        ref_wl = band_config['band'][closest_idx]
+        ref_col = f'abs_{ref_wl}'
+
+        # Get reference values and AAE
+        ref_values = result_df.loc[mask, ref_col].values
+        aae_values = result_df.loc[mask, 'AAE'].values
+
+        # Calculate using ratio
+        ratio = target_wl / ref_wl
+        result_df.loc[mask, f'abs_{target_wl}'] = calculate_specific_wavelengths_numba(
+            ref_values, aae_values, ratio)
+
+    # Set eBC values
+    result_df.loc[mask, 'eBC'] = df_valid[band_config['eBC']]
+
+    # Combine with original data
+    return pd.concat([df, result_df], axis=1)