AeroViz 0.1.21__py3-none-any.whl

AeroViz/rawDataReader/script/IGAC.py

@@ -0,0 +1,140 @@
+# read meteorological data from google sheet
+
+
+from pandas import read_csv, to_numeric, Series
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+
+
+class Reader(AbstractReader):
+    """IGAC (In-situ Gas and Aerosol Composition) Monitor Data Reader
+
+    This class handles the reading and parsing of IGAC monitor data files,
+    which provide real-time measurements of water-soluble inorganic ions in
+    particulate matter.
+
+    See full documentation at docs/source/instruments/IGAC.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'IGAC'
+
+    # =========================================================================
+    # Column Definitions
+    # =========================================================================
+    CATION_COLUMNS = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+']
+    ANION_COLUMNS = ['Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-']
+    MAIN_IONS = ['SO42-', 'NO3-', 'NH4+']
+
+    # =========================================================================
+    # Detection Limits (MDL) in ug/m3
+    # =========================================================================
+    MDL = {
+        'Na+': 0.06,
+        'NH4+': 0.05,
+        'K+': 0.05,
+        'Mg2+': 0.12,
+        'Ca2+': 0.07,
+        'Cl-': 0.07,
+        'NO2-': 0.05,
+        'NO3-': 0.11,
+        'SO42-': 0.08,
+    }
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw IGAC monitor data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the IGAC data file.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Processed IGAC data with datetime index and ion concentration columns.
+        """
+        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=True, index_col=0, na_values='-')
+
+            _df.columns = _df.keys().str.strip(' ')
+            _df.index.name = 'time'
+
+            _df = _df.apply(to_numeric, errors='coerce')
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        """
+        Perform quality control on IGAC ion composition data.
+
+        QC Rules Applied
+        ----------------
+        1. Mass Closure    : Total ion mass > PM2.5 mass
+        2. Missing Main    : Main ions (NH4+, SO42-, NO3-) not present
+        3. Below MDL       : Ion concentration below detection limit
+        4. Ion Balance     : Cation/Anion ratio outside valid range
+        """
+        _index = _df.index.copy()
+
+        # Get ion columns that exist in the data
+        ion_columns = [col for col in self.MDL.keys() if col in _df.columns]
+        df_qc = _df[ion_columns].copy()
+
+        # Calculate total ion mass for mass closure check
+        total_ions = df_qc.sum(axis=1, min_count=1)
+        pm25 = _df['PM2.5'] if 'PM2.5' in _df.columns else Series(float('inf'), index=_df.index)
+
+        # Calculate cation/anion ratio for ion balance check
+        cation_cols = [c for c in self.CATION_COLUMNS if c in df_qc.columns]
+        anion_cols = [c for c in self.ANION_COLUMNS if c in df_qc.columns]
+        cation_sum = df_qc[cation_cols].sum(axis=1, min_count=1) if cation_cols else Series(0, index=df_qc.index)
+        anion_sum = df_qc[anion_cols].sum(axis=1, min_count=1) if anion_cols else Series(1, index=df_qc.index)
+        ca_ratio = cation_sum / anion_sum.replace(0, float('nan'))
+
+        # Calculate IQR bounds for ion balance
+        q1, q3 = ca_ratio.quantile(0.25), ca_ratio.quantile(0.75)
+        iqr = q3 - q1
+        ca_lower, ca_upper = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+        qc.add_rules([
+            QCRule(
+                name='Mass Closure',
+                condition=lambda df: total_ions > pm25,
+                description='Total ion mass exceeds PM2.5 mass'
+            ),
+            QCRule(
+                name='Missing Main',
+                condition=lambda df: df[self.MAIN_IONS].isna().any(axis=1) if all(
+                    c in df.columns for c in self.MAIN_IONS) else Series(False, index=df.index),
+                description='Missing main ions (NH4+, SO42-, NO3-)'
+            ),
+            QCRule(
+                name='Below MDL',
+                condition=lambda df: Series(
+                    [any(df.loc[idx, col] < self.MDL.get(col, 0)
+                         for col in ion_columns if col in df.columns and not Series(df.loc[idx, col]).isna().any())
+                     for idx in df.index],
+                    index=df.index
+                ),
+                description='Ion concentration below detection limit'
+            ),
+            QCRule(
+                name='Ion Balance',
+                condition=lambda df: (ca_ratio < ca_lower) | (ca_ratio > ca_upper) | ca_ratio.isna(),
+                description='Cation/Anion ratio outside valid range'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Log QC summary
+        summary = qc.get_summary(df_qc)
+        self.logger.info(f"{self.nam} QC Summary:")
+        for _, row in summary.iterrows():
+            self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        return df_qc.reindex(_index)

AeroViz/rawDataReader/script/MA350.py

@@ -0,0 +1,179 @@
+from pandas import read_csv, to_numeric, concat, Series
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+from AeroViz.rawDataReader.core.pre_process import _absCoe
+
+
+class Reader(AbstractReader):
+    """MA350 Aethalometer Data Reader
+
+    A specialized reader for MA350 Aethalometer data files, which measure
+    black carbon at multiple wavelengths and provide source apportionment.
+
+    See full documentation at docs/source/instruments/MA350.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'MA350'
+
+    # =========================================================================
+    # Column Definitions
+    # =========================================================================
+    BC_COLUMNS = ['BC1', 'BC2', 'BC3', 'BC4', 'BC5']
+    ABS_COLUMNS = ['abs_375', 'abs_470', 'abs_528', 'abs_625', 'abs_880']
+    CAL_COLUMNS = ['abs_550', 'AAE', 'eBC']
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_BC = 0           # Minimum BC concentration (ng/m³)
+    MAX_BC = 20000       # Maximum BC concentration (ng/m³)
+    MIN_AAE = 0.7        # Minimum valid AAE (absolute value)
+    MAX_AAE = 2.0        # Maximum valid AAE (absolute value)
+
+    # =========================================================================
+    # Status Error Codes (bitwise flags)
+    # =========================================================================
+    ERROR_STATES = [
+        1,       # Power Failure
+        2,       # Start up
+        4,       # Tape advance
+        16,      # Optical saturation
+        32,      # Sample timing error
+        128,     # Flow unstable
+        256,     # Pump drive limit
+        2048,    # System busy
+        8192,    # Tape jam
+        16384,   # Tape at end
+        32768,   # Tape not ready
+        65536,   # Tape transport not ready
+        262144,  # Invalid date/time
+        524288,  # Tape error
+    ]
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw MA350 Aethalometer data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the MA350 data file.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Processed MA350 data with datetime index and standardized black carbon
+            and source apportionment columns.
+        """
+        _df = read_csv(file, parse_dates=['Date / time local'], index_col='Date / time local').rename_axis(
+            "Time")
+
+        _df = _df.rename(columns={
+            'UV BCc': 'BC1',
+            'Blue BCc': 'BC2',
+            'Green BCc': 'BC3',
+            'Red BCc': 'BC4',
+            'IR BCc': 'BC5',
+            'Biomass BCc  (ng/m^3)': 'BB mass',
+            'Fossil fuel BCc  (ng/m^3)': 'FF mass',
+            'Delta-C  (ng/m^3)': 'Delta-C',
+            'AAE': 'AAE_ref',
+            'BB (%)': 'BB',
+        })
+
+        _df = _df[
+            ['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BB mass', 'FF mass', 'Delta-C', 'AAE_ref', 'BB', 'Status']].apply(
+            to_numeric,
+            errors='coerce')
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        """
+        Perform quality control on MA350 Aethalometer raw data.
+
+        QC Rules Applied (raw data only)
+        ---------------------------------
+        1. Status Error   : Invalid instrument status codes
+        2. Invalid BC     : BC concentration outside 0-20000 ng/m³
+        3. Insufficient   : Less than 50% hourly data completeness
+
+        Note: AAE validation is done in _process() after calculation.
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+        qc.add_rules([
+            QCRule(
+                name='Status Error',
+                condition=lambda df: self.QC_control().filter_error_status(df, self.ERROR_STATES),
+                description='Invalid instrument status code detected'
+            ),
+            QCRule(
+                name='Invalid BC',
+                condition=lambda df: ((df[self.BC_COLUMNS] <= self.MIN_BC) |
+                                      (df[self.BC_COLUMNS] > self.MAX_BC)).any(axis=1),
+                description=f'BC concentration outside valid range {self.MIN_BC}-{self.MAX_BC} ng/m³'
+            ),
+            QCRule(
+                name='Insufficient',
+                condition=lambda df: self.QC_control().hourly_completeness_QC(
+                    df[self.BC_COLUMNS], freq=self.meta['freq']
+                ),
+                description='Less than 50% hourly data completeness'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Store QC summary for combined output in _process()
+        self._qc_summary = qc.get_summary(df_qc)
+
+        return df_qc.reindex(_index)
+
+    def _process(self, _df):
+        """
+        Calculate absorption coefficients and validate derived parameters.
+
+        Processing Steps
+        ----------------
+        1. Calculate absorption coefficients at each wavelength
+        2. Calculate AAE (Absorption Ångström Exponent)
+        3. Calculate eBC (equivalent Black Carbon)
+        4. Validate AAE range and update QC_Flag
+        """
+        _index = _df.index.copy()
+
+        # Calculate absorption coefficients, AAE, and eBC
+        _df_cal = _absCoe(_df[self.BC_COLUMNS], instru=self.nam, specified_band=[550])
+
+        # Combine with Status and QC_Flag
+        df_out = concat([_df_cal, _df[['Status', 'QC_Flag']]], axis=1)
+
+        # Validate AAE and update QC_Flag
+        invalid_aae = (-df_out['AAE'] < self.MIN_AAE) | (-df_out['AAE'] > self.MAX_AAE)
+        df_out = self.update_qc_flag(df_out, invalid_aae, 'Invalid AAE')
+
+        # Log combined QC summary with calculated info
+        if hasattr(self, '_qc_summary') and self._qc_summary is not None:
+            import pandas as pd
+            # Add Invalid AAE row before Valid row
+            total = len(df_out)
+            invalid_aae_row = pd.DataFrame([{
+                'Rule': 'Invalid AAE',
+                'Count': invalid_aae.sum(),
+                'Percentage': f'{invalid_aae.sum() / total * 100:.1f}%',
+                'Description': f'AAE outside valid range {self.MIN_AAE}-{self.MAX_AAE}'
+            }])
+            # Insert before Valid row (last row)
+            summary = pd.concat([self._qc_summary.iloc[:-1], invalid_aae_row, self._qc_summary.iloc[-1:]], ignore_index=True)
+            self.logger.info(f"{self.nam} QC Summary:")
+            for _, row in summary.iterrows():
+                self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        # Reorder columns
+        all_data_cols = self.BC_COLUMNS + self.ABS_COLUMNS + self.CAL_COLUMNS
+        return df_out[all_data_cols + ['QC_Flag']].reindex(_index)

AeroViz/rawDataReader/script/Minion.py

@@ -0,0 +1,218 @@
+from typing import Literal
+
+import numpy as np
+import pandas
+from pandas import DataFrame, read_excel
+
+from AeroViz.rawDataReader.config.supported_instruments import meta
+from AeroViz.rawDataReader.core import AbstractReader
+
+pandas.set_option("future.no_silent_downcasting", True)
+
+desired_order1 = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC',
+                  'CH4', 'PM10', 'PM2.5', 'WS', 'WD', 'AT', 'RH']
+
+desired_order2 = ['Benzene', 'Toluene', 'EthylBenzene', 'm/p-Xylene', 'o-Xylene']
+
+MDL_NUMBER = -999
+
+
+class Reader(AbstractReader):
+    nam = 'Minion'
+
+    # 楠梓8月數據(環境部)(空品、重金屬和氣膠可用率) -> 楠梓8月數據_level1 -> NZ_minion_XXXX
+    def _raw_reader(self, file):
+        df = read_excel(file, index_col=0, parse_dates=True)
+        df.index.name = 'Time'
+
+        # 重命名列，去除空白
+        df = df.rename(columns=lambda x: x.strip())
+
+        # 保存單位
+        self.units = df.iloc[0].copy()
+
+        # 刪除原始數據中的單位行
+        df = df.iloc[1:]
+
+        # 替換特定值
+        df = df.replace({'維護校正': '*', np.nan: '-', 'Nodata': '-', '0L': MDL_NUMBER})
+        # df = df.replace(to_replace=r'\d*\.?\d*[#]\b', value='_', regex=True)
+        df = df.replace(to_replace=r'\d*\.?\d*[L]\b', value=MDL_NUMBER, regex=True)
+
+        # 處理除了'WD'列的 0 值 替換為 '_'
+        for col in [col for col in df.columns if col != 'WD']:
+            df[col] = df[col].replace({0: MDL_NUMBER})
+
+        # replace to numeric for estimating qc rate
+        df = df.replace({'_': MDL_NUMBER})
+
+        XRF_col = list(meta.get('XRF').get('MDL').keys())
+        IGAC_col = list(meta.get('IGAC').get('MDL').keys())
+
+        # 重新排序列
+        df = self.reorder_dataframe_columns(df, [desired_order1, desired_order2, XRF_col, IGAC_col])
+
+        # 將單位行添加回 DataFrame
+        # df = concat([units.to_frame().T, df])
+
+        # save Level1 data
+        output_folder = file.parent / 'Level1'
+        output_folder.mkdir(parents=True, exist_ok=True)
+        df.to_csv(output_folder / f'{file.stem}_Level1.csv')
+
+        return df.loc[~df.index.duplicated() & df.index.notna()]
+
+    def _QC(self, _df):
+        IGAC_col = list(meta.get('IGAC').get('MDL'))
+        XRF_col = list(meta.get('XRF').get('MDL'))
+
+        # IGAC MDL QC
+        _df[IGAC_col] = self.IGAC_QAQC(_df[IGAC_col])
+
+        # XRF MDL QC
+        _df[XRF_col] = self.XRF_QAQC(_df[XRF_col])
+
+        # remove negative value
+        # _df = _df.mask((_df < 0))
+        _df = _df.mask(_df == MDL_NUMBER, np.nan)
+
+        col = [col for col in desired_order1 if col != 'WD']
+        _df[col] = self.QC_control().time_aware_rolling_iqr(_df[col])
+
+        # Calculate the mass and ion balance
+        # mass tolerance = ± 1, ions balance tolerance = ± 1
+
+        # # conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
+        # _df_salt = df.mask(df.sum(axis=1, min_count=1) > df.PM25).dropna(subset=_main).copy()
+
+        ions_mass = _df[['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'Cl-', 'NO3-', 'SO42-']].sum(axis=1)
+        element_mass = _df[XRF_col].sum(axis=1)
+
+        estimated_mass = ions_mass + element_mass
+
+        valid_mask = 2 * _df['PM2.5'] > estimated_mass
+
+        _df.loc[~valid_mask, IGAC_col + XRF_col] = np.nan
+
+        return _df
+
+    def mdlReplace_timeAware_qc(self, df: DataFrame, MDL: dict, MDL_replace) -> DataFrame:
+        # Step 1: Track MDL positions and values below threshold
+        mdl_mask = (df.eq(MDL_NUMBER) |
+                    df.apply(lambda x: x < MDL.get(x.name, float('-inf'))))
+
+        # Step 2: Convert all values below MDL to MDL_NUMBER (-999)
+        df_mdl = df.mask(mdl_mask, MDL_NUMBER)
+
+        # Step 3: Apply time_aware_IQR_QC (excluding MDL_NUMBER values)
+        df_qc = self.time_aware_IQR_QC(df_mdl.mask(df_mdl == MDL_NUMBER))
+
+        # Step 4: Handle values below MDL according to specified method
+        if MDL_replace == '0.5 * MDL':
+            for column, threshold in MDL.items():
+                if column in df.columns and threshold is not None:
+                    df_qc.loc[df_mdl[column] == MDL_NUMBER, column] = 0.5 * threshold
+                else:
+                    df_qc.loc[df_mdl[column] == MDL_NUMBER, column] = np.nan
+        else:  # 'nan'
+            df_qc = df_qc.mask(df_mdl == MDL_NUMBER, np.nan)
+
+        return df_qc
+
+    def XRF_QAQC(self,
+                 df: DataFrame,
+                 MDL_replace: Literal['nan', '0.5 * MDL'] = '0.5 * MDL'
+                 ) -> DataFrame:
+        """
+        Perform Quality Assurance and Quality Control for XRF data
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input dataframe with XRF data
+        MDL_replace : {'nan', '0.5 * MDL'}, default='nan'
+            Method to handle values below MDL:
+            - 'nan': Replace with NaN
+            - '0.5 * MDL': Replace with half of MDL value
+
+        Returns
+        -------
+        pd.DataFrame
+            Processed dataframe with QC applied and MDL values handled
+        """
+        MDL = meta.get('XRF').get('MDL')
+
+        df = self.mdlReplace_timeAware_qc(df, MDL, MDL_replace)
+
+        # 轉換單位 ng/m3 -> ug/m3
+        if df.Al.max() > 10 and df.Fe.max() > 10:
+            columns_to_convert = [col for col in MDL.keys() if col in df.columns]
+            df[columns_to_convert] = df[columns_to_convert].div(1000)
+
+        self.logger.info("")
+        self.logger.info(f"XRF QAQC: values below MDL -> {MDL_replace}")
+
+        return df
+
+    def IGAC_QAQC(self,
+                  df: DataFrame,
+                  MDL_replace: Literal['nan', '0.5 * MDL'] = '0.5 * MDL',
+                  tolerance: float = 1
+                  ) -> DataFrame:
+        """
+        Perform Quality Assurance and Quality Control for IGAC data
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            Input dataframe with IGAC data
+        MDL_replace : {'nan', '0.5 * MDL'}, default='nan'
+            Method to handle values below MDL:
+            - 'nan': Replace with NaN
+            - '0.5 * MDL': Replace with half of MDL value
+        tolerance : float, default=1
+            Tolerance value for QC checks
+
+        Returns
+        -------
+        pd.DataFrame
+            Processed dataframe with QC applied and MDL values handled
+        """
+        MDL = meta.get('IGAC').get('MDL')
+
+        df = self.mdlReplace_timeAware_qc(df, MDL, MDL_replace)
+
+        # Define the ions
+        _df = df.copy()
+        _cation, _anion, _main = (['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'],
+                                  ['Cl-', 'NO2-', 'NO3-', 'SO42-'],
+                                  ['SO42-', 'NO3-', 'NH4+'])
+
+        CA_range = ()  # CA, AC Q3=1.5 * IQR
+
+        _df['+_mole'] = _df[_cation].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1, skipna=True)
+        _df['-_mole'] = _df[_anion].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
+
+        # Avoid division by zero
+        _df['ratio'] = np.where(_df['-_mole'] != 0, _df['+_mole'] / _df['-_mole'], np.nan)
+
+        # Calculate bounds
+        lower_bound, upper_bound = 1 - tolerance, 1 + tolerance
+
+        # 根據ratio决定是否保留原始数据
+        valid_mask = ((_df['ratio'] <= upper_bound) & (_df['ratio'] >= lower_bound) &
+                      ~np.isnan(_df['+_mole']) & ~np.isnan(_df['-_mole']))
+
+        # 保留数據或將不符合的條件設為NaN
+        df.loc[~valid_mask] = np.nan
+
+        # 計算保留的数據的百分比
+        retained_percentage = (valid_mask.sum() / len(df)) * 100
+
+        self.logger.info("")
+        self.logger.info(f"Ions balance: {retained_percentage.__round__(0)}% within tolerance (±{tolerance})")
+
+        if retained_percentage < 70:
+            self.logger.warning("Warning: retained data < 70%")
+
+        return df