AeroViz 0.1.21__py3-none-any.whl

AeroViz/rawDataReader/script/Aurora.py

@@ -0,0 +1,191 @@
+import pandas as pd
+from pandas import Series, concat
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+from AeroViz.rawDataReader.core.pre_process import _scaCoe
+
+
+class Reader(AbstractReader):
+    """Aurora Integrating Nephelometer Data Reader
+
+    A specialized reader for Aurora nephelometer data files, which measure aerosol light
+    scattering properties at multiple wavelengths.
+
+    See full documentation at docs/source/instruments/Aurora.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'Aurora'
+
+    # =========================================================================
+    # Column Definitions
+    # =========================================================================
+    SCAT_COLUMNS = ['B', 'G', 'R', 'BB', 'BG', 'BR']
+    CAL_COLUMNS = ['sca_550', 'SAE']
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_SCAT_VALUE = 0       # Minimum scattering coefficient (Mm^-1)
+    MAX_SCAT_VALUE = 2000    # Maximum scattering coefficient (Mm^-1)
+
+    # Status Flag
+    STATUS_COLUMN = 'Status'  # Common status column names to check
+    STATUS_COLUMNS = ['Status', 'status', 'Error', 'error', 'Flag', 'flag']
+    STATUS_OK = 0  # Status code 0 means normal operation
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw Aurora nephelometer data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the Aurora data file.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Processed Aurora data with datetime index and standardized
+            scattering coefficient columns.
+        """
+        _df = pd.read_csv(file, low_memory=False, index_col=0)
+
+        _df.index = pd.to_datetime(_df.index, errors='coerce')
+        _df.index.name = 'time'
+
+        _df.columns = _df.keys().str.strip(' ')
+
+        # consider another csv format
+        _df = _df.rename(columns={
+            '0°σspB': 'B',
+            '0°σspG': 'G',
+            '0°σspR': 'R',
+            '90°σspB': 'BB',
+            '90°σspG': 'BG',
+            '90°σspR': 'BR',
+            'Blue': 'B',
+            'Green': 'G',
+            'Red': 'R',
+            'B_Blue': 'BB',
+            'B_Green': 'BG',
+            'B_Red': 'BR',
+        })
+
+        # Check for status column (try multiple common names)
+        status_col_name = None
+        for col_name in self.STATUS_COLUMNS:
+            if col_name in _df.columns:
+                status_col_name = col_name
+                break
+
+        _df_out = _df[['B', 'G', 'R', 'BB', 'BG', 'BR']].apply(pd.to_numeric, errors='coerce')
+
+        # Include status column in _df (will be processed by core together)
+        if status_col_name is not None:
+            _df_out[self.STATUS_COLUMN] = pd.to_numeric(_df[status_col_name], errors='coerce').astype('Int64')
+
+        _df_out = _df_out.loc[~_df_out.index.duplicated() & _df_out.index.notna()]
+
+        return _df_out
+
+    def _QC(self, _df):
+        """
+        Perform quality control on Aurora nephelometer raw data.
+
+        QC Rules Applied (raw data only)
+        ---------------------------------
+        1. Status Error      : Non-zero status code indicates instrument error
+        2. No Data           : All scattering columns are NaN
+        3. Invalid Scat Value: Scattering coefficient outside 0-2000 Mm^-1
+        4. Invalid Scat Rel. : Wavelength dependence violation (B < G < R)
+        5. Insufficient      : Less than 50% hourly data completeness
+
+        Note: SAE calculation is done in _process() after QC.
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Identify rows with all data missing (handled separately)
+        all_missing_mask = df_qc[self.SCAT_COLUMNS].isna().all(axis=1)
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+
+        qc.add_rules([
+            QCRule(
+                name='Status Error',
+                condition=lambda df: self.QC_control().filter_error_status(
+                    _df, status_column=self.STATUS_COLUMN, status_type='numeric', ok_value=self.STATUS_OK
+                ),
+                description=f'Status code is not {self.STATUS_OK} (non-zero indicates error)'
+            ),
+            QCRule(
+                name='No Data',
+                condition=lambda df: Series(all_missing_mask, index=df.index),
+                description='All scattering columns are NaN'
+            ),
+            QCRule(
+                name='Invalid Scat Value',
+                condition=lambda df: ((df[self.SCAT_COLUMNS] <= self.MIN_SCAT_VALUE) |
+                                      (df[self.SCAT_COLUMNS] > self.MAX_SCAT_VALUE)).any(axis=1),
+                description=f'Scattering coefficient outside {self.MIN_SCAT_VALUE}-{self.MAX_SCAT_VALUE} Mm^-1'
+            ),
+            QCRule(
+                name='Invalid Scat Rel',
+                condition=lambda df: (df['B'] < df['G']) & (df['G'] < df['R']),
+                description='Wavelength dependence violation (Blue < Green < Red)'
+            ),
+            QCRule(
+                name='Insufficient',
+                condition=lambda df: self.QC_control().hourly_completeness_QC(
+                    df[self.SCAT_COLUMNS], freq=self.meta['freq']
+                ),
+                description='Less than 50% hourly data completeness'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Store QC summary for combined output in _process()
+        self._qc_summary = qc.get_summary(df_qc)
+
+        return df_qc.reindex(_index)
+
+    def _process(self, _df):
+        """
+        Calculate scattering coefficients and SAE.
+
+        Processing Steps
+        ----------------
+        1. Calculate scattering coefficient at 550nm
+        2. Calculate SAE (Scattering Ångström Exponent)
+
+        Parameters
+        ----------
+        _df : pd.DataFrame
+            Quality-controlled DataFrame with scattering columns and QC_Flag
+
+        Returns
+        -------
+        pd.DataFrame
+            DataFrame with sca_550, SAE, and updated QC_Flag
+        """
+        _index = _df.index.copy()
+
+        # Calculate SAE and scattering at 550nm
+        _df_cal = _scaCoe(_df[self.SCAT_COLUMNS], instru=self.nam, specified_band=[550])
+
+        # Combine with QC_Flag
+        df_out = concat([_df_cal, _df[['QC_Flag']]], axis=1)
+
+        # Log QC summary
+        if hasattr(self, '_qc_summary') and self._qc_summary is not None:
+            self.logger.info(f"{self.nam} QC Summary:")
+            for _, row in self._qc_summary.iterrows():
+                self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        return df_out.reindex(_index)

AeroViz/rawDataReader/script/BAM1020.py

@@ -0,0 +1,90 @@
+from pandas import read_csv, to_numeric, NA
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+
+
+class Reader(AbstractReader):
+    """BAM1020 (Beta Attenuation Monitor) Data Reader
+
+    A specialized reader for BAM1020 data files, which measure PM2.5 mass concentration
+    using beta attenuation technology.
+
+    See full documentation at docs/source/instruments/BAM1020.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'BAM1020'
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_CONC = 0       # Minimum PM2.5 concentration (ug/m3)
+    MAX_CONC = 500     # Maximum PM2.5 concentration (ug/m3)
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw BAM1020 data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the BAM1020 data file.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Processed BAM1020 data with datetime index and PM2.5 concentration column.
+        """
+        PM = 'Conc'
+
+        _df = read_csv(file, parse_dates=True, index_col=0, usecols=range(0, 21))
+        _df.rename(columns={'Conc (mg/m3)': PM}, inplace=True)
+
+        # remove data when Conc = 1 or 0
+        _df[PM] = _df[PM].replace(1, NA)
+
+        _df = _df[[PM]].apply(to_numeric, errors='coerce')
+
+        # tranfer unit from mg/m3 to ug/m3
+        _df = _df * 1000
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        """
+        Perform quality control on BAM1020 data.
+
+        QC Rules Applied
+        ----------------
+        1. Invalid Conc    : Concentration outside valid range (0-500 ug/m3)
+        2. Spike           : Sudden value change (vectorized spike detection)
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+        qc.add_rules([
+            QCRule(
+                name='Invalid Conc',
+                condition=lambda df: (df['Conc'] <= self.MIN_CONC) | (df['Conc'] > self.MAX_CONC),
+                description=f'Concentration outside valid range ({self.MIN_CONC}-{self.MAX_CONC} ug/m3)'
+            ),
+            QCRule(
+                name='Spike',
+                condition=lambda df: self.QC_control().spike_detection(
+                    df[['Conc']], max_change_rate=3.0
+                ),
+                description='Sudden unreasonable value change detected'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Log QC summary
+        summary = qc.get_summary(df_qc)
+        self.logger.info(f"{self.nam} QC Summary:")
+        for _, row in summary.iterrows():
+            self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        return df_qc[['Conc', 'QC_Flag']].reindex(_index)

AeroViz/rawDataReader/script/BC1054.py

@@ -0,0 +1,161 @@
+from pandas import read_csv, to_numeric, concat
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+from AeroViz.rawDataReader.core.pre_process import _absCoe
+
+
+class Reader(AbstractReader):
+    """BC1054 Black Carbon Monitor Data Reader
+
+    A specialized reader for BC1054 data files, which measure black carbon
+    concentrations using light absorption at 10 wavelengths.
+
+    See full documentation at docs/source/instruments/BC1054.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'BC1054'
+
+    # =========================================================================
+    # Column Definitions
+    # =========================================================================
+    BC_COLUMNS = ['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']
+    ABS_COLUMNS = ['abs_370', 'abs_430', 'abs_470', 'abs_525', 'abs_565',
+                   'abs_590', 'abs_660', 'abs_700', 'abs_880', 'abs_950']
+    CAL_COLUMNS = ['abs_550', 'AAE', 'eBC']
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_BC = 0           # Minimum BC concentration (ng/m³)
+    MAX_BC = 20000       # Maximum BC concentration (ng/m³)
+    MIN_AAE = 0.7        # Minimum valid AAE (absolute value)
+    MAX_AAE = 2.0        # Maximum valid AAE (absolute value)
+
+    # =========================================================================
+    # Status Error Codes (bitwise flags)
+    # =========================================================================
+    ERROR_STATES = [
+        1,      # Power Failure
+        2,      # Digital Sensor Link Failure
+        4,      # Tape Move Failure
+        8,      # Maintenance
+        16,     # Flow Failure
+        32,     # Automatic Tape Advance
+        64,     # Detector Failure
+        256,    # Sensor Range
+        512,    # Nozzle Move Failure
+        1024,   # SPI Link Failure
+        2048,   # Calibration Audit
+        65536,  # Tape Move
+    ]
+
+    def _raw_reader(self, file):
+        """Read and parse raw BC1054 data files."""
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, parse_dates=True, index_col=0)
+            _df.columns = _df.columns.str.replace(' ', '')
+
+            _df = _df.rename(columns={
+                'BC1(ng/m3)': 'BC1', 'BC2(ng/m3)': 'BC2', 'BC3(ng/m3)': 'BC3',
+                'BC4(ng/m3)': 'BC4', 'BC5(ng/m3)': 'BC5', 'BC6(ng/m3)': 'BC6',
+                'BC7(ng/m3)': 'BC7', 'BC8(ng/m3)': 'BC8', 'BC9(ng/m3)': 'BC9',
+                'BC10(ng/m3)': 'BC10'
+            })
+
+            _df = _df[self.BC_COLUMNS + ['Status']].apply(to_numeric, errors='coerce')
+
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        """
+        Perform quality control on BC1054 raw data.
+
+        QC Rules Applied (raw data only)
+        ---------------------------------
+        1. Duplicate      : Consecutive duplicate rows removed
+        2. Status Error   : Invalid instrument status codes
+        3. Invalid BC     : BC concentration outside 0-20000 ng/m³
+        4. Insufficient   : Less than 50% hourly data completeness
+
+        Note: AAE validation is done in _process() after calculation.
+        """
+        _index = _df.index.copy()
+
+        # Remove consecutive duplicate rows
+        duplicate_rows = _df.eq(_df.shift()).all(axis=1) | _df.eq(_df.shift(-1)).all(axis=1)
+        df_qc = _df[~duplicate_rows].copy()
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+        qc.add_rules([
+            QCRule(
+                name='Status Error',
+                condition=lambda df: self.QC_control().filter_error_status(df, self.ERROR_STATES),
+                description='Invalid instrument status code detected'
+            ),
+            QCRule(
+                name='Invalid BC',
+                condition=lambda df: ((df[self.BC_COLUMNS] <= self.MIN_BC) |
+                                      (df[self.BC_COLUMNS] > self.MAX_BC)).any(axis=1),
+                description=f'BC concentration outside valid range {self.MIN_BC}-{self.MAX_BC} ng/m³'
+            ),
+            QCRule(
+                name='Insufficient',
+                condition=lambda df: self.QC_control().hourly_completeness_QC(
+                    df[self.BC_COLUMNS], freq=self.meta['freq']
+                ),
+                description='Less than 50% hourly data completeness'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Store QC summary for combined output in _process()
+        self._qc_summary = qc.get_summary(df_qc)
+
+        return df_qc.reindex(_index)
+
+    def _process(self, _df):
+        """
+        Calculate absorption coefficients and validate derived parameters.
+
+        Processing Steps
+        ----------------
+        1. Calculate absorption coefficients at each wavelength
+        2. Calculate AAE (Absorption Ångström Exponent)
+        3. Calculate eBC (equivalent Black Carbon)
+        4. Validate AAE range and update QC_Flag
+        """
+        _index = _df.index.copy()
+
+        # Calculate absorption coefficients, AAE, and eBC
+        _df_cal = _absCoe(_df[self.BC_COLUMNS], instru=self.nam, specified_band=[550])
+
+        # Combine with Status and QC_Flag
+        df_out = concat([_df_cal, _df[['Status', 'QC_Flag']]], axis=1)
+
+        # Validate AAE and update QC_Flag
+        invalid_aae = (-df_out['AAE'] < self.MIN_AAE) | (-df_out['AAE'] > self.MAX_AAE)
+        df_out = self.update_qc_flag(df_out, invalid_aae, 'Invalid AAE')
+
+        # Log combined QC summary with calculated info
+        if hasattr(self, '_qc_summary') and self._qc_summary is not None:
+            import pandas as pd
+            # Add Invalid AAE row before Valid row
+            total = len(df_out)
+            invalid_aae_row = pd.DataFrame([{
+                'Rule': 'Invalid AAE',
+                'Count': invalid_aae.sum(),
+                'Percentage': f'{invalid_aae.sum() / total * 100:.1f}%',
+                'Description': f'AAE outside valid range {self.MIN_AAE}-{self.MAX_AAE}'
+            }])
+            # Insert before Valid row (last row)
+            summary = pd.concat([self._qc_summary.iloc[:-1], invalid_aae_row, self._qc_summary.iloc[-1:]], ignore_index=True)
+            self.logger.info(f"{self.nam} QC Summary:")
+            for _, row in summary.iterrows():
+                self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        # Reorder columns
+        all_data_cols = self.BC_COLUMNS + self.ABS_COLUMNS + self.CAL_COLUMNS
+        return df_out[all_data_cols + ['QC_Flag']].reindex(_index)

AeroViz/rawDataReader/script/EPA.py

@@ -0,0 +1,79 @@
+import numpy as np
+from pandas import read_csv, to_numeric
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+
+desired_order1 = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC',
+                  'CH4', 'PM10', 'PM2.5', 'PM1', 'WS', 'WD', 'AT', 'RH']
+
+desired_order2 = ['Benzene', 'Toluene', 'EthylBenzene', 'm/p-Xylene', 'o-Xylene']
+
+
+class Reader(AbstractReader):
+    """EPA Environmental Data Reader
+
+    A specialized reader for EPA air quality monitoring data files.
+
+    See full documentation at docs/source/instruments/EPA.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'EPA'
+
+    def _raw_reader(self, file):
+        # 查詢小時值(測項).csv & 查詢小時值(直式).csv (有、無輸出有效值都可以)
+        df = read_csv(file, encoding='big5', encoding_errors='ignore', index_col=0, parse_dates=True,
+                      on_bad_lines='skip')
+
+        if len(df.groupby('測站')) > 1:
+            raise ValueError(f"Multiple stations found in the file: {df['測站'].unique()}")
+        else:
+            if '測站' in df.columns:
+                df.drop(columns=['測站'], inplace=True)
+
+            if '測項' in df.columns:
+                df = df.pivot(columns='測項', values='資料')
+
+            df.rename(columns={'AMB_TEMP': 'AT', 'WIND_SPEED': 'WS', 'WIND_DIREC': 'WD'}, inplace=True)
+            df.index.name = 'Time'
+
+            # 如果沒有將無效值拿掉就輸出 請將包含 #、L 的字串替換成 # 或 _
+            df = df.replace(to_replace=r'\d*\.?\d*[#]\b', value='#', regex=True)
+            df = df.replace(to_replace=r'\d*\.?\d*[L]\b', value='_', regex=True)
+
+            # 欄位排序
+            return self.reorder_dataframe_columns(df, [desired_order1]).apply(to_numeric, errors='coerce')
+
+    def _QC(self, _df):
+        """
+        Perform quality control on EPA data.
+
+        QC Rules Applied
+        ----------------
+        1. Negative        : Any measurement value < 0
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Get numeric columns for negative value check
+        numeric_cols = df_qc.select_dtypes(include=[np.number]).columns.tolist()
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+        qc.add_rules([
+            QCRule(
+                name='Negative',
+                condition=lambda df: (df[numeric_cols] < 0).any(axis=1) if numeric_cols else False,
+                description='Measurement value is negative (< 0)'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Log QC summary
+        summary = qc.get_summary(df_qc)
+        self.logger.info(f"{self.nam} QC Summary:")
+        for _, row in summary.iterrows():
+            self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        return df_qc.reindex(_index)

AeroViz/rawDataReader/script/GRIMM.py

@@ -0,0 +1,68 @@
+from pandas import to_datetime, read_csv
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    """ GRIMM Aerosol Spectrometer Data Reader
+
+    A specialized reader for GRIMM data files, which measure particle size distributions
+    in the range of 0.25-32 μm.
+
+    See full documentation at docs/source/instruments/GRIMM.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'GRIMM'
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw GRIMM data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the GRIMM data file.
+
+        Returns
+        -------
+        pandas.DataFrame or None
+            Processed GRIMM data with datetime index and size channels as columns.
+            Returns None if the file is empty.
+        """
+        _df = read_csv(file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
+                       dayfirst=True).rename_axis("Time")
+        _df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
+
+        if file.name.startswith("A407ST"):
+            _df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
+        else:
+            _df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
+
+        if _df.empty:
+            print(file, "is empty")
+            return None
+
+        return _df / 0.035
+
+    def _QC(self, _df):
+        """
+        Perform quality control on GRIMM data.
+
+        Parameters
+        ----------
+        _df : pandas.DataFrame
+            Raw GRIMM data with datetime index and size channels as columns.
+
+        Returns
+        -------
+        pandas.DataFrame
+            The input data unchanged.
+
+        Notes
+        -----
+        No QC filters are currently applied. Future implementations could include:
+        1. Value range checks for each size channel
+        2. Total concentration consistency checks
+        3. Time-based outlier detection
+        """
+        return _df