AeroViz 0.1.21__py3-none-any.whl

AeroViz/rawDataReader/script/NEPH.py

@@ -0,0 +1,199 @@
+import pandas as pd
+from pandas import to_datetime, read_csv, to_numeric, Series
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+from AeroViz.rawDataReader.core.pre_process import _scaCoe
+
+
+class Reader(AbstractReader):
+    """Nephelometer (NEPH) Data Reader
+
+    A specialized reader for integrating nephelometer data files, which measure
+    light scattering properties of aerosols at multiple wavelengths.
+
+    See full documentation at docs/source/instruments/NEPH.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'NEPH'
+
+    # =========================================================================
+    # Column Definitions
+    # =========================================================================
+    SCAT_COLUMNS = ['B', 'G', 'R', 'BB', 'BG', 'BR']
+    CAL_COLUMNS = ['sca_550', 'SAE']
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_SCAT_VALUE = 0       # Minimum scattering coefficient (Mm^-1)
+    MAX_SCAT_VALUE = 2000    # Maximum scattering coefficient (Mm^-1)
+
+    # Status Flag
+    STATUS_COLUMN = 'status'
+    STATUS_OK = 0  # Status code 0 means normal operation
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw Nephelometer data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the Nephelometer data file.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Processed Nephelometer data with datetime index and scattering coefficient columns.
+        """
+        _df = read_csv(file, header=None, names=range(11))
+
+        _df_grp = _df.groupby(0)
+
+        # T : time
+        _idx_tm = to_datetime(
+            _df_grp.get_group('T')[[1, 2, 3, 4, 5, 6]]
+            .map(lambda x: f"{int(x):02d}")
+            .agg(''.join, axis=1),
+            format='%Y%m%d%H%M%S'
+        )
+
+        # D : data
+        # col : 3~8 B G R BB BG BR
+        # 1e6
+        try:
+            _df_dt = _df_grp.get_group('D')[[1, 2, 3, 4, 5, 6, 7, 8]].set_index(_idx_tm)
+
+            try:
+                _df_out = (_df_dt.groupby(1).get_group('NBXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+            except KeyError:
+                _df_out = (_df_dt.groupby(1).get_group('NTXX')[[3, 4, 5, 6, 7, 8]] * 1e6).reindex(_idx_tm)
+
+            _df_out.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR']
+            _df_out.index.name = 'Time'
+
+            # Y : state
+            # col : 5 RH, col : 9 status
+            _df_st = _df_grp.get_group('Y')
+            _df_out['RH'] = _df_st[5].values
+            status_values = to_numeric(_df_st[9].values, errors='coerce').astype('Int64')
+
+            _df = _df_out[['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']].apply(to_numeric, errors='coerce')
+            # Include status as a column (will be processed by core together)
+            _df[self.STATUS_COLUMN] = status_values
+            _df = _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+            return _df
+
+        except ValueError:  # Define valid groups and find invalid indices
+            invalid_indices = _df[~_df[0].isin({'B', 'G', 'R', 'D', 'T', 'Y', 'Z'})].index
+            self.logger.warning(
+                f"\tInvalid values in {file.name}: {', '.join(f'{_}:{_df.at[_, 0]}' for _ in invalid_indices)}."
+                f" Skipping file.")
+
+            return None
+
+    def _QC(self, _df):
+        """
+        Perform quality control on Nephelometer raw data.
+
+        QC Rules Applied (raw data only)
+        ---------------------------------
+        1. Status Error      : Non-zero status code indicates instrument error
+        2. No Data           : All scattering columns are NaN
+        3. Invalid Scat Value: Scattering coefficient outside 0-2000 Mm^-1
+        4. Invalid Scat Rel. : Wavelength dependence violation (B < G < R)
+        5. Insufficient      : Less than 50% hourly data completeness
+
+        Note: SAE calculation is done in _process() after QC.
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Identify rows with all data missing (handled separately)
+        all_missing_mask = df_qc[self.SCAT_COLUMNS].isna().all(axis=1)
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+
+        qc.add_rules([
+            QCRule(
+                name='Status Error',
+                condition=lambda df: self.QC_control().filter_error_status(
+                    _df, status_column=self.STATUS_COLUMN, status_type='numeric', ok_value=self.STATUS_OK
+                ),
+                description=f'Status code is not {self.STATUS_OK} (non-zero indicates error)'
+            ),
+            QCRule(
+                name='No Data',
+                condition=lambda df: Series(all_missing_mask, index=df.index),
+                description='All scattering columns are NaN'
+            ),
+            QCRule(
+                name='Invalid Scat Value',
+                condition=lambda df: ((df[self.SCAT_COLUMNS] <= self.MIN_SCAT_VALUE) |
+                                      (df[self.SCAT_COLUMNS] > self.MAX_SCAT_VALUE)).any(axis=1),
+                description=f'Scattering coefficient outside {self.MIN_SCAT_VALUE}-{self.MAX_SCAT_VALUE} Mm^-1'
+            ),
+            QCRule(
+                name='Invalid Scat Rel',
+                condition=lambda df: (df['B'] < df['G']) & (df['G'] < df['R']),
+                description='Wavelength dependence violation (Blue < Green < Red)'
+            ),
+            QCRule(
+                name='Insufficient',
+                condition=lambda df: self.QC_control().hourly_completeness_QC(
+                    df[self.SCAT_COLUMNS], freq=self.meta['freq']
+                ),
+                description='Less than 50% hourly data completeness'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Store QC summary for combined output in _process()
+        self._qc_summary = qc.get_summary(df_qc)
+
+        return df_qc.reindex(_index)
+
+    def _process(self, _df):
+        """
+        Calculate scattering coefficients and SAE.
+
+        Processing Steps
+        ----------------
+        1. Calculate scattering coefficient at 550nm
+        2. Calculate SAE (Scattering Ångström Exponent)
+
+        Parameters
+        ----------
+        _df : pd.DataFrame
+            Quality-controlled DataFrame with scattering columns and QC_Flag
+
+        Returns
+        -------
+        pd.DataFrame
+            DataFrame with sca_550, SAE, and updated QC_Flag
+        """
+        _index = _df.index.copy()
+
+        # Calculate SAE and scattering at 550nm
+        _df_cal = _scaCoe(_df[self.SCAT_COLUMNS], instru=self.nam, specified_band=[550])
+
+        # Combine with RH and QC_Flag
+        if 'RH' in _df.columns:
+            df_out = pd.concat([_df_cal, _df[['RH', 'QC_Flag']]], axis=1)
+        else:
+            df_out = pd.concat([_df_cal, _df[['QC_Flag']]], axis=1)
+
+        # Log QC summary
+        if hasattr(self, '_qc_summary') and self._qc_summary is not None:
+            self.logger.info(f"{self.nam} QC Summary:")
+            for _, row in self._qc_summary.iterrows():
+                self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        return df_out.reindex(_index)

AeroViz/rawDataReader/script/OCEC.py

@@ -0,0 +1,173 @@
+import numpy as np
+from pandas import to_datetime, read_csv, to_numeric, Series
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+
+
+class Reader(AbstractReader):
+    """OC/EC (Organic Carbon/Elemental Carbon) Analyzer Data Reader
+
+    A specialized reader for OC/EC analyzer data files, which measure carbonaceous
+    aerosol composition using thermal and optical methods.
+
+    See full documentation at docs/source/instruments/OCEC.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'OCEC'
+
+    # =========================================================================
+    # Column Definitions
+    # =========================================================================
+    OUTPUT_COLUMNS = ['Thermal_OC', 'Thermal_EC', 'Optical_OC', 'Optical_EC', 'TC',
+                      'OC1', 'OC2', 'OC3', 'OC4', 'PC']
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_VALUE = -5       # Minimum valid value (ugC/m3)
+    MAX_VALUE = 100      # Maximum valid value (ugC/m3)
+
+    # Detection limits (MDL) for each carbon fraction
+    MDL = {
+        'Thermal_OC': 0.3,
+        'Optical_OC': 0.3,
+        'Thermal_EC': 0.015,
+        'Optical_EC': 0.015
+    }
+
+    def _raw_reader(self, file):
+        """
+        Read and parse raw OC/EC data files.
+
+        Parameters
+        ----------
+        file : Path or str
+            Path to the OC/EC data file.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Processed OC/EC data with datetime index and carbon fraction columns.
+        """
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            _df = read_csv(f, skiprows=3, on_bad_lines='skip')
+
+            _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
+            _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
+
+            if _df['time'].isna().all():
+                _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %H:%M:%S', errors='coerce')
+
+            _df = _df.set_index('time')
+
+            _df = _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+            _df.index = _df.index.round('1h')
+
+            _df = _df.rename(columns={
+                'Thermal/Optical OC (ugC/LCm^3)': 'Thermal_OC',
+                'Thermal/Optical EC (ugC/LCm^3)': 'Thermal_EC',
+                'OC=TC-BC (ugC/LCm^3)': 'Optical_OC',
+                'BC (ugC/LCm^3)': 'Optical_EC',
+                'TC (ugC/LCm^3)': 'TC',
+
+                'OC ugC/m^3 (Thermal/Optical)': 'Thermal_OC',
+                'EC ugC/m^3 (Thermal/Optical)': 'Thermal_EC',
+                'OC by diff ugC (TC-OptEC)': 'Optical_OC',
+                'OptEC ugC/m^3': 'Optical_EC',
+                'TC ugC/m^3': 'TC',
+
+                'Sample Volume Local Condition Actual m^3': 'Sample_Volume',
+
+                'OCPk1-ug C': 'OC1_raw',
+                'OCPk2-ug C': 'OC2_raw',
+                'OCPk3-ug C': 'OC3_raw',
+                'OCPk4-ug C': 'OC4_raw',
+                'Pyrolized C ug': 'PC_raw',
+
+                'ECPk1-ug C': 'EC1_raw',
+                'ECPk2-ug C': 'EC2_raw',
+                'ECPk3-ug C': 'EC3_raw',
+                'ECPk4-ug C': 'EC4_raw',
+                'ECPk5-ug C': 'EC5_raw',
+            })
+
+            _df = _df.apply(to_numeric, errors='coerce')
+
+            _df['OC1'] = _df['OC1_raw'] / _df['Sample_Volume']
+            _df['OC2'] = _df['OC2_raw'] / _df['Sample_Volume']
+            _df['OC3'] = _df['OC3_raw'] / _df['Sample_Volume']
+            _df['OC4'] = _df['OC4_raw'] / _df['Sample_Volume']
+
+            _df['PC'] = _df['Thermal_OC'] - _df['OC1'] - _df['OC2'] - _df['OC3'] - _df['OC4']
+
+            # _df['EC1'] = _df['EC1_raw'] / _df['Sample_Volume']
+            # _df['EC2'] = _df['EC2_raw'] / _df['Sample_Volume']
+            # _df['EC3'] = _df['EC3_raw'] / _df['Sample_Volume']
+            # _df['EC4'] = _df['EC4_raw'] / _df['Sample_Volume']
+            # _df['EC5'] = _df['EC5_raw'] / _df['Sample_Volume']
+
+            _df = _df[['Thermal_OC', 'Thermal_EC', 'Optical_OC', 'Optical_EC', 'TC', 'Sample_Volume',
+                       'OC1', 'OC2', 'OC3', 'OC4', 'PC']]
+
+            return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        """
+        Perform quality control on OC/EC data.
+
+        QC Rules Applied
+        ----------------
+        1. Invalid Carbon  : Carbon value outside valid range (-5-100 ugC/m3)
+        2. Below MDL       : Value below method detection limit
+        3. Spike           : Sudden value change (vectorized spike detection)
+        4. Missing OC      : Thermal_OC or Optical_OC is missing
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Pre-calculate MDL mask (below detection limit)
+        mdl_mask = Series(False, index=df_qc.index)
+        for col, threshold in self.MDL.items():
+            if col in df_qc.columns:
+                mdl_mask = mdl_mask | (df_qc[col] <= threshold)
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+        qc.add_rules([
+            QCRule(
+                name='Invalid Carbon',
+                condition=lambda df: ((df[self.OUTPUT_COLUMNS] <= self.MIN_VALUE) |
+                                      (df[self.OUTPUT_COLUMNS] > self.MAX_VALUE)).any(axis=1),
+                description=f'Carbon value outside valid range ({self.MIN_VALUE}-{self.MAX_VALUE} ugC/m3)'
+            ),
+            QCRule(
+                name='Below MDL',
+                condition=lambda df: mdl_mask.reindex(df.index).fillna(False),
+                description='Value below method detection limit'
+            ),
+            QCRule(
+                name='Spike',
+                condition=lambda df: self.QC_control().spike_detection(
+                    df[['Thermal_OC', 'Thermal_EC', 'Optical_OC', 'Optical_EC']],
+                    max_change_rate=3.0
+                ),
+                description='Sudden unreasonable value change detected'
+            ),
+            QCRule(
+                name='Missing OC',
+                condition=lambda df: df['Thermal_OC'].isna() | df['Optical_OC'].isna(),
+                description='Missing Thermal_OC or Optical_OC'
+            ),
+        ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Log QC summary
+        summary = qc.get_summary(df_qc)
+        self.logger.info(f"{self.nam} QC Summary:")
+        for _, row in summary.iterrows():
+            self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        return df_qc[self.OUTPUT_COLUMNS + ['QC_Flag']].reindex(_index)

AeroViz/rawDataReader/script/Q-ACSM.py

@@ -0,0 +1,12 @@
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    """Q-ACSM Data Reader
+
+    A specialized reader for Q-ACSM data files.
+
+    See full documentation at docs/source/instruments/Q-ACSM.md for detailed information
+    on supported formats and QC procedures.
+    """
+    nam = 'Q-ACSM'