AeroViz 0.1.21__py3-none-any.whl

AeroViz/rawDataReader/script/Xact.py

@@ -0,0 +1,244 @@
+from pandas import read_csv, to_datetime, to_numeric
+
+from AeroViz.rawDataReader.core import AbstractReader, QCRule, QCFlagBuilder
+
+
+class Reader(AbstractReader):
+    """Xact 625i XRF Analyzer Data Reader
+
+    A specialized reader for Xact 625i continuous XRF analyzer data files,
+    which measure elemental composition of particulate matter.
+    """
+    nam = 'Xact'
+
+    # Element symbols with atomic numbers (extracted from column headers)
+    ELEMENTS = [
+        'Mg', 'Al', 'Si', 'P', 'S', 'Cl', 'Ar', 'K', 'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn', 'Fe',
+        'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Rb', 'Sr', 'Y', 'Zr', 'Nb', 'Mo',
+        'Ru', 'Rh', 'Pd', 'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te', 'I', 'Cs', 'Ba', 'La', 'Ce',
+        'Pr', 'Nd', 'Pm', 'Sm', 'Eu', 'Gd', 'Tb', 'Dy', 'Ho', 'Er', 'Tm', 'Yb', 'Lu',
+        'Hf', 'Ta', 'W', 'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi', 'Th', 'Pa', 'U'
+    ]
+
+    # Environmental/status columns to keep
+    ENV_COLUMNS = [
+        'AT', 'SAMPLE_T', 'BP', 'TAPE', 'FLOW_25', 'FLOW_ACT', 'FLOW_STD', 'VOLUME',
+        'TUBE_T', 'ENCLOSURE_T', 'FILAMENT_V', 'SDD_T', 'DPP_T', 'RH',
+        'WIND', 'WIND_DIR', 'SAMPLE_TIME', 'ALARM', 'SAMPLE_TYPE'
+    ]
+
+    # =========================================================================
+    # Alarm Code Definitions
+    # =========================================================================
+    # Error codes (100-110) - indicate instrument malfunction, invalidate data
+    ERROR_CODES = {
+        100: 'Xray Voltage Error',
+        101: 'Xray Current Error',
+        102: 'Tube Temperature Error',
+        103: 'Enclosure Temperature Error',
+        104: 'Tape Error',
+        105: 'Pump Error',
+        106: 'Filter Wheel Error',
+        107: 'Dynamic Rod Error',
+        108: 'Nozzle Error',
+        109: 'Energy Calibration Error',
+        110: 'Software Error',
+    }
+
+    # Warning codes (200-203) - indicate upscale warnings
+    WARNING_CODES = {
+        200: 'Upscale Cr Warning',
+        201: 'Upscale Pb Warning',
+        202: 'Upscale Cd Warning',
+        203: 'Upscale Nb Warning',
+    }
+
+    # =========================================================================
+    # QC Thresholds
+    # =========================================================================
+    MIN_VALUE = 0
+    MAX_VALUE = 100000  # ng/m3
+
+    # Internal standard (Nb) QC parameters
+    INTERNAL_STD_ELEMENT = 'Nb'
+    INTERNAL_STD_TOLERANCE = 0.20  # ±20% from median
+
+    def _raw_reader(self, file):
+        """Read and parse raw Xact 625i XRF data files."""
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            f.readline()  # skip row 0 (element names)
+            headers = f.readline().strip().split(',')
+            headers.append('_extra_')  # data has one extra field at end
+            _df = read_csv(f, names=headers, on_bad_lines='skip')
+
+        # Parse time column
+        _df['time'] = to_datetime(_df['TIME'], format='%m/%d/%Y %H:%M:%S', errors='coerce')
+        _df = _df.set_index('time')
+        _df = _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+        # Filter out calibration samples BEFORE rounding to avoid losing valid 00:30 samples
+        # Xact does daily QA checks at midnight (00:00-00:30), SAMPLE_TYPE: 1=normal, 2=calibration
+        if 'Sample Type' in _df.columns:
+            _df = _df[_df['Sample Type'] == 1]
+
+        _df.index = _df.index.round('1h')
+
+        # Rename environmental/status columns
+        rename_map = {
+            'AT (C)': 'AT',
+            'SAMPLE (C)': 'SAMPLE_T',
+            'BP (mmHg)': 'BP',
+            'TAPE (mmHg)': 'TAPE',
+            'FLOW 25 (slpm)': 'FLOW_25',
+            'FLOW ACT (lpm)': 'FLOW_ACT',
+            'FLOW STD (slpm)': 'FLOW_STD',
+            'VOLUME (L)': 'VOLUME',
+            'TUBE (C)': 'TUBE_T',
+            'ENCLOSURE (C)': 'ENCLOSURE_T',
+            'FILAMENT (V)': 'FILAMENT_V',
+            'SDD (C)': 'SDD_T',
+            'DPP (C)': 'DPP_T',
+            'RH (%)': 'RH',
+            'WIND (m/s)': 'WIND',
+            'WIND DIR (deg)': 'WIND_DIR',
+            'SAMPLE TIME (min)': 'SAMPLE_TIME',
+            'ALARM': 'ALARM',
+            'Sample Type': 'SAMPLE_TYPE'
+        }
+
+        # Build element column rename map
+        for col in _df.columns:
+            for elem in self.ELEMENTS:
+                # Match pattern like "Mg 12 (ng/m3)" or " K 19 (ng/m3)" for concentration
+                if f'{elem} ' in col and '(ng/m3)' in col and 'uncert' not in col.lower():
+                    rename_map[col] = elem
+                # Match pattern like "Al Uncert (ng/m3)" or "Mg uncert (ng/m3)" for uncertainty
+                elif f'{elem} ' in col and 'uncert' in col.lower():
+                    rename_map[col] = f'{elem}_uncert'
+
+        _df = _df.rename(columns=rename_map)
+
+        # Select columns to keep (elements + uncertainties + environmental)
+        keep_cols = []
+        for elem in self.ELEMENTS:
+            if elem in _df.columns:
+                keep_cols.append(elem)
+            if f'{elem}_uncert' in _df.columns:
+                keep_cols.append(f'{elem}_uncert')
+        for env_col in self.ENV_COLUMNS:
+            if env_col in _df.columns:
+                keep_cols.append(env_col)
+
+        _df = _df[[col for col in keep_cols if col in _df.columns]]
+        _df = _df.apply(to_numeric, errors='coerce')
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        """Perform quality control on Xact XRF data.
+
+        QC Rules Applied
+        ----------------
+        1. Calibration Mode      : SAMPLE_TYPE != 1 indicates zero calibration
+        2. Instrument Error      : ALARM code 100-110 indicates instrument error
+        3. Upscale Warning       : ALARM code 200-203 indicates upscale warning
+        4. Invalid Value         : Element concentration outside valid range (0-100000 ng/m3)
+        5. Internal Std Drift    : Nb internal standard deviates ±20% from median
+        """
+        _index = _df.index.copy()
+        df_qc = _df.copy()
+
+        # Get element columns (exclude uncertainty and environmental columns)
+        element_cols = [col for col in df_qc.columns if col in self.ELEMENTS]
+        uncert_cols = [f'{elem}_uncert' for elem in element_cols if f'{elem}_uncert' in df_qc.columns]
+
+        # Build QC rules declaratively
+        qc = QCFlagBuilder()
+
+        # Add Calibration Mode rule (SAMPLE_TYPE: 1=normal sampling, 2=zero calibration)
+        # Note: Most calibration samples are already filtered in _raw_reader, this catches any remaining
+        if 'SAMPLE_TYPE' in df_qc.columns:
+            qc.add_rules([
+                QCRule(
+                    name='Calibration Mode',
+                    condition=lambda df: (df['SAMPLE_TYPE'] != 1) & df['SAMPLE_TYPE'].notna(),
+                    description='Instrument in calibration mode (SAMPLE_TYPE != 1)'
+                ),
+            ])
+
+        # Add Instrument Error rule (ALARM codes 100-110)
+        if 'ALARM' in df_qc.columns:
+            qc.add_rules([
+                QCRule(
+                    name='Instrument Error',
+                    condition=lambda df: df['ALARM'].isin(list(self.ERROR_CODES.keys())),
+                    description='Instrument error detected (ALARM code 100-110)'
+                ),
+                QCRule(
+                    name='Upscale Warning',
+                    condition=lambda df: df['ALARM'].isin(list(self.WARNING_CODES.keys())),
+                    description='Upscale warning detected (ALARM code 200-203)'
+                ),
+            ])
+
+        # Add Invalid Value rule
+        if element_cols:
+            qc.add_rules([
+                QCRule(
+                    name='Invalid Value',
+                    condition=lambda df, cols=element_cols: (
+                            (df[cols] < self.MIN_VALUE) | (df[cols] > self.MAX_VALUE)
+                    ).any(axis=1),
+                    description=f'Concentration outside valid range ({self.MIN_VALUE}-{self.MAX_VALUE} ng/m3)'
+                ),
+            ])
+
+        # Add Internal Standard Drift rule (Nb)
+        if self.INTERNAL_STD_ELEMENT in df_qc.columns:
+            nb_median = df_qc[self.INTERNAL_STD_ELEMENT].median()
+            lower_bound = nb_median * (1 - self.INTERNAL_STD_TOLERANCE)
+            upper_bound = nb_median * (1 + self.INTERNAL_STD_TOLERANCE)
+            qc.add_rules([
+                QCRule(
+                    name='Internal Std Drift',
+                    condition=lambda df, lb=lower_bound, ub=upper_bound: (
+                            (df[self.INTERNAL_STD_ELEMENT] < lb) | (df[self.INTERNAL_STD_ELEMENT] > ub)
+                    ),
+                    description=f'{self.INTERNAL_STD_ELEMENT} internal standard outside ±{int(self.INTERNAL_STD_TOLERANCE * 100)}% of median ({nb_median:.2f} ng/m³)'
+                ),
+            ])
+
+        # Apply all QC rules and get flagged DataFrame
+        df_qc = qc.apply(df_qc)
+
+        # Log QC summary
+        summary = qc.get_summary(df_qc)
+        self.logger.info(f"{self.nam} QC Summary:")
+        for _, row in summary.iterrows():
+            self.logger.info(f"  {row['Rule']}: {row['Count']} ({row['Percentage']})")
+
+        # Get output columns: elements + uncertainties + environmental + QC_Flag
+        output_cols = element_cols + uncert_cols + [c for c in self.ENV_COLUMNS if c in df_qc.columns] + ['QC_Flag']
+        return df_qc[[c for c in output_cols if c in df_qc.columns]].reindex(_index)
+
+    def decode_alarm(self, alarm_code):
+        """Decode ALARM code to human-readable message.
+
+        Parameters
+        ----------
+        alarm_code : int
+            The ALARM code from the Xact data
+
+        Returns
+        -------
+        str
+            Human-readable description of the alarm
+        """
+        if alarm_code == 0:
+            return 'Normal'
+        elif alarm_code in self.ERROR_CODES:
+            return self.ERROR_CODES[alarm_code]
+        elif alarm_code in self.WARNING_CODES:
+            return self.WARNING_CODES[alarm_code]
+        else:
+            return f'Unknown Alarm ({alarm_code})'

AeroViz/rawDataReader/script/__init__.py

@@ -0,0 +1,28 @@
+# Auto-import all instrument reader modules dynamically
+# New instruments only need to:
+# 1. Add a new .py file in this directory
+# 2. Add entry to supported_instruments.py meta dict
+
+import importlib
+from pathlib import Path
+
+# Get all .py files in this directory (excluding __init__.py)
+_script_dir = Path(__file__).parent
+_module_files = [
+    f.stem for f in _script_dir.glob('*.py')
+    if f.stem != '__init__' and not f.stem.startswith('_')
+]
+
+# Dynamically import each module
+__all__ = []
+for _module_name in _module_files:
+    try:
+        _module = importlib.import_module(f'.{_module_name}', package=__name__)
+        globals()[_module_name] = _module
+        __all__.append(_module_name)
+    except ImportError as e:
+        # Skip modules that fail to import (e.g., missing dependencies)
+        pass
+
+# Clean up temporary variables
+del _script_dir, _module_files, _module_name, _module, importlib, Path

AeroViz/tools/__init__.py

@@ -0,0 +1,2 @@
+from .database import DataBase
+from .dataclassifier import DataClassifier

AeroViz/tools/database.py

@@ -0,0 +1,95 @@
+from io import StringIO
+from pathlib import Path
+from typing import Literal
+
+from pandas import read_csv, DataFrame
+
+
+def load_default_chemical_data():
+    # The following data is from the chemical composition of real atmospheric particles.
+    #
+    # The six main chemical components that comprised PM2.5 are listed in the data.
+    # Here, we test the radar charts to see if we can clearly identify how the
+    # chemical components vary between the three pollutant scenarios:
+    #
+    #  1) Whole sampling period (Total)
+    #  2) Clean period (Clean)
+    #  3) Transition period (Transition)
+    #  4) Event period (Event)
+
+    data = {
+        'Sulfate': [0.01, 0.34, 0.02, 0.71],
+        'Nitrate': [0.88, 0.13, 0.34, 0.13],
+        'OC': [0.07, 0.95, 0.04, 0.05],
+        'EC': [0.20, 0.02, 0.85, 0.19],
+        'Soil': [0.20, 0.10, 0.07, 0.01],
+        'SS': [0.20, 0.10, 0.07, 0.01]
+    }
+
+    return DataFrame(data, index=['Total', 'Clean', 'Transition', 'Event'])
+
+
+def load_dataset_by_url(dataset_name: Literal["Tunghai", "Taipei"] = "Tunghai") -> DataFrame:
+    import requests
+    dataset_uris = {
+        "Tunghai": "https://raw.githubusercontent.com/alex870521/DataPlot/main/DataPlot/config/default_data.csv"
+    }
+
+    # Ensure the dataset name is valid
+    if dataset_name not in dataset_uris:
+        raise ValueError(f"Dataset {dataset_name} is not supported.")
+
+    url = dataset_uris[dataset_name]
+
+    # Make a request to the URL
+    response = requests.get(url)
+
+    if response.status_code == 200:
+        return read_csv(StringIO(response.text), na_values=('E', 'F', '-', '_', '#', '*'), index_col=0,
+                        parse_dates=True, low_memory=False)
+    else:
+        print(f"Failed to download file: {response.status_code}")
+        print(response.text)  # Print the response text for debugging
+        return DataFrame()  # Return an empty DataFrame in case of failure
+
+
+def load_dataset_local(dataset_name: Literal["Tunghai", "Taipei", "PNSD"] = "Tunghai") -> DataFrame:
+    base_dir = Path(__file__).resolve().parent.parent
+    config_dir = base_dir / 'data'
+
+    dataset_paths = {
+        "Tunghai": config_dir / 'DEFAULT_DATA.csv',
+        "Taipei": config_dir / 'DEFAULT_DATA.csv',
+        "PNSD": config_dir / 'DEFAULT_PNSD_DATA.csv'
+    }
+
+    if dataset_name not in dataset_paths:
+        raise ValueError(f"Dataset {dataset_name} is not supported.")
+
+    file_path = dataset_paths[dataset_name]
+
+    if not file_path.exists():
+        raise FileNotFoundError(f"The file {file_path} does not exist.")
+
+    return read_csv(file_path, na_values=('E', 'F', '-', '_', '#', '*'), index_col=0, parse_dates=True,
+                    low_memory=False)
+
+
+class DataBase:
+    def __new__(cls, file_path: Path | str = None, load_data: bool = False, load_PSD: bool = False):
+        print(f'Loading:\033[96m Default Data\033[0m')
+        if file_path is not None:
+            file_path = Path(file_path)
+            if file_path.exists():
+                return read_csv(file_path, na_values=('E', 'F', '-', '_', '#', '*'), index_col=0, parse_dates=True,
+                                low_memory=False)
+
+        if load_data ^ load_PSD:
+            return load_dataset_local("Tunghai") if load_data else load_dataset_local("PNSD")
+
+        else:
+            raise ValueError("Exactly one of 'load_data' or 'load_PSD' must be True.")
+
+
+if __name__ == '__main__':
+    df = DataBase("Tunghai")

AeroViz/tools/dataclassifier.py

@@ -0,0 +1,117 @@
+from datetime import datetime
+from typing import Literal, Sequence
+
+import pandas as pd
+from pandas import concat, DataFrame, Series
+
+
+class Classifier:
+    Seasons = {'2020-Summer': (datetime(2020, 9, 4), datetime(2020, 9, 21, 23)),
+               '2020-Autumn': (datetime(2020, 9, 22), datetime(2020, 12, 29, 23)),
+               '2020-Winter': (datetime(2020, 12, 30), datetime(2021, 3, 25, 23)),
+               '2021-Spring': (datetime(2021, 3, 26), datetime(2021, 5, 6, 23))}
+
+    # '2021-Summer': (datetime(2021, 5, 7), datetime(2021, 10, 16, 23))
+    # '2021-Autumn': (datetime(2021, 10, 17), datetime(2021, 12, 31, 23))
+
+    @classmethod
+    def classify(cls, df) -> DataFrame:
+        df = cls.classify_by_diurnal(df)
+        df = cls.classify_by_state(df)
+        df = cls.classify_by_season(df)
+        df = cls.classify_by_season_state(df)
+
+        return df
+
+    @classmethod
+    def classify_by_diurnal(cls, df):
+        df['Hour'] = df.index.hour
+        df['Diurnal'] = df['Hour'].apply(cls.map_diurnal)
+        return df
+
+    @classmethod
+    def classify_by_state(cls, df):
+        df['State'] = df.apply(cls.map_state, axis=1, clean_bound=df.Extinction.quantile(0.2),
+                               event_bound=df.Extinction.quantile(0.8))
+        return df
+
+    @classmethod
+    def classify_by_season(cls, df):
+        for season, (season_start, season_end) in cls.Seasons.items():
+            df.loc[season_start:season_end, 'Season'] = season
+        return df
+
+    @classmethod
+    def classify_by_season_state(cls, df):
+        for _grp, _df in df.groupby('Season'):
+            df['Season_State'] = df.apply(cls.map_state, axis=1, clean_bound=_df.Extinction.quantile(0.2),
+                                          event_bound=_df.Extinction.quantile(0.8))
+        return df
+
+    @staticmethod
+    def map_diurnal(hour):
+        return 'Day' if 7 <= hour <= 18 else 'Night'
+
+    @staticmethod
+    def map_state(row, clean_bound, event_bound):
+        return 'Event' if row['Extinction'] >= event_bound else 'Clean' if row[
+                                                                               'Extinction'] < clean_bound else 'Transition'
+
+
+class DataClassifier(Classifier):
+    """
+    Notes
+    -----
+    First, create group then return the selected statistic method.
+    If the 'by' does not exist in DataFrame, import the default DataFrame to help to sign the different group.
+
+    """
+
+    def __new__(cls,
+                df: DataFrame,
+                by: Literal["Hour", "State", "Season", "Season_state"] | str,
+                df_support: DataFrame | Series = None,
+                cut_bins: Sequence = None,
+                qcut: int = None,
+                labels: list[str] = None
+                ) -> tuple[DataFrame, DataFrame]:
+        group = cls._group_data(df, by, df_support, cut_bins, qcut, labels)
+        return cls._compute_statistics(df, group)
+
+    @staticmethod
+    def _group_data(df, by, df_support, cut_bins, qcut, labels):
+        if by not in df.columns:
+            if df_support is None:
+                raise KeyError(f"Column '{by}' does not exist in DataFrame."
+                               f"Please provide a support DataFrame or Series to help classify.")
+            else:
+                df = concat([df, Classifier.classify(df_support.copy())[by]], axis=1)
+
+        if cut_bins is not None:
+            df[f'{by}_cut'] = pd.cut(df.loc[:, f'{by}'], cut_bins,
+                                     labels=labels or (cut_bins + (cut_bins[1] - cut_bins[0]) / 2)[:-1])
+            return df.groupby(f'{by}_cut', observed=False)
+
+        elif qcut is not None:
+            df[f'{by}_qcut'] = pd.qcut(df.loc[:, f'{by}'], q=qcut, labels=labels)
+            return df.groupby(f'{by}_qcut', observed=False)
+
+        else:
+            if by == 'State':
+                return df.groupby(by)
+
+            elif by == 'Season':
+                return df.groupby(pd.Categorical(df['Season'], categories=['2020-Summer', '2020-Autumn', '2020-Winter',
+                                                                           '2021-Spring']), observed=False)
+            else:
+                return df.groupby(by, observed=False)
+
+    @staticmethod
+    def _compute_statistics(df, group):
+        mean_df = group.mean(numeric_only=True)
+        mean_df.loc['Total'] = df.mean(numeric_only=True)
+
+        std_df = group.std(numeric_only=True)
+        std_df.loc['Total'] = df.std(numeric_only=True)
+
+        return mean_df, std_df

AeroViz/tools/dataprinter.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+
+from pandas import DataFrame, Timestamp
+from tabulate import tabulate
+
+
+def data_table(df: DataFrame,
+               items: list[str] | str = None,
+               times: list[datetime | Timestamp | str] = None,
+               ):
+    """
+    This function cuts the DataFrame based on the given time periods and calculates the mean and standard deviation
+    of the specified items for each period.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        The DataFrame to be processed. It should have a DateTime index.
+    items : list[str] | str, optional
+        The columns of the DataFrame to be processed. It can be a list of column names or a single column name.
+        By default, it is ['NO', 'NO2', 'NOx'].
+    times : list[str] | str, optional
+        The time periods to cut the DataFrame. It can be a list of time strings or a single time string.
+        Each time string should be in the format of 'YYYY-MM-DD'. By default, it is ['2024-03-21', '2024-04-30'].
+
+    Returns
+    -------
+    None
+        This function doesn't return any value. It prints out a table showing the mean and standard deviation
+        of the specified items for each time period.
+    """
+    items = [items] if isinstance(items, str) else items
+    times = [times] if isinstance(times, str) else times
+    times = list(map(Timestamp, times))
+
+    times.sort()
+
+    results = []
+    periods = []
+    for i in range(len(times) + 1):
+        if i == 0:
+            df_period = df.loc[df.index <= times[i], items]
+            period_label = f'Before {times[i].date()}'
+        elif i == len(times):
+            df_period = df.loc[df.index > times[i - 1], items]
+            period_label = f'After {times[i - 1].date()}'
+        else:
+            df_period = df.loc[(df.index > times[i - 1]) & (df.index <= times[i]), items]
+            period_label = f'{times[i - 1].date()} to {times[i].date()}'
+
+        mean, std = df_period.mean().round(2).to_numpy(), df_period.std().round(2).to_numpy()
+
+        results.append([f'{m} ± {s}' for m, s in zip(mean, std)])
+        periods.append(period_label)
+
+    result = DataFrame(results, columns=items, index=periods)
+
+    print(tabulate(result, headers='keys', tablefmt='fancy_grid'))