AeroViz 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

AeroViz/rawDataReader/script/Minion.py

@@ -1,38 +1,78 @@
+from typing import Literal
+
 import numpy as np
-from pandas import read_csv, to_datetime, to_numeric
+import pandas
+from pandas import read_excel, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
+pandas.set_option("future.no_silent_downcasting", True)
+
+desired_order1 = ['SO2', 'NO', 'NOx', 'NO2', 'CO', 'O3', 'THC', 'NMHC',
+                  'CH4', 'PM10', 'PM2.5', 'WS', 'WD', 'AT', 'RH']
+
+desired_order2 = ['Benzene', 'Toluene', 'EthylBenzene', 'm/p-Xylene', 'o-Xylene']
+
+desired_order3 = ['Al', 'Si', 'P', 'S', 'Cl', 'K', 'Ca', 'Ti', 'V', 'Cr', 'Mn', 'Fe',
+                  'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Rb', 'Sr',
+                  'Y', 'Zr', 'Nb', 'Mo', 'Pd', 'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te',
+                  'Cs', 'Ba', 'La', 'Ce', 'W', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi']
+
+desired_order4 = ['NH3', 'HF', 'HCl', 'HNO2', 'HNO3', 'G-SO2',
+                  'Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+',
+                  'F-', 'Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-']
+
 
 class Reader(AbstractReader):
     nam = 'Minion'
 
     def _raw_reader(self, file):
-        with file.open('r', encoding='utf-8-sig', errors='ignore') as f:
-            _df = read_csv(f, low_memory=False, index_col=0)
+        # 讀取 Excel 文件
+        df = read_excel(file, index_col=0, parse_dates=True)
+
+        # 重命名列，去除空白
+        df = df.rename(columns=lambda x: x.strip())
+
+        # 保存單位行並給它一個名稱
+        units = df.iloc[0].copy()
 
-            _df.index = to_datetime(_df.index, errors='coerce')
-            _df.index.name = 'time'
+        # 刪除原始數據中的單位行
+        df = df.iloc[1:]
 
-            _df.columns = _df.keys().str.strip(' ')
+        # 替換特定值
+        df = df.replace({'維護校正': '*', np.nan: '-', '0L': '_', 'Nodata': '-'}, inplace=False)
+        df = df.replace(to_replace=r'\d*[#]\b', value='#', regex=True)
+        df = df.replace(to_replace=r'\d*[L]\b', value='_', regex=True)
 
-        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+        # 處理除了'WD'列的 0 值
+        non_wd_columns = [col for col in df.columns if col != 'WD']
+        df.loc[:, non_wd_columns] = df.loc[:, non_wd_columns].replace({0: '_'})
+
+        # 重新排序列
+        df = self.reorder_dataframe_columns(df, [desired_order1, desired_order2, desired_order3, desired_order4])
+
+        # 將單位行添加回 DataFrame
+        # df = concat([units.to_frame().T, df])
+
+        df.index.name = 'Time'
+
+        return df.loc[~df.index.duplicated() & df.index.notna()]
 
     def _QC(self, _df):
+        # remove negative value
+        _df = _df.mask((_df < 0).copy())
+
         # XRF QAQC
         _df = self.XRF_QAQC(_df)
 
         # ions balance
-        _df = self.ions_balance(_df)
-
-        # remove negative value
-        _df = _df.mask((_df < 0).copy())
+        _df = self.IGAC_QAQC(_df)
 
         # QC data in 6h
-        return _df.resample('6h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('6h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()
 
     # base on Xact 625i Minimum Decision Limit (MDL) for XRF in ng/m3, 60 min sample time
-    def XRF_QAQC(self, df):
+    def XRF_QAQC(self, df, MDL_replace: Literal['nan', '0.5 * MDL'] = 'nan'):
         MDL = {
             'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2,
             'Cl': 1.7, 'K': 1.2, 'Ca': 0.3, 'Ti': 1.6,
@@ -40,34 +80,68 @@ class Reader(AbstractReader):
             'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079, 'Zn': 0.067,
             'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
             'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28,
-            'Zr': 0.33, 'Nb': 0.41, 'Mo': 0.48, 'Ag': 1.9,
-            'Cd': 2.5, 'In': 3.1, 'Sn': 4.1, 'Sb': 5.2,
-            'Te': 0.6, 'I': 0.49, 'Cs': 0.37, 'Ba': 0.39,
-            'La': 0.36, 'Ce': 0.3, 'Pt': 0.12, 'Au': 0.1,
-            'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13, 'Bi': 0.13
+            'Zr': 0.33, 'Nb': 0.41, 'Mo': 0.48, 'Pd': 2.2,
+            'Ag': 1.9, 'Cd': 2.5, 'In': 3.1, 'Sn': 4.1,
+            'Sb': 5.2, 'Te': 0.6, 'Cs': 0.37, 'Ba': 0.39,
+            'La': 0.36, 'Ce': 0.3, 'W': 0.0001, 'Pt': 0.12,
+            'Au': 0.1, 'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13,
+            'Bi': 0.13
         }
-        # 將小於 MDL 值的數據替換為 NaN
+        # 將小於 MDL 值的數據替換為 nan or 5/6 MDL
         for element, threshold in MDL.items():
             if element in df.columns:
-                df[element] = df[element].where(df[element] >= threshold, np.nan)
+                rep = np.nan if MDL_replace == 'nan' else 0.5 * threshold
+                df[element] = df[element].where(df[element] >= threshold, rep)
 
         self.logger.info(f"{'=' * 60}")
         self.logger.info(f"XRF QAQC summary:")
         self.logger.info("\t\ttransform values below MDL to NaN")
         self.logger.info(f"{'=' * 60}")
 
+        # 轉換單位 ng/m3 -> ug/m3
+        if df.Al.max() > 10 and df.Fe.max() > 10:
+            # 確保 MDL.keys() 中的所有列都存在於 _df 中
+            columns_to_convert = [col for col in MDL.keys() if col in df.columns]
+
+            df[columns_to_convert] = df[columns_to_convert].div(1000)
+
         return df
 
-    def ions_balance(self, df, tolerance=0.3):
+    def IGAC_QAQC(self, df, tolerance=1):
         """
         Calculate the balance of ions in the system
         """
+        # https://www.yangyao-env.com/web/product/product_in2.jsp?pd_id=PD1640151884502
+        MDL = {
+            'HF': 0.08, 'HCl': 0.05, 'HNO2': 0.01, 'HNO3': 0.05, 'G-SO2': 0.05, 'NH3': 0.1,
+            'Na+': 0.05, 'NH4+': 0.08, 'K+': 0.08, 'Mg2+': 0.05, 'Ca2+': 0.05,
+            'F-': 0.08, 'Cl-': 0.05, 'NO2-': 0.05, 'NO3-': 0.01, 'PO43-': None, 'SO42-': 0.05,
+        }
+
+        MR = {
+            'HF': 200, 'HCl': 200, 'HNO2': 200, 'HNO3': 200, 'G-SO2': 200, 'NH3': 300,
+            'Na+': 300, 'NH4+': 300, 'K+': 300, 'Mg2+': 300, 'Ca2+': 300,
+            'F-': 300, 'Cl-': 300, 'NO2-': 300, 'NO3-': 300, 'PO43-': None, 'SO42-': 300,
+        }
+
+        _cation, _anion, _main = (['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'],
+                                  ['Cl-', 'NO2-', 'NO3-', 'SO42-'],
+                                  ['SO42-', 'NO3-', 'NH4+'])
+        # QC: replace values below MDL with 0.5 * MDL -> ions balance -> PM2.5 > main salt
+        # mass tolerance = 0.3, ions balance tolerance = 0.3
+
+        # # conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
+        # _df_salt = df.mask(df.sum(axis=1, min_count=1) > df.PM25).dropna(subset=_main).copy()
+
         # Define the ions
-        item = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'F-', 'Cl-', 'NO2-', 'NO3-', 'PO43-', 'SO42-']
+        item = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+', 'Cl-', 'NO2-', 'NO3-', 'SO42-']
 
         # Calculate the balance
-        _df = df[item].copy()
-        _df = _df.apply(lambda x: to_numeric(x, errors='coerce'))
+        _df = df[item].apply(lambda x: to_numeric(x, errors='coerce'))
+
+        # for (_key, _df_col) in _df.items():
+        #     _df[_key] = _df_col.mask(_df_col < MDL[_key], MDL[_key] / 2)
+
         _df['+_mole'] = _df[['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+']].div([23, 18, 39, (24 / 2), (40 / 2)]).sum(axis=1,
                                                                                                              skipna=True)
         _df['-_mole'] = _df[['Cl-', 'NO2-', 'NO3-', 'SO42-']].div([35.5, 46, 62, (96 / 2)]).sum(axis=1, skipna=True)
@@ -79,12 +153,8 @@ class Reader(AbstractReader):
         lower_bound, upper_bound = 1 - tolerance, 1 + tolerance
 
         # 根据ratio决定是否保留原始数据
-        valid_mask = (
-                (_df['ratio'] <= upper_bound) &
-                (_df['ratio'] >= lower_bound) &
-                ~np.isnan(_df['+_mole']) &
-                ~np.isnan(_df['-_mole'])
-        )
+        valid_mask = ((_df['ratio'] <= upper_bound) & (_df['ratio'] >= lower_bound) &
+                      ~np.isnan(_df['+_mole']) & ~np.isnan(_df['-_mole']))
 
         # 保留数据或将不符合条件的行设为NaN
         df.loc[~valid_mask, item] = np.nan
@@ -100,4 +170,6 @@ class Reader(AbstractReader):
         if retained_percentage < 70:
             self.logger.warning("Warning: The percentage of retained data is less than 70%")
 
+        # print(f"\tretain {retained_percentage.__round__(0)}% data within tolerance {tolerance}")
+
         return df

AeroViz/rawDataReader/script/NEPH.py

@@ -67,4 +67,4 @@ class Reader(AbstractReader):
         _df = _df[(_df['BB'] < _df['B']) & (_df['BG'] < _df['G']) & (_df['BR'] < _df['R'])]
 
         # QC data in 1h
-        return _df.resample('1h').apply(self.basic_QC).resample(self.meta.get("freq")).mean()
+        return _df.resample('1h').apply(self.n_sigma_QC).resample(self.meta.get("freq")).mean()

AeroViz/rawDataReader/script/{Sunset_OCEC.py → OCEC.py}

@@ -4,7 +4,7 @@ from AeroViz.rawDataReader.core import AbstractReader
 
 
 class Reader(AbstractReader):
-    nam = 'Sunset_OCEC'
+    nam = 'OCEC'
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
@@ -51,7 +51,7 @@ class Reader(AbstractReader):
     def _QC(self, _df):
         import numpy as np
 
-        _df = _df.where(_df > 0)
+        _df = _df.mask((_df <= 0) | (_df > 100)).copy()
 
         thresholds = {
             'Thermal_OC': 0.3,

AeroViz/rawDataReader/script/SMPS.py

@@ -0,0 +1,77 @@
+import csv
+
+import numpy as np
+from pandas import to_datetime, to_numeric, read_csv, isna
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+def find_header_row(file_obj, delimiter):
+    csv_reader = csv.reader(file_obj, delimiter=delimiter)
+    for skip, row in enumerate(csv_reader):
+        if row and (row[0] in ['Sample #', 'Scan Number']):
+            return skip
+    raise ValueError("Header row not found")
+
+
+def parse_date(df, date_format):
+    if 'Date' in df.columns and 'Start Time' in df.columns:
+        return to_datetime(df['Date'] + ' ' + df['Start Time'], format=date_format, errors='coerce')
+    elif 'DateTime Sample Start' in df.columns:
+        return to_datetime(df['DateTime Sample Start'], format=date_format, errors='coerce')
+    else:
+        raise ValueError("Expected date columns not found")
+
+
+class Reader(AbstractReader):
+    nam = 'SMPS'
+
+    def _raw_reader(self, file):
+        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+            if file.suffix.lower() == '.txt':
+                delimiter, date_formats = '\t', ['%m/%d/%y %X', '%m/%d/%Y %X']
+            else:  # csv
+                delimiter, date_formats = ',', ['%d/%m/%Y %X']
+
+            skip = find_header_row(f, delimiter)
+            f.seek(0)
+
+            _df = read_csv(f, sep=delimiter, skiprows=skip)
+
+            for date_format in date_formats:
+                _time_index = parse_date(_df, date_format)
+                if not isna(_time_index).all():
+                    break
+            else:
+                raise ValueError("Unable to parse dates with given formats")
+
+            # sequence the data
+            numeric_cols = [col for col in _df.columns if col.strip().replace('.', '').isdigit()]
+            numeric_cols.sort(key=lambda x: float(x.strip()))
+
+            _df.index = _time_index
+            _df.index.name = 'time'
+
+            _df_smps = _df[numeric_cols]
+            _df_smps.columns = _df_smps.columns.astype(float)
+            _df_smps = _df_smps.loc[_df_smps.index.dropna().copy()]
+
+            return _df_smps.apply(to_numeric, errors='coerce')
+
+    # QC data
+    def _QC(self, _df):
+
+        # mask out the data size lower than 7
+        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
+        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
+        _df = _df.mask(_df_size < 7)
+
+        # remove total conc. lower than 2000
+        _df = _df.mask(_df['total'] < 2000)
+
+        # remove the bin over 400 nm which num. conc. larger than 4000
+        _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2] >= 400.]
+
+        _df[_df_remv_ky] = _df[_df_remv_ky].copy().mask(_df[_df_remv_ky] > 4000.)
+
+        return _df[_df.keys()[:-1]]

AeroViz/rawDataReader/script/TEOM.py

@@ -25,9 +25,9 @@ class Reader(AbstractReader):
 
             _df = _df.set_index(to_datetime(_tm_idx, errors='coerce', format='%d - %m - %Y %X'))
 
-            _df = _df.where(_df['status'] < 1e-7)
+            _df = _df.where(_df['status'] < 1)
 
-        _df = _df[['PM_NV', 'PM_Total', 'noise', ]]
+        _df = _df[['PM_NV', 'PM_Total', 'noise']]
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 

AeroViz/rawDataReader/script/VOC.py

@@ -20,8 +20,8 @@ class Reader(AbstractReader):
 
             if invalid_keys:
                 self.logger.warning(f'{invalid_keys} are not supported keys.')
-                print(f'\n\t\t{invalid_keys} are not supported keys.'
-                      f'\n\t\tPlease check the\033[91m support_voc.md\033[0m file to use the correct name.')
+                print(f'\n\t{invalid_keys} are not supported keys.'
+                      f'\n\tPlease check the\033[91m support_voc.md\033[0m file to use the correct name.')
 
             if valid_keys:
                 return _df[valid_keys].loc[~_df.index.duplicated() & _df.index.notna()]

AeroViz/rawDataReader/script/XRF.py

@@ -0,0 +1,11 @@
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'XRF'
+
+    def _raw_reader(self, file):
+        pass
+
+    def _QC(self, _df):
+        pass

AeroViz/rawDataReader/script/__init__.py

@@ -1,9 +1,7 @@
 __all__ = [
     'NEPH',
     'Aurora',
-    'SMPS_TH',
-    'SMPS_genr',
-    'SMPS_aim11',
+    'SMPS',
     'APS_3321',
     'GRIMM',
     'AE33',
@@ -11,10 +9,10 @@ __all__ = [
     'BC1054',
     'MA350',
     'TEOM',
-    'Sunset_OCEC',
+    'OCEC',
     'IGAC',
+    'XRF',
     'VOC',
-    'Table',
-    'EPA_vertical',
+    'EPA',
     'Minion'
 ]

{AeroViz-0.1.5.dist-info → AeroViz-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.5
+Version: 0.1.7
 Summary: Aerosol science
 Home-page: https://github.com/Alex870521/AeroViz
 Author: alex
@@ -11,14 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pandas
-Requires-Dist: numpy
-Requires-Dist: matplotlib
-Requires-Dist: seaborn
-Requires-Dist: scipy
-Requires-Dist: scikit-learn
-Requires-Dist: windrose
-Requires-Dist: tabulate
+Requires-Dist: pandas ==2.2.2
+Requires-Dist: numpy ==1.26.4
+Requires-Dist: matplotlib ==3.8.4
+Requires-Dist: scipy ==1.14.0
+Requires-Dist: seaborn ==0.13.2
+Requires-Dist: scikit-learn ==1.5.1
+Requires-Dist: windrose ==1.9.2
+Requires-Dist: tabulate ==0.9.0
+Requires-Dist: rich ~=13.7.1
 
 ## <div align="center">AeroViz for Aerosol Science Visualization</div>
 
@@ -27,7 +28,7 @@ Requires-Dist: tabulate
   <img alt="Static Badge" src="https://img.shields.io/badge/python-3.12-blue?logo=python">
   <img alt="Static Badge" src="https://img.shields.io/badge/License-MIT-yellow">
   <img alt="Static Badge" src="https://img.shields.io/badge/github-updating-red?logo=github">
-  <img src="https://img.shields.io/badge/testing-green?logo=Pytest&logoColor=blue">
+  <img alt="Static Badge" src="https://img.shields.io/badge/testing-green?logo=Pytest&logoColor=blue">
 
 </p>
 
@@ -52,42 +53,66 @@ Requires-Dist: tabulate
 ## <div align="center">Installation</div>
 
 ```bash
-pip install AeroViz # ensure the python version is >= 3.12
+pip install AeroViz
 ```
 
-## <div align="center">Usage</div>
+## <div align="center">Quick Start</div>
 
 ```python
 import AeroViz
+from AeroViz import RawDataReader, DataProcess, plot
+
+# Read data from a supported instrument
+data = RawDataReader('NEPH', '/path/to/data', start='2024-01-01', end='2024-01-31')
+
+# Create a visualization
+plot.timeseries(data, y='scattering_coefficient')
 ```
 
-## <div align="center">RawDataReader Supported Instruments</div>
+For more detailed usage instructions, please refer to our [User Guide]().
+
+## RawDataReader
+
+RawDataReader supports a wide range of aerosol instruments, including NEPH, SMPS, AE33, and many more. It handles
+various file types and time resolutions, making data processing efficient and standardized.
+
+For a detailed list of supported instruments, file types, and data columns, please refer to
+our [RawDataReader Usage Guide](docs/RawDataReader_Usage_Guide.md) in the `docs` folder.
+
+### Key Features:
+
+- Supports multiple aerosol instruments
+- Applies customizable quality control measures
+- Offers flexible data filtering and resampling options
+- Enables easy data export to CSV format
 
-> [!NOTE]\
-> We are continuously working to support more instruments. Please check back for updates or contribute to our project on
-> GitHub.
+### Supported Instruments
 
 The AeroViz project currently supports data from the following instruments:
 
-- **SMPS (Scanning Mobility Particle Sizer)**
-- **APS (Aerodynamic Particle Sizer)**
-- **GRIMM (GRIMM Aerosol Technik)**
-- **TEOM (Continuous Ambient Particulate Monitor)**
-- **NEPH (Nephelometer)**
-- **Aurora (Nephelometer)**
-- **AE33 (Aethalometer Model 33)**
-- **AE43 (Aethalometer Model 43)**
-- **BC1054 (Black Carbon Monitor 1054)**
-- **MA350 (MicroAeth MA350)**
-- **OCEC (Organic Carbon Elemental Carbon Analyzer)**
-- **IGAC (In-situ Gas and Aerosol Compositions monitor)**
-- **VOC (Volatile Organic Compounds Monitor)**
+- SMPS (Scanning Mobility Particle Sizer)
+- APS (Aerodynamic Particle Sizer)
+- GRIMM (GRIMM Aerosol Technik)
+- TEOM (Continuous Ambient Particulate Monitor)
+- NEPH (Nephelometer)
+- Aurora (Nephelometer)
+- AE33 (Aethalometer Model 33)
+- AE43 (Aethalometer Model 43)
+- BC1054 (Black Carbon Monitor 1054)
+- MA350 (MicroAeth MA350)
+- OCEC (Organic Carbon Elemental Carbon Analyzer)
+- IGAC (In-situ Gas and Aerosol Compositions monitor)
+- XRF (X-ray Fluorescence Spectrometer)
+- VOC (Volatile Organic Compounds Monitor)
+
+> **Note:** We are continuously working to support more instruments. Please check back for updates or contribute to our
+> project on GitHub.
 
 ## <div align="center">DataProcess Supported Method</div>
 
 The AeroViz project currently supports the following processing methods:
 
-- **Chemistry**
+- **Chemistry**:
 - **Optical**
 - **SizeDistr**
 - **VOC**
@@ -100,7 +125,7 @@ For detailed documentation, please refer to the `docs` folder, which includes:
 
 | Documentation                              | Description                |
 |--------------------------------------------|----------------------------|
-| [User Guide](docs/user_guide.md)           | Basic usage instructions   |
+| [User Guide](docs/user_guide)              | Basic usage instructions   |
 | [Developer Guide](docs/developer_guide.md) | Developer guidelines       |
 | [API Reference](docs/api_reference.md)     | API documentation          |
 | [FAQ](docs/faq.md)                         | Frequently Asked Questions |
@@ -108,7 +133,7 @@ For detailed documentation, please refer to the `docs` folder, which includes:
 
 </div>
 
-## <div align="center">Related Dependencies</div>
+## <div align="center">Related Source</div>
 
 * #### [PyMieScatt](https://github.com/bsumlin/PyMieScatt.git)
 * #### [py-smps](https://github.com/quant-aq/py-smps.git)