AeroViz 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

AeroViz/rawDataReader/__init__.py

@@ -1,106 +1,111 @@
 from datetime import datetime
 from pathlib import Path
 
+from pandas import Grouper, Timedelta
+
 from AeroViz.rawDataReader.config.supported_instruments import meta
 from AeroViz.rawDataReader.script import *
 
 __all__ = ['RawDataReader']
 
+SUPPORTED_INSTRUMENTS = [
+    NEPH, Aurora, SMPS, GRIMM, APS_3321, AE33, AE43, BC1054,
+    MA350, TEOM, OCEC, IGAC, VOC, EPA, Minion
+]
+
 
 def RawDataReader(instrument_name: str,
                   path: Path,
-                  qc: bool = True,
-                  csv_raw: bool = True,
                   reset: bool = False,
-                  rate: bool = False,
+                  qc: bool | str = True,
+                  qc_freq: str | None = None,
+                  rate: bool = True,
                   append_data: bool = False,
-                  start: datetime | None = None,
-                  end: datetime | None = None,
-                  mean_freq='1h',
-                  csv_out=True,
+                  start: datetime = None,
+                  end: datetime = None,
+                  mean_freq: str = '1h',
+                  csv_out: bool = True,
                   ):
     """
     Factory function to instantiate the appropriate reader module for a given instrument and
     return the processed data over the specified time range.
 
-    Parameters
-    ----------
-    instrument_name : str
-        The name of the instrument for which to read data. Must be a valid key in the `meta` dictionary.
-    path : Path
-        The directory where raw data files for the instrument are stored.
-    qc : bool, optional (default=True)
-        If True, apply quality control (QC) to the raw data.
-    csv_raw : bool, optional (default=True)
-        If True, read raw data from CSV files.
-    reset : bool, optional (default=False)
-        If True, reset the state and reprocess the data from scratch.
-    rate : bool, optional (default=False)
-        If True, calculate rates from the data.
-    append_data : bool, optional (default=False)
-        If True, append new data to the existing dataset instead of overwriting it.
-    start : datetime, optional
-        Start time for filtering the data. If None, no start time filtering will be applied.
-    end : datetime, optional
-        End time for filtering the data. If None, no end time filtering will be applied.
-    mean_freq : str, optional (default='1h')
-        Resampling frequency for averaging the data. Example: '1h' for hourly mean.
-    csv_out : bool, optional (default=True)
-        If True, output the processed data as a CSV file.
-
-    Return
-    ------
-    reader_module : Reader
-        An instance of the reader module corresponding to the specified instrument, which processes
-        the data and returns it in a usable format.
-
-    Raises
-    ------
-    ValueError
-        If the `instrument_name` provided is not a valid key in the `meta` dictionary.
-
-    Example
-    -------
+    :param instrument_name: The name of the instrument for which to read data. Must be a valid key in the `meta` dictionary.
+    :param path: The directory where raw data files for the instrument are stored.
+    :param reset: If True, reset the state and reprocess the data from scratch.
+    :param qc: If True, apply quality control (QC) to the raw data.
+    :param qc_freq: Frequency at which to perform QC. Must be one of 'W', 'M', 'Q', 'Y' for weekly, monthly, quarterly, or yearly.
+    :param rate: If True, calculate rates from the data.
+    :param append_data: If True, append new data to the existing dataset instead of overwriting it.
+    :param start: Start time for filtering the data. If None, no start time filtering will be applied.
+    :param end: End time for filtering the data. If None, no end time filtering will be applied.
+    :param mean_freq: Resampling frequency for averaging the data. Example: '1h' for hourly mean.
+    :param csv_out: If True, output the processed data as a CSV file.
+
+    :return: An instance of the reader module corresponding to the specified instrument, which processes the data and returns it in a usable format.
+
+    :raises ValueError: If the `instrument_name` provided is not a valid key in the `meta` dictionary.
+    :raises ValueError: If the specified path does not exist or is not a directory.
+    :raises ValueError: If the QC frequency is invalid.
+    :raises ValueError: If start and end times are not both provided or are invalid.
+    :raises ValueError: If the mean_freq is not a valid frequency string.
+
+    :Example:
+
     To read and process data for the BC1054 instrument:
 
     >>> from pathlib import Path
     >>> from datetime import datetime
-    >>> data = RawDataReader(instrument_name='BC1054', path=Path('/path/to/data'),
-    >>>                      start=datetime(2024, 1, 1), end=datetime(2024, 2, 1))
+    >>>
+    >>> data = RawDataReader(
+    ...     instrument_name='BC1054',
+    ...     path=Path('/path/to/data'),
+    ...     start=datetime(2024, 2, 1),
+    ...     end=datetime(2024, 7, 31, 23))
     """
     # Mapping of instrument names to their respective classes
-    instrument_class_map = {
-        'NEPH': NEPH,
-        'Aurora': Aurora,
-        'SMPS_genr': SMPS_genr,
-        'SMPS_aim11': SMPS_aim11,
-        'SMPS_TH': SMPS_TH,
-        'GRIMM': GRIMM,
-        'APS_3321': APS_3321,
-        'AE33': AE33,
-        'AE43': AE43,
-        'BC1054': BC1054,
-        'MA350': MA350,
-        'TEOM': TEOM,
-        'Sunset_OCEC': Sunset_OCEC,
-        'IGAC': IGAC,
-        'VOC': VOC,
-        'Table': Table,
-        'EPA_vertical': EPA_vertical,
-        'Minion': Minion
-        # Add other instruments and their corresponding classes here
-    }
+    instrument_class_map = {cls.__name__.split('.')[-1]: cls for cls in SUPPORTED_INSTRUMENTS}
 
     # Check if the instrument name is in the map
     if instrument_name not in meta.keys():
         raise ValueError(f"Instrument name '{instrument_name}' is not valid. \nMust be one of: {list(meta.keys())}")
 
+    # 檢查 path 是否存在且是一個目錄
+    if not isinstance(path, Path):
+        path = Path(path)
+    if not path.exists() or not path.is_dir():
+        raise ValueError(f"The specified path '{path}' does not exist or is not a directory.")
+
+    # Validate the QC frequency
+    if qc_freq is not None:
+        try:
+            Grouper(freq=qc_freq)
+        except ValueError as e:
+            raise ValueError(f"Invalid frequency: {qc_freq}. Error: {str(e)}")
+        except TypeError as e:
+            raise ValueError(f"Invalid frequency type: {qc_freq}. Frequency should be a string.")
+
+    if start and end:
+        if end.hour == 0 and end.minute == 0 and end.second == 0:
+            end = end.replace(hour=23)
+    else:
+        raise ValueError("Both start and end times must be provided.")
+    if end <= start:
+        raise ValueError(f"Invalid time range: start {start} is after end {end}")
+
+    # 驗證 mean_freq 的格式是否正確
+    try:
+        Timedelta(mean_freq)
+    except ValueError:
+        raise ValueError(
+            f"Invalid mean_freq: '{mean_freq}'. It should be a valid frequency string (e.g., '1H', '30min', '1D').")
+
     # Instantiate the class and return the instance
     reader_module = instrument_class_map[instrument_name].Reader(
         path=path,
-        qc=qc,
-        csv_raw=csv_raw,
         reset=reset,
+        qc=qc,
+        qc_freq=qc_freq,
         rate=rate,
         append_data=append_data
     )

AeroViz/rawDataReader/config/supported_instruments.py

@@ -2,76 +2,64 @@
 
 meta = {
     "NEPH": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "5min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
     "Aurora": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"Scatter Coe. (550 nm)": ["G"]},
     },
 
-    "SMPS_TH": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_genr": {
-        "pattern": "*.txt",
-        "freq": "6min",
-        "deter_key": {"Bins": ["all"]},
-    },
-
-    "SMPS_aim11": {
-        "pattern": "*.csv",
+    "SMPS": {
+        "pattern": ["*.txt", "*.csv"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "GRIMM": {
-        "pattern": "*.dat",
+        "pattern": ["*.dat"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "APS_3321": {
-        "pattern": "*.TXT",
+        "pattern": ["*.txt"],
         "freq": "6min",
         "deter_key": {"Bins": ["all"]},
     },
 
     "AE33": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE33*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE33*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "AE43": {
-        "pattern": "[!ST|!CT|!FV]*[!log]_AE43*.dat",
+        "pattern": ["[!ST|!CT|!FV]*[!log]_AE43*.dat"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC6"]},
         "error_state": [],
     },
 
     "BC1054": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC9"]},
         "error_state": [1, 2, 4, 8, 16, 32, 65536],
     },
 
     "MA350": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1min",
         "deter_key": {"BC Mass Conc. (880 nm)": ["BC5"]},
     },
 
     "TEOM": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "6min",
         "deter_key": {
             "PM1.0 Mass Conc.": ["PM_Total"],
@@ -79,8 +67,8 @@ meta = {
         },
     },
 
-    "Sunset_OCEC": {
-        "pattern": "*LCRes.csv",
+    "OCEC": {
+        "pattern": ["*LCRes.csv"],
         "freq": "1h",
         "deter_key": {
             "Thermal OC": ["Thermal_OC"],
@@ -91,7 +79,7 @@ meta = {
     },
 
     "IGAC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "deter_key": {
             "Na+": ["Na+"],
@@ -107,8 +95,55 @@ meta = {
         },
     },
 
+    "XRF": {
+        "pattern": ["*.csv"],
+        "freq": "1h",
+        "deter_key": {
+            "Al": ["Al"],
+            "Si": ["Si"],
+            "P": ["P"],
+            "S": ["S"],
+            "Cl": ["Cl"],
+            "K": ["K"],
+            "Ca": ["Ca"],
+            "Ti": ["Ti"],
+            "V": ["V"],
+            "Cr": ["Cr"],
+            "Mn": ["Mn"],
+            "Fe": ["Fe"],
+            "Ni": ["Ni"],
+            "Cu": ["Cu"],
+            "Zn": ["Zn"],
+            "As": ["As"],
+            "Se": ["Se"],
+            "Br": ["Br"],
+            "Rb": ["Rb"],
+            "Sr": ["Sr"],
+            "Y": ["Y"],
+            "Zr": ["Zr"],
+            "Mo": ["Mo"],
+            "Ag": ["Ag"],
+            "Cd": ["Cd"],
+            "In": ["In"],
+            "Sn": ["Sn"],
+            "Sb": ["Sb"],
+            "Te": ["Te"],
+            "Cs": ["Cs"],
+            "Ba": ["Ba"],
+            "La": ["La"],
+            "Ce": ["Ce"],
+            "W": ["W"],
+            "Pt": ["Pt"],
+            "Au": ["Au"],
+            "Hg": ["Hg"],
+            "Tl": ["Tl"],
+            "Pb": ["Pb"],
+            "Bi": ["Bi"],
+        },
+    },
+
     "VOC": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv"],
         "freq": "1h",
         "key": [
             'Benzene', 'Toluene', 'Ethylbenzene', 'm/p-Xylene', 'o-Xylene', 'Ethane', 'Propane', 'Isobutane',
@@ -128,21 +163,18 @@ meta = {
         "deter_key": None,
     },
 
-    "Table": {
-        "pattern": "*.csv",
-        "freq": "1h",
-        "deter_key": None,
-    },
-
-    "EPA_vertical": {
-        "pattern": "*.csv",
+    "EPA": {
+        "pattern": ["*.csv"],
         "freq": "1h",
-        "deter_key": None,
+        "deter_key": {"Items": ["all"]},
     },
 
     "Minion": {
-        "pattern": "*.csv",
+        "pattern": ["*.csv", "*.xlsx"],
         "freq": "1h",
-        "deter_key": None,
+        "deter_key": {
+            "Main Salt (Na+, NH4+, Cl-, NO3-, SO42-)": ["Na+", "NH4+", "Cl-", "NO3-", "SO42-"],
+            "XRF (Al, Ti, V, Cr, Mn, Fe)": ["Al", "Ti", "V", "Cr", "Mn", "Fe"],
+        },
     },
 }