AeroViz 0.1.15__tar.gz → 0.1.16__tar.gz

{aeroviz-0.1.15 → aeroviz-0.1.16}/AeroViz/dataProcess/Optical/_scattering.py

@@ -14,7 +14,7 @@ def _scaCoe(df, instru, specified_band: list):
     df_sca = df.copy().dropna()
 
     if instru == 'Neph':
-        df_out = df_sca[['B']].copy()
+        df_out = df_sca[['G']].copy()
         df_out.columns = [f'sca_{_band}' for _band in specified_band]
     else:
         df_out = df_sca.apply(get_species_wavelength, axis=1, result_type='expand', args=(specified_band,))

{aeroviz-0.1.15 → aeroviz-0.1.16}/AeroViz/rawDataReader/core/__init__.py

@@ -1,7 +1,7 @@
 import json
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 from typing import Generator
 
@@ -179,20 +179,27 @@ class AbstractReader(ABC):
             else:
                 self.__calculate_rates(raw_data, qc_data, all_keys=True, with_log=True)
 
-            # 計算週和月的數據
+            # 使用 Grouper 對數據按週和月進行分組
             current_time = datetime.now()
-            week_mask = raw_data.index >= current_time - timedelta(days=7)
-            month_mask = raw_data.index >= current_time - timedelta(days=30)
+
+            # 按週分組 (使用星期一作為每週的開始)
+            weekly_raw_groups = raw_data.groupby(pd.Grouper(freq='W-MON'))
+            weekly_qc_groups = qc_data.groupby(pd.Grouper(freq='W-MON'))
+
+            # 按月分組 (使用月初作為每月的開始)
+            monthly_raw_groups = raw_data.groupby(pd.Grouper(freq='MS'))
+            monthly_qc_groups = qc_data.groupby(pd.Grouper(freq='MS'))
 
             # 生成報告
-            self.__generate_report(
+            self.__generate_grouped_report(
                 current_time,
-                raw_data[week_mask], qc_data[week_mask],
-                raw_data[month_mask], qc_data[month_mask]
+                weekly_raw_groups, weekly_qc_groups,
+                monthly_raw_groups, monthly_qc_groups
             )
 
-    def __generate_report(self, current_time, week_raw_data, week_qc_data, month_raw_data, month_qc_data):
-        """生成獲取率和良率的報告"""
+    def __generate_grouped_report(self, current_time, weekly_raw_groups, weekly_qc_groups,
+                                  monthly_raw_groups, monthly_qc_groups):
+        """生成基於分組數據的獲取率和良率報告"""
         report = {
             "report_time": current_time.strftime('%Y-%m-%d %H:%M:%S'),
             "instrument_info": {
@@ -200,21 +207,50 @@ class AbstractReader(ABC):
                 "instrument": self.nam
             },
             "rates": {
-                "weekly": self.__calculate_rates(week_raw_data, week_qc_data),
-                "monthly": self.__calculate_rates(month_raw_data, month_qc_data),
-            },
-            "details": {
-                "weekly": {
-                    "start_time": (current_time - timedelta(days=7)).strftime('%Y-%m-%d %H:%M:%S'),
-                    "end_time": current_time.strftime('%Y-%m-%d %H:%M:%S')
-                },
-                "monthly": {
-                    "start_time": (current_time - timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S'),
-                    "end_time": current_time.strftime('%Y-%m-%d %H:%M:%S')
-                }
+                "weekly": {},
+                "monthly": {}
             }
         }
 
+        # 處理週數據 - 使用標準週時間範圍
+        for week_start, week_raw_data in weekly_raw_groups:
+            # 獲取對應的QC數據
+            week_qc_data = weekly_qc_groups.get_group(
+                week_start) if week_start in weekly_qc_groups.groups else pd.DataFrame()
+
+            if not week_raw_data.empty:
+                # 計算標準週結束時間（週日23:59:59）
+                week_end = week_start + pd.Timedelta(days=6, hours=23, minutes=59, seconds=59)
+
+                # 使用週的開始日期作為鍵
+                period_key = week_start.strftime('%Y-%m-%d')
+
+                report["rates"]["weekly"][period_key] = {
+                    "start_time": week_start.strftime('%Y-%m-%d %H:%M:%S'),
+                    "end_time": week_end.strftime('%Y-%m-%d %H:%M:%S'),
+                    "rates": self.__calculate_rates(week_raw_data, week_qc_data)
+                }
+
+        # 處理月數據 - 使用標準月時間範圍
+        for month_start, month_raw_data in monthly_raw_groups:
+            # 獲取對應的QC數據
+            month_qc_data = monthly_qc_groups.get_group(
+                month_start) if month_start in monthly_qc_groups.groups else pd.DataFrame()
+
+            if not month_raw_data.empty:
+                # 計算標準月結束時間（月末23:59:59）
+                next_month_start = (month_start + pd.Timedelta(days=32)).replace(day=1)
+                month_end = next_month_start - pd.Timedelta(seconds=1)
+
+                # 使用月份作為鍵
+                period_key = month_start.strftime('%Y-%m')
+
+                report["rates"]["monthly"][period_key] = {
+                    "start_time": month_start.strftime('%Y-%m-%d %H:%M:%S'),
+                    "end_time": month_end.strftime('%Y-%m-%d %H:%M:%S'),
+                    "rates": self.__calculate_rates(month_raw_data, month_qc_data)
+                }
+
         # 寫入報告
         with open(self.report_out, 'w') as f:
             json.dump(report, f, indent=4)
@@ -247,13 +283,28 @@ class AbstractReader(ABC):
         # Create new time index
         new_index = pd.date_range(user_start or df_start, user_end or df_end, freq=freq, name='time')
 
-        # Process data: convert to numeric, resample, and reindex
+        # Process data: convert to numeric, resample, and reindex with controlled tolerance
         if freq in ['1min', 'min', 'T']:
-            return _df.reindex(new_index, method='nearest', tolerance='1min')
+            # 對於分鐘級數據，使用較小的tolerance，例如30秒
+            return _df.reindex(new_index, method='nearest', tolerance='30s')
         elif freq in ['1h', 'h', 'H']:
-            return _df.reindex(new_index, method='nearest', tolerance='1h')
+            # 對於小時級數據，使用30分鐘作為tolerance
+            # 這樣08:20會匹配到08:00，但不會匹配到09:00
+            return _df.reindex(new_index, method='nearest', tolerance='30min')
         else:
-            return _df.reindex(new_index, method='nearest', tolerance=freq)
+            # 對於其他頻率，tolerance設置為頻率的一半
+            if isinstance(freq, str) and freq[-1].isalpha():
+                # 如果freq格式為'數字+單位'，例如'2h'，'3min'
+                try:
+                    num = int(freq[:-1])
+                    unit = freq[-1]
+                    half_freq = f"{num // 2}{unit}" if num > 1 else f"30{'min' if unit == 'h' else 's'}"
+                    return _df.reindex(new_index, method='nearest', tolerance=half_freq)
+                except ValueError:
+                    # 無法解析freq，使用默認值
+                    return _df.reindex(new_index, method='nearest', tolerance=freq)
+            else:
+                return _df.reindex(new_index, method='nearest', tolerance=freq)
 
     def _outlier_process(self, _df):
         outlier_file = self.path / 'outlier.json'

aeroviz-0.1.16/AeroViz/rawDataReader/script/TEOM.py

@@ -0,0 +1,101 @@
+import pandas as pd
+from pandas import to_datetime, read_csv, Timedelta, to_numeric
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'TEOM'
+
+    # TEOM Output Data Formats
+    # There are three data formats from TEOM instrument output:
+    #
+    # 1. Remote download format
+    #    - Identified by 'Time Stamp' column
+    #    - Date format: 'DD - MM - YYYY HH:MM:SS'
+    #    - May contain Chinese month names requiring conversion
+    #    - Maps columns: Time Stamp → time, System status → status,
+    #      PM-2.5 base MC → PM_NV, PM-2.5 MC → PM_Total, PM-2.5 TEOM noise → noise
+    #
+    # 2. USB download or auto export format
+    #    - Identified by 'tmoStatusCondition_0' column
+    #    - Two possible time formats:
+    #      a) Standard: 'Date' and 'Time' columns (YYYY-MM-DD HH:MM:SS)
+    #      b) Alternative: 'time_stamp' column (similar to remote format)
+    #    - Maps columns: tmoStatusCondition_0 → status, tmoTEOMABaseMC_0 → PM_NV,
+    #      tmoTEOMAMC_0 → PM_Total, tmoTEOMANoise_0 → noise
+    #
+    # 3. Other formats
+    #    - Not implemented, raises NotImplementedError
+    #
+    # All formats are standardized to the same column names with timestamp as index
+
+    def _raw_reader(self, file):
+        # TEOM Data Format Handler
+        # Standardizes different TEOM data formats (remote download and USB/auto export)
+        _df = read_csv(file, skiprows=3, index_col=False)
+
+        # Chinese month name conversion dictionary
+        _time_replace = {'十一月': '11', '十二月': '12', '一月': '01', '二月': '02', '三月': '03', '四月': '04',
+                         '五月': '05', '六月': '06', '七月': '07', '八月': '08', '九月': '09', '十月': '10'}
+
+        # Try both naming conventions (will ignore columns that don't exist)
+        _df = _df.rename(columns={
+            # Remote download format
+            'Time Stamp': 'time',
+            'System status': 'status',
+            'PM-2.5 base MC': 'PM_NV',
+            'PM-2.5 MC': 'PM_Total',
+            'PM-2.5 TEOM noise': 'noise',
+            # USB/auto export format
+            'time_stamp': 'time',
+            'tmoStatusCondition_0': 'status',
+            'tmoTEOMABaseMC_0': 'PM_NV',
+            'tmoTEOMAMC_0': 'PM_Total',
+            'tmoTEOMANoise_0': 'noise'
+        })
+
+        # Handle different time formats
+        if 'time' in _df.columns:  # Remote download or auto export with time column
+            _tm_idx = _df.time
+            # Convert Chinese month names if present
+            for _ori, _rpl in _time_replace.items():
+                _tm_idx = _tm_idx.str.replace(_ori, _rpl)
+
+            _df = _df.set_index(to_datetime(_tm_idx, errors='coerce', format='%d - %m - %Y %X'))
+
+        elif 'Date' in _df.columns and 'Time' in _df.columns:  # USB download format
+            _df['time'] = pd.to_datetime(_df['Date'] + ' ' + _df['Time'],
+                                         errors='coerce', format='%Y-%m-%d %H:%M:%S')
+            _df.drop(columns=['Date', 'Time'], inplace=True)
+            _df.set_index('time', inplace=True)
+
+        else:
+            raise NotImplementedError("Unsupported TEOM data format")
+
+        # Filter and clean data
+        _df = _df.where(_df['status'] < 1)
+        _df = _df[['PM_NV', 'PM_Total', 'noise']].apply(to_numeric, errors='coerce')
+
+        # Remove duplicates and NaN indices
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    # QC data
+    def _QC(self, _df):
+        _index = _df.index.copy()
+
+        # remove negative value
+        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df <= 0))
+
+        # QC data in 1 hr
+        # use time_aware_IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='6h')
+
+        # remove data where size < 50% in 1-hr
+        points_per_hour = Timedelta('1h') / Timedelta(self.meta['freq'])
+        for _key in ['PM_Total', 'PM_NV']:
+            _size = _df[_key].dropna().resample('1h').size().reindex(_index).ffill()
+            _df[_key] = _df[_key].mask(_size < points_per_hour * 0.5)
+
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

{aeroviz-0.1.15 → aeroviz-0.1.16}/AeroViz.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: AeroViz
-Version: 0.1.15
+Version: 0.1.16
 Summary: Aerosol science
 Author-email: alex <alex870521@gmail.com>
 License: MIT
@@ -111,7 +111,7 @@ data = RawDataReader(
 ```
 
 ```pycon
-> Concole output
+> Console output
 ╔════════════════════════════════════════════════════════════════════════════════╗
 ║     Reading NEPH RAW DATA from 2024-02-01 00:00:00 to 2024-04-30 23:59:59      ║
 ╚════════════════════════════════════════════════════════════════════════════════╝

{aeroviz-0.1.15 → aeroviz-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: AeroViz
-Version: 0.1.15
+Version: 0.1.16
 Summary: Aerosol science
 Author-email: alex <alex870521@gmail.com>
 License: MIT
@@ -111,7 +111,7 @@ data = RawDataReader(
 ```
 
 ```pycon
-> Concole output
+> Console output
 ╔════════════════════════════════════════════════════════════════════════════════╗
 ║     Reading NEPH RAW DATA from 2024-02-01 00:00:00 to 2024-04-30 23:59:59      ║
 ╚════════════════════════════════════════════════════════════════════════════════╝

{aeroviz-0.1.15 → aeroviz-0.1.16}/README.md

@@ -68,7 +68,7 @@ data = RawDataReader(
 ```
 
 ```pycon
-> Concole output
+> Console output
 ╔════════════════════════════════════════════════════════════════════════════════╗
 ║     Reading NEPH RAW DATA from 2024-02-01 00:00:00 to 2024-04-30 23:59:59      ║
 ╚════════════════════════════════════════════════════════════════════════════════╝

{aeroviz-0.1.15 → aeroviz-0.1.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "AeroViz"
-version = "0.1.15"
+version = "0.1.16"
 description = "Aerosol science"
 authors = [{ name = "alex", email = "alex870521@gmail.com" }]
 license = { text = "MIT" }

aeroviz-0.1.15/AeroViz/rawDataReader/script/TEOM.py

@@ -1,64 +0,0 @@
-import pandas as pd
-from pandas import to_datetime, read_csv, Timedelta, to_numeric
-
-from AeroViz.rawDataReader.core import AbstractReader
-
-
-class Reader(AbstractReader):
-    nam = 'TEOM'
-
-    def _raw_reader(self, file):
-        _df = read_csv(file, skiprows=3, index_col=False)
-
-        if 'Time Stamp' in _df.columns:  # remote download
-            _df = _df.rename(columns={'Time Stamp': 'time',
-                                      'System status': 'status',
-                                      'PM-2.5 base MC': 'PM_NV',
-                                      'PM-2.5 MC': 'PM_Total',
-                                      'PM-2.5 TEOM noise': 'noise', })
-
-            _time_replace = {'十一月': '11', '十二月': '12', '一月': '01', '二月': '02', '三月': '03', '四月': '04',
-                             '五月': '05', '六月': '06', '七月': '07', '八月': '08', '九月': '09', '十月': '10'}
-
-            _tm_idx = _df.time
-            for _ori, _rpl in _time_replace.items():
-                _tm_idx = _tm_idx.str.replace(_ori, _rpl)
-
-            _df = _df.set_index(to_datetime(_tm_idx, errors='coerce', format='%d - %m - %Y %X'))
-
-        elif 'tmoStatusCondition_0' in _df.columns:  # usb download
-            _df['time'] = pd.to_datetime(_df['Date'] + ' ' + _df['Time'], errors='coerce', format='%Y-%m-%d %H:%M:%S')
-            _df.drop(columns=['Date', 'Time'], inplace=True)
-            _df.set_index('time', inplace=True)
-
-            _df = _df.rename(columns={'tmoStatusCondition_0': 'status',
-                                      'tmoTEOMABaseMC_0': 'PM_NV',
-                                      'tmoTEOMAMC_0': 'PM_Total',
-                                      'tmoTEOMANoise_0': 'noise', })
-        else:
-            raise NotImplementedError
-
-        _df = _df.where(_df['status'] < 1)
-        _df = _df[['PM_NV', 'PM_Total', 'noise']].apply(to_numeric, errors='coerce')
-
-        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
-
-    # QC data
-    def _QC(self, _df):
-        _index = _df.index.copy()
-
-        # remove negative value
-        _df = _df.where(_df.noise < 0.01)[['PM_NV', 'PM_Total']].mask((_df <= 0))
-
-        # QC data in 1 hr
-        # use time_aware_IQR_QC
-        _df = self.time_aware_IQR_QC(_df, time_window='6h')
-
-        # remove data where size < 50% in 1-hr
-        points_per_hour = Timedelta('1h') / Timedelta(self.meta['freq'])
-        for _key in ['PM_Total', 'PM_NV']:
-            _size = _df[_key].dropna().resample('1h').size().reindex(_index).ffill()
-            _df[_key] = _df[_key].mask(_size < points_per_hour * 0.5)
-
-        # make sure all columns have values, otherwise set to nan
-        return _df.dropna(how='any').reindex(_index)