AeroViz 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

AeroViz/rawDataReader/core/__init__.py

@@ -1,19 +1,18 @@
 import json
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Optional, Generator
+from typing import Generator
 
 import numpy as np
 import pandas as pd
-from pandas import DataFrame, concat, read_pickle, to_numeric
 from rich.console import Console
-from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn, TaskProgressColumn
+from rich.progress import Progress, TextColumn, BarColumn, SpinnerColumn, TaskProgressColumn
 
 from AeroViz.rawDataReader.config.supported_instruments import meta
 from AeroViz.rawDataReader.core.logger import ReaderLogger
-from AeroViz.rawDataReader.core.qc import DataQualityControl
+from AeroViz.rawDataReader.core.qc import QualityControl
 
 __all__ = ['AbstractReader']
 
@@ -32,45 +31,44 @@ class AbstractReader(ABC):
 
     def __init__(self,
                  path: Path | str,
-                 reset: bool = False,
-                 qc: bool = True,
-                 qc_freq: Optional[str] = None,
-                 rate: bool = True,
-                 append_data: bool = False,
+                 reset: bool | str = False,
+                 qc: bool | str = True,
                  **kwargs):
 
         self.path = Path(path)
         self.meta = meta[self.nam]
-        self.logger = ReaderLogger(self.nam, self.path)
-
-        self.reset = reset
-        self.qc = qc
-        self.qc_freq = qc_freq
-        self.rate = rate
-        self.append = append_data and reset
-
-        self.pkl_nam = self.path / f'_read_{self.nam.lower()}.pkl'
-        self.csv_nam = self.path / f'_read_{self.nam.lower()}.csv'
-        self.pkl_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.pkl'
-        self.csv_nam_raw = self.path / f'_read_{self.nam.lower()}_raw.csv'
-        self.csv_out = self.path / f'output_{self.nam.lower()}.csv'
-
-        self.size_range = kwargs.get('size_range', (11.8, 593.5))
+        output_folder = self.path / f'{self.nam.lower()}_outputs'
+        output_folder.mkdir(parents=True, exist_ok=True)
+
+        self.logger = ReaderLogger(
+            self.nam, output_folder,
+            kwargs.get('log_level').upper() if not kwargs.get('suppress_warnings') else 'ERROR')
+
+        self.reset = reset is True
+        self.append = reset == 'append'
+        self.qc = qc  # if qc, then calculate rate
+        self.qc_freq = qc if isinstance(qc, str) else None
+        self.kwargs = kwargs
+
+        self.pkl_nam = output_folder / f'_read_{self.nam.lower()}.pkl'
+        self.csv_nam = output_folder / f'_read_{self.nam.lower()}.csv'
+        self.pkl_nam_raw = output_folder / f'_read_{self.nam.lower()}_raw.pkl'
+        self.csv_nam_raw = output_folder / f'_read_{self.nam.lower()}_raw.csv'
+        self.csv_out = output_folder / f'output_{self.nam.lower()}.csv'
+        self.report_out = output_folder / 'report.json'
 
     def __call__(self,
                  start: datetime,
                  end: datetime,
                  mean_freq: str = '1h',
-                 csv_out: bool = True,
-                 ) -> DataFrame:
+                 ) -> pd.DataFrame:
 
         data = self._run(start, end)
 
         if data is not None:
-            if mean_freq:
-                data = data.resample(mean_freq).mean()
-            if csv_out:
-                data.to_csv(self.csv_out)
+            data = data.resample(mean_freq).mean()
+
+        data.to_csv(self.csv_out)
 
         return data
 
@@ -79,58 +77,147 @@ class AbstractReader(ABC):
         pass
 
     @abstractmethod
-    def _QC(self, df: DataFrame) -> DataFrame:
+    def _QC(self, df: pd.DataFrame) -> pd.DataFrame:
         return df
 
-    def _rate_calculate(self, raw_data, qc_data) -> None:
-        def __base_rate(raw_data, qc_data):
-            period_size = len(raw_data.resample('1h').mean().index)
+    def __calculate_rates(self, raw_data, qc_data, all_keys=False, with_log=False):
+        """計算獲取率、良率和總比率
 
-            for _nam, _key in self.meta['deter_key'].items():
-                _columns_key, _drop_how = (qc_data.keys(), 'all') if _key == ['all'] else (_key, 'any')
+        Args:
+            raw_data: 原始數據
+            qc_data: QC後的數據
+            all_keys: 是否計算所有 deter_key
+            with_log: 是否輸出計算日誌
+        """
+        if raw_data.empty or qc_data.empty:
+            return {'acquisition_rate': 0, 'yield_rate': 0, 'total_rate': 0}
 
-                sample_size = len(raw_data[_columns_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
-                qc_size = len(qc_data[_columns_key].resample('1h').mean().copy().dropna(how=_drop_how).index)
+        def _calculate_single_key(key_name, key_columns):
+            columns, drop_how = (qc_data.keys(), 'all') if key_columns == ['all'] else (key_columns, 'any')
 
-                # validate rate calculation
-                if period_size == 0 or sample_size == 0 or qc_size == 0:
+            # 重採樣並計算有效數據量
+            period_size = len(raw_data.resample('1h').mean().index)
+            sample_size = len(raw_data[columns].resample('1h').mean().dropna(how=drop_how).index)
+            qc_size = len(qc_data[columns].resample('1h').mean().dropna(how=drop_how).index)
+
+            # 驗證計算
+            if any([
+                period_size == 0 or sample_size == 0 or qc_size == 0,
+                period_size < sample_size,
+                sample_size < qc_size
+            ]):
+                if with_log:
                     self.logger.warning(f'\t\t No data for this period... skip')
-                    continue
-                if period_size < sample_size:
-                    self.logger.warning(f'\t\tError: Sample({sample_size}) > Period({period_size})... skip')
-                    continue
-                if sample_size < qc_size:
-                    self.logger.warning(f'\t\tError: QC({qc_size}) > Sample({sample_size})... skip')
-                    continue
-
-                else:
-                    _sample_rate = round((sample_size / period_size) * 100, 1)
-                    _valid_rate = round((qc_size / sample_size) * 100, 1)
-                    _total_rate = round((qc_size / period_size) * 100, 1)
-
-                self.logger.info(f"\t\t{self.logger.CYAN}{self.logger.ARROW} {_nam}{self.logger.RESET}")
+                return None
+
+            # 計算比率
+            sample_rate = round((sample_size / period_size) * 100, 1)
+            valid_rate = round((qc_size / sample_size) * 100, 1)
+            total_rate = round((qc_size / period_size) * 100, 1)
+
+            if with_log:
+                self.logger.info(f"\t\t> {key_name}")
                 self.logger.info(
-                    f"\t\t\t├─ {'Sample Rate':15}: {self.logger.BLUE}{_sample_rate:>6.1f}%{self.logger.RESET}")
+                    f"\t\t\t> {'Sample Rate':13}: {self.logger.BLUE}{sample_rate:>6.1f}%{self.logger.RESET}")
                 self.logger.info(
-                    f"\t\t\t├─ {'Valid  Rate':15}: {self.logger.BLUE}{_valid_rate:>6.1f}%{self.logger.RESET}")
+                    f"\t\t\t> {'Valid  Rate':13}: {self.logger.BLUE}{valid_rate:>6.1f}%{self.logger.RESET}")
                 self.logger.info(
-                    f"\t\t\t└─ {'Total  Rate':15}: {self.logger.BLUE}{_total_rate:>6.1f}%{self.logger.RESET}")
+                    f"\t\t\t> {'Total  Rate':13}: {self.logger.BLUE}{total_rate:>6.1f}%{self.logger.RESET}")
+
+            return {
+                'acquisition_rate': sample_rate,
+                'yield_rate': valid_rate,
+                'total_rate': total_rate
+            }
+
+        if all_keys:
+            # 計算所有 key 並回傳所有結果（用於日誌輸出）
+            all_results = []
+            for name, columns in self.meta['deter_key'].items():
+                result = _calculate_single_key(name, columns)
+                if result:
+                    all_results.append(result)
+
+            if not all_results:
+                return {'acquisition_rate': 0, 'yield_rate': 0, 'total_rate': 0}
+
+            # 回傳所有結果中比率最低的
+            return {
+                'acquisition_rate': min(r['acquisition_rate'] for r in all_results),
+                'yield_rate': min(r['yield_rate'] for r in all_results),
+                'total_rate': min(r['total_rate'] for r in all_results)
+            }
+        else:
+            # 計算所有 key 但只回傳最低的比率
+            min_rates = {'acquisition_rate': 200, 'yield_rate': 200, 'total_rate': 200}
+
+            for name, columns in self.meta['deter_key'].items():
+                result = _calculate_single_key(name, columns)
+                if result:
+                    min_rates['acquisition_rate'] = min(min_rates['acquisition_rate'], result['acquisition_rate'])
+                    min_rates['yield_rate'] = min(min_rates['yield_rate'], result['yield_rate'])
+                    min_rates['total_rate'] = min(min_rates['total_rate'], result['total_rate'])
+
+            # 如果沒有任何有效結果，回傳 0
+            if min_rates['acquisition_rate'] == 200 and min_rates['yield_rate'] == 200:
+                return {'acquisition_rate': 0, 'yield_rate': 0, 'total_rate': 0}
 
+            return min_rates
+
+    def _rate_calculate(self, raw_data, qc_data) -> None:
         if self.meta['deter_key'] is not None:
-            # use qc_freq to calculate each period rate
             if self.qc_freq is not None:
                 raw_data_grouped = raw_data.groupby(pd.Grouper(freq=self.qc_freq))
                 qc_data_grouped = qc_data.groupby(pd.Grouper(freq=self.qc_freq))
 
                 for (month, _sub_raw_data), (_, _sub_qc_data) in zip(raw_data_grouped, qc_data_grouped):
                     self.logger.info(
-                        f"\t{self.logger.BLUE}{self.logger.ARROW} Processing: {_sub_raw_data.index[0].strftime('%F')}"
+                        f"\t{self.logger.BLUE}> Processing: {_sub_raw_data.index[0].strftime('%F')}"
                         f" to {_sub_raw_data.index[-1].strftime('%F')}{self.logger.RESET}")
 
-                    __base_rate(_sub_raw_data, _sub_qc_data)
-
+                    self.__calculate_rates(_sub_raw_data, _sub_qc_data, all_keys=True, with_log=True)
             else:
-                __base_rate(raw_data, qc_data)
+                self.__calculate_rates(raw_data, qc_data, all_keys=True, with_log=True)
+
+            # 計算週和月的數據
+            current_time = datetime.now()
+            week_mask = raw_data.index >= current_time - timedelta(days=7)
+            month_mask = raw_data.index >= current_time - timedelta(days=30)
+
+            # 生成報告
+            self.__generate_report(
+                current_time,
+                raw_data[week_mask], qc_data[week_mask],
+                raw_data[month_mask], qc_data[month_mask]
+            )
+
+    def __generate_report(self, current_time, week_raw_data, week_qc_data, month_raw_data, month_qc_data):
+        """生成獲取率和良率的報告"""
+        report = {
+            "report_time": current_time.strftime('%Y-%m-%d %H:%M:%S'),
+            "instrument_info": {
+                "station": self.path.name[:2],
+                "instrument": self.nam
+            },
+            "rates": {
+                "weekly": self.__calculate_rates(week_raw_data, week_qc_data),
+                "monthly": self.__calculate_rates(month_raw_data, month_qc_data),
+            },
+            "details": {
+                "weekly": {
+                    "start_time": (current_time - timedelta(days=7)).strftime('%Y-%m-%d %H:%M:%S'),
+                    "end_time": current_time.strftime('%Y-%m-%d %H:%M:%S')
+                },
+                "monthly": {
+                    "start_time": (current_time - timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S'),
+                    "end_time": current_time.strftime('%Y-%m-%d %H:%M:%S')
+                }
+            }
+        }
+
+        # 寫入報告
+        with open(self.report_out, 'w') as f:
+            json.dump(report, f, indent=4)
 
     def _timeIndex_process(self, _df, user_start=None, user_end=None, append_df=None):
         """
@@ -182,7 +269,7 @@ class AbstractReader(ABC):
 
         return _df
 
-    def _save_data(self, raw_data: DataFrame, qc_data: DataFrame) -> None:
+    def _save_data(self, raw_data: pd.DataFrame, qc_data: pd.DataFrame) -> None:
         try:
             raw_data.to_pickle(self.pkl_nam_raw)
             raw_data.to_csv(self.csv_nam_raw)
@@ -205,15 +292,15 @@ class AbstractReader(ABC):
 
         try:
             with Progress(
-                    TextColumn("[bold blue]{task.description}", style="bold blue"),
+                    SpinnerColumn(finished_text="✓"),
                     BarColumn(bar_width=25, complete_style="green", finished_style="bright_green"),
-                    TaskProgressColumn(),
-                    TimeRemainingColumn(),
-                    TextColumn("{task.fields[filename]}", style="yellow"),
+                    TaskProgressColumn(style="bold", text_format="[bright_green]{task.percentage:>3.0f}%"),
+                    TextColumn("{task.description}", style="bold blue"),
+                    TextColumn("{task.fields[filename]}", style="bold blue"),
                     console=Console(force_terminal=True, color_system="auto", width=120),
                     expand=False
             ) as progress:
-                task = progress.add_task(f"{self.logger.ARROW} Reading {self.nam} files", total=len(files), filename="")
+                task = progress.add_task(f"Reading {self.nam} files:", total=len(files), filename="")
                 yield progress, task
         finally:
             # Restore logger method and output message
@@ -222,7 +309,7 @@ class AbstractReader(ABC):
                 for msg in msgs:
                     original[level](msg)
 
-    def _read_raw_files(self) -> tuple[DataFrame | None, DataFrame | None]:
+    def _read_raw_files(self) -> tuple[pd.DataFrame | None, pd.DataFrame | None]:
         files = [f
                  for file_pattern in self.meta['pattern']
                  for pattern in {file_pattern.lower(), file_pattern.upper(), file_pattern}
@@ -242,7 +329,7 @@ class AbstractReader(ABC):
                     if (df := self._raw_reader(file)) is not None and not df.empty:
                         df_list.append(df)
                     else:
-                        self.logger.warning(f"\tFile {file.name} produced an empty DataFrame or None.")
+                        self.logger.debug(f"\tFile {file.name} produced an empty DataFrame or None.")
 
                 except Exception as e:
                     self.logger.error(f"Error reading {file.name}: {e}")
@@ -250,25 +337,27 @@ class AbstractReader(ABC):
         if not df_list:
             raise ValueError(f"\033[41m\033[97mAll files were either empty or failed to read.\033[0m")
 
-        raw_data = concat(df_list, axis=0).groupby(level=0).first()
+        raw_data = pd.concat(df_list, axis=0).groupby(level=0).first()
 
-        if self.nam == 'SMPS':
+        if self.nam in ['SMPS', 'APS', 'GRIMM']:
             raw_data = raw_data.sort_index(axis=1, key=lambda x: x.astype(float))
 
-        raw_data = self._timeIndex_process(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
-        qc_data = self._QC(raw_data).apply(to_numeric, errors='coerce').copy(deep=True)
+        raw_data = self._timeIndex_process(raw_data)
+
+        raw_data = raw_data.apply(pd.to_numeric, errors='coerce').copy(deep=True)
+        qc_data = self._QC(raw_data).apply(pd.to_numeric, errors='coerce').copy(deep=True)
 
         return raw_data, qc_data
 
     def _run(self, user_start, user_end):
         # read pickle if pickle file exists and 'reset=False' or process raw data or append new data
         if self.pkl_nam_raw.exists() and self.pkl_nam.exists() and not self.reset:
-            self.logger.info_box(f"Reading {self.nam} PICKLE from {user_start} to {user_end}", color_part="PICKLE")
+            self.logger.info_box(f"Reading {self.nam} PICKLE from {user_start} to {user_end}")
 
-            _f_raw_done, _f_qc_done = read_pickle(self.pkl_nam_raw), read_pickle(self.pkl_nam)
+            _f_raw_done, _f_qc_done = pd.read_pickle(self.pkl_nam_raw), pd.read_pickle(self.pkl_nam)
 
             if self.append:
-                self.logger.info_box(f"Appending New data from {user_start} to {user_end}", color_part="New data")
+                self.logger.info_box(f"Appending New data from {user_start} to {user_end}")
 
                 _f_raw_new, _f_qc_new = self._read_raw_files()
                 _f_raw = self._timeIndex_process(_f_raw_done, append_df=_f_raw_new)
@@ -280,7 +369,7 @@ class AbstractReader(ABC):
                 return _f_qc if self.qc else _f_raw
 
         else:
-            self.logger.info_box(f"Reading {self.nam} RAW DATA from {user_start} to {user_end}", color_part="RAW DATA")
+            self.logger.info_box(f"Reading {self.nam} RAW DATA from {user_start} to {user_end}")
 
             _f_raw, _f_qc = self._read_raw_files()
 
@@ -292,25 +381,26 @@ class AbstractReader(ABC):
         # save
         self._save_data(_f_raw, _f_qc)
 
-        if self.rate:
-            self._rate_calculate(_f_raw.apply(to_numeric, errors='coerce'), _f_qc.apply(to_numeric, errors='coerce'))
+        if self.qc:
+            self._rate_calculate(_f_raw.apply(pd.to_numeric, errors='coerce'),
+                                 _f_qc.apply(pd.to_numeric, errors='coerce'))
 
         return _f_qc if self.qc else _f_raw
 
     @staticmethod
-    def reorder_dataframe_columns(df, order_lists, others_col=False):
+    def reorder_dataframe_columns(df, order_lists: list[list], keep_others: bool = False):
         new_order = []
 
         for order in order_lists:
-            # 只添加存在於DataFrame中的欄位，且不重複添加
+            # Only add column that exist in the DataFrame and do not add them repeatedly
             new_order.extend([col for col in order if col in df.columns and col not in new_order])
 
-        if others_col:
-            # 添加所有不在新順序列表中的原始欄位，保持它們的原始順序
+        if keep_others:
+            # Add all original fields not in the new order list, keeping their original order
             new_order.extend([col for col in df.columns if col not in new_order])
 
         return df[new_order]
 
     @staticmethod
     def time_aware_IQR_QC(df: pd.DataFrame, time_window='1D', log_dist=False) -> pd.DataFrame:
-        return DataQualityControl().time_aware_iqr(df, time_window=time_window, log_dist=log_dist)
+        return QualityControl().time_aware_iqr(df, time_window=time_window, log_dist=log_dist)

AeroViz/rawDataReader/core/logger.py

@@ -8,9 +8,10 @@ from pathlib import Path
 
 
 class ReaderLogger:
-    def __init__(self, name: str, log_path: Path):
+    def __init__(self, name: str, log_path: Path, log_level: str = 'INFO'):
         self.name = name
         self.log_path = log_path
+        self._log_level = getattr(logging, log_level)
 
         # 檢查是否支持顏色輸出
         self.color_support = self._check_color_support()
@@ -32,16 +33,16 @@ class ReaderLogger:
             self.RESET = ''
 
         # 檢查 Unicode 支持
-        self.unicode_support = self._setup_unicode()
+        self.unicode_support = self._check_unicode_support()
 
         # 設置框架字符
         if self.unicode_support:
-            self.BOX_TOP_LEFT = "╔"
-            self.BOX_TOP_RIGHT = "╗"
-            self.BOX_BOTTOM_LEFT = "╚"
-            self.BOX_BOTTOM_RIGHT = "╝"
-            self.BOX_HORIZONTAL = "═"
-            self.BOX_VERTICAL = "║"
+            self.BOX_TOP_LEFT = "╭"
+            self.BOX_TOP_RIGHT = "╮"
+            self.BOX_BOTTOM_LEFT = "╰"
+            self.BOX_BOTTOM_RIGHT = "╯"
+            self.BOX_HORIZONTAL = "─"
+            self.BOX_VERTICAL = "│"
             self.ARROW = "▶"
         else:
             self.BOX_TOP_LEFT = "+"
@@ -74,7 +75,7 @@ class ReaderLogger:
         # 其他系統檢查
         return hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
 
-    def _setup_unicode(self) -> bool:
+    def _check_unicode_support(self) -> bool:
         """設置 Unicode 支持"""
         if platform.system().lower() == 'windows':
             try:
@@ -92,7 +93,7 @@ class ReaderLogger:
     def _setup_logger(self) -> logging.Logger:
         """設置logger"""
         logger = logging.getLogger(self.name)
-        logger.setLevel(logging.INFO)
+        logger.setLevel(self._log_level)
 
         # 移除現有的 handlers
         for handler in logger.handlers[:]:
@@ -135,6 +136,9 @@ class ReaderLogger:
             text = text.encode('ascii', 'replace').decode('ascii')
         return text
 
+    def debug(self, msg: str):
+        self.logger.debug(self._safe_print(msg))
+
     def info(self, msg: str):
         self.logger.info(self._safe_print(msg))
 

AeroViz/rawDataReader/core/qc.py

@@ -2,7 +2,7 @@ import numpy as np
 import pandas as pd
 
 
-class DataQualityControl:
+class QualityControl:
     """A class providing various methods for data quality control and outlier detection"""
 
     @staticmethod

AeroViz/rawDataReader/script/AE33.py

@@ -8,7 +8,7 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         if file.stat().st_size / 1024 < 550:
-            self.logger.info(f'\t {file.name} may not be a whole daily data. Make sure the file is correct.')
+            self.logger.warning(f'\t {file.name} may not be a whole daily data. Make sure the file is correct.')
 
         _df = read_table(file, parse_dates={'time': [0, 1]}, index_col='time',
                          delimiter=r'\s+', skiprows=5, usecols=range(67))

AeroViz/rawDataReader/script/BAM1020.py

@@ -0,0 +1,35 @@
+from pandas import read_csv, to_numeric, NA
+
+from AeroViz.rawDataReader.core import AbstractReader
+
+
+class Reader(AbstractReader):
+    nam = 'BAM1020'
+
+    def _raw_reader(self, file):
+        PM = 'Conc'
+
+        _df = read_csv(file, parse_dates=True, index_col=0, usecols=range(0, 21))
+        _df.rename(columns={'Conc (mg/m3)': PM}, inplace=True)
+
+        # remove data when Conc = 1 or 0
+        _df[PM] = _df[PM].replace(1, NA)
+
+        _df = _df[[PM]].apply(to_numeric, errors='coerce')
+
+        # tranfer unit from mg/m3 to ug/m3
+        _df = _df * 1000
+
+        return _df.loc[~_df.index.duplicated() & _df.index.notna()]
+
+    def _QC(self, _df):
+        _index = _df.index.copy()
+
+        # remove negative value
+        _df = _df.mask((_df <= 0) | (_df > 500))
+
+        # use IQR_QC
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
+
+        # make sure all columns have values, otherwise set to nan
+        return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/NEPH.py

@@ -1,4 +1,4 @@
-from pandas import to_datetime, read_csv, DataFrame, to_numeric
+from pandas import to_datetime, read_csv, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
 
@@ -46,17 +46,13 @@ class Reader(AbstractReader):
 
                 return _df.loc[~_df.index.duplicated() & _df.index.notna()]
 
-            except ValueError:
-                # Define valid groups and find invalid indices
+            except ValueError:  # Define valid groups and find invalid indices
                 invalid_indices = _df[~_df[0].isin({'B', 'G', 'R', 'D', 'T', 'Y', 'Z'})].index
-                print("Invalid values and their indices:")
-                print("\n".join([f"Index: {idx}, Value: {_df.at[idx, 0]}" for idx in invalid_indices]))
+                self.logger.warning(
+                    f"\tInvalid values in {file.name}: {', '.join(f'{_}:{_df.at[_, 0]}' for _ in invalid_indices)}."
+                    f" Skipping file.")
 
-                # Return an empty DataFrame with specified columns if there's a length mismatch
-                _df_out = DataFrame(index=_idx_tm, columns=['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH'])
-                _df_out.index.name = 'Time'
-                print(f'\n\t\t\t Length mismatch in {file} data. Returning an empty DataFrame.')
-                return _df_out
+                return None
 
     def _QC(self, _df):
         MDL_sensitivity = {'B': .1, 'G': .1, 'R': .3}

AeroViz/rawDataReader/script/SMPS.py

@@ -36,7 +36,16 @@ class Reader(AbstractReader):
             skip = find_header_row(f, delimiter)
             f.seek(0)
 
-            _df = read_csv(f, sep=delimiter, skiprows=skip)
+            _df = read_csv(f, sep=delimiter, skiprows=skip, low_memory=False)
+
+            if 'Date' not in _df.columns:  # 資料需要轉置
+                try:
+                    _df = _df.T  # 轉置
+                    _df.columns = _df.iloc[0]  # 使用第一列作為欄位名稱
+                    _df = _df.iloc[1:]  # 移除第一列（因為已經變成欄位名稱）
+                    _df = _df.reset_index(drop=True)  # 重設索引
+                except:
+                    raise NotImplementedError('Not supported date format')
 
             for date_format in date_formats:
                 _time_index = parse_date(_df, date_format)
@@ -56,9 +65,12 @@ class Reader(AbstractReader):
             _df_smps.columns = _df_smps.columns.astype(float)
             _df_smps = _df_smps.loc[_df_smps.index.dropna().copy()]
 
-            if _df_smps.columns[0] != self.size_range[0] or _df_smps.columns[-1] != self.size_range[1]:
-                self.logger.info(f'\tSMPS file: {file.name} is not match the default size range {self.size_range}, '
-                                 f'it is ({_df_smps.columns[0]}, {_df_smps.columns[-1]})')
+            size_range = self.kwargs.get('size_range') or (11.8, 593.5)
+
+            if _df_smps.columns[0] != size_range[0] or _df_smps.columns[-1] != size_range[1]:
+                self.logger.warning(f'\tSMPS file: {file.name} is not match the setting size range {size_range}, '
+                                    f'it is ({_df_smps.columns[0]}, {_df_smps.columns[-1]}). '
+                                    f'Please run by another RawDataReader instance, and set the correct size range')
                 return None
 
             return _df_smps.apply(to_numeric, errors='coerce')
@@ -68,8 +80,10 @@ class Reader(AbstractReader):
         _df = _df.copy()
         _index = _df.index.copy()
 
-        size_range_mask = (_df.columns.astype(float) >= self.size_range[0]) & (
-                _df.columns.astype(float) <= self.size_range[1])
+        size_range = self.kwargs.get('size_range') or (11.8, 593.5)
+
+        size_range_mask = (_df.columns.astype(float) >= size_range[0]) & (
+                _df.columns.astype(float) <= size_range[1])
         _df = _df.loc[:, size_range_mask]
 
         # mask out the data size lower than 7

AeroViz/rawDataReader/script/TEOM.py

@@ -1,3 +1,4 @@
+import pandas as pd
 from pandas import to_datetime, read_csv, Timedelta, to_numeric
 
 from AeroViz.rawDataReader.core import AbstractReader
@@ -7,9 +8,9 @@ class Reader(AbstractReader):
     nam = 'TEOM'
 
     def _raw_reader(self, file):
-        with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3, index_col=False)
+        _df = read_csv(file, skiprows=3, index_col=False)
 
+        if 'Time Stamp' in _df.columns:  # remote download
             _df = _df.rename(columns={'Time Stamp': 'time',
                                       'System status': 'status',
                                       'PM-2.5 base MC': 'PM_NV',
@@ -25,8 +26,19 @@ class Reader(AbstractReader):
 
             _df = _df.set_index(to_datetime(_tm_idx, errors='coerce', format='%d - %m - %Y %X'))
 
-            _df = _df.where(_df['status'] < 1)
+        elif 'tmoStatusCondition_0' in _df.columns:  # usb download
+            _df['time'] = pd.to_datetime(_df['Date'] + ' ' + _df['Time'], errors='coerce', format='%Y-%m-%d %H:%M:%S')
+            _df.drop(columns=['Date', 'Time'], inplace=True)
+            _df.set_index('time', inplace=True)
 
+            _df = _df.rename(columns={'tmoStatusCondition_0': 'status',
+                                      'tmoTEOMABaseMC_0': 'PM_NV',
+                                      'tmoTEOMAMC_0': 'PM_Total',
+                                      'tmoTEOMANoise_0': 'noise', })
+        else:
+            raise NotImplementedError
+
+        _df = _df.where(_df['status'] < 1)
         _df = _df[['PM_NV', 'PM_Total', 'noise']].apply(to_numeric, errors='coerce')
 
         return _df.loc[~_df.index.duplicated() & _df.index.notna()]

AeroViz/rawDataReader/script/__init__.py

@@ -8,6 +8,7 @@ __all__ = [
     'AE43',
     'BC1054',
     'MA350',
+    'BAM1020',
     'TEOM',
     'OCEC',
     'IGAC',