AeroViz 0.1.0__py3-none-any.whl

AeroViz/process/script/IMPROVE.py

@@ -0,0 +1,161 @@
+from typing import Literal
+
+from pathlib import Path
+from pandas import read_csv, concat, read_json
+
+from AeroViz.process.core import DataProc
+from AeroViz.tools.datareader import DataReader
+
+
+class ImproveProc(DataProc):
+	"""
+	A class for process improved chemical data.
+
+	Parameters:
+	-----------
+	reset : bool, optional
+		If True, resets the process. Default is False.
+	filename : str, optional
+		The name of the file to process. Default is None.
+	version : str, optional
+		The version of the data process. Should be one of 'revised' or 'modified'.
+		Default is None.
+
+	Methods:
+	--------
+	revised(_df):
+		Calculate revised version of particle contribution.
+
+	modified(_df):
+		Calculate modified version of particle contribution.
+
+	gas(_df):
+		Calculate gas contribution.
+
+	frh(_RH, version=None):
+		Helper function to get frh values based on relative humidity (RH) and version.
+
+	process_data():
+		Process data and save the result.
+
+	Attributes:
+	-----------
+	DEFAULT_PATH : Path
+		The default path for data files.
+
+	Examples:
+	---------
+	>>> df = ImproveProc(reset=True, filename='revised_IMPROVE.csv', version='revised').process_data()
+
+	"""
+
+	def __init__(self, file_paths: list[Path | str] = None):
+		super().__init__()
+		self.file_paths = [Path(fp) for fp in file_paths]
+
+	@staticmethod
+	def frh(_RH):
+		_frh = read_json(Path(__file__).parent.parent.parent / 'plot' / 'utils' / 'fRH.json')
+		if _RH is not None:
+			if _RH > 95:
+				_RH = 95
+			_RH = round(_RH)
+			return _frh.loc[_RH].values.T
+
+		return 1, 1, 1, 1
+
+	def revised(self, _df):
+		def mode(Mass):
+			L_mode = Mass ** 2 / 20 if Mass < 20 else Mass
+			S_mode = Mass - L_mode if Mass < 20 else 0
+
+			return L_mode, S_mode
+
+		_frh, _frhss, _frhs, _frhl = self.frh(_df['RH'])
+
+		L_AS, S_AS = mode(_df['AS'])
+		L_AN, S_AN = mode(_df['AN'])
+		L_OM, S_OM = mode(_df['OM'])
+
+		_df['AS_ext_dry'] = 2.2 * 1 * S_AS + 4.8 * 1 * L_AS
+		_df['AN_ext_dry'] = 2.4 * 1 * S_AN + 5.1 * 1 * L_AN
+		_df['OM_ext_dry'] = 2.8 * S_OM + 6.1 * L_OM
+		_df['Soil_ext_dry'] = 1 * _df['Soil']
+		_df['SS_ext_dry'] = 1.7 * 1 * _df['SS']
+		_df['EC_ext_dry'] = 10 * _df['EC']
+		_df['total_ext_dry'] = sum(_df['AS_ext_dry':'EC_ext_dry'])
+
+		_df['AS_ext'] = (2.2 * _frhs * S_AS) + (4.8 * _frhl * L_AS)
+		_df['AN_ext'] = (2.4 * _frhs * S_AN) + (5.1 * _frhl * L_AN)
+		_df['OM_ext'] = (2.8 * S_OM) + (6.1 * L_OM)
+		_df['Soil_ext'] = (1 * _df['Soil'])
+		_df['SS_ext'] = (1.7 * _frhss * _df['SS'])
+		_df['EC_ext'] = (10 * _df['EC'])
+		_df['total_ext'] = sum(_df['AS_ext':'EC_ext'])
+
+		_df['ALWC_AS_ext'] = _df['AS_ext'] - _df['AS_ext_dry']
+		_df['ALWC_AN_ext'] = _df['AN_ext'] - _df['AN_ext_dry']
+		_df['ALWC_SS_ext'] = _df['SS_ext'] - _df['SS_ext_dry']
+		_df['ALWC_ext'] = _df['total_ext'] - _df['total_ext_dry']
+
+		_df['fRH_IMPR'] = _df['total_ext'] / _df['total_ext_dry']
+
+		return _df['AS_ext_dry':]
+
+	def modified(self, _df):
+		_frh, _frhss, _frhs, _frhl = self.frh(_df['RH'])
+
+		_df['AS_ext_dry'] = 3 * 1 * _df['AS']
+		_df['AN_ext_dry'] = 3 * 1 * _df['AN']
+		_df['OM_ext_dry'] = 4 * _df['OM']
+		_df['Soil_ext_dry'] = 1 * _df['Soil']
+		_df['SS_ext_dry'] = 1.7 * 1 * _df['SS']
+		_df['EC_ext_dry'] = 10 * _df['EC']
+		_df['total_ext_dry'] = sum(_df['AS_ext_dry':'EC_ext_dry'])
+
+		_df['AS_ext'] = (3 * _frh * _df['AS'])
+		_df['AN_ext'] = (3 * _frh * _df['AN'])
+		_df['OM_ext'] = (4 * _df['OM'])
+		_df['Soil_ext'] = (1 * _df['Soil'])
+		_df['SS_ext'] = (1.7 * _frhss * _df['SS'])
+		_df['EC_ext'] = (10 * _df['EC'])
+		_df['total_ext'] = sum(_df['AS_ext':'EC_ext'])
+
+		_df['ALWC_AS_ext'] = _df['AS_ext'] - _df['AS_ext_dry']
+		_df['ALWC_AN_ext'] = _df['AN_ext'] - _df['AN_ext_dry']
+		_df['ALWC_SS_ext'] = _df['SS_ext'] - _df['SS_ext_dry']
+		_df['ALWC_ext'] = _df['total_ext'] - _df['total_ext_dry']
+
+		_df['fRH_IMPR'] = _df['total_ext'] / _df['total_ext_dry']
+
+		return _df['AS_ext_dry':]
+
+	@staticmethod
+	def gas(_df):
+		_df['ScatteringByGas'] = (11.4 * 293 / (273 + _df['AT']))
+		_df['AbsorptionByGas'] = (0.33 * _df['NO2'])
+		_df['ExtinctionByGas'] = _df['ScatteringByGas'] + _df['AbsorptionByGas']
+		return _df['ScatteringByGas':]
+
+	def process_data(self, reset: bool = False, save_file: Path | str = None,
+					 version: Literal["revised", "modified"] = "revised"):
+		save_file = Path(save_file)
+		if save_file.exists() and not reset:
+			return read_csv(save_file, parse_dates=['Time'], index_col='Time')
+		else:
+			# data_files = ['EPB.csv', 'IMPACT.csv', 'chemical.csv']
+			df = concat([DataReader(file) for file in self.file_paths], axis=1)
+
+			# particle contribution '銨不足不納入計算'
+			improve_input_df = df.loc[df['NH4_status'] != 'Deficiency', ['AS', 'AN', 'OM', 'Soil', 'SS', 'EC', 'RH']]
+
+			df_improve = improve_input_df.dropna().copy().apply(self.revised if version == 'revised' else self.modified,
+																axis=1)
+
+			# gas contribution
+			df_ext_gas = df[['NO2', 'AT']].dropna().copy().apply(self.gas, axis=1)
+
+			_df = concat([df_improve, df_ext_gas], axis=1).reindex(df.index.copy())
+			_df.to_csv(save_file)
+
+			return _df

AeroViz/process/script/Others.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+
+import numpy as np
+from pandas import read_csv, concat, DataFrame
+
+from AeroViz.process.core import DataProc
+from AeroViz.tools.datareader import DataReader
+
+
+class OthersProc(DataProc):
+	"""
+	A class for process impact data.
+
+	Parameters:
+	-----------
+	reset : bool, optional
+		If True, resets the process. Default is False.
+	filename : str, optional
+		The name of the file to process. Default is None.
+
+	Methods:
+	--------
+	process_data():
+		Process data and save the result.
+
+	Attributes:
+	-----------
+	DEFAULT_PATH : Path
+		The default path for data files.
+
+	Examples:
+	---------
+	>>> df = OthersProc().process_data(reset=True, filename=None)
+
+	"""
+
+	def __init__(self, file_paths: Path | list[Path | str] = None):
+		super().__init__()
+		self.file_paths = [Path(fp) for fp in file_paths]
+
+	def process_data(self, reset: bool = False, save_file: Path | str = None) -> DataFrame:
+		save_file = Path(save_file)
+		if save_file.exists() and not reset:
+			return read_csv(save_file, parse_dates=['Time'], index_col='Time')
+		else:
+			df = concat([DataReader(file) for file in self.file_paths], axis=1)
+
+			results = DataFrame(index=df.index)
+
+			results['PG'] = df[
+				['Scattering', 'Absorption', 'ScatteringByGas', 'AbsorptionByGas']].dropna().copy().apply(np.sum,
+																										  axis=1)
+			results['MAC'] = df['Absorption'] / df['T_EC']
+			results['Ox'] = df['NO2'] + df['O3']
+			results['N2O5_tracer'] = df['NO2'] * df['O3']
+			results['Vis_cal'] = 1096 / df['Extinction']
+			# results['fRH_Mix'] = df['Bext'] / df['Extinction']
+			# results['fRH_PNSD'] = df['Bext_internal'] / df['Bext_dry']
+			results['fRH_IMPR'] = df['total_ext'] / df['total_ext_dry']
+			results['OCEC_ratio'] = df['O_OC'] / df['O_EC']
+			results['PM1/PM25'] = np.where(df['PM1'] / df['PM25'] < 1, df['PM1'] / df['PM25'], np.nan)
+			# results['MEE_PNSD'] = df['Bext_internal'] / df['PM25']
+			# results['MEE_dry_PNSD'] = df['Bext_dry'] / df['PM25']
+
+			return results

AeroViz/process/script/PSD.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+
+from pandas import concat, read_csv, DataFrame
+
+from AeroViz.process.core import DataProc
+from AeroViz.process.core.SizeDist import SizeDist
+from AeroViz.process.script.AbstractDistCalc import DistributionCalculator
+
+
+class ParticleSizeDistProc(DataProc):
+	"""
+	A class for process particle size distribution (PSD) data.
+
+	Parameters
+	----------
+	filename : str, optional
+		The name of the PSD data file.
+		Defaults to 'PNSD_dNdlogdp.csv' in the default path.
+
+	Attributes
+	----------
+	file_path : Path
+		The directory path where the PSD data file is located.
+
+	psd : SizeDist
+		The SizeDist object.
+
+	Methods
+	-------
+	process_data(filename='PSD.csv')
+		Process and save overall PSD properties.
+
+	Examples
+	--------
+	Example 1: Use default path and filename
+	>>> psd_data = ParticleSizeDistProc(filename='PNSD_dNdlogdp.csv').process_data(reset=True)
+	"""
+
+	def __init__(self, file_path: Path | str = None):
+		super().__init__()
+		self.file_path = Path(file_path)
+
+		self.psd = SizeDist(read_csv(file_path, parse_dates=['Time'], index_col='Time'))
+
+	def process_data(self, reset: bool = False, save_file: Path | str = None) -> DataFrame:
+		save_file = Path(save_file)
+		if save_file.exists() and not reset:
+			return read_csv(save_file, parse_dates=['Time'], index_col='Time')
+
+		number = DistributionCalculator('number', self.psd).useApply()
+		surface = DistributionCalculator('surface', self.psd).useApply()
+		volume = DistributionCalculator('volume', self.psd).useApply()
+
+		surface.to_csv(save_file.parent / 'PSSD_dSdlogdp.csv')
+		volume.to_csv(save_file.parent / 'PVSD_dVdlogdp.csv')
+
+		result_df = concat(
+			[DistributionCalculator('property', SizeDist(data=number, weighting='n')).useApply(),
+			 DistributionCalculator('property', SizeDist(data=surface, weighting='s')).useApply(),
+			 DistributionCalculator('property', SizeDist(data=volume, weighting='v')).useApply()
+			 ], axis=1)
+
+		result_df.to_csv(save_file)
+		return result_df
+
+
+class ExtinctionDistProc(DataProc):
+
+	def __init__(self, file_path: Path | str = 'PNSD_dNdlogdp.csv', file_path_chem: Path | str = 'chemical.csv'):
+		super().__init__()
+		self.file_path = Path(file_path)
+		self.file_path_chem = Path(file_path_chem)
+
+		self.psd = SizeDist(read_csv(file_path, parse_dates=['Time'], index_col='Time'))
+		self.RI = read_csv(file_path_chem, parse_dates=['Time'], index_col='Time')[['n_dry', 'n_amb', 'k_dry', 'k_amb',
+																					'AS_volume_ratio',
+																					'AN_volume_ratio',
+																					'OM_volume_ratio',
+																					'Soil_volume_ratio',
+																					'SS_volume_ratio',
+																					'EC_volume_ratio',
+																					'ALWC_volume_ratio']]
+
+	def process_data(self, reset: bool = False, save_file: Path | str = 'PESD.csv'):
+		save_file = Path(save_file)
+		if save_file.exists() and not reset:
+			return read_csv(save_file, parse_dates=['Time']).set_index('Time')
+
+		ext_internal = DistributionCalculator('extinction', self.psd, self.RI, method='internal',
+											  result_type='extinction').useApply()
+		ext_external = DistributionCalculator('extinction', self.psd, self.RI, method='external',
+											  result_type='extinction').useApply()
+
+		ext_internal.to_csv(save_file.parent / 'PESD_dextdlogdp_internal.csv')
+		ext_external.to_csv(save_file.parent / 'PESD_dextdlogdp_external.csv')
+
+		result_df = concat([
+			DistributionCalculator('property', SizeDist(data=ext_internal, weighting='ext_in')).useApply(),
+			DistributionCalculator('property', SizeDist(data=ext_internal, weighting='ext_ex')).useApply(),
+		], axis=1)
+
+		result_df.to_csv(save_file)
+		return result_df

AeroViz/process/script/PSD_dry.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+
+import numpy as np
+from pandas import DataFrame, read_csv, concat
+
+from AeroViz.process.core import DataProc
+from AeroViz.process.core.SizeDist import SizeDist
+from AeroViz.tools import DataReader
+
+
+class DryPSDProc(DataProc):
+	"""
+	A class for process impact data.
+
+	Parameters
+	----------
+	reset : bool, optional
+		If True, resets the process. Default is False.
+	filename : str, optional
+		The name of the file to process. Default is None.
+
+	Methods
+	-------
+	process_data():
+		Process data and save the result.
+
+	Attributes
+	----------
+	DEFAULT_PATH : Path
+		The default path for data files.
+
+
+	Examples
+	--------
+	>>> df = DryPSDProc(reset=True, filename='PNSD_dNdlogdp_dry.csv').process_data()
+	"""
+
+	def __init__(self, file_path: Path | str = 'PNSD_dNdlogdp.csv', file_path_chem: Path | str = 'chemical.csv'):
+		super().__init__()
+		self.file_path = Path(file_path)
+		self.file_path_chem = Path(file_path_chem)
+
+		self.psd = SizeDist(read_csv(file_path, parse_dates=['Time'], index_col='Time'))
+		self.RI = read_csv(file_path_chem, parse_dates=['Time'], index_col='Time')[['n_dry', 'n_amb', 'k_dry', 'k_amb',
+																					'AS_volume_ratio',
+																					'AN_volume_ratio',
+																					'OM_volume_ratio',
+																					'Soil_volume_ratio',
+																					'SS_volume_ratio',
+																					'EC_volume_ratio',
+																					'ALWC_volume_ratio']]
+
+	def process_data(self, reset: bool = False, save_filename: Path | str = None) -> DataFrame:
+		save_filename = Path(save_filename)
+		if save_filename.exists() and not reset:
+			return read_csv(save_filename, parse_dates=['Time']).set_index('Time')
+		_df = concat([self.psd, self.RI], axis=1)
+		_df.to_csv(save_filename)
+		return _df
+
+
+def dry_PNSD_process(dist, dp, **kwargs):
+	ndp = np.array(dist[:np.size(dp)])
+	gRH = resolved_gRH(dp, dist['gRH'], uniform=True)
+
+	dry_dp = dp / gRH
+	belong_which_ibin = np.digitize(dry_dp, dp) - 1
+
+	result = {}
+	for i, (ibin, dn) in enumerate(zip(belong_which_ibin, ndp)):
+		if dp[ibin] not in result:
+			result[dp[ibin]] = []
+		result[dp[ibin]].append(ndp[i])
+
+	dry_ndp = []
+	for key, val in result.items():
+		dry_ndp.append(sum(val) / len(val))
+
+	return np.array(dry_ndp)
+
+
+def resolved_gRH(dp, gRH=1.31, uniform=True):
+	if uniform:
+		return np.array([gRH] * dp.size)
+
+	else:
+		lognorm_dist = lambda x, geoMean, geoStd: (gRH / (np.log10(geoStd) * np.sqrt(2 * np.pi))) * np.exp(
+			-(x - np.log10(geoMean)) ** 2 / (2 * np.log10(geoStd) ** 2))
+		abc = lognorm_dist(np.log10(dp), 200, 2.0)
+		return np.where(abc < 1, 1, abc)
+
+
+if __name__ == '__main__':
+	pass

AeroViz/process/script/__init__.py

@@ -0,0 +1,5 @@
+from .Chemical import ChemicalProc
+from .IMPACT import ImpactProc
+from .IMPROVE import ImproveProc
+from .Others import OthersProc
+from .PSD import ParticleSizeDistProc, ExtinctionDistProc

AeroViz/process/script/retrieve_RI.py

@@ -0,0 +1,70 @@
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+
+from AeroViz.process.core.SizeDist import SizeDist
+from AeroViz.process.method import Mie_PESD
+
+
+def retrieve_RI(_df: DataFrame,
+				_PNSD: DataFrame,
+				nMin: float = 1.33,
+				nMax: float = 1.60,
+				kMin: float = 0.00,
+				kMax: float = 0.60,
+				spaceSize: int = 31,
+				dlogdp: float = 0.014
+				) -> DataFrame:
+	nRange = np.linspace(nMin, nMax, num=spaceSize)
+	kRange = np.linspace(kMin, kMax, spaceSize)
+	Delta_array = np.zeros((spaceSize, spaceSize))
+	# 同一時間除了折射率其餘數據皆相同 因此在折射率的迴圈外
+	bext_mea, bsca_mea, babs_mea = _df['Extinction'], _df['Scattering'], _df['Absorption']
+
+	dp = SizeDist(data=_PNSD).dp
+	for ki, k in enumerate(kRange):
+		for ni, n in enumerate(nRange):
+			m = n + (1j * k)
+			ndp = np.array(_df[3:])
+
+			ext_dist, sca_dist, abs_dist = Mie_PESD(m, 550, dp, ndp)
+
+			bext_cal = sum(ext_dist) * dlogdp
+			bsca_cal = sum(sca_dist) * dlogdp
+			babs_cal = sum(abs_dist) * dlogdp
+
+			Delta_array[ni][ki] = ((babs_mea - babs_cal) / 18.23) ** 2 + ((bsca_mea - bsca_cal) / 83.67) ** 2
+
+	min_delta = Delta_array.argmin()
+	next_n = nRange[(min_delta // spaceSize)]
+	next_k = kRange[(min_delta % spaceSize)]
+
+	# 將網格變小
+	nMin_small = next_n - 0.02 if next_n > 1.33 else 1.33
+	nMax_small = next_n + 0.02
+	kMin_small = next_k - 0.04 if next_k > 0.04 else 0
+	kMax_small = next_k + 0.04
+	spaceSize_small = 41
+
+	nRange_small = np.linspace(nMin_small, nMax_small, spaceSize_small)
+	kRange_small = np.linspace(kMin_small, kMax_small, spaceSize_small)
+	Delta_array_small = np.zeros((spaceSize_small, spaceSize_small))
+	# 所有數據與大網格一致所以使用上方便數即可
+	for ki, k in enumerate(kRange_small):
+		for ni, n in enumerate(nRange_small):
+			m = n + (1j * k)
+			ndp = np.array(_df[3:])
+			ext_dist, sca_dist, abs_dist = Mie_PESD(m, 550, dp, ndp)
+
+			bext_cal = sum(ext_dist) * dlogdp
+			bsca_cal = sum(sca_dist) * dlogdp
+			babs_cal = sum(abs_dist) * dlogdp
+
+			Delta_array_small[ni][ki] = ((bext_mea - bext_cal) / 18.23) ** 2 + ((bsca_mea - bsca_cal) / 83.67) ** 2
+
+	min_delta_small = Delta_array_small.argmin()
+	_df['re_real'] = (nRange_small[(min_delta_small // spaceSize_small)])
+	_df['re_imaginary'] = (kRange_small[(min_delta_small % spaceSize_small)])
+
+	print(f'\t\tReal part:{_df['re_real']}\tIm part:{_df['re_imaginary']}', end='')
+	return _df['re_real':]

AeroViz/rawDataReader/__init__.py

@@ -0,0 +1,68 @@
+from datetime import datetime
+
+from AeroViz.rawDataReader.script import *
+from AeroViz.rawDataReader.utils.config import meta
+
+__all__ = ['RawDataReader']
+
+
+def RawDataReader(instrument_name: str,
+				  _path,
+				  QC: bool = True,
+				  csv_raw: bool = True,
+				  reset: bool = False,
+				  rate: bool = False,
+				  append_data: bool = False,
+				  update_meta=None,
+				  start: datetime | None = None,
+				  end: datetime | None = None,
+				  mean_freq='1h',
+				  csv_out=True,
+				  **kwargs
+				  ):
+	# Mapping of instrument names to their respective classes
+	instrument_class_map = {
+		'NEPH': NEPH,
+		'Aurora': Aurora,
+		'Table': Table,
+		'EPA_vertical': EPA_vertical,
+		'APS_3321': APS_3321,
+		'SMPS_TH': SMPS_TH,
+		'AE33': AE33,
+		'AE43': AE43,
+		'BC1054': BC1054,
+		'MA350': MA350,
+		'TEOM': TEOM,
+		'OCEC_RES': OCEC_RES,
+		'OCEC_LCRES': OCEC_LCRES,
+		'IGAC_TH': IGAC_TH,
+		'IGAC_ZM': IGAC_ZM,
+		'VOC_TH': VOC_TH,
+		'VOC_ZM': VOC_ZM,
+		'SMPS_genr': SMPS_genr,
+		'SMPS_aim11': SMPS_aim11,
+		'GRIMM': GRIMM
+		# Add other instruments and their corresponding classes here
+	}
+
+	# Check if the instrument name is in the map
+	if instrument_name not in meta.keys():
+		raise ValueError(f"Instrument name '{instrument_name}' is not valid. \nMust be one of: {list(meta.keys())}")
+
+	# Instantiate the class and return the instance
+	reader_module = instrument_class_map[instrument_name].Reader(
+		_path=_path,
+		QC=QC,
+		csv_raw=csv_raw,
+		reset=reset,
+		rate=rate,
+		append_data=append_data,
+		update_meta=update_meta
+	)
+	return reader_module(
+		start=start,
+		end=end,
+		mean_freq=mean_freq,
+		csv_out=csv_out,
+		**kwargs
+	)