AeroViz 0.1.3__py3-none-any.whl → 0.1.3b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of AeroViz might be problematic. Click here for more details.
- AeroViz/__init__.py +4 -4
- AeroViz/dataProcess/Chemistry/__init__.py +38 -38
- AeroViz/dataProcess/Chemistry/_calculate.py +15 -15
- AeroViz/dataProcess/Chemistry/_isoropia.py +69 -68
- AeroViz/dataProcess/Chemistry/_mass_volume.py +158 -158
- AeroViz/dataProcess/Chemistry/_ocec.py +109 -109
- AeroViz/dataProcess/Chemistry/_partition.py +19 -18
- AeroViz/dataProcess/Chemistry/_teom.py +8 -11
- AeroViz/dataProcess/Optical/_IMPROVE.py +40 -39
- AeroViz/dataProcess/Optical/__init__.py +35 -35
- AeroViz/dataProcess/Optical/_absorption.py +35 -35
- AeroViz/dataProcess/Optical/_extinction.py +25 -24
- AeroViz/dataProcess/Optical/_mie.py +5 -6
- AeroViz/dataProcess/Optical/_mie_sd.py +89 -90
- AeroViz/dataProcess/Optical/_scattering.py +16 -16
- AeroViz/dataProcess/SizeDistr/__init__.py +37 -37
- AeroViz/dataProcess/SizeDistr/__merge.py +159 -158
- AeroViz/dataProcess/SizeDistr/_merge.py +155 -154
- AeroViz/dataProcess/SizeDistr/_merge_v1.py +162 -161
- AeroViz/dataProcess/SizeDistr/_merge_v2.py +153 -152
- AeroViz/dataProcess/SizeDistr/_merge_v3.py +326 -326
- AeroViz/dataProcess/SizeDistr/_merge_v4.py +272 -274
- AeroViz/dataProcess/SizeDistr/_size_distr.py +51 -51
- AeroViz/dataProcess/VOC/__init__.py +7 -7
- AeroViz/dataProcess/VOC/_potential_par.py +53 -55
- AeroViz/dataProcess/__init__.py +4 -4
- AeroViz/dataProcess/core/__init__.py +59 -58
- AeroViz/plot/__init__.py +6 -1
- AeroViz/plot/bar.py +126 -0
- AeroViz/plot/box.py +68 -0
- AeroViz/plot/distribution/distribution.py +421 -427
- AeroViz/plot/meteorology/meteorology.py +240 -292
- AeroViz/plot/optical/__init__.py +0 -1
- AeroViz/plot/optical/optical.py +230 -230
- AeroViz/plot/pie.py +198 -0
- AeroViz/plot/regression.py +210 -0
- AeroViz/plot/scatter.py +99 -0
- AeroViz/plot/templates/__init__.py +0 -3
- AeroViz/plot/templates/contour.py +25 -25
- AeroViz/plot/templates/corr_matrix.py +86 -93
- AeroViz/plot/templates/diurnal_pattern.py +24 -24
- AeroViz/plot/templates/koschmieder.py +106 -106
- AeroViz/plot/templates/metal_heatmap.py +34 -34
- AeroViz/plot/timeseries/timeseries.py +53 -60
- AeroViz/plot/utils/__init__.py +2 -1
- AeroViz/plot/utils/_color.py +57 -57
- AeroViz/plot/utils/_unit.py +48 -48
- AeroViz/plot/utils/plt_utils.py +92 -0
- AeroViz/plot/utils/sklearn_utils.py +49 -0
- AeroViz/plot/violin.py +79 -0
- AeroViz/process/__init__.py +15 -15
- AeroViz/process/core/DataProc.py +9 -9
- AeroViz/process/core/SizeDist.py +81 -81
- AeroViz/process/method/PyMieScatt_update.py +488 -488
- AeroViz/process/method/mie_theory.py +231 -229
- AeroViz/process/method/prop.py +40 -40
- AeroViz/process/script/AbstractDistCalc.py +103 -103
- AeroViz/process/script/Chemical.py +166 -166
- AeroViz/process/script/IMPACT.py +40 -40
- AeroViz/process/script/IMPROVE.py +152 -152
- AeroViz/process/script/Others.py +45 -45
- AeroViz/process/script/PSD.py +26 -26
- AeroViz/process/script/PSD_dry.py +69 -70
- AeroViz/process/script/retrieve_RI.py +50 -51
- AeroViz/rawDataReader/__init__.py +57 -57
- AeroViz/rawDataReader/core/__init__.py +328 -326
- AeroViz/rawDataReader/script/AE33.py +18 -18
- AeroViz/rawDataReader/script/AE43.py +20 -20
- AeroViz/rawDataReader/script/APS_3321.py +30 -30
- AeroViz/rawDataReader/script/Aurora.py +23 -23
- AeroViz/rawDataReader/script/BC1054.py +40 -40
- AeroViz/rawDataReader/script/EPA_vertical.py +9 -9
- AeroViz/rawDataReader/script/GRIMM.py +21 -21
- AeroViz/rawDataReader/script/IGAC_TH.py +67 -67
- AeroViz/rawDataReader/script/IGAC_ZM.py +59 -59
- AeroViz/rawDataReader/script/MA350.py +39 -39
- AeroViz/rawDataReader/script/NEPH.py +74 -74
- AeroViz/rawDataReader/script/OCEC_LCRES.py +21 -21
- AeroViz/rawDataReader/script/OCEC_RES.py +16 -16
- AeroViz/rawDataReader/script/SMPS_TH.py +25 -25
- AeroViz/rawDataReader/script/SMPS_aim11.py +32 -32
- AeroViz/rawDataReader/script/SMPS_genr.py +31 -31
- AeroViz/rawDataReader/script/TEOM.py +28 -28
- AeroViz/rawDataReader/script/Table.py +12 -12
- AeroViz/rawDataReader/script/VOC_TH.py +16 -16
- AeroViz/rawDataReader/script/VOC_ZM.py +28 -28
- AeroViz/rawDataReader/script/__init__.py +20 -20
- AeroViz/rawDataReader/utils/config.py +161 -161
- AeroViz/tools/database.py +65 -65
- AeroViz/tools/dataclassifier.py +106 -106
- AeroViz/tools/dataprinter.py +51 -51
- AeroViz/tools/datareader.py +38 -38
- {AeroViz-0.1.3.dist-info → AeroViz-0.1.3b0.dist-info}/METADATA +5 -4
- AeroViz-0.1.3b0.dist-info/RECORD +110 -0
- AeroViz/config/__init__.py +0 -0
- AeroViz/plot/improve/__init__.py +0 -1
- AeroViz/plot/improve/improve.py +0 -240
- AeroViz/plot/optical/aethalometer.py +0 -77
- AeroViz/plot/templates/event_evolution.py +0 -65
- AeroViz/plot/templates/regression.py +0 -256
- AeroViz/plot/templates/scatter.py +0 -130
- AeroViz/plot/templates/templates.py +0 -398
- AeroViz/plot/utils/_decorator.py +0 -74
- AeroViz-0.1.3.dist-info/RECORD +0 -111
- {AeroViz-0.1.3.dist-info → AeroViz-0.1.3b0.dist-info}/LICENSE +0 -0
- {AeroViz-0.1.3.dist-info → AeroViz-0.1.3b0.dist-info}/WHEEL +0 -0
- {AeroViz-0.1.3.dist-info → AeroViz-0.1.3b0.dist-info}/top_level.txt +0 -0
|
@@ -4,28 +4,28 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reader(AbstractReader):
|
|
7
|
-
|
|
7
|
+
nam = 'AE33'
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
def _raw_reader(self, _file):
|
|
10
|
+
_df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
|
|
11
|
+
delimiter=r'\s+', skiprows=5, usecols=range(67))
|
|
12
|
+
_df.columns = _df.columns.str.strip(';')
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
# remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
|
|
15
|
+
if not self._oth_set.get('ignore_err', False):
|
|
16
|
+
_df = _df.where((_df['Status'] != 0) | (_df['Status'] != 128) | (_df['Status'] != 256)).copy()
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']]
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
def _QC(self, _df):
|
|
21
|
+
# remove negative value
|
|
22
|
+
_df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
# QC data in 5 min
|
|
25
|
+
def _QC_func(df):
|
|
26
|
+
_df_ave, _df_std = df.mean(), df.std()
|
|
27
|
+
_df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
return df.mask(_df_lowb | _df_highb).copy()
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
return _df.resample('5min').apply(_QC_func).resample('1h').mean()
|
|
@@ -4,31 +4,31 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reader(AbstractReader):
|
|
7
|
-
|
|
7
|
+
nam = 'AE43'
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
def _raw_reader(self, _file):
|
|
10
|
+
_df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
|
|
11
|
+
_df_id = _df['SetupID'].iloc[-1]
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
# get last SetupID data
|
|
14
|
+
_df = _df.groupby('SetupID').get_group(_df_id)[
|
|
15
|
+
['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']].copy()
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
# remove data without Status=0
|
|
18
|
+
_df = _df.where(_df['Status'] == 0).copy()
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
# QC data
|
|
23
|
+
def _QC(self, _df):
|
|
24
|
+
# remove negative value
|
|
25
|
+
_df = _df.mask((_df < 0).copy())
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
# QC data in 5 min
|
|
28
|
+
def _QC_func(df):
|
|
29
|
+
_df_ave, _df_std = df.mean(), df.std()
|
|
30
|
+
_df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
return df.mask(_df_lowb | _df_highb).copy()
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
return _df.resample('5min').apply(_QC_func).resample('1h').mean()
|
|
@@ -5,43 +5,43 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class Reader(AbstractReader):
|
|
8
|
-
|
|
8
|
+
nam = 'APS_3321'
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
def _raw_reader(self, _file):
|
|
11
|
+
with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
|
|
12
|
+
_df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
|
|
13
|
+
_key = list(_df.keys()[3:54]) ## 542 ~ 1981
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
## create new keys
|
|
16
|
+
_newkey = {}
|
|
17
|
+
for _k in _key:
|
|
18
|
+
_newkey[_k] = float(_k).__round__(4)
|
|
19
|
+
# _newkey['Mode(m)'] = 'mode'
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
## get new dataframe
|
|
22
|
+
_df = _df[_newkey.keys()].rename(_newkey, axis=1)
|
|
23
|
+
# _df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
_df_idx = to_datetime(_df.index, errors='coerce')
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
return _df.set_index(_df_idx).loc[_df_idx.dropna()]
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
29
|
+
## QC data
|
|
30
|
+
def _QC(self, _df):
|
|
31
|
+
## mask out the data size lower than 7
|
|
32
|
+
_df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
|
|
33
|
+
_df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
|
|
34
|
+
_df = _df.mask(_df_size < 7)
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
## remove total conc. lower than 700
|
|
37
|
+
_df = _df.mask(_df['total'] > 700)
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
# not confirmed
|
|
40
|
+
"""
|
|
41
|
+
## remove the bin over 4000 nm which num. conc. larger than 1
|
|
42
|
+
# _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2]>=4.]
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
# _df_1hr[_df_remv_ky] = _df_1hr[_df_remv_ky].copy().mask(_df_1hr[_df_remv_ky]>1.)
|
|
45
|
+
# """
|
|
46
46
|
|
|
47
|
-
|
|
47
|
+
return _df[_df.keys()[:-1]]
|
|
@@ -4,35 +4,35 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reader(AbstractReader):
|
|
7
|
-
|
|
7
|
+
nam = 'Aurora'
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
def _raw_reader(self, _file):
|
|
10
|
+
with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
|
|
11
|
+
_df = read_csv(f, low_memory=False, index_col=0)
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
_df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
|
|
14
|
+
_df.index.name = 'time'
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
_df.columns = _df.keys().str.strip(' ')
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
_df = _df.loc[
|
|
19
|
+
_df.index.dropna(), ['0°σspB', '0°σspG', '0°σspR', '90°σspB', '90°σspG', '90°σspR', 'RH']].copy()
|
|
20
|
+
_df.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
return _df
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
## QC data
|
|
25
|
+
def _QC(self, _df):
|
|
26
|
+
## remove negative value
|
|
27
|
+
_df = _df.mask((_df <= 0).copy())
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
29
|
+
## call by _QC function
|
|
30
|
+
## QC data in 1 hr
|
|
31
|
+
def _QC_func(_df_1hr):
|
|
32
|
+
_df_ave = _df_1hr.mean()
|
|
33
|
+
_df_std = _df_1hr.std()
|
|
34
|
+
_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
return _df_1hr.mask(_df_lowb | _df_highb).copy()
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
return _df.resample('1h', group_keys=False).apply(_QC_func)
|
|
@@ -4,43 +4,43 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reader(AbstractReader):
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
7
|
+
nam = 'BC1054'
|
|
8
|
+
|
|
9
|
+
def _raw_reader(self, _file):
|
|
10
|
+
with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
|
|
11
|
+
_df = read_csv(f, parse_dates=['Time'], index_col='Time')
|
|
12
|
+
|
|
13
|
+
_df = _df.rename(columns={
|
|
14
|
+
'BC1(ng/m3)': 'BC1',
|
|
15
|
+
'BC2(ng/m3)': 'BC2',
|
|
16
|
+
'BC3(ng/m3)': 'BC3',
|
|
17
|
+
'BC4(ng/m3)': 'BC4',
|
|
18
|
+
'BC5(ng/m3)': 'BC5',
|
|
19
|
+
'BC6(ng/m3)': 'BC6',
|
|
20
|
+
'BC7(ng/m3)': 'BC7',
|
|
21
|
+
'BC8(ng/m3)': 'BC8',
|
|
22
|
+
'BC9(ng/m3)': 'BC9',
|
|
23
|
+
'BC10(ng/m3)': 'BC10'
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
# remove data without Status=32 (Automatic Tape Advance), 65536 (Tape Move)
|
|
27
|
+
# if not self._oth_set.get('ignore_err', False):
|
|
28
|
+
# _df = _df.where((_df['Status'] != 32) | (_df['Status'] != 65536)).copy()
|
|
29
|
+
|
|
30
|
+
return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10', 'Status']]
|
|
31
|
+
|
|
32
|
+
# QC data
|
|
33
|
+
def _QC(self, _df):
|
|
34
|
+
# remove negative value
|
|
35
|
+
_df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
|
|
36
|
+
|
|
37
|
+
# call by _QC function
|
|
38
|
+
# QC data in 1 hr
|
|
39
|
+
def _QC_func(_df_1hr):
|
|
40
|
+
_df_ave = _df_1hr.mean()
|
|
41
|
+
_df_std = _df_1hr.std()
|
|
42
|
+
_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
|
|
43
|
+
|
|
44
|
+
return _df_1hr.mask(_df_lowb | _df_highb).copy()
|
|
45
|
+
|
|
46
|
+
return _df.resample('1h', group_keys=False).apply(_QC_func).resample('5min').mean()
|
|
@@ -4,15 +4,15 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reader(AbstractReader):
|
|
7
|
-
|
|
7
|
+
nam = 'EPA_vertical'
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
def _raw_reader(self, _file):
|
|
10
|
+
with _file.open('r', encoding='big5', errors='ignore') as f:
|
|
11
|
+
_df = read_csv(f, names=['time', 'station', 'comp', 'data', None], skiprows=1, na_values=['-'],
|
|
12
|
+
parse_dates=['time'], index_col='time')
|
|
13
|
+
_df['data'] = to_numeric(_df['data'], errors='coerce')
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
_df_piv = _df.pivot_table(values='data', columns='comp', index='time')
|
|
16
|
+
_df_piv.index.name = 'time'
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
return _df_piv
|
|
@@ -4,32 +4,32 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reader(AbstractReader):
|
|
7
|
-
|
|
7
|
+
nam = 'GRIMM'
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
def _raw_reader(self, _file):
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
_df = read_csv(_file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
|
|
12
|
+
dayfirst=True).rename_axis("Time")
|
|
13
|
+
_df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
if _file.name.startswith("A407ST"):
|
|
16
|
+
_df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
|
|
17
|
+
else:
|
|
18
|
+
_df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
if _df.empty:
|
|
21
|
+
print(_file, "is empty")
|
|
22
|
+
return None
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
return _df / 0.035
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
26
|
+
def _QC(self, _df):
|
|
27
|
+
# QC data in 1 hr
|
|
28
|
+
def _QC_func(_df_1hr):
|
|
29
|
+
_df_ave = _df_1hr.mean()
|
|
30
|
+
_df_std = _df_1hr.std()
|
|
31
|
+
_df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
return _df_1hr.mask(_df_lowb | _df_highb).copy()
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
return _df.resample('5min').apply(_QC_func).resample('1h').mean()
|
|
@@ -8,97 +8,97 @@ from AeroViz.rawDataReader.core import AbstractReader
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class Reader(AbstractReader):
|
|
11
|
-
|
|
11
|
+
nam = 'IGAC_TH'
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
def _raw_reader(self, _file):
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
self.meta['freq'] = self._oth_set.get('data_freq') or self.meta['freq']
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
|
|
18
|
+
_df = read_csv(f, low_memory=False, index_col=0)
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
_df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
|
|
21
|
+
_df.index.name = 'time'
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
_df.columns = _df.keys().str.strip(' ')
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
_df = _df.loc[_df.index.dropna()].copy()
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
return _df.loc[~_df.index.duplicated()]
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
## QC data
|
|
30
|
+
def _QC(self, _df):
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
32
|
+
## QC parameter, function (MDL SE LE)
|
|
33
|
+
_mdl = {
|
|
34
|
+
'Na+': 0.05,
|
|
35
|
+
'NH4+': 0.05,
|
|
36
|
+
'K+': 0.05,
|
|
37
|
+
'Mg2+': 0.05,
|
|
38
|
+
'Ca2+': 0.05,
|
|
39
|
+
'Cl-': 0.05,
|
|
40
|
+
'NO2-': 0.05,
|
|
41
|
+
'NO3-': 0.05,
|
|
42
|
+
'SO42-': 0.05,
|
|
43
|
+
}
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
def _se_le(_df_, _log=False):
|
|
46
|
+
_df_ = np.log10(_df_) if _log else _df_
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
_df_qua = _df_.quantile([.25, .75])
|
|
49
|
+
_df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
|
|
50
|
+
_df_iqr = _df_q3 - _df_q1
|
|
51
51
|
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
_se = concat([_df_q1 - 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
|
|
53
|
+
_le = concat([_df_q3 + 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
|
|
54
54
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
if _log:
|
|
56
|
+
return 10 ** _se, 10 ** _le
|
|
57
|
+
return _se, _le
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
_cation, _anion, _main = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'], ['Cl-', 'NO2-', 'NO3-', 'SO42-', ], ['SO42-',
|
|
60
|
+
'NO3-',
|
|
61
|
+
'NH4+']
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
_df_salt = _df[_mdl.keys()].copy()
|
|
64
|
+
_df_pm = _df['PM2.5'].copy()
|
|
65
65
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
## lower than PM2.5
|
|
67
|
+
## conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
|
|
68
|
+
_df_salt = _df_salt.mask(_df_salt.sum(axis=1, min_count=1) > _df_pm).dropna(subset=_main).copy()
|
|
69
69
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
## mdl
|
|
71
|
+
for (_key, _df_col), _mdl_val in zip(_df_salt.items(), _mdl.values()):
|
|
72
|
+
_df_salt[_key] = _df_col.mask(_df_col < _mdl_val, _mdl_val / 2)
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
## group by time (per month)
|
|
75
|
+
_df_salt['tm'] = _df_salt.index.strftime('%Y-%m')
|
|
76
76
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
_df_lst = []
|
|
78
|
+
for _ky, _df_grp in _df_salt.groupby('tm'):
|
|
79
|
+
_df_grp = _df_grp[_mdl.keys()].copy()
|
|
80
80
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
## calculate SE LE
|
|
82
|
+
## salt < LE
|
|
83
|
+
_se, _le = _se_le(_df_grp, _log=True)
|
|
84
|
+
_df_grp = _df_grp.mask(_df_grp > _le).copy()
|
|
85
85
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
## C/A, A/C
|
|
87
|
+
_rat_CA = (_df_grp[_cation].sum(axis=1) / _df_grp[_anion].sum(axis=1)).to_frame()
|
|
88
|
+
_rat_AC = (1 / _rat_CA).copy()
|
|
89
89
|
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
_se, _le = _se_le(_rat_CA, )
|
|
91
|
+
_cond_CA = (_rat_CA < _le) & (_rat_CA > 0)
|
|
92
92
|
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
_se, _le = _se_le(_rat_AC, )
|
|
94
|
+
_cond_AC = (_rat_AC < _le) & (_rat_AC > 0)
|
|
95
95
|
|
|
96
|
-
|
|
96
|
+
_df_grp = _df_grp.where((_cond_CA * _cond_AC)[0]).copy()
|
|
97
97
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
## conc. of main salt > SE
|
|
99
|
+
_se, _le = _se_le(_df_grp[_main], _log=True)
|
|
100
|
+
_df_grp[_main] = _df_grp[_main].mask(_df_grp[_main] < _se).copy()
|
|
101
101
|
|
|
102
|
-
|
|
102
|
+
_df_lst.append(_df_grp)
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
return concat(_df_lst).reindex(_df.index)
|