AeroViz 0.1.2__py3-none-any.whl → 0.1.3b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of AeroViz might be problematic. Click here for more details.
- AeroViz/__init__.py +4 -4
- AeroViz/config/DEFAULT_DATA.csv +1417 -0
- AeroViz/config/DEFAULT_PNSD_DATA.csv +1417 -0
- AeroViz/dataProcess/Chemistry/__init__.py +38 -38
- AeroViz/dataProcess/Chemistry/_calculate.py +15 -15
- AeroViz/dataProcess/Chemistry/_isoropia.py +69 -68
- AeroViz/dataProcess/Chemistry/_mass_volume.py +158 -158
- AeroViz/dataProcess/Chemistry/_ocec.py +109 -109
- AeroViz/dataProcess/Chemistry/_partition.py +19 -18
- AeroViz/dataProcess/Chemistry/_teom.py +8 -11
- AeroViz/dataProcess/Optical/_IMPROVE.py +40 -39
- AeroViz/dataProcess/Optical/__init__.py +35 -35
- AeroViz/dataProcess/Optical/_absorption.py +35 -35
- AeroViz/dataProcess/Optical/_extinction.py +25 -24
- AeroViz/dataProcess/Optical/_mie.py +5 -6
- AeroViz/dataProcess/Optical/_mie_sd.py +89 -90
- AeroViz/dataProcess/Optical/_scattering.py +16 -16
- AeroViz/dataProcess/SizeDistr/__init__.py +37 -37
- AeroViz/dataProcess/SizeDistr/__merge.py +159 -158
- AeroViz/dataProcess/SizeDistr/_merge.py +155 -154
- AeroViz/dataProcess/SizeDistr/_merge_v1.py +162 -161
- AeroViz/dataProcess/SizeDistr/_merge_v2.py +153 -152
- AeroViz/dataProcess/SizeDistr/_merge_v3.py +326 -326
- AeroViz/dataProcess/SizeDistr/_merge_v4.py +272 -274
- AeroViz/dataProcess/SizeDistr/_size_distr.py +51 -51
- AeroViz/dataProcess/VOC/__init__.py +7 -7
- AeroViz/dataProcess/VOC/_potential_par.py +53 -55
- AeroViz/dataProcess/VOC/voc_par.json +464 -0
- AeroViz/dataProcess/__init__.py +4 -4
- AeroViz/dataProcess/core/__init__.py +59 -58
- AeroViz/plot/__init__.py +6 -1
- AeroViz/plot/bar.py +126 -0
- AeroViz/plot/box.py +68 -0
- AeroViz/plot/distribution/distribution.py +421 -427
- AeroViz/plot/meteorology/meteorology.py +240 -292
- AeroViz/plot/optical/__init__.py +0 -1
- AeroViz/plot/optical/optical.py +230 -230
- AeroViz/plot/pie.py +198 -0
- AeroViz/plot/regression.py +210 -0
- AeroViz/plot/scatter.py +99 -0
- AeroViz/plot/templates/__init__.py +0 -3
- AeroViz/plot/templates/contour.py +25 -25
- AeroViz/plot/templates/corr_matrix.py +86 -93
- AeroViz/plot/templates/diurnal_pattern.py +24 -24
- AeroViz/plot/templates/koschmieder.py +106 -106
- AeroViz/plot/templates/metal_heatmap.py +34 -34
- AeroViz/plot/timeseries/timeseries.py +53 -60
- AeroViz/plot/utils/__init__.py +2 -1
- AeroViz/plot/utils/_color.py +57 -57
- AeroViz/plot/utils/_unit.py +48 -48
- AeroViz/plot/utils/fRH.json +390 -0
- AeroViz/plot/utils/plt_utils.py +92 -0
- AeroViz/plot/utils/sklearn_utils.py +49 -0
- AeroViz/plot/utils/units.json +84 -0
- AeroViz/plot/violin.py +79 -0
- AeroViz/process/__init__.py +15 -15
- AeroViz/process/core/DataProc.py +9 -9
- AeroViz/process/core/SizeDist.py +81 -81
- AeroViz/process/method/PyMieScatt_update.py +488 -488
- AeroViz/process/method/mie_theory.py +231 -229
- AeroViz/process/method/prop.py +40 -40
- AeroViz/process/script/AbstractDistCalc.py +103 -103
- AeroViz/process/script/Chemical.py +166 -166
- AeroViz/process/script/IMPACT.py +40 -40
- AeroViz/process/script/IMPROVE.py +152 -152
- AeroViz/process/script/Others.py +45 -45
- AeroViz/process/script/PSD.py +26 -26
- AeroViz/process/script/PSD_dry.py +69 -70
- AeroViz/process/script/retrieve_RI.py +50 -51
- AeroViz/rawDataReader/__init__.py +57 -57
- AeroViz/rawDataReader/core/__init__.py +328 -326
- AeroViz/rawDataReader/script/AE33.py +18 -18
- AeroViz/rawDataReader/script/AE43.py +20 -20
- AeroViz/rawDataReader/script/APS_3321.py +30 -30
- AeroViz/rawDataReader/script/Aurora.py +23 -23
- AeroViz/rawDataReader/script/BC1054.py +40 -40
- AeroViz/rawDataReader/script/EPA_vertical.py +9 -9
- AeroViz/rawDataReader/script/GRIMM.py +21 -21
- AeroViz/rawDataReader/script/IGAC_TH.py +67 -67
- AeroViz/rawDataReader/script/IGAC_ZM.py +59 -59
- AeroViz/rawDataReader/script/MA350.py +39 -39
- AeroViz/rawDataReader/script/NEPH.py +74 -74
- AeroViz/rawDataReader/script/OCEC_LCRES.py +21 -21
- AeroViz/rawDataReader/script/OCEC_RES.py +16 -16
- AeroViz/rawDataReader/script/SMPS_TH.py +25 -25
- AeroViz/rawDataReader/script/SMPS_aim11.py +32 -32
- AeroViz/rawDataReader/script/SMPS_genr.py +31 -31
- AeroViz/rawDataReader/script/TEOM.py +28 -28
- AeroViz/rawDataReader/script/Table.py +12 -12
- AeroViz/rawDataReader/script/VOC_TH.py +16 -16
- AeroViz/rawDataReader/script/VOC_ZM.py +28 -28
- AeroViz/rawDataReader/script/__init__.py +20 -20
- AeroViz/rawDataReader/utils/config.py +161 -161
- AeroViz/tools/database.py +65 -65
- AeroViz/tools/dataclassifier.py +106 -106
- AeroViz/tools/dataprinter.py +51 -51
- AeroViz/tools/datareader.py +38 -38
- {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/METADATA +5 -4
- AeroViz-0.1.3b0.dist-info/RECORD +110 -0
- AeroViz/config/__init__.py +0 -0
- AeroViz/plot/improve/__init__.py +0 -1
- AeroViz/plot/improve/improve.py +0 -240
- AeroViz/plot/optical/aethalometer.py +0 -77
- AeroViz/plot/templates/event_evolution.py +0 -65
- AeroViz/plot/templates/regression.py +0 -256
- AeroViz/plot/templates/scatter.py +0 -130
- AeroViz/plot/templates/templates.py +0 -398
- AeroViz/plot/utils/_decorator.py +0 -74
- AeroViz-0.1.2.dist-info/RECORD +0 -106
- {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/LICENSE +0 -0
- {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/WHEEL +0 -0
- {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/top_level.txt +0 -0
|
@@ -16,24 +16,24 @@ import warnings
|
|
|
16
16
|
warnings.filterwarnings("ignore")
|
|
17
17
|
|
|
18
18
|
__all__ = [
|
|
19
|
-
|
|
19
|
+
'_merge_SMPS_APS',
|
|
20
20
|
|
|
21
21
|
]
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def _test_plot(ax, smps, aps, unp, shft):
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
fs = 22.
|
|
26
|
+
font_dic = dict(fontsize=fs, math_fontfamily='custom')
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
ax.plot(smps, c='#2693ff', label='smps')
|
|
29
|
+
ax.plot(aps, c='#ff4c4d', label='aps_ori')
|
|
30
|
+
ax.plot(aps.index / shft, aps.values, c='#ff181b', label='aps_shft', ls='--')
|
|
31
|
+
ax.plot(unp, c='#333333', label='unp')
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
# ax.tick_params(which='major', length=7, labelsize=fs-2.5)
|
|
34
|
+
# ax.tick_params(which='minor', length=4.5)
|
|
35
|
+
# ax.spines[['right', 'top']].set_visible(False)
|
|
36
|
+
ax.set(xlim=(11.8, 2500), xscale='log')
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
# ax.set_xlabel('', **font_dic)
|
|
@@ -52,37 +52,37 @@ def _test_plot(ax, smps, aps, unp, shft):
|
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
def test_plot(smps, aps, unp, shft):
|
|
55
|
-
|
|
55
|
+
from matplotlib.pyplot import subplots, show, rcParams
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
57
|
+
## parameter
|
|
58
|
+
fs = 22.
|
|
59
|
+
# font_fam = 'DejaVu Sans'
|
|
60
|
+
font_fam = 'Times New Roman'
|
|
61
|
+
rcParams['font.sans-serif'] = font_fam
|
|
62
|
+
rcParams['mathtext.fontset'] = 'custom'
|
|
63
|
+
font_dic = dict(fontsize=fs, math_fontfamily='custom')
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
## plot
|
|
66
|
+
fig, axes = subplots(3, 1)
|
|
67
67
|
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
ds_fc = lambda _dt: _dt * _dt.index ** 2 * np.pi
|
|
69
|
+
dv_fc = lambda _dt: _dt * _dt.index ** 3 * np.pi / 6
|
|
70
70
|
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
axes[0].set_title(shft, **font_dic)
|
|
72
|
+
# axes[0].legend(framealpha=0, fontsize=fs * .6)
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
_test_plot(axes[0], smps, aps, unp, shft)
|
|
75
|
+
_test_plot(axes[1], ds_fc(smps), ds_fc(aps), ds_fc(unp), shft)
|
|
76
|
+
_test_plot(axes[2], dv_fc(smps), dv_fc(aps), dv_fc(unp), shft)
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
show()
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
def _powerlaw_fit(_coeA, _coeB, _aps, _idx, _factor):
|
|
82
|
-
|
|
82
|
+
# breakpoint()
|
|
83
83
|
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
_smps_fit_df = _coeA * (_aps.keys().values / _factor) ** _coeB
|
|
85
|
+
return DataFrame(((_smps_fit_df.copy() - _aps.copy()) ** 2).sum(axis=1), columns=[_idx])
|
|
86
86
|
|
|
87
87
|
|
|
88
88
|
## Calculate S2
|
|
@@ -92,427 +92,427 @@ def _powerlaw_fit(_coeA, _coeB, _aps, _idx, _factor):
|
|
|
92
92
|
## return : S2
|
|
93
93
|
# def _S2_calculate_dN(_smps, _aps):
|
|
94
94
|
def _powerlaw_fit_dN(_smps, _aps, _alg_type):
|
|
95
|
-
|
|
95
|
+
print(f"\t\t{dtm.now().strftime('%m/%d %X')} : \033[92moverlap range fitting : {_alg_type}\033[0m")
|
|
96
96
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
## overlap fitting
|
|
98
|
+
## parmeter
|
|
99
|
+
_dt_indx = _smps.index
|
|
100
100
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
## use SMPS data apply power law fitting
|
|
102
|
+
## y = Ax^B, A = e**coefa, B = coefb, x = logx, y = logy
|
|
103
|
+
## ref : http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html
|
|
104
|
+
## power law fit to SMPS num conc at upper bins to log curve
|
|
105
105
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
106
|
+
## coefficient A, B
|
|
107
|
+
_smps_qc_cond = ((_smps != 0) & np.isfinite(_smps))
|
|
108
|
+
_smps_qc = _smps.where(_smps_qc_cond)
|
|
109
109
|
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
_size = _smps_qc_cond.sum(axis=1)
|
|
111
|
+
_size = _size.where(_size != 0.).copy()
|
|
112
112
|
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
_logx, _logy = n.log(_smps_qc.keys()._data.astype(float)), n.log(_smps_qc)
|
|
114
|
+
_x, _y, _xy, _xx = _logx.sum(), _logy.sum(axis=1), (_logx * _logy).sum(axis=1), (_logx ** 2).sum()
|
|
115
115
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
116
|
+
_coeB = ((_size * _xy - _x * _y) / (_size * _xx - _x ** 2.))
|
|
117
|
+
_coeA = n.exp((_y - _coeB * _x) / _size).values.reshape(-1, 1)
|
|
118
|
+
_coeB = _coeB.values.reshape(-1, 1)
|
|
119
119
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
## rebuild shift smps data by coe. A, B
|
|
121
|
+
## x_shift = (y_ori/A)**(1/B)
|
|
122
|
+
_aps_shift_x = (_aps / _coeA) ** (1 / _coeB)
|
|
123
|
+
_aps_shift_x = _aps_shift_x.where(np.isfinite(_aps_shift_x))
|
|
124
124
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
125
|
+
## the least squares of diameter
|
|
126
|
+
## the shift factor which the closest to 1
|
|
127
|
+
_shift_val = np.arange(0.3, 3.05, .05) ** .5
|
|
128
|
+
# _shift_val = np.arange(0.9, 1.805, .005)**.5
|
|
129
129
|
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
_shift_factor = DataFrame(columns=range(_shift_val.size), index=_aps_shift_x.index)
|
|
131
|
+
_shift_factor.loc[:, :] = _shift_val
|
|
132
132
|
|
|
133
|
-
|
|
134
|
-
|
|
133
|
+
# _dropna_idx = _shift_factor.dropna(how='all').index.copy()
|
|
134
|
+
_dropna_idx = _aps_shift_x.dropna(how='all').index.copy()
|
|
135
135
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
## use the target function to get the similar aps and smps bin
|
|
137
|
+
## S2 = sum( (smps_fit_line(dia) - aps(dia*shift_factor) )**2 )
|
|
138
|
+
## assumption : the same diameter between smps and aps should get the same conc.
|
|
139
139
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
140
|
+
## be sure they art in log value
|
|
141
|
+
_S2 = DataFrame(index=_aps_shift_x.index)
|
|
142
|
+
_dia_table = DataFrame(n.full(_aps_shift_x.shape, _aps_shift_x.keys()),
|
|
143
|
+
columns=_aps_shift_x.keys(), index=_aps_shift_x.index)
|
|
144
144
|
|
|
145
|
-
|
|
145
|
+
pool = Pool(cpu_count())
|
|
146
146
|
|
|
147
|
-
|
|
147
|
+
_S2 = pool.starmap(partial(_powerlaw_fit, _coeA, _coeB, _aps), list(enumerate(_shift_val)))
|
|
148
148
|
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
pool.close()
|
|
150
|
+
pool.join()
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
152
|
+
S2 = concat(_S2, axis=1)[np.arange(_shift_val.size)]
|
|
153
|
+
# S2 /= S2.max(axis=1).to_frame().values
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
155
|
+
shift_factor_dN = DataFrame(
|
|
156
|
+
_shift_factor.loc[_dropna_idx].values[range(len(_dropna_idx)), S2.loc[_dropna_idx].idxmin(axis=1).values],
|
|
157
|
+
index=_dropna_idx).reindex(_dt_indx).astype(float)
|
|
158
158
|
|
|
159
|
-
|
|
159
|
+
shift_factor_dN = shift_factor_dN.mask((shift_factor_dN ** 2 < 0.6) | (shift_factor_dN ** 2 > 2.6))
|
|
160
160
|
|
|
161
|
-
|
|
161
|
+
return shift_factor_dN
|
|
162
162
|
|
|
163
163
|
|
|
164
164
|
def _corr_fc(_aps_dia, _smps_dia, _smps_dn, _aps_dn, _smooth, _idx, _sh):
|
|
165
|
-
|
|
166
|
-
|
|
165
|
+
ds_fc = lambda _dt: _dt * _dt.index ** 2 * np.pi
|
|
166
|
+
dv_fc = lambda _dt: _dt * _dt.index ** 3 * np.pi / 6
|
|
167
167
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
168
|
+
_aps_sh = _aps_dia / _sh
|
|
169
|
+
_aps_sh_inp = _aps_sh.where((_aps_sh >= 500) & (_aps_sh <= 1500.)).copy()
|
|
170
|
+
_aps_sh_corr = _aps_sh.where((_aps_sh >= _smps_dia[-1]) & (_aps_sh <= 1500.)).copy()
|
|
171
171
|
|
|
172
|
-
|
|
172
|
+
corr_x = np.append(_smps_dia, _aps_sh_corr.dropna())
|
|
173
173
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
174
|
+
input_x = np.append(_smps_dia, _aps_sh_inp.dropna())
|
|
175
|
+
input_y = concat([_smps_dn, _aps_dn.iloc[:, ~np.isnan(_aps_sh_inp)]], axis=1)
|
|
176
|
+
input_y.columns = input_x
|
|
177
177
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
178
|
+
input_x.sort()
|
|
179
|
+
input_y = input_y[input_x]
|
|
180
|
+
corr_y = input_y[corr_x]
|
|
181
181
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
182
|
+
S2_lst = []
|
|
183
|
+
for (_tm, _inp_y_dn), (_tm, _cor_y_dn) in zip(input_y.dropna(how='all').iterrows(),
|
|
184
|
+
corr_y.dropna(how='all').iterrows()):
|
|
185
|
+
## corr(spec_data, spec_spline)
|
|
186
|
+
_spl_dt = [unvpline(input_x, _inp_y, s=_smooth)(corr_x) for _inp_y in
|
|
187
|
+
[_inp_y_dn, ds_fc(_inp_y_dn), dv_fc(_inp_y_dn)]]
|
|
188
|
+
_cor_dt = [_cor_y_dn, ds_fc(_cor_y_dn), dv_fc(_cor_y_dn)]
|
|
189
189
|
|
|
190
|
-
|
|
190
|
+
_cor_all = sum([np.corrcoef(_cor, _spl)[0, 1] for _cor, _spl in zip(_cor_dt, _spl_dt)])
|
|
191
191
|
|
|
192
|
-
|
|
192
|
+
S2_lst.append((3 - _cor_all) / 3)
|
|
193
193
|
|
|
194
|
-
|
|
194
|
+
return DataFrame(S2_lst, columns=[_idx])
|
|
195
195
|
|
|
196
196
|
|
|
197
197
|
# def _S2_calculate_dSdV(_smps, _aps, _shft_dn, _S2, smps_ori, aps_ori):
|
|
198
198
|
# def _S2_calculate_dSdV(_smps, _aps, smps_ori=None):
|
|
199
199
|
def _corr_with_dNdSdV(_smps, _aps, _alg_type):
|
|
200
|
-
|
|
200
|
+
print(f"\t\t{dtm.now().strftime('%m/%d %X')} : \033[92moverlap range correlation : {_alg_type}\033[0m")
|
|
201
201
|
|
|
202
|
-
|
|
203
|
-
|
|
202
|
+
_smps_dia = _smps.keys().astype(float)
|
|
203
|
+
_aps_dia = _aps.keys().astype(float)
|
|
204
204
|
|
|
205
|
-
|
|
206
|
-
|
|
205
|
+
all_index = _smps.index.copy()
|
|
206
|
+
qc_index = DatetimeIndex(set(_smps.dropna(how='all').index) & set(_aps.dropna(how='all').index)).sort_values()
|
|
207
207
|
|
|
208
|
-
|
|
209
|
-
|
|
208
|
+
_smps_dn = _smps.loc[qc_index].copy()
|
|
209
|
+
_aps_dn = _aps.loc[qc_index].copy()
|
|
210
210
|
|
|
211
|
-
|
|
212
|
-
|
|
211
|
+
ds_fc = lambda _dt: _dt * _dt.index ** 2 * np.pi
|
|
212
|
+
dv_fc = lambda _dt: _dt * _dt.index ** 3 * np.pi / 6
|
|
213
213
|
|
|
214
|
-
|
|
215
|
-
|
|
214
|
+
_std_bin = np.geomspace(11.8, 19810, 230)
|
|
215
|
+
_merge_bin = _std_bin[(_std_bin >= _smps_dia[-1]) & (_std_bin < 1500)].copy()
|
|
216
216
|
|
|
217
|
-
|
|
217
|
+
_smooth = 50
|
|
218
218
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
219
|
+
_shift_val = np.arange(0.5, 2.605, .005) ** .5
|
|
220
|
+
_shift_val = np.arange(0.9, 2.01, .01) ** .5
|
|
221
|
+
_shift_val = np.arange(0.9, 2.65, .05) ** .5
|
|
222
222
|
|
|
223
|
-
|
|
224
|
-
|
|
223
|
+
## spline fitting with shift aps and smps
|
|
224
|
+
pool = Pool(cpu_count())
|
|
225
225
|
|
|
226
|
-
|
|
227
|
-
|
|
226
|
+
S2_lst = pool.starmap(partial(_corr_fc, _aps_dia, _smps_dia, _smps_dn, _aps_dn, _smooth),
|
|
227
|
+
list(enumerate(_shift_val)))
|
|
228
228
|
|
|
229
|
-
|
|
230
|
-
|
|
229
|
+
pool.close()
|
|
230
|
+
pool.join()
|
|
231
231
|
|
|
232
|
-
|
|
233
|
-
|
|
232
|
+
S2_table = concat(S2_lst, axis=1).set_index(qc_index)[np.arange(_shift_val.size)].astype(float).dropna()
|
|
233
|
+
min_shft = S2_table.idxmin(axis=1).values
|
|
234
234
|
|
|
235
|
-
|
|
235
|
+
return DataFrame(_shift_val[min_shft.astype(int)], index=S2_table.index).astype(float).reindex(_smps.index)
|
|
236
236
|
|
|
237
237
|
|
|
238
238
|
## Create merge data
|
|
239
239
|
## shift all smps bin and remove the aps bin which smaller than the latest old smps bin
|
|
240
240
|
## Return : merge bins, merge data, density
|
|
241
241
|
def _merge_data(_smps_ori, _aps_ori, _shift_ori, _smps_lb, _aps_hb, _shift_mode, _alg_type):
|
|
242
|
-
|
|
242
|
+
print(f"\t\t{dtm.now().strftime('%m/%d %X')} : \033[92mcreate merge data : {_shift_mode} and {_alg_type}\033[0m")
|
|
243
243
|
|
|
244
|
-
|
|
245
|
-
|
|
244
|
+
_ori_idx = _smps_ori.index
|
|
245
|
+
_merge_idx = _smps_ori.loc[_aps_ori.dropna(how='all').index].dropna(how='all').index
|
|
246
246
|
|
|
247
|
-
|
|
248
|
-
|
|
247
|
+
_corr_aps_cond = _aps_ori.keys() < 700
|
|
248
|
+
_corr_aps_ky = _aps_ori.keys()[_corr_aps_cond]
|
|
249
249
|
|
|
250
|
-
|
|
251
|
-
|
|
250
|
+
_uni_idx = DatetimeIndex(set(_smps_ori.dropna(how='all').index) & set(_aps_ori.dropna(how='all').index) &
|
|
251
|
+
set(_shift_ori.dropna(how='all').index)).sort_values()
|
|
252
252
|
|
|
253
|
-
|
|
253
|
+
_smps, _aps, _shift = _smps_ori.loc[_merge_idx], _aps_ori.loc[_merge_idx], _shift_ori.loc[_merge_idx].values
|
|
254
254
|
|
|
255
|
-
|
|
256
|
-
|
|
255
|
+
## parameter
|
|
256
|
+
_smps_key, _aps_key = _smps.keys()._data.astype(float), _aps.keys()._data.astype(float)
|
|
257
257
|
|
|
258
|
-
|
|
259
|
-
|
|
258
|
+
_cntr = 1000
|
|
259
|
+
_bin_lb = _smps_key[-1]
|
|
260
260
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
261
|
+
## make shift bins
|
|
262
|
+
_smps_bin = n.full(_smps.shape, _smps_key)
|
|
263
|
+
_aps_bin = n.full(_aps.shape, _aps_key)
|
|
264
264
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
265
|
+
_std_bin = n.geomspace(_smps_key[0], _aps_key[-1], 230)
|
|
266
|
+
_std_bin_merge = _std_bin[(_std_bin < _cntr) & (_std_bin > _bin_lb)]
|
|
267
|
+
_std_bin_inte1 = _std_bin[_std_bin <= _bin_lb]
|
|
268
|
+
_std_bin_inte2 = _std_bin[_std_bin >= _cntr]
|
|
269
|
+
# breakpoint()
|
|
270
|
+
if _shift_mode == 'mobility':
|
|
271
|
+
_aps_bin /= _shift
|
|
272
272
|
|
|
273
|
-
|
|
274
|
-
|
|
273
|
+
elif _shift_mode == 'aerodynamic':
|
|
274
|
+
_smps_bin *= _shift
|
|
275
275
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
276
|
+
## merge
|
|
277
|
+
_merge_lst, _corr_lst = [], []
|
|
278
|
+
for _bin_smps, _bin_aps, _dt_smps, _dt_aps, _sh in zip(_smps_bin, _aps_bin, _smps.values, _aps.values, _shift):
|
|
279
|
+
## keep complete smps bins and data
|
|
280
|
+
## remove the aps bin data lower than smps bin
|
|
281
|
+
_condi = _bin_aps >= _bin_smps[-1]
|
|
282
282
|
|
|
283
|
-
|
|
284
|
-
|
|
283
|
+
_merge_bin = n.hstack((_bin_smps, _bin_aps[_condi]))
|
|
284
|
+
_merge_dt = n.hstack((_dt_smps, _dt_aps[_condi]))
|
|
285
285
|
|
|
286
|
-
|
|
286
|
+
_merge_fit_loc = (_merge_bin < 1500) & (_merge_bin > _smps_lb)
|
|
287
287
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
288
|
+
## coeA and coeB
|
|
289
|
+
_unvpl_fc = unvpline(n.log(_merge_bin[_merge_fit_loc]), n.log(_merge_dt[_merge_fit_loc]), s=50)
|
|
290
|
+
_inte_fc = interp1d(_merge_bin, _merge_dt, kind='linear', fill_value='extrapolate')
|
|
291
291
|
|
|
292
|
-
|
|
293
|
-
|
|
292
|
+
_merge_dt_fit = n.hstack((_inte_fc(_std_bin_inte1), n.exp(_unvpl_fc(n.log(_std_bin_merge))),
|
|
293
|
+
_inte_fc(_std_bin_inte2)))
|
|
294
294
|
|
|
295
|
-
|
|
296
|
-
|
|
295
|
+
_merge_lst.append(_merge_dt_fit)
|
|
296
|
+
_corr_lst.append(interp1d(_std_bin, _merge_dt_fit)(_bin_aps[_corr_aps_cond]))
|
|
297
297
|
|
|
298
|
-
|
|
299
|
-
|
|
298
|
+
_df_merge = DataFrame(_merge_lst, columns=_std_bin, index=_merge_idx)
|
|
299
|
+
_df_merge = _df_merge.mask(_df_merge < 0)
|
|
300
300
|
|
|
301
|
-
|
|
301
|
+
_df_corr = DataFrame(_corr_lst, columns=_corr_aps_ky, index=_merge_idx) / _aps_ori.loc[_merge_idx, _corr_aps_ky]
|
|
302
302
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
303
|
+
## process output df
|
|
304
|
+
## average, align with index
|
|
305
|
+
def _out_df(*_df_arg, **_df_kwarg):
|
|
306
|
+
_df = DataFrame(*_df_arg, **_df_kwarg).reindex(_ori_idx)
|
|
307
|
+
_df.index.name = 'time'
|
|
308
|
+
return _df
|
|
309
309
|
|
|
310
|
-
|
|
310
|
+
return _out_df(_df_merge), _out_df(_shift_ori ** 2), _out_df(_df_corr)
|
|
311
311
|
|
|
312
312
|
|
|
313
313
|
def merge_SMPS_APS(df_smps, df_aps, aps_unit='um', smps_overlap_lowbound=500, aps_fit_highbound=1000, dndsdv_alg=True):
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
314
|
+
merge_data, merge_data_dn, merge_data_dsdv, merge_data_cor_dn, density, density_dn, density_dsdv, density_cor_dn = [
|
|
315
|
+
DataFrame(
|
|
316
|
+
[
|
|
317
|
+
np.nan])] * 8
|
|
318
318
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
319
|
+
## set to the same units
|
|
320
|
+
smps, aps_ori = df_smps.copy(), df_aps.copy()
|
|
321
|
+
smps.columns = smps.keys().to_numpy(float)
|
|
322
|
+
aps_ori.columns = aps_ori.keys().to_numpy(float)
|
|
323
323
|
|
|
324
|
-
|
|
325
|
-
|
|
324
|
+
if aps_unit == 'um':
|
|
325
|
+
aps_ori.columns = aps_ori.keys() * 1e3
|
|
326
326
|
|
|
327
|
-
|
|
328
|
-
|
|
327
|
+
den_lst, mer_lst = [], []
|
|
328
|
+
aps_input = aps_ori.loc[:, (aps_ori.keys() > 700) & (aps_ori.keys() < 1000)].copy()
|
|
329
329
|
|
|
330
|
-
|
|
331
|
-
|
|
330
|
+
# aps_over = aps[ aps.keys()[aps.keys() < 1000] ].copy()
|
|
331
|
+
smps_over = smps[smps.keys()[smps.keys() > 500]].copy()
|
|
332
332
|
|
|
333
|
-
|
|
333
|
+
for _count in range(2):
|
|
334
334
|
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
335
|
+
## shift data calculate
|
|
336
|
+
if _count == 0:
|
|
337
|
+
alg_type = 'dn'
|
|
338
|
+
shift = _powerlaw_fit_dN(smps_over, aps_input)
|
|
339
339
|
|
|
340
|
-
|
|
341
|
-
|
|
340
|
+
if dndsdv_alg:
|
|
341
|
+
shift_dsdv = _corr_with_dNdSdV(smps_over, aps_input).mask(shift.isna())
|
|
342
342
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
343
|
+
else:
|
|
344
|
+
alg_type = 'cor_dndsdv'
|
|
345
|
+
shift_cor = _powerlaw_fit_dN(smps_over, aps_input)
|
|
346
346
|
|
|
347
|
-
|
|
348
|
-
|
|
347
|
+
if dndsdv_alg:
|
|
348
|
+
shift = _corr_with_dNdSdV(smps_over, aps_input).mask(shift_cor.isna())
|
|
349
349
|
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
350
|
+
## merge aps and smps
|
|
351
|
+
## 1. power law fit (dn) -> return dn data and aps correct factor
|
|
352
|
+
## 2. correaltion with dn, ds, dv -> return corrected dn_ds_dv data
|
|
353
|
+
if (alg_type == 'dn') | dndsdv_alg:
|
|
354
|
+
merge_arg = (smps, aps_ori, shift, smps_overlap_lowbound, aps_fit_highbound)
|
|
355
355
|
|
|
356
|
-
|
|
357
|
-
|
|
356
|
+
merge_data, density, _corr = _merge_data(*merge_arg, 'mobility', _alg_type=alg_type)
|
|
357
|
+
density.columns = ['density']
|
|
358
358
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
359
|
+
## without aps correct
|
|
360
|
+
if _count == 0:
|
|
361
|
+
## merge aps and smps
|
|
362
|
+
## dn_ds_dv data
|
|
363
|
+
if dndsdv_alg:
|
|
364
|
+
merge_arg = (smps, aps_ori, shift_dsdv, smps_overlap_lowbound, aps_fit_highbound)
|
|
365
365
|
|
|
366
|
-
|
|
367
|
-
|
|
366
|
+
merge_data_dsdv, density_dsdv, _ = _merge_data(*merge_arg, 'mobility', _alg_type='dndsdv')
|
|
367
|
+
density_dsdv.columns = ['density']
|
|
368
368
|
|
|
369
|
-
|
|
370
|
-
|
|
369
|
+
## dn data
|
|
370
|
+
merge_data_dn, density_dn = merge_data.copy(), density.copy()
|
|
371
371
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
372
|
+
## correct aps data
|
|
373
|
+
corr = _corr.resample('1d').mean().reindex(smps.index).ffill()
|
|
374
|
+
corr = corr.mask(corr < 1, 1)
|
|
375
|
+
aps_ori.loc[:, corr.keys()] *= corr
|
|
376
376
|
|
|
377
|
-
|
|
377
|
+
aps_input = aps_ori.copy()
|
|
378
378
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
379
|
+
## with aps correct
|
|
380
|
+
else:
|
|
381
|
+
## merge aps and smps
|
|
382
|
+
## dn data
|
|
383
|
+
merge_arg = (smps, aps_ori, shift_cor, smps_overlap_lowbound, aps_fit_highbound)
|
|
384
384
|
|
|
385
|
-
|
|
386
|
-
|
|
385
|
+
merge_data_cor_dn, density_cor_dn, _ = _merge_data(*merge_arg, 'mobility', _alg_type='cor_dn')
|
|
386
|
+
density_cor_dn.columns = ['density']
|
|
387
387
|
|
|
388
|
-
|
|
389
|
-
|
|
388
|
+
out_rho = concat([density_dn, density_cor_dn, density_dsdv, density], axis=1)
|
|
389
|
+
out_rho.columns = ['dn', 'cor_dn', 'dndsdv', 'cor_dndsdv']
|
|
390
390
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
391
|
+
## out
|
|
392
|
+
out_dic = {
|
|
393
|
+
'data_cor_dndsdv': merge_data,
|
|
394
|
+
'data_dn': merge_data_dn,
|
|
395
|
+
'data_dndsdv': merge_data_dsdv,
|
|
396
|
+
'data_cor_dn': merge_data_cor_dn,
|
|
397
397
|
|
|
398
|
-
|
|
398
|
+
'density': out_rho,
|
|
399
399
|
|
|
400
|
-
|
|
400
|
+
# 'data_all_aer' : merge_data_aer,
|
|
401
401
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
402
|
+
# 'density_cor_dndsdv' : density,
|
|
403
|
+
# 'density_dn' : density_dn,
|
|
404
|
+
# 'density_dndsdv' : density_dsdv,
|
|
405
|
+
# 'density_cor_dn' : density_cor_dn,
|
|
406
|
+
}
|
|
407
407
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
408
|
+
## process data
|
|
409
|
+
for _nam, _df in out_dic.items():
|
|
410
|
+
out_dic[_nam] = _df.reindex(smps.index).copy()
|
|
411
411
|
|
|
412
|
-
|
|
412
|
+
return out_dic
|
|
413
413
|
|
|
414
414
|
|
|
415
415
|
def merge_SMPS_APS(df_smps, df_aps, aps_unit='um', smps_overlap_lowbound=500, aps_fit_highbound=1000, dndsdv_alg=True):
|
|
416
|
-
|
|
416
|
+
# merge_data, merge_data_dn, merge_data_dsdv, merge_data_cor_dn, density, density_dn, density_dsdv, density_cor_dn = [DataFrame([np.nan])] * 8
|
|
417
417
|
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
418
|
+
## set to the same units
|
|
419
|
+
smps, aps = df_smps.copy(), df_aps.copy()
|
|
420
|
+
smps.columns = smps.keys().to_numpy(float)
|
|
421
|
+
aps.columns = aps.keys().to_numpy(float)
|
|
422
422
|
|
|
423
|
-
|
|
424
|
-
|
|
423
|
+
if aps_unit == 'um':
|
|
424
|
+
aps.columns = aps.keys() * 1e3
|
|
425
425
|
|
|
426
|
-
|
|
426
|
+
oth_typ = dict()
|
|
427
427
|
|
|
428
|
-
|
|
429
|
-
|
|
428
|
+
aps_input = aps.copy()
|
|
429
|
+
aps_over = aps_input.loc[:, (aps.keys() > 700) & (aps.keys() < 1000)].copy()
|
|
430
430
|
|
|
431
|
-
|
|
432
|
-
|
|
431
|
+
smps_input = smps.copy()
|
|
432
|
+
smps_over = smps_input[smps.keys()[smps.keys() > 500]].copy()
|
|
433
433
|
|
|
434
|
-
|
|
434
|
+
for _count in range(2):
|
|
435
435
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
436
|
+
## shift data calculate
|
|
437
|
+
## original
|
|
438
|
+
if _count == 0:
|
|
439
|
+
alg_type = 'dn'
|
|
440
|
+
shift = _powerlaw_fit_dN(smps_over, aps_over, alg_type)
|
|
441
441
|
|
|
442
|
-
|
|
443
|
-
|
|
442
|
+
if dndsdv_alg:
|
|
443
|
+
shift_dsdv = _corr_with_dNdSdV(smps_over, aps_over, 'dndsdv').mask(shift.isna())
|
|
444
444
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
445
|
+
## aps correct
|
|
446
|
+
else:
|
|
447
|
+
alg_type = 'cor_dndsdv'
|
|
448
|
+
shift_cor = _powerlaw_fit_dN(smps_over, aps_over, 'cor_dn')
|
|
449
449
|
|
|
450
|
-
|
|
451
|
-
|
|
450
|
+
if dndsdv_alg:
|
|
451
|
+
shift = _corr_with_dNdSdV(smps_over, aps_over, alg_type).mask(shift_cor.isna())
|
|
452
452
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
453
|
+
## merge aps and smps
|
|
454
|
+
## 1. power law fit (dn) -> return dn data and aps correct factor
|
|
455
|
+
## 2. correaltion with dn, ds, dv -> return corrected dn_ds_dv data
|
|
456
|
+
if (alg_type == 'dn') | dndsdv_alg:
|
|
457
|
+
merge_arg = (smps_input, aps_input, shift, smps_overlap_lowbound, aps_fit_highbound)
|
|
458
458
|
|
|
459
|
-
|
|
460
|
-
|
|
459
|
+
merge_data, density, _corr = _merge_data(*merge_arg, 'mobility', _alg_type=alg_type)
|
|
460
|
+
density.columns = ['density']
|
|
461
461
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
462
|
+
## without aps correct
|
|
463
|
+
if _count == 0:
|
|
464
|
+
## merge aps and smps
|
|
465
|
+
## dn_ds_dv data
|
|
466
|
+
if dndsdv_alg:
|
|
467
|
+
alg_type = 'dndsdv'
|
|
468
|
+
merge_arg = (smps_input, aps_input, shift_dsdv, smps_overlap_lowbound, aps_fit_highbound)
|
|
469
469
|
|
|
470
|
-
|
|
471
|
-
|
|
470
|
+
merge_data_dsdv, density_dsdv, _ = _merge_data(*merge_arg, 'mobility', _alg_type=alg_type)
|
|
471
|
+
density_dsdv.columns = ['density']
|
|
472
472
|
|
|
473
|
-
|
|
474
|
-
|
|
473
|
+
## dn data
|
|
474
|
+
merge_data_dn, density_dn = merge_data.copy(), density.copy()
|
|
475
475
|
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
476
|
+
## correct aps data
|
|
477
|
+
corr = _corr.resample('1d').mean().reindex(smps.index).ffill()
|
|
478
|
+
corr = corr.mask(corr < 1, 1)
|
|
479
479
|
|
|
480
|
-
|
|
481
|
-
|
|
480
|
+
aps_input.loc[:, corr.keys()] *= corr
|
|
481
|
+
aps_over = aps_input.copy()
|
|
482
482
|
|
|
483
483
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
484
|
+
## with aps correct
|
|
485
|
+
else:
|
|
486
|
+
## merge aps and smps
|
|
487
|
+
## dn data
|
|
488
|
+
alg_type = 'cor_dn'
|
|
489
|
+
merge_arg = (smps_input, aps_input, shift_cor, smps_overlap_lowbound, aps_fit_highbound)
|
|
490
490
|
|
|
491
|
-
|
|
492
|
-
|
|
491
|
+
merge_data_cor_dn, density_cor_dn, _ = _merge_data(*merge_arg, 'mobility', _alg_type=alg_type)
|
|
492
|
+
density_cor_dn.columns = ['density']
|
|
493
493
|
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
494
|
+
## out
|
|
495
|
+
out_rho = concat([density_dn, density_cor_dn, density_dsdv, density], axis=1)
|
|
496
|
+
out_rho.columns = ['dn', 'cor_dn', 'dndsdv', 'cor_dndsdv']
|
|
497
497
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
498
|
+
out_dic = {
|
|
499
|
+
'data_cor_dndsdv': merge_data,
|
|
500
|
+
'data_dn': merge_data_dn,
|
|
501
|
+
'data_dndsdv': merge_data_dsdv,
|
|
502
|
+
'data_cor_dn': merge_data_cor_dn,
|
|
503
503
|
|
|
504
|
-
|
|
504
|
+
'density': out_rho,
|
|
505
505
|
|
|
506
|
-
|
|
506
|
+
# 'data_all_aer' : merge_data_aer,
|
|
507
507
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
508
|
+
# 'density_cor_dndsdv' : density,
|
|
509
|
+
# 'density_dn' : density_dn,
|
|
510
|
+
# 'density_dndsdv' : density_dsdv,
|
|
511
|
+
# 'density_cor_dn' : density_cor_dn,
|
|
512
|
+
}
|
|
513
513
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
514
|
+
## process data
|
|
515
|
+
for _nam, _df in out_dic.items():
|
|
516
|
+
out_dic[_nam] = _df.reindex(smps.index).copy()
|
|
517
517
|
|
|
518
|
-
|
|
518
|
+
return out_dic
|