AeroViz 0.1.2__py3-none-any.whl → 0.1.3b0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of AeroViz might be problematic. Click here for more details.

Files changed (112) hide show
  1. AeroViz/__init__.py +4 -4
  2. AeroViz/config/DEFAULT_DATA.csv +1417 -0
  3. AeroViz/config/DEFAULT_PNSD_DATA.csv +1417 -0
  4. AeroViz/dataProcess/Chemistry/__init__.py +38 -38
  5. AeroViz/dataProcess/Chemistry/_calculate.py +15 -15
  6. AeroViz/dataProcess/Chemistry/_isoropia.py +69 -68
  7. AeroViz/dataProcess/Chemistry/_mass_volume.py +158 -158
  8. AeroViz/dataProcess/Chemistry/_ocec.py +109 -109
  9. AeroViz/dataProcess/Chemistry/_partition.py +19 -18
  10. AeroViz/dataProcess/Chemistry/_teom.py +8 -11
  11. AeroViz/dataProcess/Optical/_IMPROVE.py +40 -39
  12. AeroViz/dataProcess/Optical/__init__.py +35 -35
  13. AeroViz/dataProcess/Optical/_absorption.py +35 -35
  14. AeroViz/dataProcess/Optical/_extinction.py +25 -24
  15. AeroViz/dataProcess/Optical/_mie.py +5 -6
  16. AeroViz/dataProcess/Optical/_mie_sd.py +89 -90
  17. AeroViz/dataProcess/Optical/_scattering.py +16 -16
  18. AeroViz/dataProcess/SizeDistr/__init__.py +37 -37
  19. AeroViz/dataProcess/SizeDistr/__merge.py +159 -158
  20. AeroViz/dataProcess/SizeDistr/_merge.py +155 -154
  21. AeroViz/dataProcess/SizeDistr/_merge_v1.py +162 -161
  22. AeroViz/dataProcess/SizeDistr/_merge_v2.py +153 -152
  23. AeroViz/dataProcess/SizeDistr/_merge_v3.py +326 -326
  24. AeroViz/dataProcess/SizeDistr/_merge_v4.py +272 -274
  25. AeroViz/dataProcess/SizeDistr/_size_distr.py +51 -51
  26. AeroViz/dataProcess/VOC/__init__.py +7 -7
  27. AeroViz/dataProcess/VOC/_potential_par.py +53 -55
  28. AeroViz/dataProcess/VOC/voc_par.json +464 -0
  29. AeroViz/dataProcess/__init__.py +4 -4
  30. AeroViz/dataProcess/core/__init__.py +59 -58
  31. AeroViz/plot/__init__.py +6 -1
  32. AeroViz/plot/bar.py +126 -0
  33. AeroViz/plot/box.py +68 -0
  34. AeroViz/plot/distribution/distribution.py +421 -427
  35. AeroViz/plot/meteorology/meteorology.py +240 -292
  36. AeroViz/plot/optical/__init__.py +0 -1
  37. AeroViz/plot/optical/optical.py +230 -230
  38. AeroViz/plot/pie.py +198 -0
  39. AeroViz/plot/regression.py +210 -0
  40. AeroViz/plot/scatter.py +99 -0
  41. AeroViz/plot/templates/__init__.py +0 -3
  42. AeroViz/plot/templates/contour.py +25 -25
  43. AeroViz/plot/templates/corr_matrix.py +86 -93
  44. AeroViz/plot/templates/diurnal_pattern.py +24 -24
  45. AeroViz/plot/templates/koschmieder.py +106 -106
  46. AeroViz/plot/templates/metal_heatmap.py +34 -34
  47. AeroViz/plot/timeseries/timeseries.py +53 -60
  48. AeroViz/plot/utils/__init__.py +2 -1
  49. AeroViz/plot/utils/_color.py +57 -57
  50. AeroViz/plot/utils/_unit.py +48 -48
  51. AeroViz/plot/utils/fRH.json +390 -0
  52. AeroViz/plot/utils/plt_utils.py +92 -0
  53. AeroViz/plot/utils/sklearn_utils.py +49 -0
  54. AeroViz/plot/utils/units.json +84 -0
  55. AeroViz/plot/violin.py +79 -0
  56. AeroViz/process/__init__.py +15 -15
  57. AeroViz/process/core/DataProc.py +9 -9
  58. AeroViz/process/core/SizeDist.py +81 -81
  59. AeroViz/process/method/PyMieScatt_update.py +488 -488
  60. AeroViz/process/method/mie_theory.py +231 -229
  61. AeroViz/process/method/prop.py +40 -40
  62. AeroViz/process/script/AbstractDistCalc.py +103 -103
  63. AeroViz/process/script/Chemical.py +166 -166
  64. AeroViz/process/script/IMPACT.py +40 -40
  65. AeroViz/process/script/IMPROVE.py +152 -152
  66. AeroViz/process/script/Others.py +45 -45
  67. AeroViz/process/script/PSD.py +26 -26
  68. AeroViz/process/script/PSD_dry.py +69 -70
  69. AeroViz/process/script/retrieve_RI.py +50 -51
  70. AeroViz/rawDataReader/__init__.py +57 -57
  71. AeroViz/rawDataReader/core/__init__.py +328 -326
  72. AeroViz/rawDataReader/script/AE33.py +18 -18
  73. AeroViz/rawDataReader/script/AE43.py +20 -20
  74. AeroViz/rawDataReader/script/APS_3321.py +30 -30
  75. AeroViz/rawDataReader/script/Aurora.py +23 -23
  76. AeroViz/rawDataReader/script/BC1054.py +40 -40
  77. AeroViz/rawDataReader/script/EPA_vertical.py +9 -9
  78. AeroViz/rawDataReader/script/GRIMM.py +21 -21
  79. AeroViz/rawDataReader/script/IGAC_TH.py +67 -67
  80. AeroViz/rawDataReader/script/IGAC_ZM.py +59 -59
  81. AeroViz/rawDataReader/script/MA350.py +39 -39
  82. AeroViz/rawDataReader/script/NEPH.py +74 -74
  83. AeroViz/rawDataReader/script/OCEC_LCRES.py +21 -21
  84. AeroViz/rawDataReader/script/OCEC_RES.py +16 -16
  85. AeroViz/rawDataReader/script/SMPS_TH.py +25 -25
  86. AeroViz/rawDataReader/script/SMPS_aim11.py +32 -32
  87. AeroViz/rawDataReader/script/SMPS_genr.py +31 -31
  88. AeroViz/rawDataReader/script/TEOM.py +28 -28
  89. AeroViz/rawDataReader/script/Table.py +12 -12
  90. AeroViz/rawDataReader/script/VOC_TH.py +16 -16
  91. AeroViz/rawDataReader/script/VOC_ZM.py +28 -28
  92. AeroViz/rawDataReader/script/__init__.py +20 -20
  93. AeroViz/rawDataReader/utils/config.py +161 -161
  94. AeroViz/tools/database.py +65 -65
  95. AeroViz/tools/dataclassifier.py +106 -106
  96. AeroViz/tools/dataprinter.py +51 -51
  97. AeroViz/tools/datareader.py +38 -38
  98. {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/METADATA +5 -4
  99. AeroViz-0.1.3b0.dist-info/RECORD +110 -0
  100. AeroViz/config/__init__.py +0 -0
  101. AeroViz/plot/improve/__init__.py +0 -1
  102. AeroViz/plot/improve/improve.py +0 -240
  103. AeroViz/plot/optical/aethalometer.py +0 -77
  104. AeroViz/plot/templates/event_evolution.py +0 -65
  105. AeroViz/plot/templates/regression.py +0 -256
  106. AeroViz/plot/templates/scatter.py +0 -130
  107. AeroViz/plot/templates/templates.py +0 -398
  108. AeroViz/plot/utils/_decorator.py +0 -74
  109. AeroViz-0.1.2.dist-info/RECORD +0 -106
  110. {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/LICENSE +0 -0
  111. {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/WHEEL +0 -0
  112. {AeroViz-0.1.2.dist-info → AeroViz-0.1.3b0.dist-info}/top_level.txt +0 -0
@@ -4,28 +4,28 @@ from AeroViz.rawDataReader.core import AbstractReader
4
4
 
5
5
 
6
6
  class Reader(AbstractReader):
7
- nam = 'AE33'
7
+ nam = 'AE33'
8
8
 
9
- def _raw_reader(self, _file):
10
- _df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
11
- delimiter=r'\s+', skiprows=5, usecols=range(67))
12
- _df.columns = _df.columns.str.strip(';')
9
+ def _raw_reader(self, _file):
10
+ _df = read_table(_file, parse_dates={'time': [0, 1]}, index_col='time',
11
+ delimiter=r'\s+', skiprows=5, usecols=range(67))
12
+ _df.columns = _df.columns.str.strip(';')
13
13
 
14
- # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
15
- if not self._oth_set.get('ignore_err', False):
16
- _df = _df.where((_df['Status'] != 0) | (_df['Status'] != 128) | (_df['Status'] != 256)).copy()
14
+ # remove data without Status=0, 128 (Not much filter tape), 256 (Not much filter tape)
15
+ if not self._oth_set.get('ignore_err', False):
16
+ _df = _df.where((_df['Status'] != 0) | (_df['Status'] != 128) | (_df['Status'] != 256)).copy()
17
17
 
18
- return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']]
18
+ return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']]
19
19
 
20
- def _QC(self, _df):
21
- # remove negative value
22
- _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
20
+ def _QC(self, _df):
21
+ # remove negative value
22
+ _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']].mask((_df < 0).copy())
23
23
 
24
- # QC data in 5 min
25
- def _QC_func(df):
26
- _df_ave, _df_std = df.mean(), df.std()
27
- _df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
24
+ # QC data in 5 min
25
+ def _QC_func(df):
26
+ _df_ave, _df_std = df.mean(), df.std()
27
+ _df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
28
28
 
29
- return df.mask(_df_lowb | _df_highb).copy()
29
+ return df.mask(_df_lowb | _df_highb).copy()
30
30
 
31
- return _df.resample('5min').apply(_QC_func).resample('1h').mean()
31
+ return _df.resample('5min').apply(_QC_func).resample('1h').mean()
@@ -4,31 +4,31 @@ from AeroViz.rawDataReader.core import AbstractReader
4
4
 
5
5
 
6
6
  class Reader(AbstractReader):
7
- nam = 'AE43'
7
+ nam = 'AE43'
8
8
 
9
- def _raw_reader(self, _file):
10
- _df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
11
- _df_id = _df['SetupID'].iloc[-1]
9
+ def _raw_reader(self, _file):
10
+ _df = read_csv(_file, parse_dates={'time': ['StartTime']}, index_col='time')
11
+ _df_id = _df['SetupID'].iloc[-1]
12
12
 
13
- # get last SetupID data
14
- _df = _df.groupby('SetupID').get_group(_df_id)[
15
- ['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']].copy()
13
+ # get last SetupID data
14
+ _df = _df.groupby('SetupID').get_group(_df_id)[
15
+ ['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'Status']].copy()
16
16
 
17
- # remove data without Status=0
18
- _df = _df.where(_df['Status'] == 0).copy()
17
+ # remove data without Status=0
18
+ _df = _df.where(_df['Status'] == 0).copy()
19
19
 
20
- return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
20
+ return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7']]
21
21
 
22
- # QC data
23
- def _QC(self, _df):
24
- # remove negative value
25
- _df = _df.mask((_df < 0).copy())
22
+ # QC data
23
+ def _QC(self, _df):
24
+ # remove negative value
25
+ _df = _df.mask((_df < 0).copy())
26
26
 
27
- # QC data in 5 min
28
- def _QC_func(df):
29
- _df_ave, _df_std = df.mean(), df.std()
30
- _df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
27
+ # QC data in 5 min
28
+ def _QC_func(df):
29
+ _df_ave, _df_std = df.mean(), df.std()
30
+ _df_lowb, _df_highb = df < (_df_ave - _df_std * 1.5), df > (_df_ave + _df_std * 1.5)
31
31
 
32
- return df.mask(_df_lowb | _df_highb).copy()
32
+ return df.mask(_df_lowb | _df_highb).copy()
33
33
 
34
- return _df.resample('5min').apply(_QC_func).resample('1h').mean()
34
+ return _df.resample('5min').apply(_QC_func).resample('1h').mean()
@@ -5,43 +5,43 @@ from AeroViz.rawDataReader.core import AbstractReader
5
5
 
6
6
 
7
7
  class Reader(AbstractReader):
8
- nam = 'APS_3321'
8
+ nam = 'APS_3321'
9
9
 
10
- def _raw_reader(self, _file):
11
- with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
12
- _df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
13
- _key = list(_df.keys()[3:54]) ## 542 ~ 1981
10
+ def _raw_reader(self, _file):
11
+ with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
12
+ _df = read_table(f, skiprows=6, parse_dates={'Time': ['Date', 'Start Time']}).set_index('Time')
13
+ _key = list(_df.keys()[3:54]) ## 542 ~ 1981
14
14
 
15
- ## create new keys
16
- _newkey = {}
17
- for _k in _key:
18
- _newkey[_k] = float(_k).__round__(4)
19
- # _newkey['Mode(m)'] = 'mode'
15
+ ## create new keys
16
+ _newkey = {}
17
+ for _k in _key:
18
+ _newkey[_k] = float(_k).__round__(4)
19
+ # _newkey['Mode(m)'] = 'mode'
20
20
 
21
- ## get new dataframe
22
- _df = _df[_newkey.keys()].rename(_newkey, axis=1)
23
- # _df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
21
+ ## get new dataframe
22
+ _df = _df[_newkey.keys()].rename(_newkey, axis=1)
23
+ # _df['total'] = _df[list(_newkey.values())[:-1]].sum(axis=1)*(n.diff(n.log(_df.keys()[:-1].to_numpy(float))).mean()).copy()
24
24
 
25
- _df_idx = to_datetime(_df.index, errors='coerce')
25
+ _df_idx = to_datetime(_df.index, errors='coerce')
26
26
 
27
- return _df.set_index(_df_idx).loc[_df_idx.dropna()]
27
+ return _df.set_index(_df_idx).loc[_df_idx.dropna()]
28
28
 
29
- ## QC data
30
- def _QC(self, _df):
31
- ## mask out the data size lower than 7
32
- _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
33
- _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
34
- _df = _df.mask(_df_size < 7)
29
+ ## QC data
30
+ def _QC(self, _df):
31
+ ## mask out the data size lower than 7
32
+ _df['total'] = _df.sum(axis=1, min_count=1) * (n.diff(n.log(_df.keys().to_numpy(float)))).mean()
33
+ _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
34
+ _df = _df.mask(_df_size < 7)
35
35
 
36
- ## remove total conc. lower than 700
37
- _df = _df.mask(_df['total'] > 700)
36
+ ## remove total conc. lower than 700
37
+ _df = _df.mask(_df['total'] > 700)
38
38
 
39
- # not confirmed
40
- """
41
- ## remove the bin over 4000 nm which num. conc. larger than 1
42
- # _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2]>=4.]
39
+ # not confirmed
40
+ """
41
+ ## remove the bin over 4000 nm which num. conc. larger than 1
42
+ # _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2]>=4.]
43
43
 
44
- # _df_1hr[_df_remv_ky] = _df_1hr[_df_remv_ky].copy().mask(_df_1hr[_df_remv_ky]>1.)
45
- # """
44
+ # _df_1hr[_df_remv_ky] = _df_1hr[_df_remv_ky].copy().mask(_df_1hr[_df_remv_ky]>1.)
45
+ # """
46
46
 
47
- return _df[_df.keys()[:-1]]
47
+ return _df[_df.keys()[:-1]]
@@ -4,35 +4,35 @@ from AeroViz.rawDataReader.core import AbstractReader
4
4
 
5
5
 
6
6
  class Reader(AbstractReader):
7
- nam = 'Aurora'
7
+ nam = 'Aurora'
8
8
 
9
- def _raw_reader(self, _file):
10
- with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
11
- _df = read_csv(f, low_memory=False, index_col=0)
9
+ def _raw_reader(self, _file):
10
+ with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
11
+ _df = read_csv(f, low_memory=False, index_col=0)
12
12
 
13
- _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
14
- _df.index.name = 'time'
13
+ _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
14
+ _df.index.name = 'time'
15
15
 
16
- _df.columns = _df.keys().str.strip(' ')
16
+ _df.columns = _df.keys().str.strip(' ')
17
17
 
18
- _df = _df.loc[
19
- _df.index.dropna(), ['0°σspB', '0°σspG', '0°σspR', '90°σspB', '90°σspG', '90°σspR', 'RH']].copy()
20
- _df.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
18
+ _df = _df.loc[
19
+ _df.index.dropna(), ['0°σspB', '0°σspG', '0°σspR', '90°σspB', '90°σspG', '90°σspR', 'RH']].copy()
20
+ _df.columns = ['B', 'G', 'R', 'BB', 'BG', 'BR', 'RH']
21
21
 
22
- return _df
22
+ return _df
23
23
 
24
- ## QC data
25
- def _QC(self, _df):
26
- ## remove negative value
27
- _df = _df.mask((_df <= 0).copy())
24
+ ## QC data
25
+ def _QC(self, _df):
26
+ ## remove negative value
27
+ _df = _df.mask((_df <= 0).copy())
28
28
 
29
- ## call by _QC function
30
- ## QC data in 1 hr
31
- def _QC_func(_df_1hr):
32
- _df_ave = _df_1hr.mean()
33
- _df_std = _df_1hr.std()
34
- _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
29
+ ## call by _QC function
30
+ ## QC data in 1 hr
31
+ def _QC_func(_df_1hr):
32
+ _df_ave = _df_1hr.mean()
33
+ _df_std = _df_1hr.std()
34
+ _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
35
35
 
36
- return _df_1hr.mask(_df_lowb | _df_highb).copy()
36
+ return _df_1hr.mask(_df_lowb | _df_highb).copy()
37
37
 
38
- return _df.resample('1h', group_keys=False).apply(_QC_func)
38
+ return _df.resample('1h', group_keys=False).apply(_QC_func)
@@ -4,43 +4,43 @@ from AeroViz.rawDataReader.core import AbstractReader
4
4
 
5
5
 
6
6
  class Reader(AbstractReader):
7
- nam = 'BC1054'
8
-
9
- def _raw_reader(self, _file):
10
- with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
11
- _df = read_csv(f, parse_dates=['Time'], index_col='Time')
12
-
13
- _df = _df.rename(columns={
14
- 'BC1(ng/m3)': 'BC1',
15
- 'BC2(ng/m3)': 'BC2',
16
- 'BC3(ng/m3)': 'BC3',
17
- 'BC4(ng/m3)': 'BC4',
18
- 'BC5(ng/m3)': 'BC5',
19
- 'BC6(ng/m3)': 'BC6',
20
- 'BC7(ng/m3)': 'BC7',
21
- 'BC8(ng/m3)': 'BC8',
22
- 'BC9(ng/m3)': 'BC9',
23
- 'BC10(ng/m3)': 'BC10'
24
- })
25
-
26
- # remove data without Status=32 (Automatic Tape Advance), 65536 (Tape Move)
27
- # if not self._oth_set.get('ignore_err', False):
28
- # _df = _df.where((_df['Status'] != 32) | (_df['Status'] != 65536)).copy()
29
-
30
- return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10', 'Status']]
31
-
32
- # QC data
33
- def _QC(self, _df):
34
- # remove negative value
35
- _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
36
-
37
- # call by _QC function
38
- # QC data in 1 hr
39
- def _QC_func(_df_1hr):
40
- _df_ave = _df_1hr.mean()
41
- _df_std = _df_1hr.std()
42
- _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
43
-
44
- return _df_1hr.mask(_df_lowb | _df_highb).copy()
45
-
46
- return _df.resample('1h', group_keys=False).apply(_QC_func).resample('5min').mean()
7
+ nam = 'BC1054'
8
+
9
+ def _raw_reader(self, _file):
10
+ with open(_file, 'r', encoding='utf-8', errors='ignore') as f:
11
+ _df = read_csv(f, parse_dates=['Time'], index_col='Time')
12
+
13
+ _df = _df.rename(columns={
14
+ 'BC1(ng/m3)': 'BC1',
15
+ 'BC2(ng/m3)': 'BC2',
16
+ 'BC3(ng/m3)': 'BC3',
17
+ 'BC4(ng/m3)': 'BC4',
18
+ 'BC5(ng/m3)': 'BC5',
19
+ 'BC6(ng/m3)': 'BC6',
20
+ 'BC7(ng/m3)': 'BC7',
21
+ 'BC8(ng/m3)': 'BC8',
22
+ 'BC9(ng/m3)': 'BC9',
23
+ 'BC10(ng/m3)': 'BC10'
24
+ })
25
+
26
+ # remove data without Status=32 (Automatic Tape Advance), 65536 (Tape Move)
27
+ # if not self._oth_set.get('ignore_err', False):
28
+ # _df = _df.where((_df['Status'] != 32) | (_df['Status'] != 65536)).copy()
29
+
30
+ return _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10', 'Status']]
31
+
32
+ # QC data
33
+ def _QC(self, _df):
34
+ # remove negative value
35
+ _df = _df[['BC1', 'BC2', 'BC3', 'BC4', 'BC5', 'BC6', 'BC7', 'BC8', 'BC9', 'BC10']].mask((_df < 0).copy())
36
+
37
+ # call by _QC function
38
+ # QC data in 1 hr
39
+ def _QC_func(_df_1hr):
40
+ _df_ave = _df_1hr.mean()
41
+ _df_std = _df_1hr.std()
42
+ _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
43
+
44
+ return _df_1hr.mask(_df_lowb | _df_highb).copy()
45
+
46
+ return _df.resample('1h', group_keys=False).apply(_QC_func).resample('5min').mean()
@@ -4,15 +4,15 @@ from AeroViz.rawDataReader.core import AbstractReader
4
4
 
5
5
 
6
6
  class Reader(AbstractReader):
7
- nam = 'EPA_vertical'
7
+ nam = 'EPA_vertical'
8
8
 
9
- def _raw_reader(self, _file):
10
- with _file.open('r', encoding='big5', errors='ignore') as f:
11
- _df = read_csv(f, names=['time', 'station', 'comp', 'data', None], skiprows=1, na_values=['-'],
12
- parse_dates=['time'], index_col='time')
13
- _df['data'] = to_numeric(_df['data'], errors='coerce')
9
+ def _raw_reader(self, _file):
10
+ with _file.open('r', encoding='big5', errors='ignore') as f:
11
+ _df = read_csv(f, names=['time', 'station', 'comp', 'data', None], skiprows=1, na_values=['-'],
12
+ parse_dates=['time'], index_col='time')
13
+ _df['data'] = to_numeric(_df['data'], errors='coerce')
14
14
 
15
- _df_piv = _df.pivot_table(values='data', columns='comp', index='time')
16
- _df_piv.index.name = 'time'
15
+ _df_piv = _df.pivot_table(values='data', columns='comp', index='time')
16
+ _df_piv.index.name = 'time'
17
17
 
18
- return _df_piv
18
+ return _df_piv
@@ -4,32 +4,32 @@ from AeroViz.rawDataReader.core import AbstractReader
4
4
 
5
5
 
6
6
  class Reader(AbstractReader):
7
- nam = 'GRIMM'
7
+ nam = 'GRIMM'
8
8
 
9
- def _raw_reader(self, _file):
9
+ def _raw_reader(self, _file):
10
10
 
11
- _df = read_csv(_file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
12
- dayfirst=True).rename_axis("Time")
13
- _df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
11
+ _df = read_csv(_file, header=233, delimiter='\t', index_col=0, parse_dates=[0], encoding='ISO-8859-1',
12
+ dayfirst=True).rename_axis("Time")
13
+ _df.index = to_datetime(_df.index, format="%d/%m/%Y %H:%M:%S", dayfirst=True)
14
14
 
15
- if _file.name.startswith("A407ST"):
16
- _df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
17
- else:
18
- _df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
15
+ if _file.name.startswith("A407ST"):
16
+ _df.drop(_df.columns[0:11].tolist() + _df.columns[128:].tolist(), axis=1, inplace=True)
17
+ else:
18
+ _df.drop(_df.columns[0:11].tolist() + _df.columns[-5:].tolist(), axis=1, inplace=True)
19
19
 
20
- if _df.empty:
21
- print(_file, "is empty")
22
- return None
20
+ if _df.empty:
21
+ print(_file, "is empty")
22
+ return None
23
23
 
24
- return _df / 0.035
24
+ return _df / 0.035
25
25
 
26
- def _QC(self, _df):
27
- # QC data in 1 hr
28
- def _QC_func(_df_1hr):
29
- _df_ave = _df_1hr.mean()
30
- _df_std = _df_1hr.std()
31
- _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
26
+ def _QC(self, _df):
27
+ # QC data in 1 hr
28
+ def _QC_func(_df_1hr):
29
+ _df_ave = _df_1hr.mean()
30
+ _df_std = _df_1hr.std()
31
+ _df_lowb, _df_highb = _df_1hr < (_df_ave - _df_std * 1.5), _df_1hr > (_df_ave + _df_std * 1.5)
32
32
 
33
- return _df_1hr.mask(_df_lowb | _df_highb).copy()
33
+ return _df_1hr.mask(_df_lowb | _df_highb).copy()
34
34
 
35
- return _df.resample('5min').apply(_QC_func).resample('1h').mean()
35
+ return _df.resample('5min').apply(_QC_func).resample('1h').mean()
@@ -8,97 +8,97 @@ from AeroViz.rawDataReader.core import AbstractReader
8
8
 
9
9
 
10
10
  class Reader(AbstractReader):
11
- nam = 'IGAC_TH'
11
+ nam = 'IGAC_TH'
12
12
 
13
- def _raw_reader(self, _file):
13
+ def _raw_reader(self, _file):
14
14
 
15
- self.meta['freq'] = self._oth_set.get('data_freq') or self.meta['freq']
15
+ self.meta['freq'] = self._oth_set.get('data_freq') or self.meta['freq']
16
16
 
17
- with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
18
- _df = read_csv(f, low_memory=False, index_col=0)
17
+ with (_file).open('r', encoding='utf-8-sig', errors='ignore') as f:
18
+ _df = read_csv(f, low_memory=False, index_col=0)
19
19
 
20
- _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
21
- _df.index.name = 'time'
20
+ _df.index = to_datetime(_df.index, errors='coerce', format=self._oth_set.get('date_format') or 'mixed')
21
+ _df.index.name = 'time'
22
22
 
23
- _df.columns = _df.keys().str.strip(' ')
23
+ _df.columns = _df.keys().str.strip(' ')
24
24
 
25
- _df = _df.loc[_df.index.dropna()].copy()
25
+ _df = _df.loc[_df.index.dropna()].copy()
26
26
 
27
- return _df.loc[~_df.index.duplicated()]
27
+ return _df.loc[~_df.index.duplicated()]
28
28
 
29
- ## QC data
30
- def _QC(self, _df):
29
+ ## QC data
30
+ def _QC(self, _df):
31
31
 
32
- ## QC parameter, function (MDL SE LE)
33
- _mdl = {
34
- 'Na+': 0.05,
35
- 'NH4+': 0.05,
36
- 'K+': 0.05,
37
- 'Mg2+': 0.05,
38
- 'Ca2+': 0.05,
39
- 'Cl-': 0.05,
40
- 'NO2-': 0.05,
41
- 'NO3-': 0.05,
42
- 'SO42-': 0.05,
43
- }
32
+ ## QC parameter, function (MDL SE LE)
33
+ _mdl = {
34
+ 'Na+': 0.05,
35
+ 'NH4+': 0.05,
36
+ 'K+': 0.05,
37
+ 'Mg2+': 0.05,
38
+ 'Ca2+': 0.05,
39
+ 'Cl-': 0.05,
40
+ 'NO2-': 0.05,
41
+ 'NO3-': 0.05,
42
+ 'SO42-': 0.05,
43
+ }
44
44
 
45
- def _se_le(_df_, _log=False):
46
- _df_ = np.log10(_df_) if _log else _df_
45
+ def _se_le(_df_, _log=False):
46
+ _df_ = np.log10(_df_) if _log else _df_
47
47
 
48
- _df_qua = _df_.quantile([.25, .75])
49
- _df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
50
- _df_iqr = _df_q3 - _df_q1
48
+ _df_qua = _df_.quantile([.25, .75])
49
+ _df_q1, _df_q3 = _df_qua.loc[.25].copy(), _df_qua.loc[.75].copy()
50
+ _df_iqr = _df_q3 - _df_q1
51
51
 
52
- _se = concat([_df_q1 - 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
53
- _le = concat([_df_q3 + 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
52
+ _se = concat([_df_q1 - 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
53
+ _le = concat([_df_q3 + 1.5 * _df_iqr] * len(_df_), axis=1).T.set_index(_df_.index)
54
54
 
55
- if _log:
56
- return 10 ** _se, 10 ** _le
57
- return _se, _le
55
+ if _log:
56
+ return 10 ** _se, 10 ** _le
57
+ return _se, _le
58
58
 
59
- _cation, _anion, _main = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'], ['Cl-', 'NO2-', 'NO3-', 'SO42-', ], ['SO42-',
60
- 'NO3-',
61
- 'NH4+']
59
+ _cation, _anion, _main = ['Na+', 'NH4+', 'K+', 'Mg2+', 'Ca2+'], ['Cl-', 'NO2-', 'NO3-', 'SO42-', ], ['SO42-',
60
+ 'NO3-',
61
+ 'NH4+']
62
62
 
63
- _df_salt = _df[_mdl.keys()].copy()
64
- _df_pm = _df['PM2.5'].copy()
63
+ _df_salt = _df[_mdl.keys()].copy()
64
+ _df_pm = _df['PM2.5'].copy()
65
65
 
66
- ## lower than PM2.5
67
- ## conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
68
- _df_salt = _df_salt.mask(_df_salt.sum(axis=1, min_count=1) > _df_pm).dropna(subset=_main).copy()
66
+ ## lower than PM2.5
67
+ ## conc. of main salt should be present at the same time (NH4+, SO42-, NO3-)
68
+ _df_salt = _df_salt.mask(_df_salt.sum(axis=1, min_count=1) > _df_pm).dropna(subset=_main).copy()
69
69
 
70
- ## mdl
71
- for (_key, _df_col), _mdl_val in zip(_df_salt.items(), _mdl.values()):
72
- _df_salt[_key] = _df_col.mask(_df_col < _mdl_val, _mdl_val / 2)
70
+ ## mdl
71
+ for (_key, _df_col), _mdl_val in zip(_df_salt.items(), _mdl.values()):
72
+ _df_salt[_key] = _df_col.mask(_df_col < _mdl_val, _mdl_val / 2)
73
73
 
74
- ## group by time (per month)
75
- _df_salt['tm'] = _df_salt.index.strftime('%Y-%m')
74
+ ## group by time (per month)
75
+ _df_salt['tm'] = _df_salt.index.strftime('%Y-%m')
76
76
 
77
- _df_lst = []
78
- for _ky, _df_grp in _df_salt.groupby('tm'):
79
- _df_grp = _df_grp[_mdl.keys()].copy()
77
+ _df_lst = []
78
+ for _ky, _df_grp in _df_salt.groupby('tm'):
79
+ _df_grp = _df_grp[_mdl.keys()].copy()
80
80
 
81
- ## calculate SE LE
82
- ## salt < LE
83
- _se, _le = _se_le(_df_grp, _log=True)
84
- _df_grp = _df_grp.mask(_df_grp > _le).copy()
81
+ ## calculate SE LE
82
+ ## salt < LE
83
+ _se, _le = _se_le(_df_grp, _log=True)
84
+ _df_grp = _df_grp.mask(_df_grp > _le).copy()
85
85
 
86
- ## C/A, A/C
87
- _rat_CA = (_df_grp[_cation].sum(axis=1) / _df_grp[_anion].sum(axis=1)).to_frame()
88
- _rat_AC = (1 / _rat_CA).copy()
86
+ ## C/A, A/C
87
+ _rat_CA = (_df_grp[_cation].sum(axis=1) / _df_grp[_anion].sum(axis=1)).to_frame()
88
+ _rat_AC = (1 / _rat_CA).copy()
89
89
 
90
- _se, _le = _se_le(_rat_CA, )
91
- _cond_CA = (_rat_CA < _le) & (_rat_CA > 0)
90
+ _se, _le = _se_le(_rat_CA, )
91
+ _cond_CA = (_rat_CA < _le) & (_rat_CA > 0)
92
92
 
93
- _se, _le = _se_le(_rat_AC, )
94
- _cond_AC = (_rat_AC < _le) & (_rat_AC > 0)
93
+ _se, _le = _se_le(_rat_AC, )
94
+ _cond_AC = (_rat_AC < _le) & (_rat_AC > 0)
95
95
 
96
- _df_grp = _df_grp.where((_cond_CA * _cond_AC)[0]).copy()
96
+ _df_grp = _df_grp.where((_cond_CA * _cond_AC)[0]).copy()
97
97
 
98
- ## conc. of main salt > SE
99
- _se, _le = _se_le(_df_grp[_main], _log=True)
100
- _df_grp[_main] = _df_grp[_main].mask(_df_grp[_main] < _se).copy()
98
+ ## conc. of main salt > SE
99
+ _se, _le = _se_le(_df_grp[_main], _log=True)
100
+ _df_grp[_main] = _df_grp[_main].mask(_df_grp[_main] < _se).copy()
101
101
 
102
- _df_lst.append(_df_grp)
102
+ _df_lst.append(_df_grp)
103
103
 
104
- return concat(_df_lst).reindex(_df.index)
104
+ return concat(_df_lst).reindex(_df.index)