metradar 0.1.5__py3-none-any.whl → 0.1.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metradar/__init__.py +4 -2
- metradar/config.py +53 -0
- metradar/core/__init__.py +9 -0
- metradar/{get_cross_section_from_pyart.py → core/get_cross_section.py} +5 -157
- metradar/{mosaic_merge.py → core/mosaic_merge.py} +3 -1
- metradar/{oa_dig_func.py → core/oa_dig_func.py} +83 -333
- metradar/graph/__init__.py +9 -0
- metradar/{mosaic_quickdraw.py → graph/draw_comp_mosaic.py} +71 -68
- metradar/{draw_mosaic_new.py → graph/draw_latlon_func.py} +177 -173
- metradar/graph/draw_radar_aws.py +212 -0
- metradar/{draw_radar_comp_func.py → graph/draw_radar_comp_func.py} +319 -248
- metradar/graph/parse_pal.py +157 -0
- metradar/io/__init__.py +9 -0
- metradar/{cnrad_level2.py → io/cnrad_level2.py} +45 -2
- metradar/{decode_fmt_pyart.py → io/decode_fmt_pyart.py} +19 -3
- metradar/{decode_pup_rose.py → io/decode_pup_rose.py} +241 -684
- metradar/{read_new_mosaic_func.py → io/read_new_mosaic_func.py} +30 -2
- metradar/io/read_swan.py +250 -0
- metradar/{rose_structer.py → io/rose_structer.py} +2 -0
- metradar/project/__init__.py +9 -0
- metradar/project/make_mosaic/__init__.py +9 -0
- metradar/project/make_mosaic/batch_draw_mosaic.py +32 -0
- metradar/{make_mosaic_mp_archive.py → project/make_mosaic/make_mosaic_func.py} +144 -175
- metradar/project/make_mosaic/make_mosaic_mp.ini +29 -0
- metradar/project/make_mosaic/make_mosaic_mp.py +70 -0
- metradar/project/make_vpr_aws/__init__.py +9 -0
- metradar/project/make_vpr_aws/construct_aws_refvpr_mainprog.ini +39 -0
- metradar/project/make_vpr_aws/construct_aws_refvpr_mainprog.py +565 -0
- metradar/project/nowcasting/__init__.py +9 -0
- metradar/project/nowcasting/nowcast_by_pysteps.py +214 -0
- metradar/{trans_nc_pgmb.py → project/nowcasting/trans_mosaic_pgmb.py} +19 -17
- metradar/project/qpe/Archive /346/250/241/345/274/217/350/257/264/346/230/216.txt" +2 -0
- metradar/project/qpe/__init__.py +9 -0
- metradar/project/qpe/archive_main_qpe_cfg.ini +91 -0
- metradar/project/qpe/do_s1.sh +6 -0
- metradar/project/qpe/do_s2.sh +6 -0
- metradar/project/qpe/do_s3.sh +6 -0
- metradar/project/qpe/do_s4.sh +6 -0
- metradar/project/qpe/do_s5.sh +6 -0
- metradar/project/qpe/exec_all.sh +11 -0
- metradar/project/qpe/get_rainrate_func.py +80 -0
- metradar/project/qpe/main_qpe_cfg.ini +85 -0
- metradar/project/qpe/s1_download_radar_region_cmadaas.py +123 -0
- metradar/project/qpe/s2_pre_process_single_radar.py +183 -0
- metradar/project/qpe/s3_trans_rainrate_to_qpe.py +499 -0
- metradar/project/qpe/s4_mosaic_qpe.py +523 -0
- metradar/project/qpe/s5_draw_qpe_mosaic.py +308 -0
- metradar/project/wind_retrieval/__init__.py +9 -0
- metradar/project/wind_retrieval/config_3dwind.ini +45 -0
- metradar/{main_pydda.py → project/wind_retrieval/main_pydda.py} +152 -149
- metradar/util/__init__.py +9 -0
- metradar/{comm_func.py → util/comm_func.py} +1 -41
- metradar/util/exceptions.py +50 -0
- metradar/util/geo_transforms_pyart.py +627 -0
- metradar/{get_tlogp_from_sharppy.py → util/get_tlogp_from_sharppy.py} +16 -5
- metradar/{parse_pal.py → util/parse_pal.py} +147 -147
- metradar/util/radar_common.py +16 -0
- metradar/{trans_new_mosaic_nc.py → util/trans_new_mosaic_nc.py} +1 -1
- metradar-0.1.8.dist-info/METADATA +91 -0
- metradar-0.1.8.dist-info/RECORD +67 -0
- {metradar-0.1.5.dist-info → metradar-0.1.8.dist-info}/WHEEL +1 -1
- metradar-0.1.8.dist-info/licenses/LICENSE +21 -0
- metradar/grid.py +0 -281
- metradar/grid_data.py +0 -64
- metradar/oa_couhua.py +0 -166
- metradar/read_new_mosaic.py +0 -33
- metradar/retrieve_cmadaas.py +0 -3126
- metradar/retrieve_micaps_server.py +0 -2061
- metradar-0.1.5.dist-info/METADATA +0 -37
- metradar-0.1.5.dist-info/RECORD +0 -33
- /metradar/{pgmb_io.py → io/pgmb_io.py} +0 -0
- /metradar/{exceptions.py → project/make_vpr_aws/exceptions.py} +0 -0
- /metradar/{geo_transforms_pyart.py → project/make_vpr_aws/geo_transforms_pyart.py} +0 -0
- /metradar/{make_gif.py → util/make_gif.py} +0 -0
- {metradar-0.1.5.dist-info → metradar-0.1.8.dist-info}/top_level.txt +0 -0
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
制作雷达拼图,多进程
|
|
3
|
-
朱文剑
|
|
4
|
-
2021
|
|
1
|
+
# _*_ coding: utf-8 _*_
|
|
5
2
|
|
|
6
|
-
|
|
3
|
+
# 制作三维拼图的核心函数
|
|
7
4
|
|
|
8
|
-
# %%
|
|
9
5
|
|
|
10
6
|
import pyart
|
|
11
7
|
import os
|
|
@@ -13,15 +9,23 @@ import time
|
|
|
13
9
|
import pandas as pd
|
|
14
10
|
from datetime import datetime,timedelta
|
|
15
11
|
import numpy as np
|
|
16
|
-
from multiprocessing import
|
|
17
|
-
from mosaic_merge import mosaic_merge
|
|
12
|
+
from multiprocessing import Pool
|
|
13
|
+
from metradar.core.mosaic_merge import mosaic_merge
|
|
14
|
+
|
|
18
15
|
import configparser
|
|
19
16
|
import warnings
|
|
20
17
|
import glob
|
|
21
18
|
warnings.filterwarnings('ignore')
|
|
22
|
-
|
|
23
|
-
import
|
|
24
|
-
from decode_fmt_pyart import read_cnrad_fmt
|
|
19
|
+
|
|
20
|
+
from metradar.util.radar_common import RADAR_COMMON
|
|
21
|
+
from metradar.io.decode_fmt_pyart import read_cnrad_fmt
|
|
22
|
+
radarcommon = RADAR_COMMON()
|
|
23
|
+
|
|
24
|
+
from metradar.config import CONFIG
|
|
25
|
+
|
|
26
|
+
# 资源文件路径
|
|
27
|
+
RESOURCES_PATH = CONFIG.get('SETTING','RESOURCES_PATH')
|
|
28
|
+
RADAR_FILE = RESOURCES_PATH + '/stations/radars.csv'
|
|
25
29
|
|
|
26
30
|
class MAKE_RADAR_MOSAIC:
|
|
27
31
|
|
|
@@ -49,25 +53,29 @@ class MAKE_RADAR_MOSAIC:
|
|
|
49
53
|
|
|
50
54
|
return config
|
|
51
55
|
|
|
52
|
-
def __init__(self,rcfile)
|
|
56
|
+
def __init__(self,rcfile):
|
|
53
57
|
self.berror = False
|
|
54
58
|
if not os.path.exists(rcfile):
|
|
55
59
|
print(rcfile + ' not exists!')
|
|
56
60
|
self.berror = True
|
|
57
|
-
|
|
61
|
+
|
|
62
|
+
|
|
58
63
|
config = self._get_config_from_rcfile(rcfile)
|
|
59
64
|
self.run_mode = config['RUN_MODE']['run_mode']
|
|
65
|
+
|
|
60
66
|
|
|
61
67
|
self.radar_mode = config['COMMON_SETTING']['radar_mode']
|
|
62
|
-
self.radar_sitesfile =
|
|
68
|
+
self.radar_sitesfile = RADAR_FILE
|
|
63
69
|
|
|
64
70
|
if not os.path.exists(self.radar_sitesfile):
|
|
65
|
-
print(self.radar_sitesfile + '
|
|
71
|
+
print("雷达站点文件:" + self.radar_sitesfile + ' 不存在,请检查!')
|
|
66
72
|
self.berror = True
|
|
67
|
-
|
|
73
|
+
|
|
68
74
|
|
|
69
75
|
self.realtime_peroids = int(config['REAL_TIME']['realtime_peroids'])
|
|
70
76
|
|
|
77
|
+
self.filename_type = config['COMMON_SETTING']['filename_type']
|
|
78
|
+
|
|
71
79
|
self.center_lon = float(config['COMMON_SETTING']['center_lon'])
|
|
72
80
|
self.center_lat = float(config['COMMON_SETTING']['center_lat'])
|
|
73
81
|
self.mosaic_range = float(config['COMMON_SETTING']['mosaic_range'])
|
|
@@ -85,9 +93,8 @@ class MAKE_RADAR_MOSAIC:
|
|
|
85
93
|
self.radars = config['COMMON_SETTING']['radars'].split(',')
|
|
86
94
|
self.radars = [tt.strip(' ') for tt in self.radars]
|
|
87
95
|
else:
|
|
88
|
-
|
|
89
|
-
#
|
|
90
|
-
# 将km半径大致除以100,换算成°
|
|
96
|
+
# Automatically calculate the radar station name within the target range according to longitude, latitude and radius
|
|
97
|
+
#
|
|
91
98
|
minlon = self.center_lon - self.mosaic_range / 100.0
|
|
92
99
|
maxlon = self.center_lon + self.mosaic_range / 100.0
|
|
93
100
|
minlat = self.center_lat - self.mosaic_range / 100.0
|
|
@@ -95,12 +102,11 @@ class MAKE_RADAR_MOSAIC:
|
|
|
95
102
|
|
|
96
103
|
# load radar site file
|
|
97
104
|
|
|
98
|
-
#
|
|
105
|
+
# Find the corresponding Chinese name according to the digital station number
|
|
99
106
|
|
|
100
107
|
frad = open(self.radar_sitesfile,'rt',encoding='UTF-8')
|
|
101
108
|
|
|
102
109
|
infos = frad.readlines()
|
|
103
|
-
radsite_num = {}
|
|
104
110
|
radname_gr2 = {}
|
|
105
111
|
staname_chn={}
|
|
106
112
|
radarlat={}
|
|
@@ -113,13 +119,13 @@ class MAKE_RADAR_MOSAIC:
|
|
|
113
119
|
curlon = float(tmps[4])
|
|
114
120
|
curlat = float(tmps[3])
|
|
115
121
|
if curlon > minlon and curlon < maxlon and curlat > minlat and curlat < maxlat:
|
|
116
|
-
|
|
122
|
+
|
|
117
123
|
radname_gr2[tmps[2]] = tmps[0]
|
|
118
124
|
staname_chn[tmps[2]] = tmps[1]
|
|
119
125
|
radarlat[tmps[2]]=float(tmps[3])
|
|
120
126
|
radarlon[tmps[2]]=float(tmps[4])
|
|
121
127
|
radheight[tmps[2]]=float(tmps[5])
|
|
122
|
-
self.radars = list(
|
|
128
|
+
self.radars = list(staname_chn.values())
|
|
123
129
|
|
|
124
130
|
self.input_path_realtime = config['REAL_TIME']['input_path_realtime']
|
|
125
131
|
self.output_path_realtime = config['REAL_TIME']['output_path_realtime']
|
|
@@ -136,56 +142,31 @@ class MAKE_RADAR_MOSAIC:
|
|
|
136
142
|
except:
|
|
137
143
|
self.berror = True
|
|
138
144
|
print('starttime set error!')
|
|
139
|
-
|
|
145
|
+
|
|
140
146
|
|
|
141
147
|
try:
|
|
142
148
|
et = datetime.strptime(self.endtime,'%Y%m%d%H%M')
|
|
143
149
|
except:
|
|
144
150
|
self.berror = True
|
|
145
151
|
print('endtime set error!')
|
|
146
|
-
return None
|
|
147
152
|
|
|
148
|
-
|
|
149
153
|
|
|
150
154
|
# check time step
|
|
151
155
|
if self.tstep < 0 or self.tstep > 60:
|
|
152
156
|
print('tstep set error, should between 0 and 60')
|
|
153
157
|
self.berror = True
|
|
154
|
-
|
|
158
|
+
|
|
155
159
|
|
|
156
160
|
if (et - st).total_seconds() < self.tstep * 60:
|
|
157
161
|
print('seconds between endtime and starttime is less than tstep, error!')
|
|
158
162
|
self.berror = True
|
|
159
|
-
return None
|
|
160
|
-
|
|
161
163
|
|
|
162
|
-
|
|
164
|
+
if self.berror:
|
|
165
|
+
return radarcommon.RETURN_CODE_PARAM_ERROR
|
|
163
166
|
|
|
164
167
|
|
|
165
168
|
def make_mosaic(self, params):
|
|
166
169
|
|
|
167
|
-
'''
|
|
168
|
-
params={
|
|
169
|
-
# 原始数据路径
|
|
170
|
-
'rootpath':rootpath,
|
|
171
|
-
# 输出数据路径
|
|
172
|
-
'outpath': outpath,
|
|
173
|
-
# 参与拼图的雷达站点
|
|
174
|
-
'radars': ['NJJS','HAJS','BBAH','MASR','HFAH','CZJS','TZJS'],
|
|
175
|
-
'origin_latitude':center_lat, # 拼图的中心纬度
|
|
176
|
-
'origin_longitude':center_lon, # 拼图的中心经度
|
|
177
|
-
'mosaic_range':mosaic_range, # 拼图半径,km,其实是正方形
|
|
178
|
-
'hor_reso':1, # 水平方向分辨率,km
|
|
179
|
-
'ver_reso':0.5, # 垂直方向分辨率,km
|
|
180
|
-
'bdebug':True,
|
|
181
|
-
'timestr':timestr,
|
|
182
|
-
'bot_z_lev':0.5, # 最底层所在高度
|
|
183
|
-
'top_z_lev':16, # 最顶层所在高度
|
|
184
|
-
'breplace':True,
|
|
185
|
-
'outname': 'mosaic_%s.nc'%timestr
|
|
186
|
-
}
|
|
187
|
-
'''
|
|
188
|
-
|
|
189
170
|
rootpath=params['rootpath']
|
|
190
171
|
outpath=params['outpath']
|
|
191
172
|
radars=params['radars']
|
|
@@ -201,51 +182,39 @@ class MAKE_RADAR_MOSAIC:
|
|
|
201
182
|
top_z_lev = params['top_z_lev']
|
|
202
183
|
breplace = params['breplace']
|
|
203
184
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
# timestr = '201607190112'
|
|
185
|
+
|
|
186
|
+
os.makedirs(outpath,exist_ok=True)
|
|
187
|
+
# timestr = '20220415155500'
|
|
208
188
|
if not breplace and os.path.exists(outpath + os.sep + outname):
|
|
209
189
|
print(outname + ' already exits!')
|
|
210
190
|
return False
|
|
211
|
-
curt = datetime.strptime(timestr,'%Y%m%d%H%M')
|
|
191
|
+
curt = datetime.strptime(timestr,'%Y%m%d%H%M%S')
|
|
212
192
|
curt = curt.timestamp()
|
|
213
193
|
timeinfo=dict()
|
|
214
194
|
fileinfo=dict()
|
|
215
195
|
nvalidradars = 0
|
|
216
196
|
validradars=[]
|
|
197
|
+
|
|
198
|
+
# 遍历所有目录
|
|
199
|
+
|
|
217
200
|
for radar in radars:
|
|
218
|
-
curpath = rootpath + os.sep + radar
|
|
219
|
-
if not os.path.exists(curpath):
|
|
220
|
-
# print(curpath + ' not exists!')
|
|
221
|
-
# radars.remove(radar)
|
|
222
|
-
continue
|
|
223
|
-
|
|
224
|
-
curfiles = os.listdir(curpath)
|
|
225
|
-
if len(curfiles)==0:
|
|
226
|
-
print(curpath + ' file number = %d'%len(curfiles))
|
|
227
|
-
continue
|
|
228
201
|
|
|
229
202
|
validfiles = []
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
203
|
+
validpaths = []
|
|
204
|
+
for root, dirs, files in os.walk(rootpath):
|
|
205
|
+
for file in files:
|
|
206
|
+
if file.endswith('.bz2'):
|
|
207
|
+
if file.find(radar) < 0:
|
|
208
|
+
continue
|
|
209
|
+
validfiles.append(file)
|
|
210
|
+
validpaths.append(root)
|
|
211
|
+
|
|
236
212
|
curfiles = sorted(validfiles)
|
|
237
213
|
tmptime=[]
|
|
238
214
|
tmpfile=[]
|
|
239
215
|
for file in curfiles:
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
# curfiles.remove(file)
|
|
243
|
-
continue
|
|
244
|
-
|
|
245
|
-
# cft = datetime.strptime(file[5:5+14],'%Y%m%d%H%M%S')
|
|
246
|
-
# Z_RADR_I_Z9090_20230703193348_O_DOR_SB_CAP_FMT.bin.bz2
|
|
247
|
-
# cft = datetime.strptime(file[5:13]+file[14:20],'%Y%m%d%H%M%S')
|
|
248
|
-
cft = datetime.strptime(file[15:29],'%Y%m%d%H%M%S')
|
|
216
|
+
curpath = validpaths[validfiles.index(file)]
|
|
217
|
+
cft = datetime.strptime(file.split('_')[4],'%Y%m%d%H%M%S')
|
|
249
218
|
tmptime.append(cft.timestamp())
|
|
250
219
|
tmpfile.append(curpath + os.sep + file)
|
|
251
220
|
if len(tmptime)==0:
|
|
@@ -256,6 +225,7 @@ class MAKE_RADAR_MOSAIC:
|
|
|
256
225
|
|
|
257
226
|
nvalidradars +=1
|
|
258
227
|
validradars.append(radar)
|
|
228
|
+
|
|
259
229
|
used_files = []
|
|
260
230
|
if nvalidradars == 0:
|
|
261
231
|
print('no valid files')
|
|
@@ -272,16 +242,19 @@ class MAKE_RADAR_MOSAIC:
|
|
|
272
242
|
idx = idx[0]
|
|
273
243
|
if abs(tmpt[idx]) > 240:
|
|
274
244
|
continue
|
|
245
|
+
|
|
275
246
|
used_files.append(fileinfo[radar][idx])
|
|
276
247
|
# assert isinstance(filenames,list)
|
|
277
248
|
|
|
278
249
|
if len(used_files)==0:
|
|
279
250
|
print('Error: len(used_files)==0')
|
|
280
|
-
return
|
|
251
|
+
return radarcommon.RETURN_CODE_DATA_ERROR
|
|
252
|
+
|
|
281
253
|
xgridnum = int(2*mr/hor_reso) + 1
|
|
282
254
|
ygridnum = int(2*mr/hor_reso) + 1
|
|
283
255
|
zgridnum = int((top_z_lev - bot_z_lev)/ver_reso) + 1
|
|
284
|
-
print('xgridnum = %d, ygridnum = %d, zgridnum = %d'%(xgridnum,ygridnum,zgridnum))
|
|
256
|
+
print('top_z_lev = %.1f, bot_z_lev = %.1f, xgridnum = %d, ygridnum = %d, zgridnum = %d'%(top_z_lev,bot_z_lev,xgridnum,ygridnum,zgridnum))
|
|
257
|
+
|
|
285
258
|
radars=[]
|
|
286
259
|
for filename in used_files:
|
|
287
260
|
try:
|
|
@@ -291,7 +264,7 @@ class MAKE_RADAR_MOSAIC:
|
|
|
291
264
|
print(filename + ' read error!')
|
|
292
265
|
continue
|
|
293
266
|
radars.append(radar)
|
|
294
|
-
print('add file %s'%filename)
|
|
267
|
+
# print('add file %s'%filename)
|
|
295
268
|
|
|
296
269
|
sttime = time.time()
|
|
297
270
|
grid = pyart.map.grid_from_radars(
|
|
@@ -299,7 +272,7 @@ class MAKE_RADAR_MOSAIC:
|
|
|
299
272
|
grid_origin = [origin_latitude,origin_longitude],
|
|
300
273
|
grid_shape=(zgridnum, ygridnum, xgridnum),
|
|
301
274
|
grid_limits=((bot_z_lev*1000, top_z_lev*1000), (-1.0*mr*1000, 1.0*mr*1000), (-1.0*mr*1000, 1.0*mr*1000)),
|
|
302
|
-
fields=['reflectivity'
|
|
275
|
+
fields=['reflectivity'])
|
|
303
276
|
|
|
304
277
|
edtime = time.time()
|
|
305
278
|
if bdebug:
|
|
@@ -322,7 +295,7 @@ class MAKE_RADAR_MOSAIC:
|
|
|
322
295
|
# gx.reflectivity.data=gx.reflectivity.astype(np.uint8)
|
|
323
296
|
# gx = gx.drop_vars('ROI')
|
|
324
297
|
# cref = np.nanmax(gx.reflectivity.data[0,:,:,:],axis=0)
|
|
325
|
-
# #
|
|
298
|
+
# # add var to Dataset
|
|
326
299
|
# gx["cref"]=(['y', 'x'],cref,{'scale':2,'offset':64,'decode':'dBZ = (cref-64)/2'})
|
|
327
300
|
# # gx = gx.drop_vars('reflectivity')
|
|
328
301
|
# gx.to_netcdf(outpath + os.sep + outname)
|
|
@@ -331,15 +304,32 @@ class MAKE_RADAR_MOSAIC:
|
|
|
331
304
|
|
|
332
305
|
print(outpath + os.sep + outname + ' done!')
|
|
333
306
|
|
|
307
|
+
|
|
308
|
+
return radarcommon.RETURN_CODE_SUCESSED
|
|
309
|
+
|
|
334
310
|
def do_realtime(self,):
|
|
335
|
-
if os.path.exists('
|
|
336
|
-
|
|
337
|
-
|
|
311
|
+
if os.path.exists('file_mosaic.lock'):
|
|
312
|
+
# check time If more than 10 minutes have passed, delete and run
|
|
313
|
+
fin = open('file_mosaic.lock','rt')
|
|
314
|
+
tstr = fin.readline()
|
|
315
|
+
fin.close()
|
|
316
|
+
brestart = False
|
|
317
|
+
if len(tstr) == 0:
|
|
318
|
+
brestart = True
|
|
319
|
+
ctt = datetime.strptime(tstr,'%Y%m%d%H%M%S')
|
|
320
|
+
if datetime.utcnow().timestamp() - ctt.timestamp() > 60*10:
|
|
321
|
+
brestart = True
|
|
322
|
+
|
|
323
|
+
if brestart:
|
|
324
|
+
os.remove('file_mosaic.lock')
|
|
325
|
+
else:
|
|
326
|
+
print('do_realtime is still running')
|
|
327
|
+
return radarcommon.RETURN_CODE_SUCESSED
|
|
338
328
|
|
|
339
|
-
print('
|
|
329
|
+
print('entering do_realtime')
|
|
340
330
|
|
|
341
|
-
#
|
|
342
|
-
lockfile = open('
|
|
331
|
+
# Create a lock file to prevent the program from running repeatedly
|
|
332
|
+
lockfile = open('file_mosaic.lock','wt')
|
|
343
333
|
lockfile.write(datetime.utcnow().strftime('%Y%m%d%H%M%S'))
|
|
344
334
|
lockfile.close()
|
|
345
335
|
|
|
@@ -357,36 +347,34 @@ class MAKE_RADAR_MOSAIC:
|
|
|
357
347
|
|
|
358
348
|
allparams = []
|
|
359
349
|
ct = datetime.utcnow()
|
|
360
|
-
newct = ct - timedelta(minutes= ct.minute % 6 )
|
|
361
|
-
timestr = newct.strftime('%Y%m%d%H%M')
|
|
350
|
+
newct = ct - timedelta(minutes= (ct.minute % 6 + 6))
|
|
351
|
+
timestr = newct.strftime('%Y%m%d%H%M') + '00'
|
|
362
352
|
print('processing : ' + timestr)
|
|
363
353
|
|
|
364
354
|
alloutfiles=[]
|
|
365
|
-
|
|
355
|
+
aa = self.filename_type.split('$')
|
|
356
|
+
mergename = outpath + os.sep + '%s%s%s'%(aa[0],timestr,aa[2])
|
|
366
357
|
if not breplace and os.path.exists(mergename):
|
|
367
358
|
print(mergename + ' already exists!')
|
|
368
|
-
return
|
|
359
|
+
return radarcommon.RETURN_CODE_SUCESSED
|
|
369
360
|
|
|
370
361
|
for nn in range(len(levels)-1):
|
|
371
|
-
|
|
372
|
-
outname = '
|
|
373
|
-
#
|
|
362
|
+
# 'mosaic_%s_%d_tmp.nc'%(timestr,nn+1)
|
|
363
|
+
outname = '%s%s_%d_tmp%s'%(aa[0],timestr,nn+1,aa[2])
|
|
364
|
+
# It is divided into multiple processes. Each process calculates a piece of puzzle on height, and then splices it
|
|
374
365
|
params={
|
|
375
|
-
|
|
376
|
-
'
|
|
377
|
-
# 输出数据路径
|
|
378
|
-
'outpath': outpath,
|
|
379
|
-
# 参与拼图的雷达站点
|
|
366
|
+
'rootpath':rootpath,
|
|
367
|
+
'outpath': outpath,
|
|
380
368
|
'radars':radars,
|
|
381
|
-
'origin_latitude':center_lat,
|
|
382
|
-
'origin_longitude':center_lon,
|
|
383
|
-
'mosaic_range':mosaic_range,
|
|
384
|
-
'hor_reso':hor_reso,
|
|
385
|
-
'ver_reso':ver_reso,
|
|
369
|
+
'origin_latitude':center_lat,
|
|
370
|
+
'origin_longitude':center_lon,
|
|
371
|
+
'mosaic_range':mosaic_range,
|
|
372
|
+
'hor_reso':hor_reso,
|
|
373
|
+
'ver_reso':ver_reso,
|
|
386
374
|
'bdebug':bshow_debug,
|
|
387
|
-
'timestr':timestr,
|
|
388
|
-
'bot_z_lev':levels[nn] + ver_reso,
|
|
389
|
-
'top_z_lev':levels[nn+1],
|
|
375
|
+
'timestr':timestr,
|
|
376
|
+
'bot_z_lev':levels[nn] + ver_reso,
|
|
377
|
+
'top_z_lev':levels[nn+1],
|
|
390
378
|
'breplace':breplace,
|
|
391
379
|
'outname':outname,
|
|
392
380
|
}
|
|
@@ -401,50 +389,53 @@ class MAKE_RADAR_MOSAIC:
|
|
|
401
389
|
pools.close()
|
|
402
390
|
pools.join()
|
|
403
391
|
|
|
404
|
-
#
|
|
392
|
+
# merge data
|
|
405
393
|
|
|
406
394
|
newgrid = mosaic_merge(files = alloutfiles)
|
|
407
395
|
# write to file
|
|
408
396
|
if newgrid is not None:
|
|
409
397
|
newgrid.write(mergename)
|
|
410
398
|
print(mergename + ' saved!')
|
|
411
|
-
# time.sleep(2)
|
|
412
399
|
|
|
400
|
+
|
|
413
401
|
else:
|
|
414
402
|
print('merge error!')
|
|
403
|
+
|
|
415
404
|
# delete temp files
|
|
416
405
|
files = glob.glob(outpath + os.sep + '*tmp.nc')
|
|
417
406
|
if len(files) >0:
|
|
418
407
|
for file in files:
|
|
419
408
|
os.remove(file)
|
|
420
409
|
|
|
421
|
-
#
|
|
422
|
-
if os.path.exists('
|
|
410
|
+
# Delete the lock file and allow the program to run
|
|
411
|
+
if os.path.exists('file_mosaic.lock'): os.remove('file_mosaic.lock')
|
|
423
412
|
|
|
413
|
+
|
|
424
414
|
|
|
425
415
|
def do_archive(self,):
|
|
426
416
|
|
|
427
417
|
st = datetime.strptime(self.starttime,'%Y%m%d%H%M')
|
|
428
418
|
et = datetime.strptime(self.endtime,'%Y%m%d%H%M')
|
|
429
|
-
periods = int(((et - st).total_seconds() / 60)/self.tstep)
|
|
419
|
+
periods = int(((et - st).total_seconds() / 60)/self.tstep)
|
|
430
420
|
print('periods = %d'%periods)
|
|
431
421
|
dt = pd.date_range(st,freq='%dmin'%self.tstep,periods=periods)
|
|
432
422
|
dt = pd.to_datetime(dt)
|
|
433
423
|
alltimes=[tt.strftime('%Y%m%d%H%M') for tt in dt]
|
|
434
424
|
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
425
|
+
rootpath = self.input_path_archive
|
|
426
|
+
outpath = self.output_path_archive
|
|
427
|
+
|
|
428
|
+
radars = self.radars
|
|
429
|
+
center_lat = self.center_lat
|
|
430
|
+
center_lon = self.center_lon
|
|
431
|
+
mosaic_range = self.mosaic_range
|
|
432
|
+
hor_reso = self.hori_reso
|
|
433
|
+
ver_reso = self.verti_reso
|
|
434
|
+
bshow_debug = self.bshow_debuginfo
|
|
435
|
+
breplace = self.breplace
|
|
436
|
+
levels = self.multi_levels
|
|
437
|
+
|
|
438
|
+
# levels KM
|
|
448
439
|
# levels = [0, 5, 9, 12, 16]
|
|
449
440
|
# levels = [0, 7, 10, 13, 16, 20] 82s
|
|
450
441
|
# levels = [0, 7, 10, 13, 16, 20]
|
|
@@ -456,51 +447,46 @@ class MAKE_RADAR_MOSAIC:
|
|
|
456
447
|
print('processing : ' + timestr)
|
|
457
448
|
|
|
458
449
|
alloutfiles=[]
|
|
459
|
-
rootpath = self.input_path_archive + os.sep + timestr[0:4] + os.sep + timestr[0:8]
|
|
460
|
-
outpath = self.output_path_archive
|
|
461
|
-
|
|
462
|
-
if not os.path.exists(outpath):
|
|
463
|
-
os.makedirs(outpath)
|
|
464
450
|
mergename = outpath + os.sep + 'mosaic_%s.nc'%timestr
|
|
465
|
-
if not
|
|
451
|
+
if not breplace and os.path.exists(mergename):
|
|
466
452
|
print(mergename + ' already exists!')
|
|
467
453
|
continue
|
|
468
454
|
|
|
469
|
-
for nn in range(len(
|
|
455
|
+
for nn in range(len(levels)-1):
|
|
470
456
|
|
|
471
457
|
outname = 'mosaic_%s_%d_tmp.nc'%(timestr,nn+1)
|
|
472
|
-
|
|
458
|
+
|
|
473
459
|
params={
|
|
474
|
-
|
|
460
|
+
|
|
475
461
|
'rootpath':rootpath,
|
|
476
|
-
|
|
462
|
+
|
|
477
463
|
'outpath': outpath,
|
|
478
|
-
|
|
479
|
-
'radars':
|
|
480
|
-
'origin_latitude':
|
|
481
|
-
'origin_longitude':
|
|
482
|
-
'mosaic_range':
|
|
483
|
-
'hor_reso':
|
|
484
|
-
'ver_reso':
|
|
485
|
-
'bdebug':
|
|
486
|
-
'timestr':timestr,
|
|
487
|
-
'bot_z_lev':
|
|
488
|
-
'top_z_lev':
|
|
489
|
-
'breplace':
|
|
464
|
+
|
|
465
|
+
'radars':radars,
|
|
466
|
+
'origin_latitude':center_lat,
|
|
467
|
+
'origin_longitude':center_lon,
|
|
468
|
+
'mosaic_range':mosaic_range,
|
|
469
|
+
'hor_reso':hor_reso,
|
|
470
|
+
'ver_reso':ver_reso,
|
|
471
|
+
'bdebug':bshow_debug,
|
|
472
|
+
'timestr':timestr,
|
|
473
|
+
'bot_z_lev':levels[nn] + ver_reso,
|
|
474
|
+
'top_z_lev':levels[nn+1],
|
|
475
|
+
'breplace':breplace,
|
|
490
476
|
'outname':outname,
|
|
491
477
|
}
|
|
492
478
|
|
|
493
479
|
alloutfiles.append(outpath + os.sep + outname)
|
|
494
480
|
allparams.append(params)
|
|
495
481
|
|
|
496
|
-
MAXP=len(
|
|
482
|
+
MAXP=len(levels)-1
|
|
497
483
|
pools = Pool(MAXP)
|
|
498
484
|
|
|
499
485
|
pools.map(self.make_mosaic, allparams)
|
|
500
486
|
pools.close()
|
|
501
487
|
pools.join()
|
|
502
488
|
|
|
503
|
-
#
|
|
489
|
+
# merge data
|
|
504
490
|
|
|
505
491
|
newgrid = mosaic_merge(files = alloutfiles)
|
|
506
492
|
if newgrid is not None:
|
|
@@ -516,23 +502,6 @@ class MAKE_RADAR_MOSAIC:
|
|
|
516
502
|
if len(files) >0:
|
|
517
503
|
for file in files:
|
|
518
504
|
os.remove(file)
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
job_thread.start()
|
|
523
|
-
|
|
524
|
-
if __name__ == "__main__":
|
|
525
|
-
|
|
526
|
-
print('starting mosaic program......')
|
|
527
|
-
|
|
528
|
-
_make_mosaic = MAKE_RADAR_MOSAIC('make_mosaic_mp_archive.ini')
|
|
529
|
-
|
|
530
|
-
if not _make_mosaic.berror:
|
|
531
|
-
if _make_mosaic.run_mode == 'archive':
|
|
532
|
-
_make_mosaic.do_archive()
|
|
533
|
-
else:
|
|
534
|
-
print('run mode is not archive!')
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
# %%
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
return radarcommon.RETURN_CODE_SUCESSED
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[RUN_MODE]
|
|
2
|
+
run_mode = archive
|
|
3
|
+
|
|
4
|
+
[REAL_TIME]
|
|
5
|
+
input_path_realtime = X:/radar/Archives
|
|
6
|
+
output_path_realtime = E:/metradar_test/realtime_mosaic
|
|
7
|
+
realtime_peroids = 3
|
|
8
|
+
|
|
9
|
+
[ARCHIVE]
|
|
10
|
+
input_path_archive = /mnt/e/metradar_test/vpr/FMT
|
|
11
|
+
output_path_archive = /mnt/e/metradar_test/vpr/mosaic/20230731_daxing
|
|
12
|
+
starttime = 202307310700
|
|
13
|
+
endtime = 202307311200
|
|
14
|
+
tstep = 6
|
|
15
|
+
|
|
16
|
+
[COMMON_SETTING]
|
|
17
|
+
filename_type = Z_RADR_TDMOSAIC_$YYYYMMDDhhmmss$.nc
|
|
18
|
+
radar_mode = manul
|
|
19
|
+
radars = Z9010,Z9220,Z9311,Z9317,Z9024,Z9025,
|
|
20
|
+
center_lon = 116.347
|
|
21
|
+
center_lat = 39.458
|
|
22
|
+
mosaic_range = 50
|
|
23
|
+
hori_reso = 1.0
|
|
24
|
+
verti_reso = 0.5
|
|
25
|
+
multi_levels = 0, 3,6, 9, 12, 15, 18, 20
|
|
26
|
+
mosaic_vars = REF,ZDR
|
|
27
|
+
breplace = 0
|
|
28
|
+
bshow_debuginfo = 1
|
|
29
|
+
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# _*_ coding: utf-8 _*_
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
'''
|
|
5
|
+
make radarmosaic
|
|
6
|
+
wenjianzhu
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
'''
|
|
10
|
+
|
|
11
|
+
# %%
|
|
12
|
+
import os
|
|
13
|
+
from datetime import datetime,timedelta
|
|
14
|
+
import glob
|
|
15
|
+
import schedule
|
|
16
|
+
import time
|
|
17
|
+
import threading
|
|
18
|
+
from metradar.project.make_mosaic.make_mosaic_func import MAKE_RADAR_MOSAIC
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run_threaded(job_func):
|
|
22
|
+
job_thread = threading.Thread(target=job_func)
|
|
23
|
+
job_thread.start()
|
|
24
|
+
|
|
25
|
+
if __name__ == "__main__":
|
|
26
|
+
|
|
27
|
+
print('starting mosaic program......')
|
|
28
|
+
with open('current_pid_mosaic.txt','wt') as f:
|
|
29
|
+
f.write('当前进程号为: %s'%str(os.getpid()) + ' ,' + datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
|
|
30
|
+
|
|
31
|
+
# 路径需要根据实际情况进行修改
|
|
32
|
+
_make_mosaic = MAKE_RADAR_MOSAIC('/home/wjzhu/OneDrive/PythonCode/MyWork/metradar/metradar/project/make_mosaic/make_mosaic_mp.ini')
|
|
33
|
+
|
|
34
|
+
if not _make_mosaic.berror:
|
|
35
|
+
|
|
36
|
+
if _make_mosaic.run_mode == 'realtime':
|
|
37
|
+
# Delete the lock file and allow the program to run
|
|
38
|
+
if os.path.exists('file_mosaic.lock'): os.remove('file_mosaic.lock')
|
|
39
|
+
|
|
40
|
+
if not os.path.exists(_make_mosaic.output_path_realtime):
|
|
41
|
+
os.makedirs(_make_mosaic.output_path_realtime)
|
|
42
|
+
|
|
43
|
+
if not os.path.exists(_make_mosaic.output_path_archive):
|
|
44
|
+
os.makedirs(_make_mosaic.output_path_archive)
|
|
45
|
+
|
|
46
|
+
# delete tmp.nc
|
|
47
|
+
# delete temp files
|
|
48
|
+
files = glob.glob(_make_mosaic.output_path_realtime + os.sep + '*tmp.nc')
|
|
49
|
+
if len(files) >0:
|
|
50
|
+
for file in files:
|
|
51
|
+
os.remove(file)
|
|
52
|
+
|
|
53
|
+
files = glob.glob(_make_mosaic.output_path_archive + os.sep + '*tmp.nc')
|
|
54
|
+
if len(files) >0:
|
|
55
|
+
for file in files:
|
|
56
|
+
os.remove(file)
|
|
57
|
+
|
|
58
|
+
# create task
|
|
59
|
+
run_threaded(_make_mosaic.do_realtime)
|
|
60
|
+
schedule.every(_make_mosaic.realtime_peroids*60).seconds.do(run_threaded, _make_mosaic.do_realtime)
|
|
61
|
+
while True:
|
|
62
|
+
schedule.run_pending()
|
|
63
|
+
time.sleep(1)
|
|
64
|
+
|
|
65
|
+
else: # archive mode
|
|
66
|
+
_make_mosaic.do_archive()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# %%
|
|
70
|
+
|