metradar 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metradar/__init__.py +7 -0
- metradar/cnrad_level2.py +1326 -0
- metradar/comm_func.py +135 -0
- metradar/construct_aws_refvpr_mainprog.py +515 -0
- metradar/construct_aws_refvpr_mainprog_cams.py +310 -0
- metradar/construct_aws_refvpr_mainprog_datan3d.py +386 -0
- metradar/construct_aws_refvpr_mainprog_swan.py +306 -0
- metradar/decode_fmt_pyart.py +200 -0
- metradar/decode_pup_rose.py +1993 -0
- metradar/draw_mosaic_new.py +421 -0
- metradar/draw_radar_aws_jilin_new.py +206 -0
- metradar/draw_radar_comp_func.py +1379 -0
- metradar/exceptions.py +50 -0
- metradar/geo_transforms_pyart.py +627 -0
- metradar/get_cross_section_from_pyart.py +354 -0
- metradar/get_tlogp_from_sharppy.py +93 -0
- metradar/grid.py +281 -0
- metradar/grid_data.py +64 -0
- metradar/main_pydda.py +653 -0
- metradar/make_gif.py +24 -0
- metradar/make_mosaic_mp_archive.py +538 -0
- metradar/mosaic_merge.py +64 -0
- metradar/mosaic_quickdraw.py +338 -0
- metradar/nowcast_by_pysteps.py +219 -0
- metradar/oa_couhua.py +166 -0
- metradar/oa_dig_func.py +955 -0
- metradar/parse_pal.py +148 -0
- metradar/pgmb_io.py +169 -0
- metradar/prepare_for_radar_draw.py +197 -0
- metradar/read_new_mosaic.py +33 -0
- metradar/read_new_mosaic_func.py +231 -0
- metradar/retrieve_cmadaas.py +3126 -0
- metradar/retrieve_micaps_server.py +2061 -0
- metradar/rose_structer.py +807 -0
- metradar/trans_nc_pgmb.py +62 -0
- metradar/trans_new_mosaic_nc.py +309 -0
- metradar/trans_polor2grid_func.py +203 -0
- metradar-0.1.0.dist-info/METADATA +12 -0
- metradar-0.1.0.dist-info/RECORD +41 -0
- metradar-0.1.0.dist-info/WHEEL +5 -0
- metradar-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3126 @@
|
|
|
1
|
+
# _*_ coding: utf-8 _*_
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2020 NMC Developers.
|
|
4
|
+
# Distributed under the terms of the GPL V3 License.
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
Retrieve the CMADaaS 大数据云平台 using REST API with pure python code.
|
|
8
|
+
|
|
9
|
+
refer to:
|
|
10
|
+
http://idata.cma/cmadaas/
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import fnmatch
|
|
15
|
+
import warnings
|
|
16
|
+
import json
|
|
17
|
+
import pickle
|
|
18
|
+
import hashlib
|
|
19
|
+
import uuid
|
|
20
|
+
from datetime import datetime, timedelta
|
|
21
|
+
import urllib3
|
|
22
|
+
import urllib.request
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
import numpy as np
|
|
25
|
+
import pandas as pd
|
|
26
|
+
import xarray as xr
|
|
27
|
+
from tqdm import tqdm
|
|
28
|
+
import nmc_met_io.config as CONFIG
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_rest_result(interface_id, params, url_only=False,
|
|
32
|
+
dns=None, port=None, data_format='json'):
|
|
33
|
+
"""
|
|
34
|
+
Get the http result from CAMDaaS REST api service.
|
|
35
|
+
2021/01/13, 采用hashlib.md5对输入参数进行加密, 则在url中不会出现pwd关键字, 数据传输更加安全.
|
|
36
|
+
|
|
37
|
+
:param interface_id: MUSIC interface id.
|
|
38
|
+
:param params: dictionary for MUSIC parameters.
|
|
39
|
+
可以直接指定'serviceNodeId', 'userId'和'pwd'三个参数, 若没有, 则从 .nmcdev 配置文件中读取.
|
|
40
|
+
:param url_only: if url_only = True, return url string.
|
|
41
|
+
:param dns: CMADaaS数据库DNS地址, 若不指定, 则从 .nmcdev 配置文件中读取.
|
|
42
|
+
:param port, CMADaaS数据库DNS的端口, 若不指定, 则从 .nmcdev 配置文件中读取.
|
|
43
|
+
:param data_format: MUSIC server data output format.
|
|
44
|
+
:return:
|
|
45
|
+
|
|
46
|
+
:example:
|
|
47
|
+
params = {
|
|
48
|
+
'serviceNodeId':'NMIC_MUSIC_CMADAAS',
|
|
49
|
+
'userId':'test',
|
|
50
|
+
'pwd':'123456',
|
|
51
|
+
'dataCode':'SURF_CHN_MUL_HOR_N',
|
|
52
|
+
'elements':'Datetime,Station_Id_d,Lat,Lon,PRE_24h',
|
|
53
|
+
'times':'20200910000000',
|
|
54
|
+
'dataFormat':'json',
|
|
55
|
+
'limitCnt':'10'}
|
|
56
|
+
data = get_rest_result('getSurfEleByTime', params)
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
# set MUSIC server dns port
|
|
60
|
+
if dns is None:
|
|
61
|
+
dns = CONFIG.CONFIG['CMADaaS']['DNS']
|
|
62
|
+
if port is None:
|
|
63
|
+
port = CONFIG.CONFIG['CMADaaS']['PORT']
|
|
64
|
+
|
|
65
|
+
# construct complete parameters
|
|
66
|
+
sign_params = params.copy()
|
|
67
|
+
|
|
68
|
+
# user information
|
|
69
|
+
if 'serviceNodeId' not in sign_params:
|
|
70
|
+
sign_params['serviceNodeId'] = CONFIG.CONFIG['CMADaaS']['serviceNodeId']
|
|
71
|
+
if 'userId' not in sign_params:
|
|
72
|
+
sign_params['userId'] = CONFIG.CONFIG['CMADaaS']['USER_ID']
|
|
73
|
+
if 'pwd' not in sign_params:
|
|
74
|
+
sign_params['pwd'] = CONFIG.CONFIG['CMADaaS']['PASSWORD']
|
|
75
|
+
|
|
76
|
+
# data interface Id and out data format
|
|
77
|
+
sign_params['interfaceId'] = interface_id.strip()
|
|
78
|
+
if 'dataFormat' not in sign_params:
|
|
79
|
+
sign_params['dataFormat'] = data_format.strip()
|
|
80
|
+
|
|
81
|
+
# add time stamp and nonce code
|
|
82
|
+
sign_params['timestamp'] = str(int(datetime.now().timestamp()*1000))
|
|
83
|
+
sign_params['nonce'] = str(uuid.uuid1())
|
|
84
|
+
|
|
85
|
+
# construct sign string with hashlib md5 code
|
|
86
|
+
sign_str = ""
|
|
87
|
+
keys = sorted(sign_params)
|
|
88
|
+
for key in keys:
|
|
89
|
+
sign_str = sign_str + key + "=" + str(sign_params.get(key)).strip() + "&"
|
|
90
|
+
sign_str = sign_str[:-1]
|
|
91
|
+
sign_str = sign_str + '&sign=' + hashlib.md5(sign_str.encode(encoding='UTF-8')).hexdigest().upper()
|
|
92
|
+
sign_str = sign_str.replace('&pwd='+sign_params['pwd'].strip(), '')
|
|
93
|
+
|
|
94
|
+
# construct url
|
|
95
|
+
url_str = 'http://' + dns + ':' + port + '/music-ws/api?' + sign_str
|
|
96
|
+
if url_only:
|
|
97
|
+
return url_str
|
|
98
|
+
|
|
99
|
+
# request http contents
|
|
100
|
+
http = urllib3.PoolManager()
|
|
101
|
+
req = http.request('GET', url_str)
|
|
102
|
+
if req.status != 200:
|
|
103
|
+
print('Can not access the url: ' + url_str)
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
return req.data
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _load_contents(contents):
|
|
110
|
+
"""
|
|
111
|
+
Extract information from contents.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
contents (string): [description]
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
[type]: [description]
|
|
118
|
+
"""
|
|
119
|
+
if contents is None:
|
|
120
|
+
print('Return None.')
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
contents = json.loads(contents.decode('utf-8').replace('\x00', ''), strict=False)
|
|
125
|
+
except Exception as e:
|
|
126
|
+
print(e)
|
|
127
|
+
print(contents)
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
if contents['returnCode'] != '0':
|
|
131
|
+
print(contents)
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
return contents
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def cmadaas_obs_convert_type(obs_data):
|
|
138
|
+
"""
|
|
139
|
+
Convert observation dataframe to numeric types.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
obs_data (dataframe): data frame of observations.
|
|
143
|
+
"""
|
|
144
|
+
for column in obs_data.columns:
|
|
145
|
+
if column.upper().startswith('STATION'):
|
|
146
|
+
continue
|
|
147
|
+
if column.lower() in [
|
|
148
|
+
"province", "country", "city", "cnty", "town",
|
|
149
|
+
"data_id", "rep_corr_id", "admin_code_chn"]:
|
|
150
|
+
continue
|
|
151
|
+
if column.upper() == "DATETIME":
|
|
152
|
+
obs_data[column] = pd.to_datetime(obs_data[column], format="%Y-%m-%d %H:%M:%S")
|
|
153
|
+
continue
|
|
154
|
+
obs_data[column] = pd.to_numeric(obs_data[column])
|
|
155
|
+
obs_data[column].mask(obs_data[column] >= 999990.0, inplace=True)
|
|
156
|
+
|
|
157
|
+
return obs_data
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def cmadaas_get_obs_latest_time(data_code="SURF_CHN_MUL_HOR", latestTime=12):
|
|
161
|
+
"""
|
|
162
|
+
Get the latest time of the observations.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
data_code (str, optional): dataset code, like "SURF_CHN_MUL_HOR",
|
|
166
|
+
"SURF_CHN_MUL_HOR_N". Defaults to "SURF_CHN_MUL_HOR".
|
|
167
|
+
latestTime (int, optional): latestTime > 0, like 2 is return
|
|
168
|
+
the latest time in 2 hours. Defaults to 12.
|
|
169
|
+
Returns:
|
|
170
|
+
datetime object, the latest time
|
|
171
|
+
|
|
172
|
+
Examples:
|
|
173
|
+
>>> print(cmadaas_get_obs_latest_time())
|
|
174
|
+
2020-03-12 03:00:00
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
# set retrieve parameters
|
|
178
|
+
params = {'dataCode': data_code,
|
|
179
|
+
'latestTime': str(latestTime+8)} # 由于服务器是世界时, 因此需要+8才能与当前时间比较.
|
|
180
|
+
|
|
181
|
+
# interface id
|
|
182
|
+
interface_id = "getSurfLatestTime"
|
|
183
|
+
|
|
184
|
+
# retrieve data contents
|
|
185
|
+
contents = get_rest_result(interface_id, params)
|
|
186
|
+
contents = _load_contents(contents)
|
|
187
|
+
|
|
188
|
+
# construct pandas DataFrame
|
|
189
|
+
data = pd.DataFrame(contents['DS'])
|
|
190
|
+
time = pd.to_datetime(data['Datetime'], format="%Y%m%d%H%M%S")
|
|
191
|
+
|
|
192
|
+
return time[0]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def cmadaas_get_obs_files(times, data_code="SURF_CMPA_RT_NC", out_dir=None, pbar=True):
|
|
196
|
+
"""
|
|
197
|
+
Download surface observation file.
|
|
198
|
+
|
|
199
|
+
注意cmadass设置了ip访问次数, 如果下载的文件数量过多, 则会返回"-5004 Reach the hor access line."
|
|
200
|
+
表示单位小时内检索过于频繁,请降低检索频次. 另外, 也要注意下载的文件大小, 如果文件只有几K,表示下载不成功, 删除重新下载.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
times (str): times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'; or
|
|
204
|
+
time range, '[YYYYMMDDHHMISS,YYYYMMDDHHMISS]'
|
|
205
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CMPA_RT_NC".
|
|
206
|
+
out_dir (str, optional): download files to out_dir. if out_dir is None,
|
|
207
|
+
the cached directory will be used. Defaults to None.
|
|
208
|
+
pbar (bool, optional): Show progress bar, default to True.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
the list of download files path.
|
|
212
|
+
|
|
213
|
+
Examples:
|
|
214
|
+
>>> out_files = cmadaas_get_obs_files('20200311000000,20200312000000')
|
|
215
|
+
>>> out_files = cmadaas_get_obs_files('[20200308000000,20200312000000]')
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
if out_dir is None:
|
|
219
|
+
out_dir = CONFIG.get_cache_file(data_code, "", name="CMADaaS")
|
|
220
|
+
|
|
221
|
+
times = times.strip()
|
|
222
|
+
if times[0] == '[':
|
|
223
|
+
# set retrieve parameters
|
|
224
|
+
params = {'dataCode': data_code,
|
|
225
|
+
'timeRange': times}
|
|
226
|
+
interface_id = "getSurfFileByTimeRange"
|
|
227
|
+
else:
|
|
228
|
+
# set retrieve parameters
|
|
229
|
+
params = {'dataCode': data_code,
|
|
230
|
+
'times': times}
|
|
231
|
+
interface_id = "getSurfFileByTime"
|
|
232
|
+
|
|
233
|
+
# retrieve data contents
|
|
234
|
+
contents = get_rest_result(interface_id, params)
|
|
235
|
+
contents = _load_contents(contents)
|
|
236
|
+
if contents is None:
|
|
237
|
+
return None
|
|
238
|
+
|
|
239
|
+
# loop every file and download
|
|
240
|
+
out_files = []
|
|
241
|
+
files = tqdm(contents['DS']) if pbar else contents['DS']
|
|
242
|
+
for file in files:
|
|
243
|
+
out_file = Path(out_dir) / file['FILE_NAME']
|
|
244
|
+
if not out_file.is_file():
|
|
245
|
+
urllib.request.urlretrieve(file['FILE_URL'], out_file)
|
|
246
|
+
out_files.append(out_file)
|
|
247
|
+
|
|
248
|
+
return out_files
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def cmadaas_obs_by_time(times, data_code="SURF_CHN_MUL_HOR_N",
|
|
252
|
+
sta_levels=None, ranges=None, order=None,
|
|
253
|
+
count=None, distinct=False, trans_type=True,
|
|
254
|
+
elements="Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM"):
|
|
255
|
+
"""
|
|
256
|
+
Retrieve station records from CMADaaS by times.
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
times (str): times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
260
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
261
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
262
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
263
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
264
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
265
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
266
|
+
list: a,b,c;
|
|
267
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
268
|
+
order (str, optional): elements order, seperated by ',', like
|
|
269
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
270
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
271
|
+
distinct (bool, optional): return unique records. Defaults to False.
|
|
272
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
273
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
274
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
275
|
+
|
|
276
|
+
Returns:
|
|
277
|
+
pandas data frame: observation records.
|
|
278
|
+
|
|
279
|
+
Examples:
|
|
280
|
+
obsdata = cmadaas_obs_by_time('20181108000000')
|
|
281
|
+
"""
|
|
282
|
+
|
|
283
|
+
# set retrieve parameters
|
|
284
|
+
params = {'dataCode': data_code,
|
|
285
|
+
'times': times,
|
|
286
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
287
|
+
'elements': elements}
|
|
288
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
289
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
290
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
291
|
+
if distinct: params['distinct'] = "true"
|
|
292
|
+
|
|
293
|
+
# Interface, refer to
|
|
294
|
+
interface_id = "getSurfEleByTime"
|
|
295
|
+
|
|
296
|
+
# retrieve data contents
|
|
297
|
+
contents = get_rest_result(interface_id, params)
|
|
298
|
+
contents = _load_contents(contents)
|
|
299
|
+
if contents is None:
|
|
300
|
+
return None
|
|
301
|
+
|
|
302
|
+
# construct pandas DataFrame
|
|
303
|
+
data = pd.DataFrame(contents['DS'])
|
|
304
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
305
|
+
|
|
306
|
+
# return
|
|
307
|
+
return data
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def cmadaas_obs_by_time_range(time_range, data_code="SURF_CHN_MUL_HOR_N",
|
|
311
|
+
sta_levels=None, ranges=None, order=None,
|
|
312
|
+
count=None, distinct=False, trans_type=True,
|
|
313
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
314
|
+
"""
|
|
315
|
+
Retrieve station records from CMADaaS by time range.
|
|
316
|
+
|
|
317
|
+
Args:
|
|
318
|
+
time_range (str): time range for retrieve, "[YYYYMMDDHHMISS, YYYYMMDDHHMISS]",
|
|
319
|
+
like"[20160901000000,20161001000000]"
|
|
320
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
321
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
322
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
323
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
324
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
325
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
326
|
+
list: a,b,c;
|
|
327
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
328
|
+
order (str, optional): elements order, seperated by ',', like
|
|
329
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
330
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
331
|
+
distinct (bool, optional): return unique records. Defaults to False.
|
|
332
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
333
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
334
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
335
|
+
|
|
336
|
+
Returns:
|
|
337
|
+
pandas data frame: observation records.
|
|
338
|
+
|
|
339
|
+
Examples:
|
|
340
|
+
>>> time_range = "[20180219000000,20180221000000]"
|
|
341
|
+
>>> sta_levels = "011,012,013"
|
|
342
|
+
>>> data_code = "SURF_CHN_MUL_DAY"
|
|
343
|
+
>>> elements = "Station_Id_C,Station_Name,Datetime,Lat,Lon,PRE_Time_0808"
|
|
344
|
+
>>> data = cmadaas_obs_by_time_range(time_range, sta_levels=sta_levels,
|
|
345
|
+
data_code=data_code, elements=elements)
|
|
346
|
+
>>> print "retrieve successfully" if data is not None else "failed"
|
|
347
|
+
retrieve successfully
|
|
348
|
+
>>> obs_data = cmadaas_obs_by_time_range(
|
|
349
|
+
"[20160901000000,20161001000000]", data_code="SURF_CHN_MUL_DAY_N",
|
|
350
|
+
elements="Station_Id_C,Station_Name,Lat,Lon,Alti,Datetime,PRE_Time_0808")
|
|
351
|
+
"""
|
|
352
|
+
# set retrieve parameters
|
|
353
|
+
params = {'dataCode': data_code,
|
|
354
|
+
'timeRange': time_range,
|
|
355
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
356
|
+
'elements': elements,}
|
|
357
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
358
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
359
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
360
|
+
if distinct: params['distinct'] = "true"
|
|
361
|
+
|
|
362
|
+
# interface id
|
|
363
|
+
interface_id = "getSurfEleByTimeRange"
|
|
364
|
+
|
|
365
|
+
# retrieve data contents
|
|
366
|
+
contents = get_rest_result(interface_id, params)
|
|
367
|
+
contents = _load_contents(contents)
|
|
368
|
+
if contents is None:
|
|
369
|
+
return None
|
|
370
|
+
|
|
371
|
+
# construct pandas DataFrame
|
|
372
|
+
data = pd.DataFrame(contents['DS'])
|
|
373
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
374
|
+
|
|
375
|
+
# return
|
|
376
|
+
return data
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def cmadaas_obs_by_time_and_id(times, data_code="SURF_CHN_MUL_HOR_N",
|
|
380
|
+
sta_levels=None, ranges=None, order=None,
|
|
381
|
+
count=None, trans_type=True,
|
|
382
|
+
elements="Station_Id_C,Datetime,TEM",
|
|
383
|
+
sta_ids="54511"):
|
|
384
|
+
"""
|
|
385
|
+
Retrieve station records from CMADaaS by times and station ID
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
times (str): time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
389
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
390
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
391
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
392
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
393
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
394
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
395
|
+
list: a,b,c;
|
|
396
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
397
|
+
order (str, optional): elements order, seperated by ',', like
|
|
398
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
399
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
400
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
401
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
402
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
403
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. Defaults to "54511".
|
|
404
|
+
|
|
405
|
+
Returns:
|
|
406
|
+
pandas data frame: observation records.
|
|
407
|
+
|
|
408
|
+
Examples:
|
|
409
|
+
>>> data = cmadaas_obs_by_time_and_id('20170318000000')
|
|
410
|
+
"""
|
|
411
|
+
|
|
412
|
+
# set retrieve parameters
|
|
413
|
+
params = {'dataCode': data_code,
|
|
414
|
+
'elements': elements,
|
|
415
|
+
'times': times,
|
|
416
|
+
'staIds': sta_ids,
|
|
417
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
418
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
419
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
420
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
421
|
+
|
|
422
|
+
# interface id
|
|
423
|
+
interface_id = "getSurfEleByTimeAndStaID"
|
|
424
|
+
|
|
425
|
+
# retrieve data contents
|
|
426
|
+
contents = get_rest_result(interface_id, params)
|
|
427
|
+
contents = _load_contents(contents)
|
|
428
|
+
if contents is None:
|
|
429
|
+
return None
|
|
430
|
+
|
|
431
|
+
# construct pandas DataFrame
|
|
432
|
+
data = pd.DataFrame(contents['DS'])
|
|
433
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
434
|
+
|
|
435
|
+
# return
|
|
436
|
+
return data
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def cmadaas_obs_by_time_range_and_id(time_range, data_code="SURF_CHN_MUL_HOR_N",
|
|
440
|
+
sta_levels=None, ranges=None, order=None,
|
|
441
|
+
count=None, trans_type=True,
|
|
442
|
+
elements="Station_Id_C,Datetime,PRE",
|
|
443
|
+
sta_ids="54511"):
|
|
444
|
+
"""
|
|
445
|
+
Retrieve station records from CMADaaS by time range and station ID
|
|
446
|
+
|
|
447
|
+
Args:
|
|
448
|
+
time_range (str): time range for retrieve, "[YYYYMMDDHHMISS, YYYYMMDDHHMISS]",
|
|
449
|
+
like"[201509010000,20150903060000]"
|
|
450
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
451
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
452
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
453
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
454
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
455
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
456
|
+
list: a,b,c;
|
|
457
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
458
|
+
order (str, optional): elements order, seperated by ',', like
|
|
459
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
460
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
461
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
462
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
463
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
464
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. Defaults to "54511".
|
|
465
|
+
|
|
466
|
+
Returns:
|
|
467
|
+
pandas data frame: observation records.
|
|
468
|
+
|
|
469
|
+
Examples:
|
|
470
|
+
>>> data = cmadaas_obs_by_time_range_and_id("[20160801000000,20160802000000]")
|
|
471
|
+
"""
|
|
472
|
+
|
|
473
|
+
# set retrieve parameters
|
|
474
|
+
params = {'dataCode': data_code,
|
|
475
|
+
'elements': elements,
|
|
476
|
+
'timeRange': time_range,
|
|
477
|
+
'staIds': sta_ids,
|
|
478
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
479
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
480
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
481
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
482
|
+
|
|
483
|
+
# interface id
|
|
484
|
+
interface_id = "getSurfEleByTimeRangeAndStaID"
|
|
485
|
+
|
|
486
|
+
# retrieve data contents
|
|
487
|
+
contents = get_rest_result(interface_id, params)
|
|
488
|
+
contents = _load_contents(contents)
|
|
489
|
+
if contents is None:
|
|
490
|
+
return None
|
|
491
|
+
|
|
492
|
+
# construct pandas DataFrame
|
|
493
|
+
data = pd.DataFrame(contents['DS'])
|
|
494
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
495
|
+
|
|
496
|
+
# return
|
|
497
|
+
return data
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def cmadaas_obs_in_rect_by_time(times, limit, data_code="SURF_CHN_MUL_HOR_N",
|
|
501
|
+
sta_levels=None, ranges=None, order=None,
|
|
502
|
+
count=None, trans_type=True,
|
|
503
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
504
|
+
"""
|
|
505
|
+
Retrieve station records from CIMISS in region by times.
|
|
506
|
+
|
|
507
|
+
Args:
|
|
508
|
+
times (str): time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
509
|
+
limit (list): map limits, [min_lat, min_lon, max_lat, max_lon]
|
|
510
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
511
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
512
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
513
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
514
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
515
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
516
|
+
list: a,b,c;
|
|
517
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
518
|
+
order (str, optional): elements order, seperated by ',', like
|
|
519
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
520
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
521
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
522
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
523
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
524
|
+
|
|
525
|
+
Returns:
|
|
526
|
+
pandas data frame: observation records.
|
|
527
|
+
|
|
528
|
+
Examples:
|
|
529
|
+
>>> data = cmadaas_obs_in_rect_by_time('20170320000000', [35, 110, 45, 120])
|
|
530
|
+
"""
|
|
531
|
+
|
|
532
|
+
# set retrieve parameters
|
|
533
|
+
params = {'dataCode': data_code,
|
|
534
|
+
'elements': elements,
|
|
535
|
+
'times': times,
|
|
536
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
537
|
+
'minLon': '{:.10f}'.format(limit[1]),
|
|
538
|
+
'maxLat': '{:.10f}'.format(limit[2]),
|
|
539
|
+
'maxLon': '{:.10f}'.format(limit[3]),
|
|
540
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
541
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
542
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
543
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
544
|
+
|
|
545
|
+
# interface id
|
|
546
|
+
interface_id = "getSurfEleInRectByTime"
|
|
547
|
+
|
|
548
|
+
# retrieve data contents
|
|
549
|
+
contents = get_rest_result(interface_id, params)
|
|
550
|
+
contents = _load_contents(contents)
|
|
551
|
+
if contents is None:
|
|
552
|
+
return None
|
|
553
|
+
|
|
554
|
+
# construct pandas DataFrame
|
|
555
|
+
data = pd.DataFrame(contents['DS'])
|
|
556
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
557
|
+
|
|
558
|
+
# return
|
|
559
|
+
return data
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def cmadaas_obs_in_rect_by_time_range(time_range, limit, data_code="SURF_CHN_MUL_HOR_N",
|
|
563
|
+
ranges=None, order=None,
|
|
564
|
+
count=None, trans_type=True,
|
|
565
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
566
|
+
"""
|
|
567
|
+
Retrieve observation records from CIMISS by rect and time range.
|
|
568
|
+
|
|
569
|
+
Args:
|
|
570
|
+
time_range (str): time for retrieve, "[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
|
|
571
|
+
limit (list): map limits, [min_lat, min_lon, max_lat, max_lon]
|
|
572
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
573
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
574
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
575
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
576
|
+
list: a,b,c;
|
|
577
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
578
|
+
order (str, optional): elements order, seperated by ',', like
|
|
579
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
580
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
581
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
582
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
583
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
584
|
+
|
|
585
|
+
Returns:
|
|
586
|
+
pandas data frame: observation records.
|
|
587
|
+
|
|
588
|
+
Examples:
|
|
589
|
+
>>> elements = ("Station_Id_C,Station_Id_d,Station_Name,"
|
|
590
|
+
"Station_levl,Datetime,Lat,Lon,PRE_Time_0808")
|
|
591
|
+
>>> time_range = "[20160801000000,20160801000000]"
|
|
592
|
+
>>> data_code = "SURF_CHN_MUL_DAY"
|
|
593
|
+
>>> data = cmadaas_obs_in_rect_by_time_range(
|
|
594
|
+
time_range,[35,110,45,120], data_code=data_code,
|
|
595
|
+
elements=elements)
|
|
596
|
+
"""
|
|
597
|
+
|
|
598
|
+
# set retrieve parameters
|
|
599
|
+
params = {'dataCode': data_code,
|
|
600
|
+
'elements': elements,
|
|
601
|
+
'timeRange': time_range,
|
|
602
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
603
|
+
'minLon': '{:.10f}'.format(limit[1]),
|
|
604
|
+
'maxLat': '{:.10f}'.format(limit[2]),
|
|
605
|
+
'maxLon': '{:.10f}'.format(limit[3]),
|
|
606
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
607
|
+
#if sta_levels is not None: params['staLevels'] = sta_levels # getSurfEleInRectByTimeRange not support this parameter.
|
|
608
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
609
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
610
|
+
|
|
611
|
+
# interface id
|
|
612
|
+
interface_id = "getSurfEleInRectByTimeRange"
|
|
613
|
+
|
|
614
|
+
# retrieve data contents
|
|
615
|
+
contents = get_rest_result(interface_id, params)
|
|
616
|
+
contents = _load_contents(contents)
|
|
617
|
+
if contents is None:
|
|
618
|
+
return None
|
|
619
|
+
|
|
620
|
+
# construct pandas DataFrame
|
|
621
|
+
data = pd.DataFrame(contents['DS'])
|
|
622
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
623
|
+
|
|
624
|
+
# return
|
|
625
|
+
return data
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def cmadaas_obs_in_admin_by_time(times, admin="110000", data_code="SURF_CHN_MUL_HOR_N",
|
|
629
|
+
sta_levels=None, ranges=None, order=None,
|
|
630
|
+
count=None, trans_type=True,
|
|
631
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
632
|
+
"""
|
|
633
|
+
Retrieve station records from CIMISS in provinces by time.
|
|
634
|
+
|
|
635
|
+
Args:
|
|
636
|
+
times (str): time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
637
|
+
admin (str, optional): administration(or province code), sperated by ",",
|
|
638
|
+
like "110000" is Beijing, "440000" is Guangdong. Defaults to "110000".
|
|
639
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
640
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
641
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
642
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
643
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
644
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
645
|
+
list: a,b,c;
|
|
646
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
647
|
+
order (str, optional): elements order, seperated by ',', like
|
|
648
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
649
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
650
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
651
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
652
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
653
|
+
|
|
654
|
+
Returns:
|
|
655
|
+
pandas data frame: observation records.
|
|
656
|
+
|
|
657
|
+
Examples:
|
|
658
|
+
>>> data = cmadaas_obs_in_admin_by_time('20200206000000', admin="110000")
|
|
659
|
+
"""
|
|
660
|
+
|
|
661
|
+
# set retrieve parameters
|
|
662
|
+
params = {'dataCode': data_code,
|
|
663
|
+
'elements': elements,
|
|
664
|
+
'times': times,
|
|
665
|
+
'adminCodes': admin,
|
|
666
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
667
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
668
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
669
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
670
|
+
|
|
671
|
+
# interface id
|
|
672
|
+
interface_id = "getSurfEleInRegionByTime"
|
|
673
|
+
|
|
674
|
+
# retrieve data contents
|
|
675
|
+
contents = get_rest_result(interface_id, params)
|
|
676
|
+
contents = _load_contents(contents)
|
|
677
|
+
if contents is None:
|
|
678
|
+
return None
|
|
679
|
+
|
|
680
|
+
# construct pandas DataFrame
|
|
681
|
+
data = pd.DataFrame(contents['DS'])
|
|
682
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
683
|
+
|
|
684
|
+
# return
|
|
685
|
+
return data
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def cmadaas_obs_in_admin_by_time_range(time_range, admin="110000", data_code="SURF_CHN_MUL_HOR_N",
|
|
689
|
+
sta_levels=None, ranges=None, order=None,
|
|
690
|
+
count=None, trans_type=True,
|
|
691
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
692
|
+
"""
|
|
693
|
+
Retrieve observation records from CIMISS by provinces and time range.
|
|
694
|
+
|
|
695
|
+
Args:
|
|
696
|
+
time_range (str): time for retrieve, "[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
|
|
697
|
+
admin (str, optional): administration(or province code), sperated by ",",
|
|
698
|
+
like "110000" is Beijing, "440000" is Guangdong. Defaults to "110000".
|
|
699
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
700
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
701
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
702
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
703
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
704
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
705
|
+
list: a,b,c;
|
|
706
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
707
|
+
order (str, optional): elements order, seperated by ',', like
|
|
708
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
709
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
710
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
711
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
712
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
713
|
+
|
|
714
|
+
Returns:
|
|
715
|
+
pandas data frame: observation records.
|
|
716
|
+
|
|
717
|
+
Examples:
|
|
718
|
+
>>> elements = ("Station_Id_C,Station_Id_d,Station_Name,"
|
|
719
|
+
"Station_levl,Datetime,Lat,Lon,PRE_Time_0808")
|
|
720
|
+
>>> time_range = "[20160801000000,20160801000000]"
|
|
721
|
+
>>> data_code = "SURF_CHN_MUL_DAY"
|
|
722
|
+
>>> data = cmadaas_obs_in_admin_by_time_range(
|
|
723
|
+
time_range,admin="110000", data_code=data_code,
|
|
724
|
+
elements=elements)
|
|
725
|
+
"""
|
|
726
|
+
|
|
727
|
+
# set retrieve parameters
|
|
728
|
+
params = {'dataCode': data_code,
|
|
729
|
+
'elements': elements,
|
|
730
|
+
'timeRange': time_range,
|
|
731
|
+
'adminCodes': admin,
|
|
732
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
733
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
734
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
735
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
736
|
+
|
|
737
|
+
# interface id
|
|
738
|
+
interface_id = "getSurfEleInRegionByTimeRange"
|
|
739
|
+
|
|
740
|
+
# retrieve data contents
|
|
741
|
+
contents = get_rest_result(interface_id, params)
|
|
742
|
+
contents = _load_contents(contents)
|
|
743
|
+
if contents is None:
|
|
744
|
+
return None
|
|
745
|
+
|
|
746
|
+
# construct pandas DataFrame
|
|
747
|
+
data = pd.DataFrame(contents['DS'])
|
|
748
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
749
|
+
|
|
750
|
+
# return
|
|
751
|
+
return data
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def cmadaas_obs_in_basin_by_time(times, basin="CJLY", data_code="SURF_CHN_MUL_HOR_N",
|
|
755
|
+
sta_levels=None, ranges=None, order=None,
|
|
756
|
+
count=None, trans_type=True,
|
|
757
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
758
|
+
"""
|
|
759
|
+
Retrieve station records from CIMISS in basin by time.
|
|
760
|
+
|
|
761
|
+
Args:
|
|
762
|
+
times (str): time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
763
|
+
basin (str, optional): basin codes, sperated by ",", like "CJLY" is Yangzi River,
|
|
764
|
+
"sta_2480" is 2480 stations. Defaults to "CJLY".
|
|
765
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
766
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
767
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
768
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
769
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
770
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
771
|
+
list: a,b,c;
|
|
772
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
773
|
+
order (str, optional): elements order, seperated by ',', like
|
|
774
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
775
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
776
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
777
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
778
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
779
|
+
|
|
780
|
+
Returns:
|
|
781
|
+
pandas data frame: observation records.
|
|
782
|
+
|
|
783
|
+
Examples:
|
|
784
|
+
>>> data = cmadaas_obs_in_basin_by_time('20200206000000', basin="CJLY")
|
|
785
|
+
"""
|
|
786
|
+
|
|
787
|
+
# set retrieve parameters
|
|
788
|
+
params = {'dataCode': data_code,
|
|
789
|
+
'elements': elements,
|
|
790
|
+
'times': times,
|
|
791
|
+
'basinCodes': basin,
|
|
792
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
793
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
794
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
795
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
796
|
+
|
|
797
|
+
# interface id
|
|
798
|
+
interface_id = "getSurfEleInBasinByTime"
|
|
799
|
+
|
|
800
|
+
# retrieve data contents
|
|
801
|
+
contents = get_rest_result(interface_id, params)
|
|
802
|
+
contents = _load_contents(contents)
|
|
803
|
+
if contents is None:
|
|
804
|
+
return None
|
|
805
|
+
|
|
806
|
+
# construct pandas DataFrame
|
|
807
|
+
data = pd.DataFrame(contents['DS'])
|
|
808
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
809
|
+
|
|
810
|
+
# return
|
|
811
|
+
return data
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def cmadaas_obs_in_basin_by_time_range(time_range, basin="CJLY", data_code="SURF_CHN_MUL_HOR_N",
|
|
815
|
+
sta_levels=None, ranges=None, order=None,
|
|
816
|
+
count=None, trans_type=True,
|
|
817
|
+
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
|
|
818
|
+
"""
|
|
819
|
+
Retrieve observation records from CIMISS by basin and time range.
|
|
820
|
+
|
|
821
|
+
Args:
|
|
822
|
+
time_range (str): time for retrieve, "[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
|
|
823
|
+
basin (str, optional): basin codes, sperated by ",", like "CJLY" is Yangzi River,
|
|
824
|
+
"sta_2480" is 2480 stations. Defaults to "CJLY".
|
|
825
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
826
|
+
sta_levels (str, optional): station levels, seperated by ',',
|
|
827
|
+
like "011,012,013" for standard, base and general stations. Defaults to None.
|
|
828
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
829
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
830
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
831
|
+
list: a,b,c;
|
|
832
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
833
|
+
order (str, optional): elements order, seperated by ',', like
|
|
834
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
835
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
836
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
837
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
838
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
839
|
+
|
|
840
|
+
Returns:
|
|
841
|
+
pandas data frame: observation records.
|
|
842
|
+
|
|
843
|
+
Examples:
|
|
844
|
+
>>> elements = ("Station_Id_C,Station_Id_d,Station_Name,"
|
|
845
|
+
"Station_levl,Datetime,Lat,Lon,PRE_Time_0808")
|
|
846
|
+
>>> time_range = "[20160801000000,20160801000000]"
|
|
847
|
+
>>> data_code = "SURF_CHN_MUL_DAY"
|
|
848
|
+
>>> data = cmadaas_obs_in_basin_by_time_range(
|
|
849
|
+
time_range, basin="CJLY", data_code=data_code,
|
|
850
|
+
elements=elements)
|
|
851
|
+
"""
|
|
852
|
+
|
|
853
|
+
# set retrieve parameters
|
|
854
|
+
params = {'dataCode': data_code,
|
|
855
|
+
'elements': elements,
|
|
856
|
+
'timeRange': time_range,
|
|
857
|
+
'basinCodes': basin,
|
|
858
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
859
|
+
if sta_levels is not None: params['staLevels'] = sta_levels
|
|
860
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
861
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
862
|
+
|
|
863
|
+
# interface id
|
|
864
|
+
# http://10.20.76.55/cimissapiweb/apicustomapiclassdefine_list.action?ids=getNafpEleGridByTimeAndLevelAndValidtime&apiclass=NAFP_API
|
|
865
|
+
interface_id = "getSurfEleInBasinByTimeRange"
|
|
866
|
+
|
|
867
|
+
# retrieve data contents
|
|
868
|
+
contents = get_rest_result(interface_id, params)
|
|
869
|
+
contents = _load_contents(contents)
|
|
870
|
+
if contents is None:
|
|
871
|
+
return None
|
|
872
|
+
|
|
873
|
+
# construct pandas DataFrame
|
|
874
|
+
data = pd.DataFrame(contents['DS'])
|
|
875
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
876
|
+
|
|
877
|
+
# return
|
|
878
|
+
return data
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def cmadaas_obs_by_period(minYear, maxYear, minMD, maxMD, data_code="SURF_CHN_MUL_HOR_N",
|
|
882
|
+
ranges=None, order=None, count=None, trans_type=True,
|
|
883
|
+
elements="Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM"):
|
|
884
|
+
"""
|
|
885
|
+
Retrieve station records from CIMISS by same period in years.
|
|
886
|
+
|
|
887
|
+
Args:
|
|
888
|
+
minYear (int): start year, like 2005
|
|
889
|
+
maxYear (int): end year, like 2005
|
|
890
|
+
minMD (str): start date, like "0125" is 01/25
|
|
891
|
+
maxMD (str): end date, like "0205" is 02/25
|
|
892
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
893
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
894
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
895
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
896
|
+
list: a,b,c;
|
|
897
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
898
|
+
order (str, optional): elements order, seperated by ',', like
|
|
899
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
900
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
901
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
902
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
903
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
904
|
+
|
|
905
|
+
Returns:
|
|
906
|
+
pandas data frame: observation records.
|
|
907
|
+
|
|
908
|
+
Examples:
|
|
909
|
+
obsdata = cmadaas_obs_by_period(2015, 2018, "0501", "0505")
|
|
910
|
+
"""
|
|
911
|
+
|
|
912
|
+
# set retrieve parameters
|
|
913
|
+
params = {'dataCode': data_code,
|
|
914
|
+
'minYear': str(int(minYear)),
|
|
915
|
+
'maxYear': str(int(maxYear)),
|
|
916
|
+
'minMD': str(minMD),
|
|
917
|
+
'maxMD': str(maxMD),
|
|
918
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
919
|
+
'elements': elements}
|
|
920
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
921
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
922
|
+
|
|
923
|
+
# Interface, refer to
|
|
924
|
+
interface_id = "getSurfEleByInHistoryBySamePeriod"
|
|
925
|
+
|
|
926
|
+
# retrieve data contents
|
|
927
|
+
contents = get_rest_result(interface_id, params)
|
|
928
|
+
contents = _load_contents(contents)
|
|
929
|
+
if contents is None:
|
|
930
|
+
return None
|
|
931
|
+
|
|
932
|
+
# construct pandas DataFrame
|
|
933
|
+
data = pd.DataFrame(contents['DS'])
|
|
934
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
935
|
+
|
|
936
|
+
# return
|
|
937
|
+
return data
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
def cmadaas_obs_by_period_and_id(minYear, maxYear, minMD, maxMD, data_code="SURF_CHN_MUL_HOR_N",
|
|
941
|
+
ranges=None, order=None, count=None, trans_type=True,
|
|
942
|
+
elements="Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM",
|
|
943
|
+
sta_ids="54511"):
|
|
944
|
+
"""
|
|
945
|
+
Retrieve station id records from CIMISS by same period in years.
|
|
946
|
+
|
|
947
|
+
Args:
|
|
948
|
+
minYear (int): start year, like 2005
|
|
949
|
+
maxYear (int): end year, like 2005
|
|
950
|
+
minMD (str): start date, like "0125" is 01/25
|
|
951
|
+
maxMD (str): end date, like "0205" is 02/25
|
|
952
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
953
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
954
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
955
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
956
|
+
list: a,b,c;
|
|
957
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
958
|
+
order (str, optional): elements order, seperated by ',', like
|
|
959
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
960
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
961
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
962
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
963
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
964
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. Defaults to "54511".
|
|
965
|
+
|
|
966
|
+
Returns:
|
|
967
|
+
pandas data frame: observation records.
|
|
968
|
+
|
|
969
|
+
Examples:
|
|
970
|
+
obsdata = cmadaas_obs_by_period_and_id(2015, 2018, "0501", "0505", sta_ids="54511")
|
|
971
|
+
"""
|
|
972
|
+
|
|
973
|
+
# set retrieve parameters
|
|
974
|
+
params = {'dataCode': data_code,
|
|
975
|
+
'minYear': str(int(minYear)),
|
|
976
|
+
'maxYear': str(int(maxYear)),
|
|
977
|
+
'minMD': str(minMD),
|
|
978
|
+
'maxMD': str(maxMD),
|
|
979
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
980
|
+
'elements': elements,
|
|
981
|
+
'staIds': sta_ids}
|
|
982
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
983
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
984
|
+
|
|
985
|
+
# Interface, refer to
|
|
986
|
+
interface_id = "getSurfEleInHistoryBySamePeriodAndStaID"
|
|
987
|
+
|
|
988
|
+
# retrieve data contents
|
|
989
|
+
contents = get_rest_result(interface_id, params)
|
|
990
|
+
contents = _load_contents(contents)
|
|
991
|
+
if contents is None:
|
|
992
|
+
return None
|
|
993
|
+
|
|
994
|
+
# construct pandas DataFrame
|
|
995
|
+
data = pd.DataFrame(contents['DS'])
|
|
996
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
997
|
+
|
|
998
|
+
# return
|
|
999
|
+
return data
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def cmadaas_obs_in_admin_by_period(minYear, maxYear, minMD, maxMD, admin="110000", data_code="SURF_CHN_MUL_HOR_N",
|
|
1003
|
+
ranges=None, order=None, count=None, trans_type=True,
|
|
1004
|
+
elements="Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM"):
|
|
1005
|
+
"""
|
|
1006
|
+
Retrieve station records from CIMISS by same period in years and in provinces.
|
|
1007
|
+
|
|
1008
|
+
Args:
|
|
1009
|
+
minYear (int): start year, like 2005
|
|
1010
|
+
maxYear (int): end year, like 2005
|
|
1011
|
+
minMD (str): start date, like "0125" is 01/25
|
|
1012
|
+
maxMD (str): end date, like "0205" is 02/25
|
|
1013
|
+
admin (str, optional): administration(or province code), sperated by ",",
|
|
1014
|
+
like "110000" is Beijing, "440000" is Guangdong. Defaults to "110000".
|
|
1015
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MUL_HOR_N".
|
|
1016
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1017
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1018
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1019
|
+
list: a,b,c;
|
|
1020
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1021
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1022
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1023
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1024
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1025
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1026
|
+
Defaults to "Station_Id_C,Station_Id_d,lat,lon,Datetime,TEM".
|
|
1027
|
+
|
|
1028
|
+
Returns:
|
|
1029
|
+
pandas data frame: observation records.
|
|
1030
|
+
|
|
1031
|
+
Examples:
|
|
1032
|
+
obsdata = cmadaas_obs_in_admin_by_period(2015, 2018, "0501", "0505", admin="110000")
|
|
1033
|
+
"""
|
|
1034
|
+
|
|
1035
|
+
# set retrieve parameters
|
|
1036
|
+
params = {'dataCode': data_code,
|
|
1037
|
+
'minYear': str(int(minYear)),
|
|
1038
|
+
'maxYear': str(int(maxYear)),
|
|
1039
|
+
'minMD': str(minMD),
|
|
1040
|
+
'maxMD': str(maxMD),
|
|
1041
|
+
'adminCodes': admin,
|
|
1042
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
1043
|
+
'elements': elements}
|
|
1044
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1045
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1046
|
+
|
|
1047
|
+
# Interface, refer to
|
|
1048
|
+
interface_id = "getSurfEleInHistoryBySamePeriodAndRegion"
|
|
1049
|
+
|
|
1050
|
+
# retrieve data contents
|
|
1051
|
+
contents = get_rest_result(interface_id, params)
|
|
1052
|
+
contents = _load_contents(contents)
|
|
1053
|
+
if contents is None:
|
|
1054
|
+
return None
|
|
1055
|
+
|
|
1056
|
+
# construct pandas DataFrame
|
|
1057
|
+
data = pd.DataFrame(contents['DS'])
|
|
1058
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1059
|
+
|
|
1060
|
+
# return
|
|
1061
|
+
return data
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def cmadaas_obs_grid_by_time(time, limit=None, data_code="SURF_CMPA_FAST_5KM",
|
|
1065
|
+
fcst_ele="PRE", zoom=None, varname='data',
|
|
1066
|
+
units=None, scale_off=None,
|
|
1067
|
+
cache=True, cache_clear=True):
|
|
1068
|
+
"""
|
|
1069
|
+
Retrieve surface analysis grid products, like CMPAS-V2.1融合降水分析实时数据产品(NC).
|
|
1070
|
+
For SURF_CMPA_RT_NC, this function will retrieve the 0.01 resolution data and take long time.
|
|
1071
|
+
该函数主要用于降水实况分析数据检索, 若需要温度、风的实况分析, 使用cmadaas_analysis_by_time.
|
|
1072
|
+
|
|
1073
|
+
:param time: analysis time, datetime object or string like "20171008000000", format: YYYYMMDDHHMISS
|
|
1074
|
+
:param limit: [min_lat, min_lon, max_lat, max_lon]
|
|
1075
|
+
:param data_code: MUSIC data code, default is "SURF_CMPA_FRT_5KM"
|
|
1076
|
+
"SURF_CMPA_RT_NC": 实时产品滞后约50~60分钟,分两种空间分辨率: 1小时/0.05°、1小时/0.01°
|
|
1077
|
+
"SURF_CMPA_FRT_5KM_10MIN"(PRE_10MIN): CMPAS-V2.1融合降水分析实时数据10分钟产品
|
|
1078
|
+
"SURF_CMPA_FAST_5KM": CMPAS-V2.1融合降水分析快速数据小时产品, 时效在15分钟以内, 包含小时降水和3小时累计降水两种要素
|
|
1079
|
+
"SURF_CMPA_FRT_5KM": CMPAS-V2.1融合降水分析实时数据小时产品(GRIB, 5km)
|
|
1080
|
+
"SURF_CMPA_FAST_5KM_DAY": CMPAS-V2.1融合降水分析快速数据日产品(GRIB, 5km)
|
|
1081
|
+
"SURF_CMPA_FRT_5KM_DAY": CMPAS-V2.1融合降水分析实时数据日产品(GRIB, 5km)
|
|
1082
|
+
:param fcst_ele: elements
|
|
1083
|
+
:param zoom: the zoom out integer > 1, like 2. 像元缩小倍数, 数值大于1, 1的整数倍.
|
|
1084
|
+
:param varname: set variable name, default is 'data'
|
|
1085
|
+
:param units: forecast element's units, defaults to retrieved units.
|
|
1086
|
+
:param scale_off: [scale, offset], return values = values*scale + offset.
|
|
1087
|
+
:param cache: cache retrieved data to local directory, default is True.
|
|
1088
|
+
:return: data, xarray type
|
|
1089
|
+
|
|
1090
|
+
:Example:
|
|
1091
|
+
>>> time_str = "20171106120000"
|
|
1092
|
+
>>> data_code = "SURF_CMPA_FAST_5KM"
|
|
1093
|
+
>>> data = cmadaas_obs_grid_by_time(time_str, data_code=data_code, fcst_ele="PRE")
|
|
1094
|
+
"""
|
|
1095
|
+
# check time_str type
|
|
1096
|
+
if isinstance(time, datetime):
|
|
1097
|
+
time_str = time.strftime("%Y%m%d%H%M%S")
|
|
1098
|
+
elif isinstance(time, str):
|
|
1099
|
+
time_str = time
|
|
1100
|
+
else:
|
|
1101
|
+
warnings.warn('The type of time is not supportted')
|
|
1102
|
+
return None
|
|
1103
|
+
|
|
1104
|
+
# retrieve data from cached file
|
|
1105
|
+
if cache:
|
|
1106
|
+
directory = os.path.join(data_code, fcst_ele)
|
|
1107
|
+
filename = time_str
|
|
1108
|
+
if limit is not None:
|
|
1109
|
+
filename = filename + '.' + str(limit).replace(" ","")
|
|
1110
|
+
cache_file = CONFIG.get_cache_file(directory, filename, name="CMADaaS", cache_clear=cache_clear)
|
|
1111
|
+
if cache_file.is_file():
|
|
1112
|
+
with open(cache_file, 'rb') as f:
|
|
1113
|
+
data = pickle.load(f)
|
|
1114
|
+
return data
|
|
1115
|
+
|
|
1116
|
+
# set retrieve parameters
|
|
1117
|
+
if limit is None:
|
|
1118
|
+
params = {'dataCode': data_code,
|
|
1119
|
+
'time': time_str,
|
|
1120
|
+
'fcstEle': fcst_ele}
|
|
1121
|
+
if zoom is not None: params['zoomOut'] = str(zoom)
|
|
1122
|
+
interface_id = "getSurfEleGridByTime"
|
|
1123
|
+
else:
|
|
1124
|
+
params = {'dataCode': data_code,
|
|
1125
|
+
'time': time_str,
|
|
1126
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
1127
|
+
"minLon": '{:.10f}'.format(limit[1]),
|
|
1128
|
+
"maxLat": '{:.10f}'.format(limit[2]),
|
|
1129
|
+
"maxLon": '{:.10f}'.format(limit[3]),
|
|
1130
|
+
'fcstEle': fcst_ele}
|
|
1131
|
+
if zoom is not None: params['zoomOut'] = str(zoom)
|
|
1132
|
+
interface_id = "getSurfEleGridInRectByTime"
|
|
1133
|
+
|
|
1134
|
+
# retrieve data contents
|
|
1135
|
+
contents = get_rest_result(interface_id, params)
|
|
1136
|
+
contents = _load_contents(contents)
|
|
1137
|
+
if contents is None:
|
|
1138
|
+
return None
|
|
1139
|
+
|
|
1140
|
+
# get time information
|
|
1141
|
+
time = datetime.strptime(time_str, '%Y%m%d%H%M%S')
|
|
1142
|
+
time = np.array([time], dtype="datetime64[ms]")
|
|
1143
|
+
|
|
1144
|
+
# extract coordinates and data
|
|
1145
|
+
start_lat = float(contents['startLat'])
|
|
1146
|
+
start_lon = float(contents['startLon'])
|
|
1147
|
+
nlon = int(contents['lonCount'])
|
|
1148
|
+
nlat = int(contents['latCount'])
|
|
1149
|
+
dlon = float(contents['lonStep'])
|
|
1150
|
+
dlat = float(contents['latStep'])
|
|
1151
|
+
lon = start_lon + np.arange(nlon) * dlon
|
|
1152
|
+
lat = start_lat + np.arange(nlat) * dlat
|
|
1153
|
+
name = contents['fieldNames']
|
|
1154
|
+
if units is None:
|
|
1155
|
+
units = contents['fieldUnits']
|
|
1156
|
+
|
|
1157
|
+
# define coordinates and variables
|
|
1158
|
+
time_coord = ('time', time)
|
|
1159
|
+
lon_coord = ('lon', lon, {
|
|
1160
|
+
'long_name':'longitude', 'units':'degrees_east',
|
|
1161
|
+
'_CoordinateAxisType':'Lon', 'axis': "X"})
|
|
1162
|
+
lat_coord = ('lat', lat, {
|
|
1163
|
+
'long_name':'latitude', 'units':'degrees_north',
|
|
1164
|
+
'_CoordinateAxisType':'Lat', 'axis': "Y"})
|
|
1165
|
+
varattrs = {'long_name': name, 'units': units}
|
|
1166
|
+
|
|
1167
|
+
# construct xarray
|
|
1168
|
+
data = np.array(contents['DS'], dtype=np.float32)
|
|
1169
|
+
data[data == 9999.] = np.nan
|
|
1170
|
+
if scale_off is not None:
|
|
1171
|
+
data = data * scale_off[0] + scale_off[1]
|
|
1172
|
+
data = data[np.newaxis, ...]
|
|
1173
|
+
data = xr.Dataset({
|
|
1174
|
+
varname:(['time', 'lat', 'lon'], data, varattrs)},
|
|
1175
|
+
coords={'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
1176
|
+
|
|
1177
|
+
# add global attributes
|
|
1178
|
+
data.attrs['Conventions'] = "CF-1.6"
|
|
1179
|
+
data.attrs['Origin'] = 'CIMISS Server by MUSIC API'
|
|
1180
|
+
|
|
1181
|
+
# cache data
|
|
1182
|
+
if cache:
|
|
1183
|
+
with open(cache_file, 'wb') as f:
|
|
1184
|
+
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
|
1185
|
+
|
|
1186
|
+
# return data
|
|
1187
|
+
return data
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def cmadaas_obs_grid_by_times(times_str, pbar=True, allExists=True, **kargs):
|
|
1191
|
+
"""
|
|
1192
|
+
Retrieve multiple surface analysis grid products, like CMPAS-V2.1融合降水分析实时数据产品(NC).
|
|
1193
|
+
|
|
1194
|
+
:param times_str: analysis time string list, like ["20200208000000", "20200209000000"], format: YYYYMMDDHHMISS
|
|
1195
|
+
:param allExists (boolean): all files should exist, or return None.
|
|
1196
|
+
:param pbar (boolean): Show progress bar, default to False.
|
|
1197
|
+
:param **kargs: key arguments passed to cmadaas_model_by_time function.
|
|
1198
|
+
:return: data, xarray type
|
|
1199
|
+
|
|
1200
|
+
Examples:
|
|
1201
|
+
>>> times = ["20200208000000", "20200209000000"]
|
|
1202
|
+
>>> data_code = "SURF_CMPA_FRT_5KM"
|
|
1203
|
+
>>> data = cmadaas_obs_grid_by_times(times, data_code=data_code, fcst_ele="PRE")
|
|
1204
|
+
"""
|
|
1205
|
+
|
|
1206
|
+
dataset = []
|
|
1207
|
+
tqdm_times_str = tqdm(times_str, desc=kargs['data_code'] + ": ") if pbar else times_str
|
|
1208
|
+
for time_str in tqdm_times_str:
|
|
1209
|
+
data = cmadaas_obs_grid_by_time(time_str, **kargs)
|
|
1210
|
+
if data:
|
|
1211
|
+
dataset.append(data)
|
|
1212
|
+
else:
|
|
1213
|
+
if allExists:
|
|
1214
|
+
warnings.warn("{} doese not exists.".format(kargs['data_code']+'/'+time_str))
|
|
1215
|
+
return None
|
|
1216
|
+
|
|
1217
|
+
return xr.concat(dataset, dim='time')
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def cmadaas_obs_by_days_of_year(data_code="SURF_CHN_DAY_MMUT_19812010",
|
|
1221
|
+
days_of_year=None, elements="Station_Name,Station_Id_C,Day_Seq,TEM_Avg",
|
|
1222
|
+
ranges=None, count=None, trans_type=True, sta_ids="54511"):
|
|
1223
|
+
"""
|
|
1224
|
+
按日序检索地面累年日值数据要素, 为累年日值专用.
|
|
1225
|
+
|
|
1226
|
+
Args:
|
|
1227
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_DAY_MMUT_19812010".
|
|
1228
|
+
mons_of_year (str, optional): months of year. Defaults to "1,2,3,4,5,6,7,8,9,10,11,12".
|
|
1229
|
+
elements (str, optional): . Defaults to "Station_Name,Station_Id_C,Day_Seq,TEM_Avg".
|
|
1230
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1231
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1232
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b.
|
|
1233
|
+
list: a,b,c;
|
|
1234
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.
|
|
1235
|
+
Defaults to None.
|
|
1236
|
+
count (str, optional): the number of maximum returned records. Defaults to None.
|
|
1237
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric.
|
|
1238
|
+
Defaults to True.
|
|
1239
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. If None, return all stations. Defaults to None.
|
|
1240
|
+
|
|
1241
|
+
Returns:
|
|
1242
|
+
pandas data frame: observation records.
|
|
1243
|
+
|
|
1244
|
+
Examples:
|
|
1245
|
+
>>> data = cmadaas_obs_by_days_of_year(sta_ids="54511")
|
|
1246
|
+
"""
|
|
1247
|
+
|
|
1248
|
+
# check days_of_year
|
|
1249
|
+
if days_of_year is None:
|
|
1250
|
+
days_of_year = ",".join((np.arange(365)+1).astype("str"))
|
|
1251
|
+
|
|
1252
|
+
# set retrieve parameters
|
|
1253
|
+
params = {'dataCode': data_code,
|
|
1254
|
+
'elements': elements,
|
|
1255
|
+
'daysOfYear': days_of_year}
|
|
1256
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1257
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1258
|
+
|
|
1259
|
+
# interface id
|
|
1260
|
+
if sta_ids is None:
|
|
1261
|
+
interface_id = "getSurfMDayEleByDaysOfYear"
|
|
1262
|
+
else:
|
|
1263
|
+
params['staIds'] = sta_ids
|
|
1264
|
+
interface_id = "getSurfMDayEleByDaysOfYearAndStaID"
|
|
1265
|
+
|
|
1266
|
+
# retrieve data contents
|
|
1267
|
+
contents = get_rest_result(interface_id, params)
|
|
1268
|
+
contents = _load_contents(contents)
|
|
1269
|
+
if contents is None:
|
|
1270
|
+
return None
|
|
1271
|
+
|
|
1272
|
+
# construct pandas DataFrame
|
|
1273
|
+
data = pd.DataFrame(contents['DS'])
|
|
1274
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1275
|
+
|
|
1276
|
+
# return
|
|
1277
|
+
return data
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def cmadaas_obs_by_pens_of_year(data_code="SURF_CHN_PEN_MMUT_19812010",
|
|
1281
|
+
pens_of_year=None, elements="Station_Name,Station_Id_C,Pen_Seq,TEM_Avg",
|
|
1282
|
+
ranges=None, count=None, trans_type=True, sta_ids="54511"):
|
|
1283
|
+
"""
|
|
1284
|
+
按候序检索地面累年候值数据要素, 为累年候值专用.
|
|
1285
|
+
|
|
1286
|
+
Args:
|
|
1287
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_PEN_MMUT_19812010".
|
|
1288
|
+
mons_of_year (str, optional): months of year. Defaults to "1,2,3,4,5,6,7,8,9,10,11,12".
|
|
1289
|
+
elements (str, optional): . Defaults to "Station_Name,Station_Id_C,Pen_Seq,TEM_Avg".
|
|
1290
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1291
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1292
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b.
|
|
1293
|
+
list: a,b,c;
|
|
1294
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.
|
|
1295
|
+
Defaults to None.
|
|
1296
|
+
count (str, optional): the number of maximum returned records. Defaults to None.
|
|
1297
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric.
|
|
1298
|
+
Defaults to True.
|
|
1299
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. If None, return all stations. Defaults to None.
|
|
1300
|
+
|
|
1301
|
+
Returns:
|
|
1302
|
+
pandas data frame: observation records.
|
|
1303
|
+
|
|
1304
|
+
Examples:
|
|
1305
|
+
>>> data = cmadaas_obs_by_pens_of_year(sta_ids="54511")
|
|
1306
|
+
"""
|
|
1307
|
+
|
|
1308
|
+
# check days_of_year
|
|
1309
|
+
if pens_of_year is None:
|
|
1310
|
+
pens_of_year = ",".join((np.arange(72)+1).astype("str"))
|
|
1311
|
+
|
|
1312
|
+
# set retrieve parameters
|
|
1313
|
+
params = {'dataCode': data_code,
|
|
1314
|
+
'elements': elements,
|
|
1315
|
+
'pensOfYear': pens_of_year}
|
|
1316
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1317
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1318
|
+
|
|
1319
|
+
# interface id
|
|
1320
|
+
if sta_ids is None:
|
|
1321
|
+
interface_id = "getSurfMPenEleByPensOfYear"
|
|
1322
|
+
else:
|
|
1323
|
+
params['staIds'] = sta_ids
|
|
1324
|
+
interface_id = "getSurfMPenEleByPensOfYearAndStaID"
|
|
1325
|
+
|
|
1326
|
+
# retrieve data contents
|
|
1327
|
+
contents = get_rest_result(interface_id, params)
|
|
1328
|
+
contents = _load_contents(contents)
|
|
1329
|
+
if contents is None:
|
|
1330
|
+
return None
|
|
1331
|
+
|
|
1332
|
+
# construct pandas DataFrame
|
|
1333
|
+
data = pd.DataFrame(contents['DS'])
|
|
1334
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1335
|
+
|
|
1336
|
+
# return
|
|
1337
|
+
return data
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
def cmadaas_obs_by_tens_of_year(data_code="SURF_CHN_TEN_MMUT_19812010",
|
|
1341
|
+
tens_of_year=None, elements="Station_Name,Station_Id_C,Ten_Seq,TEM_Avg",
|
|
1342
|
+
ranges=None, count=None, trans_type=True, sta_ids="54511"):
|
|
1343
|
+
"""
|
|
1344
|
+
按旬序检索地面累年旬值数据要素, 为累年旬值专用.
|
|
1345
|
+
|
|
1346
|
+
Args:
|
|
1347
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_TEN_MMUT_19812010".
|
|
1348
|
+
mons_of_year (str, optional): months of year. Defaults to "1,2,3,4,5,6,7,8,9,10,11,12".
|
|
1349
|
+
elements (str, optional): . Defaults to "Station_Name,Station_Id_C,Ten_Seq,TEM_Avg".
|
|
1350
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1351
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1352
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b.
|
|
1353
|
+
list: a,b,c;
|
|
1354
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.
|
|
1355
|
+
Defaults to None.
|
|
1356
|
+
count (str, optional): the number of maximum returned records. Defaults to None.
|
|
1357
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric.
|
|
1358
|
+
Defaults to True.
|
|
1359
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. If None, return all stations. Defaults to None.
|
|
1360
|
+
|
|
1361
|
+
Returns:
|
|
1362
|
+
pandas data frame: observation records.
|
|
1363
|
+
|
|
1364
|
+
Examples:
|
|
1365
|
+
>>> data = cmadaas_obs_by_tens_of_year(sta_ids="54511")
|
|
1366
|
+
"""
|
|
1367
|
+
|
|
1368
|
+
# check days_of_year
|
|
1369
|
+
if tens_of_year is None:
|
|
1370
|
+
tens_of_year = ",".join((np.arange(36)+1).astype("str"))
|
|
1371
|
+
|
|
1372
|
+
# set retrieve parameters
|
|
1373
|
+
params = {'dataCode': data_code,
|
|
1374
|
+
'elements': elements,
|
|
1375
|
+
'tensOfYear': tens_of_year}
|
|
1376
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1377
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1378
|
+
|
|
1379
|
+
# interface id
|
|
1380
|
+
if sta_ids is None:
|
|
1381
|
+
interface_id = "getSurfMTenEleByTensOfYear"
|
|
1382
|
+
else:
|
|
1383
|
+
params['staIds'] = sta_ids
|
|
1384
|
+
interface_id = "getSurfMTenEleByTensOfYearAndStaID"
|
|
1385
|
+
|
|
1386
|
+
# retrieve data contents
|
|
1387
|
+
contents = get_rest_result(interface_id, params)
|
|
1388
|
+
contents = _load_contents(contents)
|
|
1389
|
+
if contents is None:
|
|
1390
|
+
return None
|
|
1391
|
+
|
|
1392
|
+
# construct pandas DataFrame
|
|
1393
|
+
data = pd.DataFrame(contents['DS'])
|
|
1394
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1395
|
+
|
|
1396
|
+
# return
|
|
1397
|
+
return data
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def cmadaas_obs_by_months_of_year(data_code="SURF_CHN_MON_MMUT_19812010",
|
|
1401
|
+
mons_of_year="1,2,3,4,5,6,7,8,9,10,11,12",
|
|
1402
|
+
elements="Station_Name,Station_Id_C,Mon_Seq,RHU_Avg",
|
|
1403
|
+
ranges=None, count=None, trans_type=True, sta_ids=None):
|
|
1404
|
+
"""
|
|
1405
|
+
按月序检索地面累年月值数据要素, 为累年月值专用.
|
|
1406
|
+
|
|
1407
|
+
Args:
|
|
1408
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_MON_MMUT_19812010".
|
|
1409
|
+
mons_of_year (str, optional): months of year. Defaults to "1,2,3,4,5,6,7,8,9,10,11,12".
|
|
1410
|
+
elements (str, optional): . Defaults to "Station_Name,Station_Id_C,Mon_Seq,RHU_Avg".
|
|
1411
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1412
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1413
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b.
|
|
1414
|
+
list: a,b,c;
|
|
1415
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.
|
|
1416
|
+
Defaults to None.
|
|
1417
|
+
count (str, optional): the number of maximum returned records. Defaults to None.
|
|
1418
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric.
|
|
1419
|
+
Defaults to True.
|
|
1420
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. If None, return all stations. Defaults to None.
|
|
1421
|
+
|
|
1422
|
+
Returns:
|
|
1423
|
+
pandas data frame: observation records.
|
|
1424
|
+
|
|
1425
|
+
Examples:
|
|
1426
|
+
>>> data = cmadaas_obs_by_months_of_year()
|
|
1427
|
+
"""
|
|
1428
|
+
|
|
1429
|
+
# set retrieve parameters
|
|
1430
|
+
params = {'dataCode': data_code,
|
|
1431
|
+
'elements': elements,
|
|
1432
|
+
'monsOfYear': mons_of_year}
|
|
1433
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1434
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1435
|
+
|
|
1436
|
+
# interface id
|
|
1437
|
+
if sta_ids is None:
|
|
1438
|
+
interface_id = "getSurfMMonEleByMonthsOfYear"
|
|
1439
|
+
else:
|
|
1440
|
+
params['staIds'] = sta_ids
|
|
1441
|
+
interface_id = "getSurfMMonEleByMonthsOfYearAndStaID"
|
|
1442
|
+
|
|
1443
|
+
# retrieve data contents
|
|
1444
|
+
contents = get_rest_result(interface_id, params)
|
|
1445
|
+
contents = _load_contents(contents)
|
|
1446
|
+
if contents is None:
|
|
1447
|
+
return None
|
|
1448
|
+
|
|
1449
|
+
# construct pandas DataFrame
|
|
1450
|
+
data = pd.DataFrame(contents['DS'])
|
|
1451
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1452
|
+
|
|
1453
|
+
# return
|
|
1454
|
+
return data
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def cmadaas_obs_by_years(data_code="SURF_CHN_YER_MMUT_19812010",
|
|
1458
|
+
elements="Station_Name,Station_Id_C,RHU_Avg",
|
|
1459
|
+
ranges=None, count=None, trans_type=True, sta_ids=None):
|
|
1460
|
+
"""
|
|
1461
|
+
检索地面累年年值数据要素, 为累年年值专用.
|
|
1462
|
+
|
|
1463
|
+
Args:
|
|
1464
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CHN_YER_MMUT_19812010".
|
|
1465
|
+
elements (str, optional): . Defaults to "Station_Name,Station_Id_C,RHU_Avg".
|
|
1466
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1467
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1468
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b.
|
|
1469
|
+
list: a,b,c;
|
|
1470
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.
|
|
1471
|
+
Defaults to None.
|
|
1472
|
+
count (str, optional): the number of maximum returned records. Defaults to None.
|
|
1473
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric.
|
|
1474
|
+
Defaults to True.
|
|
1475
|
+
sta_ids (str, optional): station ids, 'xxxxx,xxxxx,...'. If None, return all stations. Defaults to None.
|
|
1476
|
+
|
|
1477
|
+
Returns:
|
|
1478
|
+
pandas data frame: observation records.
|
|
1479
|
+
|
|
1480
|
+
Examples:
|
|
1481
|
+
>>> data = cmadaas_obs_by_years()
|
|
1482
|
+
"""
|
|
1483
|
+
|
|
1484
|
+
# set retrieve parameters
|
|
1485
|
+
params = {'dataCode': data_code,
|
|
1486
|
+
'elements': elements}
|
|
1487
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1488
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1489
|
+
|
|
1490
|
+
# interface id
|
|
1491
|
+
if sta_ids is None:
|
|
1492
|
+
interface_id = "getSurfMYerEle"
|
|
1493
|
+
else:
|
|
1494
|
+
params['staIds'] = sta_ids
|
|
1495
|
+
interface_id = "getSurfMYerEleByStaID"
|
|
1496
|
+
|
|
1497
|
+
# retrieve data contents
|
|
1498
|
+
contents = get_rest_result(interface_id, params)
|
|
1499
|
+
contents = _load_contents(contents)
|
|
1500
|
+
if contents is None:
|
|
1501
|
+
return None
|
|
1502
|
+
|
|
1503
|
+
# construct pandas DataFrame
|
|
1504
|
+
data = pd.DataFrame(contents['DS'])
|
|
1505
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1506
|
+
|
|
1507
|
+
# return
|
|
1508
|
+
return data
|
|
1509
|
+
|
|
1510
|
+
|
|
1511
|
+
def cmadaas_get_sounding_latest_time(data_code="UPAR_CHN_MUL_FTM", latestTime=24):
|
|
1512
|
+
"""
|
|
1513
|
+
Get the latest time of the soundings.
|
|
1514
|
+
|
|
1515
|
+
Args:
|
|
1516
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1517
|
+
latestTime (int, optional): latestTime > 0, like 2 is return
|
|
1518
|
+
the latest time in 2 hours. Defaults to 12.
|
|
1519
|
+
Returns:
|
|
1520
|
+
datetime object, the latest time
|
|
1521
|
+
|
|
1522
|
+
Examples:
|
|
1523
|
+
>>> print(cmadaas_get_sounding_latest_time())
|
|
1524
|
+
2020-03-12 03:00:00
|
|
1525
|
+
"""
|
|
1526
|
+
|
|
1527
|
+
# set retrieve parameters
|
|
1528
|
+
params = {'dataCode': data_code,
|
|
1529
|
+
'latestTime': str(latestTime+8)} # 由于服务器是世界时, 因此需要+8才能与当前时间比较.
|
|
1530
|
+
|
|
1531
|
+
# interface id
|
|
1532
|
+
interface_id = "getUparLatestTime"
|
|
1533
|
+
|
|
1534
|
+
# retrieve data contents
|
|
1535
|
+
contents = get_rest_result(interface_id, params)
|
|
1536
|
+
contents = _load_contents(contents)
|
|
1537
|
+
|
|
1538
|
+
# construct pandas DataFrame
|
|
1539
|
+
data = pd.DataFrame(contents['DS'])
|
|
1540
|
+
time = pd.to_datetime(data['Datetime'], format="%Y%m%d%H%M%S")
|
|
1541
|
+
|
|
1542
|
+
return time[0]
|
|
1543
|
+
|
|
1544
|
+
|
|
1545
|
+
def cmadaas_sounding_by_time(times, data_code="UPAR_CHN_MUL_FTM", ranges=None,
|
|
1546
|
+
order=None, count=None, distinct=False, trans_type=True,
|
|
1547
|
+
elements="Station_Name,Station_Id_C,Lat,Lon,Alti,Year,Mon,Day,Hour,Min,Second,PRS_HWC,GPH,TEM,DPT,DTD,WIN_D"):
|
|
1548
|
+
"""
|
|
1549
|
+
Retrieve sounding records from CMADaaS by times.
|
|
1550
|
+
|
|
1551
|
+
Args:
|
|
1552
|
+
times (str): times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
1553
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1554
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1555
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1556
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1557
|
+
list: a,b,c;
|
|
1558
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1559
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1560
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1561
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1562
|
+
distinct (bool, optional): return unique records. Defaults to False.
|
|
1563
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1564
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1565
|
+
|
|
1566
|
+
Returns:
|
|
1567
|
+
pandas data frame: sounding records.
|
|
1568
|
+
|
|
1569
|
+
Examples:
|
|
1570
|
+
uparData = cmadaas_sounding_by_time('20220531000000')
|
|
1571
|
+
"""
|
|
1572
|
+
|
|
1573
|
+
# set retrieve parameters
|
|
1574
|
+
params = {'dataCode': data_code,
|
|
1575
|
+
'times': times,
|
|
1576
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
1577
|
+
'elements': elements}
|
|
1578
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1579
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1580
|
+
if distinct: params['distinct'] = "true"
|
|
1581
|
+
|
|
1582
|
+
# Interface, refer to
|
|
1583
|
+
interface_id = "getUparEleByTime"
|
|
1584
|
+
|
|
1585
|
+
# retrieve data contents
|
|
1586
|
+
contents = get_rest_result(interface_id, params)
|
|
1587
|
+
contents = _load_contents(contents)
|
|
1588
|
+
if contents is None:
|
|
1589
|
+
return None
|
|
1590
|
+
|
|
1591
|
+
# construct pandas DataFrame
|
|
1592
|
+
data = pd.DataFrame(contents['DS'])
|
|
1593
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1594
|
+
|
|
1595
|
+
# return
|
|
1596
|
+
return data
|
|
1597
|
+
|
|
1598
|
+
|
|
1599
|
+
def cmadaas_sounding_by_time_and_press(
|
|
1600
|
+
times, data_code="UPAR_CHN_MUL_FTM", press=500, ranges=None,
|
|
1601
|
+
order=None, count=None, distinct=False, trans_type=True,
|
|
1602
|
+
elements="Station_Name,Station_Id_C,Lat,Lon,Alti,Year,Mon,Day,Hour,Min,Second,PRS_HWC,GPH,TEM,DPT,DTD,WIN_D"):
|
|
1603
|
+
"""
|
|
1604
|
+
Retrieve sounding records from CMADaaS by times.
|
|
1605
|
+
|
|
1606
|
+
Args:
|
|
1607
|
+
times (str): times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
1608
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1609
|
+
press (str, optional): pressure level. Defaults to 500.
|
|
1610
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1611
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1612
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1613
|
+
list: a,b,c;
|
|
1614
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1615
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1616
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1617
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1618
|
+
distinct (bool, optional): return unique records. Defaults to False.
|
|
1619
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1620
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1621
|
+
|
|
1622
|
+
Returns:
|
|
1623
|
+
pandas data frame: sounding records.
|
|
1624
|
+
|
|
1625
|
+
Examples:
|
|
1626
|
+
uparData = cmadaas_sounding_by_time_and_press('20220531000000', press=500)
|
|
1627
|
+
"""
|
|
1628
|
+
|
|
1629
|
+
# set retrieve parameters
|
|
1630
|
+
params = {'dataCode': data_code,
|
|
1631
|
+
'times': times,
|
|
1632
|
+
'pLayers': str(press).strip(),
|
|
1633
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
1634
|
+
'elements': elements}
|
|
1635
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1636
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1637
|
+
if distinct: params['distinct'] = "true"
|
|
1638
|
+
|
|
1639
|
+
# Interface, refer to
|
|
1640
|
+
interface_id = "getUparEleByTimeAndPress"
|
|
1641
|
+
|
|
1642
|
+
# retrieve data contents
|
|
1643
|
+
contents = get_rest_result(interface_id, params)
|
|
1644
|
+
contents = _load_contents(contents)
|
|
1645
|
+
if contents is None:
|
|
1646
|
+
return None
|
|
1647
|
+
|
|
1648
|
+
# construct pandas DataFrame
|
|
1649
|
+
data = pd.DataFrame(contents['DS'])
|
|
1650
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1651
|
+
|
|
1652
|
+
# return
|
|
1653
|
+
return data
|
|
1654
|
+
|
|
1655
|
+
|
|
1656
|
+
def cmadaas_sounding_by_time_and_height(
|
|
1657
|
+
times, data_code="UPAR_CHN_MUL_FTM", height=5000, ranges=None,
|
|
1658
|
+
order=None, count=None, distinct=False, trans_type=True,
|
|
1659
|
+
elements="Station_Name,Station_Id_C,Lat,Lon,Alti,Year,Mon,Day,Hour,Min,Second,PRS_HWC,GPH,TEM,DPT,DTD,WIN_D"):
|
|
1660
|
+
"""
|
|
1661
|
+
Retrieve sounding records from CMADaaS by times.
|
|
1662
|
+
|
|
1663
|
+
Args:
|
|
1664
|
+
times (str): times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
1665
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1666
|
+
height (str, optional): height level. Defaults to 5000.
|
|
1667
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1668
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1669
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1670
|
+
list: a,b,c;
|
|
1671
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1672
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1673
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1674
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1675
|
+
distinct (bool, optional): return unique records. Defaults to False.
|
|
1676
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1677
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1678
|
+
|
|
1679
|
+
Returns:
|
|
1680
|
+
pandas data frame: sounding records.
|
|
1681
|
+
|
|
1682
|
+
Examples:
|
|
1683
|
+
uparData = cmadaas_sounding_by_time_and_height('20220531000000', height=5000)
|
|
1684
|
+
"""
|
|
1685
|
+
|
|
1686
|
+
# set retrieve parameters
|
|
1687
|
+
params = {'dataCode': data_code,
|
|
1688
|
+
'times': times,
|
|
1689
|
+
'hLayers': str(height).strip(),
|
|
1690
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
1691
|
+
'elements': elements}
|
|
1692
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1693
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1694
|
+
if distinct: params['distinct'] = "true"
|
|
1695
|
+
|
|
1696
|
+
# Interface, refer to
|
|
1697
|
+
interface_id = "getUparEleByTimeAndHeight"
|
|
1698
|
+
|
|
1699
|
+
# retrieve data contents
|
|
1700
|
+
contents = get_rest_result(interface_id, params)
|
|
1701
|
+
contents = _load_contents(contents)
|
|
1702
|
+
if contents is None:
|
|
1703
|
+
return None
|
|
1704
|
+
|
|
1705
|
+
# construct pandas DataFrame
|
|
1706
|
+
data = pd.DataFrame(contents['DS'])
|
|
1707
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1708
|
+
|
|
1709
|
+
# return
|
|
1710
|
+
return data
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def cmadaas_sounding_in_rect_by_time(
|
|
1714
|
+
times, limit=[3.51, 73.33, 53.33, 135.05], data_code="UPAR_CHN_MUL_FTM",
|
|
1715
|
+
ranges=None, order=None, count=None, trans_type=True,
|
|
1716
|
+
elements="Station_Name,Station_Id_C,Lat,Lon,Alti,Year,Mon,Day,Hour,Min,Second,PRS_HWC,GPH,TEM,DPT,DTD,WIN_D,WIN_S"):
|
|
1717
|
+
"""
|
|
1718
|
+
Retrieve sounding records from CIMISS in region by times.
|
|
1719
|
+
|
|
1720
|
+
Args:
|
|
1721
|
+
times (str): time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
1722
|
+
limit (list, optional): map limits, [min_lat, min_lon, max_lat, max_lon]
|
|
1723
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1724
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1725
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1726
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1727
|
+
list: a,b,c;
|
|
1728
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1729
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1730
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1731
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1732
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1733
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1734
|
+
|
|
1735
|
+
Returns:
|
|
1736
|
+
pandas data frame: sounding records.
|
|
1737
|
+
|
|
1738
|
+
Examples:
|
|
1739
|
+
>>> data = cmadaas_sounding_in_rect_by_time('20220531000000', limit=[3.51, 73.33, 53.33, 135.05])
|
|
1740
|
+
"""
|
|
1741
|
+
|
|
1742
|
+
# set retrieve parameters
|
|
1743
|
+
params = {'dataCode': data_code,
|
|
1744
|
+
'elements': elements,
|
|
1745
|
+
'times': times,
|
|
1746
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
1747
|
+
'minLon': '{:.10f}'.format(limit[1]),
|
|
1748
|
+
'maxLat': '{:.10f}'.format(limit[2]),
|
|
1749
|
+
'maxLon': '{:.10f}'.format(limit[3]),
|
|
1750
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
1751
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1752
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1753
|
+
|
|
1754
|
+
# interface id
|
|
1755
|
+
interface_id = "getUparEleInRectByTime"
|
|
1756
|
+
|
|
1757
|
+
# retrieve data contents
|
|
1758
|
+
contents = get_rest_result(interface_id, params)
|
|
1759
|
+
contents = _load_contents(contents)
|
|
1760
|
+
if contents is None:
|
|
1761
|
+
return None
|
|
1762
|
+
|
|
1763
|
+
# construct pandas DataFrame
|
|
1764
|
+
data = pd.DataFrame(contents['DS'])
|
|
1765
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1766
|
+
|
|
1767
|
+
# return
|
|
1768
|
+
return data
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
def cmadaas_sounding_by_time_and_id(
|
|
1772
|
+
times, data_code="UPAR_CHN_MUL_FTM", sta_ids='54511', ranges=None,
|
|
1773
|
+
order=None, count=None, distinct=False, trans_type=True,
|
|
1774
|
+
elements="Station_Name,Station_Id_C,Lat,Lon,Alti,Year,Mon,Day,Hour,Min,Second,PRS_HWC,GPH,TEM,DPT,DTD,WIN_D"):
|
|
1775
|
+
"""
|
|
1776
|
+
Retrieve sounding records from CMADaaS by times.
|
|
1777
|
+
|
|
1778
|
+
Args:
|
|
1779
|
+
times (str): times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
|
|
1780
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1781
|
+
sta_ids (str, optional): station id, 多个以逗号(,)分隔,区域站以字母开头. Defaults to 54511.
|
|
1782
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1783
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1784
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1785
|
+
list: a,b,c;
|
|
1786
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1787
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1788
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1789
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1790
|
+
distinct (bool, optional): return unique records. Defaults to False.
|
|
1791
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1792
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1793
|
+
|
|
1794
|
+
Returns:
|
|
1795
|
+
pandas data frame: sounding records.
|
|
1796
|
+
|
|
1797
|
+
Examples:
|
|
1798
|
+
uparData = cmadaas_sounding_by_time_and_id('20220531000000', sta_ids='54511')
|
|
1799
|
+
"""
|
|
1800
|
+
|
|
1801
|
+
# set retrieve parameters
|
|
1802
|
+
params = {'dataCode': data_code,
|
|
1803
|
+
'times': times,
|
|
1804
|
+
'staIds': str(sta_ids).strip(),
|
|
1805
|
+
'orderby': order if order is not None else "Datetime:ASC",
|
|
1806
|
+
'elements': elements}
|
|
1807
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1808
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1809
|
+
if distinct: params['distinct'] = "true"
|
|
1810
|
+
|
|
1811
|
+
# Interface, refer to
|
|
1812
|
+
interface_id = "getUparEleByTimeAndStaID"
|
|
1813
|
+
|
|
1814
|
+
# retrieve data contents
|
|
1815
|
+
contents = get_rest_result(interface_id, params)
|
|
1816
|
+
contents = _load_contents(contents)
|
|
1817
|
+
if contents is None:
|
|
1818
|
+
return None
|
|
1819
|
+
|
|
1820
|
+
# construct pandas DataFrame
|
|
1821
|
+
data = pd.DataFrame(contents['DS'])
|
|
1822
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1823
|
+
|
|
1824
|
+
# return
|
|
1825
|
+
return data
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
def cmadaas_sounding_in_rect_by_time_range(
|
|
1829
|
+
time_range, limit=[3.51, 73.33, 53.33, 135.05], data_code="UPAR_CHN_MUL_FTM",
|
|
1830
|
+
ranges=None, order=None, count=None, trans_type=True,
|
|
1831
|
+
elements="Station_Name,Station_Id_C,Lat,Lon,Alti,Year,Mon,Day,Hour,Min,Second,PRS_HWC,GPH,TEM,DPT,DTD,WIN_D,WIN_S"):
|
|
1832
|
+
"""
|
|
1833
|
+
Retrieve sounding records from CIMISS in region by times.
|
|
1834
|
+
|
|
1835
|
+
Args:
|
|
1836
|
+
time_range (str): time for retrieve, "[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
|
|
1837
|
+
limit (list, optional): map limits, [min_lat, min_lon, max_lat, max_lon]
|
|
1838
|
+
data_code (str, optional): dataset code. Defaults to "UPAR_CHN_MUL_FTM".
|
|
1839
|
+
ranges (str, optional): elements value ranges, seperated by ';'
|
|
1840
|
+
range: (a,) is >a, [a,) is >=a, (,a) is <a, (,a] is <=a, (a,b) is >a & <b,
|
|
1841
|
+
[a,b) is >=a & <b, (a,b] is >a & <=b, [a,b] is >=a & <=b
|
|
1842
|
+
list: a,b,c;
|
|
1843
|
+
e.g., "VIS:(,1000);RHU:(70,)", "Q_PRE_1h:0,3,4" is PRE quantity is credible.. Defaults to None.
|
|
1844
|
+
order (str, optional): elements order, seperated by ',', like
|
|
1845
|
+
"TEM:asc,SUM_PRE_1h:desc" is ascending order temperature first and descending PRE_1h. Defaults to None.
|
|
1846
|
+
count (int, optional): the number of maximum returned records. Defaults to None.
|
|
1847
|
+
trans_type (bool, optional): transform the return data frame's column type to datetime, numeric. Defaults to True.
|
|
1848
|
+
elements (str, optional): elements for retrieve, 'ele1,ele2,...'.
|
|
1849
|
+
|
|
1850
|
+
Returns:
|
|
1851
|
+
pandas data frame: sounding records.
|
|
1852
|
+
|
|
1853
|
+
Examples:
|
|
1854
|
+
>>> data = cmadaas_sounding_in_rect_by_time_range('[20220531000000,20220531010000]', limit=[3.51, 73.33, 53.33, 135.05])
|
|
1855
|
+
"""
|
|
1856
|
+
|
|
1857
|
+
# set retrieve parameters
|
|
1858
|
+
params = {'dataCode': data_code,
|
|
1859
|
+
'elements': elements,
|
|
1860
|
+
'timeRange': time_range,
|
|
1861
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
1862
|
+
'minLon': '{:.10f}'.format(limit[1]),
|
|
1863
|
+
'maxLat': '{:.10f}'.format(limit[2]),
|
|
1864
|
+
'maxLon': '{:.10f}'.format(limit[3]),
|
|
1865
|
+
'orderby': order if order is not None else "Datetime:ASC"}
|
|
1866
|
+
if ranges is not None: params['eleValueRanges'] = ranges
|
|
1867
|
+
if count is not None: params['limitCnt'] = str(count)
|
|
1868
|
+
|
|
1869
|
+
# interface id
|
|
1870
|
+
interface_id = "getUparEleInRectByTimeRange"
|
|
1871
|
+
|
|
1872
|
+
# retrieve data contents
|
|
1873
|
+
contents = get_rest_result(interface_id, params)
|
|
1874
|
+
contents = _load_contents(contents)
|
|
1875
|
+
if contents is None:
|
|
1876
|
+
return None
|
|
1877
|
+
|
|
1878
|
+
# construct pandas DataFrame
|
|
1879
|
+
data = pd.DataFrame(contents['DS'])
|
|
1880
|
+
if trans_type: data = cmadaas_obs_convert_type(data)
|
|
1881
|
+
|
|
1882
|
+
# return
|
|
1883
|
+
return data
|
|
1884
|
+
|
|
1885
|
+
|
|
1886
|
+
def cmadaas_analysis_by_time(time, limit=None, data_code='NAFP_CLDAS2.0_NRT_ASI_NC',
|
|
1887
|
+
levattrs={'long_name':'Height above Ground', 'units':'m'}, level_type='-',
|
|
1888
|
+
fcst_level=None, fcst_ele="TMP", zoom=None, varname='data', units=None,
|
|
1889
|
+
scale_off=None, cache=True, cache_clear=True):
|
|
1890
|
+
"""
|
|
1891
|
+
Retrieve CLDAS analysis data from CMADaaS service.
|
|
1892
|
+
|
|
1893
|
+
:param time: analysis time, like "20160817120000", format: YYYYMMDDHHMISS
|
|
1894
|
+
:param limit: [min_lat, min_lon, max_lat, max_lon]
|
|
1895
|
+
:param data_code: MUSIC data code, default is "NAFP_CLDAS2.0_NRT_ASI_NC"
|
|
1896
|
+
CLDAS2.0近实时数据产品(NC)-亚洲区域. Others like:
|
|
1897
|
+
NAFP_HRCLDAS_CHN_1KM_RT, HRCLDAS中国0.01°x0.01°逐小时实时融合实况分析产品
|
|
1898
|
+
:param fcst_level: vertical level, default is None.
|
|
1899
|
+
:param level_type: vertical level type, default is -, 表示没有层次类型
|
|
1900
|
+
:param fcst_ele: forecast element, default is 2m temperature "TAIR"
|
|
1901
|
+
:param zoom: the zoom out integer > 1, like 2. 像元缩小倍数, 数值大于1, 1的整数倍.
|
|
1902
|
+
:param varname: set variable name, default is 'data'
|
|
1903
|
+
:param units: forecast element's units, defaults to retrieved units.
|
|
1904
|
+
:param scale_off: [scale, offset], return values = values*scale + offset.
|
|
1905
|
+
:param cache: cache retrieved data to local directory, default is True.
|
|
1906
|
+
:return: xarray dataset.
|
|
1907
|
+
|
|
1908
|
+
Examples:
|
|
1909
|
+
>>> data = cmadaas_analysis_by_time(
|
|
1910
|
+
"20210123000000", data_code="NAFP_CLDAS2.0_NRT_ASI_NC",
|
|
1911
|
+
fcst_ele='TMP', units="C", scale_off=[1.0, -273.15],
|
|
1912
|
+
cache=False)
|
|
1913
|
+
>>> data = cmadaas_analysis_by_time(
|
|
1914
|
+
"20210821020000", data_code="NAFP_HRCLDAS_CHN_1KM_RT",
|
|
1915
|
+
fcst_ele='TAIR', units="C",
|
|
1916
|
+
scale_off=[1.0, -273.15], cache=False)
|
|
1917
|
+
"""
|
|
1918
|
+
# check time_str type
|
|
1919
|
+
if isinstance(time, datetime):
|
|
1920
|
+
time_str = time.strftime("%Y%m%d%H%M%S")
|
|
1921
|
+
elif isinstance(time, str):
|
|
1922
|
+
time_str = time
|
|
1923
|
+
else:
|
|
1924
|
+
warnings.warn('The type of time is not supportted')
|
|
1925
|
+
return None
|
|
1926
|
+
|
|
1927
|
+
# retrieve data from cached file
|
|
1928
|
+
if cache:
|
|
1929
|
+
directory = os.path.join(data_code, fcst_ele, str(fcst_level))
|
|
1930
|
+
filename = time_str
|
|
1931
|
+
if limit is not None:
|
|
1932
|
+
filename = filename + '.' + str(limit).replace(" ","")
|
|
1933
|
+
cache_file = CONFIG.get_cache_file(directory, filename, name="CMADaaS", cache_clear=cache_clear)
|
|
1934
|
+
if cache_file.is_file():
|
|
1935
|
+
with open(cache_file, 'rb') as f:
|
|
1936
|
+
data = pickle.load(f)
|
|
1937
|
+
return data
|
|
1938
|
+
|
|
1939
|
+
# set retrieve parameters
|
|
1940
|
+
if limit is None:
|
|
1941
|
+
params = {'dataCode': data_code,
|
|
1942
|
+
'time': time_str,
|
|
1943
|
+
'levelType': str(level_type).strip(),
|
|
1944
|
+
'fcstEle': fcst_ele}
|
|
1945
|
+
if zoom is not None: params['zoomOut'] = str(zoom)
|
|
1946
|
+
interface_id = 'getNafpAnaEleGridByTimeAndLevel'
|
|
1947
|
+
else:
|
|
1948
|
+
params = {'dataCode': data_code,
|
|
1949
|
+
'time': time_str,
|
|
1950
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
1951
|
+
"minLon": '{:.10f}'.format(limit[1]),
|
|
1952
|
+
"maxLat": '{:.10f}'.format(limit[2]),
|
|
1953
|
+
"maxLon": '{:.10f}'.format(limit[3]),
|
|
1954
|
+
'levelType': str(level_type).strip(),
|
|
1955
|
+
'fcstEle': fcst_ele}
|
|
1956
|
+
interface_id = 'getNafpAnaEleGridInRectByTimeAndLevel'
|
|
1957
|
+
if fcst_level is not None:
|
|
1958
|
+
params['fcstLevel'] = '{:d}'.format(fcst_level)
|
|
1959
|
+
|
|
1960
|
+
# retrieve data contents
|
|
1961
|
+
contents = get_rest_result(interface_id, params)
|
|
1962
|
+
contents = _load_contents(contents)
|
|
1963
|
+
if contents is None:
|
|
1964
|
+
return None
|
|
1965
|
+
|
|
1966
|
+
# get time information
|
|
1967
|
+
time = datetime.strptime(time_str, '%Y%m%d%H%M%S')
|
|
1968
|
+
time = np.array([time], dtype='datetime64[ms]')
|
|
1969
|
+
|
|
1970
|
+
# extract coordinates and data
|
|
1971
|
+
start_lat = float(contents['startLat'])
|
|
1972
|
+
start_lon = float(contents['startLon'])
|
|
1973
|
+
nlon = int(contents['lonCount'])
|
|
1974
|
+
nlat = int(contents['latCount'])
|
|
1975
|
+
dlon = float(contents['lonStep'])
|
|
1976
|
+
dlat = float(contents['latStep'])
|
|
1977
|
+
lon = start_lon + np.arange(nlon)*dlon
|
|
1978
|
+
lat = start_lat + np.arange(nlat)*dlat
|
|
1979
|
+
name = contents['fieldNames']
|
|
1980
|
+
if units is None:
|
|
1981
|
+
units = contents['fieldUnits']
|
|
1982
|
+
|
|
1983
|
+
# define coordinates and variables
|
|
1984
|
+
time_coord = ('time', time)
|
|
1985
|
+
lon_coord = ('lon', lon, {
|
|
1986
|
+
'long_name':'longitude', 'units':'degrees_east',
|
|
1987
|
+
'_CoordinateAxisType':'Lon', "axis": 'X'})
|
|
1988
|
+
lat_coord = ('lat', lat, {
|
|
1989
|
+
'long_name':'latitude', 'units':'degrees_north',
|
|
1990
|
+
'_CoordinateAxisType':'Lat', 'axis': 'Y'})
|
|
1991
|
+
if fcst_level is not None:
|
|
1992
|
+
level_coord = ('level', np.array([fcst_level]), levattrs)
|
|
1993
|
+
varattrs = {'long_name': name, 'units': units}
|
|
1994
|
+
|
|
1995
|
+
# construct xarray
|
|
1996
|
+
data = np.array(contents['DS'], dtype=np.float32)
|
|
1997
|
+
if scale_off is not None:
|
|
1998
|
+
data = data * scale_off[0] + scale_off[1]
|
|
1999
|
+
if fcst_level is None:
|
|
2000
|
+
data = data[np.newaxis, ...]
|
|
2001
|
+
data = xr.Dataset({
|
|
2002
|
+
varname:(['time', 'lat', 'lon'], data, varattrs)},
|
|
2003
|
+
coords={'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
2004
|
+
else:
|
|
2005
|
+
data = data[np.newaxis, np.newaxis, ...]
|
|
2006
|
+
data = xr.Dataset({
|
|
2007
|
+
varname:(['time', 'level', 'lat', 'lon'], data, varattrs)},
|
|
2008
|
+
coords={'time':time_coord, 'level':level_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
2009
|
+
|
|
2010
|
+
# add attributes
|
|
2011
|
+
data.attrs['Conventions'] = "CF-1.6"
|
|
2012
|
+
data.attrs['Origin'] = 'CIMISS Server by MUSIC API'
|
|
2013
|
+
|
|
2014
|
+
# cache data
|
|
2015
|
+
if cache:
|
|
2016
|
+
with open(cache_file, 'wb') as f:
|
|
2017
|
+
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
|
2018
|
+
|
|
2019
|
+
# return data
|
|
2020
|
+
return data
|
|
2021
|
+
|
|
2022
|
+
|
|
2023
|
+
def cmadaas_analysis_by_times(times_str, pbar=True, allExists=True, **kargs):
|
|
2024
|
+
"""
|
|
2025
|
+
Retrieve multiple CLDAS analysis data from CMADaaS service.
|
|
2026
|
+
|
|
2027
|
+
:param times_str: analysis time string list, like ["20200208000000", "20200209000000"], format: YYYYMMDDHHMISS
|
|
2028
|
+
:param allExists (boolean): all files should exist, or return None.
|
|
2029
|
+
:param pbar (boolean): Show progress bar, default to False.
|
|
2030
|
+
:param **kargs: key arguments passed to cmadaas_model_by_time function.
|
|
2031
|
+
:return: data, xarray type
|
|
2032
|
+
|
|
2033
|
+
Examples:
|
|
2034
|
+
>>> times = ["20200208000000", "20200209000000"]
|
|
2035
|
+
>>> data_code = "NAFP_CLDAS2.0_RT_GRB"
|
|
2036
|
+
>>> data = cmadaas_analysis_by_times(times, data_code=data_code)
|
|
2037
|
+
"""
|
|
2038
|
+
|
|
2039
|
+
dataset = []
|
|
2040
|
+
if pbar:
|
|
2041
|
+
tqdm_times_str = tqdm(times_str, desc=kargs['data_code'] + ": ")
|
|
2042
|
+
else:
|
|
2043
|
+
tqdm_times_str = times_str
|
|
2044
|
+
for time_str in tqdm_times_str:
|
|
2045
|
+
data = cmadaas_analysis_by_time(time_str, **kargs)
|
|
2046
|
+
if data:
|
|
2047
|
+
dataset.append(data)
|
|
2048
|
+
else:
|
|
2049
|
+
if allExists:
|
|
2050
|
+
warnings.warn("{} doese not exists.".format(kargs['data_code']+'/'+time_str))
|
|
2051
|
+
return None
|
|
2052
|
+
|
|
2053
|
+
return xr.concat(dataset, dim='time')
|
|
2054
|
+
|
|
2055
|
+
|
|
2056
|
+
def cmadaas_get_model_latest_time(data_code="NAFP_FOR_FTM_HIGH_EC_ANEA", latestTime=24):
|
|
2057
|
+
"""
|
|
2058
|
+
Get the latest run time of the model.
|
|
2059
|
+
注意不是所有的模式都有获取最新起步时间的接口函数.
|
|
2060
|
+
|
|
2061
|
+
Args:
|
|
2062
|
+
data_code (str, optional): dataset code, like "NAFP_FOR_FTM_HIGH_EC_ANEA".
|
|
2063
|
+
latestTime (int, optional): latestTime > 0, like 2 is return
|
|
2064
|
+
the latest time in 2 hours. Defaults to 24.
|
|
2065
|
+
Returns:
|
|
2066
|
+
datetime object, the latest time
|
|
2067
|
+
|
|
2068
|
+
Examples:
|
|
2069
|
+
>>> print(cmadaas_get_model_latest_time(data_code="NAFP_FOR_FTM_HIGH_EC_ANEA", latestTime=24))
|
|
2070
|
+
2020-03-11 12:00:00
|
|
2071
|
+
"""
|
|
2072
|
+
|
|
2073
|
+
# set retrieve parameters
|
|
2074
|
+
params = {'dataCode': data_code,
|
|
2075
|
+
'latestTime': str(latestTime+8)} # 由于服务器是世界时, 因此需要+8才能与当前时间比较.
|
|
2076
|
+
|
|
2077
|
+
# interface id
|
|
2078
|
+
interface_id = "getNafpLatestTime"
|
|
2079
|
+
|
|
2080
|
+
# retrieve data contents
|
|
2081
|
+
contents = get_rest_result(interface_id, params)
|
|
2082
|
+
contents = _load_contents(contents)
|
|
2083
|
+
if contents is None:
|
|
2084
|
+
return None
|
|
2085
|
+
|
|
2086
|
+
# construct pandas DataFrame
|
|
2087
|
+
data = pd.DataFrame(contents['DS'])
|
|
2088
|
+
time = pd.to_datetime(data['Datetime'], format="%Y%m%d%H%M%S")
|
|
2089
|
+
|
|
2090
|
+
return time[0]
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def cmadaas_model_grid(data_code, init_time, valid_time, fcst_ele, fcst_level, level_type,
|
|
2094
|
+
fcst_member=None, limit=None, varname='data', units=None,
|
|
2095
|
+
scale_off=None, cache=True, cache_clear=True,
|
|
2096
|
+
levattrs={'long_name':'height_above_ground', 'units':'m', '_CoordinateAxisType':'Height'}):
|
|
2097
|
+
"""
|
|
2098
|
+
Retrieve model grid data from CMADaaS service.
|
|
2099
|
+
refer to: http://10.20.76.55/cimissapiweb/apidataclassdefine_list.action
|
|
2100
|
+
|
|
2101
|
+
:param data_code: MUSIC data code,
|
|
2102
|
+
"NAFP_ECMF_C1D_GLB_FOR"(default): 欧洲中心数值预报产品-高分辨率C1D-全球
|
|
2103
|
+
"NAFP_ECMF_FTM_HIGH_ANEA_FOR": 欧洲中心数值预报产品-高分辨率C1D-亚洲地区
|
|
2104
|
+
......
|
|
2105
|
+
:param init_time: model run time, like "2016081712", or datetime object.
|
|
2106
|
+
:param valid_time: forecast hour, like 0 or 6
|
|
2107
|
+
:param fcst_ele: forecast element, like 2m temperature "TEM"
|
|
2108
|
+
:param fcst_level: vertical level, like 0
|
|
2109
|
+
:param level_type: forecast level type, 表示Grib数据中的层次类型, 可在云平台上查询.
|
|
2110
|
+
:param fcst_member: ensemble forecast member, 集合预报成员标识. 如果数据是集合预报, 该变量必须设置.
|
|
2111
|
+
:param limit: [min_lat, min_lon, max_lat, max_lon]
|
|
2112
|
+
:param varname: set variable name, default is 'data'
|
|
2113
|
+
:param units: forecast element's units, defaults to retrieved units.
|
|
2114
|
+
:param scale_off: [scale, offset], return values = values*scale + offset.
|
|
2115
|
+
:param cache: cache retrieved data to local directory, default is True.
|
|
2116
|
+
:param levattrs: level attributes, like:
|
|
2117
|
+
{'long_name':'height_above_ground', 'units':'m', '_CoordinateAxisType':'Height'}, default
|
|
2118
|
+
{'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'}
|
|
2119
|
+
{'long_name':'geopotential_height', 'units':'gpm', '_CoordinateAxisType':'GeoZ'}
|
|
2120
|
+
refer to https://www.unidata.ucar.edu/software/netcdf-java/current/reference/CoordinateAttributes.html
|
|
2121
|
+
:return: xarray dataset.
|
|
2122
|
+
|
|
2123
|
+
Examples:
|
|
2124
|
+
>>> data = cmadaas_model_grid("NAFP_FOR_FTM_HIGH_EC_ANEA", "2021010512", 24, 'TEM', 850, 100,
|
|
2125
|
+
units="C", scale_off=[1.0, -273.15], limit=[10.,100,30,120.],
|
|
2126
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'},
|
|
2127
|
+
cache=True)
|
|
2128
|
+
>>> data = cmadaas_model_grid("NAFP_GRAPESREPS_FOR_FTM_DIS_CHN", "2022092400", 24, 'TEM', 2, 103,
|
|
2129
|
+
fcst_member=1, units="C", scale_off=[1.0, -273.15],
|
|
2130
|
+
limit=[10., 100, 30, 120.], cache=True)
|
|
2131
|
+
"""
|
|
2132
|
+
|
|
2133
|
+
# check initial time
|
|
2134
|
+
if isinstance(init_time, datetime):
|
|
2135
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2136
|
+
elif isinstance(init_time, str):
|
|
2137
|
+
init_time_str = init_time
|
|
2138
|
+
else:
|
|
2139
|
+
warnings.warn('The type of init_time is not supportted')
|
|
2140
|
+
return None
|
|
2141
|
+
|
|
2142
|
+
# retrieve data from cached file
|
|
2143
|
+
if cache:
|
|
2144
|
+
directory = os.path.join(data_code, fcst_ele, str(fcst_level))
|
|
2145
|
+
filename = init_time_str
|
|
2146
|
+
if limit is not None:
|
|
2147
|
+
filename = filename + '_' + str(limit).replace(" ","")
|
|
2148
|
+
if fcst_member is not None:
|
|
2149
|
+
filename = filename + '_' + str(fcst_member).replace(" ","")
|
|
2150
|
+
filename = filename + '.' + str(valid_time).zfill(3)
|
|
2151
|
+
cache_file = CONFIG.get_cache_file(directory, filename, name="CMADaaS", cache_clear=cache_clear)
|
|
2152
|
+
if cache_file.is_file():
|
|
2153
|
+
with open(cache_file, 'rb') as f:
|
|
2154
|
+
data = pickle.load(f)
|
|
2155
|
+
return data
|
|
2156
|
+
|
|
2157
|
+
# set retrieve parameters
|
|
2158
|
+
if fcst_member is None:
|
|
2159
|
+
if limit is None:
|
|
2160
|
+
params = {'dataCode': data_code,
|
|
2161
|
+
'time': init_time_str + '0000',
|
|
2162
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2163
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2164
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2165
|
+
'fcstEle': fcst_ele}
|
|
2166
|
+
interface_id = 'getNafpEleGridByTimeAndLevelAndValidtime'
|
|
2167
|
+
else:
|
|
2168
|
+
params = {'dataCode': data_code,
|
|
2169
|
+
'time': init_time_str + '0000',
|
|
2170
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
2171
|
+
"minLon": '{:.10f}'.format(limit[1]),
|
|
2172
|
+
"maxLat": '{:.10f}'.format(limit[2]),
|
|
2173
|
+
"maxLon": '{:.10f}'.format(limit[3]),
|
|
2174
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2175
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2176
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2177
|
+
'fcstEle': fcst_ele}
|
|
2178
|
+
interface_id = 'getNafpEleGridInRectByTimeAndLevelAndValidtime'
|
|
2179
|
+
else:
|
|
2180
|
+
if limit is None:
|
|
2181
|
+
params = {'dataCode': data_code,
|
|
2182
|
+
'time': init_time_str + '0000',
|
|
2183
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2184
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2185
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2186
|
+
'fcstEle': fcst_ele,
|
|
2187
|
+
'fcstMember': '{:d}'.format(fcst_member)}
|
|
2188
|
+
interface_id = 'getNafpEleGridByTimeAndLevelAndValidtimeAndFcstMember'
|
|
2189
|
+
else:
|
|
2190
|
+
params = {'dataCode': data_code,
|
|
2191
|
+
'time': init_time_str + '0000',
|
|
2192
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
2193
|
+
"minLon": '{:.10f}'.format(limit[1]),
|
|
2194
|
+
"maxLat": '{:.10f}'.format(limit[2]),
|
|
2195
|
+
"maxLon": '{:.10f}'.format(limit[3]),
|
|
2196
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2197
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2198
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2199
|
+
'fcstEle': fcst_ele,
|
|
2200
|
+
'fcstMember': '{:d}'.format(fcst_member)}
|
|
2201
|
+
interface_id = 'getNafpEleGridInRectByTimeAndLevelAndValidtimeAndFcstMember'
|
|
2202
|
+
|
|
2203
|
+
# retrieve data contents
|
|
2204
|
+
contents = get_rest_result(interface_id, params)
|
|
2205
|
+
contents = _load_contents(contents)
|
|
2206
|
+
if contents is None:
|
|
2207
|
+
return None
|
|
2208
|
+
|
|
2209
|
+
# get time information
|
|
2210
|
+
init_time = datetime.strptime(init_time_str, '%Y%m%d%H')
|
|
2211
|
+
fhour = np.array([valid_time], dtype=np.float)
|
|
2212
|
+
time = init_time + timedelta(hours=fhour[0])
|
|
2213
|
+
init_time = np.array([init_time], dtype='datetime64[ms]')
|
|
2214
|
+
time = np.array([time], dtype='datetime64[ms]')
|
|
2215
|
+
|
|
2216
|
+
# extract coordinates and data
|
|
2217
|
+
start_lat = float(contents['startLat'])
|
|
2218
|
+
start_lon = float(contents['startLon'])
|
|
2219
|
+
nlon = int(contents['lonCount'])
|
|
2220
|
+
nlat = int(contents['latCount'])
|
|
2221
|
+
dlon = float(contents['lonStep'])
|
|
2222
|
+
dlat = float(contents['latStep'])
|
|
2223
|
+
lon = start_lon + np.arange(nlon)*dlon
|
|
2224
|
+
lat = start_lat + np.arange(nlat)*dlat
|
|
2225
|
+
name = contents['fieldNames']
|
|
2226
|
+
if units is None:
|
|
2227
|
+
units = contents['fieldUnits']
|
|
2228
|
+
|
|
2229
|
+
# set missing fcst_level for like fcst_leve='-'
|
|
2230
|
+
if type(fcst_level) == str:
|
|
2231
|
+
fcst_level = 0
|
|
2232
|
+
|
|
2233
|
+
# define coordinates and variables
|
|
2234
|
+
time_coord = ('time', time)
|
|
2235
|
+
lon_coord = ('lon', lon, {
|
|
2236
|
+
'long_name':'longitude', 'units':'degrees_east', '_CoordinateAxisType':'Lon'})
|
|
2237
|
+
lat_coord = ('lat', lat, {
|
|
2238
|
+
'long_name':'latitude', 'units':'degrees_north', '_CoordinateAxisType':'Lat'})
|
|
2239
|
+
if fcst_level != 0:
|
|
2240
|
+
level_coord = ('level', np.array([fcst_level]), levattrs)
|
|
2241
|
+
if fcst_member is not None:
|
|
2242
|
+
member_coord = ('realization', np.array([fcst_member]), {
|
|
2243
|
+
'long_name':'realization', units:'1'})
|
|
2244
|
+
varattrs = {'short_name': fcst_ele, 'long_name': name, 'units': units}
|
|
2245
|
+
|
|
2246
|
+
# construct xarray
|
|
2247
|
+
data = np.array(contents['DS'], dtype=np.float32)
|
|
2248
|
+
if scale_off is not None:
|
|
2249
|
+
data = data * scale_off[0] + scale_off[1]
|
|
2250
|
+
if fcst_member is None:
|
|
2251
|
+
if fcst_level == 0:
|
|
2252
|
+
data = data[np.newaxis, ...]
|
|
2253
|
+
data = xr.Dataset({
|
|
2254
|
+
varname:(['time', 'lat', 'lon'], data, varattrs)},
|
|
2255
|
+
coords={'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
2256
|
+
else:
|
|
2257
|
+
data = data[np.newaxis, np.newaxis, ...]
|
|
2258
|
+
data = xr.Dataset({
|
|
2259
|
+
varname:(['time', 'level', 'lat', 'lon'], data, varattrs)},
|
|
2260
|
+
coords={'time':time_coord, 'level':level_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
2261
|
+
else:
|
|
2262
|
+
if fcst_level == 0:
|
|
2263
|
+
data = data[np.newaxis, np.newaxis, ...]
|
|
2264
|
+
data = xr.Dataset({
|
|
2265
|
+
varname:(['realization', 'time', 'lat', 'lon'], data, varattrs)},
|
|
2266
|
+
coords={'realization':member_coord, 'time':time_coord,
|
|
2267
|
+
'lat':lat_coord, 'lon':lon_coord})
|
|
2268
|
+
else:
|
|
2269
|
+
data = data[np.newaxis, np.newaxis, np.newaxis, ...]
|
|
2270
|
+
data = xr.Dataset({
|
|
2271
|
+
varname:(['realization', 'time', 'level', 'lat', 'lon'], data, varattrs)},
|
|
2272
|
+
coords={'realization':member_coord, 'time':time_coord, 'level':level_coord,
|
|
2273
|
+
'lat':lat_coord, 'lon':lon_coord})
|
|
2274
|
+
|
|
2275
|
+
# add time coordinates
|
|
2276
|
+
data.coords['forecast_reference_time'] = init_time[0]
|
|
2277
|
+
data.coords['forecast_period'] = ('time', fhour, {
|
|
2278
|
+
'long_name':'forecast_period', 'units':'hour'})
|
|
2279
|
+
|
|
2280
|
+
# add attributes
|
|
2281
|
+
data.attrs['Conventions'] = "CF-1.6"
|
|
2282
|
+
data.attrs['Origin'] = 'CIMISS Server by MUSIC API'
|
|
2283
|
+
|
|
2284
|
+
# cache data
|
|
2285
|
+
if cache:
|
|
2286
|
+
with open(cache_file, 'wb') as f:
|
|
2287
|
+
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
|
2288
|
+
|
|
2289
|
+
# return data
|
|
2290
|
+
return data
|
|
2291
|
+
|
|
2292
|
+
|
|
2293
|
+
def cmadaas_model_grids(data_code, init_time, valid_times, fcst_ele, fcst_level, level_type,
|
|
2294
|
+
allExists=True, pbar=False, **kargs):
|
|
2295
|
+
"""
|
|
2296
|
+
Retrieve multiple valid time grids at the same initial time from CMADaaS service.
|
|
2297
|
+
|
|
2298
|
+
:param data_code: MUSIC data code,
|
|
2299
|
+
"NAFP_FOR_FTM_HIGH_EC_GLB"(default): 欧洲中心数值预报产品-高分辨率C1D-全球
|
|
2300
|
+
"NAFP_FOR_FTM_HIGH_EC_ASI": 欧洲中心数值预报产品-高分辨率C1D-亚洲地区
|
|
2301
|
+
"NAFP_FOR_FTM_HIGH_EC_ANEA": 欧洲中心数值预报产品-高分辨率C1D-东北亚地区
|
|
2302
|
+
......
|
|
2303
|
+
:param init_time: model run time, like "2016081712", or datetime object.
|
|
2304
|
+
:param valid_times: forecast hours, like [0, 6, 12, 15, 18, ...]
|
|
2305
|
+
:param fcst_ele: forecast element, like 2m temperature "TEF2"
|
|
2306
|
+
:param fcst_level: vertical level, like 0
|
|
2307
|
+
:param level_type: vertical level type, like 0
|
|
2308
|
+
:param allExists (boolean): all files should exist, or return None.
|
|
2309
|
+
:param pbar (boolean): Show progress bar, default to False.
|
|
2310
|
+
:param **kargs: key arguments passed to cmadaas_model_grid function.
|
|
2311
|
+
|
|
2312
|
+
Examples:
|
|
2313
|
+
>>> valid_times = [6*i for i in 0:13]
|
|
2314
|
+
>>> data = cmadaas_model_grids("NAFP_FOR_FTM_HIGH_EC_ANEA", "2020021512", valid_times, 'TEM', 850, 100, units="C", scale_off=[1.0, -273.15],
|
|
2315
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'})
|
|
2316
|
+
"""
|
|
2317
|
+
|
|
2318
|
+
dataset = []
|
|
2319
|
+
tqdm_valid_times = tqdm(valid_times, desc=data_code + ": ") if pbar else valid_times
|
|
2320
|
+
for valid_time in tqdm_valid_times:
|
|
2321
|
+
data = cmadaas_model_grid(data_code, init_time, valid_time, fcst_ele, fcst_level, level_type, **kargs)
|
|
2322
|
+
if data:
|
|
2323
|
+
dataset.append(data)
|
|
2324
|
+
else:
|
|
2325
|
+
if allExists:
|
|
2326
|
+
if isinstance(init_time, datetime):
|
|
2327
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2328
|
+
else:
|
|
2329
|
+
init_time_str = init_time
|
|
2330
|
+
warnings.warn("{} doese not exists.".format(data_code+'/'+init_time_str+'.'+str(valid_time).zfill(3)))
|
|
2331
|
+
return None
|
|
2332
|
+
|
|
2333
|
+
return xr.concat(dataset, dim='time')
|
|
2334
|
+
|
|
2335
|
+
|
|
2336
|
+
def cmadaas_model_points(data_code, init_time, valid_times, fcst_ele, fcst_level, level_type, points, **kargs):
|
|
2337
|
+
"""
|
|
2338
|
+
Retrieve model point time series at the same initial time from CMADaaS service.
|
|
2339
|
+
|
|
2340
|
+
:param data_code: MUSIC data code,
|
|
2341
|
+
"NAFP_FOR_FTM_HIGH_EC_GLB"(default): 欧洲中心数值预报产品-高分辨率C1D-全球
|
|
2342
|
+
"NAFP_FOR_FTM_HIGH_EC_ASI": 欧洲中心数值预报产品-高分辨率C1D-亚洲地区
|
|
2343
|
+
"NAFP_FOR_FTM_HIGH_EC_ANEA": 欧洲中心数值预报产品-高分辨率C1D-东北亚地区
|
|
2344
|
+
......
|
|
2345
|
+
:param init_time: model run time, like "2016081712", or datetime object
|
|
2346
|
+
:param valid_times: forecast hours, like [0, 6, 12, 15, 18, ...]
|
|
2347
|
+
:param fcst_ele: forecast element, like 2m temperature "TEF2", temperature "TEM"
|
|
2348
|
+
:param fcst_level: vertical level, like 0
|
|
2349
|
+
:param level_type: vertical level type, like 100
|
|
2350
|
+
:param points: dictionary, {'lon':[...], 'lat':[...]}.
|
|
2351
|
+
:param **kargs: key arguments passed to cmadaas_model_grids function.
|
|
2352
|
+
|
|
2353
|
+
Examples:
|
|
2354
|
+
>>> valid_times = [6*i for i in 0:13]
|
|
2355
|
+
>>> points = {'lon':[116.3833, 110.0], 'lat':[39.9, 32]}
|
|
2356
|
+
>>> data = cmadaas_model_points("NAFP_FOR_FTM_HIGH_EC_ANEA", "2020021512", valid_times, 'TEM', 850, 100, points,
|
|
2357
|
+
units="C", scale_off=[1.0, -273.15],
|
|
2358
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'})
|
|
2359
|
+
"""
|
|
2360
|
+
|
|
2361
|
+
data = cmadaas_model_grids(data_code, init_time, valid_times, fcst_ele, fcst_level, level_type, **kargs)
|
|
2362
|
+
if data:
|
|
2363
|
+
return data.interp(lon=('points', points['lon']), lat=('points', points['lat']))
|
|
2364
|
+
else:
|
|
2365
|
+
return None
|
|
2366
|
+
|
|
2367
|
+
|
|
2368
|
+
def cmadaas_model_3D_grid(data_code, init_time, valid_time, fcst_ele, fcst_levels, level_type, allExists=True, pbar=False, **kargs):
|
|
2369
|
+
"""
|
|
2370
|
+
Retrieve multiple level grids at the same initial time from CMADaaS service.
|
|
2371
|
+
|
|
2372
|
+
:param data_code: MUSIC data code,
|
|
2373
|
+
"NAFP_FOR_FTM_HIGH_EC_GLB"(default): 欧洲中心数值预报产品-高分辨率C1D-全球
|
|
2374
|
+
"NAFP_FOR_FTM_HIGH_EC_ASI": 欧洲中心数值预报产品-高分辨率C1D-亚洲地区
|
|
2375
|
+
"NAFP_FOR_FTM_HIGH_EC_ANEA": 欧洲中心数值预报产品-高分辨率C1D-东北亚地区
|
|
2376
|
+
......
|
|
2377
|
+
:param init_time: model run time, like "2016081712"
|
|
2378
|
+
:param valid_time: forecast hour, like 0
|
|
2379
|
+
:param fcst_ele: forecast element, like 2m temperature "TEF2", temperature "TEM"
|
|
2380
|
+
:param fcst_levels: vertical levels, like [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
|
|
2381
|
+
:param level_type: vertical level type, like 100
|
|
2382
|
+
:param allExists (boolean): all files should exist, or return None.
|
|
2383
|
+
:param pbar (boolean): Show progress bar, default to False.
|
|
2384
|
+
:param **kargs: key arguments passed to cmadaas_model_grid function.
|
|
2385
|
+
|
|
2386
|
+
Examples:
|
|
2387
|
+
>>> levels = [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
|
|
2388
|
+
>>> data = cmadaas_model_3D_grid("NAFP_FOR_FTM_HIGH_EC_ANEA", "2020021512", 24, 'TEM', levels, 100, units="C", scale_off=[1.0, -273.15],
|
|
2389
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'})
|
|
2390
|
+
"""
|
|
2391
|
+
|
|
2392
|
+
dataset = []
|
|
2393
|
+
tqdm_fcst_levels = tqdm(fcst_levels, desc=data_code + ": ") if pbar else fcst_levels
|
|
2394
|
+
for fcst_level in tqdm_fcst_levels:
|
|
2395
|
+
data = cmadaas_model_grid(data_code, init_time, valid_time, fcst_ele, fcst_level, level_type, **kargs)
|
|
2396
|
+
if data:
|
|
2397
|
+
dataset.append(data)
|
|
2398
|
+
else:
|
|
2399
|
+
if allExists:
|
|
2400
|
+
if isinstance(init_time, datetime):
|
|
2401
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2402
|
+
else:
|
|
2403
|
+
init_time_str = init_time
|
|
2404
|
+
warnings.warn("{} doese not exists.".format(data_code+'/'+init_time_str+'.'+str(valid_time).zfill(3)))
|
|
2405
|
+
return None
|
|
2406
|
+
|
|
2407
|
+
return xr.concat(dataset, dim='level')
|
|
2408
|
+
|
|
2409
|
+
|
|
2410
|
+
def cmadaas_model_3D_grids(data_code, init_time, valid_times, fcst_ele, fcst_levels, level_type, allExists=True, pbar=False, **kargs):
|
|
2411
|
+
"""
|
|
2412
|
+
Retrieve multiple time and level grids at the same initial time from CMADaaS service.
|
|
2413
|
+
|
|
2414
|
+
:param data_code: MUSIC data code,
|
|
2415
|
+
"NAFP_FOR_FTM_HIGH_EC_GLB"(default): 欧洲中心数值预报产品-高分辨率C1D-全球
|
|
2416
|
+
"NAFP_FOR_FTM_HIGH_EC_ASI": 欧洲中心数值预报产品-高分辨率C1D-亚洲地区
|
|
2417
|
+
"NAFP_FOR_FTM_HIGH_EC_ANEA": 欧洲中心数值预报产品-高分辨率C1D-东北亚地区
|
|
2418
|
+
......
|
|
2419
|
+
:param init_time: model run time, like "2016081712", or datetime object
|
|
2420
|
+
:param valid_times: forecast hour, like [0, 6, 12, 15, 18, ...]
|
|
2421
|
+
:param fcst_ele: forecast element, like 2m temperature "TEF2", temperature "TEM"
|
|
2422
|
+
:param fcst_levels: vertical levels, like [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
|
|
2423
|
+
:param level_type: vertical level type, like 100
|
|
2424
|
+
:param allExists (boolean): all files should exist, or return None.
|
|
2425
|
+
:param pbar (boolean): Show progress bar, default to False.
|
|
2426
|
+
:param **kargs: key arguments passed to cmadaas_model_grid function.
|
|
2427
|
+
|
|
2428
|
+
Examples:
|
|
2429
|
+
>>> valid_times = [6*i for i in range(13)]
|
|
2430
|
+
>>> levels = [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
|
|
2431
|
+
>>> data = cmadaas_model_3D_grids("NAFP_FOR_FTM_HIGH_EC_ANEA", "2020021512", 24, 'TEM', levels, 100, units="C", scale_off=[1.0, -273.15],
|
|
2432
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'})
|
|
2433
|
+
"""
|
|
2434
|
+
|
|
2435
|
+
dataset = []
|
|
2436
|
+
tqdm_valid_times = tqdm(valid_times, desc=data_code + ": ") if pbar else valid_times
|
|
2437
|
+
for valid_time in tqdm_valid_times:
|
|
2438
|
+
dataset_temp = []
|
|
2439
|
+
for fcst_level in fcst_levels:
|
|
2440
|
+
data = cmadaas_model_grid(data_code, init_time, valid_time, fcst_ele, fcst_level, level_type, **kargs)
|
|
2441
|
+
if data:
|
|
2442
|
+
dataset_temp.append(data)
|
|
2443
|
+
else:
|
|
2444
|
+
if allExists:
|
|
2445
|
+
if isinstance(init_time, datetime):
|
|
2446
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2447
|
+
else:
|
|
2448
|
+
init_time_str = init_time
|
|
2449
|
+
warnings.warn("{} doese not exists.".format(data_code+'/'+init_time_str+'.'+str(valid_time).zfill(3)))
|
|
2450
|
+
return None
|
|
2451
|
+
dataset.append(xr.concat(dataset_temp, dim='level'))
|
|
2452
|
+
|
|
2453
|
+
return xr.concat(dataset, dim='time')
|
|
2454
|
+
|
|
2455
|
+
|
|
2456
|
+
def cmadaas_model_profiles(data_code, init_time, valid_times, fcst_ele, fcst_levels, level_type, points, **kargs):
|
|
2457
|
+
"""
|
|
2458
|
+
Retrieve time series of vertical profile from 3D [time, level, lat, lon] grids
|
|
2459
|
+
at the same initial time from CMADaaS service.
|
|
2460
|
+
|
|
2461
|
+
:param data_code: MUSIC data code,
|
|
2462
|
+
"NAFP_FOR_FTM_HIGH_EC_GLB"(default): 欧洲中心数值预报产品-高分辨率C1D-全球
|
|
2463
|
+
"NAFP_FOR_FTM_HIGH_EC_ASI": 欧洲中心数值预报产品-高分辨率C1D-亚洲地区
|
|
2464
|
+
"NAFP_FOR_FTM_HIGH_EC_ANEA": 欧洲中心数值预报产品-高分辨率C1D-东北亚地区
|
|
2465
|
+
......
|
|
2466
|
+
:param init_time: model run time, like "2016081712", or datetime object
|
|
2467
|
+
:param valid_times: forecast hour, like [0, 6, 12, 15, 18, ...]
|
|
2468
|
+
:param fcst_ele: forecast element, like 2m temperature "TEF2", temperature "TEM"
|
|
2469
|
+
:param fcst_levels: vertical levels, like [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
|
|
2470
|
+
:param level_type: vertical level type, like 100
|
|
2471
|
+
:param point: dictionary, {'lon':[...], 'lat':[...]}.
|
|
2472
|
+
:param **kargs: key arguments passed to cmadaas_model_grid function.
|
|
2473
|
+
|
|
2474
|
+
Examples:
|
|
2475
|
+
>>> valid_times = [6*i for i in range(13)]
|
|
2476
|
+
>>> levels = [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
|
|
2477
|
+
>>> points = {'lon':[116.3833, 110.0], 'lat':[39.9, 32]}
|
|
2478
|
+
>>> data = cmadaas_model_profiles("NAFP_FOR_FTM_HIGH_EC_ANEA", "2020021512", 24, 'TEM', levels, 100, points, units="C", scale_off=[1.0, -273.15],
|
|
2479
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'})
|
|
2480
|
+
"""
|
|
2481
|
+
|
|
2482
|
+
data = cmadaas_model_3D_grids(data_code, init_time, valid_times, fcst_ele, fcst_levels, level_type, **kargs)
|
|
2483
|
+
if data:
|
|
2484
|
+
return data.interp(lon=('points', points['lon']), lat=('points', points['lat']))
|
|
2485
|
+
else:
|
|
2486
|
+
return None
|
|
2487
|
+
|
|
2488
|
+
|
|
2489
|
+
def cmadaas_model_by_time(init_time, valid_time=0, limit=None, fcst_member=None,
|
|
2490
|
+
data_code='NAFP_FOR_FTM_HIGH_EC_GLB', fcst_level=0, level_type=1,
|
|
2491
|
+
levattrs={'long_name':'pressure_level', 'units':'hPa', '_CoordinateAxisType':'Pressure'},
|
|
2492
|
+
fcst_ele="TEM", varname='data', units=None, scale_off=None, cache=True, cache_clear=True):
|
|
2493
|
+
"""
|
|
2494
|
+
Retrieve grid data from CMADaaS service.
|
|
2495
|
+
与cmadass_model_grid功能相似, 只是接口方式不同(多了一些默认参数的设置),
|
|
2496
|
+
建议使用cmadass_model_grid为主.
|
|
2497
|
+
|
|
2498
|
+
:param init_time: model run time, like "2016081712", or datetime object.
|
|
2499
|
+
:param valid_time: forecast hour, default is 0
|
|
2500
|
+
:param limit: [min_lat, min_lon, max_lat, max_lon]
|
|
2501
|
+
:param fcst_member: ensemble forecast member, 集合预报成员标识. 如果数据是集合预报, 该变量必须设置.
|
|
2502
|
+
:param varname: set variable name, default is 'data'
|
|
2503
|
+
:param data_code: MUSIC data code, default is "NAFP_FOR_FTM_HIGH_EC_GLB", 即EC高分辨率全球地面预报数据.
|
|
2504
|
+
:param fcst_level: vertical level, default is 0, 表示地面数据, 可在云平台上查询.
|
|
2505
|
+
:param level_type: forecast level type, default is 1, 表示Grib数据中的层次类型, 可在云平台上查询.
|
|
2506
|
+
:param fcst_ele: forecast element, default is 2m temperature "TEF2"
|
|
2507
|
+
:param units: forecast element's units, defaults to retrieved units.
|
|
2508
|
+
:param scale_off: [scale, offset], return values = values*scale + offset.
|
|
2509
|
+
:param cache: cache retrieved data to local directory, default is True.
|
|
2510
|
+
:return: xarray dataset.
|
|
2511
|
+
|
|
2512
|
+
Examples:
|
|
2513
|
+
>>> data = cmadaas_model_by_time("2021011512", data_code="NAFP_FOR_FTM_HIGH_EC_ANEA",
|
|
2514
|
+
fcst_level=850, level_type=100, fcst_ele='TEM', units="C",
|
|
2515
|
+
scale_off=[1.0, -273.15], cache=False)
|
|
2516
|
+
"""
|
|
2517
|
+
|
|
2518
|
+
# check initial time
|
|
2519
|
+
if isinstance(init_time, datetime):
|
|
2520
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2521
|
+
else:
|
|
2522
|
+
init_time_str = init_time
|
|
2523
|
+
|
|
2524
|
+
# retrieve data from cached file
|
|
2525
|
+
if cache:
|
|
2526
|
+
directory = os.path.join(data_code, fcst_ele, str(fcst_level))
|
|
2527
|
+
filename = init_time_str
|
|
2528
|
+
if limit is not None:
|
|
2529
|
+
filename = filename + '_' + str(limit).replace(" ","")
|
|
2530
|
+
if fcst_member is not None:
|
|
2531
|
+
filename = filename + '_' + str(fcst_member).replace(" ","")
|
|
2532
|
+
filename = filename + '.' + str(valid_time).zfill(3)
|
|
2533
|
+
cache_file = CONFIG.get_cache_file(directory, filename, name="CMADaaS", cache_clear=cache_clear)
|
|
2534
|
+
if cache_file.is_file():
|
|
2535
|
+
with open(cache_file, 'rb') as f:
|
|
2536
|
+
data = pickle.load(f)
|
|
2537
|
+
return data
|
|
2538
|
+
|
|
2539
|
+
# set retrieve parameters
|
|
2540
|
+
if fcst_member is None:
|
|
2541
|
+
if limit is None:
|
|
2542
|
+
params = {'dataCode': data_code,
|
|
2543
|
+
'time': init_time_str + '0000',
|
|
2544
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2545
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2546
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2547
|
+
'fcstEle': fcst_ele}
|
|
2548
|
+
interface_id = 'getNafpEleGridByTimeAndLevelAndValidtime'
|
|
2549
|
+
else:
|
|
2550
|
+
params = {'dataCode': data_code,
|
|
2551
|
+
'time': init_time_str + '0000',
|
|
2552
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
2553
|
+
"minLon": '{:.10f}'.format(limit[1]),
|
|
2554
|
+
"maxLat": '{:.10f}'.format(limit[2]),
|
|
2555
|
+
"maxLon": '{:.10f}'.format(limit[3]),
|
|
2556
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2557
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2558
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2559
|
+
'fcstEle': fcst_ele}
|
|
2560
|
+
interface_id = 'getNafpEleGridInRectByTimeAndLevelAndValidtime'
|
|
2561
|
+
else:
|
|
2562
|
+
if limit is None:
|
|
2563
|
+
params = {'dataCode': data_code,
|
|
2564
|
+
'time': init_time_str + '0000',
|
|
2565
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2566
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2567
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2568
|
+
'fcstEle': fcst_ele,
|
|
2569
|
+
'fcstMember': '{:d}'.format(fcst_member)}
|
|
2570
|
+
interface_id = 'getNafpEleGridByTimeAndLevelAndValidtimeAndFcstMember'
|
|
2571
|
+
else:
|
|
2572
|
+
params = {'dataCode': data_code,
|
|
2573
|
+
'time': init_time_str + '0000',
|
|
2574
|
+
'minLat': '{:.10f}'.format(limit[0]),
|
|
2575
|
+
"minLon": '{:.10f}'.format(limit[1]),
|
|
2576
|
+
"maxLat": '{:.10f}'.format(limit[2]),
|
|
2577
|
+
"maxLon": '{:.10f}'.format(limit[3]),
|
|
2578
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2579
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2580
|
+
'validTime': '{:d}'.format(valid_time),
|
|
2581
|
+
'fcstEle': fcst_ele,
|
|
2582
|
+
'fcstMember': '{:d}'.format(fcst_member)}
|
|
2583
|
+
interface_id = 'getNafpEleGridInRectByTimeAndLevelAndValidtimeAndFcstMember'
|
|
2584
|
+
|
|
2585
|
+
|
|
2586
|
+
# retrieve data contents
|
|
2587
|
+
contents = get_rest_result(interface_id, params)
|
|
2588
|
+
contents = _load_contents(contents)
|
|
2589
|
+
if contents is None:
|
|
2590
|
+
return None
|
|
2591
|
+
|
|
2592
|
+
# get time information
|
|
2593
|
+
init_time = datetime.strptime(init_time_str, '%Y%m%d%H')
|
|
2594
|
+
fhour = np.array([valid_time], dtype=np.float)
|
|
2595
|
+
time = init_time + timedelta(hours=fhour[0])
|
|
2596
|
+
init_time = np.array([init_time], dtype='datetime64[ms]')
|
|
2597
|
+
time = np.array([time], dtype='datetime64[ms]')
|
|
2598
|
+
|
|
2599
|
+
# extract coordinates and data
|
|
2600
|
+
start_lat = float(contents['startLat'])
|
|
2601
|
+
start_lon = float(contents['startLon'])
|
|
2602
|
+
nlon = int(contents['lonCount'])
|
|
2603
|
+
nlat = int(contents['latCount'])
|
|
2604
|
+
dlon = float(contents['lonStep'])
|
|
2605
|
+
dlat = float(contents['latStep'])
|
|
2606
|
+
lon = start_lon + np.arange(nlon)*dlon
|
|
2607
|
+
lat = start_lat + np.arange(nlat)*dlat
|
|
2608
|
+
name = contents['fieldNames']
|
|
2609
|
+
if units is None:
|
|
2610
|
+
units = contents['fieldUnits']
|
|
2611
|
+
|
|
2612
|
+
# set missing fcst_level for fcst_leve='-'
|
|
2613
|
+
if type(fcst_level) == str:
|
|
2614
|
+
fcst_level = 0
|
|
2615
|
+
|
|
2616
|
+
# define coordinates and variables
|
|
2617
|
+
time_coord = ('time', time)
|
|
2618
|
+
lon_coord = ('lon', lon, {
|
|
2619
|
+
'long_name':'longitude', 'units':'degrees_east',
|
|
2620
|
+
'_CoordinateAxisType':'Lon', 'axis':'X'})
|
|
2621
|
+
lat_coord = ('lat', lat, {
|
|
2622
|
+
'long_name':'latitude', 'units':'degrees_north', '_CoordinateAxisType':'Lat'})
|
|
2623
|
+
if fcst_level != 0:
|
|
2624
|
+
level_coord = ('level', np.array([fcst_level]), levattrs)
|
|
2625
|
+
if fcst_member is not None:
|
|
2626
|
+
member_coord = ('realization', np.array([fcst_member]), {
|
|
2627
|
+
'long_name':'realization', units:'1'})
|
|
2628
|
+
varattrs = {'long_name': name, 'units': units}
|
|
2629
|
+
|
|
2630
|
+
# construct xarray
|
|
2631
|
+
data = np.array(contents['DS'], dtype=np.float32)
|
|
2632
|
+
if scale_off is not None:
|
|
2633
|
+
data = data * scale_off[0] + scale_off[1]
|
|
2634
|
+
if fcst_member is None:
|
|
2635
|
+
if fcst_level == 0:
|
|
2636
|
+
data = data[np.newaxis, ...]
|
|
2637
|
+
data = xr.Dataset({
|
|
2638
|
+
varname:(['time', 'lat', 'lon'], data, varattrs)},
|
|
2639
|
+
coords={'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
2640
|
+
else:
|
|
2641
|
+
data = data[np.newaxis, np.newaxis, ...]
|
|
2642
|
+
data = xr.Dataset({
|
|
2643
|
+
varname:(['time', 'level', 'lat', 'lon'], data, varattrs)},
|
|
2644
|
+
coords={'time':time_coord, 'level':level_coord, 'lat':lat_coord, 'lon':lon_coord})
|
|
2645
|
+
else:
|
|
2646
|
+
if fcst_level == 0:
|
|
2647
|
+
data = data[np.newaxis, np.newaxis, ...]
|
|
2648
|
+
data = xr.Dataset({
|
|
2649
|
+
varname:(['realization', 'time', 'lat', 'lon'], data, varattrs)},
|
|
2650
|
+
coords={'realization':member_coord, 'time':time_coord,
|
|
2651
|
+
'lat':lat_coord, 'lon':lon_coord})
|
|
2652
|
+
else:
|
|
2653
|
+
data = data[np.newaxis, np.newaxis, np.newaxis, ...]
|
|
2654
|
+
data = xr.Dataset({
|
|
2655
|
+
varname:(['realization', 'time', 'level', 'lat', 'lon'], data, varattrs)},
|
|
2656
|
+
coords={'realization':member_coord, 'time':time_coord, 'level':level_coord,
|
|
2657
|
+
'lat':lat_coord, 'lon':lon_coord})
|
|
2658
|
+
|
|
2659
|
+
# add time coordinates
|
|
2660
|
+
data.coords['forecast_reference_time'] = init_time[0]
|
|
2661
|
+
data.coords['forecast_period'] = ('time', fhour, {
|
|
2662
|
+
'long_name':'forecast_period', 'units':'hour'})
|
|
2663
|
+
|
|
2664
|
+
# add attributes
|
|
2665
|
+
data.attrs['Conventions'] = "CF-1.6"
|
|
2666
|
+
data.attrs['Origin'] = 'CIMISS Server by MUSIC API'
|
|
2667
|
+
|
|
2668
|
+
# cache data
|
|
2669
|
+
if cache:
|
|
2670
|
+
with open(cache_file, 'wb') as f:
|
|
2671
|
+
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
|
2672
|
+
|
|
2673
|
+
# return data
|
|
2674
|
+
return data
|
|
2675
|
+
|
|
2676
|
+
|
|
2677
|
+
def cmadaas_model_by_times(init_time, valid_times=np.arange(0, 75, 6), pbar=True, allExists=True, **kargs):
|
|
2678
|
+
"""
|
|
2679
|
+
Retrieve multiple model grids from CMADaaS service.
|
|
2680
|
+
|
|
2681
|
+
:param init_time: model run time, like "2016081712", or datetime object.
|
|
2682
|
+
:param limit: [min_lat, min_lon, max_lat, max_lon]
|
|
2683
|
+
:param valid_times: forecast hours, default is [0, 6, 12, ..., 72]
|
|
2684
|
+
:param allExists (boolean): all files should exist, or return None.
|
|
2685
|
+
:param pbar (boolean): Show progress bar, default to False.
|
|
2686
|
+
:param **kargs: key arguments passed to cmadaas_model_by_time function.
|
|
2687
|
+
:return: xarray dataset.
|
|
2688
|
+
|
|
2689
|
+
Examples:
|
|
2690
|
+
>>> data = cmadaas_model_by_times("2020021512", data_code="NAFP_FOR_FTM_HIGH_EC_ANEA", time_range=[0, 72],
|
|
2691
|
+
fcst_level=850, fcst_ele='TEM', units="C", scale_off=[1.0, -273.15])
|
|
2692
|
+
"""
|
|
2693
|
+
|
|
2694
|
+
# check time
|
|
2695
|
+
if isinstance(init_time, datetime):
|
|
2696
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2697
|
+
else:
|
|
2698
|
+
init_time_str = init_time
|
|
2699
|
+
|
|
2700
|
+
dataset = []
|
|
2701
|
+
tqdm_valid_times = tqdm(valid_times, desc=kargs['data_code'] + ": ") if pbar else valid_times
|
|
2702
|
+
for valid_time in tqdm_valid_times:
|
|
2703
|
+
data = cmadaas_model_by_time(init_time_str, valid_time=valid_time, **kargs)
|
|
2704
|
+
if data:
|
|
2705
|
+
dataset.append(data)
|
|
2706
|
+
else:
|
|
2707
|
+
if allExists:
|
|
2708
|
+
warnings.warn("{} doese not exists.".format(kargs['data_code']+'/'+init_time_str))
|
|
2709
|
+
return None
|
|
2710
|
+
|
|
2711
|
+
return xr.concat(dataset, dim='time')
|
|
2712
|
+
|
|
2713
|
+
|
|
2714
|
+
def cmadaas_model_by_pionts(init_time_str, data_code='NAFP_FOR_FTM_HIGH_EC_ANEA',
|
|
2715
|
+
fcst_level=850, level_type=100, time_range=[0, 72],
|
|
2716
|
+
points="39.90/116.40", fcst_ele="TEM"):
|
|
2717
|
+
"""
|
|
2718
|
+
Retrieve grid point data from CMADaaS service.
|
|
2719
|
+
|
|
2720
|
+
:param init_time_str: model run time, like "2020020600"
|
|
2721
|
+
:param data_code: MUSIC data code, default is "NAFP_FOR_FTM_HIGH_EC_ANEA"
|
|
2722
|
+
:param fcst_level: vertical level, default is 850.
|
|
2723
|
+
:param level_type: forecast level type, default=100, 表示Grib数据中的层次类型, 可在云平台上查询.
|
|
2724
|
+
:param time_range: [minimum, maximum] forecast hour, default is [0, 72]
|
|
2725
|
+
:param points: point location "latitude/longitude", also support
|
|
2726
|
+
multiple points like "39.90/116.40,32.90/112.40"
|
|
2727
|
+
:param fcst_ele: forecast element, default is temperature "TEM"
|
|
2728
|
+
:return: pandas dataframe
|
|
2729
|
+
"""
|
|
2730
|
+
|
|
2731
|
+
# set retrieve parameters
|
|
2732
|
+
params = {'dataCode': data_code,
|
|
2733
|
+
'time': init_time_str + '0000',
|
|
2734
|
+
'fcstLevel': fcst_level if type(fcst_level) == str else '{:d}'.format(fcst_level),
|
|
2735
|
+
'levelType': level_type if type(level_type) == str else '{:d}'.format(level_type),
|
|
2736
|
+
'minVT': '{:d}'.format(time_range[0]),
|
|
2737
|
+
'maxVT': '{:d}'.format(time_range[1]),
|
|
2738
|
+
'latLons': points,
|
|
2739
|
+
'fcstEle': fcst_ele}
|
|
2740
|
+
interface_id = 'getNafpEleAtPointByTimeAndLevelAndValidtimeRange'
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
# retrieve data contents
|
|
2744
|
+
contents = get_rest_result(interface_id, params)
|
|
2745
|
+
contents = _load_contents(contents)
|
|
2746
|
+
if contents is None:
|
|
2747
|
+
return None
|
|
2748
|
+
|
|
2749
|
+
# convert to numeric
|
|
2750
|
+
data = pd.DataFrame(contents['DS'])
|
|
2751
|
+
data['Lat'] = pd.to_numeric(data['Lat'])
|
|
2752
|
+
data['Lon'] = pd.to_numeric(data['Lon'])
|
|
2753
|
+
data['Validtime'] = pd.to_datetime(data['Validtime'], format="%Y%m%d%H%M%S")
|
|
2754
|
+
data[fcst_ele] = pd.to_numeric(data[fcst_ele])
|
|
2755
|
+
|
|
2756
|
+
return data
|
|
2757
|
+
|
|
2758
|
+
|
|
2759
|
+
def cmadaas_model_by_piont_levels(init_time, fcst_levels=[1000, 925, 850, 700, 500],
|
|
2760
|
+
pbar=True, **kargs):
|
|
2761
|
+
"""
|
|
2762
|
+
Retrieve grid point data from CMADaaS service.
|
|
2763
|
+
|
|
2764
|
+
:param init_time: model run time, like "2020020600", or datetime object
|
|
2765
|
+
:param data_code: MUSIC data code, default is "NAFP_FOR_FTM_HIGH_EC_ANEA"
|
|
2766
|
+
:param fcst_levels: vertical levels, list like [1000, 950, 925, ...]
|
|
2767
|
+
:param time_range: [minimum, maximum] forecast hour, default is [0, 72]
|
|
2768
|
+
:param point: point location "latitude/longitude"
|
|
2769
|
+
:param fcst_ele: forecast element, default is temperature "TEM"
|
|
2770
|
+
:return: pandas dataframe
|
|
2771
|
+
"""
|
|
2772
|
+
|
|
2773
|
+
if isinstance(init_time, datetime):
|
|
2774
|
+
init_time_str = init_time.strftime("%Y%m%d%H")
|
|
2775
|
+
else:
|
|
2776
|
+
init_time_str = init_time
|
|
2777
|
+
|
|
2778
|
+
# loop every level
|
|
2779
|
+
tqdm_fcst_levels = tqdm(fcst_levels, desc=kargs['data_code'] + ": ") if pbar else fcst_levels
|
|
2780
|
+
data = None
|
|
2781
|
+
for fcst_level in tqdm_fcst_levels:
|
|
2782
|
+
temp = cmadaas_model_by_pionts(init_time_str, fcst_level=fcst_level, **kargs)
|
|
2783
|
+
if temp is None:
|
|
2784
|
+
return None
|
|
2785
|
+
|
|
2786
|
+
temp['level'] = fcst_level
|
|
2787
|
+
if data is None:
|
|
2788
|
+
data = temp
|
|
2789
|
+
else:
|
|
2790
|
+
data = pd.concat([data, temp])
|
|
2791
|
+
|
|
2792
|
+
data = data.pivot(index='Validtime', columns='level',values=kargs['fcst_ele'])
|
|
2793
|
+
data = xr.DataArray(data, coords=[data.index.values, data.columns.values],
|
|
2794
|
+
dims=['time', 'level'], name=kargs['fcst_ele'])
|
|
2795
|
+
data = data.loc[{'level':sorted(data.coords['level'].values, reverse=True)}]
|
|
2796
|
+
data = data.loc[{'time':sorted(data.coords['time'].values)}]
|
|
2797
|
+
|
|
2798
|
+
return data
|
|
2799
|
+
|
|
2800
|
+
|
|
2801
|
+
def cmadaas_get_model_file(time, data_code="NAFP_FOR_FTM_HIGH_EC_ANEA", fcst_ele=None,
|
|
2802
|
+
level_type='100', out_dir=None, pbar=False, just_url=False):
|
|
2803
|
+
"""
|
|
2804
|
+
Download numeric weather predication model file.
|
|
2805
|
+
|
|
2806
|
+
注意cmadass设置了ip访问次数, 如果下载的文件数量过多, 则会返回"-5004 Reach the hor access line."
|
|
2807
|
+
表示单位小时内检索过于频繁,请降低检索频次. 另外, 也要注意下载的文件大小, 如果文件只有几K,表示下载不成功, 删除重新下载.
|
|
2808
|
+
|
|
2809
|
+
Args:
|
|
2810
|
+
times (str): model initial time for retrieve,
|
|
2811
|
+
single time 'YYYYMMDDHHMISS'; or
|
|
2812
|
+
time range, '[YYYYMMDDHHMISS,YYYYMMDDHHMISS]'
|
|
2813
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CMPA_RT_NC".
|
|
2814
|
+
fcst_ele (str, optional): focast element. Defaults to None.
|
|
2815
|
+
out_dir (str, optional): download files to out_dir. if out_dir is None,
|
|
2816
|
+
the cached directory will be used. Defaults to None.
|
|
2817
|
+
pbar (bool, optional): Show progress bar, default to True.
|
|
2818
|
+
just_url (bool, optional): if just_url = True, return url string array, no files are downloaded.
|
|
2819
|
+
|
|
2820
|
+
Returns:
|
|
2821
|
+
the list of download files path.
|
|
2822
|
+
|
|
2823
|
+
Examples:
|
|
2824
|
+
>>> out_files = cmadaas_get_model_file('20210113000000', just_url=True)
|
|
2825
|
+
>>> out_files = cmadaas_get_model_file('20210113000000', fcst_ele='TEM', out_dir=".")
|
|
2826
|
+
>>> out_files = cmadaas_get_model_file('[20210111000000,20210130000000]', fcst_ele='TEM', out_dir=".")
|
|
2827
|
+
"""
|
|
2828
|
+
|
|
2829
|
+
# check initial time
|
|
2830
|
+
if isinstance(time, datetime):
|
|
2831
|
+
time_str = time.strftime("%Y%m%d%H%M%S")
|
|
2832
|
+
else:
|
|
2833
|
+
time_str = time
|
|
2834
|
+
|
|
2835
|
+
if out_dir is None:
|
|
2836
|
+
out_dir = CONFIG.get_cache_file(data_code, "", name="CMADaaS")
|
|
2837
|
+
|
|
2838
|
+
time_str = time_str.strip()
|
|
2839
|
+
if fcst_ele is None:
|
|
2840
|
+
params = {'dataCode': data_code}
|
|
2841
|
+
if time_str[0] == '[':
|
|
2842
|
+
# set retrieve parameters
|
|
2843
|
+
params['timeRange'] = time_str
|
|
2844
|
+
interface_id = "getNafpFileByTimeRange"
|
|
2845
|
+
else:
|
|
2846
|
+
# set retrieve parameters
|
|
2847
|
+
params['time'] = time_str
|
|
2848
|
+
interface_id = "getNafpFileByTime"
|
|
2849
|
+
else:
|
|
2850
|
+
params = {'dataCode': data_code,
|
|
2851
|
+
'fcstEle': fcst_ele.strip(),
|
|
2852
|
+
'levelType': str(level_type).strip()}
|
|
2853
|
+
if time_str[0] == '[':
|
|
2854
|
+
# set retrieve parameters
|
|
2855
|
+
params['timeRange'] = time_str
|
|
2856
|
+
interface_id = "getNafpFileByElementAndTimeRange"
|
|
2857
|
+
else:
|
|
2858
|
+
# set retrieve parameters
|
|
2859
|
+
params['time'] = time_str
|
|
2860
|
+
interface_id = "getNafpFileByElementAndTime"
|
|
2861
|
+
|
|
2862
|
+
# retrieve data contents
|
|
2863
|
+
contents = get_rest_result(interface_id, params)
|
|
2864
|
+
contents = _load_contents(contents)
|
|
2865
|
+
if contents is None:
|
|
2866
|
+
return None
|
|
2867
|
+
|
|
2868
|
+
# just return the url
|
|
2869
|
+
if just_url:
|
|
2870
|
+
return contents['DS']
|
|
2871
|
+
|
|
2872
|
+
# loop every file and download
|
|
2873
|
+
out_files = []
|
|
2874
|
+
files = tqdm(contents['DS']) if pbar else contents['DS']
|
|
2875
|
+
for file in files:
|
|
2876
|
+
out_file = Path(out_dir) / file['FILE_NAME']
|
|
2877
|
+
if not out_file.is_file():
|
|
2878
|
+
urllib.request.urlretrieve(file['FILE_URL'], out_file)
|
|
2879
|
+
out_files.append(out_file)
|
|
2880
|
+
|
|
2881
|
+
return out_files
|
|
2882
|
+
|
|
2883
|
+
|
|
2884
|
+
def cmadaas_get_model_file_with_filter(
|
|
2885
|
+
time, data_code="NAFP_FOR_FTM_HIGH_EC_ANEA",
|
|
2886
|
+
filter=None, out_dir=None, pbar=False, just_url=False):
|
|
2887
|
+
"""
|
|
2888
|
+
Download numeric weather predication model file.
|
|
2889
|
+
与cmadaas_get_model_file函数不同, 本程序增加了对检索文件的过滤.
|
|
2890
|
+
|
|
2891
|
+
Args:
|
|
2892
|
+
times (str): model initial time for retrieve,
|
|
2893
|
+
single time 'YYYYMMDDHHMISS'; or
|
|
2894
|
+
time range, '[YYYYMMDDHHMISS,YYYYMMDDHHMISS]'
|
|
2895
|
+
data_code (str, optional): dataset code. Defaults to "SURF_CMPA_RT_NC".
|
|
2896
|
+
filter (str, optional): filename's filter string. Defaults to None.
|
|
2897
|
+
out_dir (str, optional): download files to out_dir. if out_dir is None,
|
|
2898
|
+
the cached directory will be used. Defaults to None.
|
|
2899
|
+
pbar (bool, optional): Show progress bar, default to True.
|
|
2900
|
+
just_url (bool, optional): if just_url = True, return url string array, no files are downloaded.
|
|
2901
|
+
|
|
2902
|
+
Returns:
|
|
2903
|
+
the list of download files path.
|
|
2904
|
+
|
|
2905
|
+
Examples:
|
|
2906
|
+
>>> out_files = cmadaas_get_model_file_with_filter(
|
|
2907
|
+
'20220920000000', data_code="NAFP_GRAPESREPS_FOR_FTM_DIS_CHN",
|
|
2908
|
+
filter="*_TEM_103_*_4_4.*", just_url=True)
|
|
2909
|
+
"""
|
|
2910
|
+
|
|
2911
|
+
# check initial time
|
|
2912
|
+
if isinstance(time, datetime):
|
|
2913
|
+
time_str = time.strftime("%Y%m%d%H%M%S")
|
|
2914
|
+
else:
|
|
2915
|
+
time_str = time
|
|
2916
|
+
time_str = time_str.strip()
|
|
2917
|
+
|
|
2918
|
+
if out_dir is None:
|
|
2919
|
+
out_dir = CONFIG.get_cache_file(data_code, "", name="CMADaaS")
|
|
2920
|
+
|
|
2921
|
+
params = {'dataCode': data_code}
|
|
2922
|
+
if time_str[0] == '[':
|
|
2923
|
+
# set retrieve parameters
|
|
2924
|
+
params['timeRange'] = time_str
|
|
2925
|
+
interface_id = "getNafpFileByTimeRange"
|
|
2926
|
+
else:
|
|
2927
|
+
# set retrieve parameters
|
|
2928
|
+
params['time'] = time_str
|
|
2929
|
+
interface_id = "getNafpFileByTime"
|
|
2930
|
+
|
|
2931
|
+
# retrieve data contents
|
|
2932
|
+
contents = get_rest_result(interface_id, params)
|
|
2933
|
+
contents = _load_contents(contents)
|
|
2934
|
+
if contents is None:
|
|
2935
|
+
return None
|
|
2936
|
+
|
|
2937
|
+
# filter contents
|
|
2938
|
+
files = []
|
|
2939
|
+
if filter is not None:
|
|
2940
|
+
for file in contents['DS']:
|
|
2941
|
+
if fnmatch.fnmatch(file['FILE_NAME'], filter):
|
|
2942
|
+
files.append(file)
|
|
2943
|
+
else:
|
|
2944
|
+
files = contents['DS']
|
|
2945
|
+
if len(files) < 1:
|
|
2946
|
+
return None
|
|
2947
|
+
|
|
2948
|
+
# just return the url
|
|
2949
|
+
if just_url:
|
|
2950
|
+
return files
|
|
2951
|
+
|
|
2952
|
+
# loop every file and download
|
|
2953
|
+
out_files = []
|
|
2954
|
+
files = tqdm(files) if pbar else files
|
|
2955
|
+
for file in files:
|
|
2956
|
+
out_file = Path(out_dir) / file['FILE_NAME']
|
|
2957
|
+
if not out_file.is_file():
|
|
2958
|
+
urllib.request.urlretrieve(file['FILE_URL'], out_file)
|
|
2959
|
+
out_files.append(out_file)
|
|
2960
|
+
|
|
2961
|
+
return out_files
|
|
2962
|
+
|
|
2963
|
+
def cmadaas_get_radar_vol(starttime,endtime,sta_ids,data_code,outdir):
|
|
2964
|
+
interface_id = 'getRadaFileByTimeRangeAndStaId'
|
|
2965
|
+
starttime = datetime.strptime(starttime, '%Y%m%d%H%M%S')
|
|
2966
|
+
endtime = datetime.strptime(endtime, '%Y%m%d%H%M%S')
|
|
2967
|
+
curtime = starttime
|
|
2968
|
+
allfilepaths=[]
|
|
2969
|
+
allfilenames=[]
|
|
2970
|
+
while curtime <= endtime:
|
|
2971
|
+
|
|
2972
|
+
times = curtime.strftime('%Y%m%d%H0000')
|
|
2973
|
+
tmpt = curtime + timedelta(hours=1)
|
|
2974
|
+
tmptstr = tmpt.strftime('%Y%m%d%H0000')
|
|
2975
|
+
time_range = '[' + times + ',' + tmptstr + ']'
|
|
2976
|
+
# time_range='[20180301000000,20201101000000]'
|
|
2977
|
+
print(time_range)
|
|
2978
|
+
for sta in sta_ids:
|
|
2979
|
+
params = {'timeRange': time_range,
|
|
2980
|
+
'dataCode': data_code,
|
|
2981
|
+
'staIds': sta,
|
|
2982
|
+
'elements': 'FILE_NAME',
|
|
2983
|
+
}
|
|
2984
|
+
contents = get_rest_result(interface_id, params, data_format='json')
|
|
2985
|
+
if contents == None:
|
|
2986
|
+
print('No Valid data: ' + sta)
|
|
2987
|
+
continue
|
|
2988
|
+
|
|
2989
|
+
json_dic1 = json.loads(contents, strict=False)
|
|
2990
|
+
if json_dic1['returnCode'] == '0':
|
|
2991
|
+
|
|
2992
|
+
dd = json_dic1['DS']
|
|
2993
|
+
http = urllib3.PoolManager()
|
|
2994
|
+
for idx in range(len(dd)):
|
|
2995
|
+
|
|
2996
|
+
url = dd[idx]["FILE_URL"]
|
|
2997
|
+
filename = dd[idx]["FILE_NAME"]
|
|
2998
|
+
|
|
2999
|
+
newoutpath = outdir + os.sep + filename[15:19] +\
|
|
3000
|
+
os.sep + filename[15:23] + os.sep+ sta
|
|
3001
|
+
if not os.path.exists(newoutpath):
|
|
3002
|
+
os.makedirs(newoutpath)
|
|
3003
|
+
if not os.path.exists( newoutpath + os.sep + filename):
|
|
3004
|
+
# 如果不存在,才下载
|
|
3005
|
+
req = http.request("GET",url)
|
|
3006
|
+
if req.status != 200:
|
|
3007
|
+
print('Can not access the url: ' + url)
|
|
3008
|
+
|
|
3009
|
+
else:
|
|
3010
|
+
fout = open(newoutpath + os.sep + filename, 'wb')
|
|
3011
|
+
fout.write(req.data)
|
|
3012
|
+
fout.close()
|
|
3013
|
+
print('Download succeed: ' + filename)
|
|
3014
|
+
allfilepaths.append(newoutpath)
|
|
3015
|
+
allfilenames.append(filename)
|
|
3016
|
+
else:
|
|
3017
|
+
print(filename + ' already exists!')
|
|
3018
|
+
allfilepaths.append(newoutpath)
|
|
3019
|
+
allfilenames.append(filename)
|
|
3020
|
+
else:
|
|
3021
|
+
print('No Valid Record! ' + sta)
|
|
3022
|
+
print(json_dic1)
|
|
3023
|
+
curtime += timedelta(hours=1)
|
|
3024
|
+
return allfilepaths,allfilenames
|
|
3025
|
+
|
|
3026
|
+
#从天擎下载探测中心制作的三维反射率因子数据
|
|
3027
|
+
def cmadaas_get_radar_3dref(starttime,endtime,data_code,outdir):
|
|
3028
|
+
'''
|
|
3029
|
+
根据时间段检索雷达数据 interface_id = getRadaFileByTimeRange
|
|
3030
|
+
data_code = RADA_L3_MST_PRE_3REF_QC
|
|
3031
|
+
'''
|
|
3032
|
+
interface_id = 'getRadaFileByTimeRange'
|
|
3033
|
+
starttime = datetime.strptime(starttime, '%Y%m%d%H%M%S')
|
|
3034
|
+
endtime = datetime.strptime(endtime, '%Y%m%d%H%M%S')
|
|
3035
|
+
curtime = starttime
|
|
3036
|
+
allfilepaths=[]
|
|
3037
|
+
allfilenames=[]
|
|
3038
|
+
while curtime < endtime:
|
|
3039
|
+
|
|
3040
|
+
times = curtime.strftime('%Y%m%d%H0000')
|
|
3041
|
+
tmpt = curtime + timedelta(hours=1)
|
|
3042
|
+
tmptstr = tmpt.strftime('%Y%m%d%H0000')
|
|
3043
|
+
time_range = '[' + times + ',' + tmptstr + ']'
|
|
3044
|
+
# time_range='[20180301000000,20201101000000]'
|
|
3045
|
+
print(time_range)
|
|
3046
|
+
# 逐6分钟进行检测,如果已经存在了就不再查询
|
|
3047
|
+
sst = curtime
|
|
3048
|
+
ett = tmpt
|
|
3049
|
+
cst = sst
|
|
3050
|
+
bneed_download=False
|
|
3051
|
+
while cst < ett:
|
|
3052
|
+
# cpath = outdir + os.sep + cst.strftime('%Y') + os.sep + cst.strftime('%Y%m%d')
|
|
3053
|
+
cpath = outdir
|
|
3054
|
+
if not os.path.exists(cpath):
|
|
3055
|
+
bneed_download = True
|
|
3056
|
+
break
|
|
3057
|
+
else:
|
|
3058
|
+
cfile = cpath + os.sep + cst.strftime('ACHN_CAP_%Y%m%d_%H%M00') + '_3d.nc'
|
|
3059
|
+
if not os.path.exists(cfile):
|
|
3060
|
+
cfile2 = cpath + os.sep + cst.strftime('ACHN_CAP_%Y%m%d_%H%M00') + '_23.bin'
|
|
3061
|
+
if not os.path.exists(cfile2):
|
|
3062
|
+
cfile3 = cpath + os.sep + cst.strftime('ACHN_CAP_%Y%m%d_%H%M00') + '_23.nc'
|
|
3063
|
+
if not os.path.exists(cfile3):
|
|
3064
|
+
bneed_download = True
|
|
3065
|
+
break
|
|
3066
|
+
else:
|
|
3067
|
+
allfilepaths.append(cpath)
|
|
3068
|
+
else:
|
|
3069
|
+
allfilepaths.append(cpath)
|
|
3070
|
+
|
|
3071
|
+
|
|
3072
|
+
|
|
3073
|
+
cst += timedelta(minutes=6)
|
|
3074
|
+
if bneed_download:
|
|
3075
|
+
params = {'timeRange': time_range,
|
|
3076
|
+
'dataCode': data_code,
|
|
3077
|
+
'elements': 'FILE_NAME',
|
|
3078
|
+
}
|
|
3079
|
+
contents = get_rest_result(interface_id, params, data_format='json')
|
|
3080
|
+
if contents == None:
|
|
3081
|
+
print('No Valid data ')
|
|
3082
|
+
curtime += timedelta(hours=1)
|
|
3083
|
+
continue
|
|
3084
|
+
|
|
3085
|
+
json_dic1 = json.loads(contents, strict=False)
|
|
3086
|
+
if json_dic1['returnCode'] == '0':
|
|
3087
|
+
|
|
3088
|
+
dd = json_dic1['DS']
|
|
3089
|
+
http = urllib3.PoolManager()
|
|
3090
|
+
for idx in range(len(dd)):
|
|
3091
|
+
|
|
3092
|
+
url = dd[idx]["FILE_URL"]
|
|
3093
|
+
filename = dd[idx]["FILE_NAME"]
|
|
3094
|
+
|
|
3095
|
+
newoutpath = outdir + os.sep + filename[44:48] +\
|
|
3096
|
+
os.sep + filename[44:52]
|
|
3097
|
+
if not os.path.exists(newoutpath):
|
|
3098
|
+
os.makedirs(newoutpath)
|
|
3099
|
+
outname_3d = filename[35:60] + '3d.nc'
|
|
3100
|
+
outname_single = filename[35::]
|
|
3101
|
+
if os.path.exists( newoutpath + os.sep + outname_3d):
|
|
3102
|
+
print(outname_3d + ' already exists!')
|
|
3103
|
+
# allfilepaths.append(newoutpath)
|
|
3104
|
+
# allfilenames.append(filename)
|
|
3105
|
+
elif os.path.exists( newoutpath + os.sep + outname_single):
|
|
3106
|
+
print(outname_single + ' already exists!')
|
|
3107
|
+
allfilepaths.append(newoutpath)
|
|
3108
|
+
allfilenames.append(outname_single)
|
|
3109
|
+
else:
|
|
3110
|
+
# 如果不存在,才下载
|
|
3111
|
+
req = http.request("GET",url)
|
|
3112
|
+
if req.status != 200:
|
|
3113
|
+
print('Can not access the url: ' + url)
|
|
3114
|
+
else:
|
|
3115
|
+
fout = open(newoutpath + os.sep + outname_single, 'wb')
|
|
3116
|
+
fout.write(req.data)
|
|
3117
|
+
fout.close()
|
|
3118
|
+
print('Download succeed: ' + outname_single)
|
|
3119
|
+
allfilepaths.append(newoutpath)
|
|
3120
|
+
allfilenames.append(outname_single)
|
|
3121
|
+
|
|
3122
|
+
else:
|
|
3123
|
+
# print(json_dic1)
|
|
3124
|
+
print('no need to download')
|
|
3125
|
+
curtime += timedelta(hours=1)
|
|
3126
|
+
return allfilepaths,allfilenames
|