cdasws 1.8.11__py3-none-any.whl → 1.8.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cdasws/__init__.py +5 -2310
- cdasws/__main__.py +1 -1
- cdasws/cdasws.py +1833 -320
- {cdasws-1.8.11.dist-info → cdasws-1.8.12.dist-info}/METADATA +3 -1
- cdasws-1.8.12.dist-info/RECORD +13 -0
- {cdasws-1.8.11.dist-info → cdasws-1.8.12.dist-info}/licenses/LICENSE +1 -1
- cdasws-1.8.11.dist-info/RECORD +0 -13
- {cdasws-1.8.11.dist-info → cdasws-1.8.12.dist-info}/WHEEL +0 -0
- {cdasws-1.8.11.dist-info → cdasws-1.8.12.dist-info}/top_level.txt +0 -0
cdasws/__init__.py
CHANGED
|
@@ -55,96 +55,7 @@ Notes
|
|
|
55
55
|
"""
|
|
56
56
|
|
|
57
57
|
|
|
58
|
-
|
|
59
|
-
import os
|
|
60
|
-
import platform
|
|
61
|
-
import logging
|
|
62
|
-
import re
|
|
63
|
-
from importlib.util import find_spec
|
|
64
|
-
import urllib.parse
|
|
65
|
-
from urllib.parse import urlparse
|
|
66
|
-
import json
|
|
67
|
-
from operator import itemgetter
|
|
68
|
-
import time
|
|
69
|
-
from datetime import datetime, timedelta, timezone
|
|
70
|
-
import xml.etree.ElementTree as ET
|
|
71
|
-
from tempfile import mkstemp
|
|
72
|
-
from typing import Any, Callable, Dict, List, Tuple, Union
|
|
73
|
-
|
|
74
|
-
import requests
|
|
75
|
-
import dateutil.parser
|
|
76
|
-
|
|
77
|
-
from cdasws.datarepresentation import DataRepresentation
|
|
78
|
-
from cdasws.datarequest import AudioRequest, DataRequest
|
|
79
|
-
from cdasws.datarequest import CdfFormat, CdfRequest, Compression
|
|
80
|
-
from cdasws.datarequest import ImageFormat, GraphOptions, GraphRequest
|
|
81
|
-
from cdasws.datarequest import TextFormat, TextRequest, ThumbnailRequest
|
|
82
|
-
from cdasws.timeinterval import TimeInterval
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# requires python >= 3.4
|
|
86
|
-
#if find_spec('spacepy.datamodel') is not None:
|
|
87
|
-
# import spacepy.datamodel as spdm # type: ignore
|
|
88
|
-
# SPDM_AVAILABLE = True
|
|
89
|
-
#else:
|
|
90
|
-
# SPDM_AVAILABLE = False
|
|
91
|
-
# python < 3.4
|
|
92
|
-
try:
|
|
93
|
-
import spacepy.datamodel as spdm # type: ignore
|
|
94
|
-
SPDM_AVAILABLE = True
|
|
95
|
-
except ImportError:
|
|
96
|
-
SPDM_AVAILABLE = False
|
|
97
|
-
|
|
98
|
-
try:
|
|
99
|
-
from cdflib.xarray import cdf_to_xarray
|
|
100
|
-
import xarray as xr
|
|
101
|
-
CDF_XARRAY_AVAILABLE = True
|
|
102
|
-
except ImportError:
|
|
103
|
-
try:
|
|
104
|
-
import cdflib as cdf
|
|
105
|
-
import xarray as xr # pylint: disable=ungrouped-imports
|
|
106
|
-
CDF_XARRAY_AVAILABLE = True
|
|
107
|
-
def cdf_to_xarray(filename, to_datetime=False, to_unixtime=False,
|
|
108
|
-
fillval_to_nan=False):
|
|
109
|
-
"""
|
|
110
|
-
Reads a CDF into an xarray.dataset. This function exists
|
|
111
|
-
to provide compatility with cdflib >= 1.0.1 for older
|
|
112
|
-
releases of cdflib.
|
|
113
|
-
|
|
114
|
-
Parameters:
|
|
115
|
-
-----------
|
|
116
|
-
filename
|
|
117
|
-
The path to the CDF file to read.
|
|
118
|
-
to_datetime
|
|
119
|
-
Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to
|
|
120
|
-
datetime, or leave them as is.
|
|
121
|
-
to_unixtime
|
|
122
|
-
Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to
|
|
123
|
-
unixtime, or leave them as is.
|
|
124
|
-
fillval_to_nan
|
|
125
|
-
If True, any data values that match the FILLVAL
|
|
126
|
-
attribute for a variable will be set to NaN.
|
|
127
|
-
|
|
128
|
-
Returns
|
|
129
|
-
-------
|
|
130
|
-
xarray.dataset
|
|
131
|
-
An XArray Dataset object.
|
|
132
|
-
"""
|
|
133
|
-
return cdf.cdf_to_xarray(filename, to_datetime=to_datetime, # pylint: disable=no-member
|
|
134
|
-
to_unixtime=to_unixtime,
|
|
135
|
-
fillval_to_nan=fillval_to_nan)
|
|
136
|
-
except ImportError:
|
|
137
|
-
CDF_XARRAY_AVAILABLE = False
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
try:
|
|
141
|
-
import requests_cache
|
|
142
|
-
CACHE_AVAILABLE = True
|
|
143
|
-
except ImportError:
|
|
144
|
-
CACHE_AVAILABLE = False
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
__version__ = "1.8.11"
|
|
58
|
+
__version__ = "1.8.12"
|
|
148
59
|
|
|
149
60
|
|
|
150
61
|
#
|
|
@@ -168,2223 +79,7 @@ NAMESPACES = {
|
|
|
168
79
|
'cdas': NS,
|
|
169
80
|
'xhtml': XHTML_NS
|
|
170
81
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
msg: str,
|
|
176
|
-
value: Dict) -> int:
|
|
177
|
-
"""
|
|
178
|
-
A get_data progress callback which adjusts the progress value for
|
|
179
|
-
the download portion of a larger operation and then calls the
|
|
180
|
-
"real" progress callback function with this adjusted progress value.
|
|
181
|
-
|
|
182
|
-
Parameters
|
|
183
|
-
----------
|
|
184
|
-
progress
|
|
185
|
-
Measure of progress.
|
|
186
|
-
msg
|
|
187
|
-
Message describing progress of get_data call.
|
|
188
|
-
value
|
|
189
|
-
Dictionary containing the function to call and values for
|
|
190
|
-
computing the adjusted progress value.
|
|
191
|
-
Returns
|
|
192
|
-
-------
|
|
193
|
-
int
|
|
194
|
-
Flag indicating whether to continue with getting the data.
|
|
195
|
-
0 to continue. 1 to abort getting the data.
|
|
196
|
-
"""
|
|
197
|
-
progress_callback = value.get('progressCallback', None)
|
|
198
|
-
progress_user_value = value.get('progressUserValue', None)
|
|
199
|
-
adjusted_progress = value['progressStart'] + \
|
|
200
|
-
value['progressFraction'] * progress
|
|
201
|
-
|
|
202
|
-
if progress_callback is not None:
|
|
203
|
-
|
|
204
|
-
return progress_callback(adjusted_progress, msg,
|
|
205
|
-
progress_user_value)
|
|
206
|
-
return 0
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
class NullAuth(requests.auth.AuthBase): # pylint: disable=too-few-public-methods
|
|
210
|
-
"""
|
|
211
|
-
Authentication class used to cause requests to ignore any ~/.netrc
|
|
212
|
-
file. The CDAS web services do not support authentication and
|
|
213
|
-
a cdaweb (ftps) entry will cause CdasWs requests to fail with
|
|
214
|
-
a 401 error. See <https://github.com/psf/requests/issues/2773>.
|
|
215
|
-
"""
|
|
216
|
-
def __call__(self, r):
|
|
217
|
-
return r
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
class CdasWs:
|
|
221
|
-
"""
|
|
222
|
-
Class representing the web service interface to NASA's
|
|
223
|
-
Coordinated Data Analysis System (CDAS)
|
|
224
|
-
<https://cdaweb.gsfc.nasa.gov>.
|
|
225
|
-
|
|
226
|
-
Notes
|
|
227
|
-
-----
|
|
228
|
-
The logger used by this class has the class' name (CdasWs). By default,
|
|
229
|
-
it is configured with a NullHandler. Users of this class may configure
|
|
230
|
-
the logger to aid in diagnosing problems.
|
|
231
|
-
"""
|
|
232
|
-
# pylint: disable=too-many-instance-attributes
|
|
233
|
-
# pylint: disable=too-many-arguments
|
|
234
|
-
def __init__(
|
|
235
|
-
self,
|
|
236
|
-
endpoint=None,
|
|
237
|
-
timeout=None,
|
|
238
|
-
proxy=None,
|
|
239
|
-
ca_certs=None,
|
|
240
|
-
disable_ssl_certificate_validation=False,
|
|
241
|
-
user_agent=None,
|
|
242
|
-
disable_cache=False):
|
|
243
|
-
"""
|
|
244
|
-
Creates an object representing the CDAS web services.
|
|
245
|
-
|
|
246
|
-
Parameters
|
|
247
|
-
----------
|
|
248
|
-
endpoint
|
|
249
|
-
URL of the CDAS web service. If None, the default is
|
|
250
|
-
'https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/'.
|
|
251
|
-
timeout
|
|
252
|
-
Number of seconds to wait for a response from the server.
|
|
253
|
-
proxy
|
|
254
|
-
HTTP proxy information. For example,<pre>
|
|
255
|
-
proxies = {
|
|
256
|
-
'http': 'http://10.10.1.10:3128',
|
|
257
|
-
'https': 'http://10.10.1.10:1080',
|
|
258
|
-
}</pre>
|
|
259
|
-
Proxy information can also be set with environment variables.
|
|
260
|
-
For example,<pre>
|
|
261
|
-
$ export HTTP_PROXY="http://10.10.1.10:3128"
|
|
262
|
-
$ export HTTPS_PROXY="http://10.10.1.10:1080"</pre>
|
|
263
|
-
ca_certs
|
|
264
|
-
Path to certificate authority (CA) certificates that will
|
|
265
|
-
override the default bundle.
|
|
266
|
-
disable_ssl_certificate_validation
|
|
267
|
-
Flag indicating whether to validate the SSL certificate.
|
|
268
|
-
user_agent
|
|
269
|
-
A value that is appended to the HTTP User-Agent value.
|
|
270
|
-
disable_cache
|
|
271
|
-
Flag indicating whether to disable HTTP caching.
|
|
272
|
-
"""
|
|
273
|
-
|
|
274
|
-
self.logger = logging.getLogger(type(self).__name__)
|
|
275
|
-
self.logger.addHandler(logging.NullHandler())
|
|
276
|
-
|
|
277
|
-
self.logger.debug('endpoint = %s', endpoint)
|
|
278
|
-
self.logger.debug('ca_certs = %s', ca_certs)
|
|
279
|
-
self.logger.debug('disable_ssl_certificate_validation = %s',
|
|
280
|
-
disable_ssl_certificate_validation)
|
|
281
|
-
self.logger.debug('disable_cache = %s', disable_cache)
|
|
282
|
-
|
|
283
|
-
if endpoint is None:
|
|
284
|
-
self._endpoint = 'https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/'
|
|
285
|
-
else:
|
|
286
|
-
self._endpoint = endpoint
|
|
287
|
-
|
|
288
|
-
self._user_agent = 'cdasws/' + __version__ + ' (' + \
|
|
289
|
-
platform.python_implementation() + ' ' \
|
|
290
|
-
+ platform.python_version() + '; ' + platform.platform() + ')'
|
|
291
|
-
|
|
292
|
-
if user_agent is not None:
|
|
293
|
-
self._user_agent += ' (' + user_agent + ')'
|
|
294
|
-
|
|
295
|
-
self._request_headers = {
|
|
296
|
-
#'Content-Type' : 'application/json',
|
|
297
|
-
'Content-Type' : 'application/xml',
|
|
298
|
-
'Accept' : 'application/xml',
|
|
299
|
-
'User-Agent' : self._user_agent,
|
|
300
|
-
#'Accept-Encoding' : 'gzip' # only beneficial for icdfml responses
|
|
301
|
-
}
|
|
302
|
-
if CACHE_AVAILABLE and disable_cache is not True:
|
|
303
|
-
self._session = requests_cache.CachedSession('cdasws_cache',
|
|
304
|
-
cache_control=True)
|
|
305
|
-
else:
|
|
306
|
-
self._session = requests.Session()
|
|
307
|
-
|
|
308
|
-
self._session.headers.update(self._request_headers)
|
|
309
|
-
self._session.auth = NullAuth()
|
|
310
|
-
|
|
311
|
-
if ca_certs is not None:
|
|
312
|
-
self._session.verify = ca_certs
|
|
313
|
-
|
|
314
|
-
if disable_ssl_certificate_validation is True:
|
|
315
|
-
self._session.verify = False
|
|
316
|
-
|
|
317
|
-
if proxy is not None:
|
|
318
|
-
self._proxy = proxy
|
|
319
|
-
|
|
320
|
-
self._timeout = timeout
|
|
321
|
-
|
|
322
|
-
endpoint_components = urlparse(self._endpoint)
|
|
323
|
-
self._hdp_registry = endpoint_components.scheme + '://' + \
|
|
324
|
-
endpoint_components.netloc + '/registry/hdp/SscId.xql'
|
|
325
|
-
|
|
326
|
-
# pylint: enable=too-many-arguments
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
def __del__(self):
|
|
330
|
-
"""
|
|
331
|
-
Destructor. Closes all network connections.
|
|
332
|
-
"""
|
|
333
|
-
|
|
334
|
-
self.close()
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
def close(self) -> None:
|
|
338
|
-
"""
|
|
339
|
-
Closes any persistent network connections. Generally, deleting
|
|
340
|
-
this object is sufficient and calling this method is unnecessary.
|
|
341
|
-
"""
|
|
342
|
-
self._session.close()
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
def get_observatory_groups(
|
|
346
|
-
self,
|
|
347
|
-
**keywords: str
|
|
348
|
-
) -> List[Dict]:
|
|
349
|
-
"""
|
|
350
|
-
Gets descriptions of the observatory groups from the server.
|
|
351
|
-
|
|
352
|
-
Parameters
|
|
353
|
-
----------
|
|
354
|
-
keywords
|
|
355
|
-
optional keyword parameters as follows:<br>
|
|
356
|
-
<b>instrumentType</b> - an instrument type value from those
|
|
357
|
-
returned by `CdasWs.get_instrument_types`. Omitting
|
|
358
|
-
this parameter indicates that no observatories are eliminated
|
|
359
|
-
based upon their instrumentType value.
|
|
360
|
-
Returns
|
|
361
|
-
-------
|
|
362
|
-
List
|
|
363
|
-
An array of ObservatoryGroupDescription
|
|
364
|
-
dictionaries where the structure of the dictionary mirrors
|
|
365
|
-
ObservatoryGroupDescription in
|
|
366
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
367
|
-
"""
|
|
368
|
-
if 'instrumentType' in keywords:
|
|
369
|
-
url = self._endpoint + 'observatoryGroups?instrumentType=' + \
|
|
370
|
-
urllib.parse.quote(keywords['instrumentType'])
|
|
371
|
-
else:
|
|
372
|
-
url = self._endpoint + 'observatoryGroups'
|
|
373
|
-
|
|
374
|
-
self.logger.debug('request url = %s', url)
|
|
375
|
-
|
|
376
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
377
|
-
|
|
378
|
-
if response.status_code != 200:
|
|
379
|
-
|
|
380
|
-
self.logger.info('%s failed with http code %d', url,
|
|
381
|
-
response.status_code)
|
|
382
|
-
self.logger.info('response.text: %s', response.text)
|
|
383
|
-
return []
|
|
384
|
-
|
|
385
|
-
if self.logger.level <= logging.DEBUG:
|
|
386
|
-
self.logger.debug('response.text = %s', response.text)
|
|
387
|
-
|
|
388
|
-
observatory_response = ET.fromstring(response.text)
|
|
389
|
-
|
|
390
|
-
observatory_group_descriptions = []
|
|
391
|
-
for description in observatory_response.findall(\
|
|
392
|
-
'cdas:ObservatoryGroupDescription', namespaces=NAMESPACES):
|
|
393
|
-
|
|
394
|
-
observatory_ids = []
|
|
395
|
-
for observatory_id in description.findall(\
|
|
396
|
-
'cdas:ObservatoryId', namespaces=NAMESPACES):
|
|
397
|
-
|
|
398
|
-
observatory_ids.append(observatory_id.text)
|
|
399
|
-
|
|
400
|
-
observatory_group_descriptions.append({
|
|
401
|
-
'Name': description.find(\
|
|
402
|
-
'cdas:Name', namespaces=NAMESPACES).text,
|
|
403
|
-
'ObservatoryId': observatory_ids
|
|
404
|
-
})
|
|
405
|
-
|
|
406
|
-
return observatory_group_descriptions
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
def get_instrument_types(
|
|
410
|
-
self,
|
|
411
|
-
**keywords: str
|
|
412
|
-
) -> List[Dict]:
|
|
413
|
-
"""
|
|
414
|
-
Gets descriptions of the instrument types from the server.
|
|
415
|
-
|
|
416
|
-
Parameters
|
|
417
|
-
----------
|
|
418
|
-
keywords
|
|
419
|
-
optional keyword parameters as follows:<br>
|
|
420
|
-
<b>observatory</b> - an observatory value from those returned
|
|
421
|
-
by `CdasWs.get_observatories`. Omitting this parameter
|
|
422
|
-
indicates that no instrumentTypes are eliminated based upon
|
|
423
|
-
their observatory value.<br>
|
|
424
|
-
<b>observatoryGroup</b> - an observatory group value from
|
|
425
|
-
those returned by `CdasWs.get_observatory_groups`. Omitting
|
|
426
|
-
this parameter indicates that no instrumentTypes are
|
|
427
|
-
eliminated based upon their observatoryGroup value.</br>
|
|
428
|
-
Returns
|
|
429
|
-
-------
|
|
430
|
-
List
|
|
431
|
-
An array of InstrumentTypeDescription
|
|
432
|
-
dictionaries where the structure of the dictionary mirrors
|
|
433
|
-
InstrumentTypeDescription in
|
|
434
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
435
|
-
"""
|
|
436
|
-
if 'observatory' in keywords:
|
|
437
|
-
url = self._endpoint + 'instrumentTypes?observatory=' \
|
|
438
|
-
+ urllib.parse.quote(keywords['observatory'])
|
|
439
|
-
elif 'observatoryGroup' in keywords:
|
|
440
|
-
url = self._endpoint + 'instrumentTypes?observatoryGroup=' \
|
|
441
|
-
+ urllib.parse.quote(keywords['observatoryGroup'])
|
|
442
|
-
else:
|
|
443
|
-
url = self._endpoint + 'instrumentTypes'
|
|
444
|
-
|
|
445
|
-
self.logger.debug('request url = %s', url)
|
|
446
|
-
|
|
447
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
448
|
-
|
|
449
|
-
if response.status_code != 200:
|
|
450
|
-
|
|
451
|
-
self.logger.info('%s failed with http code %d', url,
|
|
452
|
-
response.status_code)
|
|
453
|
-
self.logger.info('response.text: %s', response.text)
|
|
454
|
-
return []
|
|
455
|
-
|
|
456
|
-
if self.logger.level <= logging.DEBUG:
|
|
457
|
-
self.logger.debug('response.text = %s', response.text)
|
|
458
|
-
|
|
459
|
-
instrument_response = ET.fromstring(response.text)
|
|
460
|
-
|
|
461
|
-
if self.logger.level <= logging.DEBUG:
|
|
462
|
-
self.logger.debug('instrument_response = %s',
|
|
463
|
-
ET.tostring(instrument_response))
|
|
464
|
-
|
|
465
|
-
instrument_types = []
|
|
466
|
-
for description in instrument_response.findall(\
|
|
467
|
-
'cdas:InstrumentTypeDescription', namespaces=NAMESPACES):
|
|
468
|
-
|
|
469
|
-
instrument_types.append({
|
|
470
|
-
'Name': description.find('cdas:Name',
|
|
471
|
-
namespaces=NAMESPACES).text
|
|
472
|
-
})
|
|
473
|
-
return instrument_types
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
def get_instruments(
|
|
477
|
-
self,
|
|
478
|
-
**keywords: str
|
|
479
|
-
) -> List[Dict]:
|
|
480
|
-
"""
|
|
481
|
-
Gets descriptions of the instruments from the server.
|
|
482
|
-
|
|
483
|
-
Parameters
|
|
484
|
-
----------
|
|
485
|
-
keywords
|
|
486
|
-
optional keyword parameters as follows:<br>
|
|
487
|
-
<b>observatory</b> - an observatory value from those returned
|
|
488
|
-
by `CdasWs.get_observatories`. Omitting this parameter
|
|
489
|
-
indicates that no instruments are eliminated based upon their
|
|
490
|
-
observatory value.<br>
|
|
491
|
-
<b>instrumentType</b> - an instrument type value from those
|
|
492
|
-
returned by `CdasWs.get_instrument_types`. Omitting this
|
|
493
|
-
parameter indicates that no instruments are eliminated based
|
|
494
|
-
upon their instrument type.<br>
|
|
495
|
-
Returns
|
|
496
|
-
-------
|
|
497
|
-
List
|
|
498
|
-
An array of InstrumentDescription
|
|
499
|
-
dictionaries where the structure of the dictionary mirrors
|
|
500
|
-
InstrumentDescription in
|
|
501
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
502
|
-
"""
|
|
503
|
-
if 'observatory' in keywords:
|
|
504
|
-
url = self._endpoint + 'instruments?observatory=' \
|
|
505
|
-
+ urllib.parse.quote(keywords['observatory'])
|
|
506
|
-
elif 'instrumentType' in keywords:
|
|
507
|
-
url = self._endpoint + 'instruments?instrumentType=' \
|
|
508
|
-
+ urllib.parse.quote(keywords['instrumentType'])
|
|
509
|
-
else:
|
|
510
|
-
url = self._endpoint + 'instruments'
|
|
511
|
-
|
|
512
|
-
self.logger.debug('request url = %s', url)
|
|
513
|
-
|
|
514
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
515
|
-
|
|
516
|
-
if response.status_code != 200:
|
|
517
|
-
|
|
518
|
-
self.logger.info('%s failed with http code %d', url,
|
|
519
|
-
response.status_code)
|
|
520
|
-
self.logger.info('response.text: %s', response.text)
|
|
521
|
-
return []
|
|
522
|
-
|
|
523
|
-
if self.logger.level <= logging.DEBUG:
|
|
524
|
-
self.logger.debug('response.text = %s', response.text)
|
|
525
|
-
|
|
526
|
-
instruments_response = ET.fromstring(response.text)
|
|
527
|
-
|
|
528
|
-
if self.logger.level <= logging.DEBUG:
|
|
529
|
-
self.logger.debug('instruments = %s', response.text)
|
|
530
|
-
#ET.indent(instruments_response, space=' '))
|
|
531
|
-
|
|
532
|
-
instruments = []
|
|
533
|
-
for instrument_description in instruments_response.findall(\
|
|
534
|
-
'cdas:InstrumentDescription', namespaces=NAMESPACES):
|
|
535
|
-
|
|
536
|
-
instruments.append({
|
|
537
|
-
'Name': instrument_description.find(\
|
|
538
|
-
'cdas:Name', namespaces=NAMESPACES).text,
|
|
539
|
-
'ShortDescription': instrument_description.find(\
|
|
540
|
-
'cdas:ShortDescription', namespaces=NAMESPACES).text,
|
|
541
|
-
'LongDescription': instrument_description.find(\
|
|
542
|
-
'cdas:LongDescription', namespaces=NAMESPACES).text
|
|
543
|
-
})
|
|
544
|
-
|
|
545
|
-
return instruments
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
def get_observatories(
|
|
549
|
-
self,
|
|
550
|
-
**keywords: str
|
|
551
|
-
) -> List[Dict]:
|
|
552
|
-
"""
|
|
553
|
-
Gets descriptions of the observatories from the server.
|
|
554
|
-
|
|
555
|
-
Parameters
|
|
556
|
-
----------
|
|
557
|
-
keywords
|
|
558
|
-
optional keyword parameters as follows:<br>
|
|
559
|
-
<b>instrument</b> - an instrument value from those returned
|
|
560
|
-
by `CdasWs.get_instruments`. Omitting this parameter
|
|
561
|
-
indicates that no observatories are eliminated based upon
|
|
562
|
-
their instrument value.<br>
|
|
563
|
-
<b>instrumentType</b> - in instrument type value from those
|
|
564
|
-
returned by `CdasWs.get_instrument_types`. Omitting this
|
|
565
|
-
parameter indicates that no observatories are eliminated
|
|
566
|
-
based upon their instrumentType value.<br>
|
|
567
|
-
Returns
|
|
568
|
-
-------
|
|
569
|
-
List
|
|
570
|
-
An array of ObservatoryDescriptions
|
|
571
|
-
dictionaries where the structure of the dictionary mirrors
|
|
572
|
-
ObservatoryDescription in
|
|
573
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
574
|
-
"""
|
|
575
|
-
if 'instrument' in keywords:
|
|
576
|
-
url = self._endpoint + 'observatories?instrument=' \
|
|
577
|
-
+ urllib.parse.quote(keywords['instrument'])
|
|
578
|
-
elif 'instrumentType' in keywords:
|
|
579
|
-
url = self._endpoint + 'observatories?instrumentType=' \
|
|
580
|
-
+ urllib.parse.quote(keywords['instrumentType'])
|
|
581
|
-
else:
|
|
582
|
-
url = self._endpoint + 'observatories'
|
|
583
|
-
|
|
584
|
-
self.logger.debug('request url = %s', url)
|
|
585
|
-
|
|
586
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
587
|
-
|
|
588
|
-
if response.status_code != 200:
|
|
589
|
-
|
|
590
|
-
self.logger.info('%s failed with http code %d', url,
|
|
591
|
-
response.status_code)
|
|
592
|
-
self.logger.info('response.text: %s', response.text)
|
|
593
|
-
return []
|
|
594
|
-
|
|
595
|
-
if self.logger.level <= logging.DEBUG:
|
|
596
|
-
self.logger.debug('response.text = %s', response.text)
|
|
597
|
-
|
|
598
|
-
observatory_response = ET.fromstring(response.text)
|
|
599
|
-
|
|
600
|
-
if self.logger.level <= logging.DEBUG:
|
|
601
|
-
self.logger.debug('observatories = %s', response.text)
|
|
602
|
-
|
|
603
|
-
observatories = []
|
|
604
|
-
|
|
605
|
-
for observatory in observatory_response.findall(\
|
|
606
|
-
'cdas:ObservatoryDescription', namespaces=NAMESPACES):
|
|
607
|
-
observatories.append({
|
|
608
|
-
'Name': observatory.find(\
|
|
609
|
-
'cdas:Name', namespaces=NAMESPACES).text,
|
|
610
|
-
'ShortDescription': observatory.find(\
|
|
611
|
-
'cdas:ShortDescription', namespaces=NAMESPACES).text,
|
|
612
|
-
'LongDescription': observatory.find(\
|
|
613
|
-
'cdas:LongDescription', namespaces=NAMESPACES).text
|
|
614
|
-
})
|
|
615
|
-
|
|
616
|
-
return observatories
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
def get_observatory_groups_and_instruments(
|
|
620
|
-
self,
|
|
621
|
-
**keywords: str
|
|
622
|
-
) -> List[Dict]:
|
|
623
|
-
"""
|
|
624
|
-
Gets descriptions of the observatory groups (and associated
|
|
625
|
-
instruments) from the server.
|
|
626
|
-
|
|
627
|
-
Parameters
|
|
628
|
-
----------
|
|
629
|
-
keywords
|
|
630
|
-
optional keyword parameters as follows:<br>
|
|
631
|
-
<b>instrumentType</b> - an instrument type value from those
|
|
632
|
-
returned by `CdasWs.get_instrument_types`. Omitting this
|
|
633
|
-
parameter indicates that no observatories are eliminated
|
|
634
|
-
based upon their instrumentType value.<br>
|
|
635
|
-
Returns
|
|
636
|
-
-------
|
|
637
|
-
List
|
|
638
|
-
An array of ObservatoryGroupInstrumentDescription
|
|
639
|
-
dictionaries where the structure of the dictionary mirrors
|
|
640
|
-
ObservatoryGroupInstrumentDescription in
|
|
641
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
642
|
-
"""
|
|
643
|
-
if 'instrumentType' in keywords:
|
|
644
|
-
url = self._endpoint \
|
|
645
|
-
+ 'observatoryGroupsAndInstruments?instrumentType=' \
|
|
646
|
-
+ urllib.parse.quote(keywords['instrumentType'])
|
|
647
|
-
else:
|
|
648
|
-
url = self._endpoint + 'observatoryGroupsAndInstruments'
|
|
649
|
-
|
|
650
|
-
self.logger.debug('request url = %s', url)
|
|
651
|
-
|
|
652
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
653
|
-
|
|
654
|
-
if response.status_code != 200:
|
|
655
|
-
|
|
656
|
-
self.logger.info('%s failed with http code %d', url,
|
|
657
|
-
response.status_code)
|
|
658
|
-
self.logger.info('response.text: %s', response.text)
|
|
659
|
-
return []
|
|
660
|
-
|
|
661
|
-
if self.logger.level <= logging.DEBUG:
|
|
662
|
-
self.logger.debug('response.text = %s', response.text)
|
|
663
|
-
|
|
664
|
-
observatories_response = ET.fromstring(response.text)
|
|
665
|
-
|
|
666
|
-
if self.logger.level <= logging.DEBUG:
|
|
667
|
-
self.logger.debug('observatories = %s', response.text)
|
|
668
|
-
|
|
669
|
-
o_g_i_ds = []
|
|
670
|
-
|
|
671
|
-
for o_g_i_d in observatories_response.findall(\
|
|
672
|
-
'cdas:ObservatoryGroupInstrumentDescription',\
|
|
673
|
-
namespaces=NAMESPACES):
|
|
674
|
-
|
|
675
|
-
o_g_i_d_name = o_g_i_d.find('cdas:Name',
|
|
676
|
-
namespaces=NAMESPACES).text
|
|
677
|
-
o_is = []
|
|
678
|
-
for o_i in o_g_i_d.findall('cdas:ObservatoryInstruments',
|
|
679
|
-
namespaces=NAMESPACES):
|
|
680
|
-
|
|
681
|
-
o_i_name = o_i.find('cdas:Name',
|
|
682
|
-
namespaces=NAMESPACES).text
|
|
683
|
-
i_ds = []
|
|
684
|
-
for i_d in o_i.findall('cdas:InstrumentDescription',
|
|
685
|
-
namespaces=NAMESPACES):
|
|
686
|
-
i_d_name = i_d.find('cdas:Name',
|
|
687
|
-
namespaces=NAMESPACES).text
|
|
688
|
-
i_d_short_description = \
|
|
689
|
-
i_d.find('cdas:ShortDescription',
|
|
690
|
-
namespaces=NAMESPACES).text
|
|
691
|
-
i_d_long_description = \
|
|
692
|
-
i_d.find('cdas:LongDescription',
|
|
693
|
-
namespaces=NAMESPACES).text
|
|
694
|
-
i_ds.append({
|
|
695
|
-
'Name': i_d_name,
|
|
696
|
-
'ShortDescription': i_d_short_description,
|
|
697
|
-
'LongDescription': i_d_long_description
|
|
698
|
-
})
|
|
699
|
-
o_is.append({
|
|
700
|
-
'Name': o_i_name,
|
|
701
|
-
'InstrumentDescription': i_ds
|
|
702
|
-
})
|
|
703
|
-
|
|
704
|
-
o_g_i_ds.append({
|
|
705
|
-
'Name': o_g_i_d_name,
|
|
706
|
-
'ObservatoryInstruments': o_is
|
|
707
|
-
})
|
|
708
|
-
|
|
709
|
-
return o_g_i_ds
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
# pylint: disable=too-many-branches
|
|
713
|
-
def get_datasets(
|
|
714
|
-
self,
|
|
715
|
-
**keywords: str
|
|
716
|
-
) -> List[Dict]:
|
|
717
|
-
"""
|
|
718
|
-
Gets descriptions of the specified datasets from the server.
|
|
719
|
-
|
|
720
|
-
Parameters
|
|
721
|
-
----------
|
|
722
|
-
keywords
|
|
723
|
-
optional keyword parameters as follows:<br>
|
|
724
|
-
<b>observatoryGroup</b> - an observatory group value from those
|
|
725
|
-
returned by `CdasWs.get_observatory_groups`. Omitting this
|
|
726
|
-
parameter
|
|
727
|
-
indicates that no datasets are eliminated based upon their
|
|
728
|
-
observatoryGroup value.<br>
|
|
729
|
-
<b>instrumentType</b> - an instrument type value from those
|
|
730
|
-
returned by `CdasWs.get_instrument_types`. Omitting this
|
|
731
|
-
parameter indicates that no datasets are eliminated based
|
|
732
|
-
upon their instrumentType value.<br>
|
|
733
|
-
<b>observatory</b> - an observatory name value from those
|
|
734
|
-
returned by `CdasWs.get_observatories`. Omitting this
|
|
735
|
-
parameter indicates that no datasets are eliminated based
|
|
736
|
-
upon their observatory value.<br>
|
|
737
|
-
<b>instrument</b> - an instrument value from those returned by
|
|
738
|
-
`CdasWs.get_instruments`. Omitting this parameter indicates
|
|
739
|
-
that no datasets are eliminated based upon their instrument
|
|
740
|
-
value.<br>
|
|
741
|
-
<b>startDate</b> - a datetime specifying the start of a time
|
|
742
|
-
interval. See module note about timezone value. If this
|
|
743
|
-
parameter is ommited, the time interval will begin infinitely
|
|
744
|
-
in the past.<br>
|
|
745
|
-
<b>stopDate</b> - a datetime specifying the end of a time
|
|
746
|
-
interval. See module note about timezone value. If this
|
|
747
|
-
parameter is omitted, the time interval will end infinitely
|
|
748
|
-
in the future.<br>
|
|
749
|
-
<b>id</b> - a dataset identifier. The value may be a CDAS
|
|
750
|
-
(e.g., AC_H2_MFI), DOI (e.g., 10.48322/fh85-fj47), or SPASE
|
|
751
|
-
[ResourceID] (e.g., spase://NASA/NumericalData/ACE/MAG/L2/PT1H)
|
|
752
|
-
identifier. If specified, all other keywords are ignored.<br>
|
|
753
|
-
<b>idPattern</b> - a java.util.regex compatible regular
|
|
754
|
-
expression that must match the dataset's CDAS identifier value.
|
|
755
|
-
Omitting this parameter is equivalent to `.*`.<br>
|
|
756
|
-
<b>labelPattern</b> - a java.util.regex compatible regular
|
|
757
|
-
expression that must match the dataset's CDAS label text.
|
|
758
|
-
Omitting this parameter is equivalent to `.*`. Embedded
|
|
759
|
-
matching flag expressions (e.g., `(?i)` for case insensitive
|
|
760
|
-
match mode) are supported and likely to be useful in this
|
|
761
|
-
case.<br>
|
|
762
|
-
<b>notesPattern</b> - a java.util.regex compatible regular
|
|
763
|
-
expression that must match the dataset's CDAS notes text.
|
|
764
|
-
Omitting this parameter is equivalent to `.*`. Embedded
|
|
765
|
-
matching flag expressions (e.g., `(?s)` for dotall match mode)
|
|
766
|
-
are supported and likely to be useful in this case.<br>
|
|
767
|
-
Returns
|
|
768
|
-
-------
|
|
769
|
-
List
|
|
770
|
-
A list of dictionaries containing descriptions of the datasets
|
|
771
|
-
requested. The dictionary structure is defined by the
|
|
772
|
-
DatasetDescription element in
|
|
773
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
774
|
-
"""
|
|
775
|
-
url = self._endpoint + 'datasets?'
|
|
776
|
-
|
|
777
|
-
observatory_groups = keywords.get('observatoryGroup', None)
|
|
778
|
-
if observatory_groups is not None:
|
|
779
|
-
if isinstance(observatory_groups, str):
|
|
780
|
-
observatory_groups = [observatory_groups]
|
|
781
|
-
for observatory_group in observatory_groups:
|
|
782
|
-
url = url + 'observatoryGroup=' \
|
|
783
|
-
+ urllib.parse.quote(observatory_group) + '&'
|
|
784
|
-
|
|
785
|
-
instrument_types = keywords.get('instrumentType', None)
|
|
786
|
-
if instrument_types is not None:
|
|
787
|
-
if isinstance(instrument_types, str):
|
|
788
|
-
instrument_types = [instrument_types]
|
|
789
|
-
for instrument_type in instrument_types:
|
|
790
|
-
url = url + 'instrumentType=' \
|
|
791
|
-
+ urllib.parse.quote(instrument_type) + '&'
|
|
792
|
-
|
|
793
|
-
observatories = keywords.get('observatory', None)
|
|
794
|
-
if observatories is not None:
|
|
795
|
-
if isinstance(observatories, str):
|
|
796
|
-
observatories = [observatories]
|
|
797
|
-
for observatory in observatories:
|
|
798
|
-
url = url + 'observatory=' \
|
|
799
|
-
+ urllib.parse.quote(observatory) + '&'
|
|
800
|
-
|
|
801
|
-
instruments = keywords.get('instrument', None)
|
|
802
|
-
if instruments is not None:
|
|
803
|
-
if isinstance(instruments, str):
|
|
804
|
-
instruments = [instruments]
|
|
805
|
-
for instrument in instruments:
|
|
806
|
-
url = url + 'instrument=' \
|
|
807
|
-
+ urllib.parse.quote(instrument) + '&'
|
|
808
|
-
|
|
809
|
-
if 'startDate' in keywords:
|
|
810
|
-
url = url + 'startDate=' \
|
|
811
|
-
+ urllib.parse.quote(keywords['startDate']) + '&'
|
|
812
|
-
|
|
813
|
-
if 'stopDate' in keywords:
|
|
814
|
-
url = url + 'stopDate=' \
|
|
815
|
-
+ urllib.parse.quote(keywords['stopDate']) + '&'
|
|
816
|
-
|
|
817
|
-
if 'id' in keywords:
|
|
818
|
-
url = url + 'id=' \
|
|
819
|
-
+ urllib.parse.quote(keywords['id']) + '&'
|
|
820
|
-
|
|
821
|
-
if 'idPattern' in keywords:
|
|
822
|
-
url = url + 'idPattern=' \
|
|
823
|
-
+ urllib.parse.quote(keywords['idPattern']) + '&'
|
|
824
|
-
|
|
825
|
-
if 'labelPattern' in keywords:
|
|
826
|
-
url = url + 'labelPattern=' \
|
|
827
|
-
+ urllib.parse.quote(keywords['labelPattern']) + '&'
|
|
828
|
-
|
|
829
|
-
if 'notesPattern' in keywords:
|
|
830
|
-
url = url + 'notesPattern=' \
|
|
831
|
-
+ urllib.parse.quote(keywords['notesPattern']) + '&'
|
|
832
|
-
|
|
833
|
-
self.logger.debug('request url = %s', url[:-1])
|
|
834
|
-
|
|
835
|
-
response = self._session.get(url[:-1], timeout=self._timeout)
|
|
836
|
-
|
|
837
|
-
if response.status_code != 200:
|
|
838
|
-
|
|
839
|
-
self.logger.info('%s failed with http code %d', url,
|
|
840
|
-
response.status_code)
|
|
841
|
-
self.logger.info('response.text: %s', response.text)
|
|
842
|
-
return []
|
|
843
|
-
|
|
844
|
-
if self.logger.level <= logging.DEBUG:
|
|
845
|
-
self.logger.debug('response.text = %s', response.text)
|
|
846
|
-
|
|
847
|
-
dss = ET.fromstring(response.text)
|
|
848
|
-
|
|
849
|
-
if self.logger.level <= logging.DEBUG:
|
|
850
|
-
self.logger.debug('datasets = %s', response.text)
|
|
851
|
-
|
|
852
|
-
datasets = []
|
|
853
|
-
for ds in dss.findall('cdas:DatasetDescription',
|
|
854
|
-
namespaces=NAMESPACES):
|
|
855
|
-
|
|
856
|
-
observatory_groups = []
|
|
857
|
-
for o_g in ds.findall('cdas:ObservatoryGroup',
|
|
858
|
-
namespaces=NAMESPACES):
|
|
859
|
-
observatory_groups.append(o_g.text)
|
|
860
|
-
|
|
861
|
-
instrument_types = []
|
|
862
|
-
for i_t in ds.findall('cdas:InstrumentType',
|
|
863
|
-
namespaces=NAMESPACES):
|
|
864
|
-
instrument_types.append(i_t.text)
|
|
865
|
-
|
|
866
|
-
dataset_links = []
|
|
867
|
-
for d_l in ds.findall('cdas:DatasetLink',
|
|
868
|
-
namespaces=NAMESPACES):
|
|
869
|
-
dataset_links.append({
|
|
870
|
-
'Title': d_l.find('cdas:Title',
|
|
871
|
-
namespaces=NAMESPACES).text,
|
|
872
|
-
'Text': d_l.find('cdas:Text',
|
|
873
|
-
namespaces=NAMESPACES).text,
|
|
874
|
-
'Url': d_l.find('cdas:Url',
|
|
875
|
-
namespaces=NAMESPACES).text,
|
|
876
|
-
})
|
|
877
|
-
|
|
878
|
-
observatories = []
|
|
879
|
-
for obs_elem in ds.findall('cdas:Observatory',
|
|
880
|
-
namespaces=NAMESPACES):
|
|
881
|
-
observatories.append(obs_elem.text)
|
|
882
|
-
|
|
883
|
-
instruments = []
|
|
884
|
-
for instr_elem in ds.findall('cdas:Instrument',
|
|
885
|
-
namespaces=NAMESPACES):
|
|
886
|
-
instruments.append(instr_elem.text)
|
|
887
|
-
|
|
888
|
-
dataset = {
|
|
889
|
-
'Id': ds.find('cdas:Id', namespaces=NAMESPACES).text,
|
|
890
|
-
'Observatory': observatories,
|
|
891
|
-
'Instrument': instruments,
|
|
892
|
-
'ObservatoryGroup': observatory_groups,
|
|
893
|
-
'InstrumentType': instrument_types,
|
|
894
|
-
'Label': ds.find('cdas:Label',
|
|
895
|
-
namespaces=NAMESPACES).text,
|
|
896
|
-
'TimeInterval': {
|
|
897
|
-
'Start': ds.find('cdas:TimeInterval/cdas:Start',
|
|
898
|
-
namespaces=NAMESPACES).text,
|
|
899
|
-
'End': ds.find('cdas:TimeInterval/cdas:End',
|
|
900
|
-
namespaces=NAMESPACES).text
|
|
901
|
-
},
|
|
902
|
-
'PiName': ds.find('cdas:PiName',
|
|
903
|
-
namespaces=NAMESPACES).text,
|
|
904
|
-
'PiAffiliation': ds.find('cdas:PiAffiliation',
|
|
905
|
-
namespaces=NAMESPACES).text,
|
|
906
|
-
'Notes': ds.find('cdas:Notes',
|
|
907
|
-
namespaces=NAMESPACES).text,
|
|
908
|
-
'DatasetLink': dataset_links
|
|
909
|
-
}
|
|
910
|
-
doi = ds.find('cdas:Doi', namespaces=NAMESPACES)
|
|
911
|
-
if doi is not None:
|
|
912
|
-
dataset['Doi'] = doi.text
|
|
913
|
-
|
|
914
|
-
spase_resource_id = ds.find('cdas:SpaseResourceId',
|
|
915
|
-
namespaces=NAMESPACES)
|
|
916
|
-
if spase_resource_id is not None:
|
|
917
|
-
dataset['SpaseResourceId'] = spase_resource_id.text
|
|
918
|
-
|
|
919
|
-
additional_metadata = []
|
|
920
|
-
for add_meta in ds.findall('cdas:AdditionalMetadata',
|
|
921
|
-
namespaces=NAMESPACES):
|
|
922
|
-
meta_type = add_meta.attrib['Type']
|
|
923
|
-
value = add_meta.text
|
|
924
|
-
additional_metadata.append({
|
|
925
|
-
'Type': meta_type,
|
|
926
|
-
'value': value
|
|
927
|
-
})
|
|
928
|
-
|
|
929
|
-
if len(additional_metadata) > 0:
|
|
930
|
-
dataset['AdditionalMetadata'] = additional_metadata
|
|
931
|
-
|
|
932
|
-
datasets.append(dataset)
|
|
933
|
-
|
|
934
|
-
return sorted(datasets, key=itemgetter('Id'))
|
|
935
|
-
# pylint: enable=too-many-branches
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
@staticmethod
|
|
939
|
-
def get_doi_landing_page_url(
|
|
940
|
-
doi: str
|
|
941
|
-
) -> str:
|
|
942
|
-
"""
|
|
943
|
-
Returns a URL to the given Digital Object Identifier's landing
|
|
944
|
-
page (metadata for the DOI).
|
|
945
|
-
|
|
946
|
-
Parameters
|
|
947
|
-
----------
|
|
948
|
-
doi
|
|
949
|
-
digital object identifier.
|
|
950
|
-
Returns
|
|
951
|
-
-------
|
|
952
|
-
str
|
|
953
|
-
A URL to the DOI's landing page.
|
|
954
|
-
"""
|
|
955
|
-
|
|
956
|
-
if not doi.startswith('http'):
|
|
957
|
-
return 'https://doi.org/' + doi
|
|
958
|
-
return doi
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
@staticmethod
|
|
962
|
-
def get_citation(
|
|
963
|
-
doi: str
|
|
964
|
-
) -> str:
|
|
965
|
-
"""
|
|
966
|
-
Returns the citation from doi.org for the given DOI.
|
|
967
|
-
|
|
968
|
-
Parameters
|
|
969
|
-
----------
|
|
970
|
-
doi
|
|
971
|
-
digital object identifier.
|
|
972
|
-
Returns
|
|
973
|
-
-------
|
|
974
|
-
str
|
|
975
|
-
The citation from doi.org for the given DOI.
|
|
976
|
-
"""
|
|
977
|
-
|
|
978
|
-
url = 'https://doi.org/' + doi
|
|
979
|
-
headers = {'Accept': 'text/x-bibliography; style=apa'}
|
|
980
|
-
response = requests.get(url, headers=headers,
|
|
981
|
-
timeout=30)
|
|
982
|
-
|
|
983
|
-
return response.text
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
def get_inventory(
|
|
987
|
-
self,
|
|
988
|
-
identifier: str,
|
|
989
|
-
**keywords: str
|
|
990
|
-
) -> List[TimeInterval]:
|
|
991
|
-
"""
|
|
992
|
-
Gets a description of the specified dataset's data inventory.
|
|
993
|
-
|
|
994
|
-
Parameters
|
|
995
|
-
----------
|
|
996
|
-
identifier
|
|
997
|
-
dataset identifier of data inventory to get.
|
|
998
|
-
keywords
|
|
999
|
-
optional keyword parameters as follows:<br>
|
|
1000
|
-
<b>timeInterval</b> - `timeinterval.TimeInterval` to restrict
|
|
1001
|
-
returned inventory.
|
|
1002
|
-
Returns
|
|
1003
|
-
-------
|
|
1004
|
-
List
|
|
1005
|
-
An array of `timeinterval.TimeInterval`s when data is
|
|
1006
|
-
available.
|
|
1007
|
-
"""
|
|
1008
|
-
|
|
1009
|
-
url = self._endpoint + 'datasets/' + \
|
|
1010
|
-
urllib.parse.quote(identifier, safe='') + '/inventory'
|
|
1011
|
-
|
|
1012
|
-
if 'timeInterval' in keywords:
|
|
1013
|
-
time_interval_keyword = keywords['timeInterval']
|
|
1014
|
-
url = url + '/' + \
|
|
1015
|
-
TimeInterval.basic_iso_format(time_interval_keyword.start) + \
|
|
1016
|
-
',' + \
|
|
1017
|
-
TimeInterval.basic_iso_format(time_interval_keyword.end)
|
|
1018
|
-
|
|
1019
|
-
self.logger.debug('request url = %s', url)
|
|
1020
|
-
|
|
1021
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
1022
|
-
|
|
1023
|
-
if response.status_code != 200:
|
|
1024
|
-
|
|
1025
|
-
self.logger.info('%s failed with http code %d', url,
|
|
1026
|
-
response.status_code)
|
|
1027
|
-
self.logger.info('response.text: %s', response.text)
|
|
1028
|
-
return []
|
|
1029
|
-
|
|
1030
|
-
if self.logger.level <= logging.DEBUG:
|
|
1031
|
-
self.logger.debug('response.text = %s', response.text)
|
|
1032
|
-
|
|
1033
|
-
inventory = ET.fromstring(response.text)
|
|
1034
|
-
intervals = []
|
|
1035
|
-
for inventory_desc in inventory.findall(\
|
|
1036
|
-
'cdas:InventoryDescription',
|
|
1037
|
-
namespaces=NAMESPACES):
|
|
1038
|
-
for time_interval in inventory_desc.findall(\
|
|
1039
|
-
'cdas:TimeInterval',
|
|
1040
|
-
namespaces=NAMESPACES):
|
|
1041
|
-
intervals.append(
|
|
1042
|
-
TimeInterval(
|
|
1043
|
-
time_interval.find('cdas:Start',
|
|
1044
|
-
namespaces=NAMESPACES).text,
|
|
1045
|
-
time_interval.find('cdas:End',
|
|
1046
|
-
namespaces=NAMESPACES).text
|
|
1047
|
-
)
|
|
1048
|
-
)
|
|
1049
|
-
|
|
1050
|
-
return intervals
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
def get_example_time_interval(
|
|
1054
|
-
self,
|
|
1055
|
-
identifier: str,
|
|
1056
|
-
) -> TimeInterval:
|
|
1057
|
-
"""
|
|
1058
|
-
Gets a small example time interval for the specified dataset. The
|
|
1059
|
-
interval is near the end of the dataset's data inventory. The
|
|
1060
|
-
returned interval is not guaranteed to have non-fill data for any
|
|
1061
|
-
specific variable.
|
|
1062
|
-
|
|
1063
|
-
Parameters
|
|
1064
|
-
----------
|
|
1065
|
-
identifier
|
|
1066
|
-
dataset identifier of data inventory to get.
|
|
1067
|
-
Returns
|
|
1068
|
-
-------
|
|
1069
|
-
timeinterval.TimeInterval
|
|
1070
|
-
An small example time interval that is likely, but not
|
|
1071
|
-
guaranteed, to have data or None if an interval cannot be
|
|
1072
|
-
found.
|
|
1073
|
-
"""
|
|
1074
|
-
|
|
1075
|
-
time_intervals = self.get_inventory(identifier)
|
|
1076
|
-
if len(time_intervals) < 1:
|
|
1077
|
-
return None
|
|
1078
|
-
example_interval = time_intervals[-1]
|
|
1079
|
-
if re.search('MMS[1-4]_.+_BRST_.+', identifier):
|
|
1080
|
-
time_delta = timedelta(seconds=1)
|
|
1081
|
-
else:
|
|
1082
|
-
time_delta = timedelta(hours=2)
|
|
1083
|
-
example_interval.start = example_interval.end - time_delta
|
|
1084
|
-
return example_interval
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
def get_variables(
|
|
1088
|
-
self,
|
|
1089
|
-
identifier: str
|
|
1090
|
-
) -> List[Dict]:
|
|
1091
|
-
"""
|
|
1092
|
-
Gets a description of the variables in the specified dataset.
|
|
1093
|
-
|
|
1094
|
-
Parameters
|
|
1095
|
-
----------
|
|
1096
|
-
identifier
|
|
1097
|
-
dataset identifier of data to get.
|
|
1098
|
-
Returns
|
|
1099
|
-
-------
|
|
1100
|
-
List
|
|
1101
|
-
A List of dictionary descriptions of the variables in
|
|
1102
|
-
the specified dataset. The dictionary structure is defined by
|
|
1103
|
-
the VariableDescription element in
|
|
1104
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
|
|
1105
|
-
"""
|
|
1106
|
-
|
|
1107
|
-
url = self._endpoint + 'datasets/' + \
|
|
1108
|
-
urllib.parse.quote(identifier, safe='') + '/variables'
|
|
1109
|
-
|
|
1110
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
1111
|
-
|
|
1112
|
-
if response.status_code != 200:
|
|
1113
|
-
|
|
1114
|
-
self.logger.info('%s failed with http code %d', url,
|
|
1115
|
-
response.status_code)
|
|
1116
|
-
self.logger.info('response.text: %s', response.text)
|
|
1117
|
-
return []
|
|
1118
|
-
|
|
1119
|
-
if self.logger.level <= logging.DEBUG:
|
|
1120
|
-
self.logger.debug('response.text = %s', response.text)
|
|
1121
|
-
|
|
1122
|
-
var_descriptions = ET.fromstring(response.text)
|
|
1123
|
-
|
|
1124
|
-
variables = []
|
|
1125
|
-
for var_description in var_descriptions.findall(\
|
|
1126
|
-
'cdas:VariableDescription',
|
|
1127
|
-
namespaces=NAMESPACES):
|
|
1128
|
-
name = var_description.find('cdas:Name',
|
|
1129
|
-
namespaces=NAMESPACES).text
|
|
1130
|
-
short_description = var_description.find(\
|
|
1131
|
-
'cdas:ShortDescription',
|
|
1132
|
-
namespaces=NAMESPACES).text
|
|
1133
|
-
if short_description is None:
|
|
1134
|
-
short_description = ''
|
|
1135
|
-
|
|
1136
|
-
long_description = var_description.find(\
|
|
1137
|
-
'cdas:LongDescription',
|
|
1138
|
-
namespaces=NAMESPACES).text
|
|
1139
|
-
if long_description is None:
|
|
1140
|
-
long_description = ''
|
|
1141
|
-
|
|
1142
|
-
variables.append({
|
|
1143
|
-
'Name': name,
|
|
1144
|
-
'ShortDescription': short_description,
|
|
1145
|
-
'LongDescription': long_description
|
|
1146
|
-
})
|
|
1147
|
-
|
|
1148
|
-
return variables
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
def get_variable_names(
|
|
1152
|
-
self,
|
|
1153
|
-
identifier: str
|
|
1154
|
-
) -> List[str]:
|
|
1155
|
-
"""
|
|
1156
|
-
Gets the names of the variables in the specified dataset. This
|
|
1157
|
-
method is like the get_variables method except that it only returns
|
|
1158
|
-
the variable names and not the other metadata.
|
|
1159
|
-
|
|
1160
|
-
Parameters
|
|
1161
|
-
----------
|
|
1162
|
-
identifier
|
|
1163
|
-
dataset identifier of data to get.
|
|
1164
|
-
Returns
|
|
1165
|
-
-------
|
|
1166
|
-
List
|
|
1167
|
-
A List of the names of the variables in the specified dataset.
|
|
1168
|
-
"""
|
|
1169
|
-
|
|
1170
|
-
variable_names = []
|
|
1171
|
-
for variable in self.get_variables(identifier):
|
|
1172
|
-
variable_names.append(variable['Name'])
|
|
1173
|
-
|
|
1174
|
-
return variable_names
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
@staticmethod
|
|
1178
|
-
def _get_thumbnail_description_dict(
|
|
1179
|
-
file_description_elem: ET.Element
|
|
1180
|
-
) -> Dict:
|
|
1181
|
-
"""
|
|
1182
|
-
Gets ThumbnailDescription dictionary representation from the
|
|
1183
|
-
given FileDescription element.
|
|
1184
|
-
|
|
1185
|
-
Parameters
|
|
1186
|
-
----------
|
|
1187
|
-
file_description_elem
|
|
1188
|
-
a FileDescription Element.
|
|
1189
|
-
Returns
|
|
1190
|
-
-------
|
|
1191
|
-
Dict
|
|
1192
|
-
a Dictionary representation of the ThumbnailDescription
|
|
1193
|
-
contained in the given FileDescription element.
|
|
1194
|
-
"""
|
|
1195
|
-
thumbnail_desc = file_description_elem.find(\
|
|
1196
|
-
'cdas:ThumbnailDescription',
|
|
1197
|
-
namespaces=NAMESPACES)
|
|
1198
|
-
if thumbnail_desc is not None:
|
|
1199
|
-
time_interval = thumbnail_desc.find('cdas:TimeInterval',
|
|
1200
|
-
namespaces=NAMESPACES)
|
|
1201
|
-
start = time_interval.find('cdas:Start',
|
|
1202
|
-
namespaces=NAMESPACES).text
|
|
1203
|
-
end = time_interval.find('cdas:End',
|
|
1204
|
-
namespaces=NAMESPACES).text
|
|
1205
|
-
return {
|
|
1206
|
-
'Name': thumbnail_desc.find('cdas:Name',
|
|
1207
|
-
namespaces=NAMESPACES).text,
|
|
1208
|
-
'Dataset': thumbnail_desc.find('cdas:Dataset',
|
|
1209
|
-
namespaces=NAMESPACES).text,
|
|
1210
|
-
'TimeInterval': {
|
|
1211
|
-
'Start': start,
|
|
1212
|
-
'End': end
|
|
1213
|
-
},
|
|
1214
|
-
'VarName': thumbnail_desc.find('cdas:VarName',
|
|
1215
|
-
namespaces=NAMESPACES).text,
|
|
1216
|
-
'Options': int(thumbnail_desc.find(\
|
|
1217
|
-
'cdas:Options',
|
|
1218
|
-
namespaces=NAMESPACES).text),
|
|
1219
|
-
'NumFrames': int(thumbnail_desc.find(\
|
|
1220
|
-
'cdas:NumFrames',
|
|
1221
|
-
namespaces=NAMESPACES).text),
|
|
1222
|
-
'NumRows': int(thumbnail_desc.find(\
|
|
1223
|
-
'cdas:NumRows',
|
|
1224
|
-
namespaces=NAMESPACES).text),
|
|
1225
|
-
'NumCols': int(thumbnail_desc.find(\
|
|
1226
|
-
'cdas:NumCols',
|
|
1227
|
-
namespaces=NAMESPACES).text),
|
|
1228
|
-
'TitleHeight': int(thumbnail_desc.find(\
|
|
1229
|
-
'cdas:TitleHeight',
|
|
1230
|
-
namespaces=NAMESPACES).text),
|
|
1231
|
-
'ThumbnailHeight': int(thumbnail_desc.find(\
|
|
1232
|
-
'cdas:ThumbnailHeight',
|
|
1233
|
-
namespaces=NAMESPACES).text),
|
|
1234
|
-
'ThumbnailWidth': int(thumbnail_desc.find(\
|
|
1235
|
-
'cdas:ThumbnailWidth',
|
|
1236
|
-
namespaces=NAMESPACES).text),
|
|
1237
|
-
'StartRecord': int(thumbnail_desc.find(\
|
|
1238
|
-
'cdas:StartRecord',
|
|
1239
|
-
namespaces=NAMESPACES).text),
|
|
1240
|
-
'MyScale': float(thumbnail_desc.find(\
|
|
1241
|
-
'cdas:MyScale',
|
|
1242
|
-
namespaces=NAMESPACES).text),
|
|
1243
|
-
'XyStep': float(thumbnail_desc.find(\
|
|
1244
|
-
'cdas:XyStep',
|
|
1245
|
-
namespaces=NAMESPACES).text)
|
|
1246
|
-
}
|
|
1247
|
-
return None
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
@staticmethod
|
|
1251
|
-
def _get_data_result_dict(
|
|
1252
|
-
xml_data_result: str
|
|
1253
|
-
) -> Dict:
|
|
1254
|
-
"""
|
|
1255
|
-
Gets DataResult dictionary representation from the
|
|
1256
|
-
given XML DataResult element.
|
|
1257
|
-
|
|
1258
|
-
Parameters
|
|
1259
|
-
----------
|
|
1260
|
-
xml_data_result
|
|
1261
|
-
XML representation of a DataResult.
|
|
1262
|
-
Returns
|
|
1263
|
-
-------
|
|
1264
|
-
Dict
|
|
1265
|
-
a Dictionary representation of the given XML representation
|
|
1266
|
-
of a DataResult.
|
|
1267
|
-
"""
|
|
1268
|
-
data_result = ET.fromstring(xml_data_result)
|
|
1269
|
-
file_descriptions = []
|
|
1270
|
-
for file_description in data_result.findall(\
|
|
1271
|
-
'cdas:FileDescription', namespaces=NAMESPACES):
|
|
1272
|
-
|
|
1273
|
-
dict_file_description = {
|
|
1274
|
-
'Name': file_description.find('cdas:Name',
|
|
1275
|
-
namespaces=NAMESPACES).text,
|
|
1276
|
-
'MimeType': file_description.find(\
|
|
1277
|
-
'cdas:MimeType',
|
|
1278
|
-
namespaces=NAMESPACES).text,
|
|
1279
|
-
'StartTime': file_description.find(\
|
|
1280
|
-
'cdas:StartTime',
|
|
1281
|
-
namespaces=NAMESPACES).text,
|
|
1282
|
-
'EndTime': file_description.find(\
|
|
1283
|
-
'cdas:EndTime',
|
|
1284
|
-
namespaces=NAMESPACES).text,
|
|
1285
|
-
'Length': int(file_description.find(\
|
|
1286
|
-
'cdas:Length',
|
|
1287
|
-
namespaces=NAMESPACES).text),
|
|
1288
|
-
'LastModified': file_description.find(\
|
|
1289
|
-
'cdas:LastModified',
|
|
1290
|
-
namespaces=NAMESPACES).text
|
|
1291
|
-
}
|
|
1292
|
-
thumbnail_dict = CdasWs._get_thumbnail_description_dict(\
|
|
1293
|
-
file_description)
|
|
1294
|
-
if thumbnail_dict is not None:
|
|
1295
|
-
dict_file_description['ThumbnailDescription'] = \
|
|
1296
|
-
thumbnail_dict
|
|
1297
|
-
|
|
1298
|
-
thumbnail_id_elem = file_description.find(\
|
|
1299
|
-
'cdas:ThumbnailId',
|
|
1300
|
-
namespaces=NAMESPACES)
|
|
1301
|
-
if thumbnail_id_elem is not None:
|
|
1302
|
-
dict_file_description['ThumbnailId'] = \
|
|
1303
|
-
thumbnail_id_elem.text
|
|
1304
|
-
|
|
1305
|
-
file_descriptions.append(dict_file_description)
|
|
1306
|
-
|
|
1307
|
-
if len(file_descriptions) > 0:
|
|
1308
|
-
return {
|
|
1309
|
-
'FileDescription': file_descriptions
|
|
1310
|
-
}
|
|
1311
|
-
return None
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
def get_data_result(
|
|
1315
|
-
self,
|
|
1316
|
-
data_request: DataRequest,
|
|
1317
|
-
progress_callback: Callable[[float, str, Any], int],
|
|
1318
|
-
progress_user_value: Any
|
|
1319
|
-
) -> Tuple[int, Dict]:
|
|
1320
|
-
"""
|
|
1321
|
-
Submits the given request to the server and returns the result.
|
|
1322
|
-
This is a relatively low-level method and most callers should
|
|
1323
|
-
probably use a higher-level method such as get_data.
|
|
1324
|
-
|
|
1325
|
-
Parameters
|
|
1326
|
-
----------
|
|
1327
|
-
data_request
|
|
1328
|
-
data request.
|
|
1329
|
-
progress_callback
|
|
1330
|
-
function that is called repeatedly to report the progress
|
|
1331
|
-
of getting the data. The function should return 0 if it
|
|
1332
|
-
wants to continue getting data. If it returns a non-0 value,
|
|
1333
|
-
getting the data will be aborted and the get_data() function
|
|
1334
|
-
will immediately return (204, None). The float parameter
|
|
1335
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
1336
|
-
the str parameter will contain a text message indicating
|
|
1337
|
-
the progress of this call.
|
|
1338
|
-
progressUserValue
|
|
1339
|
-
value that is passsed to the progressCallback function.
|
|
1340
|
-
Returns
|
|
1341
|
-
-------
|
|
1342
|
-
Tuple
|
|
1343
|
-
[0] contains the int HTTP status code. 200 when
|
|
1344
|
-
successful.<br>
|
|
1345
|
-
[1] contains a dictionary representing the DataResult from
|
|
1346
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
1347
|
-
or None.
|
|
1348
|
-
See Also
|
|
1349
|
-
--------
|
|
1350
|
-
CdasWs.get_data
|
|
1351
|
-
"""
|
|
1352
|
-
|
|
1353
|
-
#self.logger.debug('data_request = %s', data_request.json())
|
|
1354
|
-
self.logger.debug('data_request = %s', data_request.xml_str())
|
|
1355
|
-
|
|
1356
|
-
url = self._endpoint + 'datasets'
|
|
1357
|
-
|
|
1358
|
-
for retries in range(RETRY_LIMIT):
|
|
1359
|
-
#response = self._session.post(url, data=data_request.json(),
|
|
1360
|
-
response = self._session.post(url, data=data_request.xml_str(),
|
|
1361
|
-
timeout=self._timeout)
|
|
1362
|
-
|
|
1363
|
-
if response.status_code == 200:
|
|
1364
|
-
|
|
1365
|
-
data_result = CdasWs._get_data_result_dict(response.text)
|
|
1366
|
-
|
|
1367
|
-
if not data_result:
|
|
1368
|
-
return (response.status_code, None)
|
|
1369
|
-
|
|
1370
|
-
return (response.status_code, data_result)
|
|
1371
|
-
|
|
1372
|
-
if response.status_code == 429 or \
|
|
1373
|
-
response.status_code == 503 and \
|
|
1374
|
-
'Retry-After' in response.headers:
|
|
1375
|
-
|
|
1376
|
-
retry_after = response.headers['Retry-After']
|
|
1377
|
-
|
|
1378
|
-
self.logger.debug('429/503 status with Retry-After header: %s',
|
|
1379
|
-
retry_after)
|
|
1380
|
-
|
|
1381
|
-
if progress_callback is not None:
|
|
1382
|
-
if progress_callback(0.2, 'Waiting ' + retry_after + \
|
|
1383
|
-
's before making server request.',
|
|
1384
|
-
progress_user_value) != 0:
|
|
1385
|
-
return (204, None)
|
|
1386
|
-
|
|
1387
|
-
retry_after = int(retry_after)
|
|
1388
|
-
|
|
1389
|
-
self.logger.info('Sleeping %d seconds before making request',
|
|
1390
|
-
retry_after)
|
|
1391
|
-
time.sleep(retry_after)
|
|
1392
|
-
|
|
1393
|
-
else:
|
|
1394
|
-
self.logger.info('%s failed with http code %d', url,
|
|
1395
|
-
response.status_code)
|
|
1396
|
-
self.logger.info('data_request = %s', data_request)
|
|
1397
|
-
self.logger.info('response.text: %s', response.text)
|
|
1398
|
-
return (response.status_code, None)
|
|
1399
|
-
|
|
1400
|
-
self.logger.info('%s failed with http code %d after %d retries',
|
|
1401
|
-
url, response.status_code, retries + 1)
|
|
1402
|
-
self.logger.info('data_request = %s', data_request)
|
|
1403
|
-
self.logger.info('response.text: %s', response.text)
|
|
1404
|
-
return (response.status_code, None)
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
def get_data_file(
|
|
1408
|
-
self,
|
|
1409
|
-
dataset: str,
|
|
1410
|
-
variables: List[str],
|
|
1411
|
-
start: Union[datetime, str], end: Union[datetime, str],
|
|
1412
|
-
**keywords: Union[
|
|
1413
|
-
Dict,
|
|
1414
|
-
Callable[[float, str, Any], int],
|
|
1415
|
-
Any]
|
|
1416
|
-
) -> Tuple[int, Dict]:
|
|
1417
|
-
"""
|
|
1418
|
-
Gets the specified data file from the server.
|
|
1419
|
-
|
|
1420
|
-
Parameters
|
|
1421
|
-
----------
|
|
1422
|
-
dataset
|
|
1423
|
-
dataset identifier of data to get.
|
|
1424
|
-
variables
|
|
1425
|
-
array containing names of variables to get.
|
|
1426
|
-
start
|
|
1427
|
-
start time of data to get. See module note about timezone.
|
|
1428
|
-
end
|
|
1429
|
-
end time of data to get. See module note about timezone.
|
|
1430
|
-
keywords
|
|
1431
|
-
optional keyword parameters as follows:<br>
|
|
1432
|
-
<b>binData</b> - indicates that uniformly spaced values should
|
|
1433
|
-
be computed for scaler/vector/spectrogram data according to
|
|
1434
|
-
the given binning parameter values. binData is a Dict that
|
|
1435
|
-
may contain the following keys: interval,
|
|
1436
|
-
interpolateMissingValues, sigmaMultiplier, and/or
|
|
1437
|
-
overrideDefaultBinning with values that override the
|
|
1438
|
-
defaults.<br>
|
|
1439
|
-
<b>progressCallback</b> - is a
|
|
1440
|
-
Callable[[float, str, typing.Any], int]
|
|
1441
|
-
function that is called repeatedly to report the progress
|
|
1442
|
-
of getting the data. The function should return 0 if it
|
|
1443
|
-
wants to continue getting data. If it returns non-0 value,
|
|
1444
|
-
getting the data will be aborted and the get_data_file()
|
|
1445
|
-
function will immediately return (204, None). The float
|
|
1446
|
-
parameter is a value between 0.0 and 1.0 to indicate progress
|
|
1447
|
-
and the str parameter will contain a text message indicating
|
|
1448
|
-
the progress of this call.<br>
|
|
1449
|
-
<b>progressUserValue</b> - is an Any value that is passsed
|
|
1450
|
-
to the progressCallback function.<br>
|
|
1451
|
-
Returns
|
|
1452
|
-
-------
|
|
1453
|
-
Tuple
|
|
1454
|
-
[0] contains the int HTTP status code. 200 when
|
|
1455
|
-
successful.<br>
|
|
1456
|
-
[1] contains a dictionary representing the DataResult from
|
|
1457
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
1458
|
-
or None.
|
|
1459
|
-
Raises
|
|
1460
|
-
------
|
|
1461
|
-
ValueError
|
|
1462
|
-
If the given start/end datetime values are invalid.
|
|
1463
|
-
See Also
|
|
1464
|
-
--------
|
|
1465
|
-
CdasWs.get_data : In addition to what get_data_file does,
|
|
1466
|
-
get_data also downloads and reads the data file into memory
|
|
1467
|
-
(SpaceData or xarray.Dataset object).
|
|
1468
|
-
"""
|
|
1469
|
-
# pylint: disable=too-many-locals
|
|
1470
|
-
# pylint: disable=too-many-return-statements
|
|
1471
|
-
# pylint: enable=too-many-statements
|
|
1472
|
-
# pylint: disable=too-many-branches
|
|
1473
|
-
|
|
1474
|
-
start_datetime, end_datetime = TimeInterval.get_datetimes(start,
|
|
1475
|
-
end)
|
|
1476
|
-
|
|
1477
|
-
data_request = CdfRequest(dataset, variables,
|
|
1478
|
-
TimeInterval(start_datetime,
|
|
1479
|
-
end_datetime),
|
|
1480
|
-
3, CdfFormat.BINARY,
|
|
1481
|
-
**keywords.get('binData', {}))
|
|
1482
|
-
|
|
1483
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
1484
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
1485
|
-
|
|
1486
|
-
self.logger.debug('data_request = %s', data_request)
|
|
1487
|
-
|
|
1488
|
-
if progress_callback is not None:
|
|
1489
|
-
if progress_callback(0.1, 'Making server request.',
|
|
1490
|
-
progress_user_value) != 0:
|
|
1491
|
-
return (204, None)
|
|
1492
|
-
|
|
1493
|
-
status_code, data_result = self.get_data_result(data_request,
|
|
1494
|
-
progress_callback,
|
|
1495
|
-
progress_user_value)
|
|
1496
|
-
|
|
1497
|
-
if progress_callback is not None:
|
|
1498
|
-
if progress_callback(1.0, 'Initial server request complete.',
|
|
1499
|
-
progress_user_value) != 0:
|
|
1500
|
-
return (status_code, None)
|
|
1501
|
-
|
|
1502
|
-
return (status_code, data_result)
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
def download(
|
|
1506
|
-
self,
|
|
1507
|
-
url: str,
|
|
1508
|
-
size: int = 0,
|
|
1509
|
-
**keywords
|
|
1510
|
-
) -> str:
|
|
1511
|
-
"""
|
|
1512
|
-
Downloads the file specified by the given URL to a temporary
|
|
1513
|
-
file without reading all of it into memory. This method
|
|
1514
|
-
utilizes the connection pool and persistent HTTP connection
|
|
1515
|
-
to the CdasWs server.
|
|
1516
|
-
|
|
1517
|
-
Parameters
|
|
1518
|
-
----------
|
|
1519
|
-
url
|
|
1520
|
-
URL of file to download.
|
|
1521
|
-
size
|
|
1522
|
-
number of bytes in file to download.
|
|
1523
|
-
keywords
|
|
1524
|
-
optional keyword parameters as follows:<br>
|
|
1525
|
-
<b>progressCallback</b> - is a
|
|
1526
|
-
typing.Callable[[float, str, typing.Any], int]
|
|
1527
|
-
function that is called repeatedly to report the progress
|
|
1528
|
-
of getting the data. The function should return 0 if it
|
|
1529
|
-
wants to continue getting data. If it returns a non-0 value,
|
|
1530
|
-
getting the data will be aborted and this download() function
|
|
1531
|
-
will immediately return None. The float parameter
|
|
1532
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
1533
|
-
the str parameter will contain a text message indicating
|
|
1534
|
-
the progress of this call.<br>
|
|
1535
|
-
<b>progressUserValue</b> - is a typing.Any value that is
|
|
1536
|
-
passsed to the progressCallback function.<br>
|
|
1537
|
-
Returns
|
|
1538
|
-
-------
|
|
1539
|
-
str
|
|
1540
|
-
name of tempory file or None if there was an error.
|
|
1541
|
-
"""
|
|
1542
|
-
# pylint: disable=too-many-locals
|
|
1543
|
-
|
|
1544
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
1545
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
1546
|
-
|
|
1547
|
-
suffix = os.path.splitext(urlparse(url).path)[1]
|
|
1548
|
-
|
|
1549
|
-
file_descriptor, tmp_filename = mkstemp(suffix=suffix)
|
|
1550
|
-
|
|
1551
|
-
download_bytes = 0
|
|
1552
|
-
next_progress_report = 0.1
|
|
1553
|
-
with self._session.get(url, stream=True,
|
|
1554
|
-
timeout=self._timeout) as response:
|
|
1555
|
-
|
|
1556
|
-
file = open(tmp_filename, 'wb')
|
|
1557
|
-
for chunk in response.iter_content(chunk_size=8192):
|
|
1558
|
-
if chunk: # filter out keep-alive new chunks
|
|
1559
|
-
file.write(chunk)
|
|
1560
|
-
# file.flush()
|
|
1561
|
-
if progress_callback is not None:
|
|
1562
|
-
download_bytes += len(chunk)
|
|
1563
|
-
if size == 0:
|
|
1564
|
-
download_progress = 0.0
|
|
1565
|
-
else:
|
|
1566
|
-
download_progress = float(download_bytes) / size
|
|
1567
|
-
if download_progress > next_progress_report:
|
|
1568
|
-
next_progress_report += download_progress
|
|
1569
|
-
if progress_callback(download_progress,\
|
|
1570
|
-
'Continuing download of data.',
|
|
1571
|
-
progress_user_value) != 0:
|
|
1572
|
-
|
|
1573
|
-
file.close()
|
|
1574
|
-
os.close(file_descriptor)
|
|
1575
|
-
return None
|
|
1576
|
-
file.close()
|
|
1577
|
-
os.close(file_descriptor)
|
|
1578
|
-
|
|
1579
|
-
if progress_callback is not None:
|
|
1580
|
-
if progress_callback(0.4,
|
|
1581
|
-
'Data download complete. Reading data.',
|
|
1582
|
-
progress_user_value) != 0:
|
|
1583
|
-
return None
|
|
1584
|
-
|
|
1585
|
-
return tmp_filename
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
@staticmethod
|
|
1589
|
-
def read_data(
|
|
1590
|
-
filename: str,
|
|
1591
|
-
data_representation: DataRepresentation
|
|
1592
|
-
) -> Union['spacepy.datamodel', 'xr.Dataset']:
|
|
1593
|
-
"""
|
|
1594
|
-
Reads the data from the given file.
|
|
1595
|
-
|
|
1596
|
-
Parameters
|
|
1597
|
-
----------
|
|
1598
|
-
filename
|
|
1599
|
-
Name of file to read.
|
|
1600
|
-
data_representation
|
|
1601
|
-
Requested data representation.
|
|
1602
|
-
Returns
|
|
1603
|
-
-------
|
|
1604
|
-
spacepy.datamodel or xr.Dataset
|
|
1605
|
-
Data from file.
|
|
1606
|
-
Raises
|
|
1607
|
-
------
|
|
1608
|
-
Exception
|
|
1609
|
-
If an Exception is raise by either the spdm.fromCDF() or
|
|
1610
|
-
cdflib.cdf_to_xarray() functions.
|
|
1611
|
-
ModuleNotFoundError
|
|
1612
|
-
If the required spacepy.datamodel or the cdflib and xarray
|
|
1613
|
-
modules are not installed.
|
|
1614
|
-
"""
|
|
1615
|
-
if data_representation is None:
|
|
1616
|
-
if SPDM_AVAILABLE:
|
|
1617
|
-
return spdm.fromCDF(filename)
|
|
1618
|
-
if CDF_XARRAY_AVAILABLE:
|
|
1619
|
-
return cdf_to_xarray(filename, to_datetime=True,
|
|
1620
|
-
fillval_to_nan=True)
|
|
1621
|
-
raise ModuleNotFoundError(
|
|
1622
|
-
'neither the spacepy.datamodel nor the cdflib and '
|
|
1623
|
-
'xarray modules are installed')
|
|
1624
|
-
|
|
1625
|
-
if data_representation is DataRepresentation.SPACEPY and \
|
|
1626
|
-
not SPDM_AVAILABLE:
|
|
1627
|
-
raise ModuleNotFoundError('spacepy module must be installed')
|
|
1628
|
-
if data_representation is DataRepresentation.XARRAY and \
|
|
1629
|
-
not CDF_XARRAY_AVAILABLE:
|
|
1630
|
-
raise ModuleNotFoundError('cdflib and xarray modules must be installed')
|
|
1631
|
-
|
|
1632
|
-
if data_representation is DataRepresentation.SPACEPY:
|
|
1633
|
-
return spdm.fromCDF(filename)
|
|
1634
|
-
if data_representation is DataRepresentation.XARRAY:
|
|
1635
|
-
return cdf_to_xarray(filename, to_datetime=True,
|
|
1636
|
-
fillval_to_nan=True)
|
|
1637
|
-
return None
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
def get_data(
|
|
1641
|
-
self,
|
|
1642
|
-
dataset: str,
|
|
1643
|
-
variables: List[str],
|
|
1644
|
-
time0: Union[TimeInterval, List[TimeInterval], datetime, str],
|
|
1645
|
-
time1: Union[datetime, str] = None,
|
|
1646
|
-
**keywords: Union[
|
|
1647
|
-
Dict,
|
|
1648
|
-
DataRepresentation,
|
|
1649
|
-
Callable[[float, str, Any], int],
|
|
1650
|
-
Any]
|
|
1651
|
-
) -> Tuple[Dict, 'spdm.SpaceData', 'xarray']:
|
|
1652
|
-
"""
|
|
1653
|
-
Gets the specified data from the server. The representation
|
|
1654
|
-
of the returned data is determined as follows:<br>
|
|
1655
|
-
1. If a dataRepresentation keyword parameter is given, its
|
|
1656
|
-
value will determine the representation of the returned
|
|
1657
|
-
data. If no dataRepresenation keyword parameter is
|
|
1658
|
-
given, then<br>
|
|
1659
|
-
2. If the presence of spacepy.datamodel is found, then the data
|
|
1660
|
-
is returned in the spacepy.datamodel representation.<br>
|
|
1661
|
-
3. If the presence of the cdflib and xarray modules are found,
|
|
1662
|
-
then the data is returned in an xarray.Dataset.
|
|
1663
|
-
|
|
1664
|
-
Parameters
|
|
1665
|
-
----------
|
|
1666
|
-
dataset
|
|
1667
|
-
dataset identifier of data to get.
|
|
1668
|
-
variables
|
|
1669
|
-
array containing names of variables to get. The value
|
|
1670
|
-
ALL-VARIABLES may be used instead of specifying all the
|
|
1671
|
-
individual variable names.
|
|
1672
|
-
time0
|
|
1673
|
-
TimeInterval(s) or start time of data to get. See module
|
|
1674
|
-
note about timezone.
|
|
1675
|
-
time1
|
|
1676
|
-
when time0 is not one or more TimeInterval(s), the end time
|
|
1677
|
-
of data to get. See module note about timezone.
|
|
1678
|
-
keywords
|
|
1679
|
-
optional keyword parameters as follows:<br>
|
|
1680
|
-
<b>binData</b> - indicates that uniformly spaced values should
|
|
1681
|
-
be computed for scaler/vector/spectrogram data according to
|
|
1682
|
-
the given binning parameter values. See
|
|
1683
|
-
<https://cdaweb.gsfc.nasa.gov/CDAWeb_Binning_readme.html>
|
|
1684
|
-
for more details. binData is a Dict that
|
|
1685
|
-
may contain the following keys: interval,
|
|
1686
|
-
interpolateMissingValues, sigmaMultiplier, and/or
|
|
1687
|
-
overrideDefaultBinning with values that override the
|
|
1688
|
-
defaults.<br>
|
|
1689
|
-
<b>dataRepresentation</b> - specifies the representation of
|
|
1690
|
-
the returned data as one of
|
|
1691
|
-
`datarepresentation.DataRepresentation`.<br>
|
|
1692
|
-
<b>progressCallback</b> - is a
|
|
1693
|
-
Callable[[float, str, typing.Any], int]
|
|
1694
|
-
function that is called repeatedly to report the progress
|
|
1695
|
-
of getting the data. The function should return 0 if it
|
|
1696
|
-
wants to continue getting data. If it returns non-0 value,
|
|
1697
|
-
getting the data will be aborted and the get_data() function
|
|
1698
|
-
will immediately return (204, None). The float parameter
|
|
1699
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
1700
|
-
the str parameter will contain a text message indicating
|
|
1701
|
-
the progress of this call.<br>
|
|
1702
|
-
<b>progressUserValue</b> - is an Any value that is passsed
|
|
1703
|
-
to the progressCallback function.<br>
|
|
1704
|
-
Returns
|
|
1705
|
-
-------
|
|
1706
|
-
Tuple
|
|
1707
|
-
[0] contains a dictionary of HTTP and CDAS status information.
|
|
1708
|
-
When successful, ['http']['status_code'] will be 200.<br>
|
|
1709
|
-
[1] contains the requested data (SpaceData or xarray.Dataset
|
|
1710
|
-
object) or None.
|
|
1711
|
-
Raises
|
|
1712
|
-
------
|
|
1713
|
-
ValueError
|
|
1714
|
-
If no variables are given or if the given start/end datetime
|
|
1715
|
-
values are invalid.
|
|
1716
|
-
"""
|
|
1717
|
-
# pylint: disable=too-many-locals
|
|
1718
|
-
# pylint: disable=too-many-return-statements
|
|
1719
|
-
# pylint: disable=too-many-statements
|
|
1720
|
-
# pylint: disable=too-many-branches
|
|
1721
|
-
# pylint: disable=import-outside-toplevel
|
|
1722
|
-
|
|
1723
|
-
#import spacepy.datamodel as spdm # type: ignore
|
|
1724
|
-
|
|
1725
|
-
if len(variables) < 1:
|
|
1726
|
-
raise ValueError('at least one variable name is required')
|
|
1727
|
-
|
|
1728
|
-
if isinstance(time0, (str, datetime)):
|
|
1729
|
-
if isinstance(time1, (str, datetime)):
|
|
1730
|
-
time_intervals = [TimeInterval(time0, time1)]
|
|
1731
|
-
else:
|
|
1732
|
-
raise ValueError('time1 must be str/datetime')
|
|
1733
|
-
elif isinstance(time0, TimeInterval):
|
|
1734
|
-
time_intervals = [time0]
|
|
1735
|
-
elif isinstance(time0, list) and len(time0) > 0 and\
|
|
1736
|
-
isinstance(time0[0], TimeInterval):
|
|
1737
|
-
time_intervals = time0
|
|
1738
|
-
else:
|
|
1739
|
-
raise ValueError('invalid time0 type')
|
|
1740
|
-
|
|
1741
|
-
data_request = CdfRequest(dataset, variables,
|
|
1742
|
-
time_intervals,
|
|
1743
|
-
3, CdfFormat.BINARY,
|
|
1744
|
-
binData=keywords.get('binData', {}))
|
|
1745
|
-
|
|
1746
|
-
data_rep = keywords.get('dataRepresentation', None)
|
|
1747
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
1748
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
1749
|
-
|
|
1750
|
-
self.logger.debug('data_request = %s', data_request)
|
|
1751
|
-
|
|
1752
|
-
status = {
|
|
1753
|
-
'http': {
|
|
1754
|
-
'status_code': 204
|
|
1755
|
-
},
|
|
1756
|
-
'cdas': {
|
|
1757
|
-
'status': [],
|
|
1758
|
-
'message': [],
|
|
1759
|
-
'warning': [],
|
|
1760
|
-
'error': []
|
|
1761
|
-
}
|
|
1762
|
-
}
|
|
1763
|
-
|
|
1764
|
-
if progress_callback is not None:
|
|
1765
|
-
if progress_callback(0.1, 'Making initial server request.',
|
|
1766
|
-
progress_user_value) != 0:
|
|
1767
|
-
return (status, None)
|
|
1768
|
-
|
|
1769
|
-
status_code, data_result = self.get_data_result(data_request,
|
|
1770
|
-
progress_callback,
|
|
1771
|
-
progress_user_value)
|
|
1772
|
-
|
|
1773
|
-
status['http']['status_code'] = status_code
|
|
1774
|
-
|
|
1775
|
-
if progress_callback is not None:
|
|
1776
|
-
if progress_callback(0.3, 'Initial server request complete.',
|
|
1777
|
-
progress_user_value) != 0:
|
|
1778
|
-
return (status, None)
|
|
1779
|
-
|
|
1780
|
-
if status_code != 200:
|
|
1781
|
-
|
|
1782
|
-
self.logger.info('get_data_result failed with http code %d',
|
|
1783
|
-
status_code)
|
|
1784
|
-
self.logger.info('data_request = %s', data_request)
|
|
1785
|
-
return (status, None)
|
|
1786
|
-
|
|
1787
|
-
if not data_result:
|
|
1788
|
-
return (status, None)
|
|
1789
|
-
|
|
1790
|
-
if 'Status' in data_result:
|
|
1791
|
-
status['cdas']['status'] = data_result['Status']
|
|
1792
|
-
if 'Message' in data_result:
|
|
1793
|
-
status['cdas']['message'] = data_result['Message']
|
|
1794
|
-
if 'Warning' in data_result:
|
|
1795
|
-
status['cdas']['warning'] = data_result['Warning']
|
|
1796
|
-
if 'Error' in data_result:
|
|
1797
|
-
status['cdas']['error'] = data_result['Error']
|
|
1798
|
-
|
|
1799
|
-
if progress_callback is not None:
|
|
1800
|
-
if progress_callback(0.4, 'Beginning download of data.',
|
|
1801
|
-
progress_user_value) != 0:
|
|
1802
|
-
return (status, None)
|
|
1803
|
-
|
|
1804
|
-
file_descriptions = data_result['FileDescription']
|
|
1805
|
-
|
|
1806
|
-
data_url = file_descriptions[0]['Name']
|
|
1807
|
-
data_length = file_descriptions[0]['Length']
|
|
1808
|
-
|
|
1809
|
-
self.logger.debug('data_url = %s, data_length = %d',
|
|
1810
|
-
data_url, data_length)
|
|
1811
|
-
|
|
1812
|
-
sub_progress_control = {
|
|
1813
|
-
'progressCallback': progress_callback,
|
|
1814
|
-
'progressUserValue': progress_user_value,
|
|
1815
|
-
'progressStart': 0.4,
|
|
1816
|
-
'progressFraction': 0.1
|
|
1817
|
-
}
|
|
1818
|
-
|
|
1819
|
-
tmp_filename = self.download(data_url, data_length,
|
|
1820
|
-
progressCallback=_get_data_progress,
|
|
1821
|
-
progressUserValue=sub_progress_control)
|
|
1822
|
-
|
|
1823
|
-
try:
|
|
1824
|
-
data = self.read_data(tmp_filename, data_rep)
|
|
1825
|
-
os.remove(tmp_filename)
|
|
1826
|
-
if progress_callback is not None:
|
|
1827
|
-
if progress_callback(1.0, 'Finished reading data.',
|
|
1828
|
-
progress_user_value) != 0:
|
|
1829
|
-
return (status, None)
|
|
1830
|
-
except:
|
|
1831
|
-
self.logger.error('Exception from read_data(%s): %s, %s',
|
|
1832
|
-
tmp_filename, sys.exc_info()[0],
|
|
1833
|
-
sys.exc_info()[1])
|
|
1834
|
-
self.logger.error('CDF file has been retained.')
|
|
1835
|
-
raise
|
|
1836
|
-
return (status, data)
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
# pylint: disable=too-many-arguments
|
|
1840
|
-
def get_graph(
|
|
1841
|
-
self,
|
|
1842
|
-
dataset: str,
|
|
1843
|
-
variables: List[str],
|
|
1844
|
-
start: Union[datetime, str],
|
|
1845
|
-
end: Union[datetime, str],
|
|
1846
|
-
options: GraphOptions = None,
|
|
1847
|
-
image_format: List[ImageFormat] = None,
|
|
1848
|
-
**keywords
|
|
1849
|
-
) -> Tuple[int, Dict]:
|
|
1850
|
-
"""
|
|
1851
|
-
Gets a graphical representation of the specified data from the
|
|
1852
|
-
server.
|
|
1853
|
-
|
|
1854
|
-
Parameters
|
|
1855
|
-
----------
|
|
1856
|
-
dataset
|
|
1857
|
-
dataset identifier of data to get.
|
|
1858
|
-
variables
|
|
1859
|
-
array containing names of variables to get.
|
|
1860
|
-
start
|
|
1861
|
-
start time of data to get. See module note about timezone.
|
|
1862
|
-
end
|
|
1863
|
-
end time of data to get. See module note about timezone.
|
|
1864
|
-
options
|
|
1865
|
-
graph options.
|
|
1866
|
-
image_format
|
|
1867
|
-
image format. If None, then [ImageFormat.PNG].
|
|
1868
|
-
keywords
|
|
1869
|
-
optional keyword parameters as follows:<br>
|
|
1870
|
-
<b>binData</b> - indicates that uniformly spaced values should
|
|
1871
|
-
be computed for scaler/vector/spectrogram data according to
|
|
1872
|
-
the given binning parameter values. binData is a Dict that
|
|
1873
|
-
may contain the following keys: interval,
|
|
1874
|
-
interpolateMissingValues, sigmaMultiplier, and/or
|
|
1875
|
-
overrideDefaultBinning with values that override the
|
|
1876
|
-
defaults.<br>
|
|
1877
|
-
<b>progressCallback</b> - is a
|
|
1878
|
-
typing.Callable[[float, str, typing.Any], int]
|
|
1879
|
-
function that is called repeatedly to report the progress
|
|
1880
|
-
of getting the data. The function should return 0 if it
|
|
1881
|
-
wants to continue getting data. If it returns non-0 value,
|
|
1882
|
-
getting the data will be aborted and the get_data() function
|
|
1883
|
-
will immediately return (204, None). The float parameter
|
|
1884
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
1885
|
-
the str parameter will contain a text message indicating
|
|
1886
|
-
the progress of this call.<br>
|
|
1887
|
-
<b>progressUserValue</b> - is a typing.Any value that is
|
|
1888
|
-
passsed to the progressCallback function.<br>
|
|
1889
|
-
Returns
|
|
1890
|
-
-------
|
|
1891
|
-
Tuple
|
|
1892
|
-
[0] contains the HTTP status code value (200 when successful).<br>
|
|
1893
|
-
[1] contains a dictionary representation of a
|
|
1894
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
1895
|
-
DataResult object or None.<br>
|
|
1896
|
-
Raises
|
|
1897
|
-
------
|
|
1898
|
-
ValueError
|
|
1899
|
-
If the given start/end datetime values are invalid.
|
|
1900
|
-
"""
|
|
1901
|
-
# pylint: disable=too-many-locals
|
|
1902
|
-
# pylint: disable=too-many-return-statements
|
|
1903
|
-
# pylint: enable=too-many-statements
|
|
1904
|
-
# pylint: disable=too-many-branches
|
|
1905
|
-
|
|
1906
|
-
start_datetime, end_datetime = TimeInterval.get_datetimes(start,
|
|
1907
|
-
end)
|
|
1908
|
-
|
|
1909
|
-
request = GraphRequest(dataset, variables,
|
|
1910
|
-
TimeInterval(start_datetime, end_datetime),
|
|
1911
|
-
options, image_format,
|
|
1912
|
-
**keywords)
|
|
1913
|
-
|
|
1914
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
1915
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
1916
|
-
|
|
1917
|
-
self.logger.debug('request = %s', request)
|
|
1918
|
-
|
|
1919
|
-
if progress_callback is not None:
|
|
1920
|
-
if progress_callback(0.1, 'Making server request.',
|
|
1921
|
-
progress_user_value) != 0:
|
|
1922
|
-
return (204, None)
|
|
1923
|
-
|
|
1924
|
-
status_code, result = self.get_data_result(request, progress_callback, progress_user_value)
|
|
1925
|
-
|
|
1926
|
-
if progress_callback is not None:
|
|
1927
|
-
if progress_callback(1.0, 'Server request complete.',
|
|
1928
|
-
progress_user_value) != 0:
|
|
1929
|
-
return (status_code, None)
|
|
1930
|
-
|
|
1931
|
-
if status_code != 200:
|
|
1932
|
-
|
|
1933
|
-
self.logger.info('get_result failed with http code %d',
|
|
1934
|
-
status_code)
|
|
1935
|
-
self.logger.info('request = %s', request)
|
|
1936
|
-
return (status_code, None)
|
|
1937
|
-
|
|
1938
|
-
return (status_code, result)
|
|
1939
|
-
# pylint: enable=too-many-arguments
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
# pylint: disable=too-many-arguments
|
|
1943
|
-
def get_thumbnail(
|
|
1944
|
-
self,
|
|
1945
|
-
dataset: str,
|
|
1946
|
-
variables: List[str],
|
|
1947
|
-
start: Union[datetime, str],
|
|
1948
|
-
end: Union[datetime, str],
|
|
1949
|
-
identifier: str,
|
|
1950
|
-
thumbnail: int = 1,
|
|
1951
|
-
**keywords
|
|
1952
|
-
) -> Tuple[int, Dict]:
|
|
1953
|
-
"""
|
|
1954
|
-
Gets a graphical representation of the specified data from the
|
|
1955
|
-
server.
|
|
1956
|
-
|
|
1957
|
-
Parameters
|
|
1958
|
-
----------
|
|
1959
|
-
dataset
|
|
1960
|
-
dataset identifier of data to get.
|
|
1961
|
-
variables
|
|
1962
|
-
array containing names of variables to get.
|
|
1963
|
-
start
|
|
1964
|
-
start time of data to get. See module note about timezone.
|
|
1965
|
-
end
|
|
1966
|
-
end time of data to get. See module note about timezone.
|
|
1967
|
-
identifier
|
|
1968
|
-
thumbnail identifier (returned in a previous get_graph
|
|
1969
|
-
result).
|
|
1970
|
-
thumbnail
|
|
1971
|
-
number of thumbnail whose full size image is being requested.
|
|
1972
|
-
Thumbnail images are counted beginning at one (not zero).
|
|
1973
|
-
keywords
|
|
1974
|
-
optional keyword parameters as follows:<br>
|
|
1975
|
-
<b>progressCallback</b> - is a
|
|
1976
|
-
typing.Callable[[float, str, typing.Any], int]
|
|
1977
|
-
function that is called repeatedly to report the progress
|
|
1978
|
-
of getting the data. The function should return 0 if it
|
|
1979
|
-
wants to continue getting data. If it returns non-0 value,
|
|
1980
|
-
getting the data will be aborted and the get_data() function
|
|
1981
|
-
will immediately return (204, None). The float parameter
|
|
1982
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
1983
|
-
the str parameter will contain a text message indicating
|
|
1984
|
-
the progress of this call.<br>
|
|
1985
|
-
<b>progressUserValue</b> - is a typing.Any value that is
|
|
1986
|
-
passsed to the progressCallback function.<br>
|
|
1987
|
-
Returns
|
|
1988
|
-
-------
|
|
1989
|
-
Tuple
|
|
1990
|
-
[0] contains the HTTP status code value (200 when successful).<br>
|
|
1991
|
-
[1] contains a dictionary representation of a
|
|
1992
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
1993
|
-
DataResult object or None.<br>
|
|
1994
|
-
Raises
|
|
1995
|
-
------
|
|
1996
|
-
ValueError
|
|
1997
|
-
If the given start/end datetime values are invalid.
|
|
1998
|
-
"""
|
|
1999
|
-
# pylint: disable=too-many-locals
|
|
2000
|
-
# pylint: disable=too-many-return-statements
|
|
2001
|
-
# pylint: enable=too-many-statements
|
|
2002
|
-
# pylint: disable=too-many-branches
|
|
2003
|
-
|
|
2004
|
-
start_datetime, end_datetime = TimeInterval.get_datetimes(start,
|
|
2005
|
-
end)
|
|
2006
|
-
|
|
2007
|
-
request = ThumbnailRequest(dataset, variables,
|
|
2008
|
-
TimeInterval(start_datetime, end_datetime),
|
|
2009
|
-
identifier, thumbnail)
|
|
2010
|
-
|
|
2011
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
2012
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
2013
|
-
|
|
2014
|
-
self.logger.debug('request = %s', request)
|
|
2015
|
-
|
|
2016
|
-
if progress_callback is not None:
|
|
2017
|
-
if progress_callback(0.1, 'Making server request.',
|
|
2018
|
-
progress_user_value) != 0:
|
|
2019
|
-
return (204, None)
|
|
2020
|
-
|
|
2021
|
-
status_code, result = self.get_data_result(request,
|
|
2022
|
-
progress_callback,
|
|
2023
|
-
progress_user_value)
|
|
2024
|
-
|
|
2025
|
-
if progress_callback is not None:
|
|
2026
|
-
if progress_callback(1.0, 'Server request complete.',
|
|
2027
|
-
progress_user_value) != 0:
|
|
2028
|
-
return (status_code, None)
|
|
2029
|
-
|
|
2030
|
-
if status_code != 200:
|
|
2031
|
-
|
|
2032
|
-
self.logger.info('get_result failed with http code %d',
|
|
2033
|
-
status_code)
|
|
2034
|
-
self.logger.info('request = %s', request)
|
|
2035
|
-
return (status_code, None)
|
|
2036
|
-
|
|
2037
|
-
return (status_code, result)
|
|
2038
|
-
# pylint: enable=too-many-arguments
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
# pylint: disable=too-many-arguments
|
|
2042
|
-
def get_text(
|
|
2043
|
-
self,
|
|
2044
|
-
dataset: str,
|
|
2045
|
-
variables: List[str],
|
|
2046
|
-
start: Union[datetime, str],
|
|
2047
|
-
end: Union[datetime, str],
|
|
2048
|
-
compression: Compression = Compression.UNCOMPRESSED,
|
|
2049
|
-
text_format: TextFormat = TextFormat.PLAIN,
|
|
2050
|
-
**keywords
|
|
2051
|
-
) -> Tuple[int, Dict]:
|
|
2052
|
-
"""
|
|
2053
|
-
Gets a textual representation of the specified data from the
|
|
2054
|
-
server.
|
|
2055
|
-
|
|
2056
|
-
Parameters
|
|
2057
|
-
----------
|
|
2058
|
-
dataset
|
|
2059
|
-
dataset identifier of data to get.
|
|
2060
|
-
variables
|
|
2061
|
-
array containing names of variables to get.
|
|
2062
|
-
start
|
|
2063
|
-
start time of data to get. See module note about timezone.
|
|
2064
|
-
end
|
|
2065
|
-
end time of data to get. See module note about timezone.
|
|
2066
|
-
compression
|
|
2067
|
-
file compression.
|
|
2068
|
-
text_format
|
|
2069
|
-
text format.
|
|
2070
|
-
keywords
|
|
2071
|
-
optional keyword parameters as follows:<br>
|
|
2072
|
-
<b>binData</b> - indicates that uniformly spaced values should
|
|
2073
|
-
be computed for scaler/vector/spectrogram data according to
|
|
2074
|
-
the given binning parameter values. binData is a Dict that
|
|
2075
|
-
may contain the following keys: interval,
|
|
2076
|
-
interpolateMissingValues, sigmaMultiplier, and/or
|
|
2077
|
-
overrideDefaultBinning with values that override the
|
|
2078
|
-
defaults.<br>
|
|
2079
|
-
<b>progressCallback</b> - is a
|
|
2080
|
-
typing.Callable[[float, str, typing.Any], int]
|
|
2081
|
-
function that is called repeatedly to report the progress
|
|
2082
|
-
of getting the data. The function should return 0 if it
|
|
2083
|
-
wants to continue getting data. If it returns non-0 value,
|
|
2084
|
-
getting the data will be aborted and the get_data() function
|
|
2085
|
-
will immediately return (204, None). The float parameter
|
|
2086
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
2087
|
-
the str parameter will contain a text message indicating
|
|
2088
|
-
the progress of this call.<br>
|
|
2089
|
-
<b>progressUserValue</b> - is a typing.Any value that is
|
|
2090
|
-
passsed to the progressCallback function.<br>
|
|
2091
|
-
Returns
|
|
2092
|
-
-------
|
|
2093
|
-
Tuple
|
|
2094
|
-
[0] contains the HTTP status code value (200 when successful).<br>
|
|
2095
|
-
[1] contains a dictionary representation of a
|
|
2096
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
2097
|
-
DataResult object or None.<br>
|
|
2098
|
-
Raises
|
|
2099
|
-
------
|
|
2100
|
-
ValueError
|
|
2101
|
-
If the given start/end datetime values are invalid.
|
|
2102
|
-
"""
|
|
2103
|
-
# pylint: disable=too-many-locals
|
|
2104
|
-
# pylint: disable=too-many-return-statements
|
|
2105
|
-
# pylint: enable=too-many-statements
|
|
2106
|
-
# pylint: disable=too-many-branches
|
|
2107
|
-
|
|
2108
|
-
start_datetime, end_datetime = TimeInterval.get_datetimes(start,
|
|
2109
|
-
end)
|
|
2110
|
-
|
|
2111
|
-
request = TextRequest(dataset, variables,
|
|
2112
|
-
TimeInterval(start_datetime, end_datetime),
|
|
2113
|
-
compression, text_format,
|
|
2114
|
-
**keywords)
|
|
2115
|
-
|
|
2116
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
2117
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
2118
|
-
|
|
2119
|
-
self.logger.debug('request = %s', request)
|
|
2120
|
-
|
|
2121
|
-
if progress_callback is not None:
|
|
2122
|
-
if progress_callback(0.1, 'Making server request.',
|
|
2123
|
-
progress_user_value) != 0:
|
|
2124
|
-
return (204, None)
|
|
2125
|
-
|
|
2126
|
-
status_code, result = self.get_data_result(request,
|
|
2127
|
-
progress_callback,
|
|
2128
|
-
progress_user_value)
|
|
2129
|
-
|
|
2130
|
-
if progress_callback is not None:
|
|
2131
|
-
if progress_callback(1.0, 'Server request complete.',
|
|
2132
|
-
progress_user_value) != 0:
|
|
2133
|
-
return (status_code, None)
|
|
2134
|
-
|
|
2135
|
-
if status_code != 200:
|
|
2136
|
-
|
|
2137
|
-
self.logger.info('get_result failed with http code %d',
|
|
2138
|
-
status_code)
|
|
2139
|
-
self.logger.info('request = %s', request)
|
|
2140
|
-
return (status_code, None)
|
|
2141
|
-
|
|
2142
|
-
return (status_code, result)
|
|
2143
|
-
# pylint: enable=too-many-arguments
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
def get_audio(
|
|
2147
|
-
self,
|
|
2148
|
-
dataset: str,
|
|
2149
|
-
variables: List[str],
|
|
2150
|
-
start: Union[datetime, str],
|
|
2151
|
-
end: Union[datetime, str],
|
|
2152
|
-
**keywords
|
|
2153
|
-
) -> Tuple[int, Dict]:
|
|
2154
|
-
"""
|
|
2155
|
-
Gets an audio representation of the specified data from the
|
|
2156
|
-
server.
|
|
2157
|
-
|
|
2158
|
-
Parameters
|
|
2159
|
-
----------
|
|
2160
|
-
dataset
|
|
2161
|
-
dataset identifier of data to get.
|
|
2162
|
-
variables
|
|
2163
|
-
array containing names of variables to get.
|
|
2164
|
-
start
|
|
2165
|
-
start time of data to get. See module note about timezone.
|
|
2166
|
-
end
|
|
2167
|
-
end time of data to get. See module note about timezone.
|
|
2168
|
-
keywords
|
|
2169
|
-
optional keyword parameters as follows:<br>
|
|
2170
|
-
<b>binData</b> - indicates that uniformly spaced values should
|
|
2171
|
-
be computed for scaler/vector/spectrogram data according to
|
|
2172
|
-
the given binning parameter values. binData is a Dict that
|
|
2173
|
-
may contain the following keys: interval,
|
|
2174
|
-
interpolateMissingValues, sigmaMultiplier, and/or
|
|
2175
|
-
overrideDefaultBinning with values that override the
|
|
2176
|
-
defaults.<br>
|
|
2177
|
-
<b>progressCallback</b> - is a
|
|
2178
|
-
typing.Callable[[float, str, typing.Any], int]
|
|
2179
|
-
function that is called repeatedly to report the progress
|
|
2180
|
-
of getting the data. The function should return 0 if it
|
|
2181
|
-
wants to continue getting data. If it returns non-0 value,
|
|
2182
|
-
getting the data will be aborted and the get_data() function
|
|
2183
|
-
will immediately return (204, None). The float parameter
|
|
2184
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
2185
|
-
the str parameter will contain a text message indicating
|
|
2186
|
-
the progress of this call.<br>
|
|
2187
|
-
<b>progressUserValue</b> - is a typing.Any value that is
|
|
2188
|
-
passsed to the progressCallback function.<br>
|
|
2189
|
-
Returns
|
|
2190
|
-
-------
|
|
2191
|
-
Tuple
|
|
2192
|
-
[0] contains the HTTP status code value (200 when successful).<br>
|
|
2193
|
-
[1] contains a dictionary representation of a
|
|
2194
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
2195
|
-
DataResult object or None.<br>
|
|
2196
|
-
Raises
|
|
2197
|
-
------
|
|
2198
|
-
ValueError
|
|
2199
|
-
If the given start/end datetime values are invalid.
|
|
2200
|
-
"""
|
|
2201
|
-
# pylint: disable=too-many-locals
|
|
2202
|
-
# pylint: disable=too-many-return-statements
|
|
2203
|
-
# pylint: enable=too-many-statements
|
|
2204
|
-
# pylint: disable=too-many-branches
|
|
2205
|
-
|
|
2206
|
-
start_datetime, end_datetime = TimeInterval.get_datetimes(start,
|
|
2207
|
-
end)
|
|
2208
|
-
|
|
2209
|
-
request = AudioRequest(dataset, variables,
|
|
2210
|
-
TimeInterval(start_datetime, end_datetime),
|
|
2211
|
-
**keywords)
|
|
2212
|
-
|
|
2213
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
2214
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
2215
|
-
|
|
2216
|
-
self.logger.debug('request = %s', request)
|
|
2217
|
-
|
|
2218
|
-
if progress_callback is not None:
|
|
2219
|
-
if progress_callback(0.1, 'Making server request.',
|
|
2220
|
-
progress_user_value) != 0:
|
|
2221
|
-
return (204, None)
|
|
2222
|
-
|
|
2223
|
-
status_code, result = self.get_data_result(request,
|
|
2224
|
-
progress_callback,
|
|
2225
|
-
progress_user_value)
|
|
2226
|
-
|
|
2227
|
-
if progress_callback is not None:
|
|
2228
|
-
if progress_callback(1.0, 'Server request complete.',
|
|
2229
|
-
progress_user_value) != 0:
|
|
2230
|
-
return (status_code, None)
|
|
2231
|
-
|
|
2232
|
-
if status_code != 200:
|
|
2233
|
-
|
|
2234
|
-
self.logger.info('get_result failed with http code %d',
|
|
2235
|
-
status_code)
|
|
2236
|
-
self.logger.info('request = %s', request)
|
|
2237
|
-
return (status_code, None)
|
|
2238
|
-
|
|
2239
|
-
return (status_code, result)
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
def get_original_files(
|
|
2243
|
-
self,
|
|
2244
|
-
dataset: str,
|
|
2245
|
-
start: Union[datetime, str],
|
|
2246
|
-
end: Union[datetime, str],
|
|
2247
|
-
**keywords
|
|
2248
|
-
) -> Tuple[int, Dict]:
|
|
2249
|
-
"""
|
|
2250
|
-
Gets original data files from a dataset. Original data files
|
|
2251
|
-
lack updated meta-data and virtual variable values contained
|
|
2252
|
-
in files obtained from the `CdasWs.get_data`. Most callers
|
|
2253
|
-
should probably use `CdasWs.get_data` instead of this function.
|
|
2254
|
-
|
|
2255
|
-
Parameters
|
|
2256
|
-
----------
|
|
2257
|
-
dataset
|
|
2258
|
-
dataset identifier of data to get.
|
|
2259
|
-
start
|
|
2260
|
-
start time of data to get. See module note about timezone.
|
|
2261
|
-
end
|
|
2262
|
-
end time of data to get. See module note about timezone.
|
|
2263
|
-
keywords
|
|
2264
|
-
optional keyword parameters as follows:<br>
|
|
2265
|
-
<b>progressCallback</b> - is a
|
|
2266
|
-
typing.Callable[[float, str, typing.Any], int]
|
|
2267
|
-
function that is called repeatedly to report the progress
|
|
2268
|
-
of getting the data. The function should return 0 if it
|
|
2269
|
-
wants to continue getting data. If it returns non-0 value,
|
|
2270
|
-
getting the data will be aborted and the get_data() function
|
|
2271
|
-
will immediately return (204, None). The float parameter
|
|
2272
|
-
is a value between 0.0 and 1.0 to indicate progress and
|
|
2273
|
-
the str parameter will contain a text message indicating
|
|
2274
|
-
the progress of this call.<br>
|
|
2275
|
-
<b>progressUserValue</b> - is a typing.Any value that is
|
|
2276
|
-
passsed to the progressCallback function.<br>
|
|
2277
|
-
Returns
|
|
2278
|
-
-------
|
|
2279
|
-
Tuple
|
|
2280
|
-
[0] contains the HTTP status code value (200 when successful).<br>
|
|
2281
|
-
[1] array of dictionary representations of a
|
|
2282
|
-
<https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
|
|
2283
|
-
FileDescription objects or None.<br>
|
|
2284
|
-
Raises
|
|
2285
|
-
------
|
|
2286
|
-
ValueError
|
|
2287
|
-
If the given start/end datetime values are invalid.
|
|
2288
|
-
See Also
|
|
2289
|
-
--------
|
|
2290
|
-
CdasWs.get_data
|
|
2291
|
-
"""
|
|
2292
|
-
# pylint: disable=too-many-locals
|
|
2293
|
-
# pylint: disable=too-many-return-statements
|
|
2294
|
-
# pylint: enable=too-many-statements
|
|
2295
|
-
# pylint: disable=too-many-branches
|
|
2296
|
-
|
|
2297
|
-
start_datetime, end_datetime = TimeInterval.get_datetimes(start,
|
|
2298
|
-
end)
|
|
2299
|
-
|
|
2300
|
-
request = CdfRequest(dataset, [],
|
|
2301
|
-
TimeInterval(start_datetime, end_datetime))
|
|
2302
|
-
|
|
2303
|
-
progress_callback = keywords.get('progressCallback', None)
|
|
2304
|
-
progress_user_value = keywords.get('progressUserValue', None)
|
|
2305
|
-
|
|
2306
|
-
self.logger.debug('request = %s', request)
|
|
2307
|
-
|
|
2308
|
-
if progress_callback is not None:
|
|
2309
|
-
if progress_callback(0.1, 'Making server request.',
|
|
2310
|
-
progress_user_value) != 0:
|
|
2311
|
-
return (204, None)
|
|
2312
|
-
|
|
2313
|
-
status_code, result = self.get_data_result(request,
|
|
2314
|
-
progress_callback,
|
|
2315
|
-
progress_user_value)
|
|
2316
|
-
|
|
2317
|
-
if progress_callback is not None:
|
|
2318
|
-
if progress_callback(1.0, 'Server request complete.',
|
|
2319
|
-
progress_user_value) != 0:
|
|
2320
|
-
return (status_code, None)
|
|
2321
|
-
|
|
2322
|
-
if status_code != 200:
|
|
2323
|
-
|
|
2324
|
-
self.logger.info('get_result failed with http code %d',
|
|
2325
|
-
status_code)
|
|
2326
|
-
self.logger.info('request = %s', request)
|
|
2327
|
-
return (status_code, None)
|
|
2328
|
-
|
|
2329
|
-
return (status_code, result['FileDescription'])
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
def get_ssc_id(
|
|
2333
|
-
self,
|
|
2334
|
-
dataset: str
|
|
2335
|
-
) -> Tuple[int, Union[str, List[str]]]:
|
|
2336
|
-
"""
|
|
2337
|
-
Gets the Satellite Situation Center (SSC)
|
|
2338
|
-
<https://sscweb.gsfc.nasa.gov/> observatory identifier(s)
|
|
2339
|
-
associated with the given cdaweb dataset identifier.
|
|
2340
|
-
|
|
2341
|
-
Notes
|
|
2342
|
-
-----
|
|
2343
|
-
This method relies upon the Heliophysics Data Portal's
|
|
2344
|
-
<https://heliophysicsdata.gsfc.nasa.gov/> metadata. That metadata
|
|
2345
|
-
may be incomplete. Also, cdaweb has datasets for which SSC has
|
|
2346
|
-
no corresponding observatory (for example, ground observatory
|
|
2347
|
-
data). Callers should be prepared for negative results (200, None)
|
|
2348
|
-
from this method.
|
|
2349
|
-
|
|
2350
|
-
Parameters
|
|
2351
|
-
----------
|
|
2352
|
-
dataset
|
|
2353
|
-
cdaweb dataset identifier.
|
|
2354
|
-
Returns
|
|
2355
|
-
-------
|
|
2356
|
-
Tuple
|
|
2357
|
-
[0] contains the HTTP status code value (200 when successful).<br>
|
|
2358
|
-
[1] the SSC observatory identifier(s) associated with the given
|
|
2359
|
-
cdaweb dataset identifier or None if none is found.
|
|
2360
|
-
"""
|
|
2361
|
-
url = self._hdp_registry + '?cdawebId=' + dataset
|
|
2362
|
-
|
|
2363
|
-
self.logger.debug('request url = %s', url)
|
|
2364
|
-
|
|
2365
|
-
response = self._session.get(url, timeout=self._timeout)
|
|
2366
|
-
|
|
2367
|
-
if response.status_code != 200:
|
|
2368
|
-
|
|
2369
|
-
self.logger.info('%s failed with http code %d', url,
|
|
2370
|
-
response.status_code)
|
|
2371
|
-
self.logger.info('response.text: %s', response.text)
|
|
2372
|
-
return (response.status_code, None)
|
|
2373
|
-
|
|
2374
|
-
if self.logger.level <= logging.DEBUG:
|
|
2375
|
-
self.logger.debug('response.text = %s', response.text)
|
|
2376
|
-
|
|
2377
|
-
results = ET.fromstring(response.text)
|
|
2378
|
-
|
|
2379
|
-
ssc_id = []
|
|
2380
|
-
for ssc_id_elem in results.findall('SscId'):
|
|
2381
|
-
ssc_id.append(ssc_id_elem.text)
|
|
2382
|
-
|
|
2383
|
-
if len(ssc_id) == 0:
|
|
2384
|
-
result = None
|
|
2385
|
-
elif len(ssc_id) == 1:
|
|
2386
|
-
result = ssc_id[0]
|
|
2387
|
-
else:
|
|
2388
|
-
result = ssc_id
|
|
2389
|
-
|
|
2390
|
-
return (response.status_code, result)
|
|
82
|
+
#
|
|
83
|
+
# For backward compatibility with cdasws versions < 1.8.12.
|
|
84
|
+
#
|
|
85
|
+
from cdasws.cdasws import *
|