cdasws 1.8.11__py3-none-any.whl → 1.8.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cdasws/__init__.py CHANGED
@@ -24,7 +24,7 @@
24
24
  #
25
25
  # NOSA HEADER END
26
26
  #
27
- # Copyright (c) 2018-2025 United States Government as represented by
27
+ # Copyright (c) 2018-2026 United States Government as represented by
28
28
  # the National Aeronautics and Space Administration. No copyright is
29
29
  # claimed in the United States under Title 17, U.S.Code. All Other
30
30
  # Rights Reserved.
@@ -35,7 +35,7 @@
35
35
  Package for accessing the Coordinate Data Analysis System (CDAS)
36
36
  web services <https://cdaweb.gsfc.nasa.gov/WebServices/REST/>.<br>
37
37
 
38
- Copyright &copy; 2018-2025 United States Government as represented by the
38
+ Copyright &copy; 2018-2026 United States Government as represented by the
39
39
  National Aeronautics and Space Administration. No copyright is claimed in
40
40
  the United States under Title 17, U.S.Code. All Other Rights Reserved.
41
41
 
@@ -55,96 +55,7 @@ Notes
55
55
  """
56
56
 
57
57
 
58
- import sys
59
- import os
60
- import platform
61
- import logging
62
- import re
63
- from importlib.util import find_spec
64
- import urllib.parse
65
- from urllib.parse import urlparse
66
- import json
67
- from operator import itemgetter
68
- import time
69
- from datetime import datetime, timedelta, timezone
70
- import xml.etree.ElementTree as ET
71
- from tempfile import mkstemp
72
- from typing import Any, Callable, Dict, List, Tuple, Union
73
-
74
- import requests
75
- import dateutil.parser
76
-
77
- from cdasws.datarepresentation import DataRepresentation
78
- from cdasws.datarequest import AudioRequest, DataRequest
79
- from cdasws.datarequest import CdfFormat, CdfRequest, Compression
80
- from cdasws.datarequest import ImageFormat, GraphOptions, GraphRequest
81
- from cdasws.datarequest import TextFormat, TextRequest, ThumbnailRequest
82
- from cdasws.timeinterval import TimeInterval
83
-
84
-
85
- # requires python >= 3.4
86
- #if find_spec('spacepy.datamodel') is not None:
87
- # import spacepy.datamodel as spdm # type: ignore
88
- # SPDM_AVAILABLE = True
89
- #else:
90
- # SPDM_AVAILABLE = False
91
- # python < 3.4
92
- try:
93
- import spacepy.datamodel as spdm # type: ignore
94
- SPDM_AVAILABLE = True
95
- except ImportError:
96
- SPDM_AVAILABLE = False
97
-
98
- try:
99
- from cdflib.xarray import cdf_to_xarray
100
- import xarray as xr
101
- CDF_XARRAY_AVAILABLE = True
102
- except ImportError:
103
- try:
104
- import cdflib as cdf
105
- import xarray as xr # pylint: disable=ungrouped-imports
106
- CDF_XARRAY_AVAILABLE = True
107
- def cdf_to_xarray(filename, to_datetime=False, to_unixtime=False,
108
- fillval_to_nan=False):
109
- """
110
- Reads a CDF into an xarray.dataset. This function exists
111
- to provide compatility with cdflib >= 1.0.1 for older
112
- releases of cdflib.
113
-
114
- Parameters:
115
- -----------
116
- filename
117
- The path to the CDF file to read.
118
- to_datetime
119
- Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to
120
- datetime, or leave them as is.
121
- to_unixtime
122
- Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to
123
- unixtime, or leave them as is.
124
- fillval_to_nan
125
- If True, any data values that match the FILLVAL
126
- attribute for a variable will be set to NaN.
127
-
128
- Returns
129
- -------
130
- xarray.dataset
131
- An XArray Dataset object.
132
- """
133
- return cdf.cdf_to_xarray(filename, to_datetime=to_datetime, # pylint: disable=no-member
134
- to_unixtime=to_unixtime,
135
- fillval_to_nan=fillval_to_nan)
136
- except ImportError:
137
- CDF_XARRAY_AVAILABLE = False
138
-
139
-
140
- try:
141
- import requests_cache
142
- CACHE_AVAILABLE = True
143
- except ImportError:
144
- CACHE_AVAILABLE = False
145
-
146
-
147
- __version__ = "1.8.11"
58
+ __version__ = "1.8.13"
148
59
 
149
60
 
150
61
  #
@@ -168,2223 +79,7 @@ NAMESPACES = {
168
79
  'cdas': NS,
169
80
  'xhtml': XHTML_NS
170
81
  }
171
-
172
-
173
- def _get_data_progress(
174
- progress: float,
175
- msg: str,
176
- value: Dict) -> int:
177
- """
178
- A get_data progress callback which adjusts the progress value for
179
- the download portion of a larger operation and then calls the
180
- "real" progress callback function with this adjusted progress value.
181
-
182
- Parameters
183
- ----------
184
- progress
185
- Measure of progress.
186
- msg
187
- Message describing progress of get_data call.
188
- value
189
- Dictionary containing the function to call and values for
190
- computing the adjusted progress value.
191
- Returns
192
- -------
193
- int
194
- Flag indicating whether to continue with getting the data.
195
- 0 to continue. 1 to abort getting the data.
196
- """
197
- progress_callback = value.get('progressCallback', None)
198
- progress_user_value = value.get('progressUserValue', None)
199
- adjusted_progress = value['progressStart'] + \
200
- value['progressFraction'] * progress
201
-
202
- if progress_callback is not None:
203
-
204
- return progress_callback(adjusted_progress, msg,
205
- progress_user_value)
206
- return 0
207
-
208
-
209
- class NullAuth(requests.auth.AuthBase): # pylint: disable=too-few-public-methods
210
- """
211
- Authentication class used to cause requests to ignore any ~/.netrc
212
- file. The CDAS web services do not support authentication and
213
- a cdaweb (ftps) entry will cause CdasWs requests to fail with
214
- a 401 error. See <https://github.com/psf/requests/issues/2773>.
215
- """
216
- def __call__(self, r):
217
- return r
218
-
219
-
220
- class CdasWs:
221
- """
222
- Class representing the web service interface to NASA's
223
- Coordinated Data Analysis System (CDAS)
224
- <https://cdaweb.gsfc.nasa.gov>.
225
-
226
- Notes
227
- -----
228
- The logger used by this class has the class' name (CdasWs). By default,
229
- it is configured with a NullHandler. Users of this class may configure
230
- the logger to aid in diagnosing problems.
231
- """
232
- # pylint: disable=too-many-instance-attributes
233
- # pylint: disable=too-many-arguments
234
- def __init__(
235
- self,
236
- endpoint=None,
237
- timeout=None,
238
- proxy=None,
239
- ca_certs=None,
240
- disable_ssl_certificate_validation=False,
241
- user_agent=None,
242
- disable_cache=False):
243
- """
244
- Creates an object representing the CDAS web services.
245
-
246
- Parameters
247
- ----------
248
- endpoint
249
- URL of the CDAS web service. If None, the default is
250
- 'https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/'.
251
- timeout
252
- Number of seconds to wait for a response from the server.
253
- proxy
254
- HTTP proxy information. For example,<pre>
255
- proxies = {
256
- 'http': 'http://10.10.1.10:3128',
257
- 'https': 'http://10.10.1.10:1080',
258
- }</pre>
259
- Proxy information can also be set with environment variables.
260
- For example,<pre>
261
- $ export HTTP_PROXY="http://10.10.1.10:3128"
262
- $ export HTTPS_PROXY="http://10.10.1.10:1080"</pre>
263
- ca_certs
264
- Path to certificate authority (CA) certificates that will
265
- override the default bundle.
266
- disable_ssl_certificate_validation
267
- Flag indicating whether to validate the SSL certificate.
268
- user_agent
269
- A value that is appended to the HTTP User-Agent value.
270
- disable_cache
271
- Flag indicating whether to disable HTTP caching.
272
- """
273
-
274
- self.logger = logging.getLogger(type(self).__name__)
275
- self.logger.addHandler(logging.NullHandler())
276
-
277
- self.logger.debug('endpoint = %s', endpoint)
278
- self.logger.debug('ca_certs = %s', ca_certs)
279
- self.logger.debug('disable_ssl_certificate_validation = %s',
280
- disable_ssl_certificate_validation)
281
- self.logger.debug('disable_cache = %s', disable_cache)
282
-
283
- if endpoint is None:
284
- self._endpoint = 'https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/'
285
- else:
286
- self._endpoint = endpoint
287
-
288
- self._user_agent = 'cdasws/' + __version__ + ' (' + \
289
- platform.python_implementation() + ' ' \
290
- + platform.python_version() + '; ' + platform.platform() + ')'
291
-
292
- if user_agent is not None:
293
- self._user_agent += ' (' + user_agent + ')'
294
-
295
- self._request_headers = {
296
- #'Content-Type' : 'application/json',
297
- 'Content-Type' : 'application/xml',
298
- 'Accept' : 'application/xml',
299
- 'User-Agent' : self._user_agent,
300
- #'Accept-Encoding' : 'gzip' # only beneficial for icdfml responses
301
- }
302
- if CACHE_AVAILABLE and disable_cache is not True:
303
- self._session = requests_cache.CachedSession('cdasws_cache',
304
- cache_control=True)
305
- else:
306
- self._session = requests.Session()
307
-
308
- self._session.headers.update(self._request_headers)
309
- self._session.auth = NullAuth()
310
-
311
- if ca_certs is not None:
312
- self._session.verify = ca_certs
313
-
314
- if disable_ssl_certificate_validation is True:
315
- self._session.verify = False
316
-
317
- if proxy is not None:
318
- self._proxy = proxy
319
-
320
- self._timeout = timeout
321
-
322
- endpoint_components = urlparse(self._endpoint)
323
- self._hdp_registry = endpoint_components.scheme + '://' + \
324
- endpoint_components.netloc + '/registry/hdp/SscId.xql'
325
-
326
- # pylint: enable=too-many-arguments
327
-
328
-
329
- def __del__(self):
330
- """
331
- Destructor. Closes all network connections.
332
- """
333
-
334
- self.close()
335
-
336
-
337
- def close(self) -> None:
338
- """
339
- Closes any persistent network connections. Generally, deleting
340
- this object is sufficient and calling this method is unnecessary.
341
- """
342
- self._session.close()
343
-
344
-
345
- def get_observatory_groups(
346
- self,
347
- **keywords: str
348
- ) -> List[Dict]:
349
- """
350
- Gets descriptions of the observatory groups from the server.
351
-
352
- Parameters
353
- ----------
354
- keywords
355
- optional keyword parameters as follows:<br>
356
- <b>instrumentType</b> - an instrument type value from those
357
- returned by `CdasWs.get_instrument_types`. Omitting
358
- this parameter indicates that no observatories are eliminated
359
- based upon their instrumentType value.
360
- Returns
361
- -------
362
- List
363
- An array of ObservatoryGroupDescription
364
- dictionaries where the structure of the dictionary mirrors
365
- ObservatoryGroupDescription in
366
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
367
- """
368
- if 'instrumentType' in keywords:
369
- url = self._endpoint + 'observatoryGroups?instrumentType=' + \
370
- urllib.parse.quote(keywords['instrumentType'])
371
- else:
372
- url = self._endpoint + 'observatoryGroups'
373
-
374
- self.logger.debug('request url = %s', url)
375
-
376
- response = self._session.get(url, timeout=self._timeout)
377
-
378
- if response.status_code != 200:
379
-
380
- self.logger.info('%s failed with http code %d', url,
381
- response.status_code)
382
- self.logger.info('response.text: %s', response.text)
383
- return []
384
-
385
- if self.logger.level <= logging.DEBUG:
386
- self.logger.debug('response.text = %s', response.text)
387
-
388
- observatory_response = ET.fromstring(response.text)
389
-
390
- observatory_group_descriptions = []
391
- for description in observatory_response.findall(\
392
- 'cdas:ObservatoryGroupDescription', namespaces=NAMESPACES):
393
-
394
- observatory_ids = []
395
- for observatory_id in description.findall(\
396
- 'cdas:ObservatoryId', namespaces=NAMESPACES):
397
-
398
- observatory_ids.append(observatory_id.text)
399
-
400
- observatory_group_descriptions.append({
401
- 'Name': description.find(\
402
- 'cdas:Name', namespaces=NAMESPACES).text,
403
- 'ObservatoryId': observatory_ids
404
- })
405
-
406
- return observatory_group_descriptions
407
-
408
-
409
- def get_instrument_types(
410
- self,
411
- **keywords: str
412
- ) -> List[Dict]:
413
- """
414
- Gets descriptions of the instrument types from the server.
415
-
416
- Parameters
417
- ----------
418
- keywords
419
- optional keyword parameters as follows:<br>
420
- <b>observatory</b> - an observatory value from those returned
421
- by `CdasWs.get_observatories`. Omitting this parameter
422
- indicates that no instrumentTypes are eliminated based upon
423
- their observatory value.<br>
424
- <b>observatoryGroup</b> - an observatory group value from
425
- those returned by `CdasWs.get_observatory_groups`. Omitting
426
- this parameter indicates that no instrumentTypes are
427
- eliminated based upon their observatoryGroup value.</br>
428
- Returns
429
- -------
430
- List
431
- An array of InstrumentTypeDescription
432
- dictionaries where the structure of the dictionary mirrors
433
- InstrumentTypeDescription in
434
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
435
- """
436
- if 'observatory' in keywords:
437
- url = self._endpoint + 'instrumentTypes?observatory=' \
438
- + urllib.parse.quote(keywords['observatory'])
439
- elif 'observatoryGroup' in keywords:
440
- url = self._endpoint + 'instrumentTypes?observatoryGroup=' \
441
- + urllib.parse.quote(keywords['observatoryGroup'])
442
- else:
443
- url = self._endpoint + 'instrumentTypes'
444
-
445
- self.logger.debug('request url = %s', url)
446
-
447
- response = self._session.get(url, timeout=self._timeout)
448
-
449
- if response.status_code != 200:
450
-
451
- self.logger.info('%s failed with http code %d', url,
452
- response.status_code)
453
- self.logger.info('response.text: %s', response.text)
454
- return []
455
-
456
- if self.logger.level <= logging.DEBUG:
457
- self.logger.debug('response.text = %s', response.text)
458
-
459
- instrument_response = ET.fromstring(response.text)
460
-
461
- if self.logger.level <= logging.DEBUG:
462
- self.logger.debug('instrument_response = %s',
463
- ET.tostring(instrument_response))
464
-
465
- instrument_types = []
466
- for description in instrument_response.findall(\
467
- 'cdas:InstrumentTypeDescription', namespaces=NAMESPACES):
468
-
469
- instrument_types.append({
470
- 'Name': description.find('cdas:Name',
471
- namespaces=NAMESPACES).text
472
- })
473
- return instrument_types
474
-
475
-
476
- def get_instruments(
477
- self,
478
- **keywords: str
479
- ) -> List[Dict]:
480
- """
481
- Gets descriptions of the instruments from the server.
482
-
483
- Parameters
484
- ----------
485
- keywords
486
- optional keyword parameters as follows:<br>
487
- <b>observatory</b> - an observatory value from those returned
488
- by `CdasWs.get_observatories`. Omitting this parameter
489
- indicates that no instruments are eliminated based upon their
490
- observatory value.<br>
491
- <b>instrumentType</b> - an instrument type value from those
492
- returned by `CdasWs.get_instrument_types`. Omitting this
493
- parameter indicates that no instruments are eliminated based
494
- upon their instrument type.<br>
495
- Returns
496
- -------
497
- List
498
- An array of InstrumentDescription
499
- dictionaries where the structure of the dictionary mirrors
500
- InstrumentDescription in
501
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
502
- """
503
- if 'observatory' in keywords:
504
- url = self._endpoint + 'instruments?observatory=' \
505
- + urllib.parse.quote(keywords['observatory'])
506
- elif 'instrumentType' in keywords:
507
- url = self._endpoint + 'instruments?instrumentType=' \
508
- + urllib.parse.quote(keywords['instrumentType'])
509
- else:
510
- url = self._endpoint + 'instruments'
511
-
512
- self.logger.debug('request url = %s', url)
513
-
514
- response = self._session.get(url, timeout=self._timeout)
515
-
516
- if response.status_code != 200:
517
-
518
- self.logger.info('%s failed with http code %d', url,
519
- response.status_code)
520
- self.logger.info('response.text: %s', response.text)
521
- return []
522
-
523
- if self.logger.level <= logging.DEBUG:
524
- self.logger.debug('response.text = %s', response.text)
525
-
526
- instruments_response = ET.fromstring(response.text)
527
-
528
- if self.logger.level <= logging.DEBUG:
529
- self.logger.debug('instruments = %s', response.text)
530
- #ET.indent(instruments_response, space=' '))
531
-
532
- instruments = []
533
- for instrument_description in instruments_response.findall(\
534
- 'cdas:InstrumentDescription', namespaces=NAMESPACES):
535
-
536
- instruments.append({
537
- 'Name': instrument_description.find(\
538
- 'cdas:Name', namespaces=NAMESPACES).text,
539
- 'ShortDescription': instrument_description.find(\
540
- 'cdas:ShortDescription', namespaces=NAMESPACES).text,
541
- 'LongDescription': instrument_description.find(\
542
- 'cdas:LongDescription', namespaces=NAMESPACES).text
543
- })
544
-
545
- return instruments
546
-
547
-
548
- def get_observatories(
549
- self,
550
- **keywords: str
551
- ) -> List[Dict]:
552
- """
553
- Gets descriptions of the observatories from the server.
554
-
555
- Parameters
556
- ----------
557
- keywords
558
- optional keyword parameters as follows:<br>
559
- <b>instrument</b> - an instrument value from those returned
560
- by `CdasWs.get_instruments`. Omitting this parameter
561
- indicates that no observatories are eliminated based upon
562
- their instrument value.<br>
563
- <b>instrumentType</b> - in instrument type value from those
564
- returned by `CdasWs.get_instrument_types`. Omitting this
565
- parameter indicates that no observatories are eliminated
566
- based upon their instrumentType value.<br>
567
- Returns
568
- -------
569
- List
570
- An array of ObservatoryDescriptions
571
- dictionaries where the structure of the dictionary mirrors
572
- ObservatoryDescription in
573
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
574
- """
575
- if 'instrument' in keywords:
576
- url = self._endpoint + 'observatories?instrument=' \
577
- + urllib.parse.quote(keywords['instrument'])
578
- elif 'instrumentType' in keywords:
579
- url = self._endpoint + 'observatories?instrumentType=' \
580
- + urllib.parse.quote(keywords['instrumentType'])
581
- else:
582
- url = self._endpoint + 'observatories'
583
-
584
- self.logger.debug('request url = %s', url)
585
-
586
- response = self._session.get(url, timeout=self._timeout)
587
-
588
- if response.status_code != 200:
589
-
590
- self.logger.info('%s failed with http code %d', url,
591
- response.status_code)
592
- self.logger.info('response.text: %s', response.text)
593
- return []
594
-
595
- if self.logger.level <= logging.DEBUG:
596
- self.logger.debug('response.text = %s', response.text)
597
-
598
- observatory_response = ET.fromstring(response.text)
599
-
600
- if self.logger.level <= logging.DEBUG:
601
- self.logger.debug('observatories = %s', response.text)
602
-
603
- observatories = []
604
-
605
- for observatory in observatory_response.findall(\
606
- 'cdas:ObservatoryDescription', namespaces=NAMESPACES):
607
- observatories.append({
608
- 'Name': observatory.find(\
609
- 'cdas:Name', namespaces=NAMESPACES).text,
610
- 'ShortDescription': observatory.find(\
611
- 'cdas:ShortDescription', namespaces=NAMESPACES).text,
612
- 'LongDescription': observatory.find(\
613
- 'cdas:LongDescription', namespaces=NAMESPACES).text
614
- })
615
-
616
- return observatories
617
-
618
-
619
- def get_observatory_groups_and_instruments(
620
- self,
621
- **keywords: str
622
- ) -> List[Dict]:
623
- """
624
- Gets descriptions of the observatory groups (and associated
625
- instruments) from the server.
626
-
627
- Parameters
628
- ----------
629
- keywords
630
- optional keyword parameters as follows:<br>
631
- <b>instrumentType</b> - an instrument type value from those
632
- returned by `CdasWs.get_instrument_types`. Omitting this
633
- parameter indicates that no observatories are eliminated
634
- based upon their instrumentType value.<br>
635
- Returns
636
- -------
637
- List
638
- An array of ObservatoryGroupInstrumentDescription
639
- dictionaries where the structure of the dictionary mirrors
640
- ObservatoryGroupInstrumentDescription in
641
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
642
- """
643
- if 'instrumentType' in keywords:
644
- url = self._endpoint \
645
- + 'observatoryGroupsAndInstruments?instrumentType=' \
646
- + urllib.parse.quote(keywords['instrumentType'])
647
- else:
648
- url = self._endpoint + 'observatoryGroupsAndInstruments'
649
-
650
- self.logger.debug('request url = %s', url)
651
-
652
- response = self._session.get(url, timeout=self._timeout)
653
-
654
- if response.status_code != 200:
655
-
656
- self.logger.info('%s failed with http code %d', url,
657
- response.status_code)
658
- self.logger.info('response.text: %s', response.text)
659
- return []
660
-
661
- if self.logger.level <= logging.DEBUG:
662
- self.logger.debug('response.text = %s', response.text)
663
-
664
- observatories_response = ET.fromstring(response.text)
665
-
666
- if self.logger.level <= logging.DEBUG:
667
- self.logger.debug('observatories = %s', response.text)
668
-
669
- o_g_i_ds = []
670
-
671
- for o_g_i_d in observatories_response.findall(\
672
- 'cdas:ObservatoryGroupInstrumentDescription',\
673
- namespaces=NAMESPACES):
674
-
675
- o_g_i_d_name = o_g_i_d.find('cdas:Name',
676
- namespaces=NAMESPACES).text
677
- o_is = []
678
- for o_i in o_g_i_d.findall('cdas:ObservatoryInstruments',
679
- namespaces=NAMESPACES):
680
-
681
- o_i_name = o_i.find('cdas:Name',
682
- namespaces=NAMESPACES).text
683
- i_ds = []
684
- for i_d in o_i.findall('cdas:InstrumentDescription',
685
- namespaces=NAMESPACES):
686
- i_d_name = i_d.find('cdas:Name',
687
- namespaces=NAMESPACES).text
688
- i_d_short_description = \
689
- i_d.find('cdas:ShortDescription',
690
- namespaces=NAMESPACES).text
691
- i_d_long_description = \
692
- i_d.find('cdas:LongDescription',
693
- namespaces=NAMESPACES).text
694
- i_ds.append({
695
- 'Name': i_d_name,
696
- 'ShortDescription': i_d_short_description,
697
- 'LongDescription': i_d_long_description
698
- })
699
- o_is.append({
700
- 'Name': o_i_name,
701
- 'InstrumentDescription': i_ds
702
- })
703
-
704
- o_g_i_ds.append({
705
- 'Name': o_g_i_d_name,
706
- 'ObservatoryInstruments': o_is
707
- })
708
-
709
- return o_g_i_ds
710
-
711
-
712
- # pylint: disable=too-many-branches
713
- def get_datasets(
714
- self,
715
- **keywords: str
716
- ) -> List[Dict]:
717
- """
718
- Gets descriptions of the specified datasets from the server.
719
-
720
- Parameters
721
- ----------
722
- keywords
723
- optional keyword parameters as follows:<br>
724
- <b>observatoryGroup</b> - an observatory group value from those
725
- returned by `CdasWs.get_observatory_groups`. Omitting this
726
- parameter
727
- indicates that no datasets are eliminated based upon their
728
- observatoryGroup value.<br>
729
- <b>instrumentType</b> - an instrument type value from those
730
- returned by `CdasWs.get_instrument_types`. Omitting this
731
- parameter indicates that no datasets are eliminated based
732
- upon their instrumentType value.<br>
733
- <b>observatory</b> - an observatory name value from those
734
- returned by `CdasWs.get_observatories`. Omitting this
735
- parameter indicates that no datasets are eliminated based
736
- upon their observatory value.<br>
737
- <b>instrument</b> - an instrument value from those returned by
738
- `CdasWs.get_instruments`. Omitting this parameter indicates
739
- that no datasets are eliminated based upon their instrument
740
- value.<br>
741
- <b>startDate</b> - a datetime specifying the start of a time
742
- interval. See module note about timezone value. If this
743
- parameter is ommited, the time interval will begin infinitely
744
- in the past.<br>
745
- <b>stopDate</b> - a datetime specifying the end of a time
746
- interval. See module note about timezone value. If this
747
- parameter is omitted, the time interval will end infinitely
748
- in the future.<br>
749
- <b>id</b> - a dataset identifier. The value may be a CDAS
750
- (e.g., AC_H2_MFI), DOI (e.g., 10.48322/fh85-fj47), or SPASE
751
- [ResourceID] (e.g., spase://NASA/NumericalData/ACE/MAG/L2/PT1H)
752
- identifier. If specified, all other keywords are ignored.<br>
753
- <b>idPattern</b> - a java.util.regex compatible regular
754
- expression that must match the dataset's CDAS identifier value.
755
- Omitting this parameter is equivalent to `.*`.<br>
756
- <b>labelPattern</b> - a java.util.regex compatible regular
757
- expression that must match the dataset's CDAS label text.
758
- Omitting this parameter is equivalent to `.*`. Embedded
759
- matching flag expressions (e.g., `(?i)` for case insensitive
760
- match mode) are supported and likely to be useful in this
761
- case.<br>
762
- <b>notesPattern</b> - a java.util.regex compatible regular
763
- expression that must match the dataset's CDAS notes text.
764
- Omitting this parameter is equivalent to `.*`. Embedded
765
- matching flag expressions (e.g., `(?s)` for dotall match mode)
766
- are supported and likely to be useful in this case.<br>
767
- Returns
768
- -------
769
- List
770
- A list of dictionaries containing descriptions of the datasets
771
- requested. The dictionary structure is defined by the
772
- DatasetDescription element in
773
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
774
- """
775
- url = self._endpoint + 'datasets?'
776
-
777
- observatory_groups = keywords.get('observatoryGroup', None)
778
- if observatory_groups is not None:
779
- if isinstance(observatory_groups, str):
780
- observatory_groups = [observatory_groups]
781
- for observatory_group in observatory_groups:
782
- url = url + 'observatoryGroup=' \
783
- + urllib.parse.quote(observatory_group) + '&'
784
-
785
- instrument_types = keywords.get('instrumentType', None)
786
- if instrument_types is not None:
787
- if isinstance(instrument_types, str):
788
- instrument_types = [instrument_types]
789
- for instrument_type in instrument_types:
790
- url = url + 'instrumentType=' \
791
- + urllib.parse.quote(instrument_type) + '&'
792
-
793
- observatories = keywords.get('observatory', None)
794
- if observatories is not None:
795
- if isinstance(observatories, str):
796
- observatories = [observatories]
797
- for observatory in observatories:
798
- url = url + 'observatory=' \
799
- + urllib.parse.quote(observatory) + '&'
800
-
801
- instruments = keywords.get('instrument', None)
802
- if instruments is not None:
803
- if isinstance(instruments, str):
804
- instruments = [instruments]
805
- for instrument in instruments:
806
- url = url + 'instrument=' \
807
- + urllib.parse.quote(instrument) + '&'
808
-
809
- if 'startDate' in keywords:
810
- url = url + 'startDate=' \
811
- + urllib.parse.quote(keywords['startDate']) + '&'
812
-
813
- if 'stopDate' in keywords:
814
- url = url + 'stopDate=' \
815
- + urllib.parse.quote(keywords['stopDate']) + '&'
816
-
817
- if 'id' in keywords:
818
- url = url + 'id=' \
819
- + urllib.parse.quote(keywords['id']) + '&'
820
-
821
- if 'idPattern' in keywords:
822
- url = url + 'idPattern=' \
823
- + urllib.parse.quote(keywords['idPattern']) + '&'
824
-
825
- if 'labelPattern' in keywords:
826
- url = url + 'labelPattern=' \
827
- + urllib.parse.quote(keywords['labelPattern']) + '&'
828
-
829
- if 'notesPattern' in keywords:
830
- url = url + 'notesPattern=' \
831
- + urllib.parse.quote(keywords['notesPattern']) + '&'
832
-
833
- self.logger.debug('request url = %s', url[:-1])
834
-
835
- response = self._session.get(url[:-1], timeout=self._timeout)
836
-
837
- if response.status_code != 200:
838
-
839
- self.logger.info('%s failed with http code %d', url,
840
- response.status_code)
841
- self.logger.info('response.text: %s', response.text)
842
- return []
843
-
844
- if self.logger.level <= logging.DEBUG:
845
- self.logger.debug('response.text = %s', response.text)
846
-
847
- dss = ET.fromstring(response.text)
848
-
849
- if self.logger.level <= logging.DEBUG:
850
- self.logger.debug('datasets = %s', response.text)
851
-
852
- datasets = []
853
- for ds in dss.findall('cdas:DatasetDescription',
854
- namespaces=NAMESPACES):
855
-
856
- observatory_groups = []
857
- for o_g in ds.findall('cdas:ObservatoryGroup',
858
- namespaces=NAMESPACES):
859
- observatory_groups.append(o_g.text)
860
-
861
- instrument_types = []
862
- for i_t in ds.findall('cdas:InstrumentType',
863
- namespaces=NAMESPACES):
864
- instrument_types.append(i_t.text)
865
-
866
- dataset_links = []
867
- for d_l in ds.findall('cdas:DatasetLink',
868
- namespaces=NAMESPACES):
869
- dataset_links.append({
870
- 'Title': d_l.find('cdas:Title',
871
- namespaces=NAMESPACES).text,
872
- 'Text': d_l.find('cdas:Text',
873
- namespaces=NAMESPACES).text,
874
- 'Url': d_l.find('cdas:Url',
875
- namespaces=NAMESPACES).text,
876
- })
877
-
878
- observatories = []
879
- for obs_elem in ds.findall('cdas:Observatory',
880
- namespaces=NAMESPACES):
881
- observatories.append(obs_elem.text)
882
-
883
- instruments = []
884
- for instr_elem in ds.findall('cdas:Instrument',
885
- namespaces=NAMESPACES):
886
- instruments.append(instr_elem.text)
887
-
888
- dataset = {
889
- 'Id': ds.find('cdas:Id', namespaces=NAMESPACES).text,
890
- 'Observatory': observatories,
891
- 'Instrument': instruments,
892
- 'ObservatoryGroup': observatory_groups,
893
- 'InstrumentType': instrument_types,
894
- 'Label': ds.find('cdas:Label',
895
- namespaces=NAMESPACES).text,
896
- 'TimeInterval': {
897
- 'Start': ds.find('cdas:TimeInterval/cdas:Start',
898
- namespaces=NAMESPACES).text,
899
- 'End': ds.find('cdas:TimeInterval/cdas:End',
900
- namespaces=NAMESPACES).text
901
- },
902
- 'PiName': ds.find('cdas:PiName',
903
- namespaces=NAMESPACES).text,
904
- 'PiAffiliation': ds.find('cdas:PiAffiliation',
905
- namespaces=NAMESPACES).text,
906
- 'Notes': ds.find('cdas:Notes',
907
- namespaces=NAMESPACES).text,
908
- 'DatasetLink': dataset_links
909
- }
910
- doi = ds.find('cdas:Doi', namespaces=NAMESPACES)
911
- if doi is not None:
912
- dataset['Doi'] = doi.text
913
-
914
- spase_resource_id = ds.find('cdas:SpaseResourceId',
915
- namespaces=NAMESPACES)
916
- if spase_resource_id is not None:
917
- dataset['SpaseResourceId'] = spase_resource_id.text
918
-
919
- additional_metadata = []
920
- for add_meta in ds.findall('cdas:AdditionalMetadata',
921
- namespaces=NAMESPACES):
922
- meta_type = add_meta.attrib['Type']
923
- value = add_meta.text
924
- additional_metadata.append({
925
- 'Type': meta_type,
926
- 'value': value
927
- })
928
-
929
- if len(additional_metadata) > 0:
930
- dataset['AdditionalMetadata'] = additional_metadata
931
-
932
- datasets.append(dataset)
933
-
934
- return sorted(datasets, key=itemgetter('Id'))
935
- # pylint: enable=too-many-branches
936
-
937
-
938
- @staticmethod
939
- def get_doi_landing_page_url(
940
- doi: str
941
- ) -> str:
942
- """
943
- Returns a URL to the given Digital Object Identifier's landing
944
- page (metadata for the DOI).
945
-
946
- Parameters
947
- ----------
948
- doi
949
- digital object identifier.
950
- Returns
951
- -------
952
- str
953
- A URL to the DOI's landing page.
954
- """
955
-
956
- if not doi.startswith('http'):
957
- return 'https://doi.org/' + doi
958
- return doi
959
-
960
-
961
- @staticmethod
962
- def get_citation(
963
- doi: str
964
- ) -> str:
965
- """
966
- Returns the citation from doi.org for the given DOI.
967
-
968
- Parameters
969
- ----------
970
- doi
971
- digital object identifier.
972
- Returns
973
- -------
974
- str
975
- The citation from doi.org for the given DOI.
976
- """
977
-
978
- url = 'https://doi.org/' + doi
979
- headers = {'Accept': 'text/x-bibliography; style=apa'}
980
- response = requests.get(url, headers=headers,
981
- timeout=30)
982
-
983
- return response.text
984
-
985
-
986
- def get_inventory(
987
- self,
988
- identifier: str,
989
- **keywords: str
990
- ) -> List[TimeInterval]:
991
- """
992
- Gets a description of the specified dataset's data inventory.
993
-
994
- Parameters
995
- ----------
996
- identifier
997
- dataset identifier of data inventory to get.
998
- keywords
999
- optional keyword parameters as follows:<br>
1000
- <b>timeInterval</b> - `timeinterval.TimeInterval` to restrict
1001
- returned inventory.
1002
- Returns
1003
- -------
1004
- List
1005
- An array of `timeinterval.TimeInterval`s when data is
1006
- available.
1007
- """
1008
-
1009
- url = self._endpoint + 'datasets/' + \
1010
- urllib.parse.quote(identifier, safe='') + '/inventory'
1011
-
1012
- if 'timeInterval' in keywords:
1013
- time_interval_keyword = keywords['timeInterval']
1014
- url = url + '/' + \
1015
- TimeInterval.basic_iso_format(time_interval_keyword.start) + \
1016
- ',' + \
1017
- TimeInterval.basic_iso_format(time_interval_keyword.end)
1018
-
1019
- self.logger.debug('request url = %s', url)
1020
-
1021
- response = self._session.get(url, timeout=self._timeout)
1022
-
1023
- if response.status_code != 200:
1024
-
1025
- self.logger.info('%s failed with http code %d', url,
1026
- response.status_code)
1027
- self.logger.info('response.text: %s', response.text)
1028
- return []
1029
-
1030
- if self.logger.level <= logging.DEBUG:
1031
- self.logger.debug('response.text = %s', response.text)
1032
-
1033
- inventory = ET.fromstring(response.text)
1034
- intervals = []
1035
- for inventory_desc in inventory.findall(\
1036
- 'cdas:InventoryDescription',
1037
- namespaces=NAMESPACES):
1038
- for time_interval in inventory_desc.findall(\
1039
- 'cdas:TimeInterval',
1040
- namespaces=NAMESPACES):
1041
- intervals.append(
1042
- TimeInterval(
1043
- time_interval.find('cdas:Start',
1044
- namespaces=NAMESPACES).text,
1045
- time_interval.find('cdas:End',
1046
- namespaces=NAMESPACES).text
1047
- )
1048
- )
1049
-
1050
- return intervals
1051
-
1052
-
1053
- def get_example_time_interval(
1054
- self,
1055
- identifier: str,
1056
- ) -> TimeInterval:
1057
- """
1058
- Gets a small example time interval for the specified dataset. The
1059
- interval is near the end of the dataset's data inventory. The
1060
- returned interval is not guaranteed to have non-fill data for any
1061
- specific variable.
1062
-
1063
- Parameters
1064
- ----------
1065
- identifier
1066
- dataset identifier of data inventory to get.
1067
- Returns
1068
- -------
1069
- timeinterval.TimeInterval
1070
- An small example time interval that is likely, but not
1071
- guaranteed, to have data or None if an interval cannot be
1072
- found.
1073
- """
1074
-
1075
- time_intervals = self.get_inventory(identifier)
1076
- if len(time_intervals) < 1:
1077
- return None
1078
- example_interval = time_intervals[-1]
1079
- if re.search('MMS[1-4]_.+_BRST_.+', identifier):
1080
- time_delta = timedelta(seconds=1)
1081
- else:
1082
- time_delta = timedelta(hours=2)
1083
- example_interval.start = example_interval.end - time_delta
1084
- return example_interval
1085
-
1086
-
1087
- def get_variables(
1088
- self,
1089
- identifier: str
1090
- ) -> List[Dict]:
1091
- """
1092
- Gets a description of the variables in the specified dataset.
1093
-
1094
- Parameters
1095
- ----------
1096
- identifier
1097
- dataset identifier of data to get.
1098
- Returns
1099
- -------
1100
- List
1101
- A List of dictionary descriptions of the variables in
1102
- the specified dataset. The dictionary structure is defined by
1103
- the VariableDescription element in
1104
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
1105
- """
1106
-
1107
- url = self._endpoint + 'datasets/' + \
1108
- urllib.parse.quote(identifier, safe='') + '/variables'
1109
-
1110
- response = self._session.get(url, timeout=self._timeout)
1111
-
1112
- if response.status_code != 200:
1113
-
1114
- self.logger.info('%s failed with http code %d', url,
1115
- response.status_code)
1116
- self.logger.info('response.text: %s', response.text)
1117
- return []
1118
-
1119
- if self.logger.level <= logging.DEBUG:
1120
- self.logger.debug('response.text = %s', response.text)
1121
-
1122
- var_descriptions = ET.fromstring(response.text)
1123
-
1124
- variables = []
1125
- for var_description in var_descriptions.findall(\
1126
- 'cdas:VariableDescription',
1127
- namespaces=NAMESPACES):
1128
- name = var_description.find('cdas:Name',
1129
- namespaces=NAMESPACES).text
1130
- short_description = var_description.find(\
1131
- 'cdas:ShortDescription',
1132
- namespaces=NAMESPACES).text
1133
- if short_description is None:
1134
- short_description = ''
1135
-
1136
- long_description = var_description.find(\
1137
- 'cdas:LongDescription',
1138
- namespaces=NAMESPACES).text
1139
- if long_description is None:
1140
- long_description = ''
1141
-
1142
- variables.append({
1143
- 'Name': name,
1144
- 'ShortDescription': short_description,
1145
- 'LongDescription': long_description
1146
- })
1147
-
1148
- return variables
1149
-
1150
-
1151
- def get_variable_names(
1152
- self,
1153
- identifier: str
1154
- ) -> List[str]:
1155
- """
1156
- Gets the names of the variables in the specified dataset. This
1157
- method is like the get_variables method except that it only returns
1158
- the variable names and not the other metadata.
1159
-
1160
- Parameters
1161
- ----------
1162
- identifier
1163
- dataset identifier of data to get.
1164
- Returns
1165
- -------
1166
- List
1167
- A List of the names of the variables in the specified dataset.
1168
- """
1169
-
1170
- variable_names = []
1171
- for variable in self.get_variables(identifier):
1172
- variable_names.append(variable['Name'])
1173
-
1174
- return variable_names
1175
-
1176
-
1177
- @staticmethod
1178
- def _get_thumbnail_description_dict(
1179
- file_description_elem: ET.Element
1180
- ) -> Dict:
1181
- """
1182
- Gets ThumbnailDescription dictionary representation from the
1183
- given FileDescription element.
1184
-
1185
- Parameters
1186
- ----------
1187
- file_description_elem
1188
- a FileDescription Element.
1189
- Returns
1190
- -------
1191
- Dict
1192
- a Dictionary representation of the ThumbnailDescription
1193
- contained in the given FileDescription element.
1194
- """
1195
- thumbnail_desc = file_description_elem.find(\
1196
- 'cdas:ThumbnailDescription',
1197
- namespaces=NAMESPACES)
1198
- if thumbnail_desc is not None:
1199
- time_interval = thumbnail_desc.find('cdas:TimeInterval',
1200
- namespaces=NAMESPACES)
1201
- start = time_interval.find('cdas:Start',
1202
- namespaces=NAMESPACES).text
1203
- end = time_interval.find('cdas:End',
1204
- namespaces=NAMESPACES).text
1205
- return {
1206
- 'Name': thumbnail_desc.find('cdas:Name',
1207
- namespaces=NAMESPACES).text,
1208
- 'Dataset': thumbnail_desc.find('cdas:Dataset',
1209
- namespaces=NAMESPACES).text,
1210
- 'TimeInterval': {
1211
- 'Start': start,
1212
- 'End': end
1213
- },
1214
- 'VarName': thumbnail_desc.find('cdas:VarName',
1215
- namespaces=NAMESPACES).text,
1216
- 'Options': int(thumbnail_desc.find(\
1217
- 'cdas:Options',
1218
- namespaces=NAMESPACES).text),
1219
- 'NumFrames': int(thumbnail_desc.find(\
1220
- 'cdas:NumFrames',
1221
- namespaces=NAMESPACES).text),
1222
- 'NumRows': int(thumbnail_desc.find(\
1223
- 'cdas:NumRows',
1224
- namespaces=NAMESPACES).text),
1225
- 'NumCols': int(thumbnail_desc.find(\
1226
- 'cdas:NumCols',
1227
- namespaces=NAMESPACES).text),
1228
- 'TitleHeight': int(thumbnail_desc.find(\
1229
- 'cdas:TitleHeight',
1230
- namespaces=NAMESPACES).text),
1231
- 'ThumbnailHeight': int(thumbnail_desc.find(\
1232
- 'cdas:ThumbnailHeight',
1233
- namespaces=NAMESPACES).text),
1234
- 'ThumbnailWidth': int(thumbnail_desc.find(\
1235
- 'cdas:ThumbnailWidth',
1236
- namespaces=NAMESPACES).text),
1237
- 'StartRecord': int(thumbnail_desc.find(\
1238
- 'cdas:StartRecord',
1239
- namespaces=NAMESPACES).text),
1240
- 'MyScale': float(thumbnail_desc.find(\
1241
- 'cdas:MyScale',
1242
- namespaces=NAMESPACES).text),
1243
- 'XyStep': float(thumbnail_desc.find(\
1244
- 'cdas:XyStep',
1245
- namespaces=NAMESPACES).text)
1246
- }
1247
- return None
1248
-
1249
-
1250
- @staticmethod
1251
- def _get_data_result_dict(
1252
- xml_data_result: str
1253
- ) -> Dict:
1254
- """
1255
- Gets DataResult dictionary representation from the
1256
- given XML DataResult element.
1257
-
1258
- Parameters
1259
- ----------
1260
- xml_data_result
1261
- XML representation of a DataResult.
1262
- Returns
1263
- -------
1264
- Dict
1265
- a Dictionary representation of the given XML representation
1266
- of a DataResult.
1267
- """
1268
- data_result = ET.fromstring(xml_data_result)
1269
- file_descriptions = []
1270
- for file_description in data_result.findall(\
1271
- 'cdas:FileDescription', namespaces=NAMESPACES):
1272
-
1273
- dict_file_description = {
1274
- 'Name': file_description.find('cdas:Name',
1275
- namespaces=NAMESPACES).text,
1276
- 'MimeType': file_description.find(\
1277
- 'cdas:MimeType',
1278
- namespaces=NAMESPACES).text,
1279
- 'StartTime': file_description.find(\
1280
- 'cdas:StartTime',
1281
- namespaces=NAMESPACES).text,
1282
- 'EndTime': file_description.find(\
1283
- 'cdas:EndTime',
1284
- namespaces=NAMESPACES).text,
1285
- 'Length': int(file_description.find(\
1286
- 'cdas:Length',
1287
- namespaces=NAMESPACES).text),
1288
- 'LastModified': file_description.find(\
1289
- 'cdas:LastModified',
1290
- namespaces=NAMESPACES).text
1291
- }
1292
- thumbnail_dict = CdasWs._get_thumbnail_description_dict(\
1293
- file_description)
1294
- if thumbnail_dict is not None:
1295
- dict_file_description['ThumbnailDescription'] = \
1296
- thumbnail_dict
1297
-
1298
- thumbnail_id_elem = file_description.find(\
1299
- 'cdas:ThumbnailId',
1300
- namespaces=NAMESPACES)
1301
- if thumbnail_id_elem is not None:
1302
- dict_file_description['ThumbnailId'] = \
1303
- thumbnail_id_elem.text
1304
-
1305
- file_descriptions.append(dict_file_description)
1306
-
1307
- if len(file_descriptions) > 0:
1308
- return {
1309
- 'FileDescription': file_descriptions
1310
- }
1311
- return None
1312
-
1313
-
1314
- def get_data_result(
1315
- self,
1316
- data_request: DataRequest,
1317
- progress_callback: Callable[[float, str, Any], int],
1318
- progress_user_value: Any
1319
- ) -> Tuple[int, Dict]:
1320
- """
1321
- Submits the given request to the server and returns the result.
1322
- This is a relatively low-level method and most callers should
1323
- probably use a higher-level method such as get_data.
1324
-
1325
- Parameters
1326
- ----------
1327
- data_request
1328
- data request.
1329
- progress_callback
1330
- function that is called repeatedly to report the progress
1331
- of getting the data. The function should return 0 if it
1332
- wants to continue getting data. If it returns a non-0 value,
1333
- getting the data will be aborted and the get_data() function
1334
- will immediately return (204, None). The float parameter
1335
- is a value between 0.0 and 1.0 to indicate progress and
1336
- the str parameter will contain a text message indicating
1337
- the progress of this call.
1338
- progressUserValue
1339
- value that is passsed to the progressCallback function.
1340
- Returns
1341
- -------
1342
- Tuple
1343
- [0] contains the int HTTP status code. 200 when
1344
- successful.<br>
1345
- [1] contains a dictionary representing the DataResult from
1346
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1347
- or None.
1348
- See Also
1349
- --------
1350
- CdasWs.get_data
1351
- """
1352
-
1353
- #self.logger.debug('data_request = %s', data_request.json())
1354
- self.logger.debug('data_request = %s', data_request.xml_str())
1355
-
1356
- url = self._endpoint + 'datasets'
1357
-
1358
- for retries in range(RETRY_LIMIT):
1359
- #response = self._session.post(url, data=data_request.json(),
1360
- response = self._session.post(url, data=data_request.xml_str(),
1361
- timeout=self._timeout)
1362
-
1363
- if response.status_code == 200:
1364
-
1365
- data_result = CdasWs._get_data_result_dict(response.text)
1366
-
1367
- if not data_result:
1368
- return (response.status_code, None)
1369
-
1370
- return (response.status_code, data_result)
1371
-
1372
- if response.status_code == 429 or \
1373
- response.status_code == 503 and \
1374
- 'Retry-After' in response.headers:
1375
-
1376
- retry_after = response.headers['Retry-After']
1377
-
1378
- self.logger.debug('429/503 status with Retry-After header: %s',
1379
- retry_after)
1380
-
1381
- if progress_callback is not None:
1382
- if progress_callback(0.2, 'Waiting ' + retry_after + \
1383
- 's before making server request.',
1384
- progress_user_value) != 0:
1385
- return (204, None)
1386
-
1387
- retry_after = int(retry_after)
1388
-
1389
- self.logger.info('Sleeping %d seconds before making request',
1390
- retry_after)
1391
- time.sleep(retry_after)
1392
-
1393
- else:
1394
- self.logger.info('%s failed with http code %d', url,
1395
- response.status_code)
1396
- self.logger.info('data_request = %s', data_request)
1397
- self.logger.info('response.text: %s', response.text)
1398
- return (response.status_code, None)
1399
-
1400
- self.logger.info('%s failed with http code %d after %d retries',
1401
- url, response.status_code, retries + 1)
1402
- self.logger.info('data_request = %s', data_request)
1403
- self.logger.info('response.text: %s', response.text)
1404
- return (response.status_code, None)
1405
-
1406
-
1407
- def get_data_file(
1408
- self,
1409
- dataset: str,
1410
- variables: List[str],
1411
- start: Union[datetime, str], end: Union[datetime, str],
1412
- **keywords: Union[
1413
- Dict,
1414
- Callable[[float, str, Any], int],
1415
- Any]
1416
- ) -> Tuple[int, Dict]:
1417
- """
1418
- Gets the specified data file from the server.
1419
-
1420
- Parameters
1421
- ----------
1422
- dataset
1423
- dataset identifier of data to get.
1424
- variables
1425
- array containing names of variables to get.
1426
- start
1427
- start time of data to get. See module note about timezone.
1428
- end
1429
- end time of data to get. See module note about timezone.
1430
- keywords
1431
- optional keyword parameters as follows:<br>
1432
- <b>binData</b> - indicates that uniformly spaced values should
1433
- be computed for scaler/vector/spectrogram data according to
1434
- the given binning parameter values. binData is a Dict that
1435
- may contain the following keys: interval,
1436
- interpolateMissingValues, sigmaMultiplier, and/or
1437
- overrideDefaultBinning with values that override the
1438
- defaults.<br>
1439
- <b>progressCallback</b> - is a
1440
- Callable[[float, str, typing.Any], int]
1441
- function that is called repeatedly to report the progress
1442
- of getting the data. The function should return 0 if it
1443
- wants to continue getting data. If it returns non-0 value,
1444
- getting the data will be aborted and the get_data_file()
1445
- function will immediately return (204, None). The float
1446
- parameter is a value between 0.0 and 1.0 to indicate progress
1447
- and the str parameter will contain a text message indicating
1448
- the progress of this call.<br>
1449
- <b>progressUserValue</b> - is an Any value that is passsed
1450
- to the progressCallback function.<br>
1451
- Returns
1452
- -------
1453
- Tuple
1454
- [0] contains the int HTTP status code. 200 when
1455
- successful.<br>
1456
- [1] contains a dictionary representing the DataResult from
1457
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1458
- or None.
1459
- Raises
1460
- ------
1461
- ValueError
1462
- If the given start/end datetime values are invalid.
1463
- See Also
1464
- --------
1465
- CdasWs.get_data : In addition to what get_data_file does,
1466
- get_data also downloads and reads the data file into memory
1467
- (SpaceData or xarray.Dataset object).
1468
- """
1469
- # pylint: disable=too-many-locals
1470
- # pylint: disable=too-many-return-statements
1471
- # pylint: enable=too-many-statements
1472
- # pylint: disable=too-many-branches
1473
-
1474
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
1475
- end)
1476
-
1477
- data_request = CdfRequest(dataset, variables,
1478
- TimeInterval(start_datetime,
1479
- end_datetime),
1480
- 3, CdfFormat.BINARY,
1481
- **keywords.get('binData', {}))
1482
-
1483
- progress_callback = keywords.get('progressCallback', None)
1484
- progress_user_value = keywords.get('progressUserValue', None)
1485
-
1486
- self.logger.debug('data_request = %s', data_request)
1487
-
1488
- if progress_callback is not None:
1489
- if progress_callback(0.1, 'Making server request.',
1490
- progress_user_value) != 0:
1491
- return (204, None)
1492
-
1493
- status_code, data_result = self.get_data_result(data_request,
1494
- progress_callback,
1495
- progress_user_value)
1496
-
1497
- if progress_callback is not None:
1498
- if progress_callback(1.0, 'Initial server request complete.',
1499
- progress_user_value) != 0:
1500
- return (status_code, None)
1501
-
1502
- return (status_code, data_result)
1503
-
1504
-
1505
- def download(
1506
- self,
1507
- url: str,
1508
- size: int = 0,
1509
- **keywords
1510
- ) -> str:
1511
- """
1512
- Downloads the file specified by the given URL to a temporary
1513
- file without reading all of it into memory. This method
1514
- utilizes the connection pool and persistent HTTP connection
1515
- to the CdasWs server.
1516
-
1517
- Parameters
1518
- ----------
1519
- url
1520
- URL of file to download.
1521
- size
1522
- number of bytes in file to download.
1523
- keywords
1524
- optional keyword parameters as follows:<br>
1525
- <b>progressCallback</b> - is a
1526
- typing.Callable[[float, str, typing.Any], int]
1527
- function that is called repeatedly to report the progress
1528
- of getting the data. The function should return 0 if it
1529
- wants to continue getting data. If it returns a non-0 value,
1530
- getting the data will be aborted and this download() function
1531
- will immediately return None. The float parameter
1532
- is a value between 0.0 and 1.0 to indicate progress and
1533
- the str parameter will contain a text message indicating
1534
- the progress of this call.<br>
1535
- <b>progressUserValue</b> - is a typing.Any value that is
1536
- passsed to the progressCallback function.<br>
1537
- Returns
1538
- -------
1539
- str
1540
- name of tempory file or None if there was an error.
1541
- """
1542
- # pylint: disable=too-many-locals
1543
-
1544
- progress_callback = keywords.get('progressCallback', None)
1545
- progress_user_value = keywords.get('progressUserValue', None)
1546
-
1547
- suffix = os.path.splitext(urlparse(url).path)[1]
1548
-
1549
- file_descriptor, tmp_filename = mkstemp(suffix=suffix)
1550
-
1551
- download_bytes = 0
1552
- next_progress_report = 0.1
1553
- with self._session.get(url, stream=True,
1554
- timeout=self._timeout) as response:
1555
-
1556
- file = open(tmp_filename, 'wb')
1557
- for chunk in response.iter_content(chunk_size=8192):
1558
- if chunk: # filter out keep-alive new chunks
1559
- file.write(chunk)
1560
- # file.flush()
1561
- if progress_callback is not None:
1562
- download_bytes += len(chunk)
1563
- if size == 0:
1564
- download_progress = 0.0
1565
- else:
1566
- download_progress = float(download_bytes) / size
1567
- if download_progress > next_progress_report:
1568
- next_progress_report += download_progress
1569
- if progress_callback(download_progress,\
1570
- 'Continuing download of data.',
1571
- progress_user_value) != 0:
1572
-
1573
- file.close()
1574
- os.close(file_descriptor)
1575
- return None
1576
- file.close()
1577
- os.close(file_descriptor)
1578
-
1579
- if progress_callback is not None:
1580
- if progress_callback(0.4,
1581
- 'Data download complete. Reading data.',
1582
- progress_user_value) != 0:
1583
- return None
1584
-
1585
- return tmp_filename
1586
-
1587
-
1588
- @staticmethod
1589
- def read_data(
1590
- filename: str,
1591
- data_representation: DataRepresentation
1592
- ) -> Union['spacepy.datamodel', 'xr.Dataset']:
1593
- """
1594
- Reads the data from the given file.
1595
-
1596
- Parameters
1597
- ----------
1598
- filename
1599
- Name of file to read.
1600
- data_representation
1601
- Requested data representation.
1602
- Returns
1603
- -------
1604
- spacepy.datamodel or xr.Dataset
1605
- Data from file.
1606
- Raises
1607
- ------
1608
- Exception
1609
- If an Exception is raise by either the spdm.fromCDF() or
1610
- cdflib.cdf_to_xarray() functions.
1611
- ModuleNotFoundError
1612
- If the required spacepy.datamodel or the cdflib and xarray
1613
- modules are not installed.
1614
- """
1615
- if data_representation is None:
1616
- if SPDM_AVAILABLE:
1617
- return spdm.fromCDF(filename)
1618
- if CDF_XARRAY_AVAILABLE:
1619
- return cdf_to_xarray(filename, to_datetime=True,
1620
- fillval_to_nan=True)
1621
- raise ModuleNotFoundError(
1622
- 'neither the spacepy.datamodel nor the cdflib and '
1623
- 'xarray modules are installed')
1624
-
1625
- if data_representation is DataRepresentation.SPACEPY and \
1626
- not SPDM_AVAILABLE:
1627
- raise ModuleNotFoundError('spacepy module must be installed')
1628
- if data_representation is DataRepresentation.XARRAY and \
1629
- not CDF_XARRAY_AVAILABLE:
1630
- raise ModuleNotFoundError('cdflib and xarray modules must be installed')
1631
-
1632
- if data_representation is DataRepresentation.SPACEPY:
1633
- return spdm.fromCDF(filename)
1634
- if data_representation is DataRepresentation.XARRAY:
1635
- return cdf_to_xarray(filename, to_datetime=True,
1636
- fillval_to_nan=True)
1637
- return None
1638
-
1639
-
1640
- def get_data(
1641
- self,
1642
- dataset: str,
1643
- variables: List[str],
1644
- time0: Union[TimeInterval, List[TimeInterval], datetime, str],
1645
- time1: Union[datetime, str] = None,
1646
- **keywords: Union[
1647
- Dict,
1648
- DataRepresentation,
1649
- Callable[[float, str, Any], int],
1650
- Any]
1651
- ) -> Tuple[Dict, 'spdm.SpaceData', 'xarray']:
1652
- """
1653
- Gets the specified data from the server. The representation
1654
- of the returned data is determined as follows:<br>
1655
- 1. If a dataRepresentation keyword parameter is given, its
1656
- value will determine the representation of the returned
1657
- data. If no dataRepresenation keyword parameter is
1658
- given, then<br>
1659
- 2. If the presence of spacepy.datamodel is found, then the data
1660
- is returned in the spacepy.datamodel representation.<br>
1661
- 3. If the presence of the cdflib and xarray modules are found,
1662
- then the data is returned in an xarray.Dataset.
1663
-
1664
- Parameters
1665
- ----------
1666
- dataset
1667
- dataset identifier of data to get.
1668
- variables
1669
- array containing names of variables to get. The value
1670
- ALL-VARIABLES may be used instead of specifying all the
1671
- individual variable names.
1672
- time0
1673
- TimeInterval(s) or start time of data to get. See module
1674
- note about timezone.
1675
- time1
1676
- when time0 is not one or more TimeInterval(s), the end time
1677
- of data to get. See module note about timezone.
1678
- keywords
1679
- optional keyword parameters as follows:<br>
1680
- <b>binData</b> - indicates that uniformly spaced values should
1681
- be computed for scaler/vector/spectrogram data according to
1682
- the given binning parameter values. See
1683
- <https://cdaweb.gsfc.nasa.gov/CDAWeb_Binning_readme.html>
1684
- for more details. binData is a Dict that
1685
- may contain the following keys: interval,
1686
- interpolateMissingValues, sigmaMultiplier, and/or
1687
- overrideDefaultBinning with values that override the
1688
- defaults.<br>
1689
- <b>dataRepresentation</b> - specifies the representation of
1690
- the returned data as one of
1691
- `datarepresentation.DataRepresentation`.<br>
1692
- <b>progressCallback</b> - is a
1693
- Callable[[float, str, typing.Any], int]
1694
- function that is called repeatedly to report the progress
1695
- of getting the data. The function should return 0 if it
1696
- wants to continue getting data. If it returns non-0 value,
1697
- getting the data will be aborted and the get_data() function
1698
- will immediately return (204, None). The float parameter
1699
- is a value between 0.0 and 1.0 to indicate progress and
1700
- the str parameter will contain a text message indicating
1701
- the progress of this call.<br>
1702
- <b>progressUserValue</b> - is an Any value that is passsed
1703
- to the progressCallback function.<br>
1704
- Returns
1705
- -------
1706
- Tuple
1707
- [0] contains a dictionary of HTTP and CDAS status information.
1708
- When successful, ['http']['status_code'] will be 200.<br>
1709
- [1] contains the requested data (SpaceData or xarray.Dataset
1710
- object) or None.
1711
- Raises
1712
- ------
1713
- ValueError
1714
- If no variables are given or if the given start/end datetime
1715
- values are invalid.
1716
- """
1717
- # pylint: disable=too-many-locals
1718
- # pylint: disable=too-many-return-statements
1719
- # pylint: disable=too-many-statements
1720
- # pylint: disable=too-many-branches
1721
- # pylint: disable=import-outside-toplevel
1722
-
1723
- #import spacepy.datamodel as spdm # type: ignore
1724
-
1725
- if len(variables) < 1:
1726
- raise ValueError('at least one variable name is required')
1727
-
1728
- if isinstance(time0, (str, datetime)):
1729
- if isinstance(time1, (str, datetime)):
1730
- time_intervals = [TimeInterval(time0, time1)]
1731
- else:
1732
- raise ValueError('time1 must be str/datetime')
1733
- elif isinstance(time0, TimeInterval):
1734
- time_intervals = [time0]
1735
- elif isinstance(time0, list) and len(time0) > 0 and\
1736
- isinstance(time0[0], TimeInterval):
1737
- time_intervals = time0
1738
- else:
1739
- raise ValueError('invalid time0 type')
1740
-
1741
- data_request = CdfRequest(dataset, variables,
1742
- time_intervals,
1743
- 3, CdfFormat.BINARY,
1744
- binData=keywords.get('binData', {}))
1745
-
1746
- data_rep = keywords.get('dataRepresentation', None)
1747
- progress_callback = keywords.get('progressCallback', None)
1748
- progress_user_value = keywords.get('progressUserValue', None)
1749
-
1750
- self.logger.debug('data_request = %s', data_request)
1751
-
1752
- status = {
1753
- 'http': {
1754
- 'status_code': 204
1755
- },
1756
- 'cdas': {
1757
- 'status': [],
1758
- 'message': [],
1759
- 'warning': [],
1760
- 'error': []
1761
- }
1762
- }
1763
-
1764
- if progress_callback is not None:
1765
- if progress_callback(0.1, 'Making initial server request.',
1766
- progress_user_value) != 0:
1767
- return (status, None)
1768
-
1769
- status_code, data_result = self.get_data_result(data_request,
1770
- progress_callback,
1771
- progress_user_value)
1772
-
1773
- status['http']['status_code'] = status_code
1774
-
1775
- if progress_callback is not None:
1776
- if progress_callback(0.3, 'Initial server request complete.',
1777
- progress_user_value) != 0:
1778
- return (status, None)
1779
-
1780
- if status_code != 200:
1781
-
1782
- self.logger.info('get_data_result failed with http code %d',
1783
- status_code)
1784
- self.logger.info('data_request = %s', data_request)
1785
- return (status, None)
1786
-
1787
- if not data_result:
1788
- return (status, None)
1789
-
1790
- if 'Status' in data_result:
1791
- status['cdas']['status'] = data_result['Status']
1792
- if 'Message' in data_result:
1793
- status['cdas']['message'] = data_result['Message']
1794
- if 'Warning' in data_result:
1795
- status['cdas']['warning'] = data_result['Warning']
1796
- if 'Error' in data_result:
1797
- status['cdas']['error'] = data_result['Error']
1798
-
1799
- if progress_callback is not None:
1800
- if progress_callback(0.4, 'Beginning download of data.',
1801
- progress_user_value) != 0:
1802
- return (status, None)
1803
-
1804
- file_descriptions = data_result['FileDescription']
1805
-
1806
- data_url = file_descriptions[0]['Name']
1807
- data_length = file_descriptions[0]['Length']
1808
-
1809
- self.logger.debug('data_url = %s, data_length = %d',
1810
- data_url, data_length)
1811
-
1812
- sub_progress_control = {
1813
- 'progressCallback': progress_callback,
1814
- 'progressUserValue': progress_user_value,
1815
- 'progressStart': 0.4,
1816
- 'progressFraction': 0.1
1817
- }
1818
-
1819
- tmp_filename = self.download(data_url, data_length,
1820
- progressCallback=_get_data_progress,
1821
- progressUserValue=sub_progress_control)
1822
-
1823
- try:
1824
- data = self.read_data(tmp_filename, data_rep)
1825
- os.remove(tmp_filename)
1826
- if progress_callback is not None:
1827
- if progress_callback(1.0, 'Finished reading data.',
1828
- progress_user_value) != 0:
1829
- return (status, None)
1830
- except:
1831
- self.logger.error('Exception from read_data(%s): %s, %s',
1832
- tmp_filename, sys.exc_info()[0],
1833
- sys.exc_info()[1])
1834
- self.logger.error('CDF file has been retained.')
1835
- raise
1836
- return (status, data)
1837
-
1838
-
1839
- # pylint: disable=too-many-arguments
1840
- def get_graph(
1841
- self,
1842
- dataset: str,
1843
- variables: List[str],
1844
- start: Union[datetime, str],
1845
- end: Union[datetime, str],
1846
- options: GraphOptions = None,
1847
- image_format: List[ImageFormat] = None,
1848
- **keywords
1849
- ) -> Tuple[int, Dict]:
1850
- """
1851
- Gets a graphical representation of the specified data from the
1852
- server.
1853
-
1854
- Parameters
1855
- ----------
1856
- dataset
1857
- dataset identifier of data to get.
1858
- variables
1859
- array containing names of variables to get.
1860
- start
1861
- start time of data to get. See module note about timezone.
1862
- end
1863
- end time of data to get. See module note about timezone.
1864
- options
1865
- graph options.
1866
- image_format
1867
- image format. If None, then [ImageFormat.PNG].
1868
- keywords
1869
- optional keyword parameters as follows:<br>
1870
- <b>binData</b> - indicates that uniformly spaced values should
1871
- be computed for scaler/vector/spectrogram data according to
1872
- the given binning parameter values. binData is a Dict that
1873
- may contain the following keys: interval,
1874
- interpolateMissingValues, sigmaMultiplier, and/or
1875
- overrideDefaultBinning with values that override the
1876
- defaults.<br>
1877
- <b>progressCallback</b> - is a
1878
- typing.Callable[[float, str, typing.Any], int]
1879
- function that is called repeatedly to report the progress
1880
- of getting the data. The function should return 0 if it
1881
- wants to continue getting data. If it returns non-0 value,
1882
- getting the data will be aborted and the get_data() function
1883
- will immediately return (204, None). The float parameter
1884
- is a value between 0.0 and 1.0 to indicate progress and
1885
- the str parameter will contain a text message indicating
1886
- the progress of this call.<br>
1887
- <b>progressUserValue</b> - is a typing.Any value that is
1888
- passsed to the progressCallback function.<br>
1889
- Returns
1890
- -------
1891
- Tuple
1892
- [0] contains the HTTP status code value (200 when successful).<br>
1893
- [1] contains a dictionary representation of a
1894
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1895
- DataResult object or None.<br>
1896
- Raises
1897
- ------
1898
- ValueError
1899
- If the given start/end datetime values are invalid.
1900
- """
1901
- # pylint: disable=too-many-locals
1902
- # pylint: disable=too-many-return-statements
1903
- # pylint: enable=too-many-statements
1904
- # pylint: disable=too-many-branches
1905
-
1906
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
1907
- end)
1908
-
1909
- request = GraphRequest(dataset, variables,
1910
- TimeInterval(start_datetime, end_datetime),
1911
- options, image_format,
1912
- **keywords)
1913
-
1914
- progress_callback = keywords.get('progressCallback', None)
1915
- progress_user_value = keywords.get('progressUserValue', None)
1916
-
1917
- self.logger.debug('request = %s', request)
1918
-
1919
- if progress_callback is not None:
1920
- if progress_callback(0.1, 'Making server request.',
1921
- progress_user_value) != 0:
1922
- return (204, None)
1923
-
1924
- status_code, result = self.get_data_result(request, progress_callback, progress_user_value)
1925
-
1926
- if progress_callback is not None:
1927
- if progress_callback(1.0, 'Server request complete.',
1928
- progress_user_value) != 0:
1929
- return (status_code, None)
1930
-
1931
- if status_code != 200:
1932
-
1933
- self.logger.info('get_result failed with http code %d',
1934
- status_code)
1935
- self.logger.info('request = %s', request)
1936
- return (status_code, None)
1937
-
1938
- return (status_code, result)
1939
- # pylint: enable=too-many-arguments
1940
-
1941
-
1942
- # pylint: disable=too-many-arguments
1943
- def get_thumbnail(
1944
- self,
1945
- dataset: str,
1946
- variables: List[str],
1947
- start: Union[datetime, str],
1948
- end: Union[datetime, str],
1949
- identifier: str,
1950
- thumbnail: int = 1,
1951
- **keywords
1952
- ) -> Tuple[int, Dict]:
1953
- """
1954
- Gets a graphical representation of the specified data from the
1955
- server.
1956
-
1957
- Parameters
1958
- ----------
1959
- dataset
1960
- dataset identifier of data to get.
1961
- variables
1962
- array containing names of variables to get.
1963
- start
1964
- start time of data to get. See module note about timezone.
1965
- end
1966
- end time of data to get. See module note about timezone.
1967
- identifier
1968
- thumbnail identifier (returned in a previous get_graph
1969
- result).
1970
- thumbnail
1971
- number of thumbnail whose full size image is being requested.
1972
- Thumbnail images are counted beginning at one (not zero).
1973
- keywords
1974
- optional keyword parameters as follows:<br>
1975
- <b>progressCallback</b> - is a
1976
- typing.Callable[[float, str, typing.Any], int]
1977
- function that is called repeatedly to report the progress
1978
- of getting the data. The function should return 0 if it
1979
- wants to continue getting data. If it returns non-0 value,
1980
- getting the data will be aborted and the get_data() function
1981
- will immediately return (204, None). The float parameter
1982
- is a value between 0.0 and 1.0 to indicate progress and
1983
- the str parameter will contain a text message indicating
1984
- the progress of this call.<br>
1985
- <b>progressUserValue</b> - is a typing.Any value that is
1986
- passsed to the progressCallback function.<br>
1987
- Returns
1988
- -------
1989
- Tuple
1990
- [0] contains the HTTP status code value (200 when successful).<br>
1991
- [1] contains a dictionary representation of a
1992
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1993
- DataResult object or None.<br>
1994
- Raises
1995
- ------
1996
- ValueError
1997
- If the given start/end datetime values are invalid.
1998
- """
1999
- # pylint: disable=too-many-locals
2000
- # pylint: disable=too-many-return-statements
2001
- # pylint: enable=too-many-statements
2002
- # pylint: disable=too-many-branches
2003
-
2004
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2005
- end)
2006
-
2007
- request = ThumbnailRequest(dataset, variables,
2008
- TimeInterval(start_datetime, end_datetime),
2009
- identifier, thumbnail)
2010
-
2011
- progress_callback = keywords.get('progressCallback', None)
2012
- progress_user_value = keywords.get('progressUserValue', None)
2013
-
2014
- self.logger.debug('request = %s', request)
2015
-
2016
- if progress_callback is not None:
2017
- if progress_callback(0.1, 'Making server request.',
2018
- progress_user_value) != 0:
2019
- return (204, None)
2020
-
2021
- status_code, result = self.get_data_result(request,
2022
- progress_callback,
2023
- progress_user_value)
2024
-
2025
- if progress_callback is not None:
2026
- if progress_callback(1.0, 'Server request complete.',
2027
- progress_user_value) != 0:
2028
- return (status_code, None)
2029
-
2030
- if status_code != 200:
2031
-
2032
- self.logger.info('get_result failed with http code %d',
2033
- status_code)
2034
- self.logger.info('request = %s', request)
2035
- return (status_code, None)
2036
-
2037
- return (status_code, result)
2038
- # pylint: enable=too-many-arguments
2039
-
2040
-
2041
- # pylint: disable=too-many-arguments
2042
- def get_text(
2043
- self,
2044
- dataset: str,
2045
- variables: List[str],
2046
- start: Union[datetime, str],
2047
- end: Union[datetime, str],
2048
- compression: Compression = Compression.UNCOMPRESSED,
2049
- text_format: TextFormat = TextFormat.PLAIN,
2050
- **keywords
2051
- ) -> Tuple[int, Dict]:
2052
- """
2053
- Gets a textual representation of the specified data from the
2054
- server.
2055
-
2056
- Parameters
2057
- ----------
2058
- dataset
2059
- dataset identifier of data to get.
2060
- variables
2061
- array containing names of variables to get.
2062
- start
2063
- start time of data to get. See module note about timezone.
2064
- end
2065
- end time of data to get. See module note about timezone.
2066
- compression
2067
- file compression.
2068
- text_format
2069
- text format.
2070
- keywords
2071
- optional keyword parameters as follows:<br>
2072
- <b>binData</b> - indicates that uniformly spaced values should
2073
- be computed for scaler/vector/spectrogram data according to
2074
- the given binning parameter values. binData is a Dict that
2075
- may contain the following keys: interval,
2076
- interpolateMissingValues, sigmaMultiplier, and/or
2077
- overrideDefaultBinning with values that override the
2078
- defaults.<br>
2079
- <b>progressCallback</b> - is a
2080
- typing.Callable[[float, str, typing.Any], int]
2081
- function that is called repeatedly to report the progress
2082
- of getting the data. The function should return 0 if it
2083
- wants to continue getting data. If it returns non-0 value,
2084
- getting the data will be aborted and the get_data() function
2085
- will immediately return (204, None). The float parameter
2086
- is a value between 0.0 and 1.0 to indicate progress and
2087
- the str parameter will contain a text message indicating
2088
- the progress of this call.<br>
2089
- <b>progressUserValue</b> - is a typing.Any value that is
2090
- passsed to the progressCallback function.<br>
2091
- Returns
2092
- -------
2093
- Tuple
2094
- [0] contains the HTTP status code value (200 when successful).<br>
2095
- [1] contains a dictionary representation of a
2096
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
2097
- DataResult object or None.<br>
2098
- Raises
2099
- ------
2100
- ValueError
2101
- If the given start/end datetime values are invalid.
2102
- """
2103
- # pylint: disable=too-many-locals
2104
- # pylint: disable=too-many-return-statements
2105
- # pylint: enable=too-many-statements
2106
- # pylint: disable=too-many-branches
2107
-
2108
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2109
- end)
2110
-
2111
- request = TextRequest(dataset, variables,
2112
- TimeInterval(start_datetime, end_datetime),
2113
- compression, text_format,
2114
- **keywords)
2115
-
2116
- progress_callback = keywords.get('progressCallback', None)
2117
- progress_user_value = keywords.get('progressUserValue', None)
2118
-
2119
- self.logger.debug('request = %s', request)
2120
-
2121
- if progress_callback is not None:
2122
- if progress_callback(0.1, 'Making server request.',
2123
- progress_user_value) != 0:
2124
- return (204, None)
2125
-
2126
- status_code, result = self.get_data_result(request,
2127
- progress_callback,
2128
- progress_user_value)
2129
-
2130
- if progress_callback is not None:
2131
- if progress_callback(1.0, 'Server request complete.',
2132
- progress_user_value) != 0:
2133
- return (status_code, None)
2134
-
2135
- if status_code != 200:
2136
-
2137
- self.logger.info('get_result failed with http code %d',
2138
- status_code)
2139
- self.logger.info('request = %s', request)
2140
- return (status_code, None)
2141
-
2142
- return (status_code, result)
2143
- # pylint: enable=too-many-arguments
2144
-
2145
-
2146
- def get_audio(
2147
- self,
2148
- dataset: str,
2149
- variables: List[str],
2150
- start: Union[datetime, str],
2151
- end: Union[datetime, str],
2152
- **keywords
2153
- ) -> Tuple[int, Dict]:
2154
- """
2155
- Gets an audio representation of the specified data from the
2156
- server.
2157
-
2158
- Parameters
2159
- ----------
2160
- dataset
2161
- dataset identifier of data to get.
2162
- variables
2163
- array containing names of variables to get.
2164
- start
2165
- start time of data to get. See module note about timezone.
2166
- end
2167
- end time of data to get. See module note about timezone.
2168
- keywords
2169
- optional keyword parameters as follows:<br>
2170
- <b>binData</b> - indicates that uniformly spaced values should
2171
- be computed for scaler/vector/spectrogram data according to
2172
- the given binning parameter values. binData is a Dict that
2173
- may contain the following keys: interval,
2174
- interpolateMissingValues, sigmaMultiplier, and/or
2175
- overrideDefaultBinning with values that override the
2176
- defaults.<br>
2177
- <b>progressCallback</b> - is a
2178
- typing.Callable[[float, str, typing.Any], int]
2179
- function that is called repeatedly to report the progress
2180
- of getting the data. The function should return 0 if it
2181
- wants to continue getting data. If it returns non-0 value,
2182
- getting the data will be aborted and the get_data() function
2183
- will immediately return (204, None). The float parameter
2184
- is a value between 0.0 and 1.0 to indicate progress and
2185
- the str parameter will contain a text message indicating
2186
- the progress of this call.<br>
2187
- <b>progressUserValue</b> - is a typing.Any value that is
2188
- passsed to the progressCallback function.<br>
2189
- Returns
2190
- -------
2191
- Tuple
2192
- [0] contains the HTTP status code value (200 when successful).<br>
2193
- [1] contains a dictionary representation of a
2194
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
2195
- DataResult object or None.<br>
2196
- Raises
2197
- ------
2198
- ValueError
2199
- If the given start/end datetime values are invalid.
2200
- """
2201
- # pylint: disable=too-many-locals
2202
- # pylint: disable=too-many-return-statements
2203
- # pylint: enable=too-many-statements
2204
- # pylint: disable=too-many-branches
2205
-
2206
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2207
- end)
2208
-
2209
- request = AudioRequest(dataset, variables,
2210
- TimeInterval(start_datetime, end_datetime),
2211
- **keywords)
2212
-
2213
- progress_callback = keywords.get('progressCallback', None)
2214
- progress_user_value = keywords.get('progressUserValue', None)
2215
-
2216
- self.logger.debug('request = %s', request)
2217
-
2218
- if progress_callback is not None:
2219
- if progress_callback(0.1, 'Making server request.',
2220
- progress_user_value) != 0:
2221
- return (204, None)
2222
-
2223
- status_code, result = self.get_data_result(request,
2224
- progress_callback,
2225
- progress_user_value)
2226
-
2227
- if progress_callback is not None:
2228
- if progress_callback(1.0, 'Server request complete.',
2229
- progress_user_value) != 0:
2230
- return (status_code, None)
2231
-
2232
- if status_code != 200:
2233
-
2234
- self.logger.info('get_result failed with http code %d',
2235
- status_code)
2236
- self.logger.info('request = %s', request)
2237
- return (status_code, None)
2238
-
2239
- return (status_code, result)
2240
-
2241
-
2242
- def get_original_files(
2243
- self,
2244
- dataset: str,
2245
- start: Union[datetime, str],
2246
- end: Union[datetime, str],
2247
- **keywords
2248
- ) -> Tuple[int, Dict]:
2249
- """
2250
- Gets original data files from a dataset. Original data files
2251
- lack updated meta-data and virtual variable values contained
2252
- in files obtained from the `CdasWs.get_data`. Most callers
2253
- should probably use `CdasWs.get_data` instead of this function.
2254
-
2255
- Parameters
2256
- ----------
2257
- dataset
2258
- dataset identifier of data to get.
2259
- start
2260
- start time of data to get. See module note about timezone.
2261
- end
2262
- end time of data to get. See module note about timezone.
2263
- keywords
2264
- optional keyword parameters as follows:<br>
2265
- <b>progressCallback</b> - is a
2266
- typing.Callable[[float, str, typing.Any], int]
2267
- function that is called repeatedly to report the progress
2268
- of getting the data. The function should return 0 if it
2269
- wants to continue getting data. If it returns non-0 value,
2270
- getting the data will be aborted and the get_data() function
2271
- will immediately return (204, None). The float parameter
2272
- is a value between 0.0 and 1.0 to indicate progress and
2273
- the str parameter will contain a text message indicating
2274
- the progress of this call.<br>
2275
- <b>progressUserValue</b> - is a typing.Any value that is
2276
- passsed to the progressCallback function.<br>
2277
- Returns
2278
- -------
2279
- Tuple
2280
- [0] contains the HTTP status code value (200 when successful).<br>
2281
- [1] array of dictionary representations of a
2282
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
2283
- FileDescription objects or None.<br>
2284
- Raises
2285
- ------
2286
- ValueError
2287
- If the given start/end datetime values are invalid.
2288
- See Also
2289
- --------
2290
- CdasWs.get_data
2291
- """
2292
- # pylint: disable=too-many-locals
2293
- # pylint: disable=too-many-return-statements
2294
- # pylint: enable=too-many-statements
2295
- # pylint: disable=too-many-branches
2296
-
2297
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2298
- end)
2299
-
2300
- request = CdfRequest(dataset, [],
2301
- TimeInterval(start_datetime, end_datetime))
2302
-
2303
- progress_callback = keywords.get('progressCallback', None)
2304
- progress_user_value = keywords.get('progressUserValue', None)
2305
-
2306
- self.logger.debug('request = %s', request)
2307
-
2308
- if progress_callback is not None:
2309
- if progress_callback(0.1, 'Making server request.',
2310
- progress_user_value) != 0:
2311
- return (204, None)
2312
-
2313
- status_code, result = self.get_data_result(request,
2314
- progress_callback,
2315
- progress_user_value)
2316
-
2317
- if progress_callback is not None:
2318
- if progress_callback(1.0, 'Server request complete.',
2319
- progress_user_value) != 0:
2320
- return (status_code, None)
2321
-
2322
- if status_code != 200:
2323
-
2324
- self.logger.info('get_result failed with http code %d',
2325
- status_code)
2326
- self.logger.info('request = %s', request)
2327
- return (status_code, None)
2328
-
2329
- return (status_code, result['FileDescription'])
2330
-
2331
-
2332
- def get_ssc_id(
2333
- self,
2334
- dataset: str
2335
- ) -> Tuple[int, Union[str, List[str]]]:
2336
- """
2337
- Gets the Satellite Situation Center (SSC)
2338
- <https://sscweb.gsfc.nasa.gov/> observatory identifier(s)
2339
- associated with the given cdaweb dataset identifier.
2340
-
2341
- Notes
2342
- -----
2343
- This method relies upon the Heliophysics Data Portal's
2344
- <https://heliophysicsdata.gsfc.nasa.gov/> metadata. That metadata
2345
- may be incomplete. Also, cdaweb has datasets for which SSC has
2346
- no corresponding observatory (for example, ground observatory
2347
- data). Callers should be prepared for negative results (200, None)
2348
- from this method.
2349
-
2350
- Parameters
2351
- ----------
2352
- dataset
2353
- cdaweb dataset identifier.
2354
- Returns
2355
- -------
2356
- Tuple
2357
- [0] contains the HTTP status code value (200 when successful).<br>
2358
- [1] the SSC observatory identifier(s) associated with the given
2359
- cdaweb dataset identifier or None if none is found.
2360
- """
2361
- url = self._hdp_registry + '?cdawebId=' + dataset
2362
-
2363
- self.logger.debug('request url = %s', url)
2364
-
2365
- response = self._session.get(url, timeout=self._timeout)
2366
-
2367
- if response.status_code != 200:
2368
-
2369
- self.logger.info('%s failed with http code %d', url,
2370
- response.status_code)
2371
- self.logger.info('response.text: %s', response.text)
2372
- return (response.status_code, None)
2373
-
2374
- if self.logger.level <= logging.DEBUG:
2375
- self.logger.debug('response.text = %s', response.text)
2376
-
2377
- results = ET.fromstring(response.text)
2378
-
2379
- ssc_id = []
2380
- for ssc_id_elem in results.findall('SscId'):
2381
- ssc_id.append(ssc_id_elem.text)
2382
-
2383
- if len(ssc_id) == 0:
2384
- result = None
2385
- elif len(ssc_id) == 1:
2386
- result = ssc_id[0]
2387
- else:
2388
- result = ssc_id
2389
-
2390
- return (response.status_code, result)
82
+ #
83
+ # For backward compatibility with cdasws versions < 1.8.12.
84
+ #
85
+ from cdasws.cdasws import * # pylint: disable=wrong-import-position