cdasws 1.8.10__py3-none-any.whl → 1.8.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cdasws/__init__.py CHANGED
@@ -55,89 +55,7 @@ Notes
55
55
  """
56
56
 
57
57
 
58
- import sys
59
- import os
60
- import platform
61
- import logging
62
- import re
63
- from importlib.util import find_spec
64
- import urllib.parse
65
- from urllib.parse import urlparse
66
- import json
67
- from operator import itemgetter
68
- import time
69
- from datetime import datetime, timedelta, timezone
70
- import xml.etree.ElementTree as ET
71
- from tempfile import mkstemp
72
- from typing import Any, Callable, Dict, List, Tuple, Union
73
-
74
- import requests
75
- import dateutil.parser
76
-
77
- from cdasws.datarepresentation import DataRepresentation
78
- from cdasws.datarequest import AudioRequest, DataRequest
79
- from cdasws.datarequest import CdfFormat, CdfRequest, Compression
80
- from cdasws.datarequest import ImageFormat, GraphOptions, GraphRequest
81
- from cdasws.datarequest import TextFormat, TextRequest, ThumbnailRequest
82
- from cdasws.timeinterval import TimeInterval
83
-
84
-
85
- # requires python >= 3.4
86
- #if find_spec('spacepy.datamodel') is not None:
87
- # import spacepy.datamodel as spdm # type: ignore
88
- # SPDM_AVAILABLE = True
89
- #else:
90
- # SPDM_AVAILABLE = False
91
- # python < 3.4
92
- try:
93
- import spacepy.datamodel as spdm # type: ignore
94
- SPDM_AVAILABLE = True
95
- except ImportError:
96
- SPDM_AVAILABLE = False
97
-
98
- try:
99
- from cdflib.xarray import cdf_to_xarray
100
- import xarray as xr
101
- CDF_XARRAY_AVAILABLE = True
102
- except ImportError:
103
- try:
104
- import cdflib as cdf
105
- import xarray as xr
106
- CDF_XARRAY_AVAILABLE = True
107
- def cdf_to_xarray(filename, to_datetime=False, to_unixtime=False,
108
- fillval_to_nan=False):
109
- """
110
- Reads a CDF into an xarray.dataset. This function exists
111
- to provide compatility with cdflib >= 1.0.1 for older
112
- releases of cdflib.
113
-
114
- Parameters:
115
- -----------
116
- filename
117
- The path to the CDF file to read.
118
- to_datetime
119
- Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to
120
- datetime, or leave them as is.
121
- to_unixtime
122
- Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to
123
- unixtime, or leave them as is.
124
- fillval_to_nan
125
- If True, any data values that match the FILLVAL
126
- attribute for a variable will be set to NaN.
127
-
128
- Returns
129
- -------
130
- xarray.dataset
131
- An XArray Dataset object.
132
- """
133
- return cdf.cdf_to_xarray(filename, to_datetime=to_datetime,
134
- to_unixtime=to_unixtime,
135
- fillval_to_nan=fillval_to_nan)
136
- except ImportError:
137
- CDF_XARRAY_AVAILABLE = False
138
-
139
-
140
- __version__ = "1.8.10"
58
+ __version__ = "1.8.12"
141
59
 
142
60
 
143
61
  #
@@ -161,2213 +79,7 @@ NAMESPACES = {
161
79
  'cdas': NS,
162
80
  'xhtml': XHTML_NS
163
81
  }
164
-
165
-
166
- def _get_data_progress(
167
- progress: float,
168
- msg: str,
169
- value: Dict) -> int:
170
- """
171
- A get_data progress callback which adjusts the progress value for
172
- the download portion of a larger operation and then calls the
173
- "real" progress callback function with this adjusted progress value.
174
-
175
- Parameters
176
- ----------
177
- progress
178
- Measure of progress.
179
- msg
180
- Message describing progress of get_data call.
181
- value
182
- Dictionary containing the function to call and values for
183
- computing the adjusted progress value.
184
- Returns
185
- -------
186
- int
187
- Flag indicating whether to continue with getting the data.
188
- 0 to continue. 1 to abort getting the data.
189
- """
190
- progress_callback = value.get('progressCallback', None)
191
- progress_user_value = value.get('progressUserValue', None)
192
- adjusted_progress = value['progressStart'] + \
193
- value['progressFraction'] * progress
194
-
195
- if progress_callback is not None:
196
-
197
- return progress_callback(adjusted_progress, msg,
198
- progress_user_value)
199
- return 0
200
-
201
-
202
- class NullAuth(requests.auth.AuthBase): # pylint: disable=too-few-public-methods
203
- """
204
- Authentication class used to cause requests to ignore any ~/.netrc
205
- file. The CDAS web services do not support authentication and
206
- a cdaweb (ftps) entry will cause CdasWs requests to fail with
207
- a 401 error. See <https://github.com/psf/requests/issues/2773>.
208
- """
209
- def __call__(self, r):
210
- return r
211
-
212
-
213
- class CdasWs:
214
- """
215
- Class representing the web service interface to NASA's
216
- Coordinated Data Analysis System (CDAS)
217
- <https://cdaweb.gsfc.nasa.gov>.
218
-
219
- Notes
220
- -----
221
- The logger used by this class has the class' name (CdasWs). By default,
222
- it is configured with a NullHandler. Users of this class may configure
223
- the logger to aid in diagnosing problems.
224
- """
225
- # pylint: disable=too-many-instance-attributes
226
- # pylint: disable=too-many-arguments
227
- def __init__(
228
- self,
229
- endpoint=None,
230
- timeout=None,
231
- proxy=None,
232
- ca_certs=None,
233
- disable_ssl_certificate_validation=False,
234
- user_agent=None):
235
- """
236
- Creates an object representing the CDAS web services.
237
-
238
- Parameters
239
- ----------
240
- endpoint
241
- URL of the CDAS web service. If None, the default is
242
- 'https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/'.
243
- timeout
244
- Number of seconds to wait for a response from the server.
245
- proxy
246
- HTTP proxy information. For example,<pre>
247
- proxies = {
248
- 'http': 'http://10.10.1.10:3128',
249
- 'https': 'http://10.10.1.10:1080',
250
- }</pre>
251
- Proxy information can also be set with environment variables.
252
- For example,<pre>
253
- $ export HTTP_PROXY="http://10.10.1.10:3128"
254
- $ export HTTPS_PROXY="http://10.10.1.10:1080"</pre>
255
- ca_certs
256
- Path to certificate authority (CA) certificates that will
257
- override the default bundle.
258
- disable_ssl_certificate_validation
259
- Flag indicating whether to validate the SSL certificate.
260
- user_agent
261
- A value that is appended to the HTTP User-Agent value.
262
- """
263
-
264
- self.logger = logging.getLogger(type(self).__name__)
265
- self.logger.addHandler(logging.NullHandler())
266
-
267
- self.logger.debug('endpoint = %s', endpoint)
268
- self.logger.debug('ca_certs = %s', ca_certs)
269
- self.logger.debug('disable_ssl_certificate_validation = %s',
270
- disable_ssl_certificate_validation)
271
-
272
- if endpoint is None:
273
- self._endpoint = 'https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/'
274
- else:
275
- self._endpoint = endpoint
276
-
277
- self._user_agent = 'cdasws/' + __version__ + ' (' + \
278
- platform.python_implementation() + ' ' \
279
- + platform.python_version() + '; ' + platform.platform() + ')'
280
-
281
- if user_agent is not None:
282
- self._user_agent += ' (' + user_agent + ')'
283
-
284
- self._request_headers = {
285
- #'Content-Type' : 'application/json',
286
- 'Content-Type' : 'application/xml',
287
- 'Accept' : 'application/xml',
288
- 'User-Agent' : self._user_agent,
289
- #'Accept-Encoding' : 'gzip' # only beneficial for icdfml responses
290
- }
291
- self._session = requests.Session()
292
- self._session.headers.update(self._request_headers)
293
- self._session.auth = NullAuth()
294
-
295
- if ca_certs is not None:
296
- self._session.verify = ca_certs
297
-
298
- if disable_ssl_certificate_validation is True:
299
- self._session.verify = False
300
-
301
- if proxy is not None:
302
- self._proxy = proxy
303
-
304
- self._timeout = timeout
305
-
306
- endpoint_components = urlparse(self._endpoint)
307
- self._hdp_registry = endpoint_components.scheme + '://' + \
308
- endpoint_components.netloc + '/registry/hdp/SscId.xql'
309
-
310
- # pylint: enable=too-many-arguments
311
-
312
-
313
- def __del__(self):
314
- """
315
- Destructor. Closes all network connections.
316
- """
317
-
318
- self.close()
319
-
320
-
321
- def close(self) -> None:
322
- """
323
- Closes any persistent network connections. Generally, deleting
324
- this object is sufficient and calling this method is unnecessary.
325
- """
326
- self._session.close()
327
-
328
-
329
- def get_observatory_groups(
330
- self,
331
- **keywords: str
332
- ) -> List[Dict]:
333
- """
334
- Gets descriptions of the observatory groups from the server.
335
-
336
- Parameters
337
- ----------
338
- keywords
339
- optional keyword parameters as follows:<br>
340
- <b>instrumentType</b> - an instrument type value from those
341
- returned by `CdasWs.get_instrument_types`. Omitting
342
- this parameter indicates that no observatories are eliminated
343
- based upon their instrumentType value.
344
- Returns
345
- -------
346
- List
347
- An array of ObservatoryGroupDescription
348
- dictionaries where the structure of the dictionary mirrors
349
- ObservatoryGroupDescription in
350
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
351
- """
352
- if 'instrumentType' in keywords:
353
- url = self._endpoint + 'observatoryGroups?instrumentType=' + \
354
- urllib.parse.quote(keywords['instrumentType'])
355
- else:
356
- url = self._endpoint + 'observatoryGroups'
357
-
358
- self.logger.debug('request url = %s', url)
359
-
360
- response = self._session.get(url, timeout=self._timeout)
361
-
362
- if response.status_code != 200:
363
-
364
- self.logger.info('%s failed with http code %d', url,
365
- response.status_code)
366
- self.logger.info('response.text: %s', response.text)
367
- return []
368
-
369
- if self.logger.level <= logging.DEBUG:
370
- self.logger.debug('response.text = %s', response.text)
371
-
372
- observatory_response = ET.fromstring(response.text)
373
-
374
- observatory_group_descriptions = []
375
- for description in observatory_response.findall(\
376
- 'cdas:ObservatoryGroupDescription', namespaces=NAMESPACES):
377
-
378
- observatory_ids = []
379
- for observatory_id in description.findall(\
380
- 'cdas:ObservatoryId', namespaces=NAMESPACES):
381
-
382
- observatory_ids.append(observatory_id.text)
383
-
384
- observatory_group_descriptions.append({
385
- 'Name': description.find(\
386
- 'cdas:Name', namespaces=NAMESPACES).text,
387
- 'ObservatoryId': observatory_ids
388
- })
389
-
390
- return observatory_group_descriptions
391
-
392
-
393
- def get_instrument_types(
394
- self,
395
- **keywords: str
396
- ) -> List[Dict]:
397
- """
398
- Gets descriptions of the instrument types from the server.
399
-
400
- Parameters
401
- ----------
402
- keywords
403
- optional keyword parameters as follows:<br>
404
- <b>observatory</b> - an observatory value from those returned
405
- by `CdasWs.get_observatories`. Omitting this parameter
406
- indicates that no instrumentTypes are eliminated based upon
407
- their observatory value.<br>
408
- <b>observatoryGroup</b> - an observatory group value from
409
- those returned by `CdasWs.get_observatory_groups`. Omitting
410
- this parameter indicates that no instrumentTypes are
411
- eliminated based upon their observatoryGroup value.</br>
412
- Returns
413
- -------
414
- List
415
- An array of InstrumentTypeDescription
416
- dictionaries where the structure of the dictionary mirrors
417
- InstrumentTypeDescription in
418
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
419
- """
420
- if 'observatory' in keywords:
421
- url = self._endpoint + 'instrumentTypes?observatory=' \
422
- + urllib.parse.quote(keywords['observatory'])
423
- elif 'observatoryGroup' in keywords:
424
- url = self._endpoint + 'instrumentTypes?observatoryGroup=' \
425
- + urllib.parse.quote(keywords['observatoryGroup'])
426
- else:
427
- url = self._endpoint + 'instrumentTypes'
428
-
429
- self.logger.debug('request url = %s', url)
430
-
431
- response = self._session.get(url, timeout=self._timeout)
432
-
433
- if response.status_code != 200:
434
-
435
- self.logger.info('%s failed with http code %d', url,
436
- response.status_code)
437
- self.logger.info('response.text: %s', response.text)
438
- return []
439
-
440
- if self.logger.level <= logging.DEBUG:
441
- self.logger.debug('response.text = %s', response.text)
442
-
443
- instrument_response = ET.fromstring(response.text)
444
-
445
- if self.logger.level <= logging.DEBUG:
446
- self.logger.debug('instrument_response = %s',
447
- ET.tostring(instrument_response))
448
-
449
- instrument_types = []
450
- for description in instrument_response.findall(\
451
- 'cdas:InstrumentTypeDescription', namespaces=NAMESPACES):
452
-
453
- instrument_types.append({
454
- 'Name': description.find('cdas:Name',
455
- namespaces=NAMESPACES).text
456
- })
457
- return instrument_types
458
-
459
-
460
- def get_instruments(
461
- self,
462
- **keywords: str
463
- ) -> List[Dict]:
464
- """
465
- Gets descriptions of the instruments from the server.
466
-
467
- Parameters
468
- ----------
469
- keywords
470
- optional keyword parameters as follows:<br>
471
- <b>observatory</b> - an observatory value from those returned
472
- by `CdasWs.get_observatories`. Omitting this parameter
473
- indicates that no instruments are eliminated based upon their
474
- observatory value.<br>
475
- <b>instrumentType</b> - an instrument type value from those
476
- returned by `CdasWs.get_instrument_types`. Omitting this
477
- parameter indicates that no instruments are eliminated based
478
- upon their instrument type.<br>
479
- Returns
480
- -------
481
- List
482
- An array of InstrumentDescription
483
- dictionaries where the structure of the dictionary mirrors
484
- InstrumentDescription in
485
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
486
- """
487
- if 'observatory' in keywords:
488
- url = self._endpoint + 'instruments?observatory=' \
489
- + urllib.parse.quote(keywords['observatory'])
490
- elif 'instrumentType' in keywords:
491
- url = self._endpoint + 'instruments?instrumentType=' \
492
- + urllib.parse.quote(keywords['instrumentType'])
493
- else:
494
- url = self._endpoint + 'instruments'
495
-
496
- self.logger.debug('request url = %s', url)
497
-
498
- response = self._session.get(url, timeout=self._timeout)
499
-
500
- if response.status_code != 200:
501
-
502
- self.logger.info('%s failed with http code %d', url,
503
- response.status_code)
504
- self.logger.info('response.text: %s', response.text)
505
- return []
506
-
507
- if self.logger.level <= logging.DEBUG:
508
- self.logger.debug('response.text = %s', response.text)
509
-
510
- instruments_response = ET.fromstring(response.text)
511
-
512
- if self.logger.level <= logging.DEBUG:
513
- self.logger.debug('instruments = %s', response.text)
514
- #ET.indent(instruments_response, space=' '))
515
-
516
- instruments = []
517
- for instrument_description in instruments_response.findall(\
518
- 'cdas:InstrumentDescription', namespaces=NAMESPACES):
519
-
520
- instruments.append({
521
- 'Name': instrument_description.find(\
522
- 'cdas:Name', namespaces=NAMESPACES).text,
523
- 'ShortDescription': instrument_description.find(\
524
- 'cdas:ShortDescription', namespaces=NAMESPACES).text,
525
- 'LongDescription': instrument_description.find(\
526
- 'cdas:LongDescription', namespaces=NAMESPACES).text
527
- })
528
-
529
- return instruments
530
-
531
-
532
- def get_observatories(
533
- self,
534
- **keywords: str
535
- ) -> List[Dict]:
536
- """
537
- Gets descriptions of the observatories from the server.
538
-
539
- Parameters
540
- ----------
541
- keywords
542
- optional keyword parameters as follows:<br>
543
- <b>instrument</b> - an instrument value from those returned
544
- by `CdasWs.get_instruments`. Omitting this parameter
545
- indicates that no observatories are eliminated based upon
546
- their instrument value.<br>
547
- <b>instrumentType</b> - in instrument type value from those
548
- returned by `CdasWs.get_instrument_types`. Omitting this
549
- parameter indicates that no observatories are eliminated
550
- based upon their instrumentType value.<br>
551
- Returns
552
- -------
553
- List
554
- An array of ObservatoryDescriptions
555
- dictionaries where the structure of the dictionary mirrors
556
- ObservatoryDescription in
557
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
558
- """
559
- if 'instrument' in keywords:
560
- url = self._endpoint + 'observatories?instrument=' \
561
- + urllib.parse.quote(keywords['instrument'])
562
- elif 'instrumentType' in keywords:
563
- url = self._endpoint + 'observatories?instrumentType=' \
564
- + urllib.parse.quote(keywords['instrumentType'])
565
- else:
566
- url = self._endpoint + 'observatories'
567
-
568
- self.logger.debug('request url = %s', url)
569
-
570
- response = self._session.get(url, timeout=self._timeout)
571
-
572
- if response.status_code != 200:
573
-
574
- self.logger.info('%s failed with http code %d', url,
575
- response.status_code)
576
- self.logger.info('response.text: %s', response.text)
577
- return []
578
-
579
- if self.logger.level <= logging.DEBUG:
580
- self.logger.debug('response.text = %s', response.text)
581
-
582
- observatory_response = ET.fromstring(response.text)
583
-
584
- if self.logger.level <= logging.DEBUG:
585
- self.logger.debug('observatories = %s', response.text)
586
-
587
- observatories = []
588
-
589
- for observatory in observatory_response.findall(\
590
- 'cdas:ObservatoryDescription', namespaces=NAMESPACES):
591
- observatories.append({
592
- 'Name': observatory.find(\
593
- 'cdas:Name', namespaces=NAMESPACES).text,
594
- 'ShortDescription': observatory.find(\
595
- 'cdas:ShortDescription', namespaces=NAMESPACES).text,
596
- 'LongDescription': observatory.find(\
597
- 'cdas:LongDescription', namespaces=NAMESPACES).text
598
- })
599
-
600
- return observatories
601
-
602
-
603
- def get_observatory_groups_and_instruments(
604
- self,
605
- **keywords: str
606
- ) -> List[Dict]:
607
- """
608
- Gets descriptions of the observatory groups (and associated
609
- instruments) from the server.
610
-
611
- Parameters
612
- ----------
613
- keywords
614
- optional keyword parameters as follows:<br>
615
- <b>instrumentType</b> - an instrument type value from those
616
- returned by `CdasWs.get_instrument_types`. Omitting this
617
- parameter indicates that no observatories are eliminated
618
- based upon their instrumentType value.<br>
619
- Returns
620
- -------
621
- List
622
- An array of ObservatoryGroupInstrumentDescription
623
- dictionaries where the structure of the dictionary mirrors
624
- ObservatoryGroupInstrumentDescription in
625
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
626
- """
627
- if 'instrumentType' in keywords:
628
- url = self._endpoint \
629
- + 'observatoryGroupsAndInstruments?instrumentType=' \
630
- + urllib.parse.quote(keywords['instrumentType'])
631
- else:
632
- url = self._endpoint + 'observatoryGroupsAndInstruments'
633
-
634
- self.logger.debug('request url = %s', url)
635
-
636
- response = self._session.get(url, timeout=self._timeout)
637
-
638
- if response.status_code != 200:
639
-
640
- self.logger.info('%s failed with http code %d', url,
641
- response.status_code)
642
- self.logger.info('response.text: %s', response.text)
643
- return []
644
-
645
- if self.logger.level <= logging.DEBUG:
646
- self.logger.debug('response.text = %s', response.text)
647
-
648
- observatories_response = ET.fromstring(response.text)
649
-
650
- if self.logger.level <= logging.DEBUG:
651
- self.logger.debug('observatories = %s', response.text)
652
-
653
- o_g_i_ds = []
654
-
655
- for o_g_i_d in observatories_response.findall(\
656
- 'cdas:ObservatoryGroupInstrumentDescription',\
657
- namespaces=NAMESPACES):
658
-
659
- o_g_i_d_name = o_g_i_d.find('cdas:Name',
660
- namespaces=NAMESPACES).text
661
- o_is = []
662
- for o_i in o_g_i_d.findall('cdas:ObservatoryInstruments',
663
- namespaces=NAMESPACES):
664
-
665
- o_i_name = o_i.find('cdas:Name',
666
- namespaces=NAMESPACES).text
667
- i_ds = []
668
- for i_d in o_i.findall('cdas:InstrumentDescription',
669
- namespaces=NAMESPACES):
670
- i_d_name = i_d.find('cdas:Name',
671
- namespaces=NAMESPACES).text
672
- i_d_short_description = \
673
- i_d.find('cdas:ShortDescription',
674
- namespaces=NAMESPACES).text
675
- i_d_long_description = \
676
- i_d.find('cdas:LongDescription',
677
- namespaces=NAMESPACES).text
678
- i_ds.append({
679
- 'Name': i_d_name,
680
- 'ShortDescription': i_d_short_description,
681
- 'LongDescription': i_d_long_description
682
- })
683
- o_is.append({
684
- 'Name': o_i_name,
685
- 'InstrumentDescription': i_ds
686
- })
687
-
688
- o_g_i_ds.append({
689
- 'Name': o_g_i_d_name,
690
- 'ObservatoryInstruments': o_is
691
- })
692
-
693
- return o_g_i_ds
694
-
695
-
696
- # pylint: disable=too-many-branches
697
- def get_datasets(
698
- self,
699
- **keywords: str
700
- ) -> List[Dict]:
701
- """
702
- Gets descriptions of the specified datasets from the server.
703
-
704
- Parameters
705
- ----------
706
- keywords
707
- optional keyword parameters as follows:<br>
708
- <b>observatoryGroup</b> - an observatory group value from those
709
- returned by `CdasWs.get_observatory_groups`. Omitting this
710
- parameter
711
- indicates that no datasets are eliminated based upon their
712
- observatoryGroup value.<br>
713
- <b>instrumentType</b> - an instrument type value from those
714
- returned by `CdasWs.get_instrument_types`. Omitting this
715
- parameter indicates that no datasets are eliminated based
716
- upon their instrumentType value.<br>
717
- <b>observatory</b> - an observatory name value from those
718
- returned by `CdasWs.get_observatories`. Omitting this
719
- parameter indicates that no datasets are eliminated based
720
- upon their observatory value.<br>
721
- <b>instrument</b> - an instrument value from those returned by
722
- `CdasWs.get_instruments`. Omitting this parameter indicates
723
- that no datasets are eliminated based upon their instrument
724
- value.<br>
725
- <b>startDate</b> - a datetime specifying the start of a time
726
- interval. See module note about timezone value. If this
727
- parameter is ommited, the time interval will begin infinitely
728
- in the past.<br>
729
- <b>stopDate</b> - a datetime specifying the end of a time
730
- interval. See module note about timezone value. If this
731
- parameter is omitted, the time interval will end infinitely
732
- in the future.<br>
733
- <b>id</b> - a dataset identifier. The value may be a CDAS
734
- (e.g., AC_H2_MFI), DOI (e.g., 10.48322/fh85-fj47), or SPASE
735
- [ResourceID] (e.g., spase://NASA/NumericalData/ACE/MAG/L2/PT1H)
736
- identifier. If specified, all other keywords are ignored.<br>
737
- <b>idPattern</b> - a java.util.regex compatible regular
738
- expression that must match the dataset's CDAS identifier value.
739
- Omitting this parameter is equivalent to `.*`.<br>
740
- <b>labelPattern</b> - a java.util.regex compatible regular
741
- expression that must match the dataset's CDAS label text.
742
- Omitting this parameter is equivalent to `.*`. Embedded
743
- matching flag expressions (e.g., `(?i)` for case insensitive
744
- match mode) are supported and likely to be useful in this
745
- case.<br>
746
- <b>notesPattern</b> - a java.util.regex compatible regular
747
- expression that must match the dataset's CDAS notes text.
748
- Omitting this parameter is equivalent to `.*`. Embedded
749
- matching flag expressions (e.g., `(?s)` for dotall match mode)
750
- are supported and likely to be useful in this case.<br>
751
- Returns
752
- -------
753
- List
754
- A list of dictionaries containing descriptions of the datasets
755
- requested. The dictionary structure is defined by the
756
- DatasetDescription element in
757
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
758
- """
759
- url = self._endpoint + 'datasets?'
760
-
761
- observatory_groups = keywords.get('observatoryGroup', None)
762
- if observatory_groups is not None:
763
- if isinstance(observatory_groups, str):
764
- observatory_groups = [observatory_groups]
765
- for observatory_group in observatory_groups:
766
- url = url + 'observatoryGroup=' \
767
- + urllib.parse.quote(observatory_group) + '&'
768
-
769
- instrument_types = keywords.get('instrumentType', None)
770
- if instrument_types is not None:
771
- if isinstance(instrument_types, str):
772
- instrument_types = [instrument_types]
773
- for instrument_type in instrument_types:
774
- url = url + 'instrumentType=' \
775
- + urllib.parse.quote(instrument_type) + '&'
776
-
777
- observatories = keywords.get('observatory', None)
778
- if observatories is not None:
779
- if isinstance(observatories, str):
780
- observatories = [observatories]
781
- for observatory in observatories:
782
- url = url + 'observatory=' \
783
- + urllib.parse.quote(observatory) + '&'
784
-
785
- instruments = keywords.get('instrument', None)
786
- if instruments is not None:
787
- if isinstance(instruments, str):
788
- instruments = [instruments]
789
- for instrument in instruments:
790
- url = url + 'instrument=' \
791
- + urllib.parse.quote(instrument) + '&'
792
-
793
- if 'startDate' in keywords:
794
- url = url + 'startDate=' \
795
- + urllib.parse.quote(keywords['startDate']) + '&'
796
-
797
- if 'stopDate' in keywords:
798
- url = url + 'stopDate=' \
799
- + urllib.parse.quote(keywords['stopDate']) + '&'
800
-
801
- if 'id' in keywords:
802
- url = url + 'id=' \
803
- + urllib.parse.quote(keywords['id']) + '&'
804
-
805
- if 'idPattern' in keywords:
806
- url = url + 'idPattern=' \
807
- + urllib.parse.quote(keywords['idPattern']) + '&'
808
-
809
- if 'labelPattern' in keywords:
810
- url = url + 'labelPattern=' \
811
- + urllib.parse.quote(keywords['labelPattern']) + '&'
812
-
813
- if 'notesPattern' in keywords:
814
- url = url + 'notesPattern=' \
815
- + urllib.parse.quote(keywords['notesPattern']) + '&'
816
-
817
- self.logger.debug('request url = %s', url[:-1])
818
-
819
- response = self._session.get(url[:-1], timeout=self._timeout)
820
-
821
- if response.status_code != 200:
822
-
823
- self.logger.info('%s failed with http code %d', url,
824
- response.status_code)
825
- self.logger.info('response.text: %s', response.text)
826
- return []
827
-
828
- if self.logger.level <= logging.DEBUG:
829
- self.logger.debug('response.text = %s', response.text)
830
-
831
- dss = ET.fromstring(response.text)
832
-
833
- if self.logger.level <= logging.DEBUG:
834
- self.logger.debug('datasets = %s', response.text)
835
-
836
- datasets = []
837
- for ds in dss.findall('cdas:DatasetDescription',
838
- namespaces=NAMESPACES):
839
-
840
- observatory_groups = []
841
- for o_g in ds.findall('cdas:ObservatoryGroup',
842
- namespaces=NAMESPACES):
843
- observatory_groups.append(o_g.text)
844
-
845
- instrument_types = []
846
- for i_t in ds.findall('cdas:InstrumentType',
847
- namespaces=NAMESPACES):
848
- instrument_types.append(i_t.text)
849
-
850
- dataset_links = []
851
- for d_l in ds.findall('cdas:DatasetLink',
852
- namespaces=NAMESPACES):
853
- dataset_links.append({
854
- 'Title': d_l.find('cdas:Title',
855
- namespaces=NAMESPACES).text,
856
- 'Text': d_l.find('cdas:Text',
857
- namespaces=NAMESPACES).text,
858
- 'Url': d_l.find('cdas:Url',
859
- namespaces=NAMESPACES).text,
860
- })
861
-
862
- observatories = []
863
- for obs_elem in ds.findall('cdas:Observatory',
864
- namespaces=NAMESPACES):
865
- observatories.append(obs_elem.text)
866
-
867
- instruments = []
868
- for instr_elem in ds.findall('cdas:Instrument',
869
- namespaces=NAMESPACES):
870
- instruments.append(instr_elem.text)
871
-
872
- dataset = {
873
- 'Id': ds.find('cdas:Id', namespaces=NAMESPACES).text,
874
- 'Observatory': observatories,
875
- 'Instrument': instruments,
876
- 'ObservatoryGroup': observatory_groups,
877
- 'InstrumentType': instrument_types,
878
- 'Label': ds.find('cdas:Label',
879
- namespaces=NAMESPACES).text,
880
- 'TimeInterval': {
881
- 'Start': ds.find('cdas:TimeInterval/cdas:Start',
882
- namespaces=NAMESPACES).text,
883
- 'End': ds.find('cdas:TimeInterval/cdas:End',
884
- namespaces=NAMESPACES).text
885
- },
886
- 'PiName': ds.find('cdas:PiName',
887
- namespaces=NAMESPACES).text,
888
- 'PiAffiliation': ds.find('cdas:PiAffiliation',
889
- namespaces=NAMESPACES).text,
890
- 'Notes': ds.find('cdas:Notes',
891
- namespaces=NAMESPACES).text,
892
- 'DatasetLink': dataset_links
893
- }
894
- doi = ds.find('cdas:Doi', namespaces=NAMESPACES)
895
- if doi is not None:
896
- dataset['Doi'] = doi.text
897
-
898
- spase_resource_id = ds.find('cdas:SpaseResourceId',
899
- namespaces=NAMESPACES)
900
- if spase_resource_id is not None:
901
- dataset['SpaseResourceId'] = spase_resource_id.text
902
-
903
- additional_metadata = []
904
- for add_meta in ds.findall('cdas:AdditionalMetadata',
905
- namespaces=NAMESPACES):
906
- meta_type = add_meta.attrib['Type']
907
- value = add_meta.text
908
- additional_metadata.append({
909
- 'Type': meta_type,
910
- 'value': value
911
- })
912
-
913
- if len(additional_metadata) > 0:
914
- dataset['AdditionalMetadata'] = additional_metadata
915
-
916
- datasets.append(dataset)
917
-
918
- return sorted(datasets, key=itemgetter('Id'))
919
- # pylint: enable=too-many-branches
920
-
921
-
922
- @staticmethod
923
- def get_doi_landing_page_url(
924
- doi: str
925
- ) -> str:
926
- """
927
- Returns a URL to the given Digital Object Identifier's landing
928
- page (metadata for the DOI).
929
-
930
- Parameters
931
- ----------
932
- doi
933
- digital object identifier.
934
- Returns
935
- -------
936
- str
937
- A URL to the DOI's landing page.
938
- """
939
-
940
- if not doi.startswith('http'):
941
- return 'https://doi.org/' + doi
942
- return doi
943
-
944
-
945
- @staticmethod
946
- def get_citation(
947
- doi: str
948
- ) -> str:
949
- """
950
- Returns the citation from doi.org for the given DOI.
951
-
952
- Parameters
953
- ----------
954
- doi
955
- digital object identifier.
956
- Returns
957
- -------
958
- str
959
- The citation from doi.org for the given DOI.
960
- """
961
-
962
- url = 'https://doi.org/' + doi
963
- headers = {'Accept': 'text/x-bibliography; style=apa'}
964
- response = requests.get(url, headers=headers)
965
-
966
- return response.text
967
-
968
-
969
- def get_inventory(
970
- self,
971
- identifier: str,
972
- **keywords: str
973
- ) -> List[TimeInterval]:
974
- """
975
- Gets a description of the specified dataset's data inventory.
976
-
977
- Parameters
978
- ----------
979
- identifier
980
- dataset identifier of data inventory to get.
981
- keywords
982
- optional keyword parameters as follows:<br>
983
- <b>timeInterval</b> - `timeinterval.TimeInterval` to restrict
984
- returned inventory.
985
- Returns
986
- -------
987
- List
988
- An array of `timeinterval.TimeInterval`s when data is
989
- available.
990
- """
991
-
992
- url = self._endpoint + 'datasets/' + \
993
- urllib.parse.quote(identifier, safe='') + '/inventory'
994
-
995
- if 'timeInterval' in keywords:
996
- time_interval_keyword = keywords['timeInterval']
997
- url = url + '/' + \
998
- TimeInterval.basic_iso_format(time_interval_keyword.start) + \
999
- ',' + \
1000
- TimeInterval.basic_iso_format(time_interval_keyword.end)
1001
-
1002
- self.logger.debug('request url = %s', url)
1003
-
1004
- response = self._session.get(url, timeout=self._timeout)
1005
-
1006
- if response.status_code != 200:
1007
-
1008
- self.logger.info('%s failed with http code %d', url,
1009
- response.status_code)
1010
- self.logger.info('response.text: %s', response.text)
1011
- return []
1012
-
1013
- if self.logger.level <= logging.DEBUG:
1014
- self.logger.debug('response.text = %s', response.text)
1015
-
1016
- inventory = ET.fromstring(response.text)
1017
- intervals = []
1018
- for inventory_desc in inventory.findall(\
1019
- 'cdas:InventoryDescription',
1020
- namespaces=NAMESPACES):
1021
- for time_interval in inventory_desc.findall(\
1022
- 'cdas:TimeInterval',
1023
- namespaces=NAMESPACES):
1024
- intervals.append(
1025
- TimeInterval(
1026
- time_interval.find('cdas:Start',
1027
- namespaces=NAMESPACES).text,
1028
- time_interval.find('cdas:End',
1029
- namespaces=NAMESPACES).text
1030
- )
1031
- )
1032
-
1033
- return intervals
1034
-
1035
-
1036
- def get_example_time_interval(
1037
- self,
1038
- identifier: str,
1039
- ) -> TimeInterval:
1040
- """
1041
- Gets a small example time interval for the specified dataset. The
1042
- interval is near the end of the dataset's data inventory. The
1043
- returned interval is not guaranteed to have non-fill data for any
1044
- specific variable.
1045
-
1046
- Parameters
1047
- ----------
1048
- identifier
1049
- dataset identifier of data inventory to get.
1050
- Returns
1051
- -------
1052
- timeinterval.TimeInterval
1053
- An small example time interval that is likely, but not
1054
- guaranteed, to have data or None if an interval cannot be
1055
- found.
1056
- """
1057
-
1058
- time_intervals = self.get_inventory(identifier)
1059
- if len(time_intervals) < 1:
1060
- return None
1061
- example_interval = time_intervals[-1]
1062
- if re.search('MMS[1-4]_.+_BRST_.+', identifier):
1063
- time_delta = timedelta(seconds=1)
1064
- else:
1065
- time_delta = timedelta(hours=2)
1066
- example_interval.start = example_interval.end - time_delta
1067
- return example_interval
1068
-
1069
-
1070
- def get_variables(
1071
- self,
1072
- identifier: str
1073
- ) -> List[Dict]:
1074
- """
1075
- Gets a description of the variables in the specified dataset.
1076
-
1077
- Parameters
1078
- ----------
1079
- identifier
1080
- dataset identifier of data to get.
1081
- Returns
1082
- -------
1083
- List
1084
- A List of dictionary descriptions of the variables in
1085
- the specified dataset. The dictionary structure is defined by
1086
- the VariableDescription element in
1087
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>.
1088
- """
1089
-
1090
- url = self._endpoint + 'datasets/' + \
1091
- urllib.parse.quote(identifier, safe='') + '/variables'
1092
-
1093
- response = self._session.get(url, timeout=self._timeout)
1094
-
1095
- if response.status_code != 200:
1096
-
1097
- self.logger.info('%s failed with http code %d', url,
1098
- response.status_code)
1099
- self.logger.info('response.text: %s', response.text)
1100
- return []
1101
-
1102
- if self.logger.level <= logging.DEBUG:
1103
- self.logger.debug('response.text = %s', response.text)
1104
-
1105
- var_descriptions = ET.fromstring(response.text)
1106
-
1107
- variables = []
1108
- for var_description in var_descriptions.findall(\
1109
- 'cdas:VariableDescription',
1110
- namespaces=NAMESPACES):
1111
- name = var_description.find('cdas:Name',
1112
- namespaces=NAMESPACES).text
1113
- short_description = var_description.find(\
1114
- 'cdas:ShortDescription',
1115
- namespaces=NAMESPACES).text
1116
- if short_description is None:
1117
- short_description = ''
1118
-
1119
- long_description = var_description.find(\
1120
- 'cdas:LongDescription',
1121
- namespaces=NAMESPACES).text
1122
- if long_description is None:
1123
- long_description = ''
1124
-
1125
- variables.append({
1126
- 'Name': name,
1127
- 'ShortDescription': short_description,
1128
- 'LongDescription': long_description
1129
- })
1130
-
1131
- return variables
1132
-
1133
-
1134
- def get_variable_names(
1135
- self,
1136
- identifier: str
1137
- ) -> List[str]:
1138
- """
1139
- Gets the names of the variables in the specified dataset. This
1140
- method is like the get_variables method except that it only returns
1141
- the variable names and not the other metadata.
1142
-
1143
- Parameters
1144
- ----------
1145
- identifier
1146
- dataset identifier of data to get.
1147
- Returns
1148
- -------
1149
- List
1150
- A List of the names of the variables in the specified dataset.
1151
- """
1152
-
1153
- variable_names = []
1154
- for variable in self.get_variables(identifier):
1155
- variable_names.append(variable['Name'])
1156
-
1157
- return variable_names
1158
-
1159
-
1160
- @staticmethod
1161
- def _get_thumbnail_description_dict(
1162
- file_description_elem: ET.Element
1163
- ) -> Dict:
1164
- """
1165
- Gets ThumbnailDescription dictionary representation from the
1166
- given FileDescription element.
1167
-
1168
- Parameters
1169
- ----------
1170
- file_description_elem
1171
- a FileDescription Element.
1172
- Returns
1173
- -------
1174
- Dict
1175
- a Dictionary representation of the ThumbnailDescription
1176
- contained in the given FileDescription element.
1177
- """
1178
- thumbnail_desc = file_description_elem.find(\
1179
- 'cdas:ThumbnailDescription',
1180
- namespaces=NAMESPACES)
1181
- if thumbnail_desc is not None:
1182
- time_interval = thumbnail_desc.find('cdas:TimeInterval',
1183
- namespaces=NAMESPACES)
1184
- start = time_interval.find('cdas:Start',
1185
- namespaces=NAMESPACES).text
1186
- end = time_interval.find('cdas:End',
1187
- namespaces=NAMESPACES).text
1188
- return {
1189
- 'Name': thumbnail_desc.find('cdas:Name',
1190
- namespaces=NAMESPACES).text,
1191
- 'Dataset': thumbnail_desc.find('cdas:Dataset',
1192
- namespaces=NAMESPACES).text,
1193
- 'TimeInterval': {
1194
- 'Start': start,
1195
- 'End': end
1196
- },
1197
- 'VarName': thumbnail_desc.find('cdas:VarName',
1198
- namespaces=NAMESPACES).text,
1199
- 'Options': int(thumbnail_desc.find(\
1200
- 'cdas:Options',
1201
- namespaces=NAMESPACES).text),
1202
- 'NumFrames': int(thumbnail_desc.find(\
1203
- 'cdas:NumFrames',
1204
- namespaces=NAMESPACES).text),
1205
- 'NumRows': int(thumbnail_desc.find(\
1206
- 'cdas:NumRows',
1207
- namespaces=NAMESPACES).text),
1208
- 'NumCols': int(thumbnail_desc.find(\
1209
- 'cdas:NumCols',
1210
- namespaces=NAMESPACES).text),
1211
- 'TitleHeight': int(thumbnail_desc.find(\
1212
- 'cdas:TitleHeight',
1213
- namespaces=NAMESPACES).text),
1214
- 'ThumbnailHeight': int(thumbnail_desc.find(\
1215
- 'cdas:ThumbnailHeight',
1216
- namespaces=NAMESPACES).text),
1217
- 'ThumbnailWidth': int(thumbnail_desc.find(\
1218
- 'cdas:ThumbnailWidth',
1219
- namespaces=NAMESPACES).text),
1220
- 'StartRecord': int(thumbnail_desc.find(\
1221
- 'cdas:StartRecord',
1222
- namespaces=NAMESPACES).text),
1223
- 'MyScale': float(thumbnail_desc.find(\
1224
- 'cdas:MyScale',
1225
- namespaces=NAMESPACES).text),
1226
- 'XyStep': float(thumbnail_desc.find(\
1227
- 'cdas:XyStep',
1228
- namespaces=NAMESPACES).text)
1229
- }
1230
- return None
1231
-
1232
-
1233
- @staticmethod
1234
- def _get_data_result_dict(
1235
- xml_data_result: str
1236
- ) -> Dict:
1237
- """
1238
- Gets DataResult dictionary representation from the
1239
- given XML DataResult element.
1240
-
1241
- Parameters
1242
- ----------
1243
- xml_data_result
1244
- XML representation of a DataResult.
1245
- Returns
1246
- -------
1247
- Dict
1248
- a Dictionary representation of the given XML representation
1249
- of a DataResult.
1250
- """
1251
- data_result = ET.fromstring(xml_data_result)
1252
- file_descriptions = []
1253
- for file_description in data_result.findall(\
1254
- 'cdas:FileDescription', namespaces=NAMESPACES):
1255
-
1256
- dict_file_description = {
1257
- 'Name': file_description.find('cdas:Name',
1258
- namespaces=NAMESPACES).text,
1259
- 'MimeType': file_description.find(\
1260
- 'cdas:MimeType',
1261
- namespaces=NAMESPACES).text,
1262
- 'StartTime': file_description.find(\
1263
- 'cdas:StartTime',
1264
- namespaces=NAMESPACES).text,
1265
- 'EndTime': file_description.find(\
1266
- 'cdas:EndTime',
1267
- namespaces=NAMESPACES).text,
1268
- 'Length': int(file_description.find(\
1269
- 'cdas:Length',
1270
- namespaces=NAMESPACES).text),
1271
- 'LastModified': file_description.find(\
1272
- 'cdas:LastModified',
1273
- namespaces=NAMESPACES).text
1274
- }
1275
- thumbnail_dict = CdasWs._get_thumbnail_description_dict(\
1276
- file_description)
1277
- if thumbnail_dict is not None:
1278
- dict_file_description['ThumbnailDescription'] = \
1279
- thumbnail_dict
1280
-
1281
- thumbnail_id_elem = file_description.find(\
1282
- 'cdas:ThumbnailId',
1283
- namespaces=NAMESPACES)
1284
- if thumbnail_id_elem is not None:
1285
- dict_file_description['ThumbnailId'] = \
1286
- thumbnail_id_elem.text
1287
-
1288
- file_descriptions.append(dict_file_description)
1289
-
1290
- if len(file_descriptions) > 0:
1291
- return {
1292
- 'FileDescription': file_descriptions
1293
- }
1294
- return None
1295
-
1296
-
1297
- def get_data_result(
1298
- self,
1299
- data_request: DataRequest,
1300
- progress_callback: Callable[[float, str, Any], int],
1301
- progress_user_value: Any
1302
- ) -> Tuple[int, Dict]:
1303
- """
1304
- Submits the given request to the server and returns the result.
1305
- This is a relatively low-level method and most callers should
1306
- probably use a higher-level method such as get_data.
1307
-
1308
- Parameters
1309
- ----------
1310
- data_request
1311
- data request.
1312
- progress_callback
1313
- function that is called repeatedly to report the progress
1314
- of getting the data. The function should return 0 if it
1315
- wants to continue getting data. If it returns a non-0 value,
1316
- getting the data will be aborted and the get_data() function
1317
- will immediately return (204, None). The float parameter
1318
- is a value between 0.0 and 1.0 to indicate progress and
1319
- the str parameter will contain a text message indicating
1320
- the progress of this call.
1321
- progressUserValue
1322
- value that is passsed to the progressCallback function.
1323
- Returns
1324
- -------
1325
- Tuple
1326
- [0] contains the int HTTP status code. 200 when
1327
- successful.<br>
1328
- [1] contains a dictionary representing the DataResult from
1329
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1330
- or None.
1331
- See Also
1332
- --------
1333
- CdasWs.get_data
1334
- """
1335
-
1336
- #self.logger.debug('data_request = %s', data_request.json())
1337
- self.logger.debug('data_request = %s', data_request.xml_str())
1338
-
1339
- url = self._endpoint + 'datasets'
1340
-
1341
- for retries in range(RETRY_LIMIT):
1342
- #response = self._session.post(url, data=data_request.json(),
1343
- response = self._session.post(url, data=data_request.xml_str(),
1344
- timeout=self._timeout)
1345
-
1346
- if response.status_code == 200:
1347
-
1348
- data_result = CdasWs._get_data_result_dict(response.text)
1349
-
1350
- if not data_result:
1351
- return (response.status_code, None)
1352
-
1353
- return (response.status_code, data_result)
1354
-
1355
- if response.status_code == 429 or \
1356
- response.status_code == 503 and \
1357
- 'Retry-After' in response.headers:
1358
-
1359
- retry_after = response.headers['Retry-After']
1360
-
1361
- self.logger.debug('429/503 status with Retry-After header: %s',
1362
- retry_after)
1363
-
1364
- if progress_callback is not None:
1365
- if progress_callback(0.2, 'Waiting ' + retry_after + \
1366
- 's before making server request.',
1367
- progress_user_value) != 0:
1368
- return (204, None)
1369
-
1370
- retry_after = int(retry_after)
1371
-
1372
- self.logger.info('Sleeping %d seconds before making request',
1373
- retry_after)
1374
- time.sleep(retry_after)
1375
-
1376
- else:
1377
- self.logger.info('%s failed with http code %d', url,
1378
- response.status_code)
1379
- self.logger.info('data_request = %s', data_request)
1380
- self.logger.info('response.text: %s', response.text)
1381
- return (response.status_code, None)
1382
-
1383
- self.logger.info('%s failed with http code %d after %d retries',
1384
- url, response.status_code, retries + 1)
1385
- self.logger.info('data_request = %s', data_request)
1386
- self.logger.info('response.text: %s', response.text)
1387
- return (response.status_code, None)
1388
-
1389
-
1390
- def get_data_file(
1391
- self,
1392
- dataset: str,
1393
- variables: List[str],
1394
- start: Union[datetime, str], end: Union[datetime, str],
1395
- **keywords: Union[
1396
- Dict,
1397
- Callable[[float, str, Any], int],
1398
- Any]
1399
- ) -> Tuple[int, Dict]:
1400
- """
1401
- Gets the specified data file from the server.
1402
-
1403
- Parameters
1404
- ----------
1405
- dataset
1406
- dataset identifier of data to get.
1407
- variables
1408
- array containing names of variables to get.
1409
- start
1410
- start time of data to get. See module note about timezone.
1411
- end
1412
- end time of data to get. See module note about timezone.
1413
- keywords
1414
- optional keyword parameters as follows:<br>
1415
- <b>binData</b> - indicates that uniformly spaced values should
1416
- be computed for scaler/vector/spectrogram data according to
1417
- the given binning parameter values. binData is a Dict that
1418
- may contain the following keys: interval,
1419
- interpolateMissingValues, sigmaMultiplier, and/or
1420
- overrideDefaultBinning with values that override the
1421
- defaults.<br>
1422
- <b>progressCallback</b> - is a
1423
- Callable[[float, str, typing.Any], int]
1424
- function that is called repeatedly to report the progress
1425
- of getting the data. The function should return 0 if it
1426
- wants to continue getting data. If it returns non-0 value,
1427
- getting the data will be aborted and the get_data_file()
1428
- function will immediately return (204, None). The float
1429
- parameter is a value between 0.0 and 1.0 to indicate progress
1430
- and the str parameter will contain a text message indicating
1431
- the progress of this call.<br>
1432
- <b>progressUserValue</b> - is an Any value that is passsed
1433
- to the progressCallback function.<br>
1434
- Returns
1435
- -------
1436
- Tuple
1437
- [0] contains the int HTTP status code. 200 when
1438
- successful.<br>
1439
- [1] contains a dictionary representing the DataResult from
1440
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1441
- or None.
1442
- Raises
1443
- ------
1444
- ValueError
1445
- If the given start/end datetime values are invalid.
1446
- See Also
1447
- --------
1448
- CdasWs.get_data : In addition to what get_data_file does,
1449
- get_data also downloads and reads the data file into memory
1450
- (SpaceData or xarray.Dataset object).
1451
- """
1452
- # pylint: disable=too-many-locals
1453
- # pylint: disable=too-many-return-statements
1454
- # pylint: enable=too-many-statements
1455
- # pylint: disable=too-many-branches
1456
-
1457
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
1458
- end)
1459
-
1460
- data_request = CdfRequest(dataset, variables,
1461
- TimeInterval(start_datetime,
1462
- end_datetime),
1463
- 3, CdfFormat.BINARY,
1464
- **keywords.get('binData', {}))
1465
-
1466
- progress_callback = keywords.get('progressCallback', None)
1467
- progress_user_value = keywords.get('progressUserValue', None)
1468
-
1469
- self.logger.debug('data_request = %s', data_request)
1470
-
1471
- if progress_callback is not None:
1472
- if progress_callback(0.1, 'Making server request.',
1473
- progress_user_value) != 0:
1474
- return (204, None)
1475
-
1476
- status_code, data_result = self.get_data_result(data_request,
1477
- progress_callback,
1478
- progress_user_value)
1479
-
1480
- if progress_callback is not None:
1481
- if progress_callback(1.0, 'Initial server request complete.',
1482
- progress_user_value) != 0:
1483
- return (status_code, None)
1484
-
1485
- return (status_code, data_result)
1486
-
1487
-
1488
- def download(
1489
- self,
1490
- url: str,
1491
- size: int = 0,
1492
- **keywords
1493
- ) -> str:
1494
- """
1495
- Downloads the file specified by the given URL to a temporary
1496
- file without reading all of it into memory. This method
1497
- utilizes the connection pool and persistent HTTP connection
1498
- to the CdasWs server.
1499
-
1500
- Parameters
1501
- ----------
1502
- url
1503
- URL of file to download.
1504
- size
1505
- number of bytes in file to download.
1506
- keywords
1507
- optional keyword parameters as follows:<br>
1508
- <b>progressCallback</b> - is a
1509
- typing.Callable[[float, str, typing.Any], int]
1510
- function that is called repeatedly to report the progress
1511
- of getting the data. The function should return 0 if it
1512
- wants to continue getting data. If it returns a non-0 value,
1513
- getting the data will be aborted and this download() function
1514
- will immediately return None. The float parameter
1515
- is a value between 0.0 and 1.0 to indicate progress and
1516
- the str parameter will contain a text message indicating
1517
- the progress of this call.<br>
1518
- <b>progressUserValue</b> - is a typing.Any value that is
1519
- passsed to the progressCallback function.<br>
1520
- Returns
1521
- -------
1522
- str
1523
- name of tempory file or None if there was an error.
1524
- """
1525
- # pylint: disable=too-many-locals
1526
-
1527
- progress_callback = keywords.get('progressCallback', None)
1528
- progress_user_value = keywords.get('progressUserValue', None)
1529
-
1530
- suffix = os.path.splitext(urlparse(url).path)[1]
1531
-
1532
- file_descriptor, tmp_filename = mkstemp(suffix=suffix)
1533
-
1534
- download_bytes = 0
1535
- next_progress_report = 0.1
1536
- with self._session.get(url, stream=True,
1537
- timeout=self._timeout) as response:
1538
-
1539
- file = open(tmp_filename, 'wb')
1540
- for chunk in response.iter_content(chunk_size=8192):
1541
- if chunk: # filter out keep-alive new chunks
1542
- file.write(chunk)
1543
- # file.flush()
1544
- if progress_callback is not None:
1545
- download_bytes += len(chunk)
1546
- if size == 0:
1547
- download_progress = 0.0
1548
- else:
1549
- download_progress = float(download_bytes) / size
1550
- if download_progress > next_progress_report:
1551
- next_progress_report += download_progress
1552
- if progress_callback(download_progress,\
1553
- 'Continuing download of data.',
1554
- progress_user_value) != 0:
1555
-
1556
- file.close()
1557
- os.close(file_descriptor)
1558
- return None
1559
- file.close()
1560
- os.close(file_descriptor)
1561
-
1562
- if progress_callback is not None:
1563
- if progress_callback(0.4,
1564
- 'Data download complete. Reading data.',
1565
- progress_user_value) != 0:
1566
- return None
1567
-
1568
- return tmp_filename
1569
-
1570
-
1571
- @staticmethod
1572
- def read_data(
1573
- filename: str,
1574
- data_representation: DataRepresentation
1575
- ) -> Union['spacepy.datamodel', 'xr.Dataset']:
1576
- """
1577
- Reads the data from the given file.
1578
-
1579
- Parameters
1580
- ----------
1581
- filename
1582
- Name of file to read.
1583
- data_representation
1584
- Requested data representation.
1585
- Returns
1586
- -------
1587
- spacepy.datamodel or xr.Dataset
1588
- Data from file.
1589
- Raises
1590
- ------
1591
- Exception
1592
- If an Exception is raise by either the spdm.fromCDF() or
1593
- cdflib.cdf_to_xarray() functions.
1594
- ModuleNotFoundError
1595
- If the required spacepy.datamodel or the cdflib and xarray
1596
- modules are not installed.
1597
- """
1598
- if data_representation is None:
1599
- if SPDM_AVAILABLE:
1600
- return spdm.fromCDF(filename)
1601
- if CDF_XARRAY_AVAILABLE:
1602
- return cdf_to_xarray(filename, to_datetime=True,
1603
- fillval_to_nan=True)
1604
- raise ModuleNotFoundError(
1605
- 'neither the spacepy.datamodel nor the cdflib and '
1606
- 'xarray modules are installed')
1607
-
1608
- if data_representation is DataRepresentation.SPACEPY and \
1609
- not SPDM_AVAILABLE:
1610
- raise ModuleNotFoundError('spacepy module must be installed')
1611
- if data_representation is DataRepresentation.XARRAY and \
1612
- not CDF_XARRAY_AVAILABLE:
1613
- raise ModuleNotFoundError('cdflib and xarray modules must be installed')
1614
-
1615
- if data_representation is DataRepresentation.SPACEPY:
1616
- return spdm.fromCDF(filename)
1617
- if data_representation is DataRepresentation.XARRAY:
1618
- return cdf_to_xarray(filename, to_datetime=True,
1619
- fillval_to_nan=True)
1620
- return None
1621
-
1622
-
1623
- def get_data(
1624
- self,
1625
- dataset: str,
1626
- variables: List[str],
1627
- time0: Union[TimeInterval, List[TimeInterval], datetime, str],
1628
- time1: Union[datetime, str] = None,
1629
- **keywords: Union[
1630
- Dict,
1631
- DataRepresentation,
1632
- Callable[[float, str, Any], int],
1633
- Any]
1634
- ) -> Tuple[Dict, 'spdm.SpaceData', 'xarray']:
1635
- """
1636
- Gets the specified data from the server. The representation
1637
- of the returned data is determined as follows:<br>
1638
- 1. If a dataRepresentation keyword parameter is given, its
1639
- value will determine the representation of the returned
1640
- data. If no dataRepresenation keyword parameter is
1641
- given, then<br>
1642
- 2. If the presence of spacepy.datamodel is found, then the data
1643
- is returned in the spacepy.datamodel representation.<br>
1644
- 3. If the presence of the cdflib and xarray modules are found,
1645
- then the data is returned in an xarray.Dataset.
1646
-
1647
- Parameters
1648
- ----------
1649
- dataset
1650
- dataset identifier of data to get.
1651
- variables
1652
- array containing names of variables to get. The value
1653
- ALL-VARIABLES may be used instead of specifying all the
1654
- individual variable names.
1655
- time0
1656
- TimeInterval(s) or start time of data to get. See module
1657
- note about timezone.
1658
- time1
1659
- when time0 is not one or more TimeInterval(s), the end time
1660
- of data to get. See module note about timezone.
1661
- keywords
1662
- optional keyword parameters as follows:<br>
1663
- <b>binData</b> - indicates that uniformly spaced values should
1664
- be computed for scaler/vector/spectrogram data according to
1665
- the given binning parameter values. See
1666
- <https://cdaweb.gsfc.nasa.gov/CDAWeb_Binning_readme.html>
1667
- for more details. binData is a Dict that
1668
- may contain the following keys: interval,
1669
- interpolateMissingValues, sigmaMultiplier, and/or
1670
- overrideDefaultBinning with values that override the
1671
- defaults.<br>
1672
- <b>dataRepresentation</b> - specifies the representation of
1673
- the returned data as one of
1674
- `datarepresentation.DataRepresentation`.<br>
1675
- <b>progressCallback</b> - is a
1676
- Callable[[float, str, typing.Any], int]
1677
- function that is called repeatedly to report the progress
1678
- of getting the data. The function should return 0 if it
1679
- wants to continue getting data. If it returns non-0 value,
1680
- getting the data will be aborted and the get_data() function
1681
- will immediately return (204, None). The float parameter
1682
- is a value between 0.0 and 1.0 to indicate progress and
1683
- the str parameter will contain a text message indicating
1684
- the progress of this call.<br>
1685
- <b>progressUserValue</b> - is an Any value that is passsed
1686
- to the progressCallback function.<br>
1687
- Returns
1688
- -------
1689
- Tuple
1690
- [0] contains a dictionary of HTTP and CDAS status information.
1691
- When successful, ['http']['status_code'] will be 200.<br>
1692
- [1] contains the requested data (SpaceData or xarray.Dataset
1693
- object) or None.
1694
- Raises
1695
- ------
1696
- ValueError
1697
- If no variables are given or if the given start/end datetime
1698
- values are invalid.
1699
- """
1700
- # pylint: disable=too-many-locals
1701
- # pylint: disable=too-many-return-statements
1702
- # pylint: disable=too-many-statements
1703
- # pylint: disable=too-many-branches
1704
- # pylint: disable=import-outside-toplevel
1705
-
1706
- #import spacepy.datamodel as spdm # type: ignore
1707
-
1708
- if len(variables) < 1:
1709
- raise ValueError('at least one variable name is required')
1710
-
1711
- if isinstance(time0, (str, datetime)):
1712
- if isinstance(time1, (str, datetime)):
1713
- time_intervals = [TimeInterval(time0, time1)]
1714
- else:
1715
- raise ValueError('time1 must be str/datetime')
1716
- elif isinstance(time0, TimeInterval):
1717
- time_intervals = [time0]
1718
- elif isinstance(time0, list) and len(time0) > 0 and\
1719
- isinstance(time0[0], TimeInterval):
1720
- time_intervals = time0
1721
- else:
1722
- raise ValueError('invalid time0 type')
1723
-
1724
- data_request = CdfRequest(dataset, variables,
1725
- time_intervals,
1726
- 3, CdfFormat.BINARY,
1727
- binData=keywords.get('binData', {}))
1728
-
1729
- data_rep = keywords.get('dataRepresentation', None)
1730
- progress_callback = keywords.get('progressCallback', None)
1731
- progress_user_value = keywords.get('progressUserValue', None)
1732
-
1733
- self.logger.debug('data_request = %s', data_request)
1734
-
1735
- status = {
1736
- 'http': {
1737
- 'status_code': 204
1738
- },
1739
- 'cdas': {
1740
- 'status': [],
1741
- 'message': [],
1742
- 'warning': [],
1743
- 'error': []
1744
- }
1745
- }
1746
-
1747
- if progress_callback is not None:
1748
- if progress_callback(0.1, 'Making initial server request.',
1749
- progress_user_value) != 0:
1750
- return (status, None)
1751
-
1752
- status_code, data_result = self.get_data_result(data_request,
1753
- progress_callback,
1754
- progress_user_value)
1755
-
1756
- status['http']['status_code'] = status_code
1757
-
1758
- if progress_callback is not None:
1759
- if progress_callback(0.3, 'Initial server request complete.',
1760
- progress_user_value) != 0:
1761
- return (status, None)
1762
-
1763
- if status_code != 200:
1764
-
1765
- self.logger.info('get_data_result failed with http code %d',
1766
- status_code)
1767
- self.logger.info('data_request = %s', data_request)
1768
- return (status, None)
1769
-
1770
- if not data_result:
1771
- return (status, None)
1772
-
1773
- if 'Status' in data_result:
1774
- status['cdas']['status'] = data_result['Status']
1775
- if 'Message' in data_result:
1776
- status['cdas']['message'] = data_result['Message']
1777
- if 'Warning' in data_result:
1778
- status['cdas']['warning'] = data_result['Warning']
1779
- if 'Error' in data_result:
1780
- status['cdas']['error'] = data_result['Error']
1781
-
1782
- if progress_callback is not None:
1783
- if progress_callback(0.4, 'Beginning download of data.',
1784
- progress_user_value) != 0:
1785
- return (status, None)
1786
-
1787
- file_descriptions = data_result['FileDescription']
1788
-
1789
- data_url = file_descriptions[0]['Name']
1790
- data_length = file_descriptions[0]['Length']
1791
-
1792
- self.logger.debug('data_url = %s, data_length = %d',
1793
- data_url, data_length)
1794
-
1795
- sub_progress_control = {
1796
- 'progressCallback': progress_callback,
1797
- 'progressUserValue': progress_user_value,
1798
- 'progressStart': 0.4,
1799
- 'progressFraction': 0.1
1800
- }
1801
-
1802
- tmp_filename = self.download(data_url, data_length,
1803
- progressCallback=_get_data_progress,
1804
- progressUserValue=sub_progress_control)
1805
-
1806
- try:
1807
- data = self.read_data(tmp_filename, data_rep)
1808
- os.remove(tmp_filename)
1809
- if progress_callback is not None:
1810
- if progress_callback(1.0, 'Finished reading data.',
1811
- progress_user_value) != 0:
1812
- return (status, None)
1813
- except:
1814
- self.logger.error('Exception from read_data(%s): %s, %s',
1815
- tmp_filename, sys.exc_info()[0],
1816
- sys.exc_info()[1])
1817
- self.logger.error('CDF file has been retained.')
1818
- raise
1819
- return (status, data)
1820
-
1821
-
1822
- # pylint: disable=too-many-arguments
1823
- def get_graph(
1824
- self,
1825
- dataset: str,
1826
- variables: List[str],
1827
- start: Union[datetime, str],
1828
- end: Union[datetime, str],
1829
- options: GraphOptions = None,
1830
- image_format: List[ImageFormat] = None,
1831
- **keywords
1832
- ) -> Tuple[int, Dict]:
1833
- """
1834
- Gets a graphical representation of the specified data from the
1835
- server.
1836
-
1837
- Parameters
1838
- ----------
1839
- dataset
1840
- dataset identifier of data to get.
1841
- variables
1842
- array containing names of variables to get.
1843
- start
1844
- start time of data to get. See module note about timezone.
1845
- end
1846
- end time of data to get. See module note about timezone.
1847
- options
1848
- graph options.
1849
- image_format
1850
- image format. If None, then [ImageFormat.PNG].
1851
- keywords
1852
- optional keyword parameters as follows:<br>
1853
- <b>binData</b> - indicates that uniformly spaced values should
1854
- be computed for scaler/vector/spectrogram data according to
1855
- the given binning parameter values. binData is a Dict that
1856
- may contain the following keys: interval,
1857
- interpolateMissingValues, sigmaMultiplier, and/or
1858
- overrideDefaultBinning with values that override the
1859
- defaults.<br>
1860
- <b>progressCallback</b> - is a
1861
- typing.Callable[[float, str, typing.Any], int]
1862
- function that is called repeatedly to report the progress
1863
- of getting the data. The function should return 0 if it
1864
- wants to continue getting data. If it returns non-0 value,
1865
- getting the data will be aborted and the get_data() function
1866
- will immediately return (204, None). The float parameter
1867
- is a value between 0.0 and 1.0 to indicate progress and
1868
- the str parameter will contain a text message indicating
1869
- the progress of this call.<br>
1870
- <b>progressUserValue</b> - is a typing.Any value that is
1871
- passsed to the progressCallback function.<br>
1872
- Returns
1873
- -------
1874
- Tuple
1875
- [0] contains the HTTP status code value (200 when successful).<br>
1876
- [1] contains a dictionary representation of a
1877
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1878
- DataResult object or None.<br>
1879
- Raises
1880
- ------
1881
- ValueError
1882
- If the given start/end datetime values are invalid.
1883
- """
1884
- # pylint: disable=too-many-locals
1885
- # pylint: disable=too-many-return-statements
1886
- # pylint: enable=too-many-statements
1887
- # pylint: disable=too-many-branches
1888
-
1889
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
1890
- end)
1891
-
1892
- request = GraphRequest(dataset, variables,
1893
- TimeInterval(start_datetime, end_datetime),
1894
- options, image_format,
1895
- **keywords)
1896
-
1897
- progress_callback = keywords.get('progressCallback', None)
1898
- progress_user_value = keywords.get('progressUserValue', None)
1899
-
1900
- self.logger.debug('request = %s', request)
1901
-
1902
- if progress_callback is not None:
1903
- if progress_callback(0.1, 'Making server request.',
1904
- progress_user_value) != 0:
1905
- return (204, None)
1906
-
1907
- status_code, result = self.get_data_result(request, progress_callback, progress_user_value)
1908
-
1909
- if progress_callback is not None:
1910
- if progress_callback(1.0, 'Server request complete.',
1911
- progress_user_value) != 0:
1912
- return (status_code, None)
1913
-
1914
- if status_code != 200:
1915
-
1916
- self.logger.info('get_result failed with http code %d',
1917
- status_code)
1918
- self.logger.info('request = %s', request)
1919
- return (status_code, None)
1920
-
1921
- return (status_code, result)
1922
- # pylint: enable=too-many-arguments
1923
-
1924
-
1925
- # pylint: disable=too-many-arguments
1926
- def get_thumbnail(
1927
- self,
1928
- dataset: str,
1929
- variables: List[str],
1930
- start: Union[datetime, str],
1931
- end: Union[datetime, str],
1932
- identifier: str,
1933
- thumbnail: int = 1,
1934
- **keywords
1935
- ) -> Tuple[int, Dict]:
1936
- """
1937
- Gets a graphical representation of the specified data from the
1938
- server.
1939
-
1940
- Parameters
1941
- ----------
1942
- dataset
1943
- dataset identifier of data to get.
1944
- variables
1945
- array containing names of variables to get.
1946
- start
1947
- start time of data to get. See module note about timezone.
1948
- end
1949
- end time of data to get. See module note about timezone.
1950
- identifier
1951
- thumbnail identifier (returned in a previous get_graph
1952
- result).
1953
- thumbnail
1954
- number of thumbnail whose full size image is being requested.
1955
- Thumbnail images are counted beginning at one (not zero).
1956
- keywords
1957
- optional keyword parameters as follows:<br>
1958
- <b>progressCallback</b> - is a
1959
- typing.Callable[[float, str, typing.Any], int]
1960
- function that is called repeatedly to report the progress
1961
- of getting the data. The function should return 0 if it
1962
- wants to continue getting data. If it returns non-0 value,
1963
- getting the data will be aborted and the get_data() function
1964
- will immediately return (204, None). The float parameter
1965
- is a value between 0.0 and 1.0 to indicate progress and
1966
- the str parameter will contain a text message indicating
1967
- the progress of this call.<br>
1968
- <b>progressUserValue</b> - is a typing.Any value that is
1969
- passsed to the progressCallback function.<br>
1970
- Returns
1971
- -------
1972
- Tuple
1973
- [0] contains the HTTP status code value (200 when successful).<br>
1974
- [1] contains a dictionary representation of a
1975
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
1976
- DataResult object or None.<br>
1977
- Raises
1978
- ------
1979
- ValueError
1980
- If the given start/end datetime values are invalid.
1981
- """
1982
- # pylint: disable=too-many-locals
1983
- # pylint: disable=too-many-return-statements
1984
- # pylint: enable=too-many-statements
1985
- # pylint: disable=too-many-branches
1986
-
1987
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
1988
- end)
1989
-
1990
- request = ThumbnailRequest(dataset, variables,
1991
- TimeInterval(start_datetime, end_datetime),
1992
- identifier, thumbnail)
1993
-
1994
- progress_callback = keywords.get('progressCallback', None)
1995
- progress_user_value = keywords.get('progressUserValue', None)
1996
-
1997
- self.logger.debug('request = %s', request)
1998
-
1999
- if progress_callback is not None:
2000
- if progress_callback(0.1, 'Making server request.',
2001
- progress_user_value) != 0:
2002
- return (204, None)
2003
-
2004
- status_code, result = self.get_data_result(request,
2005
- progress_callback,
2006
- progress_user_value)
2007
-
2008
- if progress_callback is not None:
2009
- if progress_callback(1.0, 'Server request complete.',
2010
- progress_user_value) != 0:
2011
- return (status_code, None)
2012
-
2013
- if status_code != 200:
2014
-
2015
- self.logger.info('get_result failed with http code %d',
2016
- status_code)
2017
- self.logger.info('request = %s', request)
2018
- return (status_code, None)
2019
-
2020
- return (status_code, result)
2021
- # pylint: enable=too-many-arguments
2022
-
2023
-
2024
- # pylint: disable=too-many-arguments
2025
- def get_text(
2026
- self,
2027
- dataset: str,
2028
- variables: List[str],
2029
- start: Union[datetime, str],
2030
- end: Union[datetime, str],
2031
- compression: Compression = Compression.UNCOMPRESSED,
2032
- text_format: TextFormat = TextFormat.PLAIN,
2033
- **keywords
2034
- ) -> Tuple[int, Dict]:
2035
- """
2036
- Gets a textual representation of the specified data from the
2037
- server.
2038
-
2039
- Parameters
2040
- ----------
2041
- dataset
2042
- dataset identifier of data to get.
2043
- variables
2044
- array containing names of variables to get.
2045
- start
2046
- start time of data to get. See module note about timezone.
2047
- end
2048
- end time of data to get. See module note about timezone.
2049
- compression
2050
- file compression.
2051
- text_format
2052
- text format.
2053
- keywords
2054
- optional keyword parameters as follows:<br>
2055
- <b>binData</b> - indicates that uniformly spaced values should
2056
- be computed for scaler/vector/spectrogram data according to
2057
- the given binning parameter values. binData is a Dict that
2058
- may contain the following keys: interval,
2059
- interpolateMissingValues, sigmaMultiplier, and/or
2060
- overrideDefaultBinning with values that override the
2061
- defaults.<br>
2062
- <b>progressCallback</b> - is a
2063
- typing.Callable[[float, str, typing.Any], int]
2064
- function that is called repeatedly to report the progress
2065
- of getting the data. The function should return 0 if it
2066
- wants to continue getting data. If it returns non-0 value,
2067
- getting the data will be aborted and the get_data() function
2068
- will immediately return (204, None). The float parameter
2069
- is a value between 0.0 and 1.0 to indicate progress and
2070
- the str parameter will contain a text message indicating
2071
- the progress of this call.<br>
2072
- <b>progressUserValue</b> - is a typing.Any value that is
2073
- passsed to the progressCallback function.<br>
2074
- Returns
2075
- -------
2076
- Tuple
2077
- [0] contains the HTTP status code value (200 when successful).<br>
2078
- [1] contains a dictionary representation of a
2079
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
2080
- DataResult object or None.<br>
2081
- Raises
2082
- ------
2083
- ValueError
2084
- If the given start/end datetime values are invalid.
2085
- """
2086
- # pylint: disable=too-many-locals
2087
- # pylint: disable=too-many-return-statements
2088
- # pylint: enable=too-many-statements
2089
- # pylint: disable=too-many-branches
2090
-
2091
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2092
- end)
2093
-
2094
- request = TextRequest(dataset, variables,
2095
- TimeInterval(start_datetime, end_datetime),
2096
- compression, text_format,
2097
- **keywords)
2098
-
2099
- progress_callback = keywords.get('progressCallback', None)
2100
- progress_user_value = keywords.get('progressUserValue', None)
2101
-
2102
- self.logger.debug('request = %s', request)
2103
-
2104
- if progress_callback is not None:
2105
- if progress_callback(0.1, 'Making server request.',
2106
- progress_user_value) != 0:
2107
- return (204, None)
2108
-
2109
- status_code, result = self.get_data_result(request,
2110
- progress_callback,
2111
- progress_user_value)
2112
-
2113
- if progress_callback is not None:
2114
- if progress_callback(1.0, 'Server request complete.',
2115
- progress_user_value) != 0:
2116
- return (status_code, None)
2117
-
2118
- if status_code != 200:
2119
-
2120
- self.logger.info('get_result failed with http code %d',
2121
- status_code)
2122
- self.logger.info('request = %s', request)
2123
- return (status_code, None)
2124
-
2125
- return (status_code, result)
2126
- # pylint: enable=too-many-arguments
2127
-
2128
-
2129
- def get_audio(
2130
- self,
2131
- dataset: str,
2132
- variables: List[str],
2133
- start: Union[datetime, str],
2134
- end: Union[datetime, str],
2135
- **keywords
2136
- ) -> Tuple[int, Dict]:
2137
- """
2138
- Gets an audio representation of the specified data from the
2139
- server.
2140
-
2141
- Parameters
2142
- ----------
2143
- dataset
2144
- dataset identifier of data to get.
2145
- variables
2146
- array containing names of variables to get.
2147
- start
2148
- start time of data to get. See module note about timezone.
2149
- end
2150
- end time of data to get. See module note about timezone.
2151
- keywords
2152
- optional keyword parameters as follows:<br>
2153
- <b>binData</b> - indicates that uniformly spaced values should
2154
- be computed for scaler/vector/spectrogram data according to
2155
- the given binning parameter values. binData is a Dict that
2156
- may contain the following keys: interval,
2157
- interpolateMissingValues, sigmaMultiplier, and/or
2158
- overrideDefaultBinning with values that override the
2159
- defaults.<br>
2160
- <b>progressCallback</b> - is a
2161
- typing.Callable[[float, str, typing.Any], int]
2162
- function that is called repeatedly to report the progress
2163
- of getting the data. The function should return 0 if it
2164
- wants to continue getting data. If it returns non-0 value,
2165
- getting the data will be aborted and the get_data() function
2166
- will immediately return (204, None). The float parameter
2167
- is a value between 0.0 and 1.0 to indicate progress and
2168
- the str parameter will contain a text message indicating
2169
- the progress of this call.<br>
2170
- <b>progressUserValue</b> - is a typing.Any value that is
2171
- passsed to the progressCallback function.<br>
2172
- Returns
2173
- -------
2174
- Tuple
2175
- [0] contains the HTTP status code value (200 when successful).<br>
2176
- [1] contains a dictionary representation of a
2177
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
2178
- DataResult object or None.<br>
2179
- Raises
2180
- ------
2181
- ValueError
2182
- If the given start/end datetime values are invalid.
2183
- """
2184
- # pylint: disable=too-many-locals
2185
- # pylint: disable=too-many-return-statements
2186
- # pylint: enable=too-many-statements
2187
- # pylint: disable=too-many-branches
2188
-
2189
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2190
- end)
2191
-
2192
- request = AudioRequest(dataset, variables,
2193
- TimeInterval(start_datetime, end_datetime),
2194
- **keywords)
2195
-
2196
- progress_callback = keywords.get('progressCallback', None)
2197
- progress_user_value = keywords.get('progressUserValue', None)
2198
-
2199
- self.logger.debug('request = %s', request)
2200
-
2201
- if progress_callback is not None:
2202
- if progress_callback(0.1, 'Making server request.',
2203
- progress_user_value) != 0:
2204
- return (204, None)
2205
-
2206
- status_code, result = self.get_data_result(request,
2207
- progress_callback,
2208
- progress_user_value)
2209
-
2210
- if progress_callback is not None:
2211
- if progress_callback(1.0, 'Server request complete.',
2212
- progress_user_value) != 0:
2213
- return (status_code, None)
2214
-
2215
- if status_code != 200:
2216
-
2217
- self.logger.info('get_result failed with http code %d',
2218
- status_code)
2219
- self.logger.info('request = %s', request)
2220
- return (status_code, None)
2221
-
2222
- return (status_code, result)
2223
-
2224
-
2225
- def get_original_files(
2226
- self,
2227
- dataset: str,
2228
- start: Union[datetime, str],
2229
- end: Union[datetime, str],
2230
- **keywords
2231
- ) -> Tuple[int, Dict]:
2232
- """
2233
- Gets original data files from a dataset. Original data files
2234
- lack updated meta-data and virtual variable values contained
2235
- in files obtained from the `CdasWs.get_data`. Most callers
2236
- should probably use `CdasWs.get_data` instead of this function.
2237
-
2238
- Parameters
2239
- ----------
2240
- dataset
2241
- dataset identifier of data to get.
2242
- start
2243
- start time of data to get. See module note about timezone.
2244
- end
2245
- end time of data to get. See module note about timezone.
2246
- keywords
2247
- optional keyword parameters as follows:<br>
2248
- <b>progressCallback</b> - is a
2249
- typing.Callable[[float, str, typing.Any], int]
2250
- function that is called repeatedly to report the progress
2251
- of getting the data. The function should return 0 if it
2252
- wants to continue getting data. If it returns non-0 value,
2253
- getting the data will be aborted and the get_data() function
2254
- will immediately return (204, None). The float parameter
2255
- is a value between 0.0 and 1.0 to indicate progress and
2256
- the str parameter will contain a text message indicating
2257
- the progress of this call.<br>
2258
- <b>progressUserValue</b> - is a typing.Any value that is
2259
- passsed to the progressCallback function.<br>
2260
- Returns
2261
- -------
2262
- Tuple
2263
- [0] contains the HTTP status code value (200 when successful).<br>
2264
- [1] array of dictionary representations of a
2265
- <https://cdaweb.gsfc.nasa.gov/WebServices/REST/CDAS.xsd>
2266
- FileDescription objects or None.<br>
2267
- Raises
2268
- ------
2269
- ValueError
2270
- If the given start/end datetime values are invalid.
2271
- See Also
2272
- --------
2273
- CdasWs.get_data
2274
- """
2275
- # pylint: disable=too-many-locals
2276
- # pylint: disable=too-many-return-statements
2277
- # pylint: enable=too-many-statements
2278
- # pylint: disable=too-many-branches
2279
-
2280
- start_datetime, end_datetime = TimeInterval.get_datetimes(start,
2281
- end)
2282
-
2283
- request = CdfRequest(dataset, [],
2284
- TimeInterval(start_datetime, end_datetime))
2285
-
2286
- progress_callback = keywords.get('progressCallback', None)
2287
- progress_user_value = keywords.get('progressUserValue', None)
2288
-
2289
- self.logger.debug('request = %s', request)
2290
-
2291
- if progress_callback is not None:
2292
- if progress_callback(0.1, 'Making server request.',
2293
- progress_user_value) != 0:
2294
- return (204, None)
2295
-
2296
- status_code, result = self.get_data_result(request,
2297
- progress_callback,
2298
- progress_user_value)
2299
-
2300
- if progress_callback is not None:
2301
- if progress_callback(1.0, 'Server request complete.',
2302
- progress_user_value) != 0:
2303
- return (status_code, None)
2304
-
2305
- if status_code != 200:
2306
-
2307
- self.logger.info('get_result failed with http code %d',
2308
- status_code)
2309
- self.logger.info('request = %s', request)
2310
- return (status_code, None)
2311
-
2312
- return (status_code, result['FileDescription'])
2313
-
2314
-
2315
- def get_ssc_id(
2316
- self,
2317
- dataset: str
2318
- ) -> Tuple[int, Union[str, List[str]]]:
2319
- """
2320
- Gets the Satellite Situation Center (SSC)
2321
- <https://sscweb.gsfc.nasa.gov/> observatory identifier(s)
2322
- associated with the given cdaweb dataset identifier.
2323
-
2324
- Notes
2325
- -----
2326
- This method relies upon the Heliophysics Data Portal's
2327
- <https://heliophysicsdata.gsfc.nasa.gov/> metadata. That metadata
2328
- may be incomplete. Also, cdaweb has datasets for which SSC has
2329
- no corresponding observatory (for example, ground observatory
2330
- data). Callers should be prepared for negative results (200, None)
2331
- from this method.
2332
-
2333
- Parameters
2334
- ----------
2335
- dataset
2336
- cdaweb dataset identifier.
2337
- Returns
2338
- -------
2339
- Tuple
2340
- [0] contains the HTTP status code value (200 when successful).<br>
2341
- [1] the SSC observatory identifier(s) associated with the given
2342
- cdaweb dataset identifier or None if none is found.
2343
- """
2344
- url = self._hdp_registry + '?cdawebId=' + dataset
2345
-
2346
- self.logger.debug('request url = %s', url)
2347
-
2348
- response = self._session.get(url, timeout=self._timeout)
2349
-
2350
- if response.status_code != 200:
2351
-
2352
- self.logger.info('%s failed with http code %d', url,
2353
- response.status_code)
2354
- self.logger.info('response.text: %s', response.text)
2355
- return (response.status_code, None)
2356
-
2357
- if self.logger.level <= logging.DEBUG:
2358
- self.logger.debug('response.text = %s', response.text)
2359
-
2360
- results = ET.fromstring(response.text)
2361
-
2362
- ssc_id = []
2363
- for ssc_id_elem in results.findall('SscId'):
2364
- ssc_id.append(ssc_id_elem.text)
2365
-
2366
- if len(ssc_id) == 0:
2367
- result = None
2368
- elif len(ssc_id) == 1:
2369
- result = ssc_id[0]
2370
- else:
2371
- result = ssc_id
2372
-
2373
- return (response.status_code, result)
82
+ #
83
+ # For backward compatibility with cdasws versions < 1.8.12.
84
+ #
85
+ from cdasws.cdasws import *