hapiclient 0.2.6__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hapiclient-0.2.6 → hapiclient-0.2.7}/CHANGES.txt +9 -2
- {hapiclient-0.2.6 → hapiclient-0.2.7}/PKG-INFO +56 -6
- hapiclient-0.2.6/hapiclient.egg-info/PKG-INFO → hapiclient-0.2.7/README.md +40 -15
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient/__init__.py +7 -1
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient/hapi.py +83 -17
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient/hapitime.py +14 -14
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient/util.py +41 -20
- hapiclient-0.2.6/README.md → hapiclient-0.2.7/hapiclient.egg-info/PKG-INFO +65 -4
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient.egg-info/SOURCES.txt +7 -7
- {hapiclient-0.2.6 → hapiclient-0.2.7}/setup.py +6 -3
- {hapiclient-0.2.6/hapiclient → hapiclient-0.2.7}/test/test_chunking.py +85 -57
- hapiclient-0.2.7/test/test_datetime2hapitime.py +32 -0
- hapiclient-0.2.7/test/test_hapi_data_requests.py +224 -0
- hapiclient-0.2.7/test/test_hapi_metadata_requests.py +145 -0
- hapiclient-0.2.7/test/test_hapitime2datetime.py +102 -0
- hapiclient-0.2.7/test/test_hapitime_reformat.py +83 -0
- hapiclient-0.2.7/test/test_logging.py +137 -0
- hapiclient-0.2.6/hapiclient/test/__init__.py +0 -0
- hapiclient-0.2.6/hapiclient/test/compare.py +0 -240
- hapiclient-0.2.6/hapiclient/test/test_datetime2hapitime.py +0 -17
- hapiclient-0.2.6/hapiclient/test/test_hapi.py +0 -340
- hapiclient-0.2.6/hapiclient/test/test_hapitime2datetime.py +0 -124
- hapiclient-0.2.6/hapiclient/test/test_hapitime_reformat.py +0 -79
- {hapiclient-0.2.6 → hapiclient-0.2.7}/LICENSE.txt +0 -0
- {hapiclient-0.2.6 → hapiclient-0.2.7}/MANIFEST.in +0 -0
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient.egg-info/dependency_links.txt +0 -0
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient.egg-info/requires.txt +1 -1
- {hapiclient-0.2.6 → hapiclient-0.2.7}/hapiclient.egg-info/top_level.txt +0 -0
- {hapiclient-0.2.6 → hapiclient-0.2.7}/setup.cfg +0 -0
|
@@ -10,7 +10,7 @@ v0.0.6:
|
|
|
10
10
|
2018-10-04 -- Improve dateticks.py
|
|
11
11
|
2018-10-04 -- Add hapi_demo.ipynb
|
|
12
12
|
2018-10-04 -- Other documentation improvements
|
|
13
|
-
v0.0.7:
|
|
13
|
+
v0.0.7:
|
|
14
14
|
2018-10-06 -- Improve documentation
|
|
15
15
|
2018-10-06 -- Add reader tests
|
|
16
16
|
2018-10-23 -- Improve dateticks and tests
|
|
@@ -103,4 +103,11 @@ v0.2.6b1:
|
|
|
103
103
|
2023-02-04 -- 2ef7cff Unicode fix; Log string fix
|
|
104
104
|
2024-05-11 -- fba4ad3 Testing Updates
|
|
105
105
|
v0.2.6:
|
|
106
|
-
|
|
106
|
+
2023-05-24 -- Use token for PyPi in Makefile
|
|
107
|
+
v0.2.7b1:
|
|
108
|
+
2025-03-01 -- Unrecognized ISO 8601 time format: '00:00:00.Z' https://github.com/hapi-server/client-python/issues/76
|
|
109
|
+
2025-03-01 -- 'infer_datetime_format' is deprecated ... https://github.com/hapi-server/client-python/issues/78
|
|
110
|
+
2026-04-08 -- Improve NaN handling https://github.com/hapi-server/client-python/issues/88
|
|
111
|
+
2026-05-02 -- Logging and testing https://github.com/hapi-server/client-python/issues/89
|
|
112
|
+
v0.2.7:
|
|
113
|
+
2026--05-02-- Support for Python 2.7 dropped. This is last release with support for 3.5 through 3.8.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: hapiclient
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.7
|
|
4
4
|
Summary: Interface to Heliophysics data server API
|
|
5
5
|
Home-page: http://pypi.python.org/pypi/hapiclient/
|
|
6
6
|
Author: Bob Weigel
|
|
@@ -8,6 +8,20 @@ Author-email: rweigel@gmu.edu
|
|
|
8
8
|
License: LICENSE.txt
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
License-File: LICENSE.txt
|
|
11
|
+
Requires-Dist: urllib3
|
|
12
|
+
Requires-Dist: joblib
|
|
13
|
+
Requires-Dist: isodate>=0.6.0
|
|
14
|
+
Requires-Dist: pandas>=0.23
|
|
15
|
+
Requires-Dist: numpy>=1.14.3
|
|
16
|
+
Dynamic: author
|
|
17
|
+
Dynamic: author-email
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: home-page
|
|
21
|
+
Dynamic: license
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
Dynamic: requires-dist
|
|
24
|
+
Dynamic: summary
|
|
11
25
|
|
|
12
26
|
[](https://zenodo.org/badge/latestdoi/93170857)
|
|
13
27
|
[](https://app.travis-ci.com/hapi-server/client-python)
|
|
@@ -50,6 +64,7 @@ start = '2003-09-01T00:00:00'
|
|
|
50
64
|
stop = '2003-12-01T00:00:00'
|
|
51
65
|
parameters = 'DST1800'
|
|
52
66
|
opts = {'logging': True}
|
|
67
|
+
# See misc/hapi_logging_demo.py for example of using conventional Python logging.
|
|
53
68
|
|
|
54
69
|
# Get data
|
|
55
70
|
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
|
|
@@ -92,7 +107,41 @@ returns the [Numpy N-D array](https://docs.scipy.org/doc/numpy-1.15.1/user/quick
|
|
|
92
107
|
|
|
93
108
|
The HAPI client data model is intentionally basic. There is an ongoing discussion of a data model for Heliophysics data among the [PyHC community](https://heliopython.org/). When this data model is complete, a function that converts `data` and `meta` to that data model will be included in the `hapiclient` package.
|
|
94
109
|
|
|
95
|
-
# Development
|
|
110
|
+
# Development and Testing
|
|
111
|
+
|
|
112
|
+
## Testing Python versions available with tox-env
|
|
113
|
+
|
|
114
|
+
As of 2026-04-14, Python 3.8+ is available with tox-env.
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
git clone https://github.com/hapi-server/client-python
|
|
118
|
+
cd client-python; python -m pip install -e .
|
|
119
|
+
|
|
120
|
+
# Run a specific test on current Python version (default is log_level=INFO
|
|
121
|
+
# for hapiclient logger and test logger when executed this way).
|
|
122
|
+
python test/test_hapi_data_requests.py
|
|
123
|
+
|
|
124
|
+
# Run a specific test on a specific Python version with all logging
|
|
125
|
+
tox -e py311 -- test/test_hapi_data_requests.py
|
|
126
|
+
# Set log level for test logger
|
|
127
|
+
tox -e py311 -- test/test_hapi_data_requests.py --log-level=INFO
|
|
128
|
+
|
|
129
|
+
# Run a specific test on all Python versions set by tox.ini `envlist`
|
|
130
|
+
tox -- test/test_hapi_data_requests.py
|
|
131
|
+
|
|
132
|
+
# Run all tests in ./test
|
|
133
|
+
tox
|
|
134
|
+
|
|
135
|
+
# Run long-running tests
|
|
136
|
+
tox -e long-test
|
|
137
|
+
|
|
138
|
+
# Run long-running tests on a specific Python version
|
|
139
|
+
tox -e long-test --override testenv:long-test.basepython=python3.11
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Testing Python versions available with Anaconda
|
|
143
|
+
|
|
144
|
+
As of 2026-04-14, Python 3.5+ is available with Anaconda.
|
|
96
145
|
|
|
97
146
|
```bash
|
|
98
147
|
git clone https://github.com/hapi-server/client-python
|
|
@@ -117,12 +166,13 @@ make repository-test
|
|
|
117
166
|
|
|
118
167
|
To run an individual unit test in a Python session, use, e.g.,
|
|
119
168
|
|
|
120
|
-
```
|
|
121
|
-
|
|
122
|
-
test_reader_short()
|
|
169
|
+
```bash
|
|
170
|
+
cd test; python test_hapitime_reformat.py
|
|
123
171
|
```
|
|
124
172
|
|
|
125
173
|
# Contact
|
|
126
174
|
|
|
127
175
|
Submit bug reports and feature requests on the [repository issue
|
|
128
176
|
tracker](https://github.com/hapi-server/client-python/issues>).
|
|
177
|
+
|
|
178
|
+
Before submitting a pull request, please post an issue with the proposed changes for discussion prior to developing the pull request.
|
|
@@ -1,14 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: hapiclient
|
|
3
|
-
Version: 0.2.6
|
|
4
|
-
Summary: Interface to Heliophysics data server API
|
|
5
|
-
Home-page: http://pypi.python.org/pypi/hapiclient/
|
|
6
|
-
Author: Bob Weigel
|
|
7
|
-
Author-email: rweigel@gmu.edu
|
|
8
|
-
License: LICENSE.txt
|
|
9
|
-
Description-Content-Type: text/markdown
|
|
10
|
-
License-File: LICENSE.txt
|
|
11
|
-
|
|
12
1
|
[](https://zenodo.org/badge/latestdoi/93170857)
|
|
13
2
|
[](https://app.travis-ci.com/hapi-server/client-python)
|
|
14
3
|
|
|
@@ -50,6 +39,7 @@ start = '2003-09-01T00:00:00'
|
|
|
50
39
|
stop = '2003-12-01T00:00:00'
|
|
51
40
|
parameters = 'DST1800'
|
|
52
41
|
opts = {'logging': True}
|
|
42
|
+
# See misc/hapi_logging_demo.py for example of using conventional Python logging.
|
|
53
43
|
|
|
54
44
|
# Get data
|
|
55
45
|
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
|
|
@@ -92,7 +82,41 @@ returns the [Numpy N-D array](https://docs.scipy.org/doc/numpy-1.15.1/user/quick
|
|
|
92
82
|
|
|
93
83
|
The HAPI client data model is intentionally basic. There is an ongoing discussion of a data model for Heliophysics data among the [PyHC community](https://heliopython.org/). When this data model is complete, a function that converts `data` and `meta` to that data model will be included in the `hapiclient` package.
|
|
94
84
|
|
|
95
|
-
# Development
|
|
85
|
+
# Development and Testing
|
|
86
|
+
|
|
87
|
+
## Testing Python versions available with tox-env
|
|
88
|
+
|
|
89
|
+
As of 2026-04-14, Python 3.8+ is available with tox-env.
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
git clone https://github.com/hapi-server/client-python
|
|
93
|
+
cd client-python; python -m pip install -e .
|
|
94
|
+
|
|
95
|
+
# Run a specific test on current Python version (default is log_level=INFO
|
|
96
|
+
# for hapiclient logger and test logger when executed this way).
|
|
97
|
+
python test/test_hapi_data_requests.py
|
|
98
|
+
|
|
99
|
+
# Run a specific test on a specific Python version with all logging
|
|
100
|
+
tox -e py311 -- test/test_hapi_data_requests.py
|
|
101
|
+
# Set log level for test logger
|
|
102
|
+
tox -e py311 -- test/test_hapi_data_requests.py --log-level=INFO
|
|
103
|
+
|
|
104
|
+
# Run a specific test on all Python versions set by tox.ini `envlist`
|
|
105
|
+
tox -- test/test_hapi_data_requests.py
|
|
106
|
+
|
|
107
|
+
# Run all tests in ./test
|
|
108
|
+
tox
|
|
109
|
+
|
|
110
|
+
# Run long-running tests
|
|
111
|
+
tox -e long-test
|
|
112
|
+
|
|
113
|
+
# Run long-running tests on a specific Python version
|
|
114
|
+
tox -e long-test --override testenv:long-test.basepython=python3.11
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Testing Python versions available with Anaconda
|
|
118
|
+
|
|
119
|
+
As of 2026-04-14, Python 3.5+ is available with Anaconda.
|
|
96
120
|
|
|
97
121
|
```bash
|
|
98
122
|
git clone https://github.com/hapi-server/client-python
|
|
@@ -117,12 +141,13 @@ make repository-test
|
|
|
117
141
|
|
|
118
142
|
To run an individual unit test in a Python session, use, e.g.,
|
|
119
143
|
|
|
120
|
-
```
|
|
121
|
-
|
|
122
|
-
test_reader_short()
|
|
144
|
+
```bash
|
|
145
|
+
cd test; python test_hapitime_reformat.py
|
|
123
146
|
```
|
|
124
147
|
|
|
125
148
|
# Contact
|
|
126
149
|
|
|
127
150
|
Submit bug reports and feature requests on the [repository issue
|
|
128
151
|
tracker](https://github.com/hapi-server/client-python/issues>).
|
|
152
|
+
|
|
153
|
+
Before submitting a pull request, please post an issue with the proposed changes for discussion prior to developing the pull request.
|
|
@@ -10,9 +10,15 @@ from hapiclient.hapitime import datetime2hapitime
|
|
|
10
10
|
# Allow "from hapiclient import HAPIError"
|
|
11
11
|
from hapiclient.util import HAPIError
|
|
12
12
|
|
|
13
|
-
__version__ = '0.2.
|
|
13
|
+
__version__ = '0.2.7'
|
|
14
14
|
|
|
15
15
|
import sys
|
|
16
|
+
import platform
|
|
17
|
+
|
|
18
|
+
if sys.version_info < (3, 10) and platform.system() == "Darwin":
|
|
19
|
+
import warnings
|
|
20
|
+
warnings.filterwarnings("ignore", message=".*urllib3.*OpenSSL.*")
|
|
21
|
+
|
|
16
22
|
if sys.version_info[0] < 3:
|
|
17
23
|
# Python 2.7
|
|
18
24
|
reload(sys)
|
|
@@ -4,6 +4,7 @@ import sys
|
|
|
4
4
|
import json
|
|
5
5
|
import time
|
|
6
6
|
import pickle
|
|
7
|
+
import logging as _logging
|
|
7
8
|
import warnings
|
|
8
9
|
from datetime import datetime, timedelta
|
|
9
10
|
|
|
@@ -13,7 +14,7 @@ import numpy as np
|
|
|
13
14
|
from joblib import Parallel, delayed
|
|
14
15
|
|
|
15
16
|
from hapiclient.hapitime import hapitime2datetime, hapitime_reformat
|
|
16
|
-
from hapiclient.util import setopts, log, warning, error
|
|
17
|
+
from hapiclient.util import setopts, log, warning, error, configure_logging
|
|
17
18
|
from hapiclient.util import urlopen, urlretrieve, jsonparse, unicode_error_message
|
|
18
19
|
|
|
19
20
|
|
|
@@ -176,7 +177,7 @@ def hapiopts():
|
|
|
176
177
|
def hapi(*args, **kwargs):
|
|
177
178
|
"""Request data from a HAPI server.
|
|
178
179
|
|
|
179
|
-
Version: 0.2.
|
|
180
|
+
Version: 0.2.7
|
|
180
181
|
|
|
181
182
|
|
|
182
183
|
Examples
|
|
@@ -250,7 +251,7 @@ def hapi(*args, **kwargs):
|
|
|
250
251
|
Cadence = PT1M and request for
|
|
251
252
|
|
|
252
253
|
start/stop=1999-11-12T00:10:00/stop=1999-11-12T12:09:00
|
|
253
|
-
|
|
254
|
+
|
|
254
255
|
Chunk size is P1D and requested time range < 1/2 of this
|
|
255
256
|
=> Default behavior
|
|
256
257
|
|
|
@@ -344,7 +345,11 @@ def hapi(*args, **kwargs):
|
|
|
344
345
|
# Override defaults
|
|
345
346
|
opts = setopts(hapiopts(), kwargs)
|
|
346
347
|
|
|
347
|
-
assert (opts['logging']
|
|
348
|
+
assert (isinstance(opts['logging'], bool) or hasattr(opts['logging'], 'write')), \
|
|
349
|
+
"logging keyword must be True, False, or a file-like object"
|
|
350
|
+
|
|
351
|
+
configure_logging(opts)
|
|
352
|
+
|
|
348
353
|
assert (opts['cache'] in [True, False]), "cache keyword must be True of False"
|
|
349
354
|
assert (opts['usecache'] in [True, False]), "usecache keyword must be True of False"
|
|
350
355
|
assert (opts['format'] in ['binary', 'csv']), "format keyword must be 'csv' or 'binary'"
|
|
@@ -408,7 +413,7 @@ def hapi(*args, **kwargs):
|
|
|
408
413
|
# urld = url subdirectory of cachedir to store files from SERVER
|
|
409
414
|
urld = cachedir(opts["cachedir"], SERVER)
|
|
410
415
|
|
|
411
|
-
if opts["cachedir"]: log('
|
|
416
|
+
if opts["cachedir"]: log('cache subdirectory = %s' % urld, opts)
|
|
412
417
|
|
|
413
418
|
urljson = SERVER + '/info?id=' + DATASET
|
|
414
419
|
|
|
@@ -558,10 +563,13 @@ def hapi(*args, **kwargs):
|
|
|
558
563
|
|
|
559
564
|
backend = 'sequential'
|
|
560
565
|
if opts['parallel']:
|
|
566
|
+
# Note that this does not often lead to significant speed-up.
|
|
567
|
+
# Needs further testing and optimization.
|
|
561
568
|
backend = 'threading'
|
|
562
|
-
# multiprocessing
|
|
563
|
-
#
|
|
564
|
-
#
|
|
569
|
+
# multiprocessing not working.
|
|
570
|
+
#backend = 'multiprocessing'
|
|
571
|
+
# loky works, but not speed-up.
|
|
572
|
+
#backend = 'loky'
|
|
565
573
|
|
|
566
574
|
log('backend = {}'.format(backend), opts)
|
|
567
575
|
|
|
@@ -663,14 +671,18 @@ def hapi(*args, **kwargs):
|
|
|
663
671
|
if opts["usecache"] and os.path.isfile(fnamenpy):
|
|
664
672
|
# Read cached data file.
|
|
665
673
|
log('Reading %s ' % fnamenpy.replace(urld + '/', ''), opts)
|
|
674
|
+
tic = time.time()
|
|
666
675
|
f = open(fnamenpy, 'rb')
|
|
667
676
|
data = np.load(f)
|
|
668
677
|
f.close()
|
|
678
|
+
toc = time.time() - tic
|
|
669
679
|
# There is a possibility that the fnamenpy file existed but
|
|
670
680
|
# fnamepklx was not found (b/c removed). In this case, the meta
|
|
671
681
|
# returned will not have all of the "x_" information inserted below.
|
|
672
682
|
# Code that uses this information needs to account for this.
|
|
673
683
|
meta['x_totalTime'] = time.time() - tic_totalTime
|
|
684
|
+
meta['x_readTime'] = tic - tic_totalTime
|
|
685
|
+
meta['x_downloadTime'] = 0
|
|
674
686
|
return data, meta
|
|
675
687
|
|
|
676
688
|
cformats = ['csv', 'binary'] # client formats
|
|
@@ -812,13 +824,32 @@ def hapi(*args, **kwargs):
|
|
|
812
824
|
error('Malformed response? Could not read response: {}'.format(urlcsv))
|
|
813
825
|
if opts['method'] == '' or opts['method'] == 'pandas':
|
|
814
826
|
# Read file into Pandas DataFrame
|
|
827
|
+
csv_kwargs = {
|
|
828
|
+
'sep': ',',
|
|
829
|
+
'header': None,
|
|
830
|
+
'encoding': 'utf-8',
|
|
831
|
+
'skipinitialspace': True,
|
|
832
|
+
'keep_default_na': False,
|
|
833
|
+
'na_values': ['NaN', 'nan', 'Nan', 'naN', ' "NaN"', ' "nan"', ' "Nan"', ' "naN"', '"NaN"', '"nan"', '"Nan"', '"naN"']
|
|
834
|
+
}
|
|
835
|
+
"""
|
|
836
|
+
Note that this does not handle trailing whitespace after
|
|
837
|
+
any of the na_values. (There is no skiptrailingspace option).
|
|
838
|
+
Stripping trailing whitespace would require adding something
|
|
839
|
+
like
|
|
840
|
+
def strip_field(x):
|
|
841
|
+
# Strip whitespace and normalize NaN values.
|
|
842
|
+
x = x.strip()
|
|
843
|
+
return np.nan if x.lower() == "nan" else x
|
|
844
|
+
|
|
845
|
+
ncols = cols[-1][1] + 1
|
|
846
|
+
csv_kwargs["converters"] = {i: strip_field for i in range(ncols)}
|
|
847
|
+
"""
|
|
815
848
|
try:
|
|
816
|
-
df = pandas.read_csv(fnamecsv,
|
|
817
|
-
sep=',',
|
|
818
|
-
header=None,
|
|
819
|
-
encoding='utf-8')
|
|
849
|
+
df = pandas.read_csv(fnamecsv, **csv_kwargs)
|
|
820
850
|
except:
|
|
821
851
|
error('Malformed response? Could not read response: {}'.format(urlcsv))
|
|
852
|
+
|
|
822
853
|
# Allocate output N-D array (It is not possible to pass dtype=dt
|
|
823
854
|
# as computed to pandas.read_csv; pandas dtype is different
|
|
824
855
|
# from numpy's dtype.)
|
|
@@ -865,7 +896,14 @@ def hapi(*args, **kwargs):
|
|
|
865
896
|
pickle.dump(meta, f, protocol=2)
|
|
866
897
|
|
|
867
898
|
log('Writing %s' % fnamenpy, opts)
|
|
868
|
-
|
|
899
|
+
with warnings.catch_warnings():
|
|
900
|
+
# Ignore warning that occurs when saving Unicode data.
|
|
901
|
+
warnings.filterwarnings("ignore",
|
|
902
|
+
message=r"Stored array in format 3\.0.*",
|
|
903
|
+
category=UserWarning,
|
|
904
|
+
module=r"numpy\.lib\.format",
|
|
905
|
+
)
|
|
906
|
+
np.save(fnamenpy, data)
|
|
869
907
|
|
|
870
908
|
meta['x_totalTime'] = time.time() - tic_totalTime
|
|
871
909
|
|
|
@@ -963,9 +1001,28 @@ def parse_missing_length(fnamecsv, dt, cols, psizes, pnames, ptypes, opts):
|
|
|
963
1001
|
if opts['method'] == 'numpy' or opts['method'] == 'numpynolength':
|
|
964
1002
|
# If requested method was numpy, use numpynolength method.
|
|
965
1003
|
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
1004
|
+
ncols = cols[-1][1] + 1
|
|
1005
|
+
|
|
1006
|
+
def normalize_field(value):
|
|
1007
|
+
if isinstance(value, bytes):
|
|
1008
|
+
value = value.decode('utf-8')
|
|
1009
|
+
value = value.strip()
|
|
1010
|
+
if len(value) >= 2 and value[0] == '"' and value[-1] == '"':
|
|
1011
|
+
value = value[1:-1].strip()
|
|
1012
|
+
if value.lower() == 'nan':
|
|
1013
|
+
return 'nan'
|
|
1014
|
+
return value
|
|
1015
|
+
|
|
1016
|
+
converters = {i: normalize_field for i in range(ncols)}
|
|
1017
|
+
|
|
1018
|
+
table = np.genfromtxt(fnamecsv,
|
|
1019
|
+
dtype=None,
|
|
1020
|
+
deletechars='',
|
|
1021
|
+
replace_space=' ',
|
|
1022
|
+
delimiter=',',
|
|
1023
|
+
encoding='utf-8',
|
|
1024
|
+
converters=converters)
|
|
1025
|
+
|
|
969
1026
|
# table is a 1-D array. Each element is a row in the file.
|
|
970
1027
|
# - If the data types are not the same for each column,
|
|
971
1028
|
# the elements are tuples with length equal to the number
|
|
@@ -1010,8 +1067,17 @@ def parse_missing_length(fnamecsv, dt, cols, psizes, pnames, ptypes, opts):
|
|
|
1010
1067
|
if opts['method'] == '' or opts['method'] == 'pandas' or opts['method'] == 'pandasnolength':
|
|
1011
1068
|
# If requested method was pandas, use pandasnolength method.
|
|
1012
1069
|
|
|
1070
|
+
# TODO: Duplicate code.
|
|
1013
1071
|
# Read file into Pandas DataFrame
|
|
1014
|
-
|
|
1072
|
+
csv_kwargs = {
|
|
1073
|
+
'sep': ',',
|
|
1074
|
+
'header': None,
|
|
1075
|
+
'encoding': 'utf-8',
|
|
1076
|
+
'skipinitialspace': True,
|
|
1077
|
+
'keep_default_na': False,
|
|
1078
|
+
'na_values': ['NaN', 'nan', 'Nan', 'naN', ' "NaN"', ' "nan"', ' "Nan"', ' "naN"', '"NaN"', '"nan"', '"Nan"', '"naN"']
|
|
1079
|
+
}
|
|
1080
|
+
df = pandas.read_csv(fnamecsv, **csv_kwargs)
|
|
1015
1081
|
|
|
1016
1082
|
# Allocate output N-D array (It is not possible to pass dtype=dt
|
|
1017
1083
|
# as computed to pandas.read_csv, so need to create new ND array.)
|
|
@@ -22,9 +22,9 @@ def hapitime_reformat(form_to_match, given_form, logging=False):
|
|
|
22
22
|
::
|
|
23
23
|
|
|
24
24
|
start = hapitime_reformat(data['Time'][0], start)
|
|
25
|
-
|
|
25
|
+
|
|
26
26
|
Then subset using
|
|
27
|
-
|
|
27
|
+
|
|
28
28
|
::
|
|
29
29
|
|
|
30
30
|
data = data[data['Time'] >= start]
|
|
@@ -69,7 +69,7 @@ def hapitime_reformat(form_to_match, given_form, logging=False):
|
|
|
69
69
|
|
|
70
70
|
converted = hapitime_reformat(form_to_match+'Z', given_form+'Z')
|
|
71
71
|
converted = converted.strip('Z')
|
|
72
|
-
|
|
72
|
+
|
|
73
73
|
converted_fractional = '{:0<{}.{}}'.format(given_form_fractional,
|
|
74
74
|
len(form_to_match_fractional),
|
|
75
75
|
len(form_to_match_fractional))
|
|
@@ -77,11 +77,11 @@ def hapitime_reformat(form_to_match, given_form, logging=False):
|
|
|
77
77
|
|
|
78
78
|
if 'Z' in format_ref:
|
|
79
79
|
return converted + 'Z'
|
|
80
|
-
|
|
80
|
+
|
|
81
81
|
return converted
|
|
82
82
|
|
|
83
83
|
converted = dt_given.strftime(format_ref)
|
|
84
|
-
|
|
84
|
+
|
|
85
85
|
if len(converted) > len(form_to_match):
|
|
86
86
|
converted = converted[0:len(form_to_match)-1] + "Z"
|
|
87
87
|
|
|
@@ -220,14 +220,9 @@ def hapitime2datetime(Time, **kwargs):
|
|
|
220
220
|
hapitime2datetime([b'1970-01-01T00:00:00.000Z'])
|
|
221
221
|
hapitime2datetime('1970-01-01T00:00:00.000Z')
|
|
222
222
|
"""
|
|
223
|
-
from datetime import datetime
|
|
223
|
+
from datetime import datetime, timezone
|
|
224
224
|
|
|
225
|
-
|
|
226
|
-
# Python 2
|
|
227
|
-
import pytz
|
|
228
|
-
tzinfo = pytz.UTC
|
|
229
|
-
except:
|
|
230
|
-
tzinfo = datetime.timezone.utc
|
|
225
|
+
tzinfo = timezone.utc
|
|
231
226
|
|
|
232
227
|
if type(Time) == list:
|
|
233
228
|
Time = np.asarray(Time)
|
|
@@ -272,7 +267,8 @@ def hapitime2datetime(Time, **kwargs):
|
|
|
272
267
|
|
|
273
268
|
try:
|
|
274
269
|
# This is the fastest conversion option. But it will fail on YYYY-DOY
|
|
275
|
-
# format and other valid ISO 8601 dates such as 2001-01-01T00:00:03.Z
|
|
270
|
+
# format and other valid^* ISO 8601 dates such as 2001-01-01T00:00:03.Z
|
|
271
|
+
# ^*Maybe not: https://github.com/hapi-server/client-python/issues/76
|
|
276
272
|
# When infer_datetime_format is used, a TimeStamp object returned,
|
|
277
273
|
# which is the reason for the to_pydatetime() call. (When format=... is
|
|
278
274
|
# used, a datetime object is returned.)
|
|
@@ -281,7 +277,11 @@ def hapitime2datetime(Time, **kwargs):
|
|
|
281
277
|
# is the reason for the call to tz_convert(tzinfo).
|
|
282
278
|
# TODO: Use hapitime_format_str() and pass this as format=...
|
|
283
279
|
Timeo = Time[0]
|
|
284
|
-
|
|
280
|
+
pandas_major_version = int(pandas.__version__.split('.')[0])
|
|
281
|
+
if pandas_major_version < 2:
|
|
282
|
+
Time = pandas.to_datetime(Time, infer_datetime_format=True).tz_convert(tzinfo).to_pydatetime()
|
|
283
|
+
else:
|
|
284
|
+
Time = pandas.to_datetime(Time).tz_convert(tzinfo).to_pydatetime()
|
|
285
285
|
if reshape:
|
|
286
286
|
Time = np.reshape(Time, shape)
|
|
287
287
|
toc = time.time() - tic
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import logging as _logging
|
|
2
|
+
_logger = _logging.getLogger("hapiclient")
|
|
3
|
+
|
|
4
|
+
|
|
1
5
|
def setopts(defaults, given):
|
|
2
6
|
"""Override default keyword dictionary options.
|
|
3
7
|
|
|
@@ -19,34 +23,50 @@ def setopts(defaults, given):
|
|
|
19
23
|
return defaults
|
|
20
24
|
|
|
21
25
|
|
|
22
|
-
|
|
26
|
+
_INTERNAL_HANDLER_ATTR = "_hapiclient_internal_handler"
|
|
27
|
+
_INTERNAL_LEVEL_ATTR = "_hapiclient_internal_level"
|
|
28
|
+
def configure_logging(opts):
|
|
29
|
+
"""Configure the hapiclient logger based on opts['logging'].
|
|
23
30
|
|
|
24
|
-
|
|
25
|
-
|
|
31
|
+
If the hapiclient logger has been configured externally (level != NOTSET
|
|
32
|
+
or handlers present), the logging kwarg.
|
|
33
|
+
"""
|
|
34
|
+
has_user_level = _logger.level != _logging.NOTSET and \
|
|
35
|
+
_logger.level != getattr(_logger, _INTERNAL_LEVEL_ATTR, None)
|
|
36
|
+
has_user_handlers = any(
|
|
37
|
+
not getattr(handler, _INTERNAL_HANDLER_ATTR, False)
|
|
38
|
+
for handler in _logger.handlers
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
if opts['logging']:
|
|
42
|
+
_logger.setLevel(_logging.INFO)
|
|
43
|
+
setattr(_logger, _INTERNAL_LEVEL_ATTR, _logging.INFO)
|
|
44
|
+
_logger.propagate = False
|
|
45
|
+
if not _logger.handlers:
|
|
46
|
+
import sys
|
|
47
|
+
_handler = _logging.StreamHandler(sys.stdout)
|
|
48
|
+
_handler.setFormatter(_logging.Formatter("%(message)s"))
|
|
49
|
+
setattr(_handler, _INTERNAL_HANDLER_ATTR, True)
|
|
50
|
+
_logger.addHandler(_handler)
|
|
51
|
+
else:
|
|
52
|
+
if has_user_level or has_user_handlers:
|
|
53
|
+
if has_user_handlers:
|
|
54
|
+
log("Ignoring logging=%s because standard Python logger for 'hapiclient' already configured with handlers." % opts['logging'], opts)
|
|
55
|
+
else:
|
|
56
|
+
log("Ignoring logging=%s because standard Python logger for 'hapiclient' already configured with log_level != NOTSET." % opts['logging'], opts)
|
|
57
|
+
else:
|
|
58
|
+
_logger.setLevel(_logging.WARNING)
|
|
59
|
+
setattr(_logger, _INTERNAL_LEVEL_ATTR, _logging.WARNING)
|
|
26
60
|
|
|
27
61
|
|
|
28
62
|
def log(msg, opts):
|
|
29
|
-
"""
|
|
63
|
+
"""Log message using the 'hapiclient' logger."""
|
|
30
64
|
|
|
31
|
-
import os
|
|
32
65
|
import sys
|
|
33
66
|
|
|
34
|
-
if not 'logging' in opts:
|
|
35
|
-
opts = opts.copy()
|
|
36
|
-
opts['logging'] = False
|
|
37
|
-
|
|
38
67
|
pre = sys._getframe(1).f_code.co_name + '(): '
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
# Don't show full path information.
|
|
42
|
-
msg = msg.replace(opts['cachedir'] + os.path.sep, '')
|
|
43
|
-
msg = msg.replace(opts['cachedir'], '')
|
|
44
|
-
print(pre + msg)
|
|
45
|
-
elif hasattr(opts['logging'], 'write'):
|
|
46
|
-
opts['logging'].write(pre + msg + "\n")
|
|
47
|
-
opts['logging'].flush()
|
|
48
|
-
else:
|
|
49
|
-
pass # TODO: error
|
|
68
|
+
|
|
69
|
+
_logger.info(pre + msg)
|
|
50
70
|
|
|
51
71
|
|
|
52
72
|
def jsonparse(res, url):
|
|
@@ -107,6 +127,7 @@ def unicode_error_message(name):
|
|
|
107
127
|
import sys
|
|
108
128
|
msg = ""
|
|
109
129
|
if sys.version_info[0:2] <= (3, 5):
|
|
130
|
+
name = str(name)
|
|
110
131
|
if not all(ord(char) < 128 for char in name):
|
|
111
132
|
msg = "hapiclient cannot handle Unicode dataset or parameter names (" + str(name.encode('utf-8')) + ") for Python < 3.6 on Windows."
|
|
112
133
|
return msg
|
|
@@ -1,3 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hapiclient
|
|
3
|
+
Version: 0.2.7
|
|
4
|
+
Summary: Interface to Heliophysics data server API
|
|
5
|
+
Home-page: http://pypi.python.org/pypi/hapiclient/
|
|
6
|
+
Author: Bob Weigel
|
|
7
|
+
Author-email: rweigel@gmu.edu
|
|
8
|
+
License: LICENSE.txt
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE.txt
|
|
11
|
+
Requires-Dist: urllib3
|
|
12
|
+
Requires-Dist: joblib
|
|
13
|
+
Requires-Dist: isodate>=0.6.0
|
|
14
|
+
Requires-Dist: pandas>=0.23
|
|
15
|
+
Requires-Dist: numpy>=1.14.3
|
|
16
|
+
Dynamic: author
|
|
17
|
+
Dynamic: author-email
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: home-page
|
|
21
|
+
Dynamic: license
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
Dynamic: requires-dist
|
|
24
|
+
Dynamic: summary
|
|
25
|
+
|
|
1
26
|
[](https://zenodo.org/badge/latestdoi/93170857)
|
|
2
27
|
[](https://app.travis-ci.com/hapi-server/client-python)
|
|
3
28
|
|
|
@@ -39,6 +64,7 @@ start = '2003-09-01T00:00:00'
|
|
|
39
64
|
stop = '2003-12-01T00:00:00'
|
|
40
65
|
parameters = 'DST1800'
|
|
41
66
|
opts = {'logging': True}
|
|
67
|
+
# See misc/hapi_logging_demo.py for example of using conventional Python logging.
|
|
42
68
|
|
|
43
69
|
# Get data
|
|
44
70
|
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
|
|
@@ -81,7 +107,41 @@ returns the [Numpy N-D array](https://docs.scipy.org/doc/numpy-1.15.1/user/quick
|
|
|
81
107
|
|
|
82
108
|
The HAPI client data model is intentionally basic. There is an ongoing discussion of a data model for Heliophysics data among the [PyHC community](https://heliopython.org/). When this data model is complete, a function that converts `data` and `meta` to that data model will be included in the `hapiclient` package.
|
|
83
109
|
|
|
84
|
-
# Development
|
|
110
|
+
# Development and Testing
|
|
111
|
+
|
|
112
|
+
## Testing Python versions available with tox-env
|
|
113
|
+
|
|
114
|
+
As of 2026-04-14, Python 3.8+ is available with tox-env.
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
git clone https://github.com/hapi-server/client-python
|
|
118
|
+
cd client-python; python -m pip install -e .
|
|
119
|
+
|
|
120
|
+
# Run a specific test on current Python version (default is log_level=INFO
|
|
121
|
+
# for hapiclient logger and test logger when executed this way).
|
|
122
|
+
python test/test_hapi_data_requests.py
|
|
123
|
+
|
|
124
|
+
# Run a specific test on a specific Python version with all logging
|
|
125
|
+
tox -e py311 -- test/test_hapi_data_requests.py
|
|
126
|
+
# Set log level for test logger
|
|
127
|
+
tox -e py311 -- test/test_hapi_data_requests.py --log-level=INFO
|
|
128
|
+
|
|
129
|
+
# Run a specific test on all Python versions set by tox.ini `envlist`
|
|
130
|
+
tox -- test/test_hapi_data_requests.py
|
|
131
|
+
|
|
132
|
+
# Run all tests in ./test
|
|
133
|
+
tox
|
|
134
|
+
|
|
135
|
+
# Run long-running tests
|
|
136
|
+
tox -e long-test
|
|
137
|
+
|
|
138
|
+
# Run long-running tests on a specific Python version
|
|
139
|
+
tox -e long-test --override testenv:long-test.basepython=python3.11
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Testing Python versions available with Anaconda
|
|
143
|
+
|
|
144
|
+
As of 2026-04-14, Python 3.5+ is available with Anaconda.
|
|
85
145
|
|
|
86
146
|
```bash
|
|
87
147
|
git clone https://github.com/hapi-server/client-python
|
|
@@ -106,12 +166,13 @@ make repository-test
|
|
|
106
166
|
|
|
107
167
|
To run an individual unit test in a Python session, use, e.g.,
|
|
108
168
|
|
|
109
|
-
```
|
|
110
|
-
|
|
111
|
-
test_reader_short()
|
|
169
|
+
```bash
|
|
170
|
+
cd test; python test_hapitime_reformat.py
|
|
112
171
|
```
|
|
113
172
|
|
|
114
173
|
# Contact
|
|
115
174
|
|
|
116
175
|
Submit bug reports and feature requests on the [repository issue
|
|
117
176
|
tracker](https://github.com/hapi-server/client-python/issues>).
|
|
177
|
+
|
|
178
|
+
Before submitting a pull request, please post an issue with the proposed changes for discussion prior to developing the pull request.
|