sparclclient 1.2.1.dev6__tar.gz → 1.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/.gitignore +0 -3
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/.pre-commit-config.yaml +2 -2
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/PKG-INFO +2 -2
- sparclclient-1.2.2/README.md +2 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/pyproject.toml +7 -5
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/requirements-internal.txt +10 -6
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/source/conf.py +1 -5
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/Results.py +11 -2
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/__init__.py +4 -11
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/client.py +163 -32
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/exceptions.py +10 -2
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/fields.py +1 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/gather_2d.py +2 -1
- sparclclient-1.2.2/sparcl/notebooks/sparcl-examples.ipynb +2054 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/utils.py +13 -0
- sparclclient-1.2.2/tests/expected_pat.py +121 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/tests/tests_api.py +461 -83
- sparclclient-1.2.1.dev6/README.md +0 -2
- sparclclient-1.2.1.dev6/tests/expected_pat.py +0 -114
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/.github/workflows/django.yml +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/.readthedocs.yaml +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/LICENSE +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/Makefile +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/make.bat +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/requirements-client.txt +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/requirements.txt +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/source/index.rst +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/source/sparcl.rst +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/benchmarks/__init__.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/benchmarks/benchmarks.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/conf.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/resample_spectra.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/sparc.ini +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/type_conversion.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/sparcl/unsupported.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/tests/expected_dev1.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/tests/methods_tests.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/tests/utils.py +0 -0
- {sparclclient-1.2.1.dev6 → sparclclient-1.2.2}/tox.ini +0 -0
|
@@ -11,11 +11,11 @@ repos:
|
|
|
11
11
|
- id: check-yaml
|
|
12
12
|
- id: check-added-large-files
|
|
13
13
|
- repo: https://github.com/psf/black
|
|
14
|
-
rev:
|
|
14
|
+
rev: 24.2.0 # 2.10.0
|
|
15
15
|
hooks:
|
|
16
16
|
- id: black
|
|
17
17
|
- repo: https://github.com/pycqa/flake8
|
|
18
|
-
rev: 4.0.1
|
|
18
|
+
rev: 7.0.0 # rev: 4.0.1
|
|
19
19
|
hooks:
|
|
20
20
|
- id: flake8
|
|
21
21
|
exclude: ^source|^tests/expected.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sparclclient
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.2
|
|
4
4
|
Summary: A client for getting spectra and meta-data from NOIRLab.
|
|
5
5
|
Author-email: "S. Pothier" <datalab-spectro@noirlab.edu>
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -10,5 +10,5 @@ Project-URL: Documentation, https://sparclclient.readthedocs.io/en/latest/
|
|
|
10
10
|
Project-URL: Homepage, https://github.com/pypa/sparclclient
|
|
11
11
|
|
|
12
12
|
# sparclclient
|
|
13
|
-
Python Client for
|
|
13
|
+
Python Client for SPARCL (SPectra Analysis and Retrievable Catalog Lab)
|
|
14
14
|
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
# See:
|
|
2
|
-
# https://flit.pypa.io/en/latest/pyproject_toml.html
|
|
3
|
-
# https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
4
|
-
# https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/
|
|
2
|
+
# FLIT: https://flit.pypa.io/en/latest/pyproject_toml.html
|
|
5
3
|
# TOML format: https://toml.io/en/
|
|
6
|
-
|
|
4
|
+
|
|
7
5
|
#
|
|
8
6
|
# Updating PyPi:
|
|
9
7
|
# source venv/bin/activate
|
|
10
8
|
# flit build
|
|
11
|
-
# flit publish
|
|
9
|
+
# flit publish --repository sparclclient
|
|
12
10
|
|
|
13
11
|
|
|
14
12
|
[build-system]
|
|
@@ -24,6 +22,7 @@ authors = [
|
|
|
24
22
|
readme = "README.md"
|
|
25
23
|
license = {file = "LICENSE"}
|
|
26
24
|
classifiers = ["License :: OSI Approved :: MIT License"]
|
|
25
|
+
# version is found in: ~/sandbox/sparclclient/sparcl/__init__.py
|
|
27
26
|
dynamic = ["version", "description"]
|
|
28
27
|
|
|
29
28
|
[project.urls]
|
|
@@ -34,6 +33,9 @@ dynamic = ["version", "description"]
|
|
|
34
33
|
[tool.flit.module]
|
|
35
34
|
name = "sparcl"
|
|
36
35
|
|
|
36
|
+
[tool.flit.sdist]
|
|
37
|
+
exclude = ["**/UNPUBLISHED/", "**/OBSOLETE/"]
|
|
38
|
+
|
|
37
39
|
[tool.black]
|
|
38
40
|
# https://black.readthedocs.io/en/stable/usage_and_configuration/index.html
|
|
39
41
|
line-length = 79
|
|
@@ -1,19 +1,23 @@
|
|
|
1
|
-
#
|
|
1
|
+
#
|
|
2
2
|
#psutil==5.8.0 # for benchmarks
|
|
3
3
|
#! speedtest # for benchmarks
|
|
4
4
|
# packaging # for noaodatalab
|
|
5
5
|
#specutils==1.3.1
|
|
6
6
|
# jupyter-lab --ip=0.0.0.0
|
|
7
|
-
|
|
7
|
+
jupyterlab # =3.1.17
|
|
8
|
+
|
|
8
9
|
#matplotlib==3.4.3
|
|
9
10
|
#!sparclclient==0.3.9
|
|
10
11
|
|
|
11
12
|
Sphinx # ==4.1.2
|
|
12
13
|
sphinx-rtd-theme # ==0.5.2
|
|
13
14
|
pre-commit
|
|
14
|
-
build
|
|
15
15
|
pip
|
|
16
|
-
|
|
17
|
-
wheel
|
|
18
|
-
twine
|
|
16
|
+
flit
|
|
19
17
|
sphinx_mdinclude
|
|
18
|
+
|
|
19
|
+
#
|
|
20
|
+
pandas
|
|
21
|
+
matplotlib
|
|
22
|
+
ipympl
|
|
23
|
+
astropy
|
|
@@ -14,17 +14,13 @@
|
|
|
14
14
|
import os
|
|
15
15
|
import re
|
|
16
16
|
import sys
|
|
17
|
-
|
|
18
17
|
sys.path.insert(0, os.path.abspath(".."))
|
|
19
|
-
|
|
20
|
-
|
|
21
18
|
from sparcl import __version__
|
|
22
19
|
|
|
23
|
-
|
|
24
20
|
# -- Project information -----------------------------------------------------
|
|
25
21
|
|
|
26
22
|
project = "SPARCL"
|
|
27
|
-
copyright = "
|
|
23
|
+
copyright = "2024, S.Pothier, A.Jacques"
|
|
28
24
|
author = "S.Pothier, A.Jacques"
|
|
29
25
|
|
|
30
26
|
#!version = client_version
|
|
@@ -21,6 +21,15 @@ class Results(UserList):
|
|
|
21
21
|
self.fields = client.fields
|
|
22
22
|
self.to_science_fields()
|
|
23
23
|
|
|
24
|
+
# HACK 12/14/2023 -sp- to fix UUID problem presumably
|
|
25
|
+
# produced on stack version upgrade (to Django 4.2, postgres 13+)
|
|
26
|
+
# Done per AB for expediency since real solution will be easier
|
|
27
|
+
# after field-renaming is removed.
|
|
28
|
+
for rec in self.recs:
|
|
29
|
+
if "sparcl_id" in rec:
|
|
30
|
+
rec["sparcl_id"] = str(rec["sparcl_id"])
|
|
31
|
+
# END __init__()
|
|
32
|
+
|
|
24
33
|
# https://docs.python.org/3/library/collections.html#collections.deque.clear
|
|
25
34
|
def clear(self):
|
|
26
35
|
"""Delete the contents of this collection."""
|
|
@@ -132,9 +141,9 @@ class Results(UserList):
|
|
|
132
141
|
# Transform science fields to internal fields
|
|
133
142
|
new_recs = self.science_to_internal_fields()
|
|
134
143
|
# Get the ids or specids from retrieved records
|
|
135
|
-
if type(ids_og[0])
|
|
144
|
+
if type(ids_og[0]) is str:
|
|
136
145
|
ids_re = [f["sparcl_id"] for f in new_recs]
|
|
137
|
-
elif type(ids_og[0])
|
|
146
|
+
elif type(ids_og[0]) is int:
|
|
138
147
|
ids_re = [f["specid"] for f in new_recs]
|
|
139
148
|
# Enumerate the original ids
|
|
140
149
|
dict_og = {x: i for i, x in enumerate(ids_og)}
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
A client for getting spectra and meta-data from NOIRLab.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
|
|
6
5
|
# List of packages to import when "from sparcl import *" is used
|
|
7
6
|
__all__ = ["client", "align_records"]
|
|
8
7
|
|
|
@@ -26,16 +25,10 @@ __all__ = ["client", "align_records"]
|
|
|
26
25
|
|
|
27
26
|
# must mach: [N!]N(.N)*[{a|b|rc}N][.postN][.devN]
|
|
28
27
|
# Example of a correct version string: '0.4.0a3.dev35'
|
|
29
|
-
# __version__ = '0.4.0b1.dev8'
|
|
30
|
-
# __version__ = '0.4.0b1.dev10'
|
|
31
|
-
# __version__ = '1.0.0'
|
|
32
|
-
# __version__ = '1.0.0b1.dev7'
|
|
33
|
-
# __version__ = '1.0.0b1.dev8'
|
|
34
|
-
# __version__ = '1.0.0b1.dev9'
|
|
35
|
-
# __version__ = '1.0.1b2.dev1'
|
|
36
|
-
# __version__ = '1.1rc1'
|
|
37
|
-
# __version__ = '1.1rc2'
|
|
38
28
|
# __version__ = '1.1'
|
|
39
29
|
# __version__ = '1.2.0b4'
|
|
40
30
|
# __version__ = '1.2.0' # Release
|
|
41
|
-
__version__ = "1.2.
|
|
31
|
+
# __version__ = "1.2.1b3"
|
|
32
|
+
# __version__ = "1.2.1"
|
|
33
|
+
# FIRST uncommented value will be used! (so only leave one uncommented)
|
|
34
|
+
__version__ = "1.2.2"
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
"""Client module for SPARCL.
|
|
2
2
|
This module interfaces to the SPARC-Server to get spectra data.
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
# python -m unittest tests.tests_api
|
|
6
|
+
|
|
7
|
+
# ### Run tests against DEV
|
|
8
|
+
# serverurl=http://localhost:8050 python -m unittest tests.tests_api
|
|
9
|
+
#
|
|
10
|
+
# ### Run tests Against PAT Server.
|
|
11
|
+
# export serverurl=https://sparc1.datalab.noirlab.edu/
|
|
12
|
+
# python -m unittest tests.tests_api
|
|
13
|
+
|
|
5
14
|
#
|
|
6
15
|
# Doctest example:
|
|
7
16
|
# cd ~/sandbox/sparclclient
|
|
@@ -14,6 +23,7 @@ This module interfaces to the SPARC-Server to get spectra data.
|
|
|
14
23
|
from urllib.parse import urlencode, urlparse
|
|
15
24
|
from warnings import warn
|
|
16
25
|
import pickle
|
|
26
|
+
import getpass
|
|
17
27
|
|
|
18
28
|
#!from pathlib import Path
|
|
19
29
|
import tempfile
|
|
@@ -22,6 +32,9 @@ import tempfile
|
|
|
22
32
|
# External Packages
|
|
23
33
|
import requests
|
|
24
34
|
|
|
35
|
+
#!from requests.auth import HTTPBasicAuth
|
|
36
|
+
from requests.auth import AuthBase
|
|
37
|
+
|
|
25
38
|
############################################
|
|
26
39
|
# Local Packages
|
|
27
40
|
from sparcl.fields import Fields
|
|
@@ -102,6 +115,19 @@ RESERVED = set([DEFAULT, ALL])
|
|
|
102
115
|
#! return set(lists[0]).intersection(*lists[1:])
|
|
103
116
|
|
|
104
117
|
|
|
118
|
+
class TokenAuth(AuthBase):
|
|
119
|
+
"""Attaches HTTP Token Authentication to the given Request object."""
|
|
120
|
+
|
|
121
|
+
def __init__(self, token):
|
|
122
|
+
# setup any auth-related data here
|
|
123
|
+
self.token = token
|
|
124
|
+
|
|
125
|
+
def __call__(self, request):
|
|
126
|
+
# modify and return the request
|
|
127
|
+
request.headers["Authorization"] = self.token
|
|
128
|
+
return request
|
|
129
|
+
|
|
130
|
+
|
|
105
131
|
###########################
|
|
106
132
|
# ## The Client class
|
|
107
133
|
|
|
@@ -113,7 +139,7 @@ class SparclClient: # was SparclApi()
|
|
|
113
139
|
about the Client and Server that is usefule to Developers.
|
|
114
140
|
|
|
115
141
|
Args:
|
|
116
|
-
url (:obj:`str`, optional): Base URL of
|
|
142
|
+
url (:obj:`str`, optional): Base URL of SPARCL Server. Defaults
|
|
117
143
|
to 'https://astrosparcl.datalab.noirlab.edu'.
|
|
118
144
|
|
|
119
145
|
verbose (:obj:`bool`, optional): Default verbosity is set to
|
|
@@ -137,13 +163,11 @@ class SparclClient: # was SparclApi()
|
|
|
137
163
|
|
|
138
164
|
"""
|
|
139
165
|
|
|
140
|
-
KNOWN_GOOD_API_VERSION =
|
|
166
|
+
KNOWN_GOOD_API_VERSION = 12.0 # @@@ Change when Server version incremented
|
|
141
167
|
|
|
142
168
|
def __init__(
|
|
143
169
|
self,
|
|
144
170
|
*,
|
|
145
|
-
email=None,
|
|
146
|
-
password=None,
|
|
147
171
|
url=_PROD,
|
|
148
172
|
verbose=False,
|
|
149
173
|
show_curl=False,
|
|
@@ -153,11 +177,11 @@ class SparclClient: # was SparclApi()
|
|
|
153
177
|
"""Create client instance."""
|
|
154
178
|
session = requests.Session()
|
|
155
179
|
self.session = session
|
|
156
|
-
|
|
157
|
-
self.session.auth = (email, password) if email and password else None
|
|
180
|
+
self.session.auth = None
|
|
158
181
|
self.rooturl = url.rstrip("/") # eg. "http://localhost:8050"
|
|
159
182
|
self.apiurl = f"{self.rooturl}/sparc"
|
|
160
183
|
self.apiversion = None
|
|
184
|
+
self.token = None
|
|
161
185
|
self.verbose = verbose
|
|
162
186
|
self.show_curl = show_curl # Show CURL equivalent of client method
|
|
163
187
|
#!self.internal_names = internal_names
|
|
@@ -199,7 +223,6 @@ class SparclClient: # was SparclApi()
|
|
|
199
223
|
f"at {self.apiurl}."
|
|
200
224
|
)
|
|
201
225
|
raise Exception(msg)
|
|
202
|
-
# self.session = requests.Session() #@@@
|
|
203
226
|
|
|
204
227
|
self.clientversion = client_version
|
|
205
228
|
self.fields = Fields(self.apiurl)
|
|
@@ -214,13 +237,109 @@ class SparclClient: # was SparclApi()
|
|
|
214
237
|
f"(sparclclient:{self.clientversion},"
|
|
215
238
|
f" api:{self.apiversion},"
|
|
216
239
|
f" {self.apiurl},"
|
|
240
|
+
f" client_hash={ut.githash()},"
|
|
217
241
|
f" verbose={self.verbose},"
|
|
218
242
|
f" connect_timeout={self.c_timeout},"
|
|
219
243
|
f" read_timeout={self.r_timeout})"
|
|
220
244
|
)
|
|
221
245
|
|
|
246
|
+
def login(self, email, password=None):
|
|
247
|
+
"""Login to the SPARCL service.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
email (:obj:`str`): User login email.
|
|
251
|
+
|
|
252
|
+
password (:obj:`str`, optional): User SSO password.
|
|
253
|
+
If not given, the output will prompt the user
|
|
254
|
+
to enter in their SSO password.
|
|
255
|
+
|
|
256
|
+
Returns:
|
|
257
|
+
None.
|
|
258
|
+
|
|
259
|
+
Example:
|
|
260
|
+
>>>
|
|
261
|
+
>> client = SparclClient()
|
|
262
|
+
>> client.login('test_user@noirlab.edu', 'testpw')
|
|
263
|
+
Logged in successfully with email='test_user@noirlab.edu'
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
if email is None: # "logout"
|
|
267
|
+
old_email = self.session.auth[0] if self.session.auth else None
|
|
268
|
+
self.session.auth = None
|
|
269
|
+
self.token = None
|
|
270
|
+
print(
|
|
271
|
+
f"Logged-out successfully. "
|
|
272
|
+
f" Previously logged-in with email {old_email}."
|
|
273
|
+
)
|
|
274
|
+
return None
|
|
275
|
+
if password is None:
|
|
276
|
+
password = getpass.getpass(prompt="SSO Password: ")
|
|
277
|
+
url = f"{self.apiurl}/get_token/"
|
|
278
|
+
# print(f'login: get_token {url=}')
|
|
279
|
+
res = requests.post(
|
|
280
|
+
url,
|
|
281
|
+
json=dict(email=email, password=password),
|
|
282
|
+
timeout=self.timeout,
|
|
283
|
+
)
|
|
284
|
+
try:
|
|
285
|
+
res.raise_for_status()
|
|
286
|
+
#!print(f"DBG: {res.content=}")
|
|
287
|
+
self.token = res.json()
|
|
288
|
+
self.session.auth = (email, password)
|
|
289
|
+
except Exception:
|
|
290
|
+
self.session.auth = None
|
|
291
|
+
self.token = None
|
|
292
|
+
msg = (
|
|
293
|
+
"Could not login with given credentials."
|
|
294
|
+
' Reverted to "Anonymous" user.'
|
|
295
|
+
)
|
|
296
|
+
return msg
|
|
297
|
+
|
|
298
|
+
print(f"Logged in successfully with {email=}")
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
def logout(self):
|
|
302
|
+
"""Logout of the SPARCL service.
|
|
303
|
+
|
|
304
|
+
Args:
|
|
305
|
+
None.
|
|
306
|
+
|
|
307
|
+
Returns:
|
|
308
|
+
None.
|
|
309
|
+
|
|
310
|
+
Example:
|
|
311
|
+
>>> client = SparclClient()
|
|
312
|
+
>>> client.logout()
|
|
313
|
+
Logged-out successfully. Previously logged-in with email None.
|
|
314
|
+
"""
|
|
315
|
+
|
|
316
|
+
return self.login(None)
|
|
317
|
+
|
|
318
|
+
@property
|
|
319
|
+
def authorized(self):
|
|
320
|
+
auth = TokenAuth(self.token) if self.token else None
|
|
321
|
+
response = requests.get(
|
|
322
|
+
f"{self.apiurl}/auth_status/", auth=auth, timeout=self.timeout
|
|
323
|
+
)
|
|
324
|
+
auth_status = response.json()
|
|
325
|
+
#! print(f"DBG authorized: {auth_status=}")
|
|
326
|
+
|
|
327
|
+
username = auth_status.get("Loggedin_User")
|
|
328
|
+
#! all_private_drs = set(auth_status.get("All_Private_Datasets"))
|
|
329
|
+
all_public_drs = set(auth_status.get("All_Public_Datasets"))
|
|
330
|
+
auth_drs = set(auth_status.get("Authorized_Private_Datasets"))
|
|
331
|
+
res = dict(
|
|
332
|
+
Loggedin_As=username, # email
|
|
333
|
+
Authorized_Datasets=auth_drs | all_public_drs,
|
|
334
|
+
#! Unauthorized_Datasets=all_private_drs - auth_drs,
|
|
335
|
+
#! All_Private_Datasets=all_private_drs,
|
|
336
|
+
#! All_Datasets=all_drs,
|
|
337
|
+
)
|
|
338
|
+
return res
|
|
339
|
+
|
|
222
340
|
@property
|
|
223
341
|
def all_datasets(self):
|
|
342
|
+
"""Set of all DataSets available from Server"""
|
|
224
343
|
return self.fields.all_drs
|
|
225
344
|
|
|
226
345
|
def get_default_fields(self, *, dataset_list=None):
|
|
@@ -232,7 +351,7 @@ class SparclClient: # was SparclApi()
|
|
|
232
351
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
233
352
|
which to get the default fields. Defaults to None, which
|
|
234
353
|
will return the intersection of default fields in all
|
|
235
|
-
data sets hosted on the
|
|
354
|
+
data sets hosted on the SPARCL database.
|
|
236
355
|
|
|
237
356
|
Returns:
|
|
238
357
|
List of fields tagged as 'default' from DATASET_LIST.
|
|
@@ -263,7 +382,7 @@ class SparclClient: # was SparclApi()
|
|
|
263
382
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
264
383
|
which to get all fields. Defaults to None, which
|
|
265
384
|
will return the intersection of all fields in all
|
|
266
|
-
data sets hosted on the
|
|
385
|
+
data sets hosted on the SPARCL database.
|
|
267
386
|
|
|
268
387
|
Returns:
|
|
269
388
|
List of fields tagged as 'all' from DATASET_LIST.
|
|
@@ -321,7 +440,7 @@ class SparclClient: # was SparclApi()
|
|
|
321
440
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
322
441
|
which to get available fields. Defaults to None, which
|
|
323
442
|
will return the intersection of all available fields in
|
|
324
|
-
all data sets hosted on the
|
|
443
|
+
all data sets hosted on the SPARCL database.
|
|
325
444
|
|
|
326
445
|
Returns:
|
|
327
446
|
Set of fields available from data sets in DATASET_LIST.
|
|
@@ -329,7 +448,8 @@ class SparclClient: # was SparclApi()
|
|
|
329
448
|
Example:
|
|
330
449
|
>>> client = SparclClient()
|
|
331
450
|
>>> sorted(client.get_available_fields())
|
|
332
|
-
['data_release', 'datasetgroup', 'dateobs', 'dateobs_center', 'dec', '
|
|
451
|
+
['data_release', 'datasetgroup', 'dateobs', 'dateobs_center', 'dec', 'exptime', 'extra_files', 'file', 'flux', 'instrument', 'ivar', 'mask', 'model', 'ra', 'redshift', 'redshift_err', 'redshift_warning', 'site', 'sparcl_id', 'specid', 'specprimary', 'spectype', 'survey', 'targetid', 'telescope', 'updated', 'wave_sigma', 'wavelength', 'wavemax', 'wavemin']
|
|
452
|
+
|
|
333
453
|
""" # noqa: E501
|
|
334
454
|
|
|
335
455
|
drs = self.fields.all_drs if dataset_list is None else dataset_list
|
|
@@ -348,7 +468,7 @@ class SparclClient: # was SparclApi()
|
|
|
348
468
|
Example:
|
|
349
469
|
>>> client = SparclClient()
|
|
350
470
|
>>> client.version
|
|
351
|
-
|
|
471
|
+
12.0
|
|
352
472
|
"""
|
|
353
473
|
|
|
354
474
|
if self.apiversion is None:
|
|
@@ -363,12 +483,14 @@ class SparclClient: # was SparclApi()
|
|
|
363
483
|
outfields=None,
|
|
364
484
|
*,
|
|
365
485
|
constraints={}, # dict(fname) = [op, param, ...]
|
|
366
|
-
|
|
486
|
+
#! exclude_unauth = True, # Not implemented yet
|
|
367
487
|
limit=500,
|
|
368
488
|
sort=None,
|
|
489
|
+
# count=False,
|
|
490
|
+
# dataset_list=None,
|
|
369
491
|
verbose=None,
|
|
370
492
|
):
|
|
371
|
-
"""Find records in the
|
|
493
|
+
"""Find records in the SPARCL database.
|
|
372
494
|
|
|
373
495
|
Args:
|
|
374
496
|
outfields (:obj:`list`, optional): List of fields to return.
|
|
@@ -428,6 +550,7 @@ class SparclClient: # was SparclApi()
|
|
|
428
550
|
}
|
|
429
551
|
uparams = dict(
|
|
430
552
|
limit=limit,
|
|
553
|
+
#! count='Y' if count else 'N'
|
|
431
554
|
)
|
|
432
555
|
if sort is not None:
|
|
433
556
|
uparams["sort"] = sort
|
|
@@ -444,7 +567,8 @@ class SparclClient: # was SparclApi()
|
|
|
444
567
|
cmd = ut.curl_find_str(sspec, self.rooturl, qstr=qstr)
|
|
445
568
|
print(cmd)
|
|
446
569
|
|
|
447
|
-
|
|
570
|
+
auth = TokenAuth(self.token) if self.token else None
|
|
571
|
+
res = requests.post(url, json=sspec, auth=auth, timeout=self.timeout)
|
|
448
572
|
|
|
449
573
|
if res.status_code != 200:
|
|
450
574
|
if verbose and ("traceback" in res.json()):
|
|
@@ -460,14 +584,14 @@ class SparclClient: # was SparclApi()
|
|
|
460
584
|
self, uuid_list, *, dataset_list=None, countOnly=False, verbose=False
|
|
461
585
|
):
|
|
462
586
|
"""Return the subset of sparcl_ids in the given uuid_list that are
|
|
463
|
-
NOT stored in the
|
|
587
|
+
NOT stored in the SPARCL database.
|
|
464
588
|
|
|
465
589
|
Args:
|
|
466
590
|
uuid_list (:obj:`list`): List of sparcl_ids.
|
|
467
591
|
|
|
468
592
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
469
593
|
which to find missing sparcl_ids. Defaults to None, meaning
|
|
470
|
-
all data sets hosted on the
|
|
594
|
+
all data sets hosted on the SPARCL database.
|
|
471
595
|
|
|
472
596
|
countOnly (:obj:`bool`, optional): Set to True to return only
|
|
473
597
|
a count of the missing sparcl_ids from the uuid_list.
|
|
@@ -478,7 +602,7 @@ class SparclClient: # was SparclApi()
|
|
|
478
602
|
|
|
479
603
|
Returns:
|
|
480
604
|
A list of the subset of sparcl_ids in the given uuid_list that
|
|
481
|
-
are NOT stored in the
|
|
605
|
+
are NOT stored in the SPARCL database.
|
|
482
606
|
|
|
483
607
|
Example:
|
|
484
608
|
>>> client = SparclClient()
|
|
@@ -513,14 +637,14 @@ class SparclClient: # was SparclApi()
|
|
|
513
637
|
self, specid_list, *, dataset_list=None, countOnly=False, verbose=False
|
|
514
638
|
):
|
|
515
639
|
"""Return the subset of specids in the given specid_list that are
|
|
516
|
-
NOT stored in the
|
|
640
|
+
NOT stored in the SPARCL database.
|
|
517
641
|
|
|
518
642
|
Args:
|
|
519
643
|
specid_list (:obj:`list`): List of specids.
|
|
520
644
|
|
|
521
645
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
522
646
|
which to find missing specids. Defaults to None, meaning
|
|
523
|
-
all data sets hosted on the
|
|
647
|
+
all data sets hosted on the SPARCL database.
|
|
524
648
|
|
|
525
649
|
countOnly (:obj:`bool`, optional): Set to True to return only
|
|
526
650
|
a count of the missing specids from the specid_list.
|
|
@@ -531,11 +655,12 @@ class SparclClient: # was SparclApi()
|
|
|
531
655
|
|
|
532
656
|
Returns:
|
|
533
657
|
A list of the subset of specids in the given specid_list that
|
|
534
|
-
are NOT stored in the
|
|
658
|
+
are NOT stored in the SPARCL database.
|
|
535
659
|
|
|
536
660
|
Example:
|
|
537
|
-
>>> client = SparclClient(
|
|
538
|
-
>>>
|
|
661
|
+
>>> client = SparclClient()
|
|
662
|
+
>>> found = client.find(outfields=['specid'], limit=2)
|
|
663
|
+
>>> specids = [f.specid for f in found.records]
|
|
539
664
|
>>> client.missing_specids(specids + ['bad_id'])
|
|
540
665
|
['bad_id']
|
|
541
666
|
"""
|
|
@@ -597,7 +722,7 @@ class SparclClient: # was SparclApi()
|
|
|
597
722
|
limit=500,
|
|
598
723
|
verbose=None,
|
|
599
724
|
):
|
|
600
|
-
"""Retrieve spectra records from the
|
|
725
|
+
"""Retrieve spectra records from the SPARCL database by list of
|
|
601
726
|
sparcl_ids.
|
|
602
727
|
|
|
603
728
|
Args:
|
|
@@ -609,7 +734,7 @@ class SparclClient: # was SparclApi()
|
|
|
609
734
|
|
|
610
735
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
611
736
|
which to retrieve spectra data. Defaults to None, meaning all
|
|
612
|
-
data sets hosted on the
|
|
737
|
+
data sets hosted on the SPARCL database.
|
|
613
738
|
|
|
614
739
|
limit (:obj:`int`, optional): Maximum number of records to
|
|
615
740
|
return. Defaults to 500. Maximum allowed is 24,000.
|
|
@@ -622,7 +747,7 @@ class SparclClient: # was SparclApi()
|
|
|
622
747
|
|
|
623
748
|
Example:
|
|
624
749
|
>>> client = SparclClient()
|
|
625
|
-
>>> ids =
|
|
750
|
+
>>> ids = client.find(limit=1).ids
|
|
626
751
|
>>> inc = ['sparcl_id', 'flux', 'wavelength', 'model']
|
|
627
752
|
>>> ret = client.retrieve(uuid_list=ids, include=inc)
|
|
628
753
|
>>> type(ret.records[0].wavelength)
|
|
@@ -645,6 +770,7 @@ class SparclClient: # was SparclApi()
|
|
|
645
770
|
format = "pkl" # 'json',
|
|
646
771
|
chunk = 500
|
|
647
772
|
|
|
773
|
+
orig_dataset_list = dataset_list
|
|
648
774
|
if dataset_list is None:
|
|
649
775
|
dataset_list = self.fields.all_drs
|
|
650
776
|
assert isinstance(
|
|
@@ -686,8 +812,13 @@ class SparclClient: # was SparclApi()
|
|
|
686
812
|
#! "chunk_len": chunk,
|
|
687
813
|
"format": format,
|
|
688
814
|
#! "1thread": "yes", # @@@ 7.3.2023
|
|
689
|
-
"dataset_list": ",".join(dataset_list),
|
|
815
|
+
#!"dataset_list": ",".join(dataset_list),
|
|
690
816
|
}
|
|
817
|
+
# Do not put dataset_list in server call if it wasn't in client call.
|
|
818
|
+
# Else will interfer with defaulting behavior of AUTH (DLS-504).
|
|
819
|
+
if orig_dataset_list is not None:
|
|
820
|
+
uparams["dataset_list"] = ",".join(dataset_list)
|
|
821
|
+
|
|
691
822
|
qstr = urlencode(uparams)
|
|
692
823
|
|
|
693
824
|
#!url = f'{self.apiurl}/retrieve/?{qstr}'
|
|
@@ -702,9 +833,8 @@ class SparclClient: # was SparclApi()
|
|
|
702
833
|
print(cmd)
|
|
703
834
|
|
|
704
835
|
try:
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
)
|
|
836
|
+
auth = TokenAuth(self.token) if self.token else None
|
|
837
|
+
res = requests.post(url, json=ids, auth=auth, timeout=self.timeout)
|
|
708
838
|
except requests.exceptions.ConnectTimeout as reCT:
|
|
709
839
|
raise ex.UnknownSparcl(f"ConnectTimeout: {reCT}")
|
|
710
840
|
except requests.exceptions.ReadTimeout as reRT:
|
|
@@ -781,7 +911,8 @@ class SparclClient: # was SparclApi()
|
|
|
781
911
|
limit=500,
|
|
782
912
|
verbose=False,
|
|
783
913
|
):
|
|
784
|
-
"""Retrieve spectra records from the
|
|
914
|
+
"""Retrieve spectra records from the SPARCL database by list of
|
|
915
|
+
specids.
|
|
785
916
|
|
|
786
917
|
Args:
|
|
787
918
|
specid_list (:obj:`list`): List of specids.
|
|
@@ -792,7 +923,7 @@ class SparclClient: # was SparclApi()
|
|
|
792
923
|
|
|
793
924
|
dataset_list (:obj:`list`, optional): List of data sets from
|
|
794
925
|
which to retrieve spectra data. Defaults to None, meaning all
|
|
795
|
-
data sets hosted on the
|
|
926
|
+
data sets hosted on the SPARCL database.
|
|
796
927
|
|
|
797
928
|
limit (:obj:`int`, optional): Maximum number of records to
|
|
798
929
|
return. Defaults to 500. Maximum allowed is 24,000.
|
|
@@ -21,9 +21,13 @@ def genSparclException(response, verbose=False):
|
|
|
21
21
|
return UnknownField(status.get("errorMessage"))
|
|
22
22
|
elif status.get("errorCode") == "BADCONST":
|
|
23
23
|
return BadSearchConstraint(status.get("errorMessage"))
|
|
24
|
+
elif status.get("errorCode") == "NODRACCESS":
|
|
25
|
+
return AccessNotAllowed(status.get("errorMessage"))
|
|
24
26
|
else:
|
|
25
27
|
return UnknownServerError(
|
|
26
|
-
f"{status.get('errorMessage')} "
|
|
28
|
+
f"{status.get('errorMessage')} "
|
|
29
|
+
f"[{status.get('errorCode')}] "
|
|
30
|
+
f"{status.get('traceback')}"
|
|
27
31
|
)
|
|
28
32
|
|
|
29
33
|
|
|
@@ -32,7 +36,7 @@ class BaseSparclException(Exception):
|
|
|
32
36
|
|
|
33
37
|
error_code = "UNKNOWN"
|
|
34
38
|
error_message = "<NA>"
|
|
35
|
-
traceback =
|
|
39
|
+
traceback = True
|
|
36
40
|
|
|
37
41
|
def get_subclass_name(self):
|
|
38
42
|
return self.__class__.__name__
|
|
@@ -138,4 +142,8 @@ class NoIDs(BaseSparclException):
|
|
|
138
142
|
error_code = "NOIDS"
|
|
139
143
|
|
|
140
144
|
|
|
145
|
+
class AccessNotAllowed(BaseSparclException):
|
|
146
|
+
error_code = "DSDENIED"
|
|
147
|
+
|
|
148
|
+
|
|
141
149
|
# error_code values should be no bigger than 8 characters 12345678
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Align or resample spectra related fields across multiple records."""
|
|
2
|
+
|
|
2
3
|
# See client.py for Doctest example
|
|
3
4
|
#
|
|
4
5
|
# For info about problems with floating point,
|
|
@@ -194,7 +195,7 @@ def align_records(records, fields=["flux", "wavelength"], precision=7):
|
|
|
194
195
|
>>> got = client.retrieve(found.ids, include=specflds)
|
|
195
196
|
>>> ar_dict, grid = align_records(got.records, fields=specflds)
|
|
196
197
|
>>> ar_dict['model'].shape
|
|
197
|
-
(21,
|
|
198
|
+
(21, 4669)
|
|
198
199
|
|
|
199
200
|
"""
|
|
200
201
|
# Report Garbage In
|