sparclclient 1.2.1.dev7__py2.py3-none-any.whl → 1.2.2__py2.py3-none-any.whl

sparcl/Results.py

@@ -21,6 +21,15 @@ class Results(UserList):
         self.fields = client.fields
         self.to_science_fields()
 
+        # HACK 12/14/2023 -sp- to fix UUID problem presumably
+        # produced on stack version upgrade (to Django 4.2, postgres 13+)
+        # Done per AB for expediency since real solution will be easier
+        # after field-renaming is removed.
+        for rec in self.recs:
+            if "sparcl_id" in rec:
+                rec["sparcl_id"] = str(rec["sparcl_id"])
+        # END __init__()
+
     # https://docs.python.org/3/library/collections.html#collections.deque.clear
     def clear(self):
         """Delete the contents of this collection."""
@@ -132,9 +141,9 @@ class Results(UserList):
             # Transform science fields to internal fields
             new_recs = self.science_to_internal_fields()
             # Get the ids or specids from retrieved records
-            if type(ids_og[0]) == str:
+            if type(ids_og[0]) is str:
                 ids_re = [f["sparcl_id"] for f in new_recs]
-            elif type(ids_og[0]) == int:
+            elif type(ids_og[0]) is int:
                 ids_re = [f["specid"] for f in new_recs]
             # Enumerate the original ids
             dict_og = {x: i for i, x in enumerate(ids_og)}

sparcl/__init__.py

@@ -2,7 +2,6 @@
 A client for getting spectra and meta-data from NOIRLab.
 """
 
-
 # List of packages to import when "from sparcl import *" is used
 __all__ = ["client", "align_records"]
 
@@ -26,16 +25,10 @@ __all__ = ["client", "align_records"]
 
 # must mach: [N!]N(.N)*[{a|b|rc}N][.postN][.devN]
 # Example of a correct version string: '0.4.0a3.dev35'
-# __version__ = '0.4.0b1.dev8'
-# __version__ = '0.4.0b1.dev10'
-# __version__ = '1.0.0'
-# __version__ = '1.0.0b1.dev7'
-# __version__ = '1.0.0b1.dev8'
-# __version__ = '1.0.0b1.dev9'
-# __version__ = '1.0.1b2.dev1'
-# __version__ = '1.1rc1'
-# __version__ = '1.1rc2'
 # __version__ = '1.1'
 # __version__ = '1.2.0b4'
 # __version__ = '1.2.0'  # Release
-__version__ = "1.2.1.dev7"
+# __version__ = "1.2.1b3"
+# __version__ = "1.2.1"
+# FIRST uncommented value will be used! (so only leave one uncommented)
+__version__ = "1.2.2"

sparcl/client.py

@@ -1,7 +1,16 @@
 """Client module for SPARCL.
 This module interfaces to the SPARC-Server to get spectra data.
 """
+
 # python -m unittest tests.tests_api
+
+# ### Run tests against DEV
+# serverurl=http://localhost:8050 python -m unittest tests.tests_api
+#
+#  ### Run tests Against PAT Server.
+#  export serverurl=https://sparc1.datalab.noirlab.edu/
+#  python -m unittest tests.tests_api
+
 #
 # Doctest example:
 #   cd ~/sandbox/sparclclient
@@ -14,6 +23,7 @@ This module interfaces to the SPARC-Server to get spectra data.
 from urllib.parse import urlencode, urlparse
 from warnings import warn
 import pickle
+import getpass
 
 #!from pathlib import Path
 import tempfile
@@ -22,6 +32,9 @@ import tempfile
 # External Packages
 import requests
 
+#!from requests.auth import HTTPBasicAuth
+from requests.auth import AuthBase
+
 ############################################
 # Local Packages
 from sparcl.fields import Fields
@@ -102,6 +115,19 @@ RESERVED = set([DEFAULT, ALL])
 #!    return set(lists[0]).intersection(*lists[1:])
 
 
+class TokenAuth(AuthBase):
+    """Attaches HTTP Token Authentication to the given Request object."""
+
+    def __init__(self, token):
+        # setup any auth-related data here
+        self.token = token
+
+    def __call__(self, request):
+        # modify and return the request
+        request.headers["Authorization"] = self.token
+        return request
+
+
 ###########################
 # ## The Client class
 
@@ -113,7 +139,7 @@ class SparclClient:  # was SparclApi()
     about the Client and Server that is usefule to Developers.
 
     Args:
-        url (:obj:`str`, optional): Base URL of SPARC Server. Defaults
+        url (:obj:`str`, optional): Base URL of SPARCL Server. Defaults
             to 'https://astrosparcl.datalab.noirlab.edu'.
 
         verbose (:obj:`bool`, optional): Default verbosity is set to
@@ -137,13 +163,11 @@ class SparclClient:  # was SparclApi()
 
     """
 
-    KNOWN_GOOD_API_VERSION = 11.0  # @@@ Change when Server version incremented
+    KNOWN_GOOD_API_VERSION = 12.0  # @@@ Change when Server version incremented
 
     def __init__(
         self,
         *,
-        email=None,
-        password=None,
         url=_PROD,
         verbose=False,
         show_curl=False,
@@ -153,11 +177,11 @@ class SparclClient:  # was SparclApi()
         """Create client instance."""
         session = requests.Session()
         self.session = session
-
-        self.session.auth = (email, password) if email and password else None
+        self.session.auth = None
         self.rooturl = url.rstrip("/")  # eg. "http://localhost:8050"
         self.apiurl = f"{self.rooturl}/sparc"
         self.apiversion = None
+        self.token = None
         self.verbose = verbose
         self.show_curl = show_curl  # Show CURL equivalent of client method
         #!self.internal_names = internal_names
@@ -199,7 +223,6 @@ class SparclClient:  # was SparclApi()
                 f"at {self.apiurl}."
             )
             raise Exception(msg)
-        # self.session = requests.Session() #@@@
 
         self.clientversion = client_version
         self.fields = Fields(self.apiurl)
@@ -214,11 +237,106 @@ class SparclClient:  # was SparclApi()
             f"(sparclclient:{self.clientversion},"
             f" api:{self.apiversion},"
             f" {self.apiurl},"
+            f" client_hash={ut.githash()},"
             f" verbose={self.verbose},"
             f" connect_timeout={self.c_timeout},"
             f" read_timeout={self.r_timeout})"
         )
 
+    def login(self, email, password=None):
+        """Login to the SPARCL service.
+
+        Args:
+            email (:obj:`str`): User login email.
+
+            password (:obj:`str`, optional): User SSO password.
+                If not given, the output will prompt the user
+                to enter in their SSO password.
+
+        Returns:
+            None.
+
+        Example:
+            >>>
+            >> client = SparclClient()
+            >> client.login('test_user@noirlab.edu', 'testpw')
+            Logged in successfully with email='test_user@noirlab.edu'
+        """
+
+        if email is None:  # "logout"
+            old_email = self.session.auth[0] if self.session.auth else None
+            self.session.auth = None
+            self.token = None
+            print(
+                f"Logged-out successfully. "
+                f" Previously logged-in with email {old_email}."
+            )
+            return None
+        if password is None:
+            password = getpass.getpass(prompt="SSO Password: ")
+        url = f"{self.apiurl}/get_token/"
+        # print(f'login: get_token {url=}')
+        res = requests.post(
+            url,
+            json=dict(email=email, password=password),
+            timeout=self.timeout,
+        )
+        try:
+            res.raise_for_status()
+            #!print(f"DBG: {res.content=}")
+            self.token = res.json()
+            self.session.auth = (email, password)
+        except Exception:
+            self.session.auth = None
+            self.token = None
+            msg = (
+                "Could not login with given credentials."
+                ' Reverted to "Anonymous" user.'
+            )
+            return msg
+
+        print(f"Logged in successfully with {email=}")
+        return None
+
+    def logout(self):
+        """Logout of the SPARCL service.
+
+        Args:
+            None.
+
+        Returns:
+            None.
+
+        Example:
+            >>> client = SparclClient()
+            >>> client.logout()
+            Logged-out successfully.  Previously logged-in with email None.
+        """
+
+        return self.login(None)
+
+    @property
+    def authorized(self):
+        auth = TokenAuth(self.token) if self.token else None
+        response = requests.get(
+            f"{self.apiurl}/auth_status/", auth=auth, timeout=self.timeout
+        )
+        auth_status = response.json()
+        #! print(f"DBG authorized: {auth_status=}")
+
+        username = auth_status.get("Loggedin_User")
+        #! all_private_drs = set(auth_status.get("All_Private_Datasets"))
+        all_public_drs = set(auth_status.get("All_Public_Datasets"))
+        auth_drs = set(auth_status.get("Authorized_Private_Datasets"))
+        res = dict(
+            Loggedin_As=username,  # email
+            Authorized_Datasets=auth_drs | all_public_drs,
+            #! Unauthorized_Datasets=all_private_drs - auth_drs,
+            #! All_Private_Datasets=all_private_drs,
+            #! All_Datasets=all_drs,
+        )
+        return res
+
     @property
     def all_datasets(self):
         """Set of all DataSets available from Server"""
@@ -233,7 +351,7 @@ class SparclClient:  # was SparclApi()
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to get the default fields. Defaults to None, which
                 will return the intersection of default fields in all
-                data sets hosted on the SPARC database.
+                data sets hosted on the SPARCL database.
 
         Returns:
             List of fields tagged as 'default' from DATASET_LIST.
@@ -264,7 +382,7 @@ class SparclClient:  # was SparclApi()
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to get all fields. Defaults to None, which
                 will return the intersection of all fields in all
-                data sets hosted on the SPARC database.
+                data sets hosted on the SPARCL database.
 
         Returns:
             List of fields tagged as 'all' from DATASET_LIST.
@@ -322,7 +440,7 @@ class SparclClient:  # was SparclApi()
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to get available fields. Defaults to None, which
                 will return the intersection of all available fields in
-                all data sets hosted on the SPARC database.
+                all data sets hosted on the SPARCL database.
 
         Returns:
             Set of fields available from data sets in DATASET_LIST.
@@ -330,7 +448,8 @@ class SparclClient:  # was SparclApi()
         Example:
             >>> client = SparclClient()
             >>> sorted(client.get_available_fields())
-            ['data_release', 'datasetgroup', 'dateobs', 'dateobs_center', 'dec', 'dirpath', 'exptime', 'extra_files', 'filename', 'filesize', 'flux', 'instrument', 'ivar', 'mask', 'model', 'ra', 'redshift', 'redshift_err', 'redshift_warning', 'site', 'sparcl_id', 'specid', 'specprimary', 'spectype', 'survey', 'targetid', 'telescope', 'updated', 'wave_sigma', 'wavelength', 'wavemax', 'wavemin']
+            ['data_release', 'datasetgroup', 'dateobs', 'dateobs_center', 'dec', 'exptime', 'extra_files', 'file', 'flux', 'instrument', 'ivar', 'mask', 'model', 'ra', 'redshift', 'redshift_err', 'redshift_warning', 'site', 'sparcl_id', 'specid', 'specprimary', 'spectype', 'survey', 'targetid', 'telescope', 'updated', 'wave_sigma', 'wavelength', 'wavemax', 'wavemin']
+
         """  # noqa: E501
 
         drs = self.fields.all_drs if dataset_list is None else dataset_list
@@ -349,7 +468,7 @@ class SparclClient:  # was SparclApi()
         Example:
             >>> client = SparclClient()
             >>> client.version
-            9.0
+            12.0
         """
 
         if self.apiversion is None:
@@ -364,12 +483,14 @@ class SparclClient:  # was SparclApi()
         outfields=None,
         *,
         constraints={},  # dict(fname) = [op, param, ...]
-        # dataset_list=None,
+        #! exclude_unauth = True,  # Not implemented yet
         limit=500,
         sort=None,
+        # count=False,
+        # dataset_list=None,
         verbose=None,
     ):
-        """Find records in the SPARC database.
+        """Find records in the SPARCL database.
 
         Args:
             outfields (:obj:`list`, optional): List of fields to return.
@@ -429,6 +550,7 @@ class SparclClient:  # was SparclApi()
             }
         uparams = dict(
             limit=limit,
+            #! count='Y' if count else 'N'
         )
         if sort is not None:
             uparams["sort"] = sort
@@ -445,7 +567,8 @@ class SparclClient:  # was SparclApi()
             cmd = ut.curl_find_str(sspec, self.rooturl, qstr=qstr)
             print(cmd)
 
-        res = requests.post(url, json=sspec, timeout=self.timeout)
+        auth = TokenAuth(self.token) if self.token else None
+        res = requests.post(url, json=sspec, auth=auth, timeout=self.timeout)
 
         if res.status_code != 200:
             if verbose and ("traceback" in res.json()):
@@ -461,14 +584,14 @@ class SparclClient:  # was SparclApi()
         self, uuid_list, *, dataset_list=None, countOnly=False, verbose=False
     ):
         """Return the subset of sparcl_ids in the given uuid_list that are
-        NOT stored in the SPARC database.
+        NOT stored in the SPARCL database.
 
         Args:
             uuid_list (:obj:`list`): List of sparcl_ids.
 
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to find missing sparcl_ids. Defaults to None, meaning
-                all data sets hosted on the SPARC database.
+                all data sets hosted on the SPARCL database.
 
             countOnly (:obj:`bool`, optional): Set to True to return only
                 a count of the missing sparcl_ids from the uuid_list.
@@ -479,7 +602,7 @@ class SparclClient:  # was SparclApi()
 
         Returns:
             A list of the subset of sparcl_ids in the given uuid_list that
-            are NOT stored in the SPARC database.
+            are NOT stored in the SPARCL database.
 
         Example:
             >>> client = SparclClient()
@@ -514,14 +637,14 @@ class SparclClient:  # was SparclApi()
         self, specid_list, *, dataset_list=None, countOnly=False, verbose=False
     ):
         """Return the subset of specids in the given specid_list that are
-        NOT stored in the SPARC database.
+        NOT stored in the SPARCL database.
 
         Args:
             specid_list (:obj:`list`): List of specids.
 
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to find missing specids. Defaults to None, meaning
-                all data sets hosted on the SPARC database.
+                all data sets hosted on the SPARCL database.
 
             countOnly (:obj:`bool`, optional): Set to True to return only
                 a count of the missing specids from the specid_list.
@@ -532,11 +655,12 @@ class SparclClient:  # was SparclApi()
 
         Returns:
             A list of the subset of specids in the given specid_list that
-            are NOT stored in the SPARC database.
+            are NOT stored in the SPARCL database.
 
         Example:
-            >>> client = SparclClient(url=_PAT)
-            >>> specids = ['7972592460248666112', '3663710814482833408']
+            >>> client = SparclClient()
+            >>> found = client.find(outfields=['specid'], limit=2)
+            >>> specids = [f.specid for f in found.records]
             >>> client.missing_specids(specids + ['bad_id'])
             ['bad_id']
         """
@@ -598,7 +722,7 @@ class SparclClient:  # was SparclApi()
         limit=500,
         verbose=None,
     ):
-        """Retrieve spectra records from the SPARC database by list of
+        """Retrieve spectra records from the SPARCL database by list of
         sparcl_ids.
 
         Args:
@@ -610,7 +734,7 @@ class SparclClient:  # was SparclApi()
 
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to retrieve spectra data. Defaults to None, meaning all
-                data sets hosted on the SPARC database.
+                data sets hosted on the SPARCL database.
 
             limit (:obj:`int`, optional): Maximum number of records to
                 return. Defaults to 500. Maximum allowed is 24,000.
@@ -623,7 +747,7 @@ class SparclClient:  # was SparclApi()
 
         Example:
             >>> client = SparclClient()
-            >>> ids = ['00000f0b-07db-4234-892a-6e347db79c89',]
+            >>> ids = client.find(limit=1).ids
             >>> inc = ['sparcl_id', 'flux', 'wavelength', 'model']
             >>> ret = client.retrieve(uuid_list=ids, include=inc)
             >>> type(ret.records[0].wavelength)
@@ -646,6 +770,7 @@ class SparclClient:  # was SparclApi()
         format = "pkl"  # 'json',
         chunk = 500
 
+        orig_dataset_list = dataset_list
         if dataset_list is None:
             dataset_list = self.fields.all_drs
         assert isinstance(
@@ -687,8 +812,13 @@ class SparclClient:  # was SparclApi()
             #! "chunk_len": chunk,
             "format": format,
             #! "1thread": "yes",  # @@@ 7.3.2023
-            "dataset_list": ",".join(dataset_list),
+            #!"dataset_list": ",".join(dataset_list),
         }
+        # Do not put dataset_list in server call if it wasn't in client call.
+        # Else will interfer with defaulting behavior of AUTH (DLS-504).
+        if orig_dataset_list is not None:
+            uparams["dataset_list"] = ",".join(dataset_list)
+
         qstr = urlencode(uparams)
 
         #!url = f'{self.apiurl}/retrieve/?{qstr}'
@@ -703,9 +833,8 @@ class SparclClient:  # was SparclApi()
             print(cmd)
 
         try:
-            res = requests.post(
-                url, json=ids, auth=self.session.auth, timeout=self.timeout
-            )
+            auth = TokenAuth(self.token) if self.token else None
+            res = requests.post(url, json=ids, auth=auth, timeout=self.timeout)
         except requests.exceptions.ConnectTimeout as reCT:
             raise ex.UnknownSparcl(f"ConnectTimeout: {reCT}")
         except requests.exceptions.ReadTimeout as reRT:
@@ -782,7 +911,8 @@ class SparclClient:  # was SparclApi()
         limit=500,
         verbose=False,
     ):
-        """Retrieve spectra records from the SPARC database by list of specids.
+        """Retrieve spectra records from the SPARCL database by list of
+        specids.
 
         Args:
             specid_list (:obj:`list`): List of specids.
@@ -793,7 +923,7 @@ class SparclClient:  # was SparclApi()
 
             dataset_list (:obj:`list`, optional): List of data sets from
                 which to retrieve spectra data. Defaults to None, meaning all
-                data sets hosted on the SPARC database.
+                data sets hosted on the SPARCL database.
 
             limit (:obj:`int`, optional): Maximum number of records to
                 return. Defaults to 500. Maximum allowed is 24,000.

sparcl/exceptions.py

@@ -21,9 +21,13 @@ def genSparclException(response, verbose=False):
         return UnknownField(status.get("errorMessage"))
     elif status.get("errorCode") == "BADCONST":
         return BadSearchConstraint(status.get("errorMessage"))
+    elif status.get("errorCode") == "NODRACCESS":
+        return AccessNotAllowed(status.get("errorMessage"))
     else:
         return UnknownServerError(
-            f"{status.get('errorMessage')} " f"[{status.get('errorCode')}]"
+            f"{status.get('errorMessage')} "
+            f"[{status.get('errorCode')}] "
+            f"{status.get('traceback')}"
         )
 
 
@@ -32,7 +36,7 @@ class BaseSparclException(Exception):
 
     error_code = "UNKNOWN"
     error_message = "<NA>"
-    traceback = None
+    traceback = True
 
     def get_subclass_name(self):
         return self.__class__.__name__
@@ -138,4 +142,8 @@ class NoIDs(BaseSparclException):
     error_code = "NOIDS"
 
 
+class AccessNotAllowed(BaseSparclException):
+    error_code = "DSDENIED"
+
+
 # error_code values should be no bigger than 8 characters 12345678

sparcl/fields.py

@@ -1,5 +1,6 @@
 """Get Field names associated with various SPARCL conditions.
 """
+
 # Python Standard Library
 from collections import defaultdict
 

sparcl/gather_2d.py

@@ -1,4 +1,5 @@
 """Align or resample spectra related fields across multiple records."""
+
 # See client.py for Doctest example
 #
 # For info about problems with floating point,
@@ -194,7 +195,7 @@ def align_records(records, fields=["flux", "wavelength"], precision=7):
         >>> got = client.retrieve(found.ids, include=specflds)
         >>> ar_dict, grid = align_records(got.records, fields=specflds)
         >>> ar_dict['model'].shape
-        (21, 4666)
+        (21, 4669)
 
     """
     # Report Garbage In