sparclclient 1.2.0b3.dev9__py3-none-any.whl → 1.2.0b4__py3-none-any.whl

sparcl/Results.py

@@ -62,7 +62,7 @@ class Results(UserList):
                 else:
                     new = self.fields._science_name(orig, dr)
                     if new is None:
-                        keep = False
+                        keep = False # We don't have name mapping, toss rec
                     newrec[new] = rec[orig]
             if keep:
                 newrecs.append(_AttrDict(newrec))

sparcl/__init__.py

@@ -31,4 +31,4 @@ __all__ = ["client", "align_records"]
 #__version__ = '1.1rc1'
 #__version__ = '1.1rc2'
 #__version__ = '1.1'
-__version__ = '1.2.0b3.dev9'
+__version__ = '1.2.0b4'

sparcl/client.py

@@ -130,7 +130,7 @@ class SparclClient():  # was SparclApi()
 
     """
 
-    KNOWN_GOOD_API_VERSION = 8.0  # @@@ Change this on Server version increment
+    KNOWN_GOOD_API_VERSION = 9.0  # @@@ Change this on Server version increment
 
     def __init__(self, *,
                  url=_PROD,
@@ -394,7 +394,8 @@ class SparclClient():  # was SparclApi()
                 raise ex.NoCommonIdField(msg)
             outfields = [idfld]
         dataset_list = self.fields.all_drs
-        #! self._validate_science_fields(outfields, dataset_list=dataset_list) # DLS-401
+        #! self._validate_science_fields(outfields,
+        #!                               dataset_list=dataset_list) # DLS-401
         dr = list(dataset_list)[0]
         if len(constraints) > 0:
             self._validate_science_fields(constraints.keys(),
@@ -475,6 +476,56 @@ class SparclClient():  # was SparclApi()
         return ret
         # END missing()
 
+    def missing_specids(self, specid_list, *, dataset_list=None,
+                        countOnly=False, verbose=False):
+        """Return the subset of specids in the given specid_list that are
+        NOT stored in the SPARC database.
+
+        Args:
+            specid_list (:obj:`list`): List of specids.
+
+            dataset_list (:obj:`list`, optional): List of data sets from
+                which to find missing specids. Defaults to None, meaning
+                all data sets hosted on the SPARC database.
+
+            countOnly (:obj:`bool`, optional): Set to True to return only
+                a count of the missing specids from the specid_list.
+                Defaults to False.
+
+            verbose (:obj:`bool`, optional): Set to True for in-depth return
+                statement. Defaults to False.
+
+        Returns:
+            A list of the subset of specids in the given specid_list that
+            are NOT stored in the SPARC database.
+
+        Example:
+            >>> client = SparclClient(url=_PAT)
+            >>> specids = ['7972592460248666112', '3663710814482833408']
+            >>> client.missing_specids(specids + ['bad_id'])
+            ['bad_id']
+        """
+        if dataset_list is None:
+            dataset_list = self.fields.all_drs
+        assert isinstance(dataset_list, (list, set)), (
+            f'DATASET_LIST must be a list. Found {dataset_list}')
+
+        verbose = verbose or self.verbose
+        uparams = dict(dataset_list=','.join(dataset_list))
+        qstr = urlencode(uparams)
+        url = f'{self.apiurl}/missing_specids/?{qstr}'
+        specids = list(specid_list)
+        if verbose:
+            print(f'Using url="{url}"')
+        res = requests.post(url, json=specids, timeout=self.timeout)
+
+        res.raise_for_status()
+        if res.status_code != 200:
+            raise Exception(res)
+        ret = res.json()
+        return ret
+        # END missing_specids()
+
     # Include fields are Science (not internal) names. But the mapping
     # of Internal to Science name depends on DataSet.  Its possible
     # for a field (Science name) to be valid in one DataSet but not
@@ -667,6 +718,7 @@ class SparclClient():  # was SparclApi()
                            format='pkl',    # 'json',
                            include='DEFAULT',
                            dataset_list=None,
+                           limit=500,
                            verbose=False):
         """Retrieve spectra records from the SPARC database by list of specids.
 
@@ -716,7 +768,7 @@ class SparclClient():  # was SparclApi()
         dr = list(self.fields.all_drs)[0]
         idfld = self.fields._science_name('id', dr)
 
-        found = self.find([idfld], constraints=constraints)
+        found = self.find([idfld], constraints=constraints, limit=limit)
         if verbose:
             print(f'Found {found.count} matches.')
         res = self.retrieve(found.ids,
@@ -724,6 +776,7 @@ class SparclClient():  # was SparclApi()
                             format=format,
                             include=include,
                             dataset_list=dataset_list,
+                            limit=limit,
                             verbose=verbose)
         if verbose:
             print(f'Got {res.count} records.')

sparcl/gather_2d.py

@@ -1,8 +1,6 @@
 """Align or resample spectra related fields across multiple records."""
 # See client.py for Doctest example
 #
-# See:
-#   https://spectres.readthedocs.io/en/latest/
 # For info about problems with floating point,
 #   See:  https://docs.python.org/3/tutorial/floatingpoint.html
 #   Also: https://docs.python.org/3/library/decimal.html#floating-point-notes
@@ -10,41 +8,11 @@
 import math
 from decimal import Decimal
 #
-import spectres
 import numpy as np
 #
 import sparcl.client
 
 
-# Per paper, should be able to pass all flux in one call to spectres
-# https://arxiv.org/pdf/1705.05165.pdf
-# Perhaps users would rather the bins uniform (1,5,20 Angstroms?)
-def _resample_flux(records, wavstep=1):
-    smallest = math.floor(min([min(r.wavelength) for r in records]))
-    largest = math.ceil(max([max(r.wavelength) for r in records]))
-
-    #!wrange = largest - smallest
-    #new_wavs = np.fromfunction(lambda i: i + smallest, (wrange,), dtype=int)
-    #flux_2d = np.ones([len(records), wrange])
-
-    new_wavs = np.array(range(smallest, largest + 1, wavstep))
-    flux_2d = np.full([len(records), len(new_wavs)], None, dtype=float)
-
-    for idx, rec in enumerate(records):
-        flux_2d[idx] = spectres.spectres(new_wavs,
-                                         rec.wavelength,
-                                         rec.flux,
-                                         verbose=False)
-    return flux_2d, new_wavs
-
-
-def _tt0(numrecs=20):
-    client = sparcl.client.SparclClient()
-    found = client.find(constraints=dict(data_release=['BOSS-DR16']),
-                        limit=numrecs)
-    got = client.retrieve(found.ids)
-    flux_2d, new_wavs = _resample_flux(got.records)
-    return flux_2d, new_wavs
 
 
 # Map every wavelength of every record to index (ri,wi)
@@ -88,14 +56,17 @@ def _validate_wavelength_alignment(records, window, offsets, precision=None):
             else:
                 recwl = Decimal(rwl).quantize(PLACES)
             wwl = window[offsets[ri] + wi]
-            msg = (f'Wavelength in '
-                   f'Record[{ri}][{wi}] ({recwl}) does not match '
-                   f'Window[{offsets[ri]+wi} = offset[{ri}]={offsets[ri]} '
-                   f'+ {wi}]  ({wwl})'
-                   )
-            assert recwl == wwl, msg
-            # f'RecWL[{wi}] {rwl} != WindowWL[{offsets[ri+wi]}] {wwl} '
-            # f'offset={offsets[ri]}')
+            #! msg = (f'Wavelength in '
+            #!        f'Record[{ri}][{wi}] ({recwl}) does not match '
+            #!        f'Window[{offsets[ri]+wi} = offset[{ri}]={offsets[ri]} '
+            #!        f'+ {wi}]  ({wwl})'
+            #!        )
+            #! assert recwl == wwl, msg
+            if recwl != wwl:
+                msg = (f'The spectra cannot be aligned with the given'
+                       f' "precision" parameter ({precision}).'
+                       f' Try lowering the precision value.')
+                raise Exception(msg)
 
 
 # We want to align a bunch of records by wavelength into a single
@@ -127,6 +98,7 @@ def _tt1(numrecs=20, dr='BOSS-DR16'):
 
 
 # precision:: number of decimal places
+# "records" must contain "wavelength" field.
 def _wavelength_grid_offsets(records, precision=11):
     PLACES = Decimal(10) ** -precision
 
@@ -175,9 +147,14 @@ def _field_grid(records, fieldName, grid, offsets, precision=None):
 #!     ar = _flux_grid(records, grid, offsets, precision=precision)
 #!     return ar, np.array([float(x) for x in grid])
 
+def _validate_spectra_fields(records, fields):
+    spectra_fields = [client.fields.n2o['BOSS-DR16'][k] for k,v in client.fields.attrs['BOSS-DR16'].items() if v['storage']=='S']
+    [k for k in records[0].keys() if not k.startswith('_')]
+
+
 # TOP level: Intended for access from Jupyter NOTEBOOK.
 # Align spectra related field from records into one array using quantization.
-def align_records(records, fields=None, precision=7):
+def align_records(records, fields=['flux','wavelength'], precision=7):
     """Align given spectra-type fields to a common wavelength grid.
 
     Args:
@@ -186,9 +163,11 @@ def align_records(records, fields=None, precision=7):
 
         fields (:obj:`list`, optional): List of Science Field Names of
             spectra related fields to align and include in the results.
+            DEFAULT=['flux', 'wavelength']
 
         precision (:obj:`int`, optional): Number of decimal points to use for
-            quantizing wavelengths into a grid. Default=7
+            quantizing wavelengths into a grid.
+            DEFAULT=7
 
     Returns:
         tuple containing:
@@ -198,7 +177,7 @@ def align_records(records, fields=None, precision=7):
 
     Example:
         >>> client = sparcl.client.SparclClient()
-        >>> specflds = ['wavelength', 'flux', 'ivar', 'mask', 'model']
+        >>> specflds = ['wavelength', 'model']
         >>> cons = {"data_release": ['BOSS-DR16']}
         >>> found = client.find(constraints=cons, limit=21)
         >>> got = client.retrieve(found.ids, include=specflds)
@@ -207,15 +186,20 @@ def align_records(records, fields=None, precision=7):
         (21, 4670)
 
     """
+    # Report Garbage In
+    if 'wavelength' not in fields:
+        msg = (f'You must provide "wavelength" in the list provided'
+               f' in the "fields" paramter.  Got: {fields}')
+        raise Exception(msg)
+    if 'wavelength' not in records[0]:
+        msg = (f'Records must contain the "wavelength" field.'
+               f' The first record contains fields: {sorted(records[0].keys())}')
+        raise Exception(msg)
+
+    #! _validate_spectra_fields(records, fields)
     grid, offsets = _wavelength_grid_offsets(records, precision=precision)
     _validate_wavelength_alignment(records, grid, offsets, precision=precision)
 
-    # One slice for each record; each slice a 2darray(wavelength, fieldName)=fldVal
-    #! slices = list()
-    #! for rec in records:
-    #!     ar = rec_grid(rec, fields, grid, offsets, precision=None):
-    #!     slices.append(ar)
-
     # One slice for each field; each slice a 2darray(wavelength, record)=fldVal
     adict = dict()
     for fld in fields:

sparcl/resample_spectra.py

@@ -0,0 +1,36 @@
+# NOT INTENDED FOR PUBLIC USE!
+#
+# See:
+#   https://spectres.readthedocs.io/en/latest/
+
+import spectres
+
+# Per paper, should be able to pass all flux in one call to spectres
+# https://arxiv.org/pdf/1705.05165.pdf
+# Perhaps users would rather the bins uniform (1,5,20 Angstroms?)
+def _resample_flux(records, wavstep=1):
+    smallest = math.floor(min([min(r.wavelength) for r in records]))
+    largest = math.ceil(max([max(r.wavelength) for r in records]))
+
+    #!wrange = largest - smallest
+    #new_wavs = np.fromfunction(lambda i: i + smallest, (wrange,), dtype=int)
+    #flux_2d = np.ones([len(records), wrange])
+
+    new_wavs = np.array(range(smallest, largest + 1, wavstep))
+    flux_2d = np.full([len(records), len(new_wavs)], None, dtype=float)
+
+    for idx, rec in enumerate(records):
+        flux_2d[idx] = spectres.spectres(new_wavs,
+                                         rec.wavelength,
+                                         rec.flux,
+                                         verbose=False)
+    return flux_2d, new_wavs
+
+
+def _tt0(numrecs=20):
+    client = sparcl.client.SparclClient()
+    found = client.find(constraints=dict(data_release=['BOSS-DR16']),
+                        limit=numrecs)
+    got = client.retrieve(found.ids)
+    flux_2d, new_wavs = _resample_flux(got.records)
+    return flux_2d, new_wavs

{sparclclient-1.2.0b3.dev9.dist-info → sparclclient-1.2.0b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sparclclient
-Version: 1.2.0b3.dev9
+Version: 1.2.0b4
 Summary: A client for getting spectra data from NOIRLab.
 Home-page: https://github.com/astro-datalab/sparclclient
 Author: NOIRLab DataLab
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests (==2.26.0)
+Requires-Dist: requests (==2.31.0)
 Requires-Dist: numpy
 Requires-Dist: spectres
 

sparclclient-1.2.0b4.dist-info/RECORD

@@ -0,0 +1,20 @@
+sparcl/Results.py,sha256=S4jQcXnw6qfdIEL-NEsCqMK6LolFaERGxjNWIvNhACM,8039
+sparcl/__init__.py,sha256=sLkvMfJ715Hbp_Lo0vJSC0RRqDYMcvrlMXHbJBBce_M,1030
+sparcl/big_retrieve.py,sha256=q0ScH87QqPL4bz4g0hB0AO3k4c_TiuQrWjBJHqHhE60,798
+sparcl/client.py,sha256=2bmwXjLU28Apod7O7ohkwHAvcGRbURAolYq9xERj9HE,31749
+sparcl/conf.py,sha256=O9l4-vpWBZK0QjhHxjskGO8kHPxBj7mkWlchd2rot1c,953
+sparcl/dls_376.py,sha256=WvZjuZFRU0jgH3ELRrMQdslkMWiF2wFQrSag0cYii-I,887
+sparcl/exceptions.py,sha256=q7ONsLsop9OQJJCD4SEzfdsojv0yo3WQT0SluaxGOQ0,3813
+sparcl/fields.py,sha256=7MpaJQr2d1GktS7aeM4010jyLqDdKQ7BZIF9hM0IjII,5002
+sparcl/gather_2d.py,sha256=ZRr41vNHV4tnf63-QuTu04SlWv6TOzK-CeHpbt9YwOY,9254
+sparcl/resample_spectra.py,sha256=2MO-sDCCFg2eNiK6jQs2EJRu4bNnXycGV8WaOydssG4,1329
+sparcl/type_conversion.py,sha256=RX7OD1iGuuUrf-yAd0ISdiqBq4CP7QlCw0vvkAdHdsQ,13112
+sparcl/unsupported.py,sha256=vkSaK3Ppcxx6mMsqBktUjI0uS7RwBJYH2BkBABsnyIM,1867
+sparcl/utils.py,sha256=YlLUP0j4thUyEwTJAaqJ7zzsvbCxPe5EYTn9kvWGfBY,4682
+sparcl/benchmarks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sparcl/benchmarks/benchmarks.py,sha256=FPZ2KExfVWHhGt3B4VyfgOhxxsemj7OeBWJO0dyDDC4,9667
+sparclclient-1.2.0b4.dist-info/LICENSE,sha256=y10EluGMCzGs9X4oYCYyix3l6u-lawB_vlGR8qe442Q,1576
+sparclclient-1.2.0b4.dist-info/METADATA,sha256=NtJ980uIF8tOtRNULNS_Y4UlPDI9Gg3ToLzj31Hnyng,867
+sparclclient-1.2.0b4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+sparclclient-1.2.0b4.dist-info/top_level.txt,sha256=d5CZ3Duxq3MyQTB2ZqOrdtSBv4GdVceF-pOZFmsuHZY,7
+sparclclient-1.2.0b4.dist-info/RECORD,,

sparclclient-1.2.0b3.dev9.dist-info/RECORD

@@ -1,19 +0,0 @@
-sparcl/Results.py,sha256=yHetKpwujeqW2RXloo-_9d3JgTu11VLrwSZzVwqZcJU,8000
-sparcl/__init__.py,sha256=OxAYp-Ca2BNX2OhG2VesOv1QdBPa5TPbkdiaG2xp4gY,1035
-sparcl/big_retrieve.py,sha256=q0ScH87QqPL4bz4g0hB0AO3k4c_TiuQrWjBJHqHhE60,798
-sparcl/client.py,sha256=EbLkZmEnTMUh2AnijIrqIbUalAcm3LQnGM6RkBbTWUM,29684
-sparcl/conf.py,sha256=O9l4-vpWBZK0QjhHxjskGO8kHPxBj7mkWlchd2rot1c,953
-sparcl/dls_376.py,sha256=WvZjuZFRU0jgH3ELRrMQdslkMWiF2wFQrSag0cYii-I,887
-sparcl/exceptions.py,sha256=q7ONsLsop9OQJJCD4SEzfdsojv0yo3WQT0SluaxGOQ0,3813
-sparcl/fields.py,sha256=7MpaJQr2d1GktS7aeM4010jyLqDdKQ7BZIF9hM0IjII,5002
-sparcl/gather_2d.py,sha256=0AOBbjt8orrMg-IgOgIjTruHMFxlUqwgN5WTopab7Ao,9799
-sparcl/type_conversion.py,sha256=RX7OD1iGuuUrf-yAd0ISdiqBq4CP7QlCw0vvkAdHdsQ,13112
-sparcl/unsupported.py,sha256=vkSaK3Ppcxx6mMsqBktUjI0uS7RwBJYH2BkBABsnyIM,1867
-sparcl/utils.py,sha256=YlLUP0j4thUyEwTJAaqJ7zzsvbCxPe5EYTn9kvWGfBY,4682
-sparcl/benchmarks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sparcl/benchmarks/benchmarks.py,sha256=FPZ2KExfVWHhGt3B4VyfgOhxxsemj7OeBWJO0dyDDC4,9667
-sparclclient-1.2.0b3.dev9.dist-info/LICENSE,sha256=y10EluGMCzGs9X4oYCYyix3l6u-lawB_vlGR8qe442Q,1576
-sparclclient-1.2.0b3.dev9.dist-info/METADATA,sha256=6vglH_6at8TrSgsV8eoWxycL0E-zjUfyCqomQ-R3nhA,872
-sparclclient-1.2.0b3.dev9.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-sparclclient-1.2.0b3.dev9.dist-info/top_level.txt,sha256=d5CZ3Duxq3MyQTB2ZqOrdtSBv4GdVceF-pOZFmsuHZY,7
-sparclclient-1.2.0b3.dev9.dist-info/RECORD,,