sparclclient 1.2.4b2__tar.gz → 1.2.5b1__tar.gz

{sparclclient-1.2.4b2 → sparclclient-1.2.5b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sparclclient
-Version: 1.2.4b2
+Version: 1.2.5b1
 Summary: A client for getting spectra and meta-data from NOIRLab.
 Author-email: "S. Pothier" <datalab-spectro@noirlab.edu>
 Description-Content-Type: text/markdown

{sparclclient-1.2.4b2 → sparclclient-1.2.5b1}/sparcl/__init__.py

@@ -33,4 +33,5 @@ __all__ = ["client", "align_records"]
 # FIRST uncommented value will be used! (so only leave one uncommented)
 #__version__ = "1.2.2"
 #__version__ = "1.2.3"
-__version__ = "1.2.4b2"
+#__version__ = "1.2.4"
+__version__ = "1.2.5b1"

sparclclient-1.2.5b1/sparcl/benchmarks/Benchmark_SPARCL_example.ipynb

@@ -0,0 +1,290 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "17a35c00-bc12-4a45-b883-762edde6f1ce",
+   "metadata": {},
+   "source": [
+    "# Example Benchmarking for SPARCL"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f33d932-4fbd-47e9-b390-316c00fcddad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import some helpful python packages \n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import time\n",
+    "from astropy.table import Table\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "## DataLab related modules\n",
+    "from sparcl.client import SparclClient\n",
+    "from dl import queryClient as qc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e5af373-bf6c-45d2-8a96-5a6b0c2a5a4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "## Define here astrosparcl vs. astrosparcl2\n",
+    "client_astrosparcl = SparclClient()\n",
+    "client_astrosparcl2 = SparclClient(url='https://astrosparcl2.datalab.noirlab.edu')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f72f1a5a-8b33-4edb-845c-300e277e6316",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client_astrosparcl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54d1f19c-1772-4967-b45e-d2a17d549f7b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client_astrosparcl2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "85a8075d-2b30-48e1-81bb-d75eceeb096f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run_desi_query(NN, randomid=[0,1]):\n",
+    "\n",
+    "    # Query for DESI; keep only objects with a unique spectrum to avoid duplicates\n",
+    "    query = f'''SELECT targetid FROM desi_dr1.zpix \n",
+    "                WHERE zcat_nspec=1 AND random_id BETWEEN {randomid[0]} AND {randomid[1]} LIMIT {NN}'''\n",
+    "\n",
+    "    t = qc.query(sql=query, fmt='table')\n",
+    "\n",
+    "    return(t)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "85cc1d16-07a5-4d41-849e-572e277e4915",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run_sdss_query(NN, randomid=[0,1]):\n",
+    "\n",
+    "    # Query for SDSS; already unique per specobjid\n",
+    "    query = f'''SELECT specobjid FROM sdss_dr16.specobj \n",
+    "                WHERE random_id BETWEEN {randomid[0]} AND {randomid[1]} LIMIT {NN}'''\n",
+    "\n",
+    "    t = qc.query(sql=query, fmt='table')\n",
+    "\n",
+    "    return(t)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06d97329-3fd7-41b2-a623-2373a4c876d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define the fields to include in the retrieve function (common to SDSS and DESI)\n",
+    "inc = ['specid', 'redshift', 'specprimary', 'wavelength', 'flux', 'ivar']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f2620054-aa25-41e4-adc1-da37b78b5cc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_spec_timing(specids, dataset=['DESI-DR1']):\n",
+    "## Version for astrosparcl\n",
+    "## dataset = 'DESI-DR1' by default, can set 'SDSS-DR16' or others by request\n",
+    "    \n",
+    "    start_time = time.time()\n",
+    "\n",
+    "    ## Retrieve spectra\n",
+    "    results = client_astrosparcl.retrieve_by_specid(specid_list=specids, include=inc, dataset_list=dataset, limit=24000)\n",
+    "    \n",
+    "    ## Time rounded to 0.1 sec\n",
+    "    duration = np.round((time.time() - start_time), 1)\n",
+    "    print(f'Querying N={len(specids)}; retrieved N={results.count} in {duration} sec')\n",
+    "\n",
+    "    ## save space\n",
+    "    results = 0\n",
+    "\n",
+    "    return(duration)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f7382a9-cb26-4a64-a848-ef85d7dc1756",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_spec_timing_2(specids, dataset=['DESI-DR1']):\n",
+    "## Version for astrosparcl2\n",
+    "## dataset = 'DESI-DR1' by default, can set 'SDSS-DR16' or others by request\n",
+    "\n",
+    "    start_time = time.time()\n",
+    "\n",
+    "    ## Retrieve spectra\n",
+    "    results = client_astrosparcl2.retrieve_by_specid(specid_list=specids, include=inc, dataset_list=dataset, limit=24000)\n",
+    "\n",
+    "    ## Time rounded to 0.1 sec\n",
+    "    duration = np.round((time.time() - start_time), 1)\n",
+    "    print(f'Querying N={len(specids)}; retrieved N={results.count} in {duration} sec')\n",
+    "\n",
+    "    ## save space\n",
+    "    results = 0\n",
+    "\n",
+    "    return(duration)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65482bb6-d4f1-42ca-89a3-07e11f93fd2e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define the Nb to retrieve\n",
+    "## Focus on small numbers up to 200\n",
+    "Ns = [1, 2, 4, 20, 40, 200] ##, 2000, 20000] # can extend to 24,000\n",
+    "\n",
+    "out = Table()\n",
+    "out['N'] = Ns\n",
+    "# astrosparcl\n",
+    "out['T_retrieve_DESI'] = 0.0\n",
+    "out['T_retrieve_SDSS'] = 0.0\n",
+    "# astrosparcl2\n",
+    "out['T2_retrieve_DESI'] = 0.0\n",
+    "out['T2_retrieve_SDSS'] = 0.0\n",
+    "\n",
+    "for i, N in enumerate(Ns):\n",
+    "\n",
+    "    # Query for SDSS specobjid's\n",
+    "    ## NOTE: need to change code for randomid for SDSS as fct(N bins) and sample size)\n",
+    "    t_sdss = run_sdss_query(N) #, randomid=[0+i*0.1, 0.3+i*0.1])\n",
+    "    ids = t_sdss['specobjid']\n",
+    "    ids = ids.astype(int).tolist()    \n",
+    "\n",
+    "    # Call retrieve and check timing for SDSS\n",
+    "    out['T_retrieve_SDSS'][i] = get_spec_timing(ids, dataset=['SDSS-DR16','BOSS-DR16'])\n",
+    "    out['T2_retrieve_SDSS'][i] = get_spec_timing_2(ids, dataset=['SDSS-DR16','BOSS-DR16'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "862e8456-d61b-43d1-bf8a-03507c76861b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0bb31825-4a4d-48dd-8e0a-074a41b8c58d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i, N in enumerate(Ns):\n",
+    "    \n",
+    "    # Query for DESI targetid's\n",
+    "    t_desi = run_desi_query(N, randomid=[0+i*0.1, 0.1+i*0.1])\n",
+    "    ids = t_desi['targetid']\n",
+    "    ids = ids.astype(int).tolist()\n",
+    "\n",
+    "    # Call retrieve and check timing\n",
+    "    out['T_retrieve_DESI'][i] = get_spec_timing(ids)\n",
+    "    out['T2_retrieve_DESI'][i] = get_spec_timing_2(ids)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab087231-5e31-4fa5-afa9-1d0361b9962a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "990223ec-eaee-418f-bc7a-61120e33751f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#out_save = out.copy()\n",
+    "#out_save"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43354fc9-6fa9-4186-89d7-792c240f3698",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#outfile = \"results_250616_astrosparcl.fits\"\n",
+    "#out_save['N','T_retrieve_DESI','T_retrieve_SDSS'].write(outfile, overwrite=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a516ce42-82dc-4910-92c4-d2cb34fea012",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#outfile = \"results_250613_astrosparcl2.fits\"\n",
+    "#out_save['N','T2_retrieve_DESI','T2_retrieve_SDSS'].write(outfile, overwrite=False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

sparclclient-1.2.5b1/sparcl/benchmarks/sparcl_benchmarking.ipynb

@@ -0,0 +1,789 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "049ba371-1951-40c3-b874-362814854321",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# SPARCL imports\n",
+    "from sparcl.client import SparclClient\n",
+    "\n",
+    "# 3rd party imports\n",
+    "import numpy as np\n",
+    "import astropy.units as u\n",
+    "from specutils import Spectrum1D\n",
+    "from astropy.nddata import InverseVariance\n",
+    "from astropy.convolution import convolve, Gaussian1DKernel\n",
+    "%matplotlib inline\n",
+    "import matplotlib.pyplot as plt\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Data Lab imports\n",
+    "from dl import queryClient as qc\n",
+    "from dl import authClient as ac\n",
+    "from getpass import getpass\n",
+    "\n",
+    "# plots default setup\n",
+    "plt.rcParams['font.size'] = 14\n",
+    "plt.rcParams['figure.figsize'] = (14,8)\n",
+    "\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "4ba4b161-493b-4b49-ac3b-ea9ee523c3e6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(sparclclient:1.2.4, api:12.0, https://astrosparcl.datalab.noirlab.edu/sparc, client_hash=, verbose=False, connect_timeout=1.1, read_timeout=5400.0)\n",
+      "(sparclclient:1.2.4, api:12.0, https://astrosparcl2.datalab.noirlab.edu/sparc, client_hash=, verbose=False, connect_timeout=1.1, read_timeout=5400.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "client_astrosparcl = SparclClient()\n",
+    "print(client_astrosparcl)\n",
+    "client_astrosparcl2 = SparclClient(url='https://astrosparcl2.datalab.noirlab.edu')\n",
+    "print(client_astrosparcl2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "71ba4082-94ad-4965-a152-b942edf0da9f",
+   "metadata": {},
+   "source": [
+    "# client.find"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1d0d31a0-7415-4d92-b9d2-e6d49bba9d55",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out = ['sparcl_id', 'ra', 'dec', 'redshift', 'spectype', 'data_release']\n",
+    "cons1 = {'spectype': ['GALAXY'], 'redshift': [0.3, 6.2]}\n",
+    "cons2 = {'spectype': ['QSO'], 'redshift': [0.5, 7.1]}\n",
+    "\n",
+    "cons1_1 = {'ra': [93.2,140.3]}\n",
+    "cons2_2 = {'ra': [141.2,191.9]}\n",
+    "\n",
+    "cons11 = {'data_release': ['BOSS-DR16','DESI-DR1']}\n",
+    "cons22 = {'data_release': ['SDSS-DR16','DESI-DR1']}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "125471ce-a32b-44e3-b63e-975c12951829",
+   "metadata": {},
+   "source": [
+    "### client.find: limit=100,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "469f73bc-77fd-40a3-bcfa-7f11c7d8e965",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 981 ms, sys: 129 ms, total: 1.11 s\n",
+      "Wall time: 32.7 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 100000 records"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_1 = client_astrosparcl.find(outfields=out, constraints=cons1, limit=100000)\n",
+    "found_1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "0ddd1ac0-6ef9-46d5-a445-29a9642de73a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 914 ms, sys: 73.7 ms, total: 988 ms\n",
+      "Wall time: 5.47 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 100000 records"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_2 = client_astrosparcl2.find(outfields=out, constraints=cons2, limit=100000)\n",
+    "found_2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "675d12b7-bae2-4f7f-a792-a9aa58e240a4",
+   "metadata": {},
+   "source": [
+    "### client.find: limit=1,000,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "6453c7b9-db57-459d-8ffa-dddc09acd1c6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 8.44 s, sys: 967 ms, total: 9.4 s\n",
+      "Wall time: 51.5 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 1000000 records"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_2 = client_astrosparcl2.find(outfields=out, constraints=cons2, limit=1000000)\n",
+    "found_2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "b38acaba-b481-4de1-a304-33371c7b5522",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 9.03 s, sys: 820 ms, total: 9.85 s\n",
+      "Wall time: 58.8 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 1000000 records"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_1 = client_astrosparcl.find(outfields=out, constraints=cons1, limit=1000000)\n",
+    "found_1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "96455ba7-b56c-4717-8c60-49c3f06059fd",
+   "metadata": {},
+   "source": [
+    "### client.find: limit=10,000,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "63f9da26-eedf-4db7-aee0-7597e9b87503",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1min 36s, sys: 9.11 s, total: 1min 45s\n",
+      "Wall time: 6min 41s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 10000000 records"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_2 = client_astrosparcl2.find(outfields=out, constraints=cons11, limit=10000000)\n",
+    "found_2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "d0b77073-7b6f-4c2e-9be4-9ac07961f107",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1min 35s, sys: 13.2 s, total: 1min 48s\n",
+      "Wall time: 22min 52s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 10000000 records"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_1 = client_astrosparcl.find(outfields=out, constraints=cons22, limit=10000000)\n",
+    "found_1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3d9c9134-6be1-4414-96d3-e05ee3e71912",
+   "metadata": {},
+   "source": [
+    "### client.find: limit=None"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "cc4f3a46-08f8-4846-8143-ee6af634cf8f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 5min, sys: 2min 47s, total: 7min 48s\n",
+      "Wall time: 29min 1s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 30892024 records"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_2 = client_astrosparcl2.find(outfields=out, limit=None)\n",
+    "found_2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "24da8b92-3881-48c6-9936-aed48a68cdb3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 5min 30s, sys: 3min 32s, total: 9min 3s\n",
+      "Wall time: 56min 29s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Find Results: 30892024 records"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "found_1 = client_astrosparcl.find(outfields=out, limit=None)\n",
+    "found_1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65fc6f11-4bb1-4920-8410-0ffc0102a472",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3a51eafa-4a54-4dd2-9b34-46fff075f2d1",
+   "metadata": {},
+   "source": [
+    "# client.retrieve"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "efa34af0-4b22-4316-a8d8-9a4b4f7776fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define the fields to include in the retrieve function\n",
+    "inc = ['sparcl_id', 'specid', 'data_release', 'redshift', 'flux',\n",
+    "       'wavelength', 'spectype', 'ra', 'dec']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "933c543f-88bf-479c-b467-f1945f4d5ea3",
+   "metadata": {},
+   "source": [
+    "### client.retrieve: limit=1,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "bb013ba1-9012-452a-93e2-5290b6cc365e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1.8 s, sys: 1.93 s, total: 3.74 s\n",
+      "Wall time: 50.6 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 1000 records"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_2 = found_2.ids\n",
+    "results_2 = client_astrosparcl2.retrieve(uuid_list=ids_2, include=inc, limit=1000)\n",
+    "results_2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "d2230e68-9532-4819-b621-79d386bcf63c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 3.48 s, sys: 4.02 s, total: 7.5 s\n",
+      "Wall time: 1min 3s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 1000 records"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_1 = found_1.ids\n",
+    "results_1 = client_astrosparcl.retrieve(uuid_list=ids_1, include=inc, limit=1000)\n",
+    "results_1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "91d08444-a450-4103-af86-f75fbadc41da",
+   "metadata": {},
+   "source": [
+    "### client.retrieve: limit=5,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "55efbd37-7010-4e61-b2e0-d79c2c9082d0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 9.88 s, sys: 10.8 s, total: 20.7 s\n",
+      "Wall time: 10min 23s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 5000 records"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_1 = found_1.ids\n",
+    "results_1 = client_astrosparcl.retrieve(uuid_list=ids_1, include=inc, limit=5000)\n",
+    "results_1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "cf624549-2c37-41d0-bd56-17852bd39d6d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 7.77 s, sys: 7.41 s, total: 15.2 s\n",
+      "Wall time: 3min 53s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 5000 records"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_2 = found_2.ids\n",
+    "results_2 = client_astrosparcl2.retrieve(uuid_list=ids_2, include=inc, limit=5000)\n",
+    "results_2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4bedd466-a23f-458c-adaf-441dece2e794",
+   "metadata": {},
+   "source": [
+    "### client.retrieve: limit=10,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "5de660f1-16c1-4a2e-af05-a9a23b3d45b3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 16.5 s, sys: 15.6 s, total: 32.1 s\n",
+      "Wall time: 8min\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 10000 records"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_2 = found_2.ids\n",
+    "results_2 = client_astrosparcl2.retrieve(uuid_list=ids_2, include=inc, limit=10000)\n",
+    "results_2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "be7339da-feec-4d18-8bcb-7d629910a120",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 19.7 s, sys: 19.1 s, total: 38.7 s\n",
+      "Wall time: 21min 19s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 10000 records"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_1 = found_1.ids\n",
+    "results_1 = client_astrosparcl.retrieve(uuid_list=ids_1, include=inc, limit=10000)\n",
+    "results_1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "83446a5c-a01a-42a3-a8e2-9fe1a25371ce",
+   "metadata": {},
+   "source": [
+    "### client.retrieve: limit=20,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4919d147-4dfe-4da6-984b-2b530b976f09",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 38 s, sys: 35.1 s, total: 1min 13s\n",
+      "Wall time: 20min 57s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 20000 records"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_1 = found_1.ids\n",
+    "results_1 = client_astrosparcl.retrieve(uuid_list=ids_1, include=inc, limit=20000)\n",
+    "results_1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "b1d2f32d-0a00-4c08-8950-560196cd3ea2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 31.5 s, sys: 28 s, total: 59.4 s\n",
+      "Wall time: 19min 59s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 20000 records"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_2 = found_2.ids\n",
+    "results_2 = client_astrosparcl2.retrieve(uuid_list=ids_2, include=inc, limit=20000)\n",
+    "results_2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fc0f4354-67ff-45d1-9356-a28856ab1702",
+   "metadata": {},
+   "source": [
+    "### client.retrieve: limit=24,000 (max allowed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "id": "c38f2df1-27bc-457b-b671-06fcc2853909",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 46.6 s, sys: 46 s, total: 1min 32s\n",
+      "Wall time: 15min 37s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 24000 records"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_2 = found_2.ids\n",
+    "results_2 = client_astrosparcl2.retrieve(uuid_list=ids_2, include=inc, limit=24000)\n",
+    "results_2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "8118ecd9-97e2-49f5-aed9-b21dae5a2fc3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 48 s, sys: 46.5 s, total: 1min 34s\n",
+      "Wall time: 55min 34s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Retrieved Results: 24000 records"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "ids_1 = found_1.ids\n",
+    "results_1 = client_astrosparcl.retrieve(uuid_list=ids_1, include=inc, limit=24000)\n",
+    "results_1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f04b5df-fbd8-49b5-9b29-e1b522dfa96d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8d990bf9-5102-4fcc-b16d-6091bf8e0be1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{sparclclient-1.2.4b2 → sparclclient-1.2.5b1}/sparcl/client.py

@@ -337,7 +337,7 @@ class SparclClient:  # was SparclApi()
             return msg
 
         self.set_token_exp()
-        print(f"Logged in successfully with {email=}")
+        print(f"Logged in successfully with email={email}")
         return None
 
     def set_token_exp(self):
@@ -680,6 +680,8 @@ class SparclClient:  # was SparclApi()
         if res.status_code != 200:
             raise Exception(res)
         ret = res.json()
+        if countOnly:
+            return len(ret)
         return ret
         # END missing()
 
@@ -733,6 +735,8 @@ class SparclClient:  # was SparclApi()
         if res.status_code != 200:
             raise Exception(res)
         ret = res.json()
+        if countOnly:
+            return len(ret)
         return ret
         # END missing_specids()
 
@@ -944,7 +948,9 @@ class SparclClient:  # was SparclApi()
                 "spectra/sec)"
             )
             print(f'{meta["status"]}')
-
+        
+        # Format/consolodate the server messages to one message with the count of missing
+        # files
         if len(meta["status"].get("warnings", [])) > 0:
             warnings = meta["status"].get("warnings")
             if verbose:
@@ -957,7 +963,7 @@ class SparclClient:  # was SparclApi()
                     missingcount += int(matches.groups()[0])
 
             # using old style substitution to avoid issue with the {} in the message  # noqa: E501
-            warning_message = missing_message % (missingcount, limit, missingcount)   # noqa: E501
+            warning_message = missing_message % (missingcount, req_num, missingcount)   # noqa: E501
             warn(warning_message, stacklevel=2)
 
         return Retrieved(results, client=self)

{sparclclient-1.2.4b2 → sparclclient-1.2.5b1}/sparcl/exceptions.py

@@ -13,8 +13,11 @@ def genSparclException(response, verbose=False):
     # As of Python 3.10.0.alpha6, python "match" statement could be used
     # instead of if-elif-else.
     # https://docs.python.org/3.10/whatsnew/3.10.html#pep-634-structural-pattern-matching
+
     if status.get("errorCode") == "BADPATH":
         return BadPath(status.get("errorMessage"))
+    elif response.status_code == 429:  # too many requests, throttled
+        return TooManyRequests(status.get("detail"))
     elif status.get("errorCode") == "BADQUERY":
         return BadQuery(status.get("errorMessage"))
     elif status.get("errorCode") == "UNKFIELD":
@@ -64,6 +67,9 @@ class BadPath(BaseSparclException):
 
     error_code = "BADPATH"
 
+class TooManyRequests(BaseSparclException):
+
+    error_code = "TOOMANYREQUESTS"
 
 class BadQuery(BaseSparclException):
     """Bad find constraints."""