sparclclient 1.2.4b3__tar.gz → 1.2.5__tar.gz

{sparclclient-1.2.4b3 → sparclclient-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sparclclient
-Version: 1.2.4b3
+Version: 1.2.5
 Summary: A client for getting spectra and meta-data from NOIRLab.
 Author-email: "S. Pothier" <datalab-spectro@noirlab.edu>
 Description-Content-Type: text/markdown

{sparclclient-1.2.4b3 → sparclclient-1.2.5}/sparcl/__init__.py

@@ -33,4 +33,5 @@ __all__ = ["client", "align_records"]
 # FIRST uncommented value will be used! (so only leave one uncommented)
 #__version__ = "1.2.2"
 #__version__ = "1.2.3"
-__version__ = "1.2.4b3"
+#__version__ = "1.2.4"
+__version__ = "1.2.5"

sparclclient-1.2.5/sparcl/benchmarks/Benchmark_SPARCL_example.ipynb

@@ -0,0 +1,290 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "17a35c00-bc12-4a45-b883-762edde6f1ce",
+   "metadata": {},
+   "source": [
+    "# Example Benchmarking for SPARCL"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f33d932-4fbd-47e9-b390-316c00fcddad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import some helpful python packages \n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import time\n",
+    "from astropy.table import Table\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "## DataLab related modules\n",
+    "from sparcl.client import SparclClient\n",
+    "from dl import queryClient as qc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e5af373-bf6c-45d2-8a96-5a6b0c2a5a4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "## Define here astrosparcl vs. astrosparcl2\n",
+    "client_astrosparcl = SparclClient()\n",
+    "client_astrosparcl2 = SparclClient(url='https://astrosparcl2.datalab.noirlab.edu')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f72f1a5a-8b33-4edb-845c-300e277e6316",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client_astrosparcl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54d1f19c-1772-4967-b45e-d2a17d549f7b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client_astrosparcl2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "85a8075d-2b30-48e1-81bb-d75eceeb096f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run_desi_query(NN, randomid=[0,1]):\n",
+    "\n",
+    "    # Query for DESI; keep only objects with a unique spectrum to avoid duplicates\n",
+    "    query = f'''SELECT targetid FROM desi_dr1.zpix \n",
+    "                WHERE zcat_nspec=1 AND random_id BETWEEN {randomid[0]} AND {randomid[1]} LIMIT {NN}'''\n",
+    "\n",
+    "    t = qc.query(sql=query, fmt='table')\n",
+    "\n",
+    "    return(t)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "85cc1d16-07a5-4d41-849e-572e277e4915",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run_sdss_query(NN, randomid=[0,1]):\n",
+    "\n",
+    "    # Query for SDSS; already unique per specobjid\n",
+    "    query = f'''SELECT specobjid FROM sdss_dr16.specobj \n",
+    "                WHERE random_id BETWEEN {randomid[0]} AND {randomid[1]} LIMIT {NN}'''\n",
+    "\n",
+    "    t = qc.query(sql=query, fmt='table')\n",
+    "\n",
+    "    return(t)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06d97329-3fd7-41b2-a623-2373a4c876d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define the fields to include in the retrieve function (common to SDSS and DESI)\n",
+    "inc = ['specid', 'redshift', 'specprimary', 'wavelength', 'flux', 'ivar']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f2620054-aa25-41e4-adc1-da37b78b5cc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_spec_timing(specids, dataset=['DESI-DR1']):\n",
+    "## Version for astrosparcl\n",
+    "## dataset = 'DESI-DR1' by default, can set 'SDSS-DR16' or others by request\n",
+    "    \n",
+    "    start_time = time.time()\n",
+    "\n",
+    "    ## Retrieve spectra\n",
+    "    results = client_astrosparcl.retrieve_by_specid(specid_list=specids, include=inc, dataset_list=dataset, limit=24000)\n",
+    "    \n",
+    "    ## Time rounded to 0.1 sec\n",
+    "    duration = np.round((time.time() - start_time), 1)\n",
+    "    print(f'Querying N={len(specids)}; retrieved N={results.count} in {duration} sec')\n",
+    "\n",
+    "    ## save space\n",
+    "    results = 0\n",
+    "\n",
+    "    return(duration)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f7382a9-cb26-4a64-a848-ef85d7dc1756",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_spec_timing_2(specids, dataset=['DESI-DR1']):\n",
+    "## Version for astrosparcl2\n",
+    "## dataset = 'DESI-DR1' by default, can set 'SDSS-DR16' or others by request\n",
+    "\n",
+    "    start_time = time.time()\n",
+    "\n",
+    "    ## Retrieve spectra\n",
+    "    results = client_astrosparcl2.retrieve_by_specid(specid_list=specids, include=inc, dataset_list=dataset, limit=24000)\n",
+    "\n",
+    "    ## Time rounded to 0.1 sec\n",
+    "    duration = np.round((time.time() - start_time), 1)\n",
+    "    print(f'Querying N={len(specids)}; retrieved N={results.count} in {duration} sec')\n",
+    "\n",
+    "    ## save space\n",
+    "    results = 0\n",
+    "\n",
+    "    return(duration)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65482bb6-d4f1-42ca-89a3-07e11f93fd2e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define the Nb to retrieve\n",
+    "## Focus on small numbers up to 200\n",
+    "Ns = [1, 2, 4, 20, 40, 200] ##, 2000, 20000] # can extend to 24,000\n",
+    "\n",
+    "out = Table()\n",
+    "out['N'] = Ns\n",
+    "# astrosparcl\n",
+    "out['T_retrieve_DESI'] = 0.0\n",
+    "out['T_retrieve_SDSS'] = 0.0\n",
+    "# astrosparcl2\n",
+    "out['T2_retrieve_DESI'] = 0.0\n",
+    "out['T2_retrieve_SDSS'] = 0.0\n",
+    "\n",
+    "for i, N in enumerate(Ns):\n",
+    "\n",
+    "    # Query for SDSS specobjid's\n",
+    "    ## NOTE: need to change code for randomid for SDSS as fct(N bins) and sample size)\n",
+    "    t_sdss = run_sdss_query(N) #, randomid=[0+i*0.1, 0.3+i*0.1])\n",
+    "    ids = t_sdss['specobjid']\n",
+    "    ids = ids.astype(int).tolist()    \n",
+    "\n",
+    "    # Call retrieve and check timing for SDSS\n",
+    "    out['T_retrieve_SDSS'][i] = get_spec_timing(ids, dataset=['SDSS-DR16','BOSS-DR16'])\n",
+    "    out['T2_retrieve_SDSS'][i] = get_spec_timing_2(ids, dataset=['SDSS-DR16','BOSS-DR16'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "862e8456-d61b-43d1-bf8a-03507c76861b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0bb31825-4a4d-48dd-8e0a-074a41b8c58d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i, N in enumerate(Ns):\n",
+    "    \n",
+    "    # Query for DESI targetid's\n",
+    "    t_desi = run_desi_query(N, randomid=[0+i*0.1, 0.1+i*0.1])\n",
+    "    ids = t_desi['targetid']\n",
+    "    ids = ids.astype(int).tolist()\n",
+    "\n",
+    "    # Call retrieve and check timing\n",
+    "    out['T_retrieve_DESI'][i] = get_spec_timing(ids)\n",
+    "    out['T2_retrieve_DESI'][i] = get_spec_timing_2(ids)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab087231-5e31-4fa5-afa9-1d0361b9962a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "990223ec-eaee-418f-bc7a-61120e33751f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#out_save = out.copy()\n",
+    "#out_save"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43354fc9-6fa9-4186-89d7-792c240f3698",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#outfile = \"results_250616_astrosparcl.fits\"\n",
+    "#out_save['N','T_retrieve_DESI','T_retrieve_SDSS'].write(outfile, overwrite=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a516ce42-82dc-4910-92c4-d2cb34fea012",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#outfile = \"results_250613_astrosparcl2.fits\"\n",
+    "#out_save['N','T2_retrieve_DESI','T2_retrieve_SDSS'].write(outfile, overwrite=False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}