sdss-almanac 0.2.8__tar.gz → 0.2.9__tar.gz

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdss-almanac
-Version: 0.2.8
+Version: 0.2.9
 Summary: Everything we've got
 Author-email: Andy Casey <andrew.casey@monash.edu>
 License: BSD 3-Clause License
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Topic :: Documentation :: Sphinx
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.8
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: numpy

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sdss-almanac"
-version = "0.2.8"
+version = "0.2.9"
 authors = [
     { name = "Andy Casey", email = "andrew.casey@monash.edu" }
 ]
 description = "Everything we've got"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 license = { text = "BSD 3-Clause License" }
 keywords = ["astronomy", "software"]
 classifiers = [

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/src/almanac/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.2.8"
+__version__ = "0.2.9"
 
 from .config import config, get_config_path, ConfigManager
 from .logger import logger

sdss_almanac-0.2.9/src/almanac/catalog.py

@@ -0,0 +1,262 @@
+from itertools import batched
+from typing import List
+from peewee import JOIN
+
+def query_catalog(sdss_ids: List[int], batch_size: int = 1000):
+    """
+    Query the SDSS database for photometry and astrometry.
+    """
+
+    sdss_ids = sorted(list(set(sdss_ids).difference({0, -1})))
+
+    from almanac.database import is_database_available, catalogdb as db
+
+    for batch in batched(sdss_ids, batch_size):
+        q = (
+            db
+            .SDSS_ID_To_Catalog
+            .select(
+                db.SDSS_ID_To_Catalog.sdss_id,
+                db.SDSS_ID_To_Catalog.catalogid,
+                db.SDSS_ID_To_Catalog.version_id,
+                db.SDSS_ID_To_Catalog.lead,
+                db.SDSS_ID_To_Catalog.allstar_dr17_synspec_rev1,
+                db.SDSS_ID_To_Catalog.allwise,
+                db.SDSS_ID_To_Catalog.bhm_rm_v0,
+                db.SDSS_ID_To_Catalog.bhm_rm_v0_2,
+                db.SDSS_ID_To_Catalog.catwise,
+                db.SDSS_ID_To_Catalog.catwise2020,
+                db.SDSS_ID_To_Catalog.gaia_dr2_source,
+                db.SDSS_ID_To_Catalog.gaia_dr3_source,
+                db.SDSS_ID_To_Catalog.glimpse,
+                db.SDSS_ID_To_Catalog.guvcat,
+                db.SDSS_ID_To_Catalog.legacy_survey_dr10,
+                db.SDSS_ID_To_Catalog.legacy_survey_dr8,
+                db.SDSS_ID_To_Catalog.mangatarget,
+                db.SDSS_ID_To_Catalog.marvels_dr11_star,
+                db.SDSS_ID_To_Catalog.marvels_dr12_star,
+                db.SDSS_ID_To_Catalog.mastar_goodstars,
+                db.SDSS_ID_To_Catalog.panstarrs1,
+                db.SDSS_ID_To_Catalog.ps1_g18,
+                db.SDSS_ID_To_Catalog.sdss_dr13_photoobj,
+                db.SDSS_ID_To_Catalog.sdss_dr17_specobj,
+                db.SDSS_ID_To_Catalog.skymapper_dr1_1,
+                db.SDSS_ID_To_Catalog.skymapper_dr2,
+                db.SDSS_ID_To_Catalog.supercosmos,
+                db.SDSS_ID_To_Catalog.tic_v8,
+                db.SDSS_ID_To_Catalog.twomass_psc,
+                db.SDSS_ID_To_Catalog.tycho2,
+                db.SDSS_ID_To_Catalog.unwise,
+                db.Gaia_DR3.source_id,
+                db.Gaia_DR3.solution_id,
+                db.Gaia_DR3.designation,
+                db.Gaia_DR3.random_index,
+                db.Gaia_DR3.ref_epoch,
+                db.Gaia_DR3.ra,
+                db.Gaia_DR3.ra_error,
+                db.Gaia_DR3.dec,
+                db.Gaia_DR3.dec_error,
+                db.Gaia_DR3.parallax,
+                db.Gaia_DR3.parallax_error,
+                db.Gaia_DR3.parallax_over_error,
+                db.Gaia_DR3.pm,
+                db.Gaia_DR3.pmra,
+                db.Gaia_DR3.pmra_error,
+                db.Gaia_DR3.pmdec,
+                db.Gaia_DR3.pmdec_error,
+                db.Gaia_DR3.ra_dec_corr,
+                db.Gaia_DR3.ra_parallax_corr,
+                db.Gaia_DR3.ra_pmra_corr,
+                db.Gaia_DR3.ra_pmdec_corr,
+                db.Gaia_DR3.dec_parallax_corr,
+                db.Gaia_DR3.dec_pmra_corr,
+                db.Gaia_DR3.dec_pmdec_corr,
+                db.Gaia_DR3.parallax_pmra_corr,
+                db.Gaia_DR3.parallax_pmdec_corr,
+                db.Gaia_DR3.pmra_pmdec_corr,
+                db.Gaia_DR3.astrometric_n_obs_al,
+                db.Gaia_DR3.astrometric_n_obs_ac,
+                db.Gaia_DR3.astrometric_n_good_obs_al,
+                db.Gaia_DR3.astrometric_n_bad_obs_al,
+                db.Gaia_DR3.astrometric_gof_al,
+                db.Gaia_DR3.astrometric_chi2_al,
+                db.Gaia_DR3.astrometric_excess_noise,
+                db.Gaia_DR3.astrometric_excess_noise_sig,
+                db.Gaia_DR3.astrometric_params_solved,
+                db.Gaia_DR3.astrometric_primary_flag,
+                db.Gaia_DR3.nu_eff_used_in_astrometry,
+                db.Gaia_DR3.pseudocolour,
+                db.Gaia_DR3.pseudocolour_error,
+                db.Gaia_DR3.ra_pseudocolour_corr,
+                db.Gaia_DR3.dec_pseudocolour_corr,
+                db.Gaia_DR3.parallax_pseudocolour_corr,
+                db.Gaia_DR3.pmra_pseudocolour_corr,
+                db.Gaia_DR3.pmdec_pseudocolour_corr,
+                db.Gaia_DR3.astrometric_matched_transits,
+                db.Gaia_DR3.visibility_periods_used,
+                db.Gaia_DR3.astrometric_sigma5d_max,
+                db.Gaia_DR3.matched_transits,
+                db.Gaia_DR3.new_matched_transits,
+                db.Gaia_DR3.matched_transits_removed,
+                db.Gaia_DR3.ipd_gof_harmonic_amplitude,
+                db.Gaia_DR3.ipd_gof_harmonic_phase,
+                db.Gaia_DR3.ipd_frac_multi_peak,
+                db.Gaia_DR3.ipd_frac_odd_win,
+                db.Gaia_DR3.ruwe,
+                db.Gaia_DR3.scan_direction_strength_k1,
+                db.Gaia_DR3.scan_direction_strength_k2,
+                db.Gaia_DR3.scan_direction_strength_k3,
+                db.Gaia_DR3.scan_direction_strength_k4,
+                db.Gaia_DR3.scan_direction_mean_k1,
+                db.Gaia_DR3.scan_direction_mean_k2,
+                db.Gaia_DR3.scan_direction_mean_k3,
+                db.Gaia_DR3.scan_direction_mean_k4,
+                db.Gaia_DR3.duplicated_source,
+                db.Gaia_DR3.phot_g_n_obs,
+                db.Gaia_DR3.phot_g_mean_flux,
+                db.Gaia_DR3.phot_g_mean_flux_error,
+                db.Gaia_DR3.phot_g_mean_flux_over_error,
+                db.Gaia_DR3.phot_g_mean_mag,
+                db.Gaia_DR3.phot_bp_n_obs,
+                db.Gaia_DR3.phot_bp_mean_flux,
+                db.Gaia_DR3.phot_bp_mean_flux_error,
+                db.Gaia_DR3.phot_bp_mean_flux_over_error,
+                db.Gaia_DR3.phot_bp_mean_mag,
+                db.Gaia_DR3.phot_rp_n_obs,
+                db.Gaia_DR3.phot_rp_mean_flux,
+                db.Gaia_DR3.phot_rp_mean_flux_error,
+                db.Gaia_DR3.phot_rp_mean_flux_over_error,
+                db.Gaia_DR3.phot_rp_mean_mag,
+                db.Gaia_DR3.phot_bp_rp_excess_factor,
+                db.Gaia_DR3.phot_bp_n_contaminated_transits,
+                db.Gaia_DR3.phot_bp_n_blended_transits,
+                db.Gaia_DR3.phot_rp_n_contaminated_transits,
+                db.Gaia_DR3.phot_rp_n_blended_transits,
+                db.Gaia_DR3.phot_proc_mode,
+                db.Gaia_DR3.bp_rp,
+                db.Gaia_DR3.bp_g,
+                db.Gaia_DR3.g_rp,
+                db.Gaia_DR3.radial_velocity,
+                db.Gaia_DR3.radial_velocity_error,
+                db.Gaia_DR3.rv_method_used,
+                db.Gaia_DR3.rv_nb_transits,
+                db.Gaia_DR3.rv_nb_deblended_transits,
+                db.Gaia_DR3.rv_visibility_periods_used,
+                db.Gaia_DR3.rv_expected_sig_to_noise,
+                db.Gaia_DR3.rv_renormalised_gof,
+                db.Gaia_DR3.rv_chisq_pvalue,
+                db.Gaia_DR3.rv_time_duration,
+                db.Gaia_DR3.rv_amplitude_robust,
+                db.Gaia_DR3.rv_template_teff,
+                db.Gaia_DR3.rv_template_logg,
+                db.Gaia_DR3.rv_template_fe_h,
+                db.Gaia_DR3.rv_atm_param_origin,
+                db.Gaia_DR3.vbroad,
+                db.Gaia_DR3.vbroad_error,
+                db.Gaia_DR3.vbroad_nb_transits,
+                db.Gaia_DR3.grvs_mag,
+                db.Gaia_DR3.grvs_mag_error,
+                db.Gaia_DR3.grvs_mag_nb_transits,
+                db.Gaia_DR3.rvs_spec_sig_to_noise,
+                db.Gaia_DR3.phot_variable_flag,
+                db.Gaia_DR3.l,
+                db.Gaia_DR3.b,
+                db.Gaia_DR3.ecl_lon,
+                db.Gaia_DR3.ecl_lat,
+                db.Gaia_DR3.in_qso_candidates,
+                db.Gaia_DR3.in_galaxy_candidates,
+                db.Gaia_DR3.non_single_star,
+                db.Gaia_DR3.has_xp_continuous,
+                db.Gaia_DR3.has_xp_sampled,
+                db.Gaia_DR3.has_rvs,
+                db.Gaia_DR3.has_epoch_photometry,
+                db.Gaia_DR3.has_epoch_rv,
+                db.Gaia_DR3.has_mcmc_gspphot,
+                db.Gaia_DR3.has_mcmc_msc,
+                db.Gaia_DR3.in_andromeda_survey,
+                db.Gaia_DR3.classprob_dsc_combmod_quasar,
+                db.Gaia_DR3.classprob_dsc_combmod_galaxy,
+                db.Gaia_DR3.classprob_dsc_combmod_star,
+                db.Gaia_DR3.teff_gspphot,
+                db.Gaia_DR3.teff_gspphot_lower,
+                db.Gaia_DR3.teff_gspphot_upper,
+                db.Gaia_DR3.logg_gspphot,
+                db.Gaia_DR3.logg_gspphot_lower,
+                db.Gaia_DR3.logg_gspphot_upper,
+                db.Gaia_DR3.mh_gspphot,
+                db.Gaia_DR3.mh_gspphot_lower,
+                db.Gaia_DR3.mh_gspphot_upper,
+                db.Gaia_DR3.distance_gspphot,
+                db.Gaia_DR3.distance_gspphot_lower,
+                db.Gaia_DR3.distance_gspphot_upper,
+                db.Gaia_DR3.azero_gspphot,
+                db.Gaia_DR3.azero_gspphot_lower,
+                db.Gaia_DR3.azero_gspphot_upper,
+                db.Gaia_DR3.ag_gspphot,
+                db.Gaia_DR3.ag_gspphot_lower,
+                db.Gaia_DR3.ag_gspphot_upper,
+                db.Gaia_DR3.ebpminrp_gspphot,
+                db.Gaia_DR3.ebpminrp_gspphot_lower,
+                db.Gaia_DR3.ebpminrp_gspphot_upper,
+                db.Gaia_DR3.libname_gspphot,
+                db.TwoMassPSC.designation,
+                db.TwoMassPSC.j_m,
+                db.TwoMassPSC.j_cmsig,
+                db.TwoMassPSC.j_msigcom,
+                db.TwoMassPSC.j_snr,
+                db.TwoMassPSC.h_m,
+                db.TwoMassPSC.h_cmsig,
+                db.TwoMassPSC.h_msigcom,
+                db.TwoMassPSC.h_snr,
+                db.TwoMassPSC.k_m,
+                db.TwoMassPSC.k_cmsig,
+                db.TwoMassPSC.k_msigcom,
+                db.TwoMassPSC.k_snr,
+                db.TwoMassPSC.ph_qual,
+                db.TwoMassPSC.rd_flg,
+                db.TwoMassPSC.bl_flg,
+                db.TwoMassPSC.cc_flg,
+                db.TwoMassPSC.ndet,
+                db.TwoMassPSC.prox,
+                db.TwoMassPSC.pxpa,
+                db.TwoMassPSC.pxcntr,
+                db.TwoMassPSC.gal_contam,
+                db.TwoMassPSC.mp_flg,
+                db.TwoMassPSC.scan,
+                db.TwoMassPSC.glon,
+                db.TwoMassPSC.glat,
+                db.TwoMassPSC.x_scan,
+                db.TwoMassPSC.jdate,
+                db.TwoMassPSC.j_psfchi,
+                db.TwoMassPSC.h_psfchi,
+                db.TwoMassPSC.k_psfchi,
+                db.TwoMassPSC.j_m_stdap,
+                db.TwoMassPSC.j_msig_stdap,
+                db.TwoMassPSC.h_m_stdap,
+                db.TwoMassPSC.h_msig_stdap,
+                db.TwoMassPSC.k_m_stdap,
+                db.TwoMassPSC.k_msig_stdap,
+                db.TwoMassPSC.dist_edge_ns,
+                db.TwoMassPSC.dist_edge_ew,
+                db.TwoMassPSC.dist_edge_flg,
+                db.TwoMassPSC.dup_src,
+                db.TwoMassPSC.use_src,
+                db.TwoMassPSC.a,
+                db.TwoMassPSC.dist_opt,
+                db.TwoMassPSC.phi_opt,
+                db.TwoMassPSC.b_m_opt,
+                db.TwoMassPSC.vr_m_opt,
+                db.TwoMassPSC.nopt_mchs,
+                db.TwoMassPSC.ext_key,
+                db.TwoMassPSC.scan_key,
+                db.TwoMassPSC.coadd_key,
+                db.TwoMassPSC.coadd
+            )
+            .distinct(db.SDSS_ID_To_Catalog.sdss_id)
+            .join(db.Gaia_DR3, join_type=JOIN.LEFT_OUTER)
+            .switch(db.SDSS_ID_To_Catalog)
+            .join(db.TwoMassPSC, join_type=JOIN.LEFT_OUTER)
+            .where(db.SDSS_ID_To_Catalog.sdss_id.in_(tuple(batch)))
+            .dicts()
+        )
+        yield from q.iterator()

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/src/almanac/cli.py

@@ -203,29 +203,58 @@ def add(**kwargs):
 
 @add.command()
 @click.argument("input_path", type=str)
-@click.option("--mjd", default=None, type=int, help="Modified Julian Date")
-@click.option("--apo", is_flag=True, help="Apache Point Observatory")
-@click.option("--lco", is_flag=True, help="Las Campanas Observatory")
-def metadata(input_path, mjd, apo, lco, **kwargs):
-    """Add photometry and astrometry to an existing Almanac file."""
+@click.option("--mjd", default=None, type=int, help="Modified Julian date to query. Use negative values to indicate relative to current MJD")
+@click.option("--mjd-start", default=None, type=int, help="Start of MJD range to query")
+@click.option("--mjd-end", default=None, type=int, help="End of MJD range to query")
+@click.option("--date", default=None, type=str, help="Date to query (e.g., 2024-01-15)")
+@click.option("--date-start", default=None, type=str, help="Start of date range to query")
+@click.option("--date-end", default=None, type=str, help="End of date range to query")
+@click.option("--apo", is_flag=True, help="Query Apache Point Observatory data")
+@click.option("--lco", is_flag=True, help="Query Las Campanas Observatory data")
+def metadata(input_path, mjd, mjd_start, mjd_end, date, date_start, date_end, apo, lco, **kwargs):
+    """Add astrometry and photometry to an existing Almanac file."""
 
     import numpy as np
+    import h5py as h5
+    from itertools import product
     from almanac import utils
     from almanac.catalog import query_catalog
+    from almanac.data_models.metadata import SourceMetadata
+    from tqdm import tqdm
 
     observatories = utils.get_observatories(apo, lco)
+    mjds, *_ = utils.parse_mjds(
+        mjd, mjd_start, mjd_end, date, date_start, date_end, return_nones=True
+    )
+
+    sdss_ids = set()
+    with h5.File(input_path, "r") as fp:
+        if mjds is None:
+            mjds = []
+            for obs in observatories:
+                mjds.extend(fp[obs])
+            mjds = list(set(mjds))
+
+        for mjd, obs in product(mjds, observatories):
+            group = fp.get(f"{obs}/{mjd}/fibers", [])
+            for config in group:
+                if "source_id" in group[config]:
+                    continue
+                sdss_ids.update(group[config]["sdss_id"][:])
+
+    v = []
+    for row in tqdm(query_catalog(sdss_ids), total=len(sdss_ids)):
+        v.append(row)
+
+
 
-    # Open the input file and get all possible sdss_ids
-    import h5py as h5
-    from tqdm import tqdm
-    with h5.File(input_path, "r+") as fp:
-        sdss_ids = dict()
 
         if mjd is None:
             total = sum([len(fp[obs].keys()) for obs in observatories])
         else:
             total = 1 * len(observatories)
 
+        sdss_ids = dict()
         with tqdm(total=total, desc="Collecting SDSS identifiers") as pb:
             for observatory in observatories:
                 if observatory not in fp:
@@ -234,11 +263,10 @@ def metadata(input_path, mjd, apo, lco, **kwargs):
                 mjds = fp[observatory].keys() if mjd is None else [str(mjd)]
 
                 for mjd in mjds:
-                    if mjd not in fp[observatory]:
-                        continue
                     group = fp[f"{observatory}/{mjd}"]
                     if "fibers" not in group:
                         continue
+
                     for config_id in group["fibers"]:
                         config_group = group[f"fibers/{config_id}"]
 

sdss_almanac-0.2.9/src/almanac/data_models/metadata.py

@@ -0,0 +1,247 @@
+from pydantic import BaseModel, Field, computed_field, field_validator, model_validator
+from almanac.data_models.types import *
+
+class SourceMetadata(BaseModel):
+
+    # Identifiers
+    sdss_id: Int64 = Field(default=-1)
+    catalogid: Int64 = Field(default=-1)
+    version_id: int = Field(default=-1)
+    lead: str = Field(default="")
+    allstar_dr17_synspec_rev1: Str = Field(default=-1)
+    allwise: Int64 = Field(default=-1)
+    bhm_rm_v0: Int64 = Field(default=-1)
+    bhm_rm_v0_2: Int64 = Field(default=-1)
+    catwise: Str = Field(default=-1)
+    catwise2020: Str = Field(default=-1)
+    gaia_dr2_source: Int64 = Field(default=-1)
+    gaia_dr3_source: Int64 = Field(default=-1)
+    glimpse: Int64 = Field(default=-1)
+    guvcat: Int64 = Field(default=-1)
+    legacy_survey_dr10: Int64 = Field(default=-1)
+    legacy_survey_dr8: Int64 = Field(default=-1)
+    mangatarget: Str = Field(default=-1)
+    marvels_dr11_star: Str = Field(default=-1)
+    marvels_dr12_star: Int64 = Field(default=-1)
+    mastar_goodstars: Str = Field(default=-1)
+    panstarrs1: Int64 = Field(default=-1)
+    ps1_g18: Int64 = Field(default=-1)
+    sdss_dr13_photoobj: Int64 = Field(default=-1)
+    sdss_dr17_specobj: Str = Field(default=-1)
+    skymapper_dr1_1: Int64 = Field(default=-1)
+    skymapper_dr2: Int64 = Field(default=-1)
+    supercosmos: Int64 = Field(default=-1)
+    tic_v8: Int64 = Field(default=-1)
+    twomass_psc: Int64 = Field(default=-1)
+    tycho2: Str = Field(default=-1)
+    unwise: Str = Field(default=-1)
+
+    # Gaia
+    source_id: Int = Field(default=-1)
+    solution_id: Int = Field(default=-1)
+    designation: Str = Field(default=-1)
+    random_index: Float = Field(default=-1)
+    ref_epoch: Float = Field(default=-1)
+    ra: Float = Field(default=-1)
+    ra_error: Float = Field(default=-1)
+    dec: Float = Field(default=-1)
+    dec_error: Float = Field(default=-1)
+    parallax: Float = Field(default=-1)
+    parallax_error: Float = Field(default=-1)
+    parallax_over_error: Float = Field(default=-1)
+    pm: Float = Field(default=-1)
+    pmra: Float = Field(default=-1)
+    pmra_error: Float = Field(default=-1)
+    pmdec: Float = Field(default=-1)
+    pmdec_error: Float = Field(default=-1)
+    ra_dec_corr: Float = Field(default=-1)
+    ra_parallax_corr: Float = Field(default=-1)
+    ra_pmra_corr: Float = Field(default=-1)
+    ra_pmdec_corr: Float = Field(default=-1)
+    dec_parallax_corr: Float = Field(default=-1)
+    dec_pmra_corr: Float = Field(default=-1)
+    dec_pmdec_corr: Float = Field(default=-1)
+    parallax_pmra_corr: Float = Field(default=-1)
+    parallax_pmdec_corr: Float = Field(default=-1)
+    pmra_pmdec_corr: Float = Field(default=-1)
+    astrometric_n_obs_al: Int = Field(default=-1)
+    astrometric_n_obs_ac: Int = Field(default=-1)
+    astrometric_n_good_obs_al: Int = Field(default=-1)
+    astrometric_n_bad_obs_al: Int = Field(default=-1)
+    astrometric_gof_al: Float = Field(default=-1)
+    astrometric_chi2_al: Float = Field(default=-1)
+    astrometric_excess_noise: Float = Field(default=-1)
+    astrometric_excess_noise_sig: Float = Field(default=-1)
+    astrometric_params_solved: Int = Field(default=-1)
+    astrometric_primary_flag: Bool = Field(default=-1)
+    nu_eff_used_in_astrometry: Float = Field(default=-1)
+    pseudocolour: Float = Field(default=-1)
+    pseudocolour_error: Float = Field(default=-1)
+    ra_pseudocolour_corr: Float = Field(default=-1)
+    dec_pseudocolour_corr: Float = Field(default=-1)
+    parallax_pseudocolour_corr: Float = Field(default=-1)
+    pmra_pseudocolour_corr: Float = Field(default=-1)
+    pmdec_pseudocolour_corr: Float = Field(default=-1)
+    astrometric_matched_transits: Int = Field(default=-1)
+    visibility_periods_used: Int = Field(default=-1)
+    astrometric_sigma5d_max: Float = Field(default=-1)
+    matched_transits: Int = Field(default=-1)
+    new_matched_transits: Int = Field(default=-1)
+    matched_transits_removed: Int = Field(default=-1)
+    ipd_gof_harmonic_amplitude: Float = Field(default=-1)
+    ipd_gof_harmonic_phase: Float = Field(default=-1)
+    ipd_frac_multi_peak: Float = Field(default=-1)
+    ipd_frac_odd_win: Float = Field(default=-1)
+    ruwe: Float = Field(default=-1)
+    scan_direction_strength_k1: Float = Field(default=-1)
+    scan_direction_strength_k2: Float = Field(default=-1)
+    scan_direction_strength_k3: Float = Field(default=-1)
+    scan_direction_strength_k4: Float = Field(default=-1)
+    scan_direction_mean_k1: Float = Field(default=-1)
+    scan_direction_mean_k2: Float = Field(default=-1)
+    scan_direction_mean_k3: Float = Field(default=-1)
+    scan_direction_mean_k4: Float = Field(default=-1)
+    duplicated_source: Bool = Field(default=-1)
+    phot_g_n_obs: Int = Field(default=-1)
+    phot_g_mean_flux: Float = Field(default=-1)
+    phot_g_mean_flux_error: Float = Field(default=-1)
+    phot_g_mean_flux_over_error: Float = Field(default=-1)
+    phot_g_mean_mag: Float = Field(default=-1)
+    phot_bp_n_obs: Int = Field(default=-1)
+    phot_bp_mean_flux: Float = Field(default=-1)
+    phot_bp_mean_flux_error: Float = Field(default=-1)
+    phot_bp_mean_flux_over_error: Float = Field(default=-1)
+    phot_bp_mean_mag: Float = Field(default=-1)
+    phot_rp_n_obs: Int = Field(default=-1)
+    phot_rp_mean_flux: Float = Field(default=-1)
+    phot_rp_mean_flux_error: Float = Field(default=-1)
+    phot_rp_mean_flux_over_error: Float = Field(default=-1)
+    phot_rp_mean_mag: Float = Field(default=-1)
+    phot_bp_rp_excess_factor: Float = Field(default=-1)
+    phot_bp_n_contaminated_transits: Int = Field(default=-1)
+    phot_bp_n_blended_transits: Int = Field(default=-1)
+    phot_rp_n_contaminated_transits: Int = Field(default=-1)
+    phot_rp_n_blended_transits: Int = Field(default=-1)
+    phot_proc_mode: Int = Field(default=-1)
+    bp_rp: Float = Field(default=-1)
+    bp_g: Float = Field(default=-1)
+    g_rp: Float = Field(default=-1)
+    radial_velocity: Float = Field(default=-1)
+    radial_velocity_error: Float = Field(default=-1)
+    rv_method_used: Str = Field(default=-1)
+    rv_nb_transits: Int = Field(default=-1)
+    rv_nb_deblended_transits: Int = Field(default=-1)
+    rv_visibility_periods_used: Int = Field(default=-1)
+    rv_expected_sig_to_noise: Float = Field(default=-1)
+    rv_renormalised_gof: Float = Field(default=-1)
+    rv_chisq_pvalue: Float = Field(default=-1)
+    rv_time_duration: Float = Field(default=-1)
+    rv_amplitude_robust: Float = Field(default=-1)
+    rv_template_teff: Float = Field(default=-1)
+    rv_template_logg: Float = Field(default=-1)
+    rv_template_fe_h: Float = Field(default=-1)
+    rv_atm_param_origin: Str = Field(default=-1)
+    vbroad: Float = Field(default=-1)
+    vbroad_error: Float = Field(default=-1)
+    vbroad_nb_transits: Int = Field(default=-1)
+    grvs_mag: Float = Field(default=-1)
+    grvs_mag_error: Float = Field(default=-1)
+    grvs_mag_nb_transits: Int = Field(default=-1)
+    rvs_spec_sig_to_noise: Float = Field(default=-1)
+    phot_variable_flag: Str = Field(default=-1)
+    l: Float = Field(default=-1)
+    b: Float = Field(default=-1)
+    ecl_lon: Float = Field(default=-1)
+    ecl_lat: Float = Field(default=-1)
+    in_qso_candidates: Bool = Field(default=-1)
+    in_galaxy_candidates: Bool = Field(default=-1)
+    non_single_star: Bool = Field(default=-1)
+    has_xp_continuous: Bool = Field(default=-1)
+    has_xp_sampled: Bool = Field(default=-1)
+    has_rvs: Bool = Field(default=-1)
+    has_epoch_photometry: Bool = Field(default=-1)
+    has_epoch_rv: Bool = Field(default=-1)
+    has_mcmc_gspphot: Bool = Field(default=-1)
+    has_mcmc_msc: Bool = Field(default=-1)
+    in_andromeda_survey: Bool = Field(default=-1)
+    classprob_dsc_combmod_quasar: Float = Field(default=-1)
+    classprob_dsc_combmod_galaxy: Float = Field(default=-1)
+    classprob_dsc_combmod_star: Float = Field(default=-1)
+    teff_gspphot: Float = Field(default=-1)
+    teff_gspphot_lower: Float = Field(default=-1)
+    teff_gspphot_upper: Float = Field(default=-1)
+    logg_gspphot: Float = Field(default=-1)
+    logg_gspphot_lower: Float = Field(default=-1)
+    logg_gspphot_upper: Float = Field(default=-1)
+    mh_gspphot: Float = Field(default=-1)
+    mh_gspphot_lower: Float = Field(default=-1)
+    mh_gspphot_upper: Float = Field(default=-1)
+    distance_gspphot: Float = Field(default=-1)
+    distance_gspphot_lower: Float = Field(default=-1)
+    distance_gspphot_upper: Float = Field(default=-1)
+    azero_gspphot: Float = Field(default=-1)
+    azero_gspphot_lower: Float = Field(default=-1)
+    azero_gspphot_upper: Float = Field(default=-1)
+    ag_gspphot: Float = Field(default=-1)
+    ag_gspphot_lower: Float = Field(default=-1)
+    ag_gspphot_upper: Float = Field(default=-1)
+    ebpminrp_gspphot: Float = Field(default=-1)
+    ebpminrp_gspphot_lower: Float = Field(default=-1)
+    ebpminrp_gspphot_upper: Float = Field(default=-1)
+    libname_gspphot: Str = Field(default=-1)
+    j_m: Float = Field(default=-1)
+    j_cmsig: Float = Field(default=-1)
+    j_msigcom: Float = Field(default=-1)
+    j_snr: Float = Field(default=-1)
+    h_m: Float = Field(default=-1)
+    h_cmsig: Float = Field(default=-1)
+    h_msigcom: Float = Field(default=-1)
+    h_snr: Float = Field(default=-1)
+    k_m: Float = Field(default=-1)
+    k_cmsig: Float = Field(default=-1)
+    k_msigcom: Float = Field(default=-1)
+    k_snr: Float = Field(default=-1)
+    ph_qual: Str = Field(default=-1)
+    rd_flg: Str = Field(default=-1)
+    bl_flg: Str = Field(default=-1)
+    cc_flg: Str = Field(default=-1)
+    ndet: Int = Field(default=-1)
+    prox: Float = Field(default=-1)
+    pxpa: Float = Field(default=-1)
+    pxcntr: Float = Field(default=-1)
+    gal_contam: Int = Field(default=-1)
+    mp_flg: Int = Field(default=-1)
+    scan: Int = Field(default=-1)
+    glon: Float = Field(default=-1)
+    glat: Float = Field(default=-1)
+    x_scan: Float = Field(default=-1)
+    jdate: Float = Field(default=-1)
+    j_psfchi: Float = Field(default=-1)
+    h_psfchi: Float = Field(default=-1)
+    k_psfchi: Float = Field(default=-1)
+    j_m_stdap: Float = Field(default=-1)
+    j_msig_stdap: Float = Field(default=-1)
+    h_m_stdap: Float = Field(default=-1)
+    h_msig_stdap: Float = Field(default=-1)
+    k_m_stdap: Float = Field(default=-1)
+    k_msig_stdap: Float = Field(default=-1)
+    dist_edge_ns: Float = Field(default=-1)
+    dist_edge_ew: Float = Field(default=-1)
+    dist_edge_flg: Str = Field(default=-1)
+    dup_src: Int = Field(default=-1)
+    use_src: Int = Field(default=-1)
+    a: Str = Field(default=-1)
+    dist_opt: Float = Field(default=-1)
+    phi_opt: Float = Field(default=-1)
+    b_m_opt: Float = Field(default=-1)
+    vr_m_opt: Float = Field(default=-1)
+    nopt_mchs: Int = Field(default=-1)
+    ext_key: Int = Field(default=-1)
+    scan_key: Int = Field(default=-1)
+    coadd_key: Int = Field(default=-1)
+    coad: Int = Field(default=-1)
+
+    class Config:
+        validate_by_name = True
+        validate_assignment = True
+        arbitrary_types_allowed = True

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/src/almanac/data_models/types.py

@@ -4,11 +4,40 @@ from typing_extensions import Annotated
 from pydantic import BeforeValidator
 
 def validate_np_int64(v):
+    if v is None:
+        return np.int64(-1)
     if not isinstance(v, np.int64):
         return np.int64(v)
     return v
 
+def validate_int(v):
+    if v is None:
+        return -1
+    return int(v)
+
+def validate_float(v):
+    if v is None:
+        return float('nan')
+    try:
+        return float(v)
+    except (ValueError, TypeError):
+        return float('nan')
+
+def validate_str(v):
+    if v is None:
+        return ""
+    return str(v)
+
+def validate_bool(v):
+    if v is None:
+        return False
+    return bool(v)
+
 Int64 = Annotated[np.int64, BeforeValidator(validate_np_int64)]
+Int = Annotated[int, BeforeValidator(validate_int)]
+Float = Annotated[float, BeforeValidator(validate_float)]
+Str = Annotated[str, BeforeValidator(validate_str)]
+Bool = Annotated[bool, BeforeValidator(validate_bool)]
 
 
 Observatory = Literal["apo", "lco"]

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/src/almanac/utils.py

@@ -83,9 +83,16 @@ def mjd_to_datetime(mjd: float) -> datetime:
     """
     return Time(mjd, format='mjd').datetime
 
-def parse_mjds(mjd: Optional[int], mjd_start: Optional[int], mjd_end: Optional[int],
-               date: Optional[str], date_start: Optional[str], date_end: Optional[str],
-               earliest_mjd: int = 0) -> Tuple[Union[int, range, Tuple[int, ...]], int, int]:
+def parse_mjds(
+    mjd: Optional[int],
+    mjd_start: Optional[int],
+    mjd_end: Optional[int],
+    date: Optional[str],
+    date_start: Optional[str],
+    date_end: Optional[str],
+    earliest_mjd: int = 0,
+    return_nones: bool = False
+) -> Tuple[Union[int, range, Tuple[int, ...]], int, int]:
     """Parse MJD and date parameters to determine observation date range.
 
     Args:
@@ -96,6 +103,7 @@ def parse_mjds(mjd: Optional[int], mjd_start: Optional[int], mjd_end: Optional[i
         date_start: Start date string in "YYYY-MM-DD" format
         date_end: End date string in "YYYY-MM-DD" format
         earliest_mjd: Earliest allowed MJD value (default: 0)
+        return_nones: If True, return None for mjd range if no dates are specified (default: False)
 
     Returns:
         Tuple containing:
@@ -117,7 +125,10 @@ def parse_mjds(mjd: Optional[int], mjd_start: Optional[int], mjd_end: Optional[i
             "Cannot specify more than one of --mjd, --mjd-start/--mjd-end, --date, --date-start/--date-end"
         )
     if n_given == 0:
-        return ((current_mjd, ), current_mjd, current_mjd)
+        if return_nones:
+            return (None, None, None)
+        else:
+            return ((current_mjd, ), current_mjd, current_mjd)
     if mjd is not None:
         if mjd < 0:
             mjd += current_mjd

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/src/sdss_almanac.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdss-almanac
-Version: 0.2.8
+Version: 0.2.9
 Summary: Everything we've got
 Author-email: Andy Casey <andrew.casey@monash.edu>
 License: BSD 3-Clause License
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Topic :: Documentation :: Sphinx
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.8
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: numpy

{sdss_almanac-0.2.8 → sdss_almanac-0.2.9}/src/sdss_almanac.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ src/almanac/utils.py
 src/almanac/data_models/__init__.py
 src/almanac/data_models/exposure.py
 src/almanac/data_models/fps.py
+src/almanac/data_models/metadata.py
 src/almanac/data_models/plate.py
 src/almanac/data_models/types.py
 src/almanac/data_models/utils.py

sdss_almanac-0.2.8/src/almanac/catalog.py

@@ -1,262 +0,0 @@
-from itertools import batched
-from typing import List
-from peewee import JOIN
-
-def query_catalog(sdss_ids: List[int], batch_size: int = 1000):
-    """
-    Query the SDSS database for photometry and astrometry.
-    """
-
-    sdss_ids = sorted(list(set(sdss_ids).difference({0, -1})))
-
-    from almanac.database import is_database_available, catalogdb as db
-
-    for batch in batched(sdss_ids, batch_size):
-        q = (
-            db
-            .SDSS_ID_To_Catalog
-            .select(
-                db.SDSS_ID_To_Catalog.sdss_id,
-                #db.SDSS_ID_To_Catalog.catalogid,
-                #db.SDSS_ID_To_Catalog.version_id,
-                #db.SDSS_ID_To_Catalog.lead,
-                #db.SDSS_ID_To_Catalog.allstar_dr17_synspec_rev1,
-                #db.SDSS_ID_To_Catalog.allwise,
-                #db.SDSS_ID_To_Catalog.bhm_rm_v0,
-                #db.SDSS_ID_To_Catalog.bhm_rm_v0_2,
-                #db.SDSS_ID_To_Catalog.catwise,
-                #db.SDSS_ID_To_Catalog.catwise2020,
-                #db.SDSS_ID_To_Catalog.gaia_dr2_source,
-                #db.SDSS_ID_To_Catalog.gaia_dr3_source,
-                #db.SDSS_ID_To_Catalog.glimpse,
-                #db.SDSS_ID_To_Catalog.guvcat,
-                #db.SDSS_ID_To_Catalog.legacy_survey_dr10,
-                #db.SDSS_ID_To_Catalog.legacy_survey_dr8,
-                #db.SDSS_ID_To_Catalog.mangatarget,
-                #db.SDSS_ID_To_Catalog.marvels_dr11_star,
-                #db.SDSS_ID_To_Catalog.marvels_dr12_star,
-                #db.SDSS_ID_To_Catalog.mastar_goodstars,
-                #db.SDSS_ID_To_Catalog.panstarrs1,
-                #db.SDSS_ID_To_Catalog.ps1_g18,
-                #db.SDSS_ID_To_Catalog.sdss_dr13_photoobj,
-                #db.SDSS_ID_To_Catalog.sdss_dr17_specobj,
-                #db.SDSS_ID_To_Catalog.skymapper_dr1_1,
-                #db.SDSS_ID_To_Catalog.skymapper_dr2,
-                #db.SDSS_ID_To_Catalog.supercosmos,
-                #db.SDSS_ID_To_Catalog.tic_v8,
-                #db.SDSS_ID_To_Catalog.twomass_psc,
-                #db.SDSS_ID_To_Catalog.tycho2,
-                #db.SDSS_ID_To_Catalog.unwise,
-                db.Gaia_DR3.source_id,
-                #db.Gaia_DR3.solution_id,
-                #db.Gaia_DR3.designation,
-                #db.Gaia_DR3.random_index,
-                #db.Gaia_DR3.ref_epoch,
-                db.Gaia_DR3.ra,
-                #db.Gaia_DR3.ra_error,
-                db.Gaia_DR3.dec,
-                #db.Gaia_DR3.dec_error,
-                db.Gaia_DR3.parallax,
-                #db.Gaia_DR3.parallax_error,
-                #db.Gaia_DR3.parallax_over_error,
-                #db.Gaia_DR3.pm,
-                #db.Gaia_DR3.pmra,
-                #db.Gaia_DR3.pmra_error,
-                #db.Gaia_DR3.pmdec,
-                #db.Gaia_DR3.pmdec_error,
-                #db.Gaia_DR3.ra_dec_corr,
-                #db.Gaia_DR3.ra_parallax_corr,
-                #db.Gaia_DR3.ra_pmra_corr,
-                #db.Gaia_DR3.ra_pmdec_corr,
-                #db.Gaia_DR3.dec_parallax_corr,
-                #db.Gaia_DR3.dec_pmra_corr,
-                #db.Gaia_DR3.dec_pmdec_corr,
-                #db.Gaia_DR3.parallax_pmra_corr,
-                #db.Gaia_DR3.parallax_pmdec_corr,
-                #db.Gaia_DR3.pmra_pmdec_corr,
-                #db.Gaia_DR3.astrometric_n_obs_al,
-                #db.Gaia_DR3.astrometric_n_obs_ac,
-                #db.Gaia_DR3.astrometric_n_good_obs_al,
-                #db.Gaia_DR3.astrometric_n_bad_obs_al,
-                #db.Gaia_DR3.astrometric_gof_al,
-                #db.Gaia_DR3.astrometric_chi2_al,
-                #db.Gaia_DR3.astrometric_excess_noise,
-                #db.Gaia_DR3.astrometric_excess_noise_sig,
-                #db.Gaia_DR3.astrometric_params_solved,
-                #db.Gaia_DR3.astrometric_primary_flag,
-                #db.Gaia_DR3.nu_eff_used_in_astrometry,
-                #db.Gaia_DR3.pseudocolour,
-                #db.Gaia_DR3.pseudocolour_error,
-                #db.Gaia_DR3.ra_pseudocolour_corr,
-                #db.Gaia_DR3.dec_pseudocolour_corr,
-                #db.Gaia_DR3.parallax_pseudocolour_corr,
-                #db.Gaia_DR3.pmra_pseudocolour_corr,
-                #db.Gaia_DR3.pmdec_pseudocolour_corr,
-                #db.Gaia_DR3.astrometric_matched_transits,
-                #db.Gaia_DR3.visibility_periods_used,
-                #db.Gaia_DR3.astrometric_sigma5d_max,
-                #db.Gaia_DR3.matched_transits,
-                #db.Gaia_DR3.new_matched_transits,
-                #db.Gaia_DR3.matched_transits_removed,
-                #db.Gaia_DR3.ipd_gof_harmonic_amplitude,
-                #db.Gaia_DR3.ipd_gof_harmonic_phase,
-                #db.Gaia_DR3.ipd_frac_multi_peak,
-                #db.Gaia_DR3.ipd_frac_odd_win,
-                #db.Gaia_DR3.ruwe,
-                #db.Gaia_DR3.scan_direction_strength_k1,
-                #db.Gaia_DR3.scan_direction_strength_k2,
-                #db.Gaia_DR3.scan_direction_strength_k3,
-                #db.Gaia_DR3.scan_direction_strength_k4,
-                #db.Gaia_DR3.scan_direction_mean_k1,
-                #db.Gaia_DR3.scan_direction_mean_k2,
-                #db.Gaia_DR3.scan_direction_mean_k3,
-                #db.Gaia_DR3.scan_direction_mean_k4,
-                #db.Gaia_DR3.duplicated_source,
-                #db.Gaia_DR3.phot_g_n_obs,
-                #db.Gaia_DR3.phot_g_mean_flux,
-                #db.Gaia_DR3.phot_g_mean_flux_error,
-                #db.Gaia_DR3.phot_g_mean_flux_over_error,
-                #db.Gaia_DR3.phot_g_mean_mag,
-                #db.Gaia_DR3.phot_bp_n_obs,
-                #db.Gaia_DR3.phot_bp_mean_flux,
-                #db.Gaia_DR3.phot_bp_mean_flux_error,
-                #db.Gaia_DR3.phot_bp_mean_flux_over_error,
-                #db.Gaia_DR3.phot_bp_mean_mag,
-                #db.Gaia_DR3.phot_rp_n_obs,
-                #db.Gaia_DR3.phot_rp_mean_flux,
-                #db.Gaia_DR3.phot_rp_mean_flux_error,
-                #db.Gaia_DR3.phot_rp_mean_flux_over_error,
-                #db.Gaia_DR3.phot_rp_mean_mag,
-                #db.Gaia_DR3.phot_bp_rp_excess_factor,
-                #db.Gaia_DR3.phot_bp_n_contaminated_transits,
-                #db.Gaia_DR3.phot_bp_n_blended_transits,
-                #db.Gaia_DR3.phot_rp_n_contaminated_transits,
-                #db.Gaia_DR3.phot_rp_n_blended_transits,
-                #db.Gaia_DR3.phot_proc_mode,
-                db.Gaia_DR3.bp_rp,
-                db.Gaia_DR3.bp_g,
-                db.Gaia_DR3.g_rp,
-                db.Gaia_DR3.radial_velocity,
-                db.Gaia_DR3.radial_velocity_error,
-                #db.Gaia_DR3.rv_method_used,
-                #db.Gaia_DR3.rv_nb_transits,
-                #db.Gaia_DR3.rv_nb_deblended_transits,
-                #db.Gaia_DR3.rv_visibility_periods_used,
-                #db.Gaia_DR3.rv_expected_sig_to_noise,
-                #db.Gaia_DR3.rv_renormalised_gof,
-                #db.Gaia_DR3.rv_chisq_pvalue,
-                #db.Gaia_DR3.rv_time_duration,
-                #db.Gaia_DR3.rv_amplitude_robust,
-                #db.Gaia_DR3.rv_template_teff,
-                #db.Gaia_DR3.rv_template_logg,
-                #db.Gaia_DR3.rv_template_fe_h,
-                #db.Gaia_DR3.rv_atm_param_origin,
-                #db.Gaia_DR3.vbroad,
-                #db.Gaia_DR3.vbroad_error,
-                #db.Gaia_DR3.vbroad_nb_transits,
-                #db.Gaia_DR3.grvs_mag,
-                #db.Gaia_DR3.grvs_mag_error,
-                #db.Gaia_DR3.grvs_mag_nb_transits,
-                #db.Gaia_DR3.rvs_spec_sig_to_noise,
-                #db.Gaia_DR3.phot_variable_flag,
-                #db.Gaia_DR3.l,
-                #db.Gaia_DR3.b,
-                #db.Gaia_DR3.ecl_lon,
-                #db.Gaia_DR3.ecl_lat,
-                #db.Gaia_DR3.in_qso_candidates,
-                #db.Gaia_DR3.in_galaxy_candidates,
-                #db.Gaia_DR3.non_single_star,
-                #db.Gaia_DR3.has_xp_continuous,
-                #db.Gaia_DR3.has_xp_sampled,
-                #db.Gaia_DR3.has_rvs,
-                #db.Gaia_DR3.has_epoch_photometry,
-                #db.Gaia_DR3.has_epoch_rv,
-                #db.Gaia_DR3.has_mcmc_gspphot,
-                #db.Gaia_DR3.has_mcmc_msc,
-                #db.Gaia_DR3.in_andromeda_survey,
-                #db.Gaia_DR3.classprob_dsc_combmod_quasar,
-                #db.Gaia_DR3.classprob_dsc_combmod_galaxy,
-                #db.Gaia_DR3.classprob_dsc_combmod_star,
-                #db.Gaia_DR3.teff_gspphot,
-                #db.Gaia_DR3.teff_gspphot_lower,
-                #db.Gaia_DR3.teff_gspphot_upper,
-                #db.Gaia_DR3.logg_gspphot,
-                #db.Gaia_DR3.logg_gspphot_lower,
-                #db.Gaia_DR3.logg_gspphot_upper,
-                #db.Gaia_DR3.mh_gspphot,
-                #db.Gaia_DR3.mh_gspphot_lower,
-                #db.Gaia_DR3.mh_gspphot_upper,
-                #db.Gaia_DR3.distance_gspphot,
-                #db.Gaia_DR3.distance_gspphot_lower,
-                #db.Gaia_DR3.distance_gspphot_upper,
-                #db.Gaia_DR3.azero_gspphot,
-                #db.Gaia_DR3.azero_gspphot_lower,
-                #db.Gaia_DR3.azero_gspphot_upper,
-                #db.Gaia_DR3.ag_gspphot,
-                #db.Gaia_DR3.ag_gspphot_lower,
-                #db.Gaia_DR3.ag_gspphot_upper,
-                #db.Gaia_DR3.ebpminrp_gspphot,
-                #db.Gaia_DR3.ebpminrp_gspphot_lower,
-                #db.Gaia_DR3.ebpminrp_gspphot_upper,
-                #db.Gaia_DR3.libname_gspphot,
-                db.TwoMassPSC.designation,
-                db.TwoMassPSC.j_m,
-                #db.TwoMassPSC.j_cmsig,
-                #db.TwoMassPSC.j_msigcom,
-                #db.TwoMassPSC.j_snr,
-                db.TwoMassPSC.h_m,
-                #db.TwoMassPSC.h_cmsig,
-                #db.TwoMassPSC.h_msigcom,
-                #db.TwoMassPSC.h_snr,
-                db.TwoMassPSC.k_m,
-                #db.TwoMassPSC.k_cmsig,
-                #db.TwoMassPSC.k_msigcom,
-                #db.TwoMassPSC.k_snr,
-                #db.TwoMassPSC.ph_qual,
-                #db.TwoMassPSC.rd_flg,
-                #db.TwoMassPSC.bl_flg,
-                #db.TwoMassPSC.cc_flg,
-                #db.TwoMassPSC.ndet,
-                #db.TwoMassPSC.prox,
-                #db.TwoMassPSC.pxpa,
-                #db.TwoMassPSC.pxcntr,
-                #db.TwoMassPSC.gal_contam,
-                #db.TwoMassPSC.mp_flg,
-                #db.TwoMassPSC.scan,
-                #db.TwoMassPSC.glon,
-                #db.TwoMassPSC.glat,
-                #db.TwoMassPSC.x_scan,
-                #db.TwoMassPSC.jdate,
-                #db.TwoMassPSC.j_psfchi,
-                #db.TwoMassPSC.h_psfchi,
-                #db.TwoMassPSC.k_psfchi,
-                #db.TwoMassPSC.j_m_stdap,
-                #db.TwoMassPSC.j_msig_stdap,
-                #db.TwoMassPSC.h_m_stdap,
-                #db.TwoMassPSC.h_msig_stdap,
-                #db.TwoMassPSC.k_m_stdap,
-                #db.TwoMassPSC.k_msig_stdap,
-                #db.TwoMassPSC.dist_edge_ns,
-                #db.TwoMassPSC.dist_edge_ew,
-                #db.TwoMassPSC.dist_edge_flg,
-                #db.TwoMassPSC.dup_src,
-                #db.TwoMassPSC.use_src,
-                #db.TwoMassPSC.a,
-                #db.TwoMassPSC.dist_opt,
-                #db.TwoMassPSC.phi_opt,
-                #db.TwoMassPSC.b_m_opt,
-                #db.TwoMassPSC.vr_m_opt,
-                #db.TwoMassPSC.nopt_mchs,
-                #db.TwoMassPSC.ext_key,
-                #db.TwoMassPSC.scan_key,
-                #db.TwoMassPSC.coadd_key,
-                #db.TwoMassPSC.coadd
-            )
-            .distinct(db.SDSS_ID_To_Catalog.sdss_id)
-            .join(db.Gaia_DR3, join_type=JOIN.LEFT_OUTER)
-            .switch(db.SDSS_ID_To_Catalog)
-            .join(db.TwoMassPSC, join_type=JOIN.LEFT_OUTER)
-            .where(db.SDSS_ID_To_Catalog.sdss_id.in_(tuple(batch)))
-            .dicts()
-        )
-        yield from q.iterator()