rubin-sim 2.2.2__tar.gz → 2.2.4__tar.gz

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/.github/workflows/test_and_build.yaml

@@ -36,6 +36,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda install --yes --file=requirements.txt
+          conda install --yes --file=optional-requirements.txt
           conda install --yes --file=test-requirements.txt
 
       - name: install rubin_sim

{rubin_sim-2.2.2/rubin_sim.egg-info → rubin_sim-2.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rubin-sim
-Version: 2.2.2
+Version: 2.2.4
 Summary: Scheduler, survey strategy analysis, and other simulation tools for Rubin Observatory.
 License: GPL
 Project-URL: documentation, https://rubin-sim.lsst.io
@@ -14,28 +14,26 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: matplotlib
-Requires-Dist: healpy
-Requires-Dist: pandas
-Requires-Dist: numexpr
-Requires-Dist: scipy
-Requires-Dist: sqlalchemy
-Requires-Dist: astropy
-Requires-Dist: tables
-Requires-Dist: h5py
 Requires-Dist: astroplan
+Requires-Dist: astropy
 Requires-Dist: colorcet
 Requires-Dist: cycler
-Requires-Dist: george
-Requires-Dist: scikit-learn
+Requires-Dist: gitpython
+Requires-Dist: h5py
+Requires-Dist: healpy
+Requires-Dist: matplotlib
+Requires-Dist: numexpr
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pyarrow
+Requires-Dist: rubin-scheduler>=3.0
+Requires-Dist: scipy
 Requires-Dist: shapely
-Requires-Dist: skyfield
+Requires-Dist: skyfield>=1.52
 Requires-Dist: skyproj
+Requires-Dist: sqlalchemy
+Requires-Dist: tables
 Requires-Dist: tqdm
-Requires-Dist: rubin-scheduler>=3.0
-Requires-Dist: pyarrow
-Requires-Dist: gitpython
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: black>=25; extra == "test"
@@ -44,6 +42,10 @@ Requires-Dist: isort; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Provides-Extra: dev
 Requires-Dist: documenteer[guide]; extra == "dev"
+Provides-Extra: extras
+Requires-Dist: george; extra == "extras"
+Requires-Dist: scikit-learn; extra == "extras"
+Requires-Dist: pyoorb; extra == "extras"
 Dynamic: license-file
 
 # rubin_sim

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/batch/run_prenight_sims.sh

@@ -29,6 +29,7 @@ conda activate /sdf/data/rubin/shared/scheduler/envs/prenight
 
 set -o xtrace
 
+export TELESCOPE=$1
 export AWS_PROFILE=prenight
 WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/run_prenight_sims/%Y-%m-%dT%H%M%S' --utc)
 echo "Working in $WORK_DIR"
@@ -36,8 +37,9 @@ mkdir ${WORK_DIR}
 cd ${WORK_DIR}
 
 # Get ts_ocs_config
+TS_CONFIG_OCS_REPO="https://github.com/lsst-ts/ts_config_ocs"
 TS_CONFIG_OCS_VERSION=$(obs_version_at_time ts_config_ocs)
-curl --location --output ts_config_ocs.zip https://github.com/lsst-ts/ts_config_ocs/archive/${TS_CONFIG_OCS_VERSION}.zip
+curl --location --output ts_config_ocs.zip ${TS_CONFIG_OCS_REPO}/archive/${TS_CONFIG_OCS_VERSION}.zip
 unzip ts_config_ocs.zip
 mv $(find . -maxdepth 1 -type d -name ts_config_ocs\*) ts_config_ocs
 
@@ -54,16 +56,18 @@ TS_FBS_UTILS_TAG=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tag
 pip install --no-deps --target=${PACKAGE_DIR} git+https://github.com/lsst-ts/ts_fbs_utils.git@${TS_FBS_UTILS_TAG}
 
 # Get the scheduler configuration script
-SCHEDULER_CONFIG_SCRIPT=$(scheduler_config_at_time latiss)
+SCHEDULER_CONFIG_SCRIPT=$(scheduler_config_at_time ${TELESCOPE})
 
 # Get the path to prenight_sim as provided by the current environment,
 # so we do not accidentally run one from the adjusted PATH below.
 PRENIGHT_SIM=$(which prenight_sim)
 
+export SIM_ARCHIVE_LOG_FILE=${WORK_DIR}/sim_archive_log.txt
+export PRENIGHT_LOG_FILE=${WORK_DIR}/prenight_log.txt
 export PYTHONPATH=${PACKAGE_DIR}:${PYTHONPATH}
 export PATH=${PACKAGE_DIR}/bin:${PATH}
 printenv > env.out
 date --iso=s
-time ${PRENIGHT_SIM} --scheduler auxtel.pickle.xz --opsim None --script ${SCHEDULER_CONFIG_SCRIPT}
+time ${PRENIGHT_SIM} --scheduler ${TELESCOPE}.pickle.xz --opsim None --repo=${TS_CONFIG_OCS_REPO} --script ${SCHEDULER_CONFIG_SCRIPT} --config_version ${TS_CONFIG_OCS_VERSION} --telescope ${TELESCOPE} 2>&1 > ${WORK_DIR}/prenight_sim.out
 date --iso=s
 echo "******* END of run_prenight_sims.sh *********"

rubin_sim-2.2.4/optional-requirements.txt

@@ -0,0 +1,6 @@
+openorb
+openorb-data-de405
+george
+scikit-learn
+jinja2
+tornado

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/pyproject.toml

@@ -19,28 +19,26 @@ classifiers = [
 urls = {documentation = "https://rubin-sim.lsst.io", repository = "https://github.com/lsst/rubin_sim" }
 dynamic = [ "version" ]
 dependencies = [
-    "numpy",
-    "matplotlib",
-    "healpy",
-    "pandas",
-    "numexpr",
-    "scipy",
-    "sqlalchemy",
-    "astropy",
-    "tables",
-    "h5py",
     "astroplan",
+    "astropy",
     "colorcet",
     "cycler",
-    "george",
-    "scikit-learn",
+    "gitpython",
+    "h5py",
+    "healpy",
+    "matplotlib",
+    "numexpr",
+    "numpy",
+    "pandas",
+    "pyarrow",
+    "rubin-scheduler>=3.0",
+    "scipy",
     "shapely",
-    "skyfield",
+    "skyfield>=1.52",
     "skyproj",
-    "tqdm",
-    "rubin-scheduler>=3.0",
-    "pyarrow",
-    "gitpython",
+    "sqlalchemy",
+    "tables",
+    "tqdm"
 ]
 
 [project.optional-dependencies]
@@ -54,6 +52,11 @@ test = [
 dev = [
   "documenteer[guide]",
 ]
+extras = [
+    "george",
+    "scikit-learn",
+    "pyoorb",
+]
 
 [project.scripts]
 rs_download_data = "rubin_sim.data.rs_download_data:rs_download_data"
@@ -85,7 +88,7 @@ version = { attr = "setuptools_scm.get_version" }
 where = [ "" ]
 
 [tool.setuptools_scm]
-version_file = "rubin_sim/_version.py"
+version_file = "rubin_sim/version.py"
 
 
 [tool.pytest.ini_options]
@@ -98,7 +101,7 @@ exclude = "version.py"
 
 [tool.black]
 line-length = 110
-target-version = ["py311"]
+target-version = ["py312"]
 
 [tool.isort]
 profile = "black"
@@ -109,7 +112,7 @@ exclude = [
     "__init__.py",
 ]
 line-length = 110
-target-version = "py311"
+target-version = "py312"
 
 [tool.ruff.lint]
 ignore = [

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/requirements.txt

@@ -1,27 +1,23 @@
-setuptools_scm
-setuptools_scm_git_archive
-numpy
-matplotlib
+astropy-base
+astroplan
+colorcet
+conda
+cycler
+gitpython
 healpy
+h5py
+matplotlib-base
+numpy
+numexpr
 pandas
 pyarrow
-numexpr
-scipy
-sqlalchemy
-astropy
 pytables
-h5py
-astroplan
-colorcet
-cycler
-george
-scikit-learn
+rubin-scheduler >=3.0
+scipy
+setuptools_scm
+setuptools_scm_git_archive
 shapely
-skyfield
+skyfield >=1.52
 skyproj
-tornado
-jinja2
+sqlalchemy
 tqdm
-gitpython
-conda
-rubin-scheduler >=3.0

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/rubin_sim/maf/maf_contrib/static_probes_fom_summary_metric.py

@@ -4,7 +4,6 @@ import healpy as hp
 import numpy as np
 import pandas as pd
 from scipy.optimize import minimize
-from sklearn.preprocessing import StandardScaler
 
 from rubin_sim.maf.metrics.base_metric import BaseMetric
 
@@ -326,6 +325,7 @@ class StaticProbesFoMEmulatorMetric(BaseMetric):
     def run(self, data_slice, slice_point=None):
         import george
         from george import kernels
+        from sklearn.preprocessing import StandardScaler
 
         # Chop off any outliers
         good_pix = np.where(data_slice[self.col] > 0)[0]

rubin_sim-2.2.4/rubin_sim/maf/run_comparison/gather_summaries.py

@@ -0,0 +1,125 @@
+__all__ = ("combine_result_dbs", "gather_summaries")
+
+import argparse
+import glob
+import logging
+import os
+import sqlite3
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def combine_result_dbs(run_dirs, dbfilename="resultsDb_sqlite.db"):
+    """Helper function for gather_summaries
+
+    Parameters
+    ----------
+    run_dirs : `list`  [`str`]
+        A list of directories to search for MAF result databases.
+    dbfilename : `str`
+        The database filename to look for (default: resultsDb_sqlite.db).
+    """
+
+    # query to grab all the summary stats
+    sql_q = "SELECT summarystats.summary_value, "
+    sql_q += "metrics.metric_name, metrics.metric_info_label, "
+    sql_q += "metrics.slicer_name, summarystats.summary_name, metrics.run_name "
+    sql_q += "FROM summarystats INNER JOIN metrics ON metrics.metric_id=summarystats.metric_id"
+
+    all_summaries = []
+    for rdir in run_dirs:
+        fname = os.path.join(rdir, dbfilename)
+        if not os.path.isfile(fname):
+            logger.warning(f"No resultsDb database in {rdir}")
+
+        con = sqlite3.connect(fname)
+        temp_df = pd.read_sql(sql_q, con)
+        con.close()
+
+        # Make column names
+        def make_summary_name(x):
+            summary_name = " ".join(
+                [
+                    x.summary_name.strip(),
+                    x.metric_name.strip(),
+                    x.metric_info_label.strip(),
+                    x.slicer_name.strip(),
+                ]
+            )
+            summary_name = summary_name.replace("  ", " ")
+            return summary_name
+
+        temp_df["summary_names"] = temp_df.apply(make_summary_name, axis=1)
+        all_summaries.append(temp_df[["summary_names", "summary_value", "run_name"]])
+
+    # Make one big dataframe
+    all_summaries = pd.concat(all_summaries)
+    # Group by run names and drop duplicates
+    g = all_summaries.groupby(["run_name", "summary_names"]).agg({"summary_value": "last"})
+    # Convert to one row with all summary stats per run
+    result_df = g.reset_index("summary_names").pivot(columns="summary_names")
+    # That ended up as a MultiIndex which we didn't need, so fix and rename
+    result_df.columns = result_df.columns.droplevel(0).rename("metric")
+    return result_df
+
+
+def gather_summaries():
+    """Find resultsDbs in a series of directories and gather up their summary
+    stats into a single CSV or hdf5 file. Outputs one row per unique run name.
+    """
+
+    parser = argparse.ArgumentParser(
+        description="Find resultsDbs in a series of directories and "
+        "gather up their summary stats into a single CSV or hdf5 file. "
+        "Intended to run on a set of metrics run on multiple "
+        "simulations, so that each results_db has similar summary"
+        "statistics."
+    )
+    parser.add_argument(
+        "--base_dir",
+        type=str,
+        default=".",
+        help="Root directory from where to search for MAF (sub)directories.",
+    )
+    parser.add_argument(
+        "--outfile",
+        type=str,
+        default="summary",
+        help="Output file name. Default (summary)",
+    )
+    parser.add_argument(
+        "--to_csv",
+        dest="to_csv",
+        action="store_true",
+        help="Create a .csv file, instead of the default hdf file.",
+    )
+    parser.add_argument("--to_hdf", dest="to_hdf", action="store_true")
+    parser.add_argument(
+        "--dirs",
+        type=str,
+        default=None,
+        help="comma separated list of directories to use, default None",
+    )
+
+    args = parser.parse_args()
+    if args.dirs is None:
+        run_dirs = glob.glob(args.base_dir + "/*/")
+    else:
+        run_dirs = args.dirs.split(",")
+
+    # Create output file name if needed
+    if args.to_csv:
+        outfile = args.outfile + ".csv"
+    else:
+        outfile = args.outfile + ".h5"
+
+    result_df = combine_result_dbs(run_dirs)
+
+    # Save summary statistics
+    if args.to_csv:
+        result_df.to_csv(outfile)
+    else:
+        # Create a CSV file
+        result_df.to_hdf(outfile, key="stats")

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/rubin_sim/maf/stackers/coord_stackers.py

@@ -1,5 +1,6 @@
-__all__ = ("ra_dec2_alt_az", "GalacticStacker", "EclipticStacker")
+__all__ = ("ra_dec2_alt_az", "GalacticStacker", "EclipticStacker", "HealpixStacker")
 
+import healpy as hp
 import numpy as np
 from astropy import units as u
 from astropy.coordinates import SkyCoord, get_sun
@@ -186,3 +187,38 @@ class EclipticStacker(BaseStacker):
             sim_data["eclipLon"] = np.degrees(sim_data["eclipLon"])
             sim_data["eclipLat"] = np.degrees(sim_data["eclipLat"])
         return sim_data
+
+
+class HealpixStacker(BaseStacker):
+    """Add healpix id
+
+    Parameters
+    ----------
+    ra_col : str, optional
+        Name of the RA column. Default fieldRA.
+    dec_col : str, optional
+        Name of the Dec column. Default fieldDec.
+    nside : int
+        nside healpix to use. Default to 128.
+    """
+
+    cols_added = ["ring_healpix", "nest_healpix"]
+
+    def __init__(self, ra_col: str = "fieldRA", dec_col: str = "fieldDec", nside: int = 128):
+        self.cols_req = [ra_col, dec_col]
+        self.ra_col = ra_col
+        self.dec_col = dec_col
+        self.nside = nside
+        self.units = [f"healpix (ring, nside={nside})", f"healpix (nest, nside={nside})"]
+
+    def _run(self, sim_data, cols_present=False):
+        if cols_present:
+            # Column already present in data;
+            # assume it is correct and does not need recalculating.
+            return sim_data
+
+        ra = sim_data[self.ra_col]
+        decl = sim_data[self.dec_col]
+        sim_data["ring_healpix"] = hp.ang2pix(self.nside, ra, decl, nest=False, lonlat=True)
+        sim_data["nest_healpix"] = hp.ang2pix(self.nside, ra, decl, nest=True, lonlat=True)
+        return sim_data

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/rubin_sim/maf/stackers/date_stackers.py

@@ -1,11 +1,13 @@
 __all__ = (
     "ObservationStartDatetime64Stacker",
+    "ObservationStartTimestampStacker",
     "DayObsStacker",
     "DayObsMJDStacker",
     "DayObsISOStacker",
 )
 
 import numpy as np
+import pandas as pd
 from astropy.time import Time
 
 from .base_stacker import BaseStacker
@@ -36,6 +38,52 @@ class ObservationStartDatetime64Stacker(BaseStacker):
         return sim_data
 
 
+class ObservationStartTimestampStacker(BaseStacker):
+    """Add the observation start time as a pandas.Timestamp."""
+
+    cols_added = ["start_timestamp"]
+
+    def __init__(
+        self,
+        mjd_col="observationStartMJD",
+    ):
+        self.mjd_col = mjd_col
+        self.cols_req = [self.mjd_col]
+        self.units = [None]
+        self.cols_added_dtypes = ["O"]
+
+    def run(self, sim_data, override=False):
+        # Override the run from the base class, not _run,
+        # because the implementation
+        # of _add_stackers_cols in run in the base closs
+        # clobbers the type for the new column,
+        # which we really need.
+
+        visits = sim_data if isinstance(sim_data, pd.DataFrame) else pd.DataFrame(sim_data)
+
+        if "start_timestamp" in visits.columns and not override:
+            return sim_data
+
+        if len(visits[self.mjd_col]) > 0:
+            visits["start_timestamp"] = pd.to_datetime(
+                visits[self.mjd_col] + 2400000.5, origin="julian", unit="D", utc=True
+            )
+        else:
+            # If we are passed an empty series, be sure to return add an empty
+            # series of the correct type back.
+            # This is handy if the result is being passed to bokeh
+            # for plotting.
+            visits["start_timestamp"] = pd.to_datetime(2460000.5, origin="julian", unit="D", utc=True)
+
+        match sim_data:
+            case pd.DataFrame():
+                return visits
+            case dict():
+                return visits.to_dict()
+            case _:
+                return visits.to_records(index=False)
+
+
 def _compute_day_obs_mjd(mjd):
     day_obs_mjd = np.floor(mjd - 0.5).astype("int")
     return day_obs_mjd

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/rubin_sim/maf/utils/opsim_utils.py

@@ -72,7 +72,7 @@ def get_sim_data(
         elif "summary" in tables[0]:
             table_name = "summary"
         else:
-            ValueError("Could not guess table_name, set with table_name or full_sql_query kwargs")
+            raise ValueError("Could not guess table_name, set with table_name or full_sql_query kwargs")
     elif (table_name is None) & (full_sql_query is None):
         # If someone passes in a connection object with an old table_name
         # things will fail

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/rubin_sim/maf/utils/stellar_mags.py

@@ -81,7 +81,7 @@ def stellar_mags(stellar_type, rmag=19.0):
     if not hasattr(stellar_mags, "data"):
         names = ["stellar_type", "Model Name", "u-g", "g-r", "r-i", "i-z", "z-y"]
         types = [("U", 20), ("U", 35), float, float, float, float, float]
-        data = np._core.records.fromrecords(
+        data = np.rec.fromrecords(
             [
                 (
                     "O",

{rubin_sim-2.2.2 → rubin_sim-2.2.4}/rubin_sim/sim_archive/prenight.py

@@ -15,6 +15,7 @@ import lzma
 import os
 import pickle
 import typing
+import warnings
 from datetime import datetime
 from functools import partial
 from tempfile import TemporaryFile
@@ -33,14 +34,19 @@ from rubin_scheduler.site_models import Almanac
 
 from rubin_sim.sim_archive.sim_archive import drive_sim
 
-from .make_snapshot import add_make_scheduler_snapshot_args, get_scheduler, save_scheduler
+from .make_snapshot import (
+    add_make_scheduler_snapshot_args,
+    get_scheduler,
+    get_scheduler_instance_from_path,
+    save_scheduler,
+)
 
 try:
     from rubin_sim.data import get_baseline  # type: ignore
 except ModuleNotFoundError:
     get_baseline = partial(warn, "Cannot find default baseline because rubin_sim is not installed.")
 
-DEFAULT_ARCHIVE_URI = "s3://rubin-scheduler-prenight/opsim/"
+DEFAULT_ARCHIVE_URI = "s3://rubin:rubin-scheduler-prenight/opsim/"
 
 
 def _run_sim(
@@ -210,7 +216,7 @@ def run_prenights(
     opsim_db: Optional[str] = None,
     minutes_delays: tuple[float, ...] = (0, 1, 10, 60, 240),
     anomalous_overhead_seeds: tuple[int, ...] = (101, 102, 103, 104, 105),
-    sim_nights: int = 2,
+    sim_nights: int = 3,
     opsim_metadata: dict | None = None,
 ) -> None:
     """Run the set of scheduler simulations needed to prepare for a night.
@@ -234,7 +240,7 @@ def run_prenights(
     anomalous_overhead_seeds: `tuple` of `int`
         Random number seeds to use for anomalous overhead runs.
     sim_nights: `int`
-        Number of nights to simulate. Defaults to 2.
+        Number of nights to simulate. Defaults to 3.
     opsim_metadata: `dict`
         Extra metadata for the archive
     """
@@ -337,6 +343,33 @@ def prenight_sim_cli(cli_args: list = []) -> None:
         help="Archive in which to store simulation results.",
     )
     parser.add_argument("--scheduler", type=str, default=None, help="pickle file of the scheduler to run.")
+    parser.add_argument("--config_version", type=str, default=None, help="Version of ts_config_ocs used.")
+    parser.add_argument("--telescope", type=str, default=None, help="Telescope scheduled.")
+
+    # Configure logging
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    log_handlers = [stream_handler]
+
+    log_file = os.environ.get("PRENIGHT_LOG_FILE", None)
+    if log_file is not None:
+        file_handler = logging.FileHandler(log_file, mode="w")
+        file_handler.setLevel(logging.DEBUG)
+        log_handlers.append(file_handler)
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format="%(asctime)s: %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S%z",
+        handlers=[stream_handler, file_handler],
+    )
+
+    # FIXME
+    warnings.filterwarnings(
+        "ignore",
+        category=UserWarning,
+        message="IntRounded being used with a potentially too-small scale factor.",
+    )
 
     # Only pass a default if we have an opsim
     baseline = get_baseline()
@@ -356,20 +389,34 @@ def prenight_sim_cli(cli_args: list = []) -> None:
     opsim_db = None if args.opsim in ("", "None") else args.opsim
 
     scheduler_file = args.scheduler
-    if args.repo is not None or args.script is not None:
+    opsim_metadata = {"telescope": args.telescope}
+    if args.repo is not None:
         if os.path.exists(scheduler_file):
             raise ValueError(f"File {scheduler_file} already exists!")
 
-        scheduler: CoreScheduler = get_scheduler(args.repo, args.script, args.branch)
-        save_scheduler(scheduler, scheduler_file)
-
-        opsim_metadata = {
-            "opsim_config_repository": args.repo,
-            "opsim_config_script": args.script,
-            "opsim_config_branch": args.branch,
-        }
-    else:
-        opsim_metadata = None
+        if args.config_version is not None:
+            scheduler: CoreScheduler = get_scheduler_instance_from_path(args.script)
+            save_scheduler(scheduler, scheduler_file)
+            opsim_metadata.update(
+                {
+                    "opsim_config_repository": args.repo,
+                    "opsim_config_script": args.script,
+                    "opsim_config_version": args.config_version,
+                }
+            )
+        elif args.branch is not None:
+            scheduler: CoreScheduler = get_scheduler(args.repo, args.script, args.branch)
+            save_scheduler(scheduler, scheduler_file)
+
+            opsim_metadata.update(
+                {
+                    "opsim_config_repository": args.repo,
+                    "opsim_config_script": args.script,
+                    "opsim_config_branch": args.branch,
+                }
+            )
+        else:
+            raise ValueError("Either the branch or the version of ts_ocs_config must be specified")
 
     run_prenights(
         day_obs_mjd,