rubin-sim 2.6.1.dev2__py3-none-any.whl

rubin_sim/maf/utils/maf_utils.py

@@ -0,0 +1,212 @@
+__all__ = (
+    "optimal_bins",
+    "percentile_clipping",
+    "radec2pix",
+    "coadd_m5",
+    "collapse_night",
+    "load_inst_zeropoints",
+)
+
+import os
+import warnings
+
+import healpy as hp
+import numpy as np
+from rubin_scheduler.data import get_data_dir
+from rubin_scheduler.utils import SysEngVals, int_binned_stat
+from scipy.stats import binned_statistic
+
+from rubin_sim.phot_utils import Bandpass, PhotometricParameters
+
+
+def load_inst_zeropoints():
+    """Load up and return instrumental zeropoints and atmospheric
+    extinctions.
+    """
+    zp_inst = {}
+    datadir = get_data_dir()
+    for filtername in "ugrizy":
+        # set gain and exptime to 1 so the instrumental zeropoint will be in
+        # photoelectrons and per second
+        phot_params = PhotometricParameters(nexp=1, gain=1, exptime=1, bandpass=filtername)
+        bp = Bandpass()
+        bp.read_throughput(os.path.join(datadir, "throughputs/baseline/", "total_%s.dat" % filtername))
+        zp_inst[filtername] = bp.calc_zp_t(phot_params)
+
+    syseng = SysEngVals()
+    k_atm = syseng.k_atm
+
+    return zp_inst, k_atm
+
+
+def coadd_m5(mags):
+    """Coadded depth, assuming Gaussian noise."""
+    return 1.25 * np.log10(np.sum(10.0 ** (0.8 * np.array(mags))))
+
+
+def collapse_night(
+    data_slice,
+    night_col="night",
+    filter_col="filter",
+    m5_col="fiveSigmaDepth",
+    mjd_col="observationStartMJD",
+):
+    """Collapse a data_slice into per-filter, per-night values for
+    the 'night', 'filter', 'median observationStartMJD', and 'fiveSigmaDepth'.
+    """
+    filters = np.unique(data_slice[filter_col])
+    # Find the per-night, per-filter values
+    night_slice = {}
+    for filtername in filters:
+        infilt = np.where(data_slice[filter_col] == filtername)[0]
+        unight = np.unique(data_slice[night_col][infilt])
+        right = unight + 0.5
+        bins = [unight[0] - 0.5] + right.tolist()
+        coadds, be, bn = binned_statistic(
+            data_slice[night_col][infilt],
+            data_slice[m5_col][infilt],
+            bins=bins,
+            statistic=coadd_m5,
+        )
+
+        unights, median_mjd_per_night = int_binned_stat(
+            data_slice[night_col][infilt],
+            data_slice[mjd_col][infilt],
+            statistic=np.median,
+        )
+
+        night_slice[filtername] = np.array(
+            list(zip(unight, median_mjd_per_night, coadds, filtername * len(unight))),
+            dtype=[
+                (night_col, int),
+                (mjd_col, float),
+                (m5_col, float),
+                (filter_col, "U1"),
+            ],
+        )
+
+    night_slice = np.concatenate([night_slice[f] for f in night_slice])
+    night_slice.sort(order=["observationStartMJD"])
+
+    return night_slice
+
+
+def optimal_bins(datain, binmin=None, binmax=None, nbin_max=200, nbin_min=1, verbose=False):
+    """
+    Set an 'optimal' number of bins using the Freedman-Diaconis rule.
+
+    Parameters
+    ----------
+    datain : `numpy.ndarray` or `numpy.ma.MaskedArray`
+        The data for which we want to set the bin_size.
+    binmin : `float`
+        The minimum bin value to consider (if None, uses minimum data value).
+    binmax : `float`
+        The maximum bin value to consider (if None, uses maximum data value).
+    nbin_max : `int`
+        The maximum number of bins to create.
+        Sometimes the 'optimal bin_size' implies an unreasonably large number
+        of bins, if the data distribution is unusual.
+    nbin_min : `int`
+        The minimum number of bins to create. Default is 1.
+    verbose : `bool`
+        Turn off warning messages. This utility very often raises warnings
+        and these should likely be logging messages at a lower logging level,
+        but for now - just use the verbose flag to turn these off or on.
+
+    Returns
+    -------
+    nbins : `int`
+        The number of bins.
+    """
+    # if it's a masked array, only use unmasked values
+    if hasattr(datain, "compressed"):
+        data = datain.compressed()
+    else:
+        data = datain
+    # Check that any good data values remain.
+    if data.size == 0:
+        nbins = nbin_max
+        if verbose:
+            warnings.warn(
+                f"No unmasked data available for calculating optimal bin size: returning {nbins} bins"
+            )
+    # Else proceed.
+    else:
+        if binmin is None:
+            binmin = np.nanmin(data)
+        if binmax is None:
+            binmax = np.nanmax(data)
+        cond = np.where((data >= binmin) & (data <= binmax))[0]
+        # Check if any data points remain within binmin/binmax.
+        if np.size(data[cond]) == 0:
+            nbins = nbin_max
+            warnings.warn(
+                "No data available for calculating optimal bin size within range of %f, %f" % (binmin, binmax)
+                + ": returning %i bins" % (nbins)
+            )
+        else:
+            iqr = np.percentile(data[cond], 75) - np.percentile(data[cond], 25)
+            binwidth = 2 * iqr * (np.size(data[cond]) ** (-1.0 / 3.0))
+            nbins = (binmax - binmin) / binwidth
+            if nbins > nbin_max:
+                warnings.warn(
+                    "Optimal bin calculation tried to make %.0f bins, returning %i" % (nbins, nbin_max)
+                )
+                nbins = nbin_max
+            if nbins < nbin_min:
+                warnings.warn(
+                    "Optimal bin calculation tried to make %.0f bins, returning %i" % (nbins, nbin_min)
+                )
+                nbins = nbin_min
+    if np.isnan(nbins):
+        warnings.warn("Optimal bin calculation calculated NaN: returning %i" % (nbin_max))
+        nbins = nbin_max
+    return int(nbins)
+
+
+def percentile_clipping(data, percentile=95.0):
+    """Calculate the minimum and maximum values of a distribution of points,
+    after discarding data more than 'percentile' from the median.
+    This is useful for determining useful data ranges for plots.
+    Note that 'percentile' percent of the data is retained.
+
+    Parameters
+    ----------
+    data : `numpy.ndarray`
+        The data to clip.
+    percentile : `float`
+        Retain values within percentile of the median.
+
+    Returns
+    -------
+    minimum, maximum : `float`, `float`
+        The minimum and maximum values of the clipped data.
+    """
+    lower_percentile = (100 - percentile) / 2.0
+    upper_percentile = 100 - lower_percentile
+    min_value = np.percentile(data, lower_percentile)
+    max_value = np.percentile(data, upper_percentile)
+    return min_value, max_value
+
+
+def radec2pix(nside, ra, dec):
+    """Calculate the nearest healpixel ID of an RA/Dec array, assuming nside.
+
+    Parameters
+    ----------
+    nside : `int`
+        The nside value of the healpix grid.
+    ra : `numpy.ndarray`, (N,)
+        The RA values to be converted to healpix ids, in radians.
+    dec : `numpy.ndarray`, (N,)
+        The Dec values to be converted to healpix ids, in radians.
+
+    Returns
+    -------
+    hpid : `numpy.ndarray`, (N,)
+        The healpix ids.
+    """
+    lat = np.pi / 2.0 - dec
+    hpid = hp.ang2pix(nside, lat, ra)
+    return hpid

rubin_sim/maf/utils/opsim_utils.py

@@ -0,0 +1,259 @@
+__all__ = (
+    "get_sim_data",
+    "scale_benchmarks",
+    "calc_coadded_depth",
+)
+
+import os
+import sqlite3
+import urllib
+from contextlib import closing
+
+import numpy as np
+import pandas as pd
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.engine import make_url
+
+
+def get_sim_data(
+    db_con,
+    sqlconstraint=None,
+    dbcols=None,
+    stackers=None,
+    table_name=None,
+    full_sql_query=None,
+):
+    """Query an opsim database for the needed data columns
+    and run any required stackers.
+
+    Parameters
+    ----------
+    db_con : `str` or SQLAlchemy connectable, or sqlite3 connection
+        Filename to a sqlite3 file, or a connection object that
+        can be used by pandas.read_sql
+    sqlconstraint : `str` or None
+        SQL constraint to apply to query for observations.
+        Ignored if full_sql_query is set.
+    dbcols : `list` [`str`]
+        Columns required from the database. Ignored if full_sql_query is set.
+    stackers : `list` [`rubin_sim.maf.stackers`], optional
+        Stackers to be used to generate additional columns. Default None.
+    table_name : `str` (None)
+        Name of the table to query.
+        Default None will try "observations".
+        Ignored if full_sql_query is set.
+    full_sql_query : `str`
+        The full SQL query to use. Overrides sqlconstraint, dbcols, tablename.
+
+    Returns
+    -------
+    sim_data : `np.ndarray`
+        A numpy structured array with columns resulting from dbcols + stackers,
+        for observations matching the SQLconstraint.
+    """
+    if sqlconstraint is None:
+        sqlconstraint = ""
+
+    # Check that file exists
+    if isinstance(db_con, str):
+        if os.path.isfile(db_con) is False:
+            raise FileNotFoundError("No file %s" % db_con)
+
+    # Check if table is "observations" or "SummaryAllProps"
+    if (table_name is None) & (full_sql_query is None) & (isinstance(db_con, str)):
+        url = make_url("sqlite:///" + db_con)
+        eng = create_engine(url)
+        inspector = inspect(eng)
+        tables = [inspector.get_table_names(schema=schema) for schema in inspector.get_schema_names()]
+        if "observations" in tables[0]:
+            table_name = "observations"
+        elif "SummaryAllProps" in tables[0]:
+            table_name = "SummaryAllProps"
+        elif "summary" in tables[0]:
+            table_name = "summary"
+        else:
+            raise ValueError("Could not guess table_name, set with table_name or full_sql_query kwargs")
+    elif (table_name is None) & (full_sql_query is None):
+        # If someone passes in a connection object with an old table_name
+        # things will fail
+        # that's probably fine, keep people from getting fancy with old sims
+        table_name = "observations"
+
+    if full_sql_query is None:
+        col_str = "*" if dbcols is None else ", ".join(dbcols)
+
+        query = "SELECT %s FROM %s" % (col_str, table_name)
+        if len(sqlconstraint) > 0:
+            query += " WHERE %s" % (sqlconstraint)
+        query += ";"
+    else:
+        query = full_sql_query
+
+    if isinstance(db_con, sqlite3.Connection):
+        sim_data = pd.read_sql(query, db_con).to_records(index=False)
+    elif isinstance(db_con, str) and os.path.isfile(db_con):
+        with closing(sqlite3.connect(db_con)) as con:
+            sim_data = pd.read_sql(query, con).to_records(index=False)
+    elif (not isinstance(db_con, str)) or urllib.parse.urlparse(db_con).scheme != "":
+        try:
+            from lsst.resources import ResourcePath
+
+            with ResourcePath(db_con).as_local() as local_db_path:
+                with closing(sqlite3.connect(local_db_path.ospath)) as con:
+                    sim_data = pd.read_sql(query, con).to_records(index=False)
+        except ModuleNotFoundError:
+            raise RuntimeError(
+                f"Cannot read visits from {db_con}."
+                "Maybe it does not exist, or maybe you need to install lsst.resources."
+            )
+    else:
+        raise RuntimeError("Cannot find {db_con}.")
+
+    if len(sim_data) == 0:
+        raise UserWarning("No data found matching sqlconstraint %s" % (sqlconstraint))
+    # Now add the stacker columns.
+    if stackers is not None:
+        for s in stackers:
+            sim_data = s.run(sim_data)
+    return sim_data
+
+
+def scale_benchmarks(run_length, benchmark="design"):
+    """Set design and stretch values of the number of visits or
+    area of the footprint or seeing/Fwhmeff/skybrightness and single visit
+    depth (based on SRD values).
+    Scales number of visits for the length of the run, relative to 10 years.
+
+    Parameters
+    ----------
+    run_length : `float`
+        The length (in years) of the run.
+    benchmark : `str`
+        design or stretch - which version of the SRD values to return.
+
+    Returns
+    -------
+    benchmarks: `dict` of floats
+       A dictionary containing the number of visits, area of footprint,
+       seeing and FWHMeff values, skybrightness and single visit depth
+       for either the design or stretch SRD values.
+    """
+    # Set baseline (default) numbers for the baseline survey length (10 years).
+    baseline = 10.0
+
+    design = {}
+    stretch = {}
+
+    design["nvisitsTotal"] = 825
+    stretch["nvisitsTotal"] = 1000
+    design["Area"] = 18000
+    stretch["Area"] = 20000
+
+    design["nvisits"] = {"u": 56, "g": 80, "r": 184, "i": 184, "z": 160, "y": 160}
+    stretch["nvisits"] = {"u": 70, "g": 100, "r": 230, "i": 230, "z": 200, "y": 200}
+
+    # mag/sq arcsec
+    design["skybrightness"] = {
+        "u": 21.8,
+        "g": 22.0,
+        "r": 21.3,
+        "i": 20.0,
+        "z": 19.1,
+        "y": 17.5,
+    }
+    stretch["skybrightness"] = {
+        "u": 21.8,
+        "g": 22.0,
+        "r": 21.3,
+        "i": 20.0,
+        "z": 19.1,
+        "y": 17.5,
+    }
+
+    # arcsec - old seeing values
+    design["seeing"] = {"u": 0.77, "g": 0.73, "r": 0.7, "i": 0.67, "z": 0.65, "y": 0.63}
+    stretch["seeing"] = {
+        "u": 0.77,
+        "g": 0.73,
+        "r": 0.7,
+        "i": 0.67,
+        "z": 0.65,
+        "y": 0.63,
+    }
+
+    # arcsec - new FWHMeff values (scaled from old seeing)
+    design["FWHMeff"] = {
+        "u": 0.92,
+        "g": 0.87,
+        "r": 0.83,
+        "i": 0.80,
+        "z": 0.78,
+        "y": 0.76,
+    }
+    stretch["FWHMeff"] = {
+        "u": 0.92,
+        "g": 0.87,
+        "r": 0.83,
+        "i": 0.80,
+        "z": 0.78,
+        "y": 0.76,
+    }
+
+    design["singleVisitDepth"] = {
+        "u": 23.9,
+        "g": 25.0,
+        "r": 24.7,
+        "i": 24.0,
+        "z": 23.3,
+        "y": 22.1,
+    }
+    stretch["singleVisitDepth"] = {
+        "u": 24.0,
+        "g": 25.1,
+        "r": 24.8,
+        "i": 24.1,
+        "z": 23.4,
+        "y": 22.2,
+    }
+
+    # Scale the number of visits.
+    if run_length != baseline:
+        scalefactor = float(run_length) / float(baseline)
+        # Calculate scaled value for design and stretch values of nvisits,
+        # per filter.
+        for f in design["nvisits"]:
+            design["nvisits"][f] = int(np.floor(design["nvisits"][f] * scalefactor))
+            stretch["nvisits"][f] = int(np.floor(stretch["nvisits"][f] * scalefactor))
+
+    if benchmark == "design":
+        return design
+    elif benchmark == "stretch":
+        return stretch
+    else:
+        raise ValueError("Benchmark value %s not understood: use 'design' or 'stretch'" % (benchmark))
+
+
+def calc_coadded_depth(nvisits, single_visit_depth):
+    """Calculate the coadded depth expected for a given number of visits
+    and single visit depth.
+
+    Parameters
+    ----------
+    nvisits : `dict` of `int` or `float`
+        Dictionary (per filter) of number of visits
+    single_visit_depth : `dict` of `float`
+        Dictionary (per filter) of the single visit depth
+
+    Returns
+    -------
+    coadded_depth : `dict` of `float`
+        Dictionary of coadded depths per filter.
+    """
+    coadded_depth = {}
+    for f in nvisits:
+        if f not in single_visit_depth:
+            raise ValueError("Filter keys in nvisits and single_visit_depth must match")
+        coadded_depth[f] = float(1.25 * np.log10(nvisits[f] * 10 ** (0.8 * single_visit_depth[f])))
+        if not np.isfinite(coadded_depth[f]):
+            coadded_depth[f] = single_visit_depth[f]
+    return coadded_depth

rubin_sim/maf/utils/output_utils.py

@@ -0,0 +1,34 @@
+__all__ = ("name_sanitize",)
+
+
+def name_sanitize(in_string):
+    """
+    Convert a string to a more file name (and web) friendly format.
+
+    Parameters
+    ----------
+    in_string : `str`
+        The input string to be sanitized.
+        Typically these are combinations of metric names and metadata.
+
+    Returns
+    -------
+    out_string : `str`
+        The string after removal/replacement of non-friendly characters.
+    """
+    # Replace <, > and = signs.
+    out_string = in_string.replace(">", "gt").replace("<", "lt").replace("=", "eq")
+    # Remove single-spaces, strip '.'s and ','s
+    out_string = out_string.replace(" ", "_").replace(".", "_").replace(",", "")
+    # and remove / and \
+    out_string = out_string.replace("/", "_").replace("\\", "_")
+    # and remove parentheses
+    out_string = out_string.replace("(", "").replace(")", "")
+    # Remove ':' and ';"
+    out_string = out_string.replace(":", "_").replace(";", "_")
+    # Replace '%' and #
+    out_string = out_string.replace("%", "_").replace("#", "_")
+    # Remove '__'
+    while "__" in out_string:
+        out_string = out_string.replace("__", "_")
+    return out_string