rubin-sim 2.6.1.dev2__py3-none-any.whl

rubin_sim/maf/metrics/agn_time_lag_metric.py

@@ -0,0 +1,118 @@
+__all__ = ("AgnTimeLagMetric",)
+
+import numpy as np
+
+from rubin_sim.phot_utils import DustValues
+
+from .base_metric import BaseMetric
+
+
+class AgnTimeLagMetric(BaseMetric):
+    def __init__(
+        self,
+        lag=100,
+        z=1,
+        log=False,
+        threshold=2.2,
+        calc_type="mean",
+        mjd_col="observationStartMJD",
+        filter_col="band",
+        m5_col="fiveSigmaDepth",
+        dust=True,
+        g_cutoff=22.0,
+        r_cutoff=21.8,
+        metric_name=None,
+        **kwargs,
+    ):
+        self.lag = lag
+        self.z = z
+        self.log = log
+        self.threshold = threshold
+        self.calc_type = calc_type
+        self.mjd_col = mjd_col
+        self.filter_col = filter_col
+        self.m5_col = m5_col
+        if metric_name is None:
+            metric_name = f"AGN_TimeLag_{lag}_days"
+        self.dust = dust
+        self.g_cutoff = g_cutoff
+        self.r_cutoff = r_cutoff
+        if dust:
+            maps = ["DustMap"]
+            dust_properties = DustValues()
+            self.ax1 = dust_properties.ax1
+        else:
+            maps = []
+        super().__init__(
+            col=[self.mjd_col, self.filter_col, self.m5_col],
+            metric_name=metric_name,
+            maps=maps,
+            **kwargs,
+        )
+
+    # Calculate NQUIST value for time-lag and sampling time
+    # (redshift is included in formula if desired)
+    def _get_nquist_value(self, caden, lag, z):
+        return lag / ((1 + z) * caden)
+
+    def run(self, data_slice, slice_point=None):
+        # Dust extinction
+        filterlist = np.unique(data_slice[self.filter_col])
+        if self.dust:
+            m5 = np.zeros(len(data_slice))
+            for filtername in filterlist:
+                in_filt = np.where(data_slice[self.filter_col] == filtername)[0]
+                a_x = self.ax1[data_slice[self.filter_col][0]] * slice_point["ebv"]
+                m5[in_filt] = data_slice[self.m5_col][in_filt] - a_x
+        else:
+            m5 = data_slice[self.m5_col]
+
+        # Identify times which pass magnitude cuts (chosen by AGN contributors)
+        mjds = np.zeros(len(data_slice))
+        for filtername in filterlist:
+            in_filt = np.where(data_slice[self.filter_col] == filtername)[0]
+            if filtername in ("u", "i", "z", "y", "r", "g"):
+                mjds[in_filt] = data_slice[self.mjd_col][in_filt]
+            elif filtername == "g":
+                faint = np.where(m5[in_filt] > self.g_cutoff)
+                mjds[in_filt][faint] = data_slice[self.mjd_col][in_filt][faint]
+            elif filtername == "r":
+                faint = np.where(m5[in_filt] > self.r_cutoff)
+                mjds[in_filt][faint] = data_slice[self.mjd_col][in_filt][faint]
+        # Remove the visits which were not faint enough
+        mjds = mjds[np.where(mjds > 0)]
+
+        # Calculate differences in time between visits
+        mv = np.sort(mjds)
+        val = np.diff(mv)
+        # If there was only one visit; bail out now.
+        if len(val) == 0:
+            return self.badval
+
+        # Otherwise summarize the time differences as:
+        if self.calc_type == "mean":
+            val = np.mean(val)
+        elif self.calc_type == "min":
+            val = np.min(val)
+        elif self.calc_type == "max":
+            val = np.max(val)
+        else:
+            # find the greatest common divisor
+            val = np.rint(val).astype(int)
+            val = np.gcd.reduce(val)
+
+        # Will always have a value at this point
+        nquist = self._get_nquist_value(val, self.lag, self.z)
+        if self.log:
+            nquist = np.log(nquist)
+
+        # Threshold nquist value is 2.2,
+        # hence we are aiming to show values higher than threshold (2.2) value
+        threshold = self.threshold
+        if self.log:
+            threshold = np.log(threshold)
+
+        if nquist < threshold:
+            nquist = self.badval
+
+        return nquist

rubin_sim/maf/metrics/agnstructure.py

@@ -0,0 +1,141 @@
+__all__ = ("SFUncertMetric",)
+
+import warnings
+
+import numpy as np
+from astropy.stats import mad_std
+
+from rubin_sim.maf.utils import m52snr
+from rubin_sim.phot_utils import DustValues
+
+from .base_metric import BaseMetric
+
+
+class SFUncertMetric(BaseMetric):
+    """Structure Function (SF) Uncertainty Metric.
+    Developed on top of LogTGaps
+
+    Adapted from Weixiang Yu & Gordon Richards at:
+    https://github.com/RichardsGroup/
+    LSST_SF_Metric/blob/main/notebooks/00_SFErrorMetric.ipynb
+
+    Parameters
+    ----------
+    mag : `float`
+        The magnitude of the fiducial object. Default 22.
+    times_col : `str`
+        Time column name. Defaults to "observationStartMJD".
+    all_gaps : `bool`
+         Whether to use all gaps (between any two pairs of observations).
+         If False, only use consecutive paris. Defaults to True.
+    units : `str`
+        Unit of this metric. Defaults to "mag".
+    bins : `object`
+        An array of bin edges.
+        Defaults to "np.logspace(0, np.log10(3650), 16)" for a
+        total of 15 (final) bins.
+    weight : `object`
+        The weight assigned to each delta_t bin for deriving the final metric.
+        Defaults to flat weighting with sum of 1.
+        Should have length 1 less than bins.
+    snr_cut : `float`
+        Ignore observations below an SNR limit, default 5.
+    dust : `bool`
+        Apply dust extinction to the fiducial object magnitude. Default True.
+    """
+
+    def __init__(
+        self,
+        mag=22,
+        times_col="observationStartMJD",
+        m5_col="fiveSigmaDepth",
+        all_gaps=True,
+        units="mag",
+        bins=np.logspace(0, np.log10(3650), 16),
+        weight=None,
+        metric_name="Structure Function Uncert",
+        snr_cut=5,
+        filter_col="band",
+        dust=True,
+        **kwargs,
+    ):
+        # Assign metric parameters to instance object
+        self.times_col = times_col
+        self.m5_col = m5_col
+        self.filter_col = filter_col
+        self.all_gaps = all_gaps
+        self.bins = bins
+        if weight is None:
+            # If weight is none, set weight so that sum over bins = 1
+            self.weight = np.ones(len(self.bins) - 1)
+            self.weight /= self.weight.sum()
+        self.metric_name = metric_name
+        self.mag = mag
+        self.snr_cut = snr_cut
+        self.dust = dust
+
+        maps = ["DustMap"]
+        super(SFUncertMetric, self).__init__(
+            col=[self.times_col, m5_col, filter_col],
+            metric_name=self.metric_name,
+            units=units,
+            maps=maps,
+            **kwargs,
+        )
+        dust_properties = DustValues()
+        self.ax1 = dust_properties.ax1
+
+    def run(self, data_slice, slice_point=None):
+        """Code executed at each healpix pixel to compute the metric"""
+
+        df = np.unique(data_slice[self.filter_col])
+        if np.size(df) > 1:
+            msg = """Running structure function on multiple filters simultaneously.
+                     Should probably change your SQL query to limit to a single filter."""
+            warnings.warn(msg)
+        if self.dust:
+            a_x = self.ax1[data_slice[self.filter_col][0]] * slice_point["ebv"]
+            extincted_mag = self.mag + a_x
+        else:
+            extincted_mag = self.mag
+        snr = m52snr(extincted_mag, data_slice[self.m5_col])
+        bright_enough = np.where(snr > self.snr_cut)[0]
+
+        # If the total number of visits < 2, mask as bad pixel
+        if data_slice[bright_enough].size < 2:
+            return self.badval
+
+        # sort data by time column
+        order = np.argsort(data_slice[self.times_col][bright_enough])
+        times = data_slice[self.times_col][bright_enough][order]
+        # Using the simple Gaussian approximation for magnitude uncertainty.
+        mag_err = 2.5 * np.log10(1.0 + 1.0 / snr[bright_enough][order])
+
+        # check if use all gaps (between any pairs of observations)
+        if self.all_gaps:
+            # use the vectorized method
+            dt_matrix = times.reshape((1, times.size)) - times.reshape((times.size, 1))
+            dts = dt_matrix[dt_matrix > 0].flatten().astype(np.float16)
+        else:
+            dts = np.diff(times)
+
+        # bin delta_t using provided bins;
+        # if zero pair found at any delta_t bin,
+        # replace 0 with 0.01 to avoid the exploding 1/sqrt(n) term
+        # in this metric
+        result, bins = np.histogram(dts, self.bins)
+        new_result = np.where(result > 0, result, 0.01)
+
+        # compute photometric_error^2 population variance and population mean
+        # note that variance is replaced by median_absolute_deviate^2
+        # mean is replaced by median in this implementation to make it robust
+        # to outliers in simulations (e.g., dcr simulations)
+        err_var = mag_err**2
+        err_var_mu = np.median(err_var)
+        err_var_std = mad_std(err_var)
+
+        # compute SF error
+        sf_var_dt = 2 * (err_var_mu + err_var_std / np.sqrt(new_result))
+        sf_var_metric = np.sum(sf_var_dt * self.weight)
+
+        return sf_var_metric

rubin_sim/maf/metrics/area_summary_metrics.py

@@ -0,0 +1,120 @@
+__all__ = ("AreaSummaryMetric", "AreaThresholdMetric")
+
+import healpy as hp
+import numpy as np
+
+from .base_metric import BaseMetric
+
+
+class AreaSummaryMetric(BaseMetric):
+    """
+    Find the min/max of a value over the area with the 'best' results
+    in the metric.
+    This is a handy substitute for when users want to know "the WFD value".
+
+    Parameters
+    ----------
+    area : `float`
+        The area to consider (sq degrees)
+    decreasing : `bool`
+        Should the values be sorted by increasing or decreasing order.
+        For values where "larger is better", decreasing (True) is probably
+        what you want. For metrics where "smaller is better"
+        (e.g., astrometric precission), set decreasing to False.
+    reduce_func : None
+        The function to reduce the clipped values by.
+        Will default to min/max depending on the bool val of the decreasing
+        kwarg.
+    """
+
+    def __init__(
+        self,
+        col="metricdata",
+        metric_name="AreaSummary",
+        area=18000.0,
+        decreasing=True,
+        reduce_func=None,
+        **kwargs,
+    ):
+        super().__init__(col=col, metric_name=metric_name, **kwargs)
+        self.area = area
+        self.decreasing = decreasing
+        self.reduce_func = reduce_func
+        self.mask_val = np.nan  # Include so all values get passed
+        self.col = col
+        if reduce_func is None:
+            if decreasing:
+                self.reduce_func = np.min
+            else:
+                self.reduce_func = np.max
+        else:
+            self.reduce_func = reduce_func
+
+    def run(self, data_slice, slice_point=None):
+        # find out what nside we have
+        nside = hp.npix2nside(data_slice[self.col].size)
+        pix_area = hp.nside2pixarea(nside, degrees=True)
+        n_pix_needed = int(np.ceil(self.area / pix_area))
+
+        # Only use the finite data
+        data = data_slice[self.col][np.isfinite(data_slice[self.col].astype(float))]
+        order = np.argsort(data)
+        if self.decreasing:
+            order = order[::-1]
+        result = self.reduce_func(data[order][0:n_pix_needed])
+        return result
+
+
+class AreaThresholdMetric(BaseMetric):
+    """Find the amount of area on the sky that meets a given threshold value.
+
+    The area per pixel is determined from the size of the metric_values
+    array passed to the summary metric.
+    This assumes that both all values are passed and that the metric was
+    calculated with a healpix slicer.
+
+    Parameters
+    ----------
+    upper_threshold : `float` or None
+        The metric value must be below this threshold to count toward the area.
+        Default None implies no upper bound.
+    lower_threshold : `float` or None, opt
+        The metric value must be above this threshold to count toward the area.
+        Default None implies no lower bound.
+    """
+
+    def __init__(
+        self,
+        col="metricdata",
+        metric_name="AreaThreshold",
+        upper_threshold=None,
+        lower_threshold=None,
+        **kwargs,
+    ):
+        super().__init__(col=col, metric_name=metric_name, **kwargs)
+        self.upper_threshold = upper_threshold
+        self.lower_threshold = lower_threshold
+        self.mask_val = np.nan  # Include so all values get passed
+        self.col = col
+        self.units = "square degrees"
+
+    def run(self, data_slice, slice_point=None):
+        # find out what nside we have
+        nside = hp.npix2nside(data_slice.size)
+        pix_area = hp.nside2pixarea(nside, degrees=True)
+        # Look for pixels which match the critera for the thresholds
+        if self.upper_threshold is None and self.lower_threshold is None:
+            npix = len(data_slice)
+        elif self.upper_threshold is None:
+            npix = len(np.where(data_slice[self.col] > self.lower_threshold)[0])
+        elif self.lower_threshold is None:
+            npix = len(np.where(data_slice[self.col] < self.upper_threshold)[0])
+        else:
+            npix = len(
+                np.where(
+                    (data_slice[self.col] > self.lower_threshold)
+                    & (data_slice[self.col] < self.upper_threshold)
+                )[0]
+            )
+        area = pix_area * npix
+        return area

rubin_sim/maf/metrics/base_metric.py

@@ -0,0 +1,215 @@
+# Base class for metrics - defines methods which must be implemented.
+
+__all__ = ("MetricRegistry", "BaseMetric", "ColRegistry")
+
+import inspect
+import warnings
+
+import numpy as np
+
+from rubin_sim.maf.stackers.get_col_info import ColInfo
+
+
+class MetricRegistry(type):
+    """
+    Meta class for metrics, to build a registry of metric classes.
+    """
+
+    def __init__(cls, name, bases, dict):
+        super(MetricRegistry, cls).__init__(name, bases, dict)
+        if not hasattr(cls, "registry"):
+            cls.registry = {}
+        modname = inspect.getmodule(cls).__name__
+        if modname.startswith("rubin_sim.maf.metrics"):
+            modname = ""
+        else:
+            if len(modname.split(".")) > 1:
+                modname = ".".join(modname.split(".")[:-1]) + "."
+            else:
+                modname = modname + "."
+        metricname = modname + name
+        if metricname in cls.registry:
+            warnings.warn("Redefining metric %s! (there are >1 metrics with the same name)" % (metricname))
+        if metricname not in ["BaseMetric", "SimpleScalarMetric"]:
+            cls.registry[metricname] = cls
+
+    def get_class(cls, metricname):
+        return cls.registry[metricname]
+
+    def help(cls, doc=False):
+        for metricname in sorted(cls.registry):
+            if not doc:
+                print(metricname)
+            if doc:
+                print("---- ", metricname, " ----")
+                print(inspect.getdoc(cls.registry[metricname]))
+
+    def help_metric(cls, metricname):
+        print(metricname)
+        print(inspect.getdoc(cls.registry[metricname]))
+        k = inspect.signature(cls.registry[metricname])
+        print(" Metric __init__ keyword args and defaults: ")
+        print(k)
+
+
+class ColRegistry:
+    """
+    ColRegistry tracks the columns needed for all metric objects
+    (kept internally in a set).
+
+    ColRegistry.col_set : a set of all unique columns required for metrics.
+    ColRegistry.dbCols : the subset of these which come from the database.
+    ColRegistry.stackerCols : the dictionary of [columns: stacker class].
+    """
+
+    col_info = ColInfo()
+
+    def __init__(self):
+        self.col_set = set()
+        self.db_set = set()
+        self.stacker_dict = {}
+
+    def clear_reg(self):
+        """Clear the registry"""
+        self.__init__()
+
+    def add_cols(self, col_array):
+        """Add the columns in ColArray into the ColRegistry.
+
+        Add the columns in col_array into the ColRegistry set (self.col_set)
+        and identifies their source, using ColInfo
+        (rubin_sim.maf.stackers.getColInfo).
+
+        Parameters
+        ----------
+        col_array : `list`
+            list of columns used in a metric.
+        """
+        for col in col_array:
+            if col is not None:
+                self.col_set.add(col)
+                source = self.col_info.get_data_source(col)
+                if source == self.col_info.default_data_source:
+                    self.db_set.add(col)
+                else:
+                    if col not in self.stacker_dict:
+                        self.stacker_dict[col] = source
+
+
+class BaseMetric(metaclass=MetricRegistry):
+    """
+    Base class for the metrics.
+    Sets up some basic functionality for the MAF framework:
+    after __init__ every metric will record the columns (and stackers)
+    it requires into the column registry, and the metric_name,
+    metric_dtype, and units for the metric will be set.
+
+    Parameters
+    ----------
+    col : `str` or `list` [`str`]
+        Names of the data columns that the metric will use.
+        The columns required for each metric is tracked in the ColRegistry,
+        and used to retrieve data from the opsim database.
+        Can be a single string or a list.
+    metric_name : `str`
+        Name to use for the metric (optional - if not set, will be derived).
+    maps : `list` [`rubin_sim.maf.maps`]
+        The maps that the metric will need (passed from the slicer).
+    units : `str`
+        The units for the value returned by the metric (optional - if not set,
+        will be derived from the ColInfo).
+    metric_dtype : `str`
+        The type of value returned by the metric - 'int', 'float', 'object'.
+        If not set, will be derived by introspection.
+    badval : `float`
+        The value indicating "bad" values calculated by the metric.
+    """
+
+    col_registry = ColRegistry()
+    col_info = ColInfo()
+
+    def __init__(
+        self,
+        col=None,
+        metric_name=None,
+        maps=None,
+        units=None,
+        metric_dtype=None,
+        badval=-666,
+        mask_val=None,
+    ):
+        # Turn cols into numpy array so we know
+        # we can iterate over the columns.
+        self.col_name_arr = np.array(col, copy=True, ndmin=1)
+        # To support simple metrics operating on a single column,
+        # set self.colname
+        if len(self.col_name_arr) == 1:
+            self.colname = self.col_name_arr[0]
+        # Add the columns to the colRegistry.
+        self.col_registry.add_cols(self.col_name_arr)
+        # Set the maps that are needed:
+        if maps is None:
+            maps = []
+        self.maps = maps
+        # Value to return if the metric can't be computed
+        self.badval = badval
+        if mask_val is not None:
+            self.mask_val = mask_val
+        # Save a unique name for the metric.
+        self.name = metric_name
+        if self.name is None:
+            # If none provided, construct our own from the class name
+            # and the data columns.
+            self.name = (
+                self.__class__.__name__.replace("Metric", "", 1)
+                + " "
+                + ", ".join(map(str, self.col_name_arr))
+            )
+        # Set up dictionary of reduce functions (may be empty).
+        self.reduce_funcs = {}
+        self.reduce_order = {}
+        for i, r in enumerate(inspect.getmembers(self, predicate=inspect.ismethod)):
+            if r[0].startswith("reduce"):
+                # Remove the "reduce_" part of the reduce functions names.
+                reducename = r[0].replace("reduce_", "", 1)
+                self.reduce_funcs[reducename] = r[1]
+                self.reduce_order[reducename] = i
+        # Identify type of metric return value.
+        if metric_dtype is not None:
+            self.metric_dtype = metric_dtype
+        elif len(list(self.reduce_funcs.keys())) > 0:
+            self.metric_dtype = "object"
+        else:
+            self.metric_dtype = "float"
+        # Set physical units, for plotting purposes.
+        if units is None:
+            units = " ".join([self.col_info.get_units(col_name) for col_name in self.col_name_arr])
+            if len(units.replace(" ", "")) == 0:
+                units = ""
+        self.units = units
+        # Add the ability to set a comment
+        # (that could be propagated automatically to a benchmark's
+        # display caption).
+        self.comment = None
+
+        # Default to only return one metric value per slice
+        self.shape = 1
+
+    def run(self, data_slice, slice_point):
+        """Calculate metric values.
+
+        Parameters
+        ----------
+        data_slice : `numpy.ndarray`, (N,)
+           Values passed to metric by the slicer, which the metric will
+           use to calculate metric values at each slice_point.
+        slice_point : `dict` or None
+           Dictionary of slice_point metadata passed to each metric.
+           E.g. the ra/dec of the healpix pixel.
+
+        Returns
+        -------
+        metricValue : `int` `float` or `object`
+            The metric value at each slice_point.
+        """
+        raise NotImplementedError("Please implement your metric calculation.")