solarwindpy 0.0.1.dev0__py3-none-any.whl → 0.1.0__py3-none-any.whl

solarwindpy/solar_activity/lisird/extrema_calculator.py

@@ -0,0 +1,394 @@
+"""Tools for calculating extrema in LISIRD activity indices."""
+
+__all__ = ["ExtremaCalculator"]
+
+import pdb  # noqa: F401
+
+import pandas as pd
+import matplotlib as mpl
+import numpy as np
+
+from ...plotting import subplots
+
+
+class ExtremaCalculator(object):
+    r"""Determine extrema in an activity index time series.
+
+    The calculator smooths the input series with a rolling mean and finds
+    local minima and maxima based on a threshold value.
+
+    Attributes
+    ----------
+    data : pandas.Series
+        Smoothed version of the activity index.
+    raw : pandas.Series
+        Unsmoothed input data.
+    threshold : pandas.Series
+        Threshold used to classify maxima and minima.
+    extrema : pandas.Series
+        Series with ``"Max"`` or ``"Min"`` labels at the extrema times.
+    formatted_extrema : pandas.DataFrame
+        Data frame formatted as a solar cycle table with ``Min`` and ``Max``
+        columns::
+
+            ========== ============ ============
+             Interval      Min          Max
+            ========== ============ ============
+             -1         <DateTime>   <DateTime>
+              0         <DateTime>   <DateTime>
+              1         <DateTime>   <DateTime>
+              ...
+              N         <DateTime>   <DateTime>
+            ========== ============ ============
+    """
+
+    def __init__(self, name, activity_index, threshold=None, window=600):
+        r"""Create the calculator.
+
+        Parameters
+        ----------
+        name : str
+            Identifier for the activity index.
+        activity_index : pandas.Series
+            Raw activity measurements.
+        threshold : float or callable, optional
+            If a scalar, it is used directly to classify maxima and minima.
+            If a callable, it is invoked with ``activity_index`` to compute the
+            threshold. When ``None``, the value is looked up from an internal
+            table or computed with :func:`numpy.nanmedian`.
+        window : int, optional
+            Window length in days for the rolling mean.
+        """
+        self.set_name(name)
+        self.set_data(activity_index, window)
+        self.set_threshold(threshold)
+        self.find_threshold_crossings()
+        self.find_extrema()
+
+    @property
+    def data(self):
+        return self._data
+
+    @property
+    def raw(self):
+        return self._raw
+
+    @property
+    def name(self):
+        r"""Activity index name."""
+        return self._name
+
+    @property
+    def window(self):
+        return self._window
+
+    @property
+    def threshold(self):
+        return self._threshold
+
+    @property
+    def extrema_finders(self):
+        return self._extrema_finders
+
+    @property
+    def extrema(self):
+        return self._extrema
+
+    @property
+    def threshold_crossings(self):
+        return self._threshold_crossings
+
+    @property
+    def data_in_extrema_finding_intervals(self):
+        return self._data_in_extrema_finding_intervals
+
+    @property
+    def formatted_extrema(self):
+        """Extrema formatted as a solar-cycle table.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Data frame with ``Min`` and ``Max`` columns indexed by cycle::
+
+                ========== ============ ============
+                 Interval      Min          Max
+                ========== ============ ============
+                 -1         <DateTime>   <DateTime>
+                  0         <DateTime>   <DateTime>
+                  1         <DateTime>   <DateTime>
+                  ...
+                  N         <DateTime>   <DateTime>
+                ========== ============ ============
+        """
+
+        return self._formatted_extrema
+
+    def set_name(self, new):
+        if new in ("delk2", "delwb", "k2vk3", "viored", "delk1"):
+            raise ValueError(
+                "Unable to determine threshold. You need to check this one."
+            )
+        self._name = str(new)
+
+    def set_data(self, index, window):
+
+        if self.name in ("delk1", "delk2", "delwb", "emdx", "k2vk3", "k3", "viored"):
+            # We don't trust CaK before then.
+            index = index.loc["1977-01-01":]
+
+        rolled = index
+        if window is not None:
+            rolled = index.rolling("%sd" % window).mean()
+            rolled.index = rolled.index - pd.to_timedelta("%sd" % (window / 2.0))
+
+        self._raw = index
+        self._data = rolled
+        self._window = window
+
+    def _format_axis(self, ax):
+        left, _ = ax.get_xlim()
+        left = pd.to_datetime(
+            "{}-01-01".format(pd.to_datetime(mpl.dates.num2date(left)).year - 1)
+        )
+        ax.set_xlim(
+            left=mpl.dates.date2num(left),
+            right=mpl.dates.date2num(pd.to_datetime("2020-01-02")),
+        )
+        ax.xaxis.set_major_formatter(mpl.dates.DateFormatter("%Y"))
+        ax.xaxis.set_major_locator(mpl.dates.YearLocator(2))
+        ax.figure.autofmt_xdate()
+
+        hdl, lbl = ax.get_legend_handles_labels()
+        hdl = np.asarray(hdl)
+        lbl = np.asarray(lbl)
+        tk = lbl != "indicator"
+
+        ax.legend(hdl[tk], lbl[tk], loc=0, ncol=1, framealpha=0)
+        ax.set_ylabel(self.name)
+        ax.set_xlabel("Year")
+
+    def _plot_data(self, ax):
+        x = mpl.dates.date2num(self.data.index)
+        y = self.data.values
+        ax.plot(x, y, color="C0", label="Rolled")
+
+    #         x = mpl.dates.date2num(self.raw.index)
+    #         y = self.raw.values
+    #         ax.plot(x, y, color="C2", label="Raw")
+
+    def _plot_threshold(self, ax):
+        x = mpl.dates.date2num(self.data.index)
+        y = self.threshold
+        ax.plot(x, y, color="C1", label="{:.5f}".format(self.threshold.unique()[0]))
+
+    def _plot_extrema_ranges(self, ax):
+        joint = pd.concat(
+            {"cut": self.data_in_extrema_finding_intervals, "indicator": self.data},
+            axis=1,
+        )
+        gb = joint.groupby("cut")
+
+        ngroup = 0
+        for k, v in gb:
+            color = "darkorange" if ngroup % 2 else "fuchsia"
+            v.plot(ax=ax, color=color, ls="--", label=None)
+            ngroup += 1
+
+        ax.legend_.set_visible(False)
+
+    def _plot_threshold_crossings(self, ax):
+        crossings = self.threshold_crossings
+        crossings.plot(ax=ax, color="cyan", marker="P", ls="none", label="Changes")
+        ax.legend()
+
+    def _plot_extrema(self, ax):
+        maxima = self.data.loc[self.extrema.index].loc[self.extrema == "Max"]
+        minima = self.data.loc[self.extrema.index].loc[self.extrema == "Min"]
+
+        for ex, c, lbl in zip((maxima, minima), ("red", "limegreen"), ("Max", "Min")):
+            x = mpl.dates.date2num(ex.index)
+            y = ex
+            ax.plot(x, y, color=c, label=lbl, ls="none", marker="*")
+
+    def set_threshold(self, threshold):
+        from numbers import Number
+        from types import FunctionType
+
+        automatic = {
+            "LymanAlpha": 4.1,
+            "delk1": 0.62,
+            "emdx": 0.091,
+            "f107": 110.0,
+            "k3": 0.066,
+            "mg_index": 0.27,
+            "sd_70": 13.0,
+            "sl_70": 2.0,  # Actually log10(sl_70)
+            "viored": 1.29,
+        }
+
+        if threshold is None:
+            threshold = automatic.get(self.name, np.nanmedian)
+
+        if isinstance(threshold, FunctionType):
+            threshold = threshold(self.data)
+
+        elif isinstance(threshold, Number):
+            pass
+
+        threshold = pd.Series(threshold, index=self.data.index)
+        self._threshold = threshold
+
+    @staticmethod
+    def _find_extrema(threshold, cut, data):
+        joint = pd.concat({"cut": cut, "indicator": data}, axis=1)
+        gb = joint.groupby("cut")
+
+        thresh = threshold.unique()
+        assert thresh.size == 1
+        thresh = thresh[0]
+
+        maxima = {}
+        minima = {}
+        for k, v in gb:
+            # Lots of logic to ensure we only have one minima or one maxima
+            v = v.indicator
+            vclean = v.dropna()
+            if not vclean.size:
+                # No valid data in this range
+                continue
+
+            is_max = (vclean > thresh).value_counts()
+            if is_max.size > 1:
+                is_max = is_max.replace(1, np.nan).dropna()
+            assert is_max.size == 1
+
+            is_max = is_max.index[0]
+            if is_max:
+                maxima[k] = vclean.idxmax()
+            else:
+                minima[k] = vclean.idxmin()
+
+        maxima = pd.Series("Max", index=maxima.values())
+        minima = pd.Series("Min", index=minima.values())
+
+        return maxima, minima
+
+    def _validate_extrema(self, maxima, minima):
+        name = self.name
+        if name == "LymanAlpha":
+            maxima = maxima.iloc[1:]
+        elif name == "delk1":
+            minima = minima.iloc[1:-1]
+        #         elif name == "emdx":
+        #             minima = minima.iloc[2:]
+        #             maxima = maxima.iloc[:-1]
+        elif name == "f107":
+            minima = minima.iloc[:-1]
+            maxima = maxima.iloc[1:]
+        elif name == "mg_index":
+            maxima = maxima.iloc[1:]
+        elif name == "sd_70":
+            minima = minima.iloc[:-1]
+        elif name == "sl_70":
+            minima = minima.iloc[1:]
+        elif name == "viored":
+            minima = minima.iloc[1:-1]
+
+        minimum_seperation = pd.to_timedelta("1000d")
+        tk_max = maxima.index.to_series().diff() > minimum_seperation
+        tk_min = minima.index.to_series().diff() > minimum_seperation
+
+        # 0th entry diff is NaT -> False by default.
+        tk_max.iloc[0] = True
+        tk_min.iloc[0] = True
+        maxima = maxima.loc[tk_max]
+        minima = minima.loc[tk_min]
+
+        return maxima, minima
+
+    def find_threshold_crossings(self):
+        data = self.data
+        threshold = self.threshold
+
+        high = data > threshold
+        low = data < threshold
+
+        dhigh = high.astype(int).diff() != 0
+        dlow = low.astype(int).diff() != 0
+        deltas = dlow | dhigh
+        crossings = data.where(deltas).dropna()
+
+        self._threshold_crossings = crossings
+        return crossings
+
+    def cut_data_into_extrema_finding_intervals(self):
+        data = self.data
+        raw = self.raw
+        crossings = self.threshold_crossings
+
+        bins = crossings.index
+        if bins[-1] < raw.index[-1]:
+            bins = bins.append(pd.DatetimeIndex([raw.index[-1]]))
+        if bins[0] > raw.index[0]:
+            bins = bins.append(pd.DatetimeIndex([raw.index[0]]))
+
+        bins = bins.sort_values()
+
+        cut = pd.cut(data.index, bins=bins)
+        cut = pd.Series(cut, index=data.index)
+        self._data_in_extrema_finding_intervals = cut
+        return cut
+
+    @staticmethod
+    def format_extrema(extrema):
+        minima = extrema.loc[extrema == "Min"]
+        maxima = extrema.loc[extrema == "Max"]
+
+        min0 = minima.index[0]
+        max0 = maxima.index[0]
+
+        if max0 < min0:
+            minima = pd.Series(minima.index, np.arange(minima.index.size))
+            maxima = pd.Series(maxima.index, np.arange(maxima.index.size) - 1)
+        else:
+            minima = pd.Series(minima.index, np.arange(minima.index.size))
+            maxima = pd.Series(maxima.index, np.arange(maxima.index.size))
+
+        formatted = pd.concat({"Min": minima, "Max": maxima}, axis=1, names=["kind"])
+        formatted.index.name = "cycle"
+
+        return formatted
+
+    def find_extrema(self):
+        # raw = self.raw
+        data = self.data
+        threshold = self.threshold
+        cut = self.cut_data_into_extrema_finding_intervals()
+
+        maxima, minima = self._find_extrema(threshold, cut, data)  # data -> raw
+        maxima, minima = self._validate_extrema(maxima, minima)
+        extrema = pd.concat([maxima, minima], axis=0).sort_index()
+        formatted = self.format_extrema(extrema)
+
+        self._extrema = extrema
+        self._formatted_extrema = formatted
+
+    def make_plot(self, crossings=False, extrema=False, ranges=False):
+        fig, ax = subplots(scale_width=2.5)
+
+        self._plot_data(ax)
+        self._plot_threshold(ax)
+
+        if crossings:
+            self._plot_threshold_crossings(ax)
+
+        if ranges:
+            self._plot_extrema_ranges(ax)
+
+        if extrema:
+            self._plot_extrema(ax)
+
+        self._format_axis(ax)
+
+        return ax

solarwindpy/solar_activity/lisird/lisird.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python
+"""Interfaces for the LASP Interactive Solar Irradiance Data Center (LISIRD).
+
+The submodule provides classes for downloading and working with data hosted at
+`LASP <http://lasp.colorado.edu/lisird/>`_.
+"""
+
+import pdb  # noqa: F401
+import urllib
+import json
+import numpy as np
+import pandas as pd
+
+from pathlib import Path
+
+# from scipy.interpolate import InterpolatedUnivariateSpline
+
+from ..base import (
+    ID,
+    DataLoader,
+    ActivityIndicator,
+    IndicatorExtrema,
+)  # , _Loader_Dtypes_Columns
+from .extrema_calculator import ExtremaCalculator
+
+pd.set_option("mode.chained_assignment", "raise")
+
+# _m13_dtypes_columns = _Loader_Dtypes_Columns(
+# {0: int, 1: int, 2: float, 3: float, 4: float, 5: int, 6: bool},
+# ("year", "month", "year_fraction", "ssn", "std", "n_obs", "definitive")
+# )
+#
+# _m_dtypes_columns = _Loader_Dtypes_Columns(
+# {0: int, 1: int, 2: float, 3: float, 4: float, 5: int, 6: bool},
+# ["year", "month", "year_fraction", "ssn", "std", "n_obs", "definitive"]
+# )
+#
+# _d_dtypes_columns = _Loader_Dtypes_Columns(
+# {0: int, 1: int, 2: int, 3: float, 4: float, 5: float, 6: int, 7: bool},
+# ["year", "month", "day", "year_fraction", "ssn", "std", "n_obs", "definitive"
+# )
+
+
+class LISIRD_ID(ID):
+    def __init__(self, key):
+        r"""Identifier for LISIRD data products.
+
+        Parameters
+        ----------
+        key : str
+            Short name of the data set. Examples include ``"Lalpha"`` or
+            ``"f107-noaa"``.
+
+            =========== ======================== =============================
+             Key           Description                     URL
+            =========== ======================== =============================
+             Lalpha      Lyman-alpha              composite_lyman_alpha.jsond
+             CaK         Calcium K line           cak.jsond
+             f107-noaa   NOAA F10.7 flux          noaa_radio_flux.jsond
+             f107-pen    Penticton F10.7 flux     penticton_radio_flux.jsond
+             MgII        Composite Magnesium II   composite_mg_index.jsond
+            =========== ======================== =============================
+
+        URLs replace the wild card in ``http://lasp.colorado.edu/lisird/latis/*``.
+
+        Note that the CaK line should probably be served directly from
+        ``https://www.nso.edu/uncategorized/ca-ii-k-line-monitoring-program/``.
+        The quantities in CaK data are
+
+            ======== ====================================================
+             k3       Core Intensity
+             k2vk3    Relative blue K2 peak w/rt K3 instensity
+             delk1    Separation of the blue and red K1 minima (K1V-K1R)
+             delk2    Separation of the two emission maxima (K2V-K2R)
+             delwb    Wilson-Bappu parameter, width between the outer
+                      edges of the K2 emission peaks
+             emdx     Emission index equivalent width in 1 angstrom
+                      band centered on K3
+             viored   ???
+            ======== ====================================================
+
+        (<https://www.nso.edu/wp-content/uploads/2018/09/cak_paper.pdf>).
+        """
+        super(LISIRD_ID, self).__init__(key)
+
+    @property
+    def _url_base(self):
+        return r"http://lasp.colorado.edu/lisird/latis/dap/"
+
+    @property
+    def _trans_url(self):
+        trans_url = (
+            ("Lalpha", "composite_lyman_alpha.jsond"),
+            ("CaK", "cak.jsond"),
+            #             ("f107", "noaa_radio_flux.jsond"),
+            ("f107-penticton", "penticton_radio_flux.jsond"),
+            ("f107-noaa", "noaa_radio_flux.jsond"),
+            (
+                "MgII",
+                "composite_mg_index.jsond",
+            ),  # CHECK: Change to "bremen_composite_mgii.jsond" ?
+        )
+
+        return dict(trans_url)
+
+
+class LISIRDLoader(DataLoader):
+    @property
+    def data_path(self):
+        return super(LISIRDLoader, self).data_path / "lisird" / self.key
+
+    @property
+    def meta(self):
+        return self._meta
+
+    def convert_nans(self, data, meta):
+        key = self.key
+        if key in ("CaK", "MgII", "f107-noaa", "f107-penticton"):
+            self.logger.info("Prior inspection shows no missing data in `%s`.", key)
+            return
+
+        elif key == "Lalpha":
+            mv0 = np.float64(meta["irradiance"]["missing_value"])
+            mv1 = np.float64(meta["uncertainty"]["missing_value"])
+            if not mv0 == mv1:
+                raise NotImplementedError(
+                    f"Unsure how to handle mv0 ({mv0:.0f}) != mv1 ({mv1:.0f})"
+                )
+            mv = mv0
+        #         elif key == "f107":
+        #             mv = np.float64(meta["f107"]["missing_value"])
+        else:
+            raise NotImplementedError("Haven't inspected other data to convert.")
+
+        data.replace(to_replace=mv, value=np.nan, inplace=True)
+
+    def verify_monotonic_epoch(self, df):
+        epoch = df.index
+        assert isinstance(epoch, pd.DatetimeIndex)
+        epoch = epoch.to_series()
+
+        ms = df.loc[:, "milliseconds"]
+        drop = ms.duplicated()
+
+        if self.key == "f107-penticton":
+            manually_identified_bad_timestamps = [
+                1_061_657_971_200,
+                1_277_073_820_800,
+                1_568_685_597_120,
+            ]
+            manual_bad = ms.isin(manually_identified_bad_timestamps)
+            drop = drop | manual_bad
+
+        df = df.loc[~drop]
+        return df
+
+    def download_data(self, new_data_path, old_data_path):
+        key = self.key
+        url = self.url
+        self.logger.info("Downloading solar activity data: %s\nurl: %s" % (key, url))
+
+        with urllib.request.urlopen(url) as url_:
+            data = json.loads(url_.read().decode())[Path(self.url).stem]
+
+        meta = data["metadata"]
+        df = pd.DataFrame(data["data"], columns=data["parameters"])
+
+        t0 = pd.to_datetime("1970-01-01 00:00:00")
+        ms = df.pop("time")  # Time in milliseconds since 1970-01-01.
+        dt = pd.to_timedelta(ms, unit="ms")
+        t = dt.add(t0)
+        df.loc[:, "milliseconds"] = ms
+        df.index = t
+        df = df.sort_index(axis=1)
+
+        self.convert_nans(df, meta)
+        df = self.verify_monotonic_epoch(df)
+
+        d = new_data_path.with_suffix(".csv")
+        m = new_data_path.with_suffix(".json")
+
+        df.to_csv(d, sep=",", na_rep="NaN")
+        with open(m, "w") as f:
+            json.dump(meta, f, indent=4)
+
+        d_old = old_data_path.with_suffix(".csv")
+        m_old = old_data_path.with_suffix(".json")
+        try:
+            d_old.unlink()
+        except FileNotFoundError:
+            pass
+        try:
+            m_old.unlink()
+        except FileNotFoundError:
+            pass
+
+    def load_data(self):
+        super(LISIRDLoader, self).load_data()
+        #        self.logger.info("Loading %s LISIRD data", self.key)
+        #
+        #        self.maybe_update_stale_data()
+        #
+        today = pd.to_datetime("today").strftime("%Y%m%d")
+        data_path = self.data_path / today
+
+        #        data = pd.read_csv(data_path.with_suffix(".csv"))
+        #        self._data = data
+        with open(data_path.with_suffix(".json")) as f:
+            meta = json.load(f)
+
+        self._meta = meta
+        self.logger.info("Load complete")
+
+
+class LISIRD(ActivityIndicator):
+    r"""Wrapper around LISIRD data sets."""
+
+    def __init__(self, key):
+        r"""Instantiate a LISIRD data object.
+
+        Parameters
+        ----------
+        key : str
+            Identifier passed to :class:`LISIRD_ID`.
+        """
+        self._init_logger()
+        self.set_id(LISIRD_ID(key))
+        self.load_data()
+        self.set_extrema()
+
+    def set_extrema(self):
+        pass
+
+    @property
+    def meta(self):
+        return self.loader.meta
+
+    @property
+    def normalized(self):
+        pass
+
+    def run_normalization(self, norm_by="feature-scale"):
+        raise NotImplementedError(
+            r"""Need to fix normalization handling for each LISIRD
+quantity"""
+        )
+
+        # Note: "max" and "feature-scale" are the same if the min(SSN) = 0.
+        assert norm_by in ("max", "zscore", "feature-scale")
+        self.logger.info("Normalizing SSN by %s", norm_by)
+        self._norm_by = norm_by
+
+        if norm_by == "max":
+
+            def norm_fcn(g):
+                return g.divide(g.max())
+
+        elif norm_by == "zscore":
+
+            def norm_fcn(g):
+                return g.subtract(g.mean()).divide(g.std())
+
+        elif norm_by == "feature-scale":
+
+            def norm_fcn(g):
+                return g.subtract(g.min()).divide(g.max() - g.min())
+
+        normalized = {
+            k: self._run_normalization(v, norm_fcn) for k, v in self.data.items()
+        }
+        normalized = pd.concat(normalized, axis=1)
+
+        normed_interpolated = None
+        try:
+            interpolated = self.interpolated
+            normed_interpolated = {
+                k: self._run_normalization(v, norm_fcn) for k, v in interpolated.items()
+            }
+            normed_interpolated = pd.concat(normed_interpolated, axis=1)
+        except AttributeError:
+            pass
+
+        return normalized, normed_interpolated
+
+    run_normalization.__doc__ = ActivityIndicator.run_normalization
+
+    def load_data(self):
+        loader = LISIRDLoader(self.id.key, self.id.url)
+        loader.load_data()
+        self._loader = loader
+
+    def interpolate_data(self, target_index):
+        trans = {
+            "Lalpha": "irradiance",
+            "MgII": "mg_index",
+            "CaK": "emdx",  # Other CaK data is available, but unsure how to use.
+            # Per <https://www.spaceweather.gc.ca/solarflux/sx-3-en.php>, adjusted value is scaled for variation in Earth-Sun distance
+            "f107-noaa": "adjusted_flux",
+            "f107-penticton": "adjusted_flux",
+        }
+
+        source = self.data.loc[:, trans[self.id.key]].dropna(how="any", axis=0)
+        interpolated = super(LISIRD, self).interpolate_data(source, target_index)
+        self._interpolated = interpolated
+        return interpolated
+
+
+class LISIRDExtrema(IndicatorExtrema):
+    @property
+    def extrema_calculator(self):
+        r""":py:class:`ExtremaCalculator` used to calculate the extrema."""
+        return self._extrema_calculator
+
+    def load_or_set_data(self, *args, **kwargs):
+        r"""Get extrema from :py:class:`ExtremaCalculator`."""
+        ec = ExtremaCalculator(*args, **kwargs)
+        extrema = ec.formatted_extrema
+        self._data = extrema
+        self._extrema_calculator = ec