divinity-forecast 0.2.0__py3-none-any.whl

divinity/legacy.py

@@ -0,0 +1,138 @@
+"""
+The version 0.1 interface, kept so existing code keeps working.
+New code should use :class:`divinity.Forecaster`.
+"""
+
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from .forecaster import Forecaster
+
+
+class divinity:
+    """
+    Deprecated version 0.1 interface, now a thin wrapper around
+    :class:`Forecaster`.
+
+    :param forecast_length: number of steps returned by :meth:`predict`.
+    :param confidence_interval: interval coverage in percent.
+    :param seasonal_periods: candidate seasonal periods. None uses the
+        Forecaster defaults.
+    :param trend_order: [0] for no trend, anything including 1 for "auto".
+    :param kwargs: other version 0.1 arguments are accepted and ignored.
+    """
+
+    def __init__(
+        self,
+        forecast_length,
+        confidence_interval=95.0,
+        seasonal_periods=None,
+        trend_order=(0, 1),
+        **kwargs,
+    ):
+        warnings.warn(
+            "divinity.divinity is deprecated, use divinity.Forecaster",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.forecast_length = forecast_length
+        self.confidence_interval = confidence_interval
+        self.model = Forecaster(
+            seasonal_periods="auto" if seasonal_periods is None else seasonal_periods,
+            trend="auto" if max(trend_order) >= 1 else None,
+            interval=confidence_interval / 100.0,
+        )
+
+    @property
+    def features(self):
+        return self.model._features(np.arange(self.model.n_))
+
+    def fit(self, y):
+        self.model.fit(y)
+
+    def predict(self):
+        """
+        :return: numpy array of the forecast. The half width of the
+            prediction interval is stored in ``ystd``.
+        """
+        result = self.model.predict(self.forecast_length)
+        self.ypred = result["forecast"].to_numpy()
+        self.ystd = ((result["upper"] - result["lower"]) / 2).to_numpy()
+        return self.ypred
+
+    def forecast(self, steps):
+        self.forecast_length = steps
+        return self.predict()
+
+
+def gen_season(N, periods=(10, 20), sine_amplitudes=(1, 1), cosine_amplitudes=(0, 0)):
+    """
+    Sinusoidal features and their weighted sum.
+
+    :return: dict with "features" (DataFrame) and "target" (array).
+    """
+    t = np.arange(N)
+    features = {}
+    X = np.zeros(N)
+    for idx, p in enumerate(periods):
+        if p == 0 or p > N / 2:
+            warnings.warn("period {} ignored, must be in (0, N/2]".format(p))
+            continue
+        s = np.sin(2 * np.pi / p * t)
+        c = np.cos(2 * np.pi / p * t)
+        if sine_amplitudes is not None:
+            X += sine_amplitudes[idx] * s
+        if cosine_amplitudes is not None:
+            X += cosine_amplitudes[idx] * c
+        features["sin P=" + str(p)] = s
+        features["cos P=" + str(p)] = c
+    return {"features": pd.DataFrame(features), "target": X}
+
+
+def gen_trend(N, coef=(0, 1, 2), amplitudes=(1, 0.5, 0.2)):
+    """
+    Polynomial trend features and their weighted sum.
+
+    :return: dict with "features" (DataFrame) and "target" (array).
+    """
+    t = np.arange(N)
+    features = {}
+    X = np.zeros(N)
+    for idx, order in enumerate(coef):
+        f = np.array(t**order, dtype=float)
+        if amplitudes is not None:
+            X += amplitudes[idx] * f
+        features["trend order " + str(order)] = f
+    return {"features": pd.DataFrame(features), "target": X}
+
+
+def gen_fake(
+    N,
+    trend_order,
+    trend_amplitudes,
+    seasonal_periods,
+    seasonal_amplitudes_sine,
+    seasonal_amplitudes_cosine,
+    normalize="0to1",
+):
+    """
+    Synthetic trend plus seasonal series and its feature matrix. Prefer
+    :mod:`divinity.synthetic` for new code.
+
+    :return: dict with "features" (DataFrame) and "target" (array).
+    """
+    trend = gen_trend(N, coef=list(range(trend_order + 1)), amplitudes=trend_amplitudes)
+    season = gen_season(
+        N,
+        periods=seasonal_periods,
+        sine_amplitudes=seasonal_amplitudes_sine,
+        cosine_amplitudes=seasonal_amplitudes_cosine,
+    )
+    features = pd.concat([trend["features"], season["features"]], axis=1)
+    if normalize == "0to1":
+        spread = features.max() - features.min()
+        cols = spread[spread > 0].index
+        features[cols] = (features[cols] - features[cols].min()) / spread[cols]
+    return {"features": features, "target": trend["target"] + season["target"]}

divinity/periods.py

@@ -0,0 +1,106 @@
+"""
+Candidate seasonal periods, from the series frequency and from the data.
+"""
+
+import numpy as np
+import pandas as pd
+from scipy import signal
+
+
+def default_periods(freq):
+    """
+    Sensible seasonal periods (in time steps) for a pandas frequency.
+    Returns an empty list if the frequency is unknown.
+    """
+    if freq is None:
+        return []
+    offset = pd.tseries.frequencies.to_offset(freq)
+    if offset.n != 1:
+        return []
+    o = pd.offsets
+    if isinstance(offset, o.BusinessDay):
+        return [5.0, 261.0]
+    if isinstance(offset, o.Day):
+        return [7.0, 365.25]
+    if isinstance(offset, o.Hour):
+        return [24.0, 168.0]
+    if isinstance(offset, o.Minute):
+        return [60.0, 1440.0]
+    if isinstance(offset, o.Week):
+        return [365.25 / 7]
+    if isinstance(offset, (o.MonthBegin, o.MonthEnd)):
+        return [12.0]
+    if isinstance(offset, (o.QuarterBegin, o.QuarterEnd)):
+        return [4.0]
+    return []
+
+
+def detect_periods(y, max_periods=3, min_period=2.0, max_period=None, oversample=8):
+    """
+    Find the strongest periodicities in a series from its periodogram.
+
+    The series is linearly detrended first (gaps are interpolated for this
+    step only). The periodogram is zero padded so that periods falling
+    between Fourier frequencies are located accurately.
+
+    :param y: 1D array, may contain NaN.
+    :param max_periods: maximum number of periods to return.
+    :param min_period: shortest period considered, in time steps.
+    :param max_period: longest period considered. Defaults to half the
+        series length, so at least two full cycles are observed.
+    :return: list of periods, strongest first.
+    """
+    y = pd.Series(np.asarray(y, dtype=float)).interpolate(limit_direction="both")
+    y = y.to_numpy()
+    n = len(y)
+    if n < 8 or np.allclose(y, y[0]):
+        return []
+    if max_period is None:
+        max_period = n / 2
+    y = signal.detrend(y)
+    freqs, power = signal.periodogram(y, nfft=oversample * n, detrend=False)
+    with np.errstate(divide="ignore"):
+        periods = 1.0 / freqs
+    valid = (periods >= min_period) & (periods <= max_period)
+    if not np.any(valid):
+        return []
+    peaks, _ = signal.find_peaks(np.where(valid, power, 0.0))
+    peaks = peaks[valid[peaks]]
+    order = peaks[np.argsort(power[peaks])[::-1]]
+
+    found = []
+    for idx in order:
+        p = periods[idx]
+        # skip harmonics and near duplicates of periods already found
+        if any(_is_related(p, q) for q in found):
+            continue
+        found.append(float(p))
+        if len(found) == max_periods:
+            break
+    return found
+
+
+def _is_related(p, q, tol=0.05):
+    """
+    True if period p is within ``tol`` of q or of one of its harmonics q/k.
+    """
+    for k in range(1, 6):
+        if abs(p - q / k) <= tol * q / k:
+            return True
+    return False
+
+
+def merge_periods(known, detected, n, tol=0.05):
+    """
+    Combine known and detected periods, dropping detected periods that match
+    a known one and any period that cannot be seen twice in ``n`` steps.
+
+    :return: (known, detected) with invalid entries removed.
+    """
+    known = [float(p) for p in known if 2 <= p <= n / 2]
+    detected = [
+        float(p)
+        for p in detected
+        if 2 <= p <= n / 2 and not any(_is_related(p, q, tol) for q in known)
+    ]
+    return known, detected

divinity/plotting.py

@@ -0,0 +1,77 @@
+"""
+Forecast plots. Colours follow a colour blind safe categorical palette
+(history in neutral grey, forecast in blue, true future values in orange).
+"""
+
+import numpy as np
+import pandas as pd
+
+HISTORY = "#52514e"
+FORECAST = "#2a78d6"
+ACTUAL = "#eb6834"
+GRID = "#e4e3df"
+SURFACE = "#fcfcfb"
+
+
+def plot_forecast(history, forecast, actual=None, ax=None, title=None):
+    """
+    Plot history, forecast with its prediction interval, and optionally the
+    true future values.
+
+    :param history: Series of observed values.
+    :param forecast: DataFrame with columns forecast, lower and upper.
+    :param actual: optional Series or array of true future values.
+    :param ax: matplotlib Axes, a new figure is made if None.
+    :param title: optional title.
+    :return: the matplotlib Axes.
+    """
+    import matplotlib.pyplot as plt
+
+    if ax is None:
+        _, ax = plt.subplots(figsize=(9, 3.9))
+    ax.set_facecolor(SURFACE)
+    ax.figure.set_facecolor(SURFACE)
+
+    pct = int(round(100 * forecast.attrs.get("interval", 0.95)))
+    ax.plot(history.index, history.values, color=HISTORY, lw=1.5, label="history")
+    ax.fill_between(
+        forecast.index,
+        forecast["lower"],
+        forecast["upper"],
+        color=FORECAST,
+        alpha=0.15,
+        lw=0,
+        label="{}% interval".format(pct),
+    )
+    ax.plot(
+        forecast.index, forecast["forecast"], color=FORECAST, lw=2, label="forecast"
+    )
+    if actual is not None:
+        if not isinstance(actual, pd.Series):
+            actual = pd.Series(np.asarray(actual), index=forecast.index[: len(actual)])
+        ax.plot(actual.index, actual.values, color=ACTUAL, lw=1.5, label="actual")
+
+    ax.grid(True, color=GRID, lw=0.8)
+    ax.set_axisbelow(True)
+    for side in ("top", "right"):
+        ax.spines[side].set_visible(False)
+    for side in ("left", "bottom"):
+        ax.spines[side].set_color(GRID)
+    ax.tick_params(colors=HISTORY, labelsize=9)
+    if isinstance(forecast.index, pd.DatetimeIndex):
+        import matplotlib.dates as mdates
+
+        locator = mdates.AutoDateLocator()
+        ax.xaxis.set_major_locator(locator)
+        ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(locator))
+    # below the axes so it never covers the data
+    ax.legend(
+        frameon=False,
+        fontsize=9,
+        loc="upper left",
+        bbox_to_anchor=(0, -0.12),
+        ncol=4,
+    )
+    if title:
+        ax.set_title(title, loc="left", fontsize=11, color="#0b0b0b")
+    return ax

divinity/residuals.py

@@ -0,0 +1,146 @@
+"""
+Choosing the ARIMA model for whatever the trend and seasonality leave behind.
+"""
+
+import warnings
+
+import numpy as np
+import pandas as pd
+from statsmodels.tools.sm_exceptions import ConvergenceWarning
+from statsmodels.tsa.arima.model import ARIMA
+from statsmodels.tsa.adfvalues import mackinnonp
+from statsmodels.tsa.stattools import adfuller
+
+
+def has_unit_root(resid, alpha=0.05, trend=True):
+    """
+    Augmented Dickey-Fuller test for a unit root (random walk behaviour).
+
+    Set ``trend`` to True if a trend was removed from the series (or could
+    be present). Testing detrended data against the no-trend critical
+    values rejects the unit root far too often, so random walks would be
+    mistaken for mean reverting series.
+
+    The null hypothesis is a unit root, so the series is treated as a random
+    walk unless the test rejects at ``alpha``. When in doubt this errs
+    towards differencing, which gives wider but honest intervals, rather
+    than towards mean reversion, which gives confident wrong ones.
+    """
+    r = np.asarray(resid, dtype=float)
+    observed = r[~np.isnan(r)]
+    if len(observed) < 20 or np.allclose(observed, observed[0]):
+        return False
+    regression = "ct" if trend else "c"
+    if np.isnan(r).any():
+        r = fill_short_gaps(r)
+    if np.isnan(r).any():
+        p_value = _adf_with_gaps(r, regression)
+        if p_value is not None:
+            return p_value > alpha
+        # too few complete rows, fall back to interpolating the gaps
+        r = pd.Series(r).interpolate(limit_direction="both").to_numpy()
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        p_value = adfuller(r, regression=regression, autolag="AIC")[1]
+    return p_value > alpha
+
+
+def fill_short_gaps(r, max_gap=2):
+    """
+    Linearly interpolate runs of at most ``max_gap`` missing values that
+    have data on both sides. Longer gaps are left as NaN.
+    """
+    r = np.asarray(r, dtype=float).copy()
+    missing = np.isnan(r)
+    filled = pd.Series(r).interpolate(limit_area="inside").to_numpy()
+    i = 0
+    while i < len(r):
+        if missing[i]:
+            j = i
+            while j < len(r) and missing[j]:
+                j += 1
+            if j - i <= max_gap:
+                r[i:j] = filled[i:j]
+            i = j
+        else:
+            i += 1
+    return r
+
+
+def _adf_with_gaps(r, regression, max_lag=4, min_rows=30):
+    """
+    ADF test on a series with missing values, using only the rows where the
+    value and all its lags are observed. Interpolating instead would turn
+    long gaps into smooth straight lines, which look like a random walk.
+
+    The lag length is chosen by AIC on a common sample, and the p-value
+    comes from MacKinnon's tables, as in statsmodels' adfuller.
+
+    :return: p-value, or None if there are too few complete rows.
+    """
+    n = len(r)
+    dy = np.full(n, np.nan)
+    dy[1:] = np.diff(r)
+    t = np.arange(n, dtype=float)
+    rows = np.arange(max_lag + 1, n)
+    lags = np.column_stack([dy[rows - k] for k in range(1, max_lag + 1)])
+    target = dy[rows]
+    level = r[rows - 1]
+    base = [level, np.ones(len(rows))]
+    if regression == "ct":
+        base.append(t[rows] / n)
+    base = np.column_stack(base)
+    ok = ~(np.isnan(target) | np.isnan(level) | np.isnan(lags).any(axis=1))
+    if ok.sum() < min_rows:
+        return None
+    y, base, lags = target[ok], base[ok], lags[ok]
+    m = len(y)
+
+    best = None
+    for k in range(max_lag + 1):
+        X = np.hstack([base, lags[:, :k]])
+        beta, *_ = np.linalg.lstsq(X, y, rcond=None)
+        rss = float(np.sum((y - X @ beta) ** 2))
+        aic = m * np.log(max(rss, 1e-300) / m) + 2 * X.shape[1]
+        if best is None or aic < best[0]:
+            best = (aic, X, beta, rss)
+    _, X, beta, rss = best
+    sigma2 = rss / (m - X.shape[1])
+    cov = sigma2 * np.linalg.pinv(X.T @ X)
+    t_stat = beta[0] / np.sqrt(cov[0, 0])
+    return float(mackinnonp(t_stat, regression=regression, N=1))
+
+
+def fit_quietly(model):
+    """
+    Fit a statsmodels model, silencing the routine convergence chatter.
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        warnings.simplefilter("ignore", UserWarning)
+        return model.fit()
+
+
+def select_order(resid, d, max_p=2, max_q=2):
+    """
+    Pick (p, d, q) for the residuals by BIC over a small grid.
+
+    :param resid: residual series (NaN allowed).
+    :param d: differencing order decided beforehand (0 or 1).
+    :return: (order, fitted results) for the best model.
+    """
+    best = None
+    trend = "c" if d == 0 else "n"
+    for p in range(max_p + 1):
+        for q in range(max_q + 1):
+            try:
+                res = fit_quietly(ARIMA(resid, order=(p, d, q), trend=trend))
+            except (np.linalg.LinAlgError, ValueError):
+                continue
+            if not np.isfinite(res.bic):
+                continue
+            if best is None or res.bic < best[1].bic:
+                best = ((p, d, q), res)
+    if best is None:
+        return (0, d, 0), None
+    return best

divinity/selection.py

@@ -0,0 +1,157 @@
+"""
+Forward selection of trend and seasonal feature groups.
+
+Groups are scored with BIC plus an extra penalty for searching over many
+candidates (in the spirit of the extended BIC of Chen and Chen, 2008).
+Without that extra term, trying dozens of candidate periods on pure noise
+will eventually find one that looks significant by chance.
+
+Selection can also be run on quasi-differenced data (y_t - rho * y_{t-1}),
+which is feasible GLS for AR(1) errors. Ordinary least squares assumes
+independent errors, so on a random walk or a strongly autocorrelated series
+it finds "significant" trends and cycles that are not there.
+"""
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from .features import design_matrix
+
+
+@dataclass
+class SelectionResult:
+    """
+    Outcome of forward selection.
+
+    :param groups: selected FeatureGroups, in the order they were added.
+    :param history: list of (group name, score) for each accepted step,
+        starting with ("constant", score).
+    """
+
+    groups: list
+    history: list = field(default_factory=list)
+
+
+def ols_rss(X, y):
+    """
+    Residual sum of squares of an ordinary least squares fit.
+    """
+    beta, *_ = np.linalg.lstsq(X, y, rcond=None)
+    resid = y - X @ beta
+    return float(resid @ resid)
+
+
+def score(rss, n, n_params, search_penalty):
+    """
+    BIC for a Gaussian linear model plus the multiple search penalty.
+    """
+    rss = max(rss, 1e-12 * n)
+    return n * np.log(rss / n) + n_params * np.log(n) + search_penalty
+
+
+def lag1_autocorrelation(resid):
+    """
+    AR(1) coefficient of a residual series, using only adjacent pairs where
+    both values are observed. Clipped to [-0.95, 1].
+    """
+    r = np.asarray(resid, dtype=float)
+    a, b = r[1:], r[:-1]
+    ok = ~(np.isnan(a) | np.isnan(b))
+    if ok.sum() < 3 or np.sum(b[ok] ** 2) == 0:
+        return 0.0
+    rho = np.sum(a[ok] * b[ok]) / np.sum(b[ok] ** 2)
+    return float(np.clip(rho, -0.95, 1.0))
+
+
+def quasi_difference(values, rho):
+    """
+    Apply x_t - rho * x_{t-1} to a 1D or 2D array (first row is dropped).
+    """
+    values = np.asarray(values, dtype=float)
+    if rho == 0:
+        return values
+    return values[1:] - rho * values[:-1]
+
+
+def forward_select(
+    y, candidates, pool_sizes, drop_columns=(), gamma=1.0, forced=(), rho=0.0
+):
+    """
+    Greedily add the candidate group that lowers the score most, until no
+    group lowers it.
+
+    A harmonic k > 1 of a period is only eligible once harmonic k - 1 of the
+    same period has been selected.
+
+    :param y: training series (NaN allowed, those rows are ignored).
+    :param candidates: list of FeatureGroup.
+    :param pool_sizes: dict of FeatureGroup to the number of alternatives it
+        was chosen from. Each selected group adds 2 * gamma * log(pool size)
+        to the score.
+    :param drop_columns: degenerate column names to leave out.
+    :param gamma: strength of the search penalty. 0 gives plain BIC.
+    :param forced: groups that are always included and carry no penalty.
+    :param rho: AR(1) coefficient of the errors. Non zero values select on
+        quasi-differenced data.
+    :return: SelectionResult
+    """
+    y = np.asarray(y, dtype=float)
+    n_train = len(y)
+    t = np.arange(n_train)
+
+    forced = list(forced)
+    candidates = [g for g in candidates if g not in forced]
+    full = design_matrix(t, forced + candidates, n_train, drop_columns)
+    columns_of = {
+        g: [c for c in design_matrix(t[:1], [g], n_train).columns if c in full.columns]
+        for g in forced + candidates
+    }
+    candidates = [g for g in candidates if columns_of[g]]
+    col_pos = {c: i for i, c in enumerate(full.columns)}
+
+    y_t = quasi_difference(y, rho)
+    X_t = quasi_difference(np.hstack([np.ones((n_train, 1)), full.to_numpy()]), rho)
+    observed = ~np.isnan(y_t)
+    y_obs = y_t[observed]
+    X_obs = X_t[observed]
+    n = len(y_obs)
+
+    def evaluate(groups):
+        idx = [0] + [1 + col_pos[c] for g in forced + groups for c in columns_of[g]]
+        X = X_obs[:, idx]
+        penalty = sum(2 * gamma * np.log(max(pool_sizes[g], 1)) for g in groups)
+        return score(ols_rss(X, y_obs), n, X.shape[1], penalty)
+
+    selected = []
+    current = evaluate(selected)
+    history = [("constant", current)]
+    while True:
+        eligible = [
+            g
+            for g in candidates
+            if g not in selected
+            and (
+                g.kind == "trend"
+                or g.harmonic == 1
+                or any(
+                    s.kind == "season"
+                    and s.period == g.period
+                    and s.harmonic == g.harmonic - 1
+                    for s in selected
+                )
+            )
+        ]
+        # never use more parameters than about a third of the data
+        n_cols = sum(len(columns_of[g]) for g in forced + selected)
+        eligible = [g for g in eligible if n_cols + len(columns_of[g]) + 1 < n / 3]
+        if not eligible:
+            break
+        scores = [evaluate(selected + [g]) for g in eligible]
+        best = int(np.argmin(scores))
+        if scores[best] >= current:
+            break
+        selected.append(eligible[best])
+        current = scores[best]
+        history.append((eligible[best].name, current))
+    return SelectionResult(forced + selected, history)