divinity-forecast 0.2.0__py3-none-any.whl

divinity/synthetic.py

@@ -0,0 +1,217 @@
+"""
+Synthetic test series with known structure, split into history and future.
+
+These are used by the tests, the README figures and the demo notebook, and
+are handy for experimenting. Every scenario is reproducible from its seed.
+"""
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass
+class Scenario:
+    """
+    A synthetic forecasting problem.
+
+    :param name: short identifier.
+    :param description: what the series contains and what a good model
+        should do with it.
+    :param history: training series.
+    :param future: true values over the forecast horizon.
+    :param options: Forecaster arguments this scenario calls for.
+    """
+
+    name: str
+    description: str
+    history: pd.Series
+    future: pd.Series
+    options: dict = field(default_factory=dict)
+
+    @property
+    def steps(self):
+        return len(self.future)
+
+
+def _split(y, horizon):
+    return y.iloc[:-horizon], y.iloc[-horizon:]
+
+
+def trend_seasonal(seed=0, n=200, horizon=40):
+    rng = np.random.default_rng(seed)
+    t = np.arange(n + horizon)
+    y = 0.05 * t + 2 * np.sin(2 * np.pi * t / 20) + rng.normal(0, 0.5, len(t))
+    history, future = _split(pd.Series(y), horizon)
+    return Scenario(
+        "trend_seasonal",
+        "Linear trend plus a period 20 cycle and noise. The standard case.",
+        history,
+        future,
+    )
+
+
+def daily_multi_seasonal(seed=0, years=3, horizon=90):
+    rng = np.random.default_rng(seed)
+    dates = pd.date_range("2022-01-01", periods=int(365.25 * years) + horizon, freq="D")
+    t = np.arange(len(dates))
+    weekly = np.where(dates.day_of_week >= 5, -3.0, 0.6)  # sharp weekend dip
+    yearly = 4 * np.sin(2 * np.pi * t / 365.25)
+    y = 50 + 0.01 * t + weekly + yearly + rng.normal(0, 1.0, len(t))
+    history, future = _split(pd.Series(y, index=dates), horizon)
+    return Scenario(
+        "daily_multi_seasonal",
+        "Daily data with a square weekend dip and a yearly cycle. Weekly and "
+        "yearly periods come from the date frequency.",
+        history,
+        future,
+    )
+
+
+def monthly_multiplicative(seed=0, years=12, horizon=24):
+    rng = np.random.default_rng(seed)
+    dates = pd.date_range("2012-01-01", periods=12 * years + horizon, freq="MS")
+    t = np.arange(len(dates))
+    log_y = (
+        np.log(100)
+        + 0.01 * t
+        + 0.2 * np.sin(2 * np.pi * t / 12)
+        + 0.1 * np.cos(4 * np.pi * t / 12)
+        + rng.normal(0, 0.04, len(t))
+    )
+    history, future = _split(pd.Series(np.exp(log_y), index=dates), horizon)
+    return Scenario(
+        "monthly_multiplicative",
+        "Monthly growth series whose seasonal swing grows with its level. "
+        "Fitted on the log scale.",
+        history,
+        future,
+        {"log": True},
+    )
+
+
+def white_noise(seed=0, n=200, horizon=50):
+    rng = np.random.default_rng(seed)
+    history, future = _split(pd.Series(10 + rng.normal(0, 1, n + horizon)), horizon)
+    return Scenario(
+        "white_noise",
+        "Pure noise around a constant. There is nothing to find, so the "
+        "forecast should be flat with no invented seasonality or trend.",
+        history,
+        future,
+    )
+
+
+def random_walk(seed=0, n=300, horizon=60):
+    rng = np.random.default_rng(seed)
+    y = np.cumsum(rng.normal(0, 1, n + horizon))
+    history, future = _split(pd.Series(y), horizon)
+    return Scenario(
+        "random_walk",
+        "A random walk. It wanders and looks trendy by chance, so the model "
+        "should not extrapolate a confident trend and its interval should "
+        "widen with the horizon.",
+        history,
+        future,
+    )
+
+
+def ar1(seed=0, n=200, horizon=50, phi=0.85):
+    rng = np.random.default_rng(seed)
+    e = rng.normal(0, 1, n + horizon + 100)
+    u = np.zeros_like(e)
+    for i in range(1, len(e)):
+        u[i] = phi * u[i - 1] + e[i]
+    history, future = _split(pd.Series(20 + u[100:]), horizon)
+    return Scenario(
+        "ar1",
+        "Strongly autocorrelated but stationary. Forecast should decay back "
+        "towards the mean rather than run off as if it were a random walk.",
+        history,
+        future,
+    )
+
+
+def missing_data(seed=0, n=200, horizon=40):
+    base = trend_seasonal(seed, n, horizon)
+    rng = np.random.default_rng(seed + 1000)
+    history = base.history.copy()
+    history[rng.random(n) < 0.15] = np.nan
+    history.iloc[90:115] = np.nan
+    return Scenario(
+        "missing_data",
+        "The standard case with 15% of points missing at random and a " "25 step gap.",
+        history,
+        base.future,
+    )
+
+
+def unknown_period(seed=0, n=240, horizon=60):
+    rng = np.random.default_rng(seed)
+    t = np.arange(n + horizon)
+    y = 3 * np.sin(2 * np.pi * t / 17.3) + rng.normal(0, 1, len(t))
+    history, future = _split(pd.Series(y), horizon)
+    return Scenario(
+        "unknown_period",
+        "A period of 17.3 steps that nobody told the model about. It has to "
+        "find it in the periodogram.",
+        history,
+        future,
+    )
+
+
+def outliers(seed=0, n=200, horizon=40):
+    base = trend_seasonal(seed, n, horizon)
+    history = base.history.copy()
+    history.iloc[[40, 120, 170]] += np.array([15.0, -12.0, 18.0])
+    return Scenario(
+        "outliers",
+        "The standard case with three large one-off spikes.",
+        history,
+        base.future,
+    )
+
+
+def short_daily(seed=0, days=35, horizon=14):
+    rng = np.random.default_rng(seed)
+    dates = pd.date_range("2024-03-01", periods=days + horizon, freq="D")
+    weekly = np.where(dates.day_of_week >= 5, 5.0, 0.0)
+    y = 20 + weekly + rng.normal(0, 1, len(dates))
+    history, future = _split(pd.Series(y, index=dates), horizon)
+    return Scenario(
+        "short_daily",
+        "Only five weeks of daily data. The yearly default period cannot be "
+        "estimated and must be ignored, while the weekly cycle is still used.",
+        history,
+        future,
+    )
+
+
+SCENARIOS = {
+    f.__name__: f
+    for f in (
+        trend_seasonal,
+        daily_multi_seasonal,
+        monthly_multiplicative,
+        white_noise,
+        random_walk,
+        ar1,
+        missing_data,
+        unknown_period,
+        outliers,
+        short_daily,
+    )
+}
+
+
+def make(name, seed=0, **kwargs):
+    """
+    Build a scenario by name, e.g. ``make("random_walk", seed=3)``.
+    """
+    try:
+        return SCENARIOS[name](seed=seed, **kwargs)
+    except KeyError:
+        raise ValueError(
+            "unknown scenario {!r}, choose from {}".format(name, sorted(SCENARIOS))
+        ) from None

divinity_forecast-0.2.0.dist-info/METADATA

@@ -0,0 +1,251 @@
+Metadata-Version: 2.4
+Name: divinity-forecast
+Version: 0.2.0
+Summary: Automatic timeseries forecasting with honest prediction intervals
+Author-email: dstarkey23 <drds.gh1@gmail.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/drds1/divinity
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Requires-Dist: scipy>=1.10
+Requires-Dist: statsmodels>=0.14
+Requires-Dist: matplotlib>=3.7
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pre-commit>=3; extra == "dev"
+Requires-Dist: nbclient>=0.10; extra == "dev"
+Requires-Dist: nbformat>=5.10; extra == "dev"
+Requires-Dist: ipykernel>=6; extra == "dev"
+Dynamic: license-file
+
+# Divinity
+
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+Automatic timeseries forecasting with prediction intervals you can trust.
+
+Give it a series and a horizon. Divinity decides whether there is a trend, which seasonal cycles are real and how much autocorrelation is left over. It returns a forecast with an interval that widens as much as the data say it should, and no more.
+
+![Daily data with weekly and yearly cycles](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/daily_multi_seasonal.png)
+
+## Install
+
+```bash
+pip install divinity-forecast
+```
+
+The package installs as `divinity-forecast` but is imported as `divinity`. Version 0.1 was published as `divinity` and is no longer maintained there.
+
+Or from a clone, with the development tools
+
+```bash
+pip install -e ".[dev]"
+```
+
+Python 3.10 or later.
+
+## Quick start
+
+```python
+import divinity as dv
+
+model = dv.Forecaster().fit(y)      # y is a pandas Series, numpy array or list
+result = model.predict(steps=30)    # DataFrame with forecast, lower, upper
+print(model.summary())
+model.plot(result)
+```
+
+```text
+Divinity forecaster
+  observations   1095
+  frequency      D
+  log scale      no
+  trend          linear
+  seasonality    365.25 (1 harmonic), 7 (3 harmonics)
+  residuals      ARIMA(0, 0, 0)
+```
+
+Or in one line
+
+```python
+result = dv.forecast(y, steps=30)
+```
+
+`result` is indexed by future time. A Series with a `DatetimeIndex` gives future dates, anything else gives integer steps following on from the data.
+
+```text
+            forecast  lower  upper
+2024-12-31     61.48  59.55  63.41
+2025-01-01     61.55  59.62  63.49
+...
+```
+
+The [demo notebook](https://github.com/drds1/divinity/blob/master/notebooks/demo.ipynb) walks through everything below with live examples.
+
+## How it works
+
+The series is modelled as a constant, plus an optional linear trend, plus Fourier terms for each seasonal period, plus ARIMA errors (regression with ARIMA errors).
+
+1. **Candidate periods.** These come from the date frequency (7 and 365.25 for daily data, 12 for monthly, 24 and 168 for hourly and so on) and from peaks in the periodogram, so a cycle nobody mentioned can still be found. A period is only considered if the data cover at least two full cycles.
+2. **Feature selection.** Trend and seasonal terms are added one at a time while they improve BIC. Extra harmonics of a period (for sharper, non-sinusoidal shapes) are only tried once the lower harmonic is in. There is an additional penalty for searching over many candidates. Without it, testing dozens of periods on noise will find one by chance.
+3. **Random walk or not?** Before choosing any features, an augmented Dickey-Fuller test is run on what is left after removing the trend and the known periods. Detected periods are left out because they might be spurious. Testing first matters. On a random walk, spurious features soak up the wander and make the leftovers look stationary, so a test afterwards confirms its own mistake.
+4. **Allowing for autocorrelation.** Least squares assumes independent errors, and on a random walk it declares almost any trend "significant". If the series has a unit root, features are selected on differenced data. Otherwise the autocorrelation of the residuals is estimated and features are reselected on quasi-differenced data (feasible GLS), repeating until the choice settles. The ARIMA order for what remains is chosen by BIC over a small grid.
+5. **Joint fit and intervals.** Features and ARIMA errors are fitted together by maximum likelihood. Forecast intervals combine the ARIMA forecast variance with the uncertainty in the trend and seasonal coefficients, which matters most when extrapolating a trend a long way. Student t quantiles allow for the noise level being estimated, which matters for short series.
+
+The fitted model does not depend on the forecast horizon, so `predict(10)` is exactly the first ten rows of `predict(100)`.
+
+## Can it be fooled?
+
+The cases below are built to catch out a naive forecaster. Each was generated by [`divinity.synthetic`](https://github.com/drds1/divinity/blob/master/divinity/synthetic.py) with seed 0, the same seed the smoke tests use, so none of them were picked because they looked good. In every chart the orange line is the true future, which the model never saw.
+
+### The standard case
+
+A linear trend, a 20 step cycle and noise. Neither component is specified.
+
+![Trend plus seasonality](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/trend_seasonal.png)
+
+### Multiplicative seasonality
+
+The seasonal swing grows with the level. Fitting with `log=True` handles this.
+
+![Monthly multiplicative](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/monthly_multiplicative.png)
+
+### Pure noise
+
+There is nothing to find. A model that looks hard enough for structure will find some anyway. Divinity returns a flat forecast at the mean.
+
+![White noise](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/white_noise.png)
+
+### A random walk
+
+Random walks look trendy. Ordinary least squares picks a "significant" trend in the large majority of them, which is then extrapolated with false confidence. Divinity finds no trend or seasonality, switches to a differenced model and lets the interval fan out.
+
+![Random walk](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/random_walk.png)
+
+### Autocorrelated but stationary
+
+The opposite trap. The series wanders but always returns to its mean. The forecast decays back towards the mean rather than treating it as a random walk.
+
+![AR(1)](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/ar1.png)
+
+### A period nobody mentioned
+
+A cycle of 17.3 steps on integer data with no frequency. It is found from the periodogram, including the fractional part.
+
+![Unknown period](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/unknown_period.png)
+
+### Missing data
+
+15% of values missing at random plus a 25 step gap. Gaps are handled natively, with no interpolation needed.
+
+![Missing data](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/missing_data.png)
+
+### Outliers
+
+Three large one-off spikes.
+
+![Outliers](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/outliers.png)
+
+### Very short history
+
+Five weeks of daily data. The yearly default period cannot be estimated from that and is dropped automatically, while the weekly cycle is kept.
+
+![Short daily](https://raw.githubusercontent.com/drds1/divinity/master/docs/images/short_daily.png)
+
+### Across 100 random seeds per case
+
+One chart per case could still be luck, so [`benchmarks/coverage.py`](https://github.com/drds1/divinity/blob/master/benchmarks/coverage.py) refits every scenario on 100 different seeds. *Interval coverage* is how often the true future falls inside the 95% interval, so it should be close to 95%. *Structure found* is how often the model chose the components the series was built with (for example no seasonality on noise, differencing on a random walk, period 17.3 when it is hidden). *RMSE vs naive* compares the error with simply repeating the last observed value, where below 1 is better.
+
+| scenario | runs | 95% interval coverage | structure found | RMSE vs naive |
+|---|---:|---:|---:|---:|
+| `trend_seasonal` | 100 | 94.3% | 100% | 0.24 |
+| `daily_multi_seasonal` | 100 | 95.1% | 100% | 0.32 |
+| `monthly_multiplicative` | 100 | 93.9% | 97% | 0.20 |
+| `white_noise` | 100 | 94.8% | 98% | 0.75 |
+| `random_walk` | 100 | 93.9% | 94% | 1.04 |
+| `ar1` | 100 | 91.8% | 88% | 0.82 |
+| `missing_data` | 100 | 93.9% | 100% | 0.23 |
+| `unknown_period` | 100 | 95.0% | 100% | 0.27 |
+| `outliers` | 100 | 100.0% | 100% | 0.43 |
+| `short_daily` | 100 | 91.4% | 100% | 0.38 |
+
+How to read the weaker rows
+
+- **`ar1` (91.8%).** Autoregressive coefficients estimated from 200 points are biased slightly towards zero, so the intervals come out a little narrow. In 12% of runs the model reads the series as a random walk or adds a small trend. Both are hard to tell apart from a strongly autocorrelated series of this length.
+- **`random_walk` (RMSE 1.04).** For a random walk the naive forecast is the best possible point forecast, so about 1.0 is the floor. What matters here is the interval, and 94% coverage means the fan is honest.
+- **`short_daily` (91.4%).** 35 points carrying seven seasonal coefficients leaves little to estimate the noise from, even with t quantiles.
+- **`outliers` (100%).** The spikes inflate the estimated noise, so the intervals are wider than they need to be. This errs in the safe direction.
+
+Regenerate it with
+
+```bash
+python benchmarks/coverage.py --seeds 100
+```
+
+## Options
+
+```python
+dv.Forecaster(
+    seasonal_periods="auto",  # "auto" (from the frequency), a list of periods, or None
+    detect_periods=True,      # also look for periods in the data
+    max_detected_periods=3,
+    max_harmonics=3,          # Fourier harmonics per period, more allow sharper shapes
+    trend="auto",             # "auto", "linear" or None
+    arima="auto",             # "auto", a (p, d, q) tuple, or None for white noise
+    log=False,                # fit on the log scale, for growth with growing seasonality
+    interval=0.95,            # default interval coverage
+    search_penalty=1.0,       # higher is more reluctant to add seasonality, 0 is plain BIC
+)
+```
+
+Useful things on a fitted model
+
+| | |
+|---|---|
+| `predict(steps, interval=None)` | forecast DataFrame (`forecast`, `lower`, `upper`) |
+| `plot(forecast, actual=None, history=None)` | chart, returns the matplotlib Axes |
+| `summary()` | readable description of what was chosen |
+| `components_` | the same as a dict |
+| `fitted_` | in-sample one step ahead predictions |
+
+If dates are missing entirely from a `DatetimeIndex` the frequency cannot be inferred. Pass it, for example `fit(y, freq="D")`, and the gaps become missing values.
+
+## Limitations
+
+- Trends are linear. A curve or saturation will be extrapolated as a straight line.
+- Level shifts and regime changes are not modelled.
+- Outliers are not removed. They widen the interval a little instead.
+- Seasonal shape is fixed (or proportional to the level with `log=True`).
+- One series at a time, with no external regressors.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pre-commit install
+pytest -m "not slow"      # fast unit and smoke tests
+pytest                    # also the coverage checks over many seeds (a few minutes)
+python docs/make_figures.py
+```
+
+The code is in [`divinity/`](https://github.com/drds1/divinity/tree/master/divinity/), with one module per stage: `periods.py` (candidate periods), `features.py` (trend and Fourier terms), `selection.py` (feature selection), `residuals.py` (ARIMA order) and `forecaster.py` (the `Forecaster` class tying them together). `synthetic.py` holds the test scenarios.
+
+### Upgrading from 0.1
+
+The old `divinity.divinity(forecast_length=...)` class still works but is deprecated and now runs the new model underneath. Replace
+
+```python
+model = dv.divinity(forecast_length=20)
+model.fit(y)
+y_pred = model.predict()
+```
+
+with
+
+```python
+result = dv.Forecaster().fit(y).predict(20)
+y_pred = result["forecast"]
+```

divinity_forecast-0.2.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+divinity/__init__.py,sha256=IY4Z0aZa_uYUvWy0uxhjgXNLNXlb1P6HKBbcsX3majo,556
+divinity/features.py,sha256=b2B6u9WbLmucjA-30f59QV7N1t1icKOCMtN7PRIbiJE,3063
+divinity/forecaster.py,sha256=C9qFqpBlsXCugOyNctdb2cKtT0ohGesY_AHVlBCtEmo,18108
+divinity/legacy.py,sha256=OYZvniAnJqi38FxBvjdg3jJS3qOLkjAaU-Ksh7xyQfw,4385
+divinity/periods.py,sha256=u5UDf6zdF8DAfBrnOxYEoe7pCCbBrxIzo_4czAM5iNY,3394
+divinity/plotting.py,sha256=daAyOPcYjcXzCHmUAa0GnsZkQ5qhsV-qne7zwMx_Z_k,2483
+divinity/residuals.py,sha256=1DR4ah8yQ2IAGHwJ4BYIxuIkpYs-hYh1dvayXEXgGe0,5088
+divinity/selection.py,sha256=fj3KTk70BNP5k_44yCj9roIkIqcGcPhBIUEPHlPU3Do,5305
+divinity/synthetic.py,sha256=W0ob316wuFw188bX1ie9jwfEmI42KLBnUD1SngBNmR0,6535
+divinity_forecast-0.2.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+divinity_forecast-0.2.0.dist-info/METADATA,sha256=kwOocw5PHWn75jyLBhgVqZ30RgTIYFApBgaiiUsBvu4,12058
+divinity_forecast-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+divinity_forecast-0.2.0.dist-info/top_level.txt,sha256=s36saxd77ypggF7o4cQ1KM2WfKBY6b0MqDzaNbG7DZw,9
+divinity_forecast-0.2.0.dist-info/RECORD,,

divinity_forecast-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+