tscp 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tscp/__init__.py +45 -0
- tscp/data/__init__.py +19 -0
- tscp/data/archive.py +488 -0
- tscp/data/availability.py +55 -0
- tscp/data/io.py +109 -0
- tscp/data/splits.py +169 -0
- tscp/datasets/__init__.py +6 -0
- tscp/datasets/synthetic.py +177 -0
- tscp/features/__init__.py +22 -0
- tscp/features/base.py +64 -0
- tscp/features/calendar.py +105 -0
- tscp/features/combine.py +69 -0
- tscp/features/context.py +42 -0
- tscp/features/history.py +58 -0
- tscp/features/phase.py +41 -0
- tscp/imports.py +32 -0
- tscp/methods/__init__.py +48 -0
- tscp/methods/aci.py +44 -0
- tscp/methods/base.py +228 -0
- tscp/methods/compose.py +90 -0
- tscp/methods/engine.py +128 -0
- tscp/methods/io.py +106 -0
- tscp/methods/nexcp.py +42 -0
- tscp/methods/split.py +30 -0
- tscp/metrics/__init__.py +30 -0
- tscp/metrics/conditional.py +66 -0
- tscp/metrics/interval.py +75 -0
- tscp/metrics/report.py +135 -0
- tscp/nn/__init__.py +13 -0
- tscp/nn/data.py +85 -0
- tscp/nn/losses.py +63 -0
- tscp/nn/training.py +125 -0
- tscp/online/__init__.py +6 -0
- tscp/online/aci.py +53 -0
- tscp/online/base.py +67 -0
- tscp/py.typed +0 -0
- tscp/quantile/__init__.py +16 -0
- tscp/quantile/base.py +73 -0
- tscp/quantile/distribution.py +170 -0
- tscp/quantile/weighted.py +103 -0
- tscp/regions/__init__.py +15 -0
- tscp/regions/base.py +139 -0
- tscp/regions/box.py +82 -0
- tscp/regions/interval.py +62 -0
- tscp/registry.py +179 -0
- tscp/scores/__init__.py +9 -0
- tscp/scores/absolute.py +22 -0
- tscp/scores/base.py +87 -0
- tscp/scores/cqr.py +71 -0
- tscp/scores/normalized.py +74 -0
- tscp/scores/signed.py +46 -0
- tscp/sources/__init__.py +7 -0
- tscp/sources/base.py +32 -0
- tscp/sources/expanding.py +25 -0
- tscp/sources/holdout.py +18 -0
- tscp/testing/__init__.py +40 -0
- tscp/testing/archives.py +80 -0
- tscp/testing/checks.py +832 -0
- tscp/testing/components.py +28 -0
- tscp/typing.py +24 -0
- tscp/utils/__init__.py +14 -0
- tscp/utils/component.py +76 -0
- tscp/utils/config.py +135 -0
- tscp/utils/hashing.py +35 -0
- tscp/weights/__init__.py +7 -0
- tscp/weights/base.py +22 -0
- tscp/weights/exponential.py +37 -0
- tscp/weights/uniform.py +23 -0
- tscp-0.1.0.dev0.dist-info/METADATA +130 -0
- tscp-0.1.0.dev0.dist-info/RECORD +72 -0
- tscp-0.1.0.dev0.dist-info/WHEEL +4 -0
- tscp-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
tscp/features/combine.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Several feature maps side by side."""
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from torch import Tensor
|
|
5
|
+
|
|
6
|
+
from tscp.data import ResidualArchive
|
|
7
|
+
from tscp.features.base import FeatureMap
|
|
8
|
+
from tscp.registry import register
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _element(values: Tensor) -> int:
|
|
12
|
+
"""Number of element dimensions of ``[T, *element, ...]`` features or masks."""
|
|
13
|
+
return values.ndim - 2
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _spread(values: Tensor, ndim: int, target: tuple[int, ...]) -> Tensor:
|
|
17
|
+
"""Repeat ``[T, *element, ...]`` across missing leading element dimensions.
|
|
18
|
+
|
|
19
|
+
An element of ``()`` or ``(N,)`` lacks the leading dimensions of ``(H, N)``, so new
|
|
20
|
+
dimensions go right after the row dimension, not at the end.
|
|
21
|
+
"""
|
|
22
|
+
missing = len(target) - ndim
|
|
23
|
+
for _ in range(missing):
|
|
24
|
+
values = values.unsqueeze(1)
|
|
25
|
+
return values.expand(values.shape[0], *target, *values.shape[1 + len(target) :])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@register("features", "concat")
|
|
29
|
+
class Concat(FeatureMap):
|
|
30
|
+
"""Several feature maps side by side, broadcast to a common element shape.
|
|
31
|
+
|
|
32
|
+
A map per row (element ``()``) or per series (``(N,)``) is repeated over the steps
|
|
33
|
+
and series of a map per step and series (``(H, N)``), and the features are
|
|
34
|
+
concatenated. An element is valid where every map's features are.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
maps: The feature maps, at least one.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, maps: list[FeatureMap]):
|
|
41
|
+
super().__init__()
|
|
42
|
+
if not maps:
|
|
43
|
+
raise ValueError("Concat needs at least one feature map")
|
|
44
|
+
self.maps = torch.nn.ModuleList(maps)
|
|
45
|
+
|
|
46
|
+
def _target(self, parts: list[Tensor]) -> tuple[int, ...]:
|
|
47
|
+
widest = max(parts, key=_element)
|
|
48
|
+
return tuple(widest.shape[1:-1])
|
|
49
|
+
|
|
50
|
+
def forward(self, rows: ResidualArchive) -> Tensor:
|
|
51
|
+
parts = [feature_map(rows) for feature_map in self.maps]
|
|
52
|
+
target = self._target(parts)
|
|
53
|
+
spread = [_spread(part, _element(part), target) for part in parts]
|
|
54
|
+
return torch.cat(spread, dim=-1)
|
|
55
|
+
|
|
56
|
+
def valid(self, rows: ResidualArchive) -> Tensor | None:
|
|
57
|
+
parts = [feature_map(rows) for feature_map in self.maps]
|
|
58
|
+
target = self._target(parts)
|
|
59
|
+
masks = [feature_map.valid(rows) for feature_map in self.maps]
|
|
60
|
+
valid = None
|
|
61
|
+
for part, mask in zip(parts, masks, strict=True):
|
|
62
|
+
if mask is None:
|
|
63
|
+
continue
|
|
64
|
+
mask = _spread(mask.unsqueeze(-1), _element(part), target)[..., 0]
|
|
65
|
+
valid = mask if valid is None else valid & mask
|
|
66
|
+
return valid
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
__all__ = ["Concat"]
|
tscp/features/context.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Columns of the archive's context."""
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from torch import Tensor
|
|
5
|
+
|
|
6
|
+
from tscp.data import ResidualArchive
|
|
7
|
+
from tscp.features.base import FeatureMap
|
|
8
|
+
from tscp.registry import register
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@register("features", "context")
|
|
12
|
+
class ContextColumns(FeatureMap):
|
|
13
|
+
"""Columns of the archive's context: ``[T, F]``, or ``[T, N, F]`` for context per series.
|
|
14
|
+
|
|
15
|
+
Context holds covariates known at each origin, so a row may always use its own.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
names: Names of the columns, from the archive's ``context_names``; None for all.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, names: list[str] | None = None):
|
|
22
|
+
super().__init__()
|
|
23
|
+
if names is not None and not names:
|
|
24
|
+
raise ValueError("names must name at least one column, or be None for all")
|
|
25
|
+
self.names = None if names is None else list(names)
|
|
26
|
+
|
|
27
|
+
def forward(self, rows: ResidualArchive) -> Tensor:
|
|
28
|
+
if rows.context is None:
|
|
29
|
+
raise ValueError("this archive has no context")
|
|
30
|
+
context = rows.context
|
|
31
|
+
if not context.is_floating_point():
|
|
32
|
+
context = context.to(torch.get_default_dtype())
|
|
33
|
+
if self.names is None:
|
|
34
|
+
return context
|
|
35
|
+
unknown = sorted(set(self.names) - set(rows.context_names))
|
|
36
|
+
if unknown:
|
|
37
|
+
raise ValueError(f"the context has no columns {unknown}; it has {rows.context_names}")
|
|
38
|
+
index = torch.tensor([rows.context_names.index(name) for name in self.names])
|
|
39
|
+
return context[..., index.to(context.device)]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
__all__ = ["ContextColumns"]
|
tscp/features/history.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""The residuals known before each origin."""
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from torch import Tensor
|
|
5
|
+
|
|
6
|
+
from tscp.data import ResidualArchive
|
|
7
|
+
from tscp.features.base import FeatureMap, timeline_of
|
|
8
|
+
from tscp.registry import register
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@register("features", "history")
|
|
12
|
+
class ResidualHistory(FeatureMap):
|
|
13
|
+
"""The last ``lags`` known residuals of each step and series: ``[T, H, N, lags]``.
|
|
14
|
+
|
|
15
|
+
For a row with origin ``t``, lag 1 of step ``h`` is the most recent residual of step
|
|
16
|
+
``h`` whose target time has passed by ``t``, lag 2 the one before, and so on. The
|
|
17
|
+
timeline's rows are in time order, so these are the last rows among the first
|
|
18
|
+
``known_counts(t)[h]``. A lag that is not known yet, or whose target was not observed,
|
|
19
|
+
is 0 and makes the element invalid.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
lags: Number of past residuals per step, positive.
|
|
23
|
+
absolute: Give absolute residuals.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, lags: int = 1, absolute: bool = False):
|
|
27
|
+
super().__init__()
|
|
28
|
+
if lags < 1:
|
|
29
|
+
raise ValueError("lags must be positive")
|
|
30
|
+
self.lags = int(lags)
|
|
31
|
+
self.absolute = bool(absolute)
|
|
32
|
+
|
|
33
|
+
def _history(self, rows: ResidualArchive) -> tuple[Tensor, Tensor]:
|
|
34
|
+
"""Values and known mask, both ``[T, H, N, lags]``."""
|
|
35
|
+
timeline = timeline_of(rows)
|
|
36
|
+
device = timeline.device
|
|
37
|
+
counts = timeline.known_counts(rows.origins) # [T, H]
|
|
38
|
+
index = counts[..., None] - torch.arange(1, self.lags + 1, device=device) # [T, H, L]
|
|
39
|
+
safe = index.clamp(min=0)
|
|
40
|
+
steps = torch.arange(timeline.horizon, device=device)[None, :, None]
|
|
41
|
+
values = timeline.residuals.transpose(0, 1)[steps, safe] # [T, H, L, N]
|
|
42
|
+
known = (index >= 0)[..., None].expand_as(values)
|
|
43
|
+
if timeline.observed is not None:
|
|
44
|
+
known = known & timeline.observed.transpose(0, 1)[steps, safe]
|
|
45
|
+
values = torch.where(known, values, torch.zeros((), dtype=values.dtype, device=device))
|
|
46
|
+
if self.absolute:
|
|
47
|
+
values = values.abs()
|
|
48
|
+
return values.permute(0, 1, 3, 2), known.permute(0, 1, 3, 2)
|
|
49
|
+
|
|
50
|
+
def forward(self, rows: ResidualArchive) -> Tensor:
|
|
51
|
+
return self._history(rows)[0]
|
|
52
|
+
|
|
53
|
+
def valid(self, rows: ResidualArchive) -> Tensor | None:
|
|
54
|
+
known = self._history(rows)[1].all(dim=-1)
|
|
55
|
+
return None if bool(known.all()) else known
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
__all__ = ["ResidualHistory"]
|
tscp/features/phase.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Where each origin falls in a cycle of base steps."""
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from torch import Tensor
|
|
5
|
+
|
|
6
|
+
from tscp.data import ResidualArchive
|
|
7
|
+
from tscp.features.base import ENCODINGS, FeatureMap, encode_cycles
|
|
8
|
+
from tscp.registry import register
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@register("features", "phase")
|
|
12
|
+
class Phase(FeatureMap):
|
|
13
|
+
"""The position of each origin in a cycle of ``period`` base steps: ``[T, F]``.
|
|
14
|
+
|
|
15
|
+
With hourly origins that start at midnight, ``Phase(24)`` is the hour of day; with
|
|
16
|
+
daily origins that start on a Monday, ``Phase(7)`` is the weekday. It needs no
|
|
17
|
+
timestamps. The ``"integer"`` encoding gives ``(origin + offset) mod period``, one
|
|
18
|
+
feature; ``"cyclic"`` gives its sine and cosine, two features.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
period: Length of the cycle in base time steps, at least 2.
|
|
22
|
+
offset: Added to the origins before taking the remainder.
|
|
23
|
+
encoding: ``"integer"`` or ``"cyclic"``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, period: int, offset: int = 0, encoding: str = "integer"):
|
|
27
|
+
super().__init__()
|
|
28
|
+
if period < 2:
|
|
29
|
+
raise ValueError("period must be at least 2")
|
|
30
|
+
if encoding not in ENCODINGS:
|
|
31
|
+
raise ValueError(f"encoding must be one of {ENCODINGS}, not {encoding!r}")
|
|
32
|
+
self.period = int(period)
|
|
33
|
+
self.offset = int(offset)
|
|
34
|
+
self.encoding = encoding
|
|
35
|
+
|
|
36
|
+
def forward(self, rows: ResidualArchive) -> Tensor:
|
|
37
|
+
phase = torch.remainder(rows.origins + self.offset, self.period)
|
|
38
|
+
return encode_cycles(phase[:, None], torch.tensor([self.period]), self.encoding)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
__all__ = ["Phase"]
|
tscp/imports.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Guards for optional dependencies.
|
|
2
|
+
|
|
3
|
+
Core tscp needs only torch and numpy. A component that needs another package imports it
|
|
4
|
+
through :func:`require`, so the error names the extra that provides it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
import importlib.util
|
|
9
|
+
from types import ModuleType
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def has_module(name: str) -> bool:
|
|
13
|
+
"""Return whether ``name`` can be imported, without importing it."""
|
|
14
|
+
return importlib.util.find_spec(name) is not None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def require(name: str, extra: str | None = None) -> ModuleType:
|
|
18
|
+
"""Import an optional dependency, or raise an ImportError that says how to install it.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
name: Module to import, such as ``"scipy"``.
|
|
22
|
+
extra: The tscp extra that provides it, if there is one.
|
|
23
|
+
"""
|
|
24
|
+
try:
|
|
25
|
+
return importlib.import_module(name)
|
|
26
|
+
except ImportError as error:
|
|
27
|
+
hint = f"pip install 'tscp[{extra}]'" if extra else f"pip install {name}"
|
|
28
|
+
message = f"{name} is needed for this feature; install it with `{hint}`."
|
|
29
|
+
raise ImportError(message) from error
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
__all__ = ["has_module", "require"]
|
tscp/methods/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Published methods as presets: one choice per axis, run by :class:`ConformalPredictor`.
|
|
2
|
+
|
|
3
|
+
Presets are registered in :mod:`tscp.registry` under the kind ``"method"``;
|
|
4
|
+
:data:`method_map` is that table, and :func:`get_method` builds a preset by name, as
|
|
5
|
+
poolers are built in tgp. :func:`compose` builds a method from a spec of short names.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from tscp import registry
|
|
9
|
+
|
|
10
|
+
from .aci import ACI
|
|
11
|
+
from .base import AXES, ConformalPredictor
|
|
12
|
+
from .compose import compose
|
|
13
|
+
from .io import load_method, save_method
|
|
14
|
+
from .nexcp import NexCP
|
|
15
|
+
from .split import SplitCP
|
|
16
|
+
|
|
17
|
+
method_map: dict[str, type[ConformalPredictor]] = registry.entries("method")
|
|
18
|
+
"""Registered presets by short name: the live table of :mod:`tscp.registry`."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_method(name: str, *, ignore_unknown: bool = False, **kwargs) -> ConformalPredictor:
|
|
22
|
+
"""Build a registered preset by name.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
name: A registered method name, in any case; installed plugins count.
|
|
26
|
+
ignore_unknown: Drop keyword arguments the preset does not take, which helps when
|
|
27
|
+
one config sweeps several presets. By default they raise.
|
|
28
|
+
**kwargs: Arguments of the preset.
|
|
29
|
+
|
|
30
|
+
Raises:
|
|
31
|
+
ValueError: If ``name`` is not registered.
|
|
32
|
+
TypeError: If an argument is unknown and ``ignore_unknown`` is False.
|
|
33
|
+
"""
|
|
34
|
+
return registry.build("method", name, ignore_unknown=ignore_unknown, **kwargs)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"ACI",
|
|
39
|
+
"AXES",
|
|
40
|
+
"ConformalPredictor",
|
|
41
|
+
"NexCP",
|
|
42
|
+
"SplitCP",
|
|
43
|
+
"compose",
|
|
44
|
+
"get_method",
|
|
45
|
+
"load_method",
|
|
46
|
+
"method_map",
|
|
47
|
+
"save_method",
|
|
48
|
+
]
|
tscp/methods/aci.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Adaptive conformal inference on top of conformal thresholds."""
|
|
2
|
+
|
|
3
|
+
from tscp.methods.base import ConformalPredictor
|
|
4
|
+
from tscp.online import ACIUpdate
|
|
5
|
+
from tscp.registry import register
|
|
6
|
+
from tscp.scores import AbsoluteScore, Score
|
|
7
|
+
from tscp.sources import ExpandingSource, Source
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@register("method", "aci")
|
|
11
|
+
class ACI(ConformalPredictor):
|
|
12
|
+
"""Adaptive conformal inference (Gibbs & Candès, NeurIPS 2021).
|
|
13
|
+
|
|
14
|
+
The threshold at each test origin is the conformal quantile at the current level
|
|
15
|
+
``alpha_t``, which :class:`~tscp.online.ACIUpdate` moves after every released
|
|
16
|
+
outcome. The long-run miss rate converges to ``alpha`` on any sequence.
|
|
17
|
+
|
|
18
|
+
Deviation from the paper: the paper recalibrates on a recent window after refitting
|
|
19
|
+
its model. tscp never refits the forecaster; the pool here is the calibration split
|
|
20
|
+
plus every test residual already known (the expanding source).
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
gamma: Step size of the level update.
|
|
24
|
+
score: Defaults to :class:`~tscp.scores.AbsoluteScore`.
|
|
25
|
+
source: Defaults to :class:`~tscp.sources.ExpandingSource`.
|
|
26
|
+
alpha: Target miscoverage level.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
gamma: float = 0.005,
|
|
32
|
+
score: Score | None = None,
|
|
33
|
+
source: Source | None = None,
|
|
34
|
+
alpha: float = 0.1,
|
|
35
|
+
):
|
|
36
|
+
super().__init__(
|
|
37
|
+
score=score if score is not None else AbsoluteScore(),
|
|
38
|
+
source=source if source is not None else ExpandingSource(),
|
|
39
|
+
update=ACIUpdate(gamma),
|
|
40
|
+
alpha=alpha,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
__all__ = ["ACI"]
|
tscp/methods/base.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The composer that runs one component per axis as a conformal method."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from typing import Self
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import Tensor
|
|
9
|
+
|
|
10
|
+
from tscp.data import ResidualArchive, availability_mask
|
|
11
|
+
from tscp.methods.engine import OnlineEngine, replay
|
|
12
|
+
from tscp.online import OnlineUpdate
|
|
13
|
+
from tscp.quantile import Pool, QuantileModel, ScoreDistribution, WeightedQuantile
|
|
14
|
+
from tscp.regions import Region
|
|
15
|
+
from tscp.scores import Score
|
|
16
|
+
from tscp.sources import HoldoutSource, Source
|
|
17
|
+
from tscp.utils import Component
|
|
18
|
+
from tscp.weights import Weights
|
|
19
|
+
|
|
20
|
+
AXES = ("score", "source", "weights", "quantile", "update")
|
|
21
|
+
"""Attribute names of the axis components."""
|
|
22
|
+
|
|
23
|
+
_AXIS_TYPES = {
|
|
24
|
+
"score": Score,
|
|
25
|
+
"source": Source,
|
|
26
|
+
"weights": Weights,
|
|
27
|
+
"quantile": QuantileModel,
|
|
28
|
+
"update": OnlineUpdate,
|
|
29
|
+
}
|
|
30
|
+
_MAX_ELEMENTS = 2**23
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ConformalPredictor(Component):
|
|
34
|
+
"""A conformal method: one component per axis, run through a fixed lifecycle.
|
|
35
|
+
|
|
36
|
+
1. :meth:`fit` fits every trainable component on ``archive.split("fit")``, with the
|
|
37
|
+
``validation`` split if there is one, and on nothing else.
|
|
38
|
+
2. :meth:`calibrate` scores the source's rows, the calibration split.
|
|
39
|
+
3. :meth:`predict` makes one region per test origin, in residual coordinates.
|
|
40
|
+
|
|
41
|
+
Evaluation is a separate step (:func:`tscp.metrics.evaluate`).
|
|
42
|
+
|
|
43
|
+
For each chunk of test origins, :meth:`predict` asks the quantile model for the
|
|
44
|
+
conformal distribution of the score given the pool and its weights. Without an online
|
|
45
|
+
update it reads every threshold from those distributions at the score's levels. With
|
|
46
|
+
one, it answers the test origins in time order and hands the update each outcome once
|
|
47
|
+
its target time has passed (:mod:`tscp.methods.engine`). An expanding source adds each
|
|
48
|
+
test residual to the pool by the same rule. For a per-step score, step ``h`` of an
|
|
49
|
+
origin becomes known ``h`` base steps after it; for a score over the whole horizon,
|
|
50
|
+
after the last step.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
score: Axis 2, how an error becomes a number.
|
|
54
|
+
source: Axis 1, which past errors calibrate; the calibration split by default.
|
|
55
|
+
weights: Axis 3; None weighs every available pool residual equally.
|
|
56
|
+
quantile: Axis 4; :class:`~tscp.quantile.WeightedQuantile` by default.
|
|
57
|
+
update: Axis 5; None for an offline method.
|
|
58
|
+
alpha: Target miscoverage level, in (0, 1).
|
|
59
|
+
chunk_size: Most query origins whose weights are held in memory at once.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
score: Score,
|
|
65
|
+
source: Source | None = None,
|
|
66
|
+
weights: Weights | None = None,
|
|
67
|
+
quantile: QuantileModel | None = None,
|
|
68
|
+
update: OnlineUpdate | None = None,
|
|
69
|
+
alpha: float = 0.1,
|
|
70
|
+
chunk_size: int = 256,
|
|
71
|
+
):
|
|
72
|
+
super().__init__()
|
|
73
|
+
if not 0 < alpha < 1:
|
|
74
|
+
raise ValueError("alpha must lie in (0, 1)")
|
|
75
|
+
if chunk_size < 1:
|
|
76
|
+
raise ValueError("chunk_size must be positive")
|
|
77
|
+
self.score = score
|
|
78
|
+
self.source = source if source is not None else HoldoutSource()
|
|
79
|
+
self.weights = weights
|
|
80
|
+
self.quantile = quantile if quantile is not None else WeightedQuantile()
|
|
81
|
+
self.update = update
|
|
82
|
+
for axis, expected in _AXIS_TYPES.items():
|
|
83
|
+
component = getattr(self, axis)
|
|
84
|
+
if component is not None and not isinstance(component, expected):
|
|
85
|
+
raise TypeError(f"{axis} must be a {expected.__name__}, not {type(component)}")
|
|
86
|
+
self.alpha = float(alpha)
|
|
87
|
+
self.chunk_size = int(chunk_size)
|
|
88
|
+
self.fit_fingerprint: str | None = None
|
|
89
|
+
# Calibration state: moves with .to(device), stays out of state_dict().
|
|
90
|
+
self.register_buffer("pool_scores", None, persistent=False)
|
|
91
|
+
self.register_buffer("pool_origins", None, persistent=False)
|
|
92
|
+
self.register_buffer("pool_valid", None, persistent=False)
|
|
93
|
+
|
|
94
|
+
def components(self) -> dict[str, Component]:
|
|
95
|
+
"""Return the components that are set, keyed by axis."""
|
|
96
|
+
return {axis: getattr(self, axis) for axis in AXES if getattr(self, axis) is not None}
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def is_trainable(self) -> bool:
|
|
100
|
+
return any(component.is_trainable for component in self.components().values())
|
|
101
|
+
|
|
102
|
+
# -- lifecycle -----------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
def fit(self, archive: ResidualArchive) -> Self:
|
|
105
|
+
"""Fit every trainable component on the ``fit`` split, and ``validation`` if any."""
|
|
106
|
+
trainable = [axis for axis, c in self.components().items() if c.is_trainable]
|
|
107
|
+
if not trainable:
|
|
108
|
+
return self
|
|
109
|
+
if archive.splits is None or "fit" not in archive.splits:
|
|
110
|
+
names = ", ".join(trainable)
|
|
111
|
+
raise ValueError(f"{names} must be fitted, and the archive has no 'fit' split")
|
|
112
|
+
train = archive.split("fit")
|
|
113
|
+
validation = archive.split("validation") if "validation" in archive.splits else None
|
|
114
|
+
for axis in trainable:
|
|
115
|
+
getattr(self, axis).fit(train, validation)
|
|
116
|
+
# Which data the fitted state came from, for saved methods (tscp.methods.io).
|
|
117
|
+
self.fit_fingerprint = archive.fingerprint()
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def calibrate(self, archive: ResidualArchive) -> Self:
|
|
121
|
+
"""Score the calibration pool of ``archive``."""
|
|
122
|
+
pool = archive.select(self.source.rows(archive))
|
|
123
|
+
self.pool_scores = self.score(pool)
|
|
124
|
+
self.pool_origins = pool.origins
|
|
125
|
+
self.pool_valid = self.score.valid(pool)
|
|
126
|
+
return self
|
|
127
|
+
|
|
128
|
+
def predict(self, archive: ResidualArchive) -> Region:
|
|
129
|
+
"""Return one region per origin of the ``test`` split, in residual coordinates."""
|
|
130
|
+
if self.pool_scores is None:
|
|
131
|
+
raise RuntimeError("calibrate() must run before predict()")
|
|
132
|
+
rows = self.source.rows(archive)
|
|
133
|
+
if not torch.equal(archive.origins[rows], self.pool_origins):
|
|
134
|
+
raise ValueError("predict() must get the archive that calibrate() got")
|
|
135
|
+
if archive.splits is None or "test" not in archive.splits:
|
|
136
|
+
raise ValueError("predict() needs an archive with a 'test' split")
|
|
137
|
+
test_rows = archive.splits["test"]
|
|
138
|
+
query = archive.select(test_rows)
|
|
139
|
+
query_scores = self.score(query)
|
|
140
|
+
query_valid = self.score.valid(query)
|
|
141
|
+
if self.source.expanding:
|
|
142
|
+
# Test residuals join the pool; the availability mask releases each one only
|
|
143
|
+
# after its target time.
|
|
144
|
+
pool = Pool(
|
|
145
|
+
torch.cat([self.pool_scores, query_scores]),
|
|
146
|
+
archive.select(torch.cat([rows, test_rows])),
|
|
147
|
+
_concat_valid(self.pool_valid, query_valid, self.pool_scores, query_scores),
|
|
148
|
+
)
|
|
149
|
+
else:
|
|
150
|
+
pool = Pool(self.pool_scores, archive.select(rows), self.pool_valid)
|
|
151
|
+
thresholds = self._thresholds(pool, query, query_scores, query_valid)
|
|
152
|
+
return self.score.region(thresholds, query)
|
|
153
|
+
|
|
154
|
+
# -- internals -----------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
def _thresholds(
|
|
157
|
+
self,
|
|
158
|
+
pool: Pool,
|
|
159
|
+
query: ResidualArchive,
|
|
160
|
+
query_scores: Tensor,
|
|
161
|
+
query_valid: Tensor | None,
|
|
162
|
+
) -> Tensor:
|
|
163
|
+
kept = pool.kept
|
|
164
|
+
levels = self.score.levels(self.alpha, kept).to(pool.scores.device)
|
|
165
|
+
step_kept = "step" not in self.score.reduce_dims
|
|
166
|
+
# Steps after which each group of kept elements becomes known.
|
|
167
|
+
offsets = query.steps if step_kept else query.steps[-1:]
|
|
168
|
+
out = torch.empty((len(query), *kept), dtype=pool.scores.dtype, device=pool.scores.device)
|
|
169
|
+
chunks = self._distributions(pool, query, offsets, step_kept)
|
|
170
|
+
if self.update is None:
|
|
171
|
+
for start, stop, distribution in chunks:
|
|
172
|
+
out[start:stop] = distribution.threshold(levels)
|
|
173
|
+
return out
|
|
174
|
+
engine = OnlineEngine(self.update, levels, step_kept, len(offsets))
|
|
175
|
+
return replay(engine, query.origins, offsets, chunks, query_scores, query_valid, out)
|
|
176
|
+
|
|
177
|
+
def _distributions(
|
|
178
|
+
self, pool: Pool, query: ResidualArchive, offsets: Tensor, step_kept: bool
|
|
179
|
+
) -> Iterator[tuple[int, int, ScoreDistribution]]:
|
|
180
|
+
"""Yield ``(start, stop, distribution)`` for consecutive chunks of the queries."""
|
|
181
|
+
uniform = self.weights is None and not self.source.expanding and pool.valid is None
|
|
182
|
+
chunk = self.chunk_size
|
|
183
|
+
if not uniform:
|
|
184
|
+
chunk = max(1, min(chunk, _MAX_ELEMENTS // (len(pool) * math.prod(pool.kept))))
|
|
185
|
+
for start in range(0, len(query), chunk):
|
|
186
|
+
stop = min(start + chunk, len(query))
|
|
187
|
+
rows = query.select(torch.arange(start, stop, device=query.device))
|
|
188
|
+
weights = None if uniform else self._weights(pool, rows, offsets, step_kept)
|
|
189
|
+
yield start, stop, self.quantile(pool, weights, rows)
|
|
190
|
+
|
|
191
|
+
def _weights(
|
|
192
|
+
self, pool: Pool, rows: ResidualArchive, offsets: Tensor, step_kept: bool
|
|
193
|
+
) -> Tensor:
|
|
194
|
+
"""Weights ``[c, P, ...]`` for the ``c`` query ``rows``, with every mask applied."""
|
|
195
|
+
kept_ndim = len(pool.kept)
|
|
196
|
+
device = pool.scores.device
|
|
197
|
+
weights = torch.ones(len(rows), len(pool), dtype=torch.float64, device=device)
|
|
198
|
+
if self.weights is not None:
|
|
199
|
+
weights = self.weights(pool.rows, rows).to(torch.float64)
|
|
200
|
+
weights = weights.reshape(*weights.shape, *([1] * (2 + kept_ndim - weights.ndim)))
|
|
201
|
+
if self.source.expanding:
|
|
202
|
+
mask = availability_mask(pool.rows.origins, rows.origins, offsets)
|
|
203
|
+
weights = weights * _group_mask(mask, step_kept, kept_ndim)
|
|
204
|
+
if pool.valid is not None:
|
|
205
|
+
weights = weights * pool.valid.unsqueeze(0)
|
|
206
|
+
return weights
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _group_mask(mask: Tensor, step_kept: bool, kept_ndim: int) -> Tensor:
|
|
210
|
+
"""Reshape a ``[Q, P, G]`` availability mask to broadcast over ``[Q, P, *kept]``."""
|
|
211
|
+
if step_kept: # one group per step, and step is the first kept dimension
|
|
212
|
+
return mask.reshape(*mask.shape, *([1] * (kept_ndim - 1)))
|
|
213
|
+
return mask[..., 0].reshape(*mask.shape[:2], *([1] * kept_ndim))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _concat_valid(
|
|
217
|
+
pool_valid: Tensor | None, query_valid: Tensor | None, pool_scores: Tensor, query_scores: Tensor
|
|
218
|
+
) -> Tensor | None:
|
|
219
|
+
if pool_valid is None and query_valid is None:
|
|
220
|
+
return None
|
|
221
|
+
if pool_valid is None:
|
|
222
|
+
pool_valid = torch.ones_like(pool_scores, dtype=torch.bool)
|
|
223
|
+
if query_valid is None:
|
|
224
|
+
query_valid = torch.ones_like(query_scores, dtype=torch.bool)
|
|
225
|
+
return torch.cat([pool_valid, query_valid])
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
__all__ = ["AXES", "ConformalPredictor"]
|
tscp/methods/compose.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Methods built from specs of short names, such as specs read from YAML."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from tscp import registry
|
|
7
|
+
from tscp.methods.base import AXES, ConformalPredictor
|
|
8
|
+
from tscp.utils import Component, from_config
|
|
9
|
+
|
|
10
|
+
_SETTINGS = ("alpha", "chunk_size")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def compose(spec: Mapping[str, Any]) -> ConformalPredictor:
|
|
14
|
+
"""Build a method from a spec.
|
|
15
|
+
|
|
16
|
+
A spec names either a registered preset with its arguments::
|
|
17
|
+
|
|
18
|
+
{"method": "nexcp", "decay": 0.99, "alpha": 0.1}
|
|
19
|
+
|
|
20
|
+
or one component per axis, run by :class:`~tscp.methods.ConformalPredictor`::
|
|
21
|
+
|
|
22
|
+
{
|
|
23
|
+
"score": "signed",
|
|
24
|
+
"source": "expanding",
|
|
25
|
+
"weights": {"name": "exponential", "decay": 0.99},
|
|
26
|
+
"update": {"name": "aci", "gamma": 0.005},
|
|
27
|
+
"alpha": 0.1,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
A component can be given as its short name, as ``{"name": ..., **arguments}``, as a
|
|
31
|
+
full config ``{"class": ..., "args": ...}`` (what
|
|
32
|
+
:meth:`~tscp.utils.ConfigMixin.to_config` returns), or as a component. Inside
|
|
33
|
+
arguments, a component of another kind states its kind:
|
|
34
|
+
``{"kind": "features", "name": "phase", "period": 24}``. Names come from
|
|
35
|
+
:mod:`tscp.registry`, so installed plugins can be named too.
|
|
36
|
+
|
|
37
|
+
Raises:
|
|
38
|
+
ValueError: If the spec has unknown keys, no score, or names nothing registered.
|
|
39
|
+
"""
|
|
40
|
+
spec = dict(spec)
|
|
41
|
+
if "method" in spec:
|
|
42
|
+
name = spec.pop("method")
|
|
43
|
+
arguments = {
|
|
44
|
+
key: _component(key, value) if key in AXES else _resolve(value)
|
|
45
|
+
for key, value in spec.items()
|
|
46
|
+
}
|
|
47
|
+
return registry.build("method", name, **arguments)
|
|
48
|
+
unknown = sorted(set(spec) - set(AXES) - set(_SETTINGS))
|
|
49
|
+
if unknown:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f"unknown keys {unknown} in a spec; use 'method' for a preset, or the axes "
|
|
52
|
+
f"{AXES} and the settings {_SETTINGS}"
|
|
53
|
+
)
|
|
54
|
+
if "score" not in spec:
|
|
55
|
+
raise ValueError("a spec needs a 'score', or a 'method' that names a preset")
|
|
56
|
+
arguments = {
|
|
57
|
+
key: _component(key, value) if key in AXES else value for key, value in spec.items()
|
|
58
|
+
}
|
|
59
|
+
return ConformalPredictor(**arguments)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _component(kind: str, value: Any) -> Any:
|
|
63
|
+
"""Build a component of ``kind`` from any of the forms a spec allows."""
|
|
64
|
+
if value is None or isinstance(value, Component):
|
|
65
|
+
return value
|
|
66
|
+
if isinstance(value, str):
|
|
67
|
+
return registry.build(kind, value)
|
|
68
|
+
if isinstance(value, Mapping):
|
|
69
|
+
if set(value) == {"class", "args"}:
|
|
70
|
+
return from_config(value)
|
|
71
|
+
if "name" in value:
|
|
72
|
+
arguments = {k: _resolve(v) for k, v in value.items() if k not in ("name", "kind")}
|
|
73
|
+
return registry.build(value.get("kind", kind), value["name"], **arguments)
|
|
74
|
+
raise ValueError(f"cannot build a {kind} from {value!r}")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _resolve(value: Any) -> Any:
|
|
78
|
+
"""Build the components nested in an argument value; leave other values alone."""
|
|
79
|
+
if isinstance(value, Mapping):
|
|
80
|
+
if set(value) == {"class", "args"}:
|
|
81
|
+
return from_config(value)
|
|
82
|
+
if "kind" in value and "name" in value:
|
|
83
|
+
return _component(value["kind"], value)
|
|
84
|
+
return {key: _resolve(item) for key, item in value.items()}
|
|
85
|
+
if isinstance(value, list):
|
|
86
|
+
return [_resolve(item) for item in value]
|
|
87
|
+
return value
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
__all__ = ["compose"]
|