spatialcv 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spatialcv/__init__.py +59 -0
- spatialcv/_base.py +105 -0
- spatialcv/_coords.py +201 -0
- spatialcv/blocks.py +152 -0
- spatialcv/buffer.py +158 -0
- spatialcv/cluster.py +143 -0
- spatialcv/datasets.py +101 -0
- spatialcv/diagnostics.py +151 -0
- spatialcv/nndm.py +470 -0
- spatialcv/plotting.py +83 -0
- spatialcv/py.typed +0 -0
- spatialcv/variogram.py +273 -0
- spatialcv-0.1.0.dist-info/METADATA +311 -0
- spatialcv-0.1.0.dist-info/RECORD +17 -0
- spatialcv-0.1.0.dist-info/WHEEL +5 -0
- spatialcv-0.1.0.dist-info/licenses/LICENSE +21 -0
- spatialcv-0.1.0.dist-info/top_level.txt +1 -0
spatialcv/__init__.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""spatialcv: honest cross-validation for models on spatial data.
|
|
2
|
+
|
|
3
|
+
scikit-learn compatible splitters that keep spatially autocorrelated neighbours out of
|
|
4
|
+
the training data of each test fold, plus tools to choose the block size or buffer and to
|
|
5
|
+
check whether the CV is as hard as the real prediction task.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from ._coords import EARTH_RADIUS_M, pairwise_distances
|
|
9
|
+
from .blocks import SpatialBlockKFold
|
|
10
|
+
from .buffer import BufferedLeaveOneOut, SpatialKFoldWithBuffer
|
|
11
|
+
from .cluster import EnvironmentalBlockKFold, SpatialClusterKFold
|
|
12
|
+
from .datasets import gaussian_random_field, make_prediction_grid, make_spatial_regression
|
|
13
|
+
from .diagnostics import FoldDiagnostics, fold_diagnostics
|
|
14
|
+
from .nndm import (
|
|
15
|
+
KNNDMKFold,
|
|
16
|
+
KNNDMResult,
|
|
17
|
+
NNDMLeaveOneOut,
|
|
18
|
+
NNDMResult,
|
|
19
|
+
knndm,
|
|
20
|
+
nndm,
|
|
21
|
+
wasserstein_distance,
|
|
22
|
+
)
|
|
23
|
+
from .plotting import plot_folds
|
|
24
|
+
from .variogram import (
|
|
25
|
+
Variogram,
|
|
26
|
+
empirical_variogram,
|
|
27
|
+
estimate_autocorrelation_range,
|
|
28
|
+
fit_variogram_model,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
__version__ = "0.1.0"
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"EARTH_RADIUS_M",
|
|
35
|
+
"BufferedLeaveOneOut",
|
|
36
|
+
"EnvironmentalBlockKFold",
|
|
37
|
+
"FoldDiagnostics",
|
|
38
|
+
"KNNDMKFold",
|
|
39
|
+
"KNNDMResult",
|
|
40
|
+
"NNDMLeaveOneOut",
|
|
41
|
+
"NNDMResult",
|
|
42
|
+
"SpatialBlockKFold",
|
|
43
|
+
"SpatialClusterKFold",
|
|
44
|
+
"SpatialKFoldWithBuffer",
|
|
45
|
+
"Variogram",
|
|
46
|
+
"__version__",
|
|
47
|
+
"empirical_variogram",
|
|
48
|
+
"estimate_autocorrelation_range",
|
|
49
|
+
"fit_variogram_model",
|
|
50
|
+
"fold_diagnostics",
|
|
51
|
+
"gaussian_random_field",
|
|
52
|
+
"knndm",
|
|
53
|
+
"make_prediction_grid",
|
|
54
|
+
"make_spatial_regression",
|
|
55
|
+
"nndm",
|
|
56
|
+
"pairwise_distances",
|
|
57
|
+
"plot_folds",
|
|
58
|
+
"wasserstein_distance",
|
|
59
|
+
]
|
spatialcv/_base.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Shared base class for the spatial splitters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
from typing import Any, Iterator, List, Optional, Sequence, Tuple
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
from sklearn.model_selection import BaseCrossValidator
|
|
10
|
+
|
|
11
|
+
from ._coords import check_metric, resolve_coords
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _short(value: Any) -> str:
|
|
15
|
+
if isinstance(value, np.ndarray):
|
|
16
|
+
return f"array(shape={value.shape})"
|
|
17
|
+
if hasattr(value, "shape") and hasattr(value, "__len__"):
|
|
18
|
+
return f"{type(value).__name__}(shape={tuple(value.shape)})"
|
|
19
|
+
return repr(value)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SpatialSplitter(BaseCrossValidator): # type: ignore[misc]
|
|
23
|
+
"""Base class: scikit-learn ``split`` / ``get_n_splits`` API plus coordinate lookup.
|
|
24
|
+
|
|
25
|
+
Subclasses either implement :meth:`_fold_labels` (partitioning splitters) or override
|
|
26
|
+
:meth:`split` directly (splitters that drop training points, such as buffered ones).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
coords: Any = None
|
|
30
|
+
coord_columns: Optional[Sequence[Any]] = None
|
|
31
|
+
metric: str = "euclidean"
|
|
32
|
+
|
|
33
|
+
def _get_coords(self, X: Any) -> np.ndarray:
|
|
34
|
+
check_metric(self.metric)
|
|
35
|
+
return resolve_coords(X, self.coords, self.coord_columns, self.metric)
|
|
36
|
+
|
|
37
|
+
# -- partitioning splitters -------------------------------------------------------
|
|
38
|
+
def _fold_labels(self, X: Any, y: Any = None, groups: Any = None) -> np.ndarray:
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
|
|
41
|
+
def assign_folds(self, X: Any = None, y: Any = None, groups: Any = None) -> np.ndarray:
|
|
42
|
+
"""Return the test-fold index (0 .. n_splits-1) of every sample."""
|
|
43
|
+
return self._fold_labels(X, y, groups)
|
|
44
|
+
|
|
45
|
+
def split(
|
|
46
|
+
self, X: Any, y: Any = None, groups: Any = None
|
|
47
|
+
) -> Iterator[Tuple[np.ndarray, np.ndarray]]:
|
|
48
|
+
"""Generate ``(train_index, test_index)`` pairs, one per fold."""
|
|
49
|
+
labels = self._fold_labels(X, y, groups)
|
|
50
|
+
indices = np.arange(len(labels))
|
|
51
|
+
for fold in range(int(labels.max()) + 1):
|
|
52
|
+
test = labels == fold
|
|
53
|
+
yield indices[~test], indices[test]
|
|
54
|
+
|
|
55
|
+
def get_n_splits(self, X: Any = None, y: Any = None, groups: Any = None) -> int:
|
|
56
|
+
"""Number of splitting iterations."""
|
|
57
|
+
return int(self.n_splits)
|
|
58
|
+
|
|
59
|
+
def __repr__(self) -> str:
|
|
60
|
+
sig = inspect.signature(type(self).__init__)
|
|
61
|
+
parts: List[str] = []
|
|
62
|
+
for name in sig.parameters:
|
|
63
|
+
if name == "self":
|
|
64
|
+
continue
|
|
65
|
+
parts.append(f"{name}={_short(getattr(self, name, None))}")
|
|
66
|
+
return f"{type(self).__name__}({', '.join(parts)})"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def check_n_splits(n_splits: int) -> int:
|
|
70
|
+
if not isinstance(n_splits, (int, np.integer)) or isinstance(n_splits, bool):
|
|
71
|
+
raise ValueError(f"n_splits must be an integer, got {n_splits!r}")
|
|
72
|
+
if n_splits < 2:
|
|
73
|
+
raise ValueError(f"n_splits must be at least 2, got {n_splits}")
|
|
74
|
+
return int(n_splits)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def assign_groups_to_folds(
|
|
78
|
+
sizes: np.ndarray, n_splits: int, rng: np.random.RandomState, balance: bool
|
|
79
|
+
) -> np.ndarray:
|
|
80
|
+
"""Assign ``len(sizes)`` groups (blocks or clusters) to ``n_splits`` folds.
|
|
81
|
+
|
|
82
|
+
``balance=False``: groups are shuffled and dealt round-robin, so every fold receives the
|
|
83
|
+
same number of groups (+/- 1). ``balance=True``: groups are visited from largest to
|
|
84
|
+
smallest (random order among equal sizes) and each goes to the fold that currently holds
|
|
85
|
+
the fewest samples (longest-processing-time greedy), which equalises fold sizes.
|
|
86
|
+
"""
|
|
87
|
+
n_groups = len(sizes)
|
|
88
|
+
if n_groups < n_splits:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
f"Only {n_groups} non-empty spatial groups for n_splits={n_splits}; "
|
|
91
|
+
"use smaller blocks / more clusters or fewer folds."
|
|
92
|
+
)
|
|
93
|
+
order = rng.permutation(n_groups)
|
|
94
|
+
out = np.empty(n_groups, dtype=int)
|
|
95
|
+
if not balance:
|
|
96
|
+
out[order] = np.arange(n_groups) % n_splits
|
|
97
|
+
return out
|
|
98
|
+
order = order[np.argsort(-sizes[order], kind="stable")]
|
|
99
|
+
load = np.zeros(n_splits)
|
|
100
|
+
for g in order:
|
|
101
|
+
candidates = np.flatnonzero(load == np.min(load))
|
|
102
|
+
f = int(candidates[rng.randint(len(candidates))])
|
|
103
|
+
out[g] = f
|
|
104
|
+
load[f] += sizes[g]
|
|
105
|
+
return out
|
spatialcv/_coords.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Coordinate handling, distances and nearest-neighbour queries.
|
|
2
|
+
|
|
3
|
+
Two coordinate systems are supported:
|
|
4
|
+
|
|
5
|
+
* ``metric="euclidean"``: planar coordinates (for example metres in a projected CRS).
|
|
6
|
+
* ``metric="haversine"``: geographic ``(lon, lat)`` pairs in decimal degrees. Distances are
|
|
7
|
+
great-circle distances in metres on a sphere with the mean Earth radius.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import warnings
|
|
13
|
+
from typing import Any, List, Optional, Sequence
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from sklearn.neighbors import BallTree, KDTree
|
|
17
|
+
|
|
18
|
+
#: Mean Earth radius in metres (IUGG), used for haversine distances.
|
|
19
|
+
EARTH_RADIUS_M = 6_371_008.8
|
|
20
|
+
|
|
21
|
+
_METRICS = ("euclidean", "haversine")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def check_metric(metric: str) -> str:
|
|
25
|
+
"""Validate a distance metric name."""
|
|
26
|
+
if metric not in _METRICS:
|
|
27
|
+
raise ValueError(f"metric must be one of {_METRICS}, got {metric!r}")
|
|
28
|
+
return metric
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _is_geo(obj: Any) -> bool:
|
|
32
|
+
"""Duck-type test for a geopandas GeoSeries / GeoDataFrame."""
|
|
33
|
+
geom = getattr(obj, "geometry", obj)
|
|
34
|
+
return hasattr(geom, "geom_type") and hasattr(geom, "centroid")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _geo_to_xy(obj: Any) -> np.ndarray:
|
|
38
|
+
geom = getattr(obj, "geometry", obj)
|
|
39
|
+
if bool(np.all(np.asarray(geom.geom_type) == "Point")):
|
|
40
|
+
pts = geom
|
|
41
|
+
else:
|
|
42
|
+
with warnings.catch_warnings():
|
|
43
|
+
# Centroids in a geographic CRS are approximate; that is fine for fold building.
|
|
44
|
+
warnings.simplefilter("ignore")
|
|
45
|
+
pts = geom.centroid
|
|
46
|
+
return np.column_stack([np.asarray(pts.x, dtype=float), np.asarray(pts.y, dtype=float)])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def as_coords(coords: Any, metric: str = "euclidean") -> np.ndarray:
|
|
50
|
+
"""Convert ``coords`` (array-like of shape (n, 2) or a GeoSeries) to a float array."""
|
|
51
|
+
arr = _geo_to_xy(coords) if _is_geo(coords) else np.asarray(coords, dtype=float)
|
|
52
|
+
if arr.ndim != 2 or arr.shape[1] != 2:
|
|
53
|
+
raise ValueError(f"coordinates must have shape (n_samples, 2), got {arr.shape}")
|
|
54
|
+
if not np.all(np.isfinite(arr)):
|
|
55
|
+
raise ValueError("coordinates contain NaN or infinite values")
|
|
56
|
+
if metric == "haversine":
|
|
57
|
+
if np.any(np.abs(arr[:, 1]) > 90.0) or np.any(np.abs(arr[:, 0]) > 360.0):
|
|
58
|
+
raise ValueError(
|
|
59
|
+
"metric='haversine' expects (lon, lat) in decimal degrees; "
|
|
60
|
+
"found latitudes outside [-90, 90] or longitudes outside [-360, 360]"
|
|
61
|
+
)
|
|
62
|
+
return arr
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def resolve_coords(
|
|
66
|
+
X: Any,
|
|
67
|
+
coords: Any = None,
|
|
68
|
+
coord_columns: Optional[Sequence[Any]] = None,
|
|
69
|
+
metric: str = "euclidean",
|
|
70
|
+
) -> np.ndarray:
|
|
71
|
+
"""Find the sample coordinates for a splitter.
|
|
72
|
+
|
|
73
|
+
Resolution order:
|
|
74
|
+
|
|
75
|
+
1. ``coords`` given explicitly (array of shape (n, 2) or a GeoSeries/GeoDataFrame);
|
|
76
|
+
2. ``coord_columns``: two column names (DataFrame) or integer positions inside ``X``;
|
|
77
|
+
3. ``X`` itself when it is a GeoDataFrame/GeoSeries (point geometry or centroids);
|
|
78
|
+
4. ``X`` itself when it has exactly two columns.
|
|
79
|
+
"""
|
|
80
|
+
if coords is not None:
|
|
81
|
+
arr = as_coords(coords, metric)
|
|
82
|
+
elif coord_columns is not None:
|
|
83
|
+
if X is None:
|
|
84
|
+
raise ValueError("coord_columns requires X")
|
|
85
|
+
cols = list(coord_columns)
|
|
86
|
+
if len(cols) != 2:
|
|
87
|
+
raise ValueError("coord_columns must name exactly two columns")
|
|
88
|
+
if hasattr(X, "iloc"):
|
|
89
|
+
if all(c in X.columns for c in cols):
|
|
90
|
+
arr = np.asarray(X[cols], dtype=float)
|
|
91
|
+
else:
|
|
92
|
+
arr = np.asarray(X.iloc[:, cols], dtype=float)
|
|
93
|
+
else:
|
|
94
|
+
arr = np.asarray(X)[:, cols].astype(float)
|
|
95
|
+
arr = as_coords(arr, metric)
|
|
96
|
+
elif X is not None and _is_geo(X):
|
|
97
|
+
arr = as_coords(X, metric)
|
|
98
|
+
elif X is not None:
|
|
99
|
+
arr = np.asarray(X)
|
|
100
|
+
if arr.ndim != 2 or arr.shape[1] != 2:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"Cannot find sample coordinates: pass coords=(n, 2) array, "
|
|
103
|
+
"coord_columns=[x_col, y_col], a GeoDataFrame as X, or an X with "
|
|
104
|
+
"exactly two columns."
|
|
105
|
+
)
|
|
106
|
+
arr = as_coords(arr.astype(float), metric)
|
|
107
|
+
else:
|
|
108
|
+
raise ValueError("No coordinates: pass coords= or X")
|
|
109
|
+
if X is not None and not _is_geo(X) and hasattr(X, "__len__") and len(X) != len(arr):
|
|
110
|
+
raise ValueError(f"coords has {len(arr)} rows but X has {len(X)} samples")
|
|
111
|
+
return arr
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def to_planar(coords: np.ndarray, metric: str) -> np.ndarray:
|
|
115
|
+
"""Return planar coordinates in metres (equirectangular projection for lon/lat)."""
|
|
116
|
+
if metric == "euclidean":
|
|
117
|
+
return np.asarray(coords, dtype=float)
|
|
118
|
+
lon = np.radians(coords[:, 0])
|
|
119
|
+
lat = np.radians(coords[:, 1])
|
|
120
|
+
lat0 = float(np.mean(lat))
|
|
121
|
+
lon0 = float(np.mean(lon))
|
|
122
|
+
x = EARTH_RADIUS_M * (lon - lon0) * np.cos(lat0)
|
|
123
|
+
y = EARTH_RADIUS_M * (lat - lat0)
|
|
124
|
+
return np.column_stack([x, y])
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def to_unit_sphere(coords: np.ndarray) -> np.ndarray:
|
|
128
|
+
"""(lon, lat) degrees to 3D unit vectors (for clustering on the sphere)."""
|
|
129
|
+
lon = np.radians(coords[:, 0])
|
|
130
|
+
lat = np.radians(coords[:, 1])
|
|
131
|
+
return np.column_stack([np.cos(lat) * np.cos(lon), np.cos(lat) * np.sin(lon), np.sin(lat)])
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def pairwise_distances(a: np.ndarray, b: np.ndarray, metric: str = "euclidean") -> np.ndarray:
|
|
135
|
+
"""Dense distance matrix of shape (len(a), len(b))."""
|
|
136
|
+
a = np.asarray(a, dtype=float)
|
|
137
|
+
b = np.asarray(b, dtype=float)
|
|
138
|
+
if metric == "euclidean":
|
|
139
|
+
return np.hypot(a[:, 0, None] - b[None, :, 0], a[:, 1, None] - b[None, :, 1])
|
|
140
|
+
lon1 = np.radians(a[:, 0])[:, None]
|
|
141
|
+
lat1 = np.radians(a[:, 1])[:, None]
|
|
142
|
+
lon2 = np.radians(b[:, 0])[None, :]
|
|
143
|
+
lat2 = np.radians(b[:, 1])[None, :]
|
|
144
|
+
h = (
|
|
145
|
+
np.sin((lat2 - lat1) / 2.0) ** 2
|
|
146
|
+
+ np.cos(lat1) * np.cos(lat2) * np.sin((lon2 - lon1) / 2.0) ** 2
|
|
147
|
+
)
|
|
148
|
+
return 2.0 * EARTH_RADIUS_M * np.arcsin(np.sqrt(np.clip(h, 0.0, 1.0)))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class _Tree:
|
|
152
|
+
"""Nearest-neighbour index in metres for either metric."""
|
|
153
|
+
|
|
154
|
+
def __init__(self, ref: np.ndarray, metric: str) -> None:
|
|
155
|
+
self.metric = metric
|
|
156
|
+
if metric == "haversine":
|
|
157
|
+
self._scale = EARTH_RADIUS_M
|
|
158
|
+
self._tree: Any = BallTree(np.radians(ref[:, ::-1]), metric="haversine")
|
|
159
|
+
else:
|
|
160
|
+
self._scale = 1.0
|
|
161
|
+
self._tree = KDTree(np.asarray(ref, dtype=float))
|
|
162
|
+
|
|
163
|
+
def _q(self, pts: np.ndarray) -> np.ndarray:
|
|
164
|
+
pts = np.asarray(pts, dtype=float)
|
|
165
|
+
return np.radians(pts[:, ::-1]) if self.metric == "haversine" else pts
|
|
166
|
+
|
|
167
|
+
def query(self, pts: np.ndarray, k: int = 1) -> np.ndarray:
|
|
168
|
+
dist, _ = self._tree.query(self._q(pts), k=k)
|
|
169
|
+
return np.asarray(dist) * self._scale
|
|
170
|
+
|
|
171
|
+
def query_radius(self, pts: np.ndarray, r: float) -> List[np.ndarray]:
|
|
172
|
+
ind = self._tree.query_radius(self._q(pts), r=r / self._scale)
|
|
173
|
+
return list(ind)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def nearest_distance(query: np.ndarray, ref: np.ndarray, metric: str = "euclidean") -> np.ndarray:
|
|
177
|
+
"""Distance from each ``query`` point to its nearest ``ref`` point."""
|
|
178
|
+
if len(ref) == 0:
|
|
179
|
+
return np.full(len(query), np.inf)
|
|
180
|
+
if len(query) == 0:
|
|
181
|
+
return np.empty(0)
|
|
182
|
+
return _Tree(ref, metric).query(query, k=1)[:, 0]
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def nearest_neighbour_loo(coords: np.ndarray, metric: str = "euclidean") -> np.ndarray:
|
|
186
|
+
"""Distance from every point to its nearest *other* point (leave-one-out)."""
|
|
187
|
+
if len(coords) < 2:
|
|
188
|
+
raise ValueError("need at least two points")
|
|
189
|
+
return _Tree(coords, metric).query(coords, k=2)[:, 1]
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def within_distance(
|
|
193
|
+
query: np.ndarray, ref: np.ndarray, radius: float, metric: str = "euclidean"
|
|
194
|
+
) -> np.ndarray:
|
|
195
|
+
"""Sorted unique indices of ``ref`` points within ``radius`` (inclusive) of any query."""
|
|
196
|
+
if len(query) == 0 or len(ref) == 0:
|
|
197
|
+
return np.empty(0, dtype=int)
|
|
198
|
+
hits = _Tree(ref, metric).query_radius(query, radius)
|
|
199
|
+
if not hits:
|
|
200
|
+
return np.empty(0, dtype=int)
|
|
201
|
+
return np.unique(np.concatenate(hits).astype(int))
|
spatialcv/blocks.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Spatial block cross-validation (Roberts et al. 2017; Valavi et al. 2019)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from typing import Any, Optional, Sequence, Tuple
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
from sklearn.utils import check_random_state
|
|
10
|
+
|
|
11
|
+
from ._base import SpatialSplitter, assign_groups_to_folds, check_n_splits
|
|
12
|
+
from ._coords import to_planar
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _hex_cells(xy: np.ndarray, size: float) -> Tuple[np.ndarray, np.ndarray]:
|
|
16
|
+
"""Axial (row, column) of pointy-top hexagons with area ``size**2``."""
|
|
17
|
+
radius = size * math.sqrt(2.0 / (3.0 * math.sqrt(3.0)))
|
|
18
|
+
x = xy[:, 0]
|
|
19
|
+
y = xy[:, 1]
|
|
20
|
+
q = (math.sqrt(3.0) / 3.0 * x - y / 3.0) / radius
|
|
21
|
+
r = (2.0 / 3.0 * y) / radius
|
|
22
|
+
# cube rounding
|
|
23
|
+
cx, cz = q, r
|
|
24
|
+
cy = -cx - cz
|
|
25
|
+
rx, ry, rz = np.round(cx), np.round(cy), np.round(cz)
|
|
26
|
+
dx, dy, dz = np.abs(rx - cx), np.abs(ry - cy), np.abs(rz - cz)
|
|
27
|
+
fix_x = (dx > dy) & (dx > dz)
|
|
28
|
+
fix_z = ~fix_x & ~(dy > dz)
|
|
29
|
+
rx = np.where(fix_x, -ry - rz, rx)
|
|
30
|
+
rz = np.where(fix_z, -rx - ry, rz)
|
|
31
|
+
return rz.astype(np.int64), rx.astype(np.int64)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def block_ids(
|
|
35
|
+
xy: np.ndarray, block_size: float, shape: str = "square", origin: Any = None
|
|
36
|
+
) -> Tuple[np.ndarray, np.ndarray]:
|
|
37
|
+
"""Grid (row, column) of the block that contains each planar point."""
|
|
38
|
+
origin = xy.min(axis=0) if origin is None else np.asarray(origin, dtype=float)
|
|
39
|
+
rel = xy - origin
|
|
40
|
+
if shape == "square":
|
|
41
|
+
col = np.floor(rel[:, 0] / block_size).astype(np.int64)
|
|
42
|
+
row = np.floor(rel[:, 1] / block_size).astype(np.int64)
|
|
43
|
+
return row, col
|
|
44
|
+
if shape == "hex":
|
|
45
|
+
return _hex_cells(rel, block_size)
|
|
46
|
+
raise ValueError(f"shape must be 'square' or 'hex', got {shape!r}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class SpatialBlockKFold(SpatialSplitter):
|
|
50
|
+
"""K-fold cross-validation with whole spatial blocks held out together.
|
|
51
|
+
|
|
52
|
+
The study area is covered with square or hexagonal blocks of side ``block_size``
|
|
53
|
+
(hexagons have the same area as the squares). Every block becomes a unit that is
|
|
54
|
+
assigned to one test fold, so test samples are surrounded by other test samples rather
|
|
55
|
+
than by training samples. Choose ``block_size`` at least as large as the range of
|
|
56
|
+
spatial autocorrelation (see :func:`spatialcv.estimate_autocorrelation_range`), as
|
|
57
|
+
recommended by Roberts et al. (2017) and Valavi et al. (2019).
|
|
58
|
+
|
|
59
|
+
Parameters
|
|
60
|
+
----------
|
|
61
|
+
n_splits : int, default=5
|
|
62
|
+
Number of folds.
|
|
63
|
+
block_size : float, optional
|
|
64
|
+
Block side in coordinate units (metres for ``metric="haversine"``). Defaults to
|
|
65
|
+
the larger side of the bounding box divided by ``2 * n_splits``.
|
|
66
|
+
shape : {"square", "hex"}, default="square"
|
|
67
|
+
assignment : {"random", "systematic"}, default="random"
|
|
68
|
+
``"random"`` shuffles the occupied blocks into folds. ``"systematic"`` numbers the
|
|
69
|
+
grid cells row by row and assigns cell ``i`` to fold ``i % n_splits`` (blockCV's
|
|
70
|
+
systematic selection); it is deterministic.
|
|
71
|
+
balance : bool, default=False
|
|
72
|
+
With random assignment, assign blocks greedily (largest first, to the currently
|
|
73
|
+
smallest fold) so the folds hold similar numbers of samples.
|
|
74
|
+
coords : array-like of shape (n_samples, 2) or GeoSeries, optional
|
|
75
|
+
Sample coordinates. See *Coordinates* in the README for the lookup rules.
|
|
76
|
+
coord_columns : sequence of two column names or positions in ``X``, optional
|
|
77
|
+
metric : {"euclidean", "haversine"}, default="euclidean"
|
|
78
|
+
``"haversine"`` means ``(lon, lat)`` degrees; blocks are then laid out on a local
|
|
79
|
+
equirectangular projection in metres.
|
|
80
|
+
origin : (float, float), optional
|
|
81
|
+
Lower-left corner of the grid (planar units); defaults to the data minimum.
|
|
82
|
+
random_state : int, RandomState instance or None
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
def __init__(
|
|
86
|
+
self,
|
|
87
|
+
n_splits: int = 5,
|
|
88
|
+
*,
|
|
89
|
+
block_size: Optional[float] = None,
|
|
90
|
+
shape: str = "square",
|
|
91
|
+
assignment: str = "random",
|
|
92
|
+
balance: bool = False,
|
|
93
|
+
coords: Any = None,
|
|
94
|
+
coord_columns: Optional[Sequence[Any]] = None,
|
|
95
|
+
metric: str = "euclidean",
|
|
96
|
+
origin: Optional[Tuple[float, float]] = None,
|
|
97
|
+
random_state: Any = None,
|
|
98
|
+
) -> None:
|
|
99
|
+
self.n_splits = check_n_splits(n_splits)
|
|
100
|
+
self.block_size = block_size
|
|
101
|
+
self.shape = shape
|
|
102
|
+
self.assignment = assignment
|
|
103
|
+
self.balance = balance
|
|
104
|
+
self.coords = coords
|
|
105
|
+
self.coord_columns = coord_columns
|
|
106
|
+
self.metric = metric
|
|
107
|
+
self.origin = origin
|
|
108
|
+
self.random_state = random_state
|
|
109
|
+
|
|
110
|
+
def _resolved_block_size(self, xy: np.ndarray) -> float:
|
|
111
|
+
if self.block_size is None:
|
|
112
|
+
extent = float(np.max(xy.max(axis=0) - xy.min(axis=0)))
|
|
113
|
+
if extent <= 0:
|
|
114
|
+
raise ValueError("all samples share one location; cannot build blocks")
|
|
115
|
+
return extent / (2 * self.n_splits) * (1.0 + 1e-9)
|
|
116
|
+
if not self.block_size > 0:
|
|
117
|
+
raise ValueError(f"block_size must be positive, got {self.block_size}")
|
|
118
|
+
return float(self.block_size)
|
|
119
|
+
|
|
120
|
+
def block_index(self, X: Any = None) -> np.ndarray:
|
|
121
|
+
"""Integer id of the block containing each sample (0 .. n_blocks-1)."""
|
|
122
|
+
xy = to_planar(self._get_coords(X), self.metric)
|
|
123
|
+
row, col = block_ids(xy, self._resolved_block_size(xy), self.shape, self.origin)
|
|
124
|
+
pairs = np.column_stack([row, col])
|
|
125
|
+
_, inverse = np.unique(pairs, axis=0, return_inverse=True)
|
|
126
|
+
return np.asarray(inverse).reshape(-1)
|
|
127
|
+
|
|
128
|
+
def _fold_labels(self, X: Any, y: Any = None, groups: Any = None) -> np.ndarray:
|
|
129
|
+
if self.assignment not in ("random", "systematic"):
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f"assignment must be 'random' or 'systematic', got {self.assignment!r}"
|
|
132
|
+
)
|
|
133
|
+
xy = to_planar(self._get_coords(X), self.metric)
|
|
134
|
+
size = self._resolved_block_size(xy)
|
|
135
|
+
row, col = block_ids(xy, size, self.shape, self.origin)
|
|
136
|
+
if self.assignment == "systematic":
|
|
137
|
+
if self.balance:
|
|
138
|
+
raise ValueError("balance=True is only available with assignment='random'")
|
|
139
|
+
r0, c0 = row - row.min(), col - col.min()
|
|
140
|
+
ncols = int(c0.max()) + 1
|
|
141
|
+
labels = ((r0 * ncols + c0) % self.n_splits).astype(int)
|
|
142
|
+
if len(np.unique(labels)) < self.n_splits:
|
|
143
|
+
raise ValueError(
|
|
144
|
+
"Systematic assignment left a fold empty; use smaller blocks or fewer folds."
|
|
145
|
+
)
|
|
146
|
+
return labels
|
|
147
|
+
pairs = np.column_stack([row, col])
|
|
148
|
+
_, inverse, counts = np.unique(pairs, axis=0, return_inverse=True, return_counts=True)
|
|
149
|
+
inverse = np.asarray(inverse).reshape(-1)
|
|
150
|
+
rng = check_random_state(self.random_state)
|
|
151
|
+
fold_of_block = assign_groups_to_folds(counts, self.n_splits, rng, self.balance)
|
|
152
|
+
return fold_of_block[inverse]
|
spatialcv/buffer.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Buffered (dead-zone) cross-validation.
|
|
2
|
+
|
|
3
|
+
Training samples closer than ``buffer`` to any test sample are dropped from that split, so
|
|
4
|
+
the model cannot learn the test value from its spatially autocorrelated neighbours
|
|
5
|
+
(Le Rest et al. 2014; Roberts et al. 2017; Pohjankukka et al. 2017 "spatial k-fold CV
|
|
6
|
+
with dead zones").
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any, Iterator, Optional, Sequence, Tuple
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
from sklearn.model_selection import KFold
|
|
15
|
+
|
|
16
|
+
from ._base import SpatialSplitter, check_n_splits
|
|
17
|
+
from ._coords import _Tree
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _check_buffer(buffer: float) -> float:
|
|
21
|
+
try:
|
|
22
|
+
value = float(buffer)
|
|
23
|
+
except (TypeError, ValueError):
|
|
24
|
+
raise ValueError(f"buffer must be a non-negative number, got {buffer!r}") from None
|
|
25
|
+
if not np.isfinite(value) or value < 0:
|
|
26
|
+
raise ValueError(f"buffer must be a non-negative number, got {buffer!r}")
|
|
27
|
+
return value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _apply_buffer(
|
|
31
|
+
tree: _Tree, xy: np.ndarray, train: np.ndarray, test: np.ndarray, buffer: float
|
|
32
|
+
) -> np.ndarray:
|
|
33
|
+
hits = tree.query_radius(xy[test], buffer)
|
|
34
|
+
if hits:
|
|
35
|
+
excluded = np.unique(np.concatenate(hits).astype(int))
|
|
36
|
+
train = np.setdiff1d(train, excluded, assume_unique=True)
|
|
37
|
+
return train
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class BufferedLeaveOneOut(SpatialSplitter):
|
|
41
|
+
"""Leave-one-out with a circular dead zone around the test sample.
|
|
42
|
+
|
|
43
|
+
For sample ``i`` the test set is ``[i]`` and the training set is every sample farther
|
|
44
|
+
than ``buffer`` from it (distance ``<= buffer`` is excluded). Test sets partition the
|
|
45
|
+
data, so :func:`sklearn.model_selection.cross_val_predict` works.
|
|
46
|
+
|
|
47
|
+
Parameters
|
|
48
|
+
----------
|
|
49
|
+
buffer : float
|
|
50
|
+
Dead-zone radius in coordinate units (metres for ``metric="haversine"``).
|
|
51
|
+
coords, coord_columns, metric
|
|
52
|
+
See :class:`SpatialBlockKFold`.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
buffer: float,
|
|
58
|
+
*,
|
|
59
|
+
coords: Any = None,
|
|
60
|
+
coord_columns: Optional[Sequence[Any]] = None,
|
|
61
|
+
metric: str = "euclidean",
|
|
62
|
+
) -> None:
|
|
63
|
+
self.buffer = buffer
|
|
64
|
+
self.coords = coords
|
|
65
|
+
self.coord_columns = coord_columns
|
|
66
|
+
self.metric = metric
|
|
67
|
+
|
|
68
|
+
def get_n_splits(self, X: Any = None, y: Any = None, groups: Any = None) -> int:
|
|
69
|
+
"""Number of samples (one split per sample)."""
|
|
70
|
+
return len(self._get_coords(X))
|
|
71
|
+
|
|
72
|
+
def split(
|
|
73
|
+
self, X: Any, y: Any = None, groups: Any = None
|
|
74
|
+
) -> Iterator[Tuple[np.ndarray, np.ndarray]]:
|
|
75
|
+
buffer = _check_buffer(self.buffer)
|
|
76
|
+
xy = self._get_coords(X)
|
|
77
|
+
n = len(xy)
|
|
78
|
+
tree = _Tree(xy, self.metric)
|
|
79
|
+
neighbours = tree.query_radius(xy, buffer)
|
|
80
|
+
indices = np.arange(n)
|
|
81
|
+
for i in range(n):
|
|
82
|
+
mask = np.ones(n, dtype=bool)
|
|
83
|
+
mask[neighbours[i]] = False
|
|
84
|
+
mask[i] = False
|
|
85
|
+
train = indices[mask]
|
|
86
|
+
if len(train) == 0:
|
|
87
|
+
raise ValueError(f"buffer={buffer} leaves no training data for sample {i}")
|
|
88
|
+
yield train, np.array([i])
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class SpatialKFoldWithBuffer(SpatialSplitter):
|
|
92
|
+
"""Any K-fold scheme plus a dead-zone buffer around each test fold.
|
|
93
|
+
|
|
94
|
+
By default the folds come from a shuffled :class:`sklearn.model_selection.KFold`
|
|
95
|
+
(the "spatial buffer v-fold" of ``spatialsample``); pass ``cv`` to buffer any other
|
|
96
|
+
splitter, for example :class:`SpatialBlockKFold`. For every split, training samples
|
|
97
|
+
within ``buffer`` (inclusive) of *any* test sample are removed.
|
|
98
|
+
|
|
99
|
+
Parameters
|
|
100
|
+
----------
|
|
101
|
+
n_splits : int, default=5
|
|
102
|
+
Used only when ``cv`` is None.
|
|
103
|
+
buffer : float, default=0.0
|
|
104
|
+
cv : cross-validator, optional
|
|
105
|
+
Splitter that produces the folds. It is called with the same ``X, y, groups``.
|
|
106
|
+
shuffle : bool, default=True
|
|
107
|
+
Passed to the default ``KFold``.
|
|
108
|
+
coords, coord_columns, metric
|
|
109
|
+
See :class:`SpatialBlockKFold`.
|
|
110
|
+
random_state : int, RandomState instance or None
|
|
111
|
+
Passed to the default ``KFold``.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def __init__(
|
|
115
|
+
self,
|
|
116
|
+
n_splits: int = 5,
|
|
117
|
+
*,
|
|
118
|
+
buffer: float = 0.0,
|
|
119
|
+
cv: Any = None,
|
|
120
|
+
shuffle: bool = True,
|
|
121
|
+
coords: Any = None,
|
|
122
|
+
coord_columns: Optional[Sequence[Any]] = None,
|
|
123
|
+
metric: str = "euclidean",
|
|
124
|
+
random_state: Any = None,
|
|
125
|
+
) -> None:
|
|
126
|
+
self.n_splits = check_n_splits(n_splits)
|
|
127
|
+
self.buffer = buffer
|
|
128
|
+
self.cv = cv
|
|
129
|
+
self.shuffle = shuffle
|
|
130
|
+
self.coords = coords
|
|
131
|
+
self.coord_columns = coord_columns
|
|
132
|
+
self.metric = metric
|
|
133
|
+
self.random_state = random_state
|
|
134
|
+
|
|
135
|
+
def _base_cv(self) -> Any:
|
|
136
|
+
if self.cv is not None:
|
|
137
|
+
return self.cv
|
|
138
|
+
return KFold(
|
|
139
|
+
self.n_splits,
|
|
140
|
+
shuffle=self.shuffle,
|
|
141
|
+
random_state=self.random_state if self.shuffle else None,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def get_n_splits(self, X: Any = None, y: Any = None, groups: Any = None) -> int:
|
|
145
|
+
return int(self._base_cv().get_n_splits(X, y, groups))
|
|
146
|
+
|
|
147
|
+
def split(
|
|
148
|
+
self, X: Any, y: Any = None, groups: Any = None
|
|
149
|
+
) -> Iterator[Tuple[np.ndarray, np.ndarray]]:
|
|
150
|
+
buffer = _check_buffer(self.buffer)
|
|
151
|
+
xy = self._get_coords(X)
|
|
152
|
+
tree = _Tree(xy, self.metric)
|
|
153
|
+
data = X if X is not None else xy
|
|
154
|
+
for train, test in self._base_cv().split(data, y, groups):
|
|
155
|
+
train = _apply_buffer(tree, xy, np.asarray(train), np.asarray(test), buffer)
|
|
156
|
+
if len(train) == 0:
|
|
157
|
+
raise ValueError(f"buffer={buffer} leaves a fold without training data")
|
|
158
|
+
yield train, np.asarray(test)
|