gorkha 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gorkha-0.0.1/LICENSE +21 -0
- gorkha-0.0.1/PKG-INFO +76 -0
- gorkha-0.0.1/README.md +32 -0
- gorkha-0.0.1/pyproject.toml +54 -0
- gorkha-0.0.1/setup.cfg +4 -0
- gorkha-0.0.1/src/gorkha/__init__.py +0 -0
- gorkha-0.0.1/src/gorkha/baselines.py +154 -0
- gorkha-0.0.1/src/gorkha/campaign.py +263 -0
- gorkha-0.0.1/src/gorkha/checks.py +102 -0
- gorkha-0.0.1/src/gorkha/design.py +64 -0
- gorkha-0.0.1/src/gorkha/download.py +215 -0
- gorkha-0.0.1/src/gorkha/features.py +246 -0
- gorkha-0.0.1/src/gorkha/gnn.py +162 -0
- gorkha-0.0.1/src/gorkha/graph.py +111 -0
- gorkha-0.0.1/src/gorkha/join.py +201 -0
- gorkha-0.0.1/src/gorkha/labels.py +45 -0
- gorkha-0.0.1/src/gorkha/metrics.py +38 -0
- gorkha-0.0.1/src/gorkha/paths.py +32 -0
- gorkha-0.0.1/src/gorkha/plots.py +104 -0
- gorkha-0.0.1/src/gorkha/roads.py +147 -0
- gorkha-0.0.1/src/gorkha/survey_sim.py +70 -0
- gorkha-0.0.1/src/gorkha/wards.py +175 -0
- gorkha-0.0.1/src/gorkha.egg-info/PKG-INFO +76 -0
- gorkha-0.0.1/src/gorkha.egg-info/SOURCES.txt +27 -0
- gorkha-0.0.1/src/gorkha.egg-info/dependency_links.txt +1 -0
- gorkha-0.0.1/src/gorkha.egg-info/requires.txt +29 -0
- gorkha-0.0.1/src/gorkha.egg-info/top_level.txt +1 -0
- gorkha-0.0.1/tests/test_baselines.py +105 -0
- gorkha-0.0.1/tests/test_join.py +28 -0
gorkha-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Angshuman Deb
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
gorkha-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gorkha
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: GORKHA: Graph-based Operational Reconnaissance Kit for Human-in-the-loop Assessment. Post-earthquake damage estimation and survey planning from sparse field surveys.
|
|
5
|
+
Author-email: Angshuman Deb <angshuman311@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/angshuman311/gorkha
|
|
8
|
+
Project-URL: Issues, https://github.com/angshuman311/gorkha/issues
|
|
9
|
+
Keywords: earthquake,damage assessment,graph neural network,kriging,survey planning,Nepal
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: geopandas
|
|
18
|
+
Requires-Dist: shapely
|
|
19
|
+
Requires-Dist: pyogrio
|
|
20
|
+
Requires-Dist: rasterio
|
|
21
|
+
Requires-Dist: rasterstats
|
|
22
|
+
Requires-Dist: pandas
|
|
23
|
+
Requires-Dist: pyarrow
|
|
24
|
+
Requires-Dist: numpy
|
|
25
|
+
Requires-Dist: scikit-learn
|
|
26
|
+
Requires-Dist: scipy
|
|
27
|
+
Requires-Dist: pykrige
|
|
28
|
+
Requires-Dist: libpysal
|
|
29
|
+
Requires-Dist: esda
|
|
30
|
+
Requires-Dist: matplotlib
|
|
31
|
+
Requires-Dist: requests
|
|
32
|
+
Requires-Dist: rapidfuzz
|
|
33
|
+
Requires-Dist: joblib
|
|
34
|
+
Provides-Extra: deep
|
|
35
|
+
Requires-Dist: torch; extra == "deep"
|
|
36
|
+
Requires-Dist: torch_geometric; extra == "deep"
|
|
37
|
+
Requires-Dist: torchgeo; extra == "deep"
|
|
38
|
+
Provides-Extra: sar
|
|
39
|
+
Requires-Dist: asf_search; extra == "sar"
|
|
40
|
+
Requires-Dist: hyp3_sdk; extra == "sar"
|
|
41
|
+
Provides-Extra: test
|
|
42
|
+
Requires-Dist: pytest; extra == "test"
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# GORKHA
|
|
46
|
+
|
|
47
|
+
**Graph-based Operational Reconnaissance Kit for Human-in-the-loop Assessment**
|
|
48
|
+
|
|
49
|
+
GORKHA estimates building damage in all wards of an earthquake region from a small number
|
|
50
|
+
of field surveys, and tells the survey teams where to go next. It uses a graph of the wards,
|
|
51
|
+
the early data that are available without a visit (ShakeMap, terrain, satellite images), and
|
|
52
|
+
the road network with its blockages.
|
|
53
|
+
|
|
54
|
+
The test case is the 2015 Gorkha earthquake sequence in Nepal (945 wards, 11 districts).
|
|
55
|
+
|
|
56
|
+
Status: research code for a CS230 course project. The interfaces change without notice.
|
|
57
|
+
|
|
58
|
+
## Install
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install gorkha
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
For the deep learning parts: `pip install "gorkha[deep]"`. For the Sentinel-1 processing at
|
|
65
|
+
the Alaska Satellite Facility: `pip install "gorkha[sar]"`.
|
|
66
|
+
|
|
67
|
+
## Data
|
|
68
|
+
|
|
69
|
+
The scripts download public data: the Nepal damage survey (NPC and Kathmandu Living Labs),
|
|
70
|
+
OpenStreetMap boundaries and roads (ODbL), USGS ShakeMaps, the NASA ARIA Damage Proxy Map,
|
|
71
|
+
the USGS landslide inventory, the Copernicus DEM, Sentinel-1 and Sentinel-2 images. The
|
|
72
|
+
Sentinel-1 coherence needs a free NASA Earthdata account.
|
|
73
|
+
|
|
74
|
+
## License
|
|
75
|
+
|
|
76
|
+
MIT. See LICENSE. The data sets have their own licenses (see the data section).
|
gorkha-0.0.1/README.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# GORKHA
|
|
2
|
+
|
|
3
|
+
**Graph-based Operational Reconnaissance Kit for Human-in-the-loop Assessment**
|
|
4
|
+
|
|
5
|
+
GORKHA estimates building damage in all wards of an earthquake region from a small number
|
|
6
|
+
of field surveys, and tells the survey teams where to go next. It uses a graph of the wards,
|
|
7
|
+
the early data that are available without a visit (ShakeMap, terrain, satellite images), and
|
|
8
|
+
the road network with its blockages.
|
|
9
|
+
|
|
10
|
+
The test case is the 2015 Gorkha earthquake sequence in Nepal (945 wards, 11 districts).
|
|
11
|
+
|
|
12
|
+
Status: research code for a CS230 course project. The interfaces change without notice.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install gorkha
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
For the deep learning parts: `pip install "gorkha[deep]"`. For the Sentinel-1 processing at
|
|
21
|
+
the Alaska Satellite Facility: `pip install "gorkha[sar]"`.
|
|
22
|
+
|
|
23
|
+
## Data
|
|
24
|
+
|
|
25
|
+
The scripts download public data: the Nepal damage survey (NPC and Kathmandu Living Labs),
|
|
26
|
+
OpenStreetMap boundaries and roads (ODbL), USGS ShakeMaps, the NASA ARIA Damage Proxy Map,
|
|
27
|
+
the USGS landslide inventory, the Copernicus DEM, Sentinel-1 and Sentinel-2 images. The
|
|
28
|
+
Sentinel-1 coherence needs a free NASA Earthdata account.
|
|
29
|
+
|
|
30
|
+
## License
|
|
31
|
+
|
|
32
|
+
MIT. See LICENSE. The data sets have their own licenses (see the data section).
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "gorkha"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "GORKHA: Graph-based Operational Reconnaissance Kit for Human-in-the-loop Assessment. Post-earthquake damage estimation and survey planning from sparse field surveys."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
authors = [{ name = "Angshuman Deb", email = "angshuman311@gmail.com" }]
|
|
8
|
+
keywords = ["earthquake", "damage assessment", "graph neural network", "kriging", "survey planning", "Nepal"]
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
13
|
+
"Intended Audience :: Science/Research",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
16
|
+
]
|
|
17
|
+
dependencies = [
|
|
18
|
+
"geopandas",
|
|
19
|
+
"shapely",
|
|
20
|
+
"pyogrio",
|
|
21
|
+
"rasterio",
|
|
22
|
+
"rasterstats",
|
|
23
|
+
"pandas",
|
|
24
|
+
"pyarrow",
|
|
25
|
+
"numpy",
|
|
26
|
+
"scikit-learn",
|
|
27
|
+
"scipy",
|
|
28
|
+
"pykrige",
|
|
29
|
+
"libpysal",
|
|
30
|
+
"esda",
|
|
31
|
+
"matplotlib",
|
|
32
|
+
"requests",
|
|
33
|
+
"rapidfuzz",
|
|
34
|
+
"joblib",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
deep = ["torch", "torch_geometric", "torchgeo"]
|
|
39
|
+
sar = ["asf_search", "hyp3_sdk"]
|
|
40
|
+
test = ["pytest"]
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/angshuman311/gorkha"
|
|
44
|
+
Issues = "https://github.com/angshuman311/gorkha/issues"
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = ["setuptools>=68"]
|
|
48
|
+
build-backend = "setuptools.build_meta"
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.packages.find]
|
|
51
|
+
where = ["src"]
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
testpaths = ["tests"]
|
gorkha-0.0.1/setup.cfg
ADDED
|
File without changes
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Baseline models. Each model sees only the labels of the surveyed wards (protocol A)."""
|
|
2
|
+
|
|
3
|
+
import warnings
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from pykrige.ok import OrdinaryKriging
|
|
7
|
+
from scipy.optimize import least_squares
|
|
8
|
+
from scipy.spatial.distance import pdist
|
|
9
|
+
from sklearn.ensemble import RandomForestRegressor
|
|
10
|
+
from sklearn.linear_model import Ridge, RidgeCV
|
|
11
|
+
from sklearn.neural_network import MLPRegressor
|
|
12
|
+
|
|
13
|
+
MODELS = ["mean", "ok", "ridge", "mlp", "forest", "rk", "rk_loo", "rk_gdif"]
|
|
14
|
+
MODEL_LABELS = {
|
|
15
|
+
"mean": "Mean of the surveyed wards",
|
|
16
|
+
"ok": "Ordinary kriging (no features)",
|
|
17
|
+
"ridge": "Ridge regression",
|
|
18
|
+
"mlp": "MLP",
|
|
19
|
+
"forest": "Random forest",
|
|
20
|
+
"rk": "Regression kriging, all features",
|
|
21
|
+
"rk_loo": "Regression kriging, all features, leave-one-out residuals",
|
|
22
|
+
"rk_gdif": "Regression kriging, G-DIF covariates",
|
|
23
|
+
}
|
|
24
|
+
RIDGE_ALPHAS = np.logspace(-2, 3, 16)
|
|
25
|
+
|
|
26
|
+
# Trend covariates of the G-DIF code (GDIF_nb.Rmd): shaking intensity, elevation, and the
|
|
27
|
+
# Damage Proxy Map. The engineering forecast of G-DIF is not available here.
|
|
28
|
+
GDIF_FEATURES = ["mmi", "mmi_after", "elev_mean", "dpm_cover", "dpm_p90"]
|
|
29
|
+
|
|
30
|
+
# Variogram fit, with the defaults of the R package gstat (the G-DIF code uses gstat):
|
|
31
|
+
# cutoff at one third of the diagonal of the data, 15 lags, fit weights N / h^2.
|
|
32
|
+
N_LAGS = 15
|
|
33
|
+
CUTOFF_SHARE = 1.0 / 3.0
|
|
34
|
+
# A small survey has too few ward pairs in the cutoff. Then the fit uses all pairs.
|
|
35
|
+
MIN_PAIRS = 40
|
|
36
|
+
PAIRS_PER_LAG = 20
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def exponential(h, nugget, psill, range_):
|
|
40
|
+
"""Exponential variogram in the pykrige convention (practical range)."""
|
|
41
|
+
return nugget + psill * (1.0 - np.exp(-3.0 * h / range_))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def experimental_variogram(xy: np.ndarray, z: np.ndarray) -> tuple:
|
|
45
|
+
"""Lag distance, semivariance, and pair count for each lag that has a pair."""
|
|
46
|
+
d = pdist(xy)
|
|
47
|
+
g = 0.5 * pdist(z[:, None], "sqeuclidean")
|
|
48
|
+
cutoff = float(np.hypot(*np.ptp(xy, axis=0)) * CUTOFF_SHARE)
|
|
49
|
+
if (d <= cutoff).sum() < MIN_PAIRS:
|
|
50
|
+
cutoff = float(d.max())
|
|
51
|
+
inside = d <= cutoff
|
|
52
|
+
n_lags = int(np.clip(inside.sum() // PAIRS_PER_LAG, 4, N_LAGS))
|
|
53
|
+
lag = np.minimum((d[inside] / cutoff * n_lags).astype(int), n_lags - 1)
|
|
54
|
+
count = np.bincount(lag, minlength=n_lags)
|
|
55
|
+
has = count > 0
|
|
56
|
+
h = np.bincount(lag, weights=d[inside], minlength=n_lags)[has] / count[has]
|
|
57
|
+
gamma = np.bincount(lag, weights=g[inside], minlength=n_lags)[has] / count[has]
|
|
58
|
+
return h, gamma, count[has], cutoff
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def fit_variogram(xy: np.ndarray, z: np.ndarray) -> dict:
|
|
62
|
+
"""Weighted least squares fit of an exponential variogram with a nugget."""
|
|
63
|
+
h, gamma, count, cutoff = experimental_variogram(xy, z)
|
|
64
|
+
var = float(np.var(z, ddof=1))
|
|
65
|
+
if len(h) < 3:
|
|
66
|
+
return {"psill": 0.0, "range": cutoff, "nugget": var}
|
|
67
|
+
weight = np.sqrt(count) / h
|
|
68
|
+
top = max(float(gamma.max()), var)
|
|
69
|
+
fit = least_squares(
|
|
70
|
+
lambda p: weight * (exponential(h, *p) - gamma),
|
|
71
|
+
x0=[0.25 * var, 0.75 * var, 0.5 * cutoff],
|
|
72
|
+
bounds=([0.0, 0.0, 0.5 * h.min()], [1.5 * top, 3.0 * top, 3.0 * cutoff]),
|
|
73
|
+
)
|
|
74
|
+
nugget, psill, range_ = fit.x
|
|
75
|
+
return {"psill": float(psill), "range": float(range_), "nugget": float(nugget)}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def krige(xy_s: np.ndarray, z_s: np.ndarray, xy_u: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
79
|
+
"""Ordinary kriging with a variogram fitted on the surveyed wards.
|
|
80
|
+
|
|
81
|
+
Return the prediction and the kriging variance at `xy_u`.
|
|
82
|
+
"""
|
|
83
|
+
if np.ptp(z_s) < 1e-9:
|
|
84
|
+
return np.full(len(xy_u), z_s.mean()), np.zeros(len(xy_u))
|
|
85
|
+
with warnings.catch_warnings():
|
|
86
|
+
warnings.simplefilter("ignore")
|
|
87
|
+
model = OrdinaryKriging(xy_s[:, 0], xy_s[:, 1], z_s, variogram_model="exponential",
|
|
88
|
+
variogram_parameters=fit_variogram(xy_s, z_s),
|
|
89
|
+
enable_plotting=False, verbose=False)
|
|
90
|
+
z, ss = model.execute("points", xy_u[:, 0], xy_u[:, 1], backend="vectorized")
|
|
91
|
+
return np.asarray(z, dtype=float), np.maximum(np.asarray(ss, dtype=float), 0.0)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def ridge_loo_residuals(xs: np.ndarray, ys: np.ndarray, alpha: float) -> np.ndarray:
|
|
95
|
+
"""Leave-one-out residuals of a ridge regression with an intercept (closed form)."""
|
|
96
|
+
n = len(ys)
|
|
97
|
+
xc = xs - xs.mean(axis=0)
|
|
98
|
+
u, sv, _ = np.linalg.svd(xc, full_matrices=False)
|
|
99
|
+
shrink = sv ** 2 / (sv ** 2 + alpha)
|
|
100
|
+
leverage = np.einsum("ij,j,ij->i", u, shrink, u) + 1.0 / n
|
|
101
|
+
fitted = u @ (shrink * (u.T @ (ys - ys.mean()))) + ys.mean()
|
|
102
|
+
return (ys - fitted) / np.maximum(1.0 - leverage, 1e-6)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def predict_all(x: np.ndarray, xy: np.ndarray, y: np.ndarray, surveyed: np.ndarray,
|
|
106
|
+
seed: int = 0, gdif_columns: list | None = None,
|
|
107
|
+
limits: tuple = (0.0, 1.0)) -> dict:
|
|
108
|
+
"""Fit each baseline on the surveyed wards and predict all wards.
|
|
109
|
+
|
|
110
|
+
`x` is the standardized design matrix of all wards. The scaler uses the features of
|
|
111
|
+
all wards, which are known before a survey. It uses no label.
|
|
112
|
+
`gdif_columns` gives the columns of `x` for the G-DIF covariates.
|
|
113
|
+
`limits` gives the permitted range of the target (a fraction, or a damage grade).
|
|
114
|
+
Return {model: (prediction, standard deviation or None)}.
|
|
115
|
+
"""
|
|
116
|
+
s, n = surveyed, len(y)
|
|
117
|
+
ys = y[s]
|
|
118
|
+
out = {}
|
|
119
|
+
|
|
120
|
+
out["mean"] = (np.full(n, ys.mean()), np.full(n, ys.std(ddof=1)))
|
|
121
|
+
|
|
122
|
+
z, var = krige(xy[s], ys, xy)
|
|
123
|
+
out["ok"] = (z, np.sqrt(var))
|
|
124
|
+
|
|
125
|
+
alpha = RidgeCV(alphas=RIDGE_ALPHAS).fit(x[s], ys).alpha_
|
|
126
|
+
trend = Ridge(alpha=alpha).fit(x[s], ys).predict(x)
|
|
127
|
+
out["ridge"] = (trend, None)
|
|
128
|
+
|
|
129
|
+
with warnings.catch_warnings():
|
|
130
|
+
warnings.simplefilter("ignore")
|
|
131
|
+
mlp = MLPRegressor(hidden_layer_sizes=(16,), alpha=1.0, solver="lbfgs", max_iter=500,
|
|
132
|
+
random_state=seed).fit(x[s], ys)
|
|
133
|
+
out["mlp"] = (mlp.predict(x), None)
|
|
134
|
+
|
|
135
|
+
forest = RandomForestRegressor(n_estimators=100, min_samples_leaf=2, max_features=0.5,
|
|
136
|
+
random_state=seed, n_jobs=1).fit(x[s], ys)
|
|
137
|
+
out["forest"] = (forest.predict(x), None)
|
|
138
|
+
|
|
139
|
+
# Regression kriging: the ridge trend plus ordinary kriging of the trend residuals.
|
|
140
|
+
r, var = krige(xy[s], ys - trend[s], xy)
|
|
141
|
+
out["rk"] = (trend + r, np.sqrt(var))
|
|
142
|
+
|
|
143
|
+
# Variant: the residual of a surveyed ward comes from a trend fitted without that ward.
|
|
144
|
+
# An in-sample residual is too small when the survey has few wards.
|
|
145
|
+
r, var = krige(xy[s], ridge_loo_residuals(x[s], ys, alpha), xy)
|
|
146
|
+
out["rk_loo"] = (trend + r, np.sqrt(var))
|
|
147
|
+
|
|
148
|
+
if gdif_columns is not None:
|
|
149
|
+
xg = x[:, gdif_columns]
|
|
150
|
+
trend_g = RidgeCV(alphas=RIDGE_ALPHAS).fit(xg[s], ys).predict(xg)
|
|
151
|
+
r, var = krige(xy[s], ys - trend_g[s], xy)
|
|
152
|
+
out["rk_gdif"] = (trend_g + r, np.sqrt(var))
|
|
153
|
+
|
|
154
|
+
return {k: (np.clip(p, *limits), sd) for k, (p, sd) in out.items()}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Simulated survey campaigns: where do the teams go next, and what does it cost?
|
|
2
|
+
|
|
3
|
+
A campaign has rounds. In each round, a method selects one ward for each team. The teams
|
|
4
|
+
travel on the open roads (shortest travel time), survey the wards, and the labels become
|
|
5
|
+
visible. The cost of a round is the travel time plus a fixed survey time for each ward.
|
|
6
|
+
A ward that no open road reaches costs a helicopter trip.
|
|
7
|
+
|
|
8
|
+
Three methods (decision of 2026-10-07):
|
|
9
|
+
- "random": regression kriging estimate (survey and feature table), random wards.
|
|
10
|
+
- "kriging": regression kriging estimate, wards with the largest kriging variance. The
|
|
11
|
+
roads matter only for the route to the selected wards.
|
|
12
|
+
- "sherpa": GNN estimate with uncertainty. A second network gives a visit score for each
|
|
13
|
+
ward. It sees the road state and learns from damage maps that the model draws from its
|
|
14
|
+
own uncertainty.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import torch
|
|
19
|
+
from sklearn.linear_model import Ridge, RidgeCV
|
|
20
|
+
from torch import nn
|
|
21
|
+
from torch_geometric.nn import GATv2Conv
|
|
22
|
+
|
|
23
|
+
from . import baselines, gnn
|
|
24
|
+
from .roads import HELICOPTER_HOURS, SURVEY_HOURS
|
|
25
|
+
|
|
26
|
+
# ---------------------------------------------------------------- kriging
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def kriging_variance(xy_s: np.ndarray, xy_all: np.ndarray, p: dict) -> np.ndarray:
|
|
30
|
+
"""Ordinary kriging variance at all wards for surveyed positions and a fixed variogram."""
|
|
31
|
+
def gamma(h):
|
|
32
|
+
return np.where(h > 0, p["nugget"] + p["psill"] * (1 - np.exp(-3 * h / p["range"])), 0.0)
|
|
33
|
+
m = len(xy_s)
|
|
34
|
+
a = np.ones((m + 1, m + 1))
|
|
35
|
+
a[:m, :m] = gamma(np.linalg.norm(xy_s[:, None] - xy_s[None], axis=2))
|
|
36
|
+
a[m, m] = 0.0
|
|
37
|
+
b = np.ones((m + 1, len(xy_all)))
|
|
38
|
+
b[:m] = gamma(np.linalg.norm(xy_s[:, None] - xy_all[None], axis=2))
|
|
39
|
+
w = np.linalg.solve(a + 1e-10 * np.eye(m + 1), b)
|
|
40
|
+
return np.maximum((w * b).sum(axis=0), 0.0)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def regression_kriging(x, xy, y, surveyed):
|
|
44
|
+
"""Ridge trend on the features and kriging of the leave-one-out residuals.
|
|
45
|
+
|
|
46
|
+
Return the estimate, the standard deviation, and the residuals at the surveyed wards.
|
|
47
|
+
"""
|
|
48
|
+
alpha = RidgeCV(alphas=baselines.RIDGE_ALPHAS).fit(x[surveyed], y[surveyed]).alpha_
|
|
49
|
+
trend = Ridge(alpha=alpha).fit(x[surveyed], y[surveyed]).predict(x)
|
|
50
|
+
resid = baselines.ridge_loo_residuals(x[surveyed], y[surveyed], alpha)
|
|
51
|
+
r, var = baselines.krige(xy[surveyed], resid, xy)
|
|
52
|
+
return np.clip(trend + r, 0, 1), np.sqrt(var), resid
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def select_kriging(xy, resid, surveyed, n_pick: int) -> list:
|
|
56
|
+
"""Wards with the largest kriging variance of the residuals, one after the other.
|
|
57
|
+
|
|
58
|
+
After each selection, the variance is calculated again as if that ward were surveyed.
|
|
59
|
+
"""
|
|
60
|
+
p = baselines.fit_variogram(xy[surveyed], resid)
|
|
61
|
+
if p["psill"] <= 1e-9:
|
|
62
|
+
p = {"psill": max(np.var(resid), 1e-4), "range": 30.0, "nugget": 0.0}
|
|
63
|
+
have = surveyed.copy()
|
|
64
|
+
picks = []
|
|
65
|
+
for _ in range(n_pick):
|
|
66
|
+
var = kriging_variance(xy[have], xy, p)
|
|
67
|
+
var[have] = -1.0
|
|
68
|
+
i = int(np.argmax(var))
|
|
69
|
+
picks.append(i)
|
|
70
|
+
have[i] = True
|
|
71
|
+
return picks
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ---------------------------------------------------------------- travel and cost
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def nearest_team_hours(travel: np.ndarray, teams: list) -> np.ndarray:
|
|
78
|
+
"""Cost in hours for the nearest team to reach and survey each ward."""
|
|
79
|
+
t = travel[teams].min(axis=0)
|
|
80
|
+
return np.where(np.isfinite(t), t, HELICOPTER_HOURS) + SURVEY_HOURS
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def assign_and_move(picks: list, teams: list, travel: np.ndarray) -> tuple[float, list, int]:
|
|
84
|
+
"""Give each selected ward to a team (shortest travel first). Return cost, positions, flights."""
|
|
85
|
+
teams, left, cost, flights = list(teams), list(picks), 0.0, 0
|
|
86
|
+
free = list(range(len(teams)))
|
|
87
|
+
while left and free:
|
|
88
|
+
t = travel[np.ix_([teams[k] for k in free], left)]
|
|
89
|
+
t = np.where(np.isfinite(t), t, HELICOPTER_HOURS + 1e3) # a flight only if no road
|
|
90
|
+
a, b = np.unravel_index(np.argmin(t), t.shape)
|
|
91
|
+
hours = t[a, b]
|
|
92
|
+
if hours >= 1e3:
|
|
93
|
+
hours, flights = HELICOPTER_HOURS, flights + 1
|
|
94
|
+
cost += hours + SURVEY_HOURS
|
|
95
|
+
teams[free[a]] = left[b]
|
|
96
|
+
free.pop(a)
|
|
97
|
+
left.pop(b)
|
|
98
|
+
return cost, teams, flights
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# ---------------------------------------------------------------- SHERPA visit score
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class VisitScorer(nn.Module):
|
|
105
|
+
"""Network for the visit score (output 3). Edges carry the distance and the road time."""
|
|
106
|
+
|
|
107
|
+
def __init__(self, n_in: int = 6, hidden: int = 32):
|
|
108
|
+
super().__init__()
|
|
109
|
+
self.inp = nn.Linear(n_in, hidden)
|
|
110
|
+
self.convs = nn.ModuleList(GATv2Conv(hidden, hidden // 4, heads=4, edge_dim=3) for _ in range(3))
|
|
111
|
+
self.out = nn.Linear(hidden, 1)
|
|
112
|
+
|
|
113
|
+
def forward(self, x, edge_index, edge_attr):
|
|
114
|
+
h = torch.relu(self.inp(x))
|
|
115
|
+
for conv in self.convs:
|
|
116
|
+
h = h + torch.relu(conv(h, edge_index, edge_attr))
|
|
117
|
+
return self.out(h).squeeze(-1)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def kernel_matrix(xy: np.ndarray, range_km: float) -> np.ndarray:
|
|
121
|
+
d = np.linalg.norm(xy[:, None] - xy[None], axis=2)
|
|
122
|
+
return np.exp(-3 * d / range_km)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def imagined_benefit(sd, kern, cost, travel, surveyed, rng, n_maps: int = 24, lookahead: float = 0.5,
|
|
126
|
+
chol=None):
|
|
127
|
+
"""Benefit for each hour of a visit to each ward, on damage maps drawn from the model.
|
|
128
|
+
|
|
129
|
+
A map is the model estimate plus an error with the model standard deviation and a
|
|
130
|
+
spatial correlation. A visit to ward i shows its error r_i. The estimate of ward j then
|
|
131
|
+
moves by kern_ij * r_i. The benefit is the decrease of the absolute error over all
|
|
132
|
+
wards. The second term is the best benefit for each hour that is reachable after i.
|
|
133
|
+
"""
|
|
134
|
+
n = len(sd)
|
|
135
|
+
if chol is None:
|
|
136
|
+
chol = np.linalg.cholesky(kern + 1e-6 * np.eye(n))
|
|
137
|
+
benefit = np.zeros(n)
|
|
138
|
+
for _ in range(n_maps):
|
|
139
|
+
r = sd * (chol @ rng.standard_normal(n))
|
|
140
|
+
after = np.abs(r[None, :] - kern * r[:, None]) # row i: errors after a visit to i
|
|
141
|
+
gain = (np.abs(r)[None, :] - after)
|
|
142
|
+
gain[:, surveyed] = 0.0
|
|
143
|
+
benefit += gain.sum(axis=1)
|
|
144
|
+
benefit /= n_maps
|
|
145
|
+
benefit[surveyed] = 0.0
|
|
146
|
+
rate = benefit / cost
|
|
147
|
+
step = np.where(np.isfinite(travel), travel, HELICOPTER_HOURS) + SURVEY_HOURS
|
|
148
|
+
follow = (benefit[None, :] / step)
|
|
149
|
+
np.fill_diagonal(follow, 0.0)
|
|
150
|
+
return rate + lookahead * follow.max(axis=1) * (benefit > 0)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def learned_scores(sd, mean, surveyed, xy, pairs, travel, teams, range_km, seed) -> np.ndarray:
|
|
154
|
+
"""Train the visit scorer on imagined maps and return its score for each ward."""
|
|
155
|
+
rng = np.random.default_rng([11, seed])
|
|
156
|
+
torch.manual_seed(seed)
|
|
157
|
+
kern = kernel_matrix(xy, range_km)
|
|
158
|
+
chol = np.linalg.cholesky(kern + 1e-6 * np.eye(len(sd)))
|
|
159
|
+
cost = nearest_team_hours(travel, teams)
|
|
160
|
+
reach = np.isfinite(travel[teams].min(axis=0)).astype(float)
|
|
161
|
+
x = np.column_stack([sd / sd.mean(), mean, surveyed.astype(float), np.log(cost), reach,
|
|
162
|
+
np.log1p(np.isfinite(travel).sum(axis=1))])
|
|
163
|
+
t_edge = travel[pairs[:, 0], pairs[:, 1]]
|
|
164
|
+
open_edge = np.isfinite(t_edge)
|
|
165
|
+
attr = np.column_stack([np.linalg.norm(xy[pairs[:, 0]] - xy[pairs[:, 1]], axis=1) / 10,
|
|
166
|
+
np.where(open_edge, np.log1p(t_edge), 0.0), open_edge.astype(float)])
|
|
167
|
+
dev = gnn.DEVICE
|
|
168
|
+
to = lambda a, t=torch.float32: torch.as_tensor(np.ascontiguousarray(a), dtype=t, device=dev)
|
|
169
|
+
xt = to(x)
|
|
170
|
+
edges = to(np.concatenate([pairs, pairs[:, ::-1]]).T, torch.long)
|
|
171
|
+
attr_t = to(np.concatenate([attr, attr]))
|
|
172
|
+
free = torch.as_tensor(~surveyed, device=dev)
|
|
173
|
+
model = VisitScorer(x.shape[1]).to(dev)
|
|
174
|
+
opt = torch.optim.Adam(model.parameters(), lr=5e-3, weight_decay=1e-4)
|
|
175
|
+
for _ in range(120):
|
|
176
|
+
target = imagined_benefit(sd, kern, cost, travel, surveyed, rng, n_maps=4, chol=chol)
|
|
177
|
+
target = to(target / (target[~surveyed].mean() + 1e-9))
|
|
178
|
+
loss = ((model(xt, edges, attr_t) - target)[free] ** 2).mean()
|
|
179
|
+
opt.zero_grad()
|
|
180
|
+
loss.backward()
|
|
181
|
+
opt.step()
|
|
182
|
+
with torch.no_grad():
|
|
183
|
+
score = model(xt, edges, attr_t).cpu().numpy()
|
|
184
|
+
score[surveyed] = -np.inf
|
|
185
|
+
return score
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def select_sherpa(sd, mean, surveyed, xy, pairs, travel, teams, n_pick, range_km, seed,
|
|
189
|
+
learned: bool = True) -> list:
|
|
190
|
+
"""Select wards one after the other. After each selection, the team moves there in the
|
|
191
|
+
plan, and the uncertainty near the selected ward decreases."""
|
|
192
|
+
sd, teams, have = sd.copy(), list(teams), surveyed.copy()
|
|
193
|
+
kern = kernel_matrix(xy, range_km)
|
|
194
|
+
rng = np.random.default_rng([13, seed])
|
|
195
|
+
picks = []
|
|
196
|
+
for k in range(n_pick):
|
|
197
|
+
if learned and k == 0:
|
|
198
|
+
base = learned_scores(sd, mean, have, xy, pairs, travel, teams, range_km, seed)
|
|
199
|
+
base_cost = nearest_team_hours(travel, teams)
|
|
200
|
+
if learned:
|
|
201
|
+
# The score is a benefit for each hour. Update it for the moved teams and for the
|
|
202
|
+
# decreased uncertainty without a new training in the same round.
|
|
203
|
+
cost = nearest_team_hours(travel, teams)
|
|
204
|
+
score = base * (base_cost / cost) * (sd / sd0 if k else 1.0)
|
|
205
|
+
else:
|
|
206
|
+
cost = nearest_team_hours(travel, teams)
|
|
207
|
+
score = imagined_benefit(sd, kern, cost, travel, have, rng)
|
|
208
|
+
score = np.where(have, -np.inf, score)
|
|
209
|
+
i = int(np.argmax(score))
|
|
210
|
+
picks.append(i)
|
|
211
|
+
if k == 0:
|
|
212
|
+
sd0 = sd.copy()
|
|
213
|
+
t = travel[teams, i]
|
|
214
|
+
teams[int(np.argmin(np.where(np.isfinite(t), t, 1e6)))] = i
|
|
215
|
+
have[i] = True
|
|
216
|
+
sd = sd * (1 - kern[i])
|
|
217
|
+
sd0 = np.maximum(sd0, 1e-9)
|
|
218
|
+
return picks
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# ---------------------------------------------------------------- campaign
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def run(method: str, x, xy, y, pairs, edges, travel, start_wards: list, rounds: int, seed: int = 0,
|
|
225
|
+
members: int = 3, learned: bool = True, x_sherpa=None) -> list:
|
|
226
|
+
"""Run one campaign. Return one record for each round (round 0 is the start).
|
|
227
|
+
|
|
228
|
+
`x` is the feature table (input 3) for the kriging cases. `x_sherpa` is the input of
|
|
229
|
+
SHERPA: the feature table with the image vectors (inputs 3 and 4). If it is None,
|
|
230
|
+
SHERPA uses `x`.
|
|
231
|
+
"""
|
|
232
|
+
x_sherpa = x if x_sherpa is None else x_sherpa
|
|
233
|
+
n = len(y)
|
|
234
|
+
rng = np.random.default_rng([17, seed])
|
|
235
|
+
surveyed = np.zeros(n, dtype=bool)
|
|
236
|
+
surveyed[start_wards] = True
|
|
237
|
+
teams, cost, flights, records = list(start_wards), 0.0, 0, []
|
|
238
|
+
for rnd in range(rounds + 1):
|
|
239
|
+
if method == "sherpa":
|
|
240
|
+
mean, sd, _ = gnn.fit_predict(x_sherpa, xy, y, surveyed, edges, seed=seed * 100 + rnd,
|
|
241
|
+
members=members)
|
|
242
|
+
else:
|
|
243
|
+
mean, sd, resid = regression_kriging(x, xy, y, surveyed)
|
|
244
|
+
err = np.abs(mean - y)[~surveyed]
|
|
245
|
+
records.append({"method": method, "seed": seed, "round": rnd, "n_surveyed": int(surveyed.sum()),
|
|
246
|
+
"cost_hours": cost, "flights": flights, "mae": float(err.mean()),
|
|
247
|
+
"coverage_90": float((err <= 1.6449 * sd[~surveyed]).mean())})
|
|
248
|
+
if rnd == rounds:
|
|
249
|
+
break
|
|
250
|
+
if method == "random":
|
|
251
|
+
picks = list(rng.choice(np.flatnonzero(~surveyed), size=len(teams), replace=False))
|
|
252
|
+
elif method == "kriging":
|
|
253
|
+
picks = select_kriging(xy, resid, surveyed, len(teams))
|
|
254
|
+
else:
|
|
255
|
+
p = baselines.fit_variogram(xy[surveyed], y[surveyed] - mean[surveyed])
|
|
256
|
+
range_km = float(np.clip(p["range"], 10.0, 80.0))
|
|
257
|
+
picks = select_sherpa(sd, mean, surveyed, xy, pairs, travel, teams, len(teams), range_km,
|
|
258
|
+
seed * 100 + rnd, learned=learned)
|
|
259
|
+
step_cost, teams, f = assign_and_move(picks, teams, travel)
|
|
260
|
+
cost += step_cost
|
|
261
|
+
flights += f
|
|
262
|
+
surveyed[picks] = True
|
|
263
|
+
return records
|