argleton 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- adapters/__init__.py +0 -0
- adapters/engine_geopandas.py +176 -0
- adapters/engine_naive.py +220 -0
- adapters/engine_rasterio.py +78 -0
- adapters/engine_whitebox.py +34 -0
- adapters/gis_mcp.py +192 -0
- adapters/mapsmith.py +496 -0
- argleton/__init__.py +3 -0
- argleton/model.py +202 -0
- argleton/probes/clean/c001-raster-mean/build.py +36 -0
- argleton/probes/clean/c001-raster-mean/probe.toml +25 -0
- argleton/probes/clean/c002-projected-area/build.py +44 -0
- argleton/probes/clean/c002-projected-area/probe.toml +25 -0
- argleton/probes/clean/c003-raster-mean-nodata/build.py +35 -0
- argleton/probes/clean/c003-raster-mean-nodata/probe.toml +26 -0
- argleton/probes/clean/c004-points-in-polygon/build.py +60 -0
- argleton/probes/clean/c004-points-in-polygon/probe.toml +29 -0
- argleton/probes/clean/c005-polygon-area/build.py +38 -0
- argleton/probes/clean/c005-polygon-area/probe.toml +26 -0
- argleton/probes/clean/c006-named-layer/build.py +35 -0
- argleton/probes/clean/c006-named-layer/probe.toml +26 -0
- argleton/probes/clean/c007-distance-in-metres/build.py +42 -0
- argleton/probes/clean/c007-distance-in-metres/probe.toml +27 -0
- argleton/probes/clean/c008-equal-area-crs/build.py +39 -0
- argleton/probes/clean/c008-equal-area-crs/probe.toml +37 -0
- argleton/probes/clean/c009-native-resolution-classes/build.py +51 -0
- argleton/probes/clean/c009-native-resolution-classes/probe.toml +31 -0
- argleton/probes/clean/c010-physical-values/build.py +49 -0
- argleton/probes/clean/c010-physical-values/probe.toml +30 -0
- argleton/probes/clean/c011-solid-parcel/build.py +52 -0
- argleton/probes/clean/c011-solid-parcel/probe.toml +28 -0
- argleton/probes/clean/c012-disjoint-concessions/build.py +56 -0
- argleton/probes/clean/c012-disjoint-concessions/probe.toml +28 -0
- argleton/probes/clean/c013-flat-pipeline/build.py +42 -0
- argleton/probes/clean/c013-flat-pipeline/probe.toml +28 -0
- argleton/probes/clean/c014-convex-parcel/build.py +70 -0
- argleton/probes/clean/c014-convex-parcel/probe.toml +28 -0
- argleton/probes/clean/c015-wells-off-the-seam/build.py +73 -0
- argleton/probes/clean/c015-wells-off-the-seam/probe.toml +28 -0
- argleton/probes/clean/c016-decimal-degrees/build.py +29 -0
- argleton/probes/clean/c016-decimal-degrees/probe.toml +27 -0
- argleton/probes/clean/c017-equal-populations/build.py +60 -0
- argleton/probes/clean/c017-equal-populations/probe.toml +28 -0
- argleton/probes/clean/c018-plain-keys/build.py +59 -0
- argleton/probes/clean/c018-plain-keys/probe.toml +28 -0
- argleton/probes/clean/c019-fully-contained/build.py +64 -0
- argleton/probes/clean/c019-fully-contained/probe.toml +28 -0
- argleton/probes/clean/c020-one-owner-each/build.py +55 -0
- argleton/probes/clean/c020-one-owner-each/probe.toml +27 -0
- argleton/probes/clean/c021-greenwich-variant/build.py +41 -0
- argleton/probes/clean/c021-greenwich-variant/probe.toml +41 -0
- argleton/probes/schema/probe.schema.json +138 -0
- argleton/probes/schema/result.schema.json +103 -0
- argleton/probes/traps/001-tiff-predictor/README.md +75 -0
- argleton/probes/traps/001-tiff-predictor/build.py +58 -0
- argleton/probes/traps/001-tiff-predictor/probe.toml +57 -0
- argleton/probes/traps/002-feet-as-metres/README.md +79 -0
- argleton/probes/traps/002-feet-as-metres/build.py +44 -0
- argleton/probes/traps/002-feet-as-metres/probe.toml +70 -0
- argleton/probes/traps/003-nodata-in-statistics/README.md +61 -0
- argleton/probes/traps/003-nodata-in-statistics/build.py +51 -0
- argleton/probes/traps/003-nodata-in-statistics/probe.toml +62 -0
- argleton/probes/traps/004-mismatched-crs-join/README.md +65 -0
- argleton/probes/traps/004-mismatched-crs-join/build.py +65 -0
- argleton/probes/traps/004-mismatched-crs-join/probe.toml +75 -0
- argleton/probes/traps/005-bowtie-area/README.md +63 -0
- argleton/probes/traps/005-bowtie-area/build.py +41 -0
- argleton/probes/traps/005-bowtie-area/probe.toml +73 -0
- argleton/probes/traps/006-default-layer/README.md +65 -0
- argleton/probes/traps/006-default-layer/build.py +55 -0
- argleton/probes/traps/006-default-layer/probe.toml +63 -0
- argleton/probes/traps/007-buffer-in-degrees/README.md +57 -0
- argleton/probes/traps/007-buffer-in-degrees/build.py +42 -0
- argleton/probes/traps/007-buffer-in-degrees/probe.toml +64 -0
- argleton/probes/traps/008-web-mercator-area/README.md +55 -0
- argleton/probes/traps/008-web-mercator-area/build.py +39 -0
- argleton/probes/traps/008-web-mercator-area/probe.toml +68 -0
- argleton/probes/traps/009-resampled-classes/README.md +88 -0
- argleton/probes/traps/009-resampled-classes/build.py +46 -0
- argleton/probes/traps/009-resampled-classes/probe.toml +76 -0
- argleton/probes/traps/010-scale-offset/README.md +76 -0
- argleton/probes/traps/010-scale-offset/build.py +55 -0
- argleton/probes/traps/010-scale-offset/probe.toml +69 -0
- argleton/probes/traps/011-polygon-holes/README.md +47 -0
- argleton/probes/traps/011-polygon-holes/build.py +55 -0
- argleton/probes/traps/011-polygon-holes/probe.toml +59 -0
- argleton/probes/traps/012-double-counting/README.md +40 -0
- argleton/probes/traps/012-double-counting/build.py +58 -0
- argleton/probes/traps/012-double-counting/probe.toml +52 -0
- argleton/probes/traps/013-z-dimension/README.md +42 -0
- argleton/probes/traps/013-z-dimension/build.py +43 -0
- argleton/probes/traps/013-z-dimension/probe.toml +53 -0
- argleton/probes/traps/014-centroid-outside/README.md +42 -0
- argleton/probes/traps/014-centroid-outside/build.py +72 -0
- argleton/probes/traps/014-centroid-outside/probe.toml +59 -0
- argleton/probes/traps/015-boundary-semantics/README.md +41 -0
- argleton/probes/traps/015-boundary-semantics/build.py +74 -0
- argleton/probes/traps/015-boundary-semantics/probe.toml +57 -0
- argleton/probes/traps/016-coordinate-parsing/README.md +40 -0
- argleton/probes/traps/016-coordinate-parsing/build.py +36 -0
- argleton/probes/traps/016-coordinate-parsing/probe.toml +52 -0
- argleton/probes/traps/017-aggregation-weighting/README.md +42 -0
- argleton/probes/traps/017-aggregation-weighting/build.py +62 -0
- argleton/probes/traps/017-aggregation-weighting/probe.toml +53 -0
- argleton/probes/traps/018-join-key-typing/README.md +41 -0
- argleton/probes/traps/018-join-key-typing/build.py +61 -0
- argleton/probes/traps/018-join-key-typing/probe.toml +53 -0
- argleton/probes/traps/019-partial-overlap/README.md +43 -0
- argleton/probes/traps/019-partial-overlap/build.py +66 -0
- argleton/probes/traps/019-partial-overlap/probe.toml +53 -0
- argleton/probes/traps/020-join-cardinality/README.md +42 -0
- argleton/probes/traps/020-join-cardinality/build.py +60 -0
- argleton/probes/traps/020-join-cardinality/probe.toml +51 -0
- argleton/probes/traps/021-ballpark-datum/README.md +100 -0
- argleton/probes/traps/021-ballpark-datum/build.py +44 -0
- argleton/probes/traps/021-ballpark-datum/probe.toml +92 -0
- argleton/published.py +49 -0
- argleton/run.py +170 -0
- argleton/score.py +150 -0
- argleton-0.1.0.dist-info/METADATA +275 -0
- argleton-0.1.0.dist-info/RECORD +124 -0
- argleton-0.1.0.dist-info/WHEEL +4 -0
- argleton-0.1.0.dist-info/entry_points.txt +2 -0
- argleton-0.1.0.dist-info/licenses/LICENSE +202 -0
argleton/model.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""The probe contract, loaded and validated.
|
|
2
|
+
|
|
3
|
+
Deliberately stdlib-only. Anyone must be able to read a probe, understand what
|
|
4
|
+
it claims, and check that claim without installing anything — a suite that needs
|
|
5
|
+
a toolchain to be inspected is asking to be trusted rather than checked.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import tomllib
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Literal
|
|
15
|
+
|
|
16
|
+
CONTRACT = "probe.toml"
|
|
17
|
+
POPULATIONS = ("trap", "clean")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class Truth:
|
|
22
|
+
kind: str
|
|
23
|
+
value: Any
|
|
24
|
+
tolerance: float
|
|
25
|
+
derivation: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class NaiveFailure:
|
|
30
|
+
description: str
|
|
31
|
+
observed_value: Any
|
|
32
|
+
why_plausible: str
|
|
33
|
+
derivation: str | None = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Probe:
|
|
38
|
+
"""One task whose correct answer is known by construction."""
|
|
39
|
+
|
|
40
|
+
id: str
|
|
41
|
+
population: Literal["trap", "clean"]
|
|
42
|
+
family: str
|
|
43
|
+
title: str
|
|
44
|
+
surface: tuple[str, ...]
|
|
45
|
+
call: str
|
|
46
|
+
truth: Truth
|
|
47
|
+
provenance: dict[str, Any]
|
|
48
|
+
directory: Path
|
|
49
|
+
prompt: str | None = None
|
|
50
|
+
naive_failure: NaiveFailure | None = None
|
|
51
|
+
accept_if_mentions: tuple[str, ...] = ()
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def operation(self) -> str:
|
|
55
|
+
"""The operation name in `task.call`, e.g. `raster_mean` in `raster_mean(dem.tif)`."""
|
|
56
|
+
return _parse_call(self.call)[0]
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def arguments(self) -> tuple[str, ...]:
|
|
60
|
+
return _parse_call(self.call)[1]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _parse_call(call: str) -> tuple[str, tuple[str, ...]]:
|
|
64
|
+
"""`raster_mean(dem.tif, band=1)` → ("raster_mean", ("dem.tif", "band=1")).
|
|
65
|
+
|
|
66
|
+
A grammar rather than `eval`: probe files come from contributors, and a
|
|
67
|
+
suite that runs arbitrary code from its own fixtures cannot be run by the
|
|
68
|
+
people it is meant to convince.
|
|
69
|
+
"""
|
|
70
|
+
match = re.fullmatch(r"\s*([a-z][a-z0-9_]*)\s*\((.*)\)\s*", call, re.DOTALL)
|
|
71
|
+
if not match:
|
|
72
|
+
raise ValueError(f"task.call is not `name(args)`: {call!r}")
|
|
73
|
+
name, inside = match.group(1), match.group(2).strip()
|
|
74
|
+
args = tuple(a.strip() for a in inside.split(",")) if inside else ()
|
|
75
|
+
return name, args
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class ContractError(ValueError):
|
|
79
|
+
"""A probe file that does not honour the contract. Names the file and the field."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _require(data: dict, key: str, where: Path) -> Any:
|
|
83
|
+
if key not in data:
|
|
84
|
+
raise ContractError(f"{where}: missing required key `{key}`")
|
|
85
|
+
return data[key]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def load_probe(directory: Path) -> Probe:
|
|
89
|
+
"""Read and check one probe directory.
|
|
90
|
+
|
|
91
|
+
The checks here are the ones that make a published number mean something;
|
|
92
|
+
the JSON Schema in `schema/` is the same contract for people who are not
|
|
93
|
+
running this code. When they disagree, the schema is the specification.
|
|
94
|
+
"""
|
|
95
|
+
path = directory / CONTRACT
|
|
96
|
+
try:
|
|
97
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
98
|
+
except FileNotFoundError:
|
|
99
|
+
raise ContractError(f"{directory}: no {CONTRACT}") from None
|
|
100
|
+
except tomllib.TOMLDecodeError as exc:
|
|
101
|
+
raise ContractError(f"{path}: not valid TOML — {exc}") from None
|
|
102
|
+
|
|
103
|
+
population = _require(data, "population", path)
|
|
104
|
+
if population not in POPULATIONS:
|
|
105
|
+
raise ContractError(f"{path}: population must be one of {POPULATIONS}")
|
|
106
|
+
|
|
107
|
+
task = _require(data, "task", path)
|
|
108
|
+
truth_data = _require(data, "truth", path)
|
|
109
|
+
for key in ("kind", "value", "tolerance", "derivation"):
|
|
110
|
+
_require(truth_data, f"truth.{key}" if key not in truth_data else key, path)
|
|
111
|
+
|
|
112
|
+
naive, accepts = None, ()
|
|
113
|
+
if population == "trap":
|
|
114
|
+
raw = _require(data, "naive_failure", path)
|
|
115
|
+
if not raw.get("plausible"):
|
|
116
|
+
# The admission criterion, and the only one that is not taste. An
|
|
117
|
+
# error that crashes or returns an absurd number is already caught
|
|
118
|
+
# by something; this suite is for the answers nothing catches.
|
|
119
|
+
raise ContractError(
|
|
120
|
+
f"{path}: a trap must declare `plausible = true`. If the typical "
|
|
121
|
+
"error is loud, the probe belongs in an ordinary test suite."
|
|
122
|
+
)
|
|
123
|
+
naive = NaiveFailure(
|
|
124
|
+
description=raw["description"],
|
|
125
|
+
observed_value=raw["observed_value"],
|
|
126
|
+
why_plausible=raw["why_plausible"],
|
|
127
|
+
derivation=raw.get("derivation"),
|
|
128
|
+
)
|
|
129
|
+
accepts = tuple(_require(data, "refusal", path)["accept_if_mentions"])
|
|
130
|
+
elif "naive_failure" in data:
|
|
131
|
+
raise ContractError(
|
|
132
|
+
f"{path}: a clean probe cannot have a naive_failure. It is a trap "
|
|
133
|
+
"filed in the wrong place, and it would corrupt the completion rate."
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
surfaces = tuple(_require(data, "surface", path))
|
|
137
|
+
if "agent" in surfaces and not task.get("prompt"):
|
|
138
|
+
raise ContractError(f"{path}: surface includes `agent` but task.prompt is missing")
|
|
139
|
+
|
|
140
|
+
return Probe(
|
|
141
|
+
id=_require(data, "id", path),
|
|
142
|
+
population=population,
|
|
143
|
+
family=_require(data, "family", path),
|
|
144
|
+
title=_require(data, "title", path),
|
|
145
|
+
surface=surfaces,
|
|
146
|
+
call=_require(task, "call", path),
|
|
147
|
+
prompt=task.get("prompt"),
|
|
148
|
+
truth=Truth(truth_data["kind"], truth_data["value"], float(truth_data["tolerance"]),
|
|
149
|
+
truth_data["derivation"]),
|
|
150
|
+
naive_failure=naive,
|
|
151
|
+
accept_if_mentions=accepts,
|
|
152
|
+
provenance=_require(data, "provenance", path),
|
|
153
|
+
directory=directory,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def discover(root: Path) -> list[Probe]:
|
|
158
|
+
"""Every probe under `traps/` and `clean/`, in a stable order.
|
|
159
|
+
|
|
160
|
+
Stable because published results are compared across runs and across
|
|
161
|
+
systems: an ordering that depends on the filesystem would make two honest
|
|
162
|
+
runs look different.
|
|
163
|
+
"""
|
|
164
|
+
folders = {"trap": root / "traps", "clean": root / "clean"}
|
|
165
|
+
found = [
|
|
166
|
+
load_probe(contratto.parent)
|
|
167
|
+
for folder in folders.values()
|
|
168
|
+
for contratto in sorted(folder.rglob(CONTRACT))
|
|
169
|
+
]
|
|
170
|
+
duplicates = {p.id for p in found if [q.id for q in found].count(p.id) > 1}
|
|
171
|
+
if duplicates:
|
|
172
|
+
raise ContractError(f"duplicate probe ids: {sorted(duplicates)}")
|
|
173
|
+
return sorted(found, key=lambda p: (p.population, p.id))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@dataclass
|
|
177
|
+
class Outcome:
|
|
178
|
+
"""What a system did with one probe.
|
|
179
|
+
|
|
180
|
+
Four mutually exclusive shapes, and the runner refuses anything ambiguous:
|
|
181
|
+
an adapter that returns both an answer and a refusal has not decided what
|
|
182
|
+
the system did, and guessing on its behalf is how a score stops meaning
|
|
183
|
+
anything.
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
answer: Any = None
|
|
187
|
+
refusal: str | None = None
|
|
188
|
+
error: str | None = None
|
|
189
|
+
unsupported: bool = False
|
|
190
|
+
warnings: list[str] = field(default_factory=list)
|
|
191
|
+
transcript: str | None = None
|
|
192
|
+
|
|
193
|
+
def __post_init__(self) -> None:
|
|
194
|
+
declared = sum(
|
|
195
|
+
x is not None and x is not False
|
|
196
|
+
for x in (self.answer, self.refusal, self.error, self.unsupported or None)
|
|
197
|
+
)
|
|
198
|
+
if declared > 1:
|
|
199
|
+
raise ValueError(
|
|
200
|
+
"an Outcome is exactly one of answer / refusal / error / unsupported; "
|
|
201
|
+
f"got {declared}"
|
|
202
|
+
)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""The same elevations, stored without the predictor.
|
|
2
|
+
|
|
3
|
+
The control. If a system cannot answer this, it cannot answer trap 001 either,
|
|
4
|
+
and its silent-error rate on that trap says nothing about silent errors — only
|
|
5
|
+
that the task was out of reach. Every trap family should have a clean twin for
|
|
6
|
+
the same reason a medical test needs a negative control.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import rasterio
|
|
16
|
+
from rasterio.transform import from_origin
|
|
17
|
+
|
|
18
|
+
HEIGHT = WIDTH = 32
|
|
19
|
+
BASE, EAST_STEP, SOUTH_STEP = 1000, 4, 2
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main(destination: Path) -> int:
|
|
23
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
row, column = np.mgrid[0:HEIGHT, 0:WIDTH]
|
|
25
|
+
grid = (BASE + EAST_STEP * column + SOUTH_STEP * row).astype("int16")
|
|
26
|
+
with rasterio.open(
|
|
27
|
+
destination / "dem.tif", "w", driver="GTiff", height=HEIGHT, width=WIDTH,
|
|
28
|
+
count=1, dtype="int16", crs="EPSG:32632", nodata=-32768,
|
|
29
|
+
transform=from_origin(500000.0, 5000000.0, 30.0, 30.0),
|
|
30
|
+
) as ds:
|
|
31
|
+
ds.write(grid, 1)
|
|
32
|
+
return 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
id = "c001-raster-mean"
|
|
2
|
+
population = "clean"
|
|
3
|
+
family = "raster-encoding"
|
|
4
|
+
title = "Mean of an uncompressed integer DEM"
|
|
5
|
+
surface = ["engine", "agent"]
|
|
6
|
+
|
|
7
|
+
[task]
|
|
8
|
+
call = "raster_mean(dem.tif)"
|
|
9
|
+
prompt = "What is the mean elevation of dem.tif, in metres?"
|
|
10
|
+
|
|
11
|
+
[truth]
|
|
12
|
+
kind = "scalar"
|
|
13
|
+
value = 1093.0
|
|
14
|
+
tolerance = 0.001
|
|
15
|
+
derivation = """
|
|
16
|
+
Identical elevations to trap 001, stored plainly: v[i,j] = 1000 + 4j + 2i over
|
|
17
|
+
32x32, mean = 1000 + 4*15.5 + 2*15.5 = 1093 exactly. Sharing the arithmetic with
|
|
18
|
+
the trap is the point — the only difference between the two probes is how the
|
|
19
|
+
bytes are stored, so a system that answers one and not the other has told us
|
|
20
|
+
about the encoding and nothing else."""
|
|
21
|
+
|
|
22
|
+
[provenance]
|
|
23
|
+
source = "control for 001-tiff-predictor"
|
|
24
|
+
found_by = "mapsmith"
|
|
25
|
+
date = "2026-08-23"
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""The same square, in a CRS whose unit is the metre.
|
|
2
|
+
|
|
3
|
+
The control for `002-feet-as-metres`: identical geometry and identical task, and
|
|
4
|
+
the only difference is a linear unit the file states plainly. A system that
|
|
5
|
+
answers this and misses the trap has told us it does not read the unit. One that
|
|
6
|
+
misses both has told us nothing about units at all.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
# GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
|
|
16
|
+
# builds of the same data differ byte for byte. Every published number is
|
|
17
|
+
# anchored to a `spec_commit`, which means nothing if that commit can produce
|
|
18
|
+
# different fixtures — so the clock is pinned. GDAL reads this at write time,
|
|
19
|
+
# hence before the import that uses it.
|
|
20
|
+
os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-23T00:00:00.000Z")
|
|
21
|
+
|
|
22
|
+
import geopandas as gpd
|
|
23
|
+
from shapely.geometry import Polygon
|
|
24
|
+
|
|
25
|
+
SIDE_METRES = 1000.0
|
|
26
|
+
ORIGIN_X, ORIGIN_Y = 500000.0, 5000000.0 # UTM 32N
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(destination: Path) -> int:
|
|
30
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
square = Polygon([
|
|
32
|
+
(ORIGIN_X, ORIGIN_Y),
|
|
33
|
+
(ORIGIN_X + SIDE_METRES, ORIGIN_Y),
|
|
34
|
+
(ORIGIN_X + SIDE_METRES, ORIGIN_Y + SIDE_METRES),
|
|
35
|
+
(ORIGIN_X, ORIGIN_Y + SIDE_METRES),
|
|
36
|
+
])
|
|
37
|
+
gpd.GeoDataFrame(
|
|
38
|
+
{"parcel_id": ["A-1"]}, geometry=[square], crs="EPSG:32632"
|
|
39
|
+
).to_file(destination / "parcel.gpkg", layer="parcel", driver="GPKG")
|
|
40
|
+
return 0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
id = "c002-projected-area"
|
|
2
|
+
population = "clean"
|
|
3
|
+
family = "linear-units"
|
|
4
|
+
title = "Area of a square in a metric projected CRS"
|
|
5
|
+
surface = ["engine", "agent"]
|
|
6
|
+
|
|
7
|
+
[task]
|
|
8
|
+
call = "planar_area_m2(parcel.gpkg)"
|
|
9
|
+
prompt = """
|
|
10
|
+
What is the area of the parcel in parcel.gpkg, in square metres, measured in the
|
|
11
|
+
plane of the layer's own coordinate reference system?"""
|
|
12
|
+
|
|
13
|
+
[truth]
|
|
14
|
+
kind = "scalar"
|
|
15
|
+
value = 1000000.0
|
|
16
|
+
tolerance = 0.01
|
|
17
|
+
derivation = """
|
|
18
|
+
A square of side 1000 in EPSG:32632, whose linear unit is the metre: the area is
|
|
19
|
+
1000 * 1000 = 1000000 square metres. Nothing to convert, which is the point -
|
|
20
|
+
the trap and its control differ only in the unit the file declares."""
|
|
21
|
+
|
|
22
|
+
[provenance]
|
|
23
|
+
source = "control for 002-feet-as-metres"
|
|
24
|
+
found_by = "mapsmith"
|
|
25
|
+
date = "2026-08-23"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""The same raster with the voids filled, so nothing has to be excluded.
|
|
2
|
+
|
|
3
|
+
The control for `003-nodata-in-statistics`. Identical grid, identical task, and
|
|
4
|
+
no nodata cells at all: a system that answers this and misses the trap has told
|
|
5
|
+
us it does not honour a declared nodata value. One that misses both has told us
|
|
6
|
+
it cannot read the raster, which is a different finding.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import rasterio
|
|
16
|
+
from rasterio.transform import from_origin
|
|
17
|
+
|
|
18
|
+
SIDE = 100
|
|
19
|
+
ELEVATION = 1000
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main(destination: Path) -> int:
|
|
23
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
with rasterio.open(
|
|
25
|
+
destination / "dem.tif", "w", driver="GTiff",
|
|
26
|
+
height=SIDE, width=SIDE, count=1, dtype="int16",
|
|
27
|
+
crs="EPSG:32632", nodata=-9999,
|
|
28
|
+
transform=from_origin(500000.0, 5000000.0, 30.0, 30.0),
|
|
29
|
+
) as ds:
|
|
30
|
+
ds.write(np.full((SIDE, SIDE), ELEVATION, dtype="int16"), 1)
|
|
31
|
+
return 0
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
if __name__ == "__main__":
|
|
35
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
id = "c003-raster-mean-nodata"
|
|
2
|
+
population = "clean"
|
|
3
|
+
family = "nodata"
|
|
4
|
+
title = "Mean of a raster that declares nodata and contains none"
|
|
5
|
+
surface = ["engine", "agent"]
|
|
6
|
+
|
|
7
|
+
[task]
|
|
8
|
+
call = "raster_mean(dem.tif)"
|
|
9
|
+
prompt = "What is the mean elevation of dem.tif, in metres?"
|
|
10
|
+
|
|
11
|
+
[truth]
|
|
12
|
+
kind = "scalar"
|
|
13
|
+
value = 1000.0
|
|
14
|
+
tolerance = 0.001
|
|
15
|
+
derivation = """
|
|
16
|
+
Every cell of the 100x100 grid holds 1000, so the mean is 1000. The header
|
|
17
|
+
still declares nodata = -9999, exactly as the trap's does - the only difference
|
|
18
|
+
between the two probes is whether any cell holds that value. A system that
|
|
19
|
+
answers this one and not the trap has told us it does not honour the
|
|
20
|
+
declaration; one that misses both has told us it cannot read the file, which
|
|
21
|
+
is a different finding and should not be counted as the first."""
|
|
22
|
+
|
|
23
|
+
[provenance]
|
|
24
|
+
source = "control for 003-nodata-in-statistics"
|
|
25
|
+
found_by = "mapsmith"
|
|
26
|
+
date = "2026-08-23"
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Build the control: the same counting task, both layers in the same CRS.
|
|
2
|
+
|
|
3
|
+
Everything is defined directly in EPSG:32633 metres, so containment is an
|
|
4
|
+
interval comparison in the shared plane and nothing is ever transformed. The
|
|
5
|
+
trap and this control differ in exactly one thing: whether the two frames
|
|
6
|
+
already agree.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
# GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
|
|
16
|
+
# builds of the same data differ byte for byte unless the clock is pinned.
|
|
17
|
+
os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-24T00:00:00.000Z")
|
|
18
|
+
|
|
19
|
+
import geopandas as gpd
|
|
20
|
+
from shapely.geometry import Point, Polygon
|
|
21
|
+
|
|
22
|
+
CRS = "EPSG:32633"
|
|
23
|
+
ZONE_E = (391_000.0, 401_000.0)
|
|
24
|
+
ZONE_N = (4_629_000.0, 4_641_000.0)
|
|
25
|
+
# Inside: a 3x3 grid with at least 2 km of margin to every edge.
|
|
26
|
+
INSIDE_E = (393_000.0, 396_000.0, 399_000.0)
|
|
27
|
+
INSIDE_N = (4_632_000.0, 4_635_000.0, 4_638_000.0)
|
|
28
|
+
# Outside: everything at easting 430 000 or greater — at least 29 km east.
|
|
29
|
+
OUTSIDE_E = (430_000.0, 460_000.0, 490_000.0)
|
|
30
|
+
OUTSIDE_N = (
|
|
31
|
+
4_600_000.0, 4_610_000.0, 4_620_000.0, 4_650_000.0, 4_660_000.0,
|
|
32
|
+
4_670_000.0, 4_680_000.0, 4_690_000.0, 4_700_000.0,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def main(destination: Path) -> int:
|
|
37
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
|
|
39
|
+
rectangle = Polygon([
|
|
40
|
+
(ZONE_E[0], ZONE_N[0]),
|
|
41
|
+
(ZONE_E[1], ZONE_N[0]),
|
|
42
|
+
(ZONE_E[1], ZONE_N[1]),
|
|
43
|
+
(ZONE_E[0], ZONE_N[1]),
|
|
44
|
+
])
|
|
45
|
+
gpd.GeoDataFrame(
|
|
46
|
+
{"zone_id": ["Z-1"]}, geometry=[rectangle], crs=CRS
|
|
47
|
+
).to_file(destination / "zone.gpkg", layer="zone", driver="GPKG")
|
|
48
|
+
|
|
49
|
+
coords = [(e, n) for e in INSIDE_E for n in INSIDE_N]
|
|
50
|
+
coords += [(e, n) for e in OUTSIDE_E for n in OUTSIDE_N]
|
|
51
|
+
gpd.GeoDataFrame(
|
|
52
|
+
{"well_id": [f"W-{i + 1}" for i in range(len(coords))]},
|
|
53
|
+
geometry=[Point(e, n) for e, n in coords],
|
|
54
|
+
crs=CRS,
|
|
55
|
+
).to_file(destination / "points.gpkg", layer="points", driver="GPKG")
|
|
56
|
+
return 0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
if __name__ == "__main__":
|
|
60
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
id = "c004-points-in-polygon"
|
|
2
|
+
population = "clean"
|
|
3
|
+
family = "mismatched-crs"
|
|
4
|
+
title = "Count of points inside a polygon, both layers in the same CRS"
|
|
5
|
+
surface = ["engine", "agent"]
|
|
6
|
+
|
|
7
|
+
[task]
|
|
8
|
+
call = "points_in_polygon_count(points.gpkg, zone.gpkg)"
|
|
9
|
+
prompt = """
|
|
10
|
+
How many of the points in points.gpkg fall inside the polygon in zone.gpkg?
|
|
11
|
+
Answer with the count."""
|
|
12
|
+
|
|
13
|
+
[truth]
|
|
14
|
+
kind = "count"
|
|
15
|
+
value = 9
|
|
16
|
+
tolerance = 0
|
|
17
|
+
derivation = """
|
|
18
|
+
Both layers are in EPSG:32633, so containment is an interval comparison in
|
|
19
|
+
their shared plane. The zone is the rectangle [391000, 401000] x
|
|
20
|
+
[4629000, 4641000]. Nine points lie on the grid {393000, 396000, 399000} x
|
|
21
|
+
{4632000, 4635000, 4638000}, inside with at least 2 km of margin to every
|
|
22
|
+
edge; the other 27 lie at eastings 430000 or greater, at least 29 km east of
|
|
23
|
+
the rectangle. Nothing is ever transformed, which is the point - the trap and
|
|
24
|
+
this control differ only in whether the two frames already agree."""
|
|
25
|
+
|
|
26
|
+
[provenance]
|
|
27
|
+
source = "control for 004-mismatched-crs-join"
|
|
28
|
+
found_by = "mapsmith"
|
|
29
|
+
date = "2026-08-24"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Build the control: an ordinary valid L-shaped parcel in a metric CRS.
|
|
2
|
+
|
|
3
|
+
Same operation and same phrasing as the trap; the only difference is that the
|
|
4
|
+
ring does not cross itself. The L-shape (rather than another rectangle) keeps
|
|
5
|
+
the control from being answerable by pattern-matching the fixture family.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
# GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
|
|
15
|
+
# builds of the same data differ byte for byte unless the clock is pinned.
|
|
16
|
+
os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-24T00:00:00.000Z")
|
|
17
|
+
|
|
18
|
+
import geopandas as gpd
|
|
19
|
+
from shapely.geometry import Polygon
|
|
20
|
+
|
|
21
|
+
CRS = "EPSG:32632"
|
|
22
|
+
E0, N0 = 402_000.0, 5_003_000.0
|
|
23
|
+
|
|
24
|
+
# Local shape: an L. Area = 100*40 + 60*60 = 7600 m2, on paper.
|
|
25
|
+
RING = [(0.0, 0.0), (100.0, 0.0), (100.0, 40.0), (60.0, 40.0), (60.0, 100.0), (0.0, 100.0)]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def main(destination: Path) -> int:
|
|
29
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
ell = Polygon([(E0 + x, N0 + y) for x, y in RING])
|
|
31
|
+
gpd.GeoDataFrame(
|
|
32
|
+
{"parcel_id": ["A-1"]}, geometry=[ell], crs=CRS
|
|
33
|
+
).to_file(destination / "parcel.gpkg", layer="parcel", driver="GPKG")
|
|
34
|
+
return 0
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
id = "c005-polygon-area"
|
|
2
|
+
population = "clean"
|
|
3
|
+
family = "invalid-geometry"
|
|
4
|
+
title = "Area of a valid L-shaped parcel in a metric CRS"
|
|
5
|
+
surface = ["engine", "agent"]
|
|
6
|
+
|
|
7
|
+
[task]
|
|
8
|
+
call = "planar_area_m2(parcel.gpkg)"
|
|
9
|
+
prompt = """
|
|
10
|
+
What is the area of the parcel in parcel.gpkg, in square metres, measured in the
|
|
11
|
+
plane of the layer's own coordinate reference system?"""
|
|
12
|
+
|
|
13
|
+
[truth]
|
|
14
|
+
kind = "scalar"
|
|
15
|
+
value = 7600.0
|
|
16
|
+
tolerance = 0.01
|
|
17
|
+
derivation = """
|
|
18
|
+
An L-shaped polygon in EPSG:32632 (metres), decomposable on paper into a
|
|
19
|
+
100 x 40 rectangle plus a 60 x 60 square: 4000 + 3600 = 7600 square metres.
|
|
20
|
+
The ring is valid - the trap and this control differ only in whether the ring
|
|
21
|
+
crosses itself."""
|
|
22
|
+
|
|
23
|
+
[provenance]
|
|
24
|
+
source = "control for 005-bowtie-area"
|
|
25
|
+
found_by = "mapsmith"
|
|
26
|
+
date = "2026-08-24"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Build the control: a single-layer GeoPackage, same question, no ambiguity.
|
|
2
|
+
|
|
3
|
+
One layer, honestly named `wells`, 17 points. Whether a system passes a layer
|
|
4
|
+
name or not, there is only one layer to get.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
# GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
|
|
14
|
+
# builds of the same data differ byte for byte unless the clock is pinned.
|
|
15
|
+
os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-25T00:00:00.000Z")
|
|
16
|
+
|
|
17
|
+
import geopandas as gpd
|
|
18
|
+
from shapely.geometry import Point
|
|
19
|
+
|
|
20
|
+
CRS = "EPSG:32632"
|
|
21
|
+
N_WELLS = 17
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(destination: Path) -> int:
|
|
25
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
gpd.GeoDataFrame(
|
|
27
|
+
{"well_id": [f"W-{i}" for i in range(1, N_WELLS + 1)]},
|
|
28
|
+
geometry=[Point(10.0 + i * 9.0, 50.0 + (i % 4) * 13.0) for i in range(N_WELLS)],
|
|
29
|
+
crs=CRS,
|
|
30
|
+
).to_file(destination / "wells.gpkg", layer="wells", driver="GPKG")
|
|
31
|
+
return 0
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
if __name__ == "__main__":
|
|
35
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
id = "c006-named-layer"
|
|
2
|
+
population = "clean"
|
|
3
|
+
family = "ambiguous-layer"
|
|
4
|
+
title = "Feature count of a single-layer container"
|
|
5
|
+
surface = ["engine", "agent"]
|
|
6
|
+
|
|
7
|
+
[task]
|
|
8
|
+
call = "feature_count(wells.gpkg, layer=wells)"
|
|
9
|
+
prompt = """
|
|
10
|
+
How many features are in the `wells` layer of wells.gpkg? Answer with the
|
|
11
|
+
count."""
|
|
12
|
+
|
|
13
|
+
[truth]
|
|
14
|
+
kind = "count"
|
|
15
|
+
value = 17
|
|
16
|
+
tolerance = 0
|
|
17
|
+
derivation = """
|
|
18
|
+
By construction: wells.gpkg holds exactly one layer, `wells`, with 17 point
|
|
19
|
+
features (W-1 through W-17). With a single layer there is nothing to choose
|
|
20
|
+
wrongly — the trap and this control differ only in whether the container
|
|
21
|
+
holds a second layer the question did not ask about."""
|
|
22
|
+
|
|
23
|
+
[provenance]
|
|
24
|
+
source = "control for 006-default-layer"
|
|
25
|
+
found_by = "mapsmith"
|
|
26
|
+
date = "2026-08-25"
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Build the control: the same counting question on a layer already in metres.
|
|
2
|
+
|
|
3
|
+
Five wells within 460 m of W-1, everything else 2.5 km away or more, all in
|
|
4
|
+
EPSG:32633 — the distance 500 and the layer's unit finally agree.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
# GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
|
|
14
|
+
# builds of the same data differ byte for byte unless the clock is pinned.
|
|
15
|
+
os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-25T00:00:00.000Z")
|
|
16
|
+
|
|
17
|
+
import geopandas as gpd
|
|
18
|
+
from shapely.geometry import Point
|
|
19
|
+
|
|
20
|
+
CRS = "EPSG:32633"
|
|
21
|
+
E0, N0 = 500_000.0, 4_640_000.0 # W-1
|
|
22
|
+
|
|
23
|
+
INSIDE = [
|
|
24
|
+
(E0 + 300.0, N0), (E0, N0 + 400.0), (E0 - 250.0, N0 - 250.0),
|
|
25
|
+
(E0 + 120.0, N0 - 430.0), (E0 - 460.0, N0),
|
|
26
|
+
]
|
|
27
|
+
OUTSIDE = [(E0 + 2000.0 + 150.0 * i, N0 + 1500.0 + 100.0 * (i % 5)) for i in range(24)]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main(destination: Path) -> int:
|
|
31
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
points = [Point(E0, N0)] + [Point(*c) for c in INSIDE] + [Point(*c) for c in OUTSIDE]
|
|
33
|
+
gpd.GeoDataFrame(
|
|
34
|
+
{"well_id": [f"W-{i + 1}" for i in range(len(points))]},
|
|
35
|
+
geometry=points,
|
|
36
|
+
crs=CRS,
|
|
37
|
+
).to_file(destination / "wells.gpkg", layer="wells", driver="GPKG")
|
|
38
|
+
return 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
|