argleton 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. adapters/__init__.py +0 -0
  2. adapters/engine_geopandas.py +176 -0
  3. adapters/engine_naive.py +220 -0
  4. adapters/engine_rasterio.py +78 -0
  5. adapters/engine_whitebox.py +34 -0
  6. adapters/gis_mcp.py +192 -0
  7. adapters/mapsmith.py +496 -0
  8. argleton/__init__.py +3 -0
  9. argleton/model.py +202 -0
  10. argleton/probes/clean/c001-raster-mean/build.py +36 -0
  11. argleton/probes/clean/c001-raster-mean/probe.toml +25 -0
  12. argleton/probes/clean/c002-projected-area/build.py +44 -0
  13. argleton/probes/clean/c002-projected-area/probe.toml +25 -0
  14. argleton/probes/clean/c003-raster-mean-nodata/build.py +35 -0
  15. argleton/probes/clean/c003-raster-mean-nodata/probe.toml +26 -0
  16. argleton/probes/clean/c004-points-in-polygon/build.py +60 -0
  17. argleton/probes/clean/c004-points-in-polygon/probe.toml +29 -0
  18. argleton/probes/clean/c005-polygon-area/build.py +38 -0
  19. argleton/probes/clean/c005-polygon-area/probe.toml +26 -0
  20. argleton/probes/clean/c006-named-layer/build.py +35 -0
  21. argleton/probes/clean/c006-named-layer/probe.toml +26 -0
  22. argleton/probes/clean/c007-distance-in-metres/build.py +42 -0
  23. argleton/probes/clean/c007-distance-in-metres/probe.toml +27 -0
  24. argleton/probes/clean/c008-equal-area-crs/build.py +39 -0
  25. argleton/probes/clean/c008-equal-area-crs/probe.toml +37 -0
  26. argleton/probes/clean/c009-native-resolution-classes/build.py +51 -0
  27. argleton/probes/clean/c009-native-resolution-classes/probe.toml +31 -0
  28. argleton/probes/clean/c010-physical-values/build.py +49 -0
  29. argleton/probes/clean/c010-physical-values/probe.toml +30 -0
  30. argleton/probes/clean/c011-solid-parcel/build.py +52 -0
  31. argleton/probes/clean/c011-solid-parcel/probe.toml +28 -0
  32. argleton/probes/clean/c012-disjoint-concessions/build.py +56 -0
  33. argleton/probes/clean/c012-disjoint-concessions/probe.toml +28 -0
  34. argleton/probes/clean/c013-flat-pipeline/build.py +42 -0
  35. argleton/probes/clean/c013-flat-pipeline/probe.toml +28 -0
  36. argleton/probes/clean/c014-convex-parcel/build.py +70 -0
  37. argleton/probes/clean/c014-convex-parcel/probe.toml +28 -0
  38. argleton/probes/clean/c015-wells-off-the-seam/build.py +73 -0
  39. argleton/probes/clean/c015-wells-off-the-seam/probe.toml +28 -0
  40. argleton/probes/clean/c016-decimal-degrees/build.py +29 -0
  41. argleton/probes/clean/c016-decimal-degrees/probe.toml +27 -0
  42. argleton/probes/clean/c017-equal-populations/build.py +60 -0
  43. argleton/probes/clean/c017-equal-populations/probe.toml +28 -0
  44. argleton/probes/clean/c018-plain-keys/build.py +59 -0
  45. argleton/probes/clean/c018-plain-keys/probe.toml +28 -0
  46. argleton/probes/clean/c019-fully-contained/build.py +64 -0
  47. argleton/probes/clean/c019-fully-contained/probe.toml +28 -0
  48. argleton/probes/clean/c020-one-owner-each/build.py +55 -0
  49. argleton/probes/clean/c020-one-owner-each/probe.toml +27 -0
  50. argleton/probes/clean/c021-greenwich-variant/build.py +41 -0
  51. argleton/probes/clean/c021-greenwich-variant/probe.toml +41 -0
  52. argleton/probes/schema/probe.schema.json +138 -0
  53. argleton/probes/schema/result.schema.json +103 -0
  54. argleton/probes/traps/001-tiff-predictor/README.md +75 -0
  55. argleton/probes/traps/001-tiff-predictor/build.py +58 -0
  56. argleton/probes/traps/001-tiff-predictor/probe.toml +57 -0
  57. argleton/probes/traps/002-feet-as-metres/README.md +79 -0
  58. argleton/probes/traps/002-feet-as-metres/build.py +44 -0
  59. argleton/probes/traps/002-feet-as-metres/probe.toml +70 -0
  60. argleton/probes/traps/003-nodata-in-statistics/README.md +61 -0
  61. argleton/probes/traps/003-nodata-in-statistics/build.py +51 -0
  62. argleton/probes/traps/003-nodata-in-statistics/probe.toml +62 -0
  63. argleton/probes/traps/004-mismatched-crs-join/README.md +65 -0
  64. argleton/probes/traps/004-mismatched-crs-join/build.py +65 -0
  65. argleton/probes/traps/004-mismatched-crs-join/probe.toml +75 -0
  66. argleton/probes/traps/005-bowtie-area/README.md +63 -0
  67. argleton/probes/traps/005-bowtie-area/build.py +41 -0
  68. argleton/probes/traps/005-bowtie-area/probe.toml +73 -0
  69. argleton/probes/traps/006-default-layer/README.md +65 -0
  70. argleton/probes/traps/006-default-layer/build.py +55 -0
  71. argleton/probes/traps/006-default-layer/probe.toml +63 -0
  72. argleton/probes/traps/007-buffer-in-degrees/README.md +57 -0
  73. argleton/probes/traps/007-buffer-in-degrees/build.py +42 -0
  74. argleton/probes/traps/007-buffer-in-degrees/probe.toml +64 -0
  75. argleton/probes/traps/008-web-mercator-area/README.md +55 -0
  76. argleton/probes/traps/008-web-mercator-area/build.py +39 -0
  77. argleton/probes/traps/008-web-mercator-area/probe.toml +68 -0
  78. argleton/probes/traps/009-resampled-classes/README.md +88 -0
  79. argleton/probes/traps/009-resampled-classes/build.py +46 -0
  80. argleton/probes/traps/009-resampled-classes/probe.toml +76 -0
  81. argleton/probes/traps/010-scale-offset/README.md +76 -0
  82. argleton/probes/traps/010-scale-offset/build.py +55 -0
  83. argleton/probes/traps/010-scale-offset/probe.toml +69 -0
  84. argleton/probes/traps/011-polygon-holes/README.md +47 -0
  85. argleton/probes/traps/011-polygon-holes/build.py +55 -0
  86. argleton/probes/traps/011-polygon-holes/probe.toml +59 -0
  87. argleton/probes/traps/012-double-counting/README.md +40 -0
  88. argleton/probes/traps/012-double-counting/build.py +58 -0
  89. argleton/probes/traps/012-double-counting/probe.toml +52 -0
  90. argleton/probes/traps/013-z-dimension/README.md +42 -0
  91. argleton/probes/traps/013-z-dimension/build.py +43 -0
  92. argleton/probes/traps/013-z-dimension/probe.toml +53 -0
  93. argleton/probes/traps/014-centroid-outside/README.md +42 -0
  94. argleton/probes/traps/014-centroid-outside/build.py +72 -0
  95. argleton/probes/traps/014-centroid-outside/probe.toml +59 -0
  96. argleton/probes/traps/015-boundary-semantics/README.md +41 -0
  97. argleton/probes/traps/015-boundary-semantics/build.py +74 -0
  98. argleton/probes/traps/015-boundary-semantics/probe.toml +57 -0
  99. argleton/probes/traps/016-coordinate-parsing/README.md +40 -0
  100. argleton/probes/traps/016-coordinate-parsing/build.py +36 -0
  101. argleton/probes/traps/016-coordinate-parsing/probe.toml +52 -0
  102. argleton/probes/traps/017-aggregation-weighting/README.md +42 -0
  103. argleton/probes/traps/017-aggregation-weighting/build.py +62 -0
  104. argleton/probes/traps/017-aggregation-weighting/probe.toml +53 -0
  105. argleton/probes/traps/018-join-key-typing/README.md +41 -0
  106. argleton/probes/traps/018-join-key-typing/build.py +61 -0
  107. argleton/probes/traps/018-join-key-typing/probe.toml +53 -0
  108. argleton/probes/traps/019-partial-overlap/README.md +43 -0
  109. argleton/probes/traps/019-partial-overlap/build.py +66 -0
  110. argleton/probes/traps/019-partial-overlap/probe.toml +53 -0
  111. argleton/probes/traps/020-join-cardinality/README.md +42 -0
  112. argleton/probes/traps/020-join-cardinality/build.py +60 -0
  113. argleton/probes/traps/020-join-cardinality/probe.toml +51 -0
  114. argleton/probes/traps/021-ballpark-datum/README.md +100 -0
  115. argleton/probes/traps/021-ballpark-datum/build.py +44 -0
  116. argleton/probes/traps/021-ballpark-datum/probe.toml +92 -0
  117. argleton/published.py +49 -0
  118. argleton/run.py +170 -0
  119. argleton/score.py +150 -0
  120. argleton-0.1.0.dist-info/METADATA +275 -0
  121. argleton-0.1.0.dist-info/RECORD +124 -0
  122. argleton-0.1.0.dist-info/WHEEL +4 -0
  123. argleton-0.1.0.dist-info/entry_points.txt +2 -0
  124. argleton-0.1.0.dist-info/licenses/LICENSE +202 -0
argleton/model.py ADDED
@@ -0,0 +1,202 @@
1
+ """The probe contract, loaded and validated.
2
+
3
+ Deliberately stdlib-only. Anyone must be able to read a probe, understand what
4
+ it claims, and check that claim without installing anything — a suite that needs
5
+ a toolchain to be inspected is asking to be trusted rather than checked.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ import tomllib
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+ from typing import Any, Literal
15
+
16
+ CONTRACT = "probe.toml"
17
+ POPULATIONS = ("trap", "clean")
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class Truth:
22
+ kind: str
23
+ value: Any
24
+ tolerance: float
25
+ derivation: str
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class NaiveFailure:
30
+ description: str
31
+ observed_value: Any
32
+ why_plausible: str
33
+ derivation: str | None = None
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class Probe:
38
+ """One task whose correct answer is known by construction."""
39
+
40
+ id: str
41
+ population: Literal["trap", "clean"]
42
+ family: str
43
+ title: str
44
+ surface: tuple[str, ...]
45
+ call: str
46
+ truth: Truth
47
+ provenance: dict[str, Any]
48
+ directory: Path
49
+ prompt: str | None = None
50
+ naive_failure: NaiveFailure | None = None
51
+ accept_if_mentions: tuple[str, ...] = ()
52
+
53
+ @property
54
+ def operation(self) -> str:
55
+ """The operation name in `task.call`, e.g. `raster_mean` in `raster_mean(dem.tif)`."""
56
+ return _parse_call(self.call)[0]
57
+
58
+ @property
59
+ def arguments(self) -> tuple[str, ...]:
60
+ return _parse_call(self.call)[1]
61
+
62
+
63
+ def _parse_call(call: str) -> tuple[str, tuple[str, ...]]:
64
+ """`raster_mean(dem.tif, band=1)` → ("raster_mean", ("dem.tif", "band=1")).
65
+
66
+ A grammar rather than `eval`: probe files come from contributors, and a
67
+ suite that runs arbitrary code from its own fixtures cannot be run by the
68
+ people it is meant to convince.
69
+ """
70
+ match = re.fullmatch(r"\s*([a-z][a-z0-9_]*)\s*\((.*)\)\s*", call, re.DOTALL)
71
+ if not match:
72
+ raise ValueError(f"task.call is not `name(args)`: {call!r}")
73
+ name, inside = match.group(1), match.group(2).strip()
74
+ args = tuple(a.strip() for a in inside.split(",")) if inside else ()
75
+ return name, args
76
+
77
+
78
+ class ContractError(ValueError):
79
+ """A probe file that does not honour the contract. Names the file and the field."""
80
+
81
+
82
+ def _require(data: dict, key: str, where: Path) -> Any:
83
+ if key not in data:
84
+ raise ContractError(f"{where}: missing required key `{key}`")
85
+ return data[key]
86
+
87
+
88
+ def load_probe(directory: Path) -> Probe:
89
+ """Read and check one probe directory.
90
+
91
+ The checks here are the ones that make a published number mean something;
92
+ the JSON Schema in `schema/` is the same contract for people who are not
93
+ running this code. When they disagree, the schema is the specification.
94
+ """
95
+ path = directory / CONTRACT
96
+ try:
97
+ data = tomllib.loads(path.read_text(encoding="utf-8"))
98
+ except FileNotFoundError:
99
+ raise ContractError(f"{directory}: no {CONTRACT}") from None
100
+ except tomllib.TOMLDecodeError as exc:
101
+ raise ContractError(f"{path}: not valid TOML — {exc}") from None
102
+
103
+ population = _require(data, "population", path)
104
+ if population not in POPULATIONS:
105
+ raise ContractError(f"{path}: population must be one of {POPULATIONS}")
106
+
107
+ task = _require(data, "task", path)
108
+ truth_data = _require(data, "truth", path)
109
+ for key in ("kind", "value", "tolerance", "derivation"):
110
+ _require(truth_data, f"truth.{key}" if key not in truth_data else key, path)
111
+
112
+ naive, accepts = None, ()
113
+ if population == "trap":
114
+ raw = _require(data, "naive_failure", path)
115
+ if not raw.get("plausible"):
116
+ # The admission criterion, and the only one that is not taste. An
117
+ # error that crashes or returns an absurd number is already caught
118
+ # by something; this suite is for the answers nothing catches.
119
+ raise ContractError(
120
+ f"{path}: a trap must declare `plausible = true`. If the typical "
121
+ "error is loud, the probe belongs in an ordinary test suite."
122
+ )
123
+ naive = NaiveFailure(
124
+ description=raw["description"],
125
+ observed_value=raw["observed_value"],
126
+ why_plausible=raw["why_plausible"],
127
+ derivation=raw.get("derivation"),
128
+ )
129
+ accepts = tuple(_require(data, "refusal", path)["accept_if_mentions"])
130
+ elif "naive_failure" in data:
131
+ raise ContractError(
132
+ f"{path}: a clean probe cannot have a naive_failure. It is a trap "
133
+ "filed in the wrong place, and it would corrupt the completion rate."
134
+ )
135
+
136
+ surfaces = tuple(_require(data, "surface", path))
137
+ if "agent" in surfaces and not task.get("prompt"):
138
+ raise ContractError(f"{path}: surface includes `agent` but task.prompt is missing")
139
+
140
+ return Probe(
141
+ id=_require(data, "id", path),
142
+ population=population,
143
+ family=_require(data, "family", path),
144
+ title=_require(data, "title", path),
145
+ surface=surfaces,
146
+ call=_require(task, "call", path),
147
+ prompt=task.get("prompt"),
148
+ truth=Truth(truth_data["kind"], truth_data["value"], float(truth_data["tolerance"]),
149
+ truth_data["derivation"]),
150
+ naive_failure=naive,
151
+ accept_if_mentions=accepts,
152
+ provenance=_require(data, "provenance", path),
153
+ directory=directory,
154
+ )
155
+
156
+
157
+ def discover(root: Path) -> list[Probe]:
158
+ """Every probe under `traps/` and `clean/`, in a stable order.
159
+
160
+ Stable because published results are compared across runs and across
161
+ systems: an ordering that depends on the filesystem would make two honest
162
+ runs look different.
163
+ """
164
+ folders = {"trap": root / "traps", "clean": root / "clean"}
165
+ found = [
166
+ load_probe(contratto.parent)
167
+ for folder in folders.values()
168
+ for contratto in sorted(folder.rglob(CONTRACT))
169
+ ]
170
+ duplicates = {p.id for p in found if [q.id for q in found].count(p.id) > 1}
171
+ if duplicates:
172
+ raise ContractError(f"duplicate probe ids: {sorted(duplicates)}")
173
+ return sorted(found, key=lambda p: (p.population, p.id))
174
+
175
+
176
+ @dataclass
177
+ class Outcome:
178
+ """What a system did with one probe.
179
+
180
+ Four mutually exclusive shapes, and the runner refuses anything ambiguous:
181
+ an adapter that returns both an answer and a refusal has not decided what
182
+ the system did, and guessing on its behalf is how a score stops meaning
183
+ anything.
184
+ """
185
+
186
+ answer: Any = None
187
+ refusal: str | None = None
188
+ error: str | None = None
189
+ unsupported: bool = False
190
+ warnings: list[str] = field(default_factory=list)
191
+ transcript: str | None = None
192
+
193
+ def __post_init__(self) -> None:
194
+ declared = sum(
195
+ x is not None and x is not False
196
+ for x in (self.answer, self.refusal, self.error, self.unsupported or None)
197
+ )
198
+ if declared > 1:
199
+ raise ValueError(
200
+ "an Outcome is exactly one of answer / refusal / error / unsupported; "
201
+ f"got {declared}"
202
+ )
@@ -0,0 +1,36 @@
1
+ """The same elevations, stored without the predictor.
2
+
3
+ The control. If a system cannot answer this, it cannot answer trap 001 either,
4
+ and its silent-error rate on that trap says nothing about silent errors — only
5
+ that the task was out of reach. Every trap family should have a clean twin for
6
+ the same reason a medical test needs a negative control.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+ import rasterio
16
+ from rasterio.transform import from_origin
17
+
18
+ HEIGHT = WIDTH = 32
19
+ BASE, EAST_STEP, SOUTH_STEP = 1000, 4, 2
20
+
21
+
22
+ def main(destination: Path) -> int:
23
+ destination.mkdir(parents=True, exist_ok=True)
24
+ row, column = np.mgrid[0:HEIGHT, 0:WIDTH]
25
+ grid = (BASE + EAST_STEP * column + SOUTH_STEP * row).astype("int16")
26
+ with rasterio.open(
27
+ destination / "dem.tif", "w", driver="GTiff", height=HEIGHT, width=WIDTH,
28
+ count=1, dtype="int16", crs="EPSG:32632", nodata=-32768,
29
+ transform=from_origin(500000.0, 5000000.0, 30.0, 30.0),
30
+ ) as ds:
31
+ ds.write(grid, 1)
32
+ return 0
33
+
34
+
35
+ if __name__ == "__main__":
36
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
@@ -0,0 +1,25 @@
1
+ id = "c001-raster-mean"
2
+ population = "clean"
3
+ family = "raster-encoding"
4
+ title = "Mean of an uncompressed integer DEM"
5
+ surface = ["engine", "agent"]
6
+
7
+ [task]
8
+ call = "raster_mean(dem.tif)"
9
+ prompt = "What is the mean elevation of dem.tif, in metres?"
10
+
11
+ [truth]
12
+ kind = "scalar"
13
+ value = 1093.0
14
+ tolerance = 0.001
15
+ derivation = """
16
+ Identical elevations to trap 001, stored plainly: v[i,j] = 1000 + 4j + 2i over
17
+ 32x32, mean = 1000 + 4*15.5 + 2*15.5 = 1093 exactly. Sharing the arithmetic with
18
+ the trap is the point — the only difference between the two probes is how the
19
+ bytes are stored, so a system that answers one and not the other has told us
20
+ about the encoding and nothing else."""
21
+
22
+ [provenance]
23
+ source = "control for 001-tiff-predictor"
24
+ found_by = "mapsmith"
25
+ date = "2026-08-23"
@@ -0,0 +1,44 @@
1
+ """The same square, in a CRS whose unit is the metre.
2
+
3
+ The control for `002-feet-as-metres`: identical geometry and identical task, and
4
+ the only difference is a linear unit the file states plainly. A system that
5
+ answers this and misses the trap has told us it does not read the unit. One that
6
+ misses both has told us nothing about units at all.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ # GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
16
+ # builds of the same data differ byte for byte. Every published number is
17
+ # anchored to a `spec_commit`, which means nothing if that commit can produce
18
+ # different fixtures — so the clock is pinned. GDAL reads this at write time,
19
+ # hence before the import that uses it.
20
+ os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-23T00:00:00.000Z")
21
+
22
+ import geopandas as gpd
23
+ from shapely.geometry import Polygon
24
+
25
+ SIDE_METRES = 1000.0
26
+ ORIGIN_X, ORIGIN_Y = 500000.0, 5000000.0 # UTM 32N
27
+
28
+
29
+ def main(destination: Path) -> int:
30
+ destination.mkdir(parents=True, exist_ok=True)
31
+ square = Polygon([
32
+ (ORIGIN_X, ORIGIN_Y),
33
+ (ORIGIN_X + SIDE_METRES, ORIGIN_Y),
34
+ (ORIGIN_X + SIDE_METRES, ORIGIN_Y + SIDE_METRES),
35
+ (ORIGIN_X, ORIGIN_Y + SIDE_METRES),
36
+ ])
37
+ gpd.GeoDataFrame(
38
+ {"parcel_id": ["A-1"]}, geometry=[square], crs="EPSG:32632"
39
+ ).to_file(destination / "parcel.gpkg", layer="parcel", driver="GPKG")
40
+ return 0
41
+
42
+
43
+ if __name__ == "__main__":
44
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
@@ -0,0 +1,25 @@
1
+ id = "c002-projected-area"
2
+ population = "clean"
3
+ family = "linear-units"
4
+ title = "Area of a square in a metric projected CRS"
5
+ surface = ["engine", "agent"]
6
+
7
+ [task]
8
+ call = "planar_area_m2(parcel.gpkg)"
9
+ prompt = """
10
+ What is the area of the parcel in parcel.gpkg, in square metres, measured in the
11
+ plane of the layer's own coordinate reference system?"""
12
+
13
+ [truth]
14
+ kind = "scalar"
15
+ value = 1000000.0
16
+ tolerance = 0.01
17
+ derivation = """
18
+ A square of side 1000 in EPSG:32632, whose linear unit is the metre: the area is
19
+ 1000 * 1000 = 1000000 square metres. Nothing to convert, which is the point -
20
+ the trap and its control differ only in the unit the file declares."""
21
+
22
+ [provenance]
23
+ source = "control for 002-feet-as-metres"
24
+ found_by = "mapsmith"
25
+ date = "2026-08-23"
@@ -0,0 +1,35 @@
1
+ """The same raster with the voids filled, so nothing has to be excluded.
2
+
3
+ The control for `003-nodata-in-statistics`. Identical grid, identical task, and
4
+ no nodata cells at all: a system that answers this and misses the trap has told
5
+ us it does not honour a declared nodata value. One that misses both has told us
6
+ it cannot read the raster, which is a different finding.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+ import rasterio
16
+ from rasterio.transform import from_origin
17
+
18
+ SIDE = 100
19
+ ELEVATION = 1000
20
+
21
+
22
+ def main(destination: Path) -> int:
23
+ destination.mkdir(parents=True, exist_ok=True)
24
+ with rasterio.open(
25
+ destination / "dem.tif", "w", driver="GTiff",
26
+ height=SIDE, width=SIDE, count=1, dtype="int16",
27
+ crs="EPSG:32632", nodata=-9999,
28
+ transform=from_origin(500000.0, 5000000.0, 30.0, 30.0),
29
+ ) as ds:
30
+ ds.write(np.full((SIDE, SIDE), ELEVATION, dtype="int16"), 1)
31
+ return 0
32
+
33
+
34
+ if __name__ == "__main__":
35
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
@@ -0,0 +1,26 @@
1
+ id = "c003-raster-mean-nodata"
2
+ population = "clean"
3
+ family = "nodata"
4
+ title = "Mean of a raster that declares nodata and contains none"
5
+ surface = ["engine", "agent"]
6
+
7
+ [task]
8
+ call = "raster_mean(dem.tif)"
9
+ prompt = "What is the mean elevation of dem.tif, in metres?"
10
+
11
+ [truth]
12
+ kind = "scalar"
13
+ value = 1000.0
14
+ tolerance = 0.001
15
+ derivation = """
16
+ Every cell of the 100x100 grid holds 1000, so the mean is 1000. The header
17
+ still declares nodata = -9999, exactly as the trap's does - the only difference
18
+ between the two probes is whether any cell holds that value. A system that
19
+ answers this one and not the trap has told us it does not honour the
20
+ declaration; one that misses both has told us it cannot read the file, which
21
+ is a different finding and should not be counted as the first."""
22
+
23
+ [provenance]
24
+ source = "control for 003-nodata-in-statistics"
25
+ found_by = "mapsmith"
26
+ date = "2026-08-23"
@@ -0,0 +1,60 @@
1
+ """Build the control: the same counting task, both layers in the same CRS.
2
+
3
+ Everything is defined directly in EPSG:32633 metres, so containment is an
4
+ interval comparison in the shared plane and nothing is ever transformed. The
5
+ trap and this control differ in exactly one thing: whether the two frames
6
+ already agree.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ # GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
16
+ # builds of the same data differ byte for byte unless the clock is pinned.
17
+ os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-24T00:00:00.000Z")
18
+
19
+ import geopandas as gpd
20
+ from shapely.geometry import Point, Polygon
21
+
22
+ CRS = "EPSG:32633"
23
+ ZONE_E = (391_000.0, 401_000.0)
24
+ ZONE_N = (4_629_000.0, 4_641_000.0)
25
+ # Inside: a 3x3 grid with at least 2 km of margin to every edge.
26
+ INSIDE_E = (393_000.0, 396_000.0, 399_000.0)
27
+ INSIDE_N = (4_632_000.0, 4_635_000.0, 4_638_000.0)
28
+ # Outside: everything at easting 430 000 or greater — at least 29 km east.
29
+ OUTSIDE_E = (430_000.0, 460_000.0, 490_000.0)
30
+ OUTSIDE_N = (
31
+ 4_600_000.0, 4_610_000.0, 4_620_000.0, 4_650_000.0, 4_660_000.0,
32
+ 4_670_000.0, 4_680_000.0, 4_690_000.0, 4_700_000.0,
33
+ )
34
+
35
+
36
+ def main(destination: Path) -> int:
37
+ destination.mkdir(parents=True, exist_ok=True)
38
+
39
+ rectangle = Polygon([
40
+ (ZONE_E[0], ZONE_N[0]),
41
+ (ZONE_E[1], ZONE_N[0]),
42
+ (ZONE_E[1], ZONE_N[1]),
43
+ (ZONE_E[0], ZONE_N[1]),
44
+ ])
45
+ gpd.GeoDataFrame(
46
+ {"zone_id": ["Z-1"]}, geometry=[rectangle], crs=CRS
47
+ ).to_file(destination / "zone.gpkg", layer="zone", driver="GPKG")
48
+
49
+ coords = [(e, n) for e in INSIDE_E for n in INSIDE_N]
50
+ coords += [(e, n) for e in OUTSIDE_E for n in OUTSIDE_N]
51
+ gpd.GeoDataFrame(
52
+ {"well_id": [f"W-{i + 1}" for i in range(len(coords))]},
53
+ geometry=[Point(e, n) for e, n in coords],
54
+ crs=CRS,
55
+ ).to_file(destination / "points.gpkg", layer="points", driver="GPKG")
56
+ return 0
57
+
58
+
59
+ if __name__ == "__main__":
60
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
@@ -0,0 +1,29 @@
1
+ id = "c004-points-in-polygon"
2
+ population = "clean"
3
+ family = "mismatched-crs"
4
+ title = "Count of points inside a polygon, both layers in the same CRS"
5
+ surface = ["engine", "agent"]
6
+
7
+ [task]
8
+ call = "points_in_polygon_count(points.gpkg, zone.gpkg)"
9
+ prompt = """
10
+ How many of the points in points.gpkg fall inside the polygon in zone.gpkg?
11
+ Answer with the count."""
12
+
13
+ [truth]
14
+ kind = "count"
15
+ value = 9
16
+ tolerance = 0
17
+ derivation = """
18
+ Both layers are in EPSG:32633, so containment is an interval comparison in
19
+ their shared plane. The zone is the rectangle [391000, 401000] x
20
+ [4629000, 4641000]. Nine points lie on the grid {393000, 396000, 399000} x
21
+ {4632000, 4635000, 4638000}, inside with at least 2 km of margin to every
22
+ edge; the other 27 lie at eastings 430000 or greater, at least 29 km east of
23
+ the rectangle. Nothing is ever transformed, which is the point - the trap and
24
+ this control differ only in whether the two frames already agree."""
25
+
26
+ [provenance]
27
+ source = "control for 004-mismatched-crs-join"
28
+ found_by = "mapsmith"
29
+ date = "2026-08-24"
@@ -0,0 +1,38 @@
1
+ """Build the control: an ordinary valid L-shaped parcel in a metric CRS.
2
+
3
+ Same operation and same phrasing as the trap; the only difference is that the
4
+ ring does not cross itself. The L-shape (rather than another rectangle) keeps
5
+ the control from being answerable by pattern-matching the fixture family.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ # GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
15
+ # builds of the same data differ byte for byte unless the clock is pinned.
16
+ os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-24T00:00:00.000Z")
17
+
18
+ import geopandas as gpd
19
+ from shapely.geometry import Polygon
20
+
21
+ CRS = "EPSG:32632"
22
+ E0, N0 = 402_000.0, 5_003_000.0
23
+
24
+ # Local shape: an L. Area = 100*40 + 60*60 = 7600 m2, on paper.
25
+ RING = [(0.0, 0.0), (100.0, 0.0), (100.0, 40.0), (60.0, 40.0), (60.0, 100.0), (0.0, 100.0)]
26
+
27
+
28
+ def main(destination: Path) -> int:
29
+ destination.mkdir(parents=True, exist_ok=True)
30
+ ell = Polygon([(E0 + x, N0 + y) for x, y in RING])
31
+ gpd.GeoDataFrame(
32
+ {"parcel_id": ["A-1"]}, geometry=[ell], crs=CRS
33
+ ).to_file(destination / "parcel.gpkg", layer="parcel", driver="GPKG")
34
+ return 0
35
+
36
+
37
+ if __name__ == "__main__":
38
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
@@ -0,0 +1,26 @@
1
+ id = "c005-polygon-area"
2
+ population = "clean"
3
+ family = "invalid-geometry"
4
+ title = "Area of a valid L-shaped parcel in a metric CRS"
5
+ surface = ["engine", "agent"]
6
+
7
+ [task]
8
+ call = "planar_area_m2(parcel.gpkg)"
9
+ prompt = """
10
+ What is the area of the parcel in parcel.gpkg, in square metres, measured in the
11
+ plane of the layer's own coordinate reference system?"""
12
+
13
+ [truth]
14
+ kind = "scalar"
15
+ value = 7600.0
16
+ tolerance = 0.01
17
+ derivation = """
18
+ An L-shaped polygon in EPSG:32632 (metres), decomposable on paper into a
19
+ 100 x 40 rectangle plus a 60 x 60 square: 4000 + 3600 = 7600 square metres.
20
+ The ring is valid - the trap and this control differ only in whether the ring
21
+ crosses itself."""
22
+
23
+ [provenance]
24
+ source = "control for 005-bowtie-area"
25
+ found_by = "mapsmith"
26
+ date = "2026-08-24"
@@ -0,0 +1,35 @@
1
+ """Build the control: a single-layer GeoPackage, same question, no ambiguity.
2
+
3
+ One layer, honestly named `wells`, 17 points. Whether a system passes a layer
4
+ name or not, there is only one layer to get.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ import sys
11
+ from pathlib import Path
12
+
13
+ # GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
14
+ # builds of the same data differ byte for byte unless the clock is pinned.
15
+ os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-25T00:00:00.000Z")
16
+
17
+ import geopandas as gpd
18
+ from shapely.geometry import Point
19
+
20
+ CRS = "EPSG:32632"
21
+ N_WELLS = 17
22
+
23
+
24
+ def main(destination: Path) -> int:
25
+ destination.mkdir(parents=True, exist_ok=True)
26
+ gpd.GeoDataFrame(
27
+ {"well_id": [f"W-{i}" for i in range(1, N_WELLS + 1)]},
28
+ geometry=[Point(10.0 + i * 9.0, 50.0 + (i % 4) * 13.0) for i in range(N_WELLS)],
29
+ crs=CRS,
30
+ ).to_file(destination / "wells.gpkg", layer="wells", driver="GPKG")
31
+ return 0
32
+
33
+
34
+ if __name__ == "__main__":
35
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))
@@ -0,0 +1,26 @@
1
+ id = "c006-named-layer"
2
+ population = "clean"
3
+ family = "ambiguous-layer"
4
+ title = "Feature count of a single-layer container"
5
+ surface = ["engine", "agent"]
6
+
7
+ [task]
8
+ call = "feature_count(wells.gpkg, layer=wells)"
9
+ prompt = """
10
+ How many features are in the `wells` layer of wells.gpkg? Answer with the
11
+ count."""
12
+
13
+ [truth]
14
+ kind = "count"
15
+ value = 17
16
+ tolerance = 0
17
+ derivation = """
18
+ By construction: wells.gpkg holds exactly one layer, `wells`, with 17 point
19
+ features (W-1 through W-17). With a single layer there is nothing to choose
20
+ wrongly — the trap and this control differ only in whether the container
21
+ holds a second layer the question did not ask about."""
22
+
23
+ [provenance]
24
+ source = "control for 006-default-layer"
25
+ found_by = "mapsmith"
26
+ date = "2026-08-25"
@@ -0,0 +1,42 @@
1
+ """Build the control: the same counting question on a layer already in metres.
2
+
3
+ Five wells within 460 m of W-1, everything else 2.5 km away or more, all in
4
+ EPSG:32633 — the distance 500 and the layer's unit finally agree.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ import sys
11
+ from pathlib import Path
12
+
13
+ # GeoPackage stamps the write time into `gpkg_contents.last_change`, so two
14
+ # builds of the same data differ byte for byte unless the clock is pinned.
15
+ os.environ.setdefault("OGR_CURRENT_DATE", "2026-08-25T00:00:00.000Z")
16
+
17
+ import geopandas as gpd
18
+ from shapely.geometry import Point
19
+
20
+ CRS = "EPSG:32633"
21
+ E0, N0 = 500_000.0, 4_640_000.0 # W-1
22
+
23
+ INSIDE = [
24
+ (E0 + 300.0, N0), (E0, N0 + 400.0), (E0 - 250.0, N0 - 250.0),
25
+ (E0 + 120.0, N0 - 430.0), (E0 - 460.0, N0),
26
+ ]
27
+ OUTSIDE = [(E0 + 2000.0 + 150.0 * i, N0 + 1500.0 + 100.0 * (i % 5)) for i in range(24)]
28
+
29
+
30
+ def main(destination: Path) -> int:
31
+ destination.mkdir(parents=True, exist_ok=True)
32
+ points = [Point(E0, N0)] + [Point(*c) for c in INSIDE] + [Point(*c) for c in OUTSIDE]
33
+ gpd.GeoDataFrame(
34
+ {"well_id": [f"W-{i + 1}" for i in range(len(points))]},
35
+ geometry=points,
36
+ crs=CRS,
37
+ ).to_file(destination / "wells.gpkg", layer="wells", driver="GPKG")
38
+ return 0
39
+
40
+
41
+ if __name__ == "__main__":
42
+ raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))