argleton 0.1.0__py3-none-any.whl

argleton/run.py

@@ -0,0 +1,170 @@
+"""The runner: build the fixtures, ask a system, write a result nobody has to trust.
+
+Fixtures are built, never vendored. The repo stays in kilobytes, anyone can
+regenerate them and check they are what we say they are, and rerunning the whole
+suite costs nothing — which is half of why a third party can contest our numbers
+in an afternoon instead of taking them on faith.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import subprocess
+import sys
+import tempfile
+from dataclasses import asdict
+from datetime import UTC, datetime
+from pathlib import Path
+
+from .model import Outcome, Probe, discover
+from .score import Verdict, judge, summarise
+
+
+def _default_root() -> Path:
+    """Where the probes are: the repository if we are in it, the package if not.
+
+    Two layouts, and the difference is not cosmetic. Run from a checkout, the
+    probes are `traps/` and `clean/` beside the package and that is what a
+    contributor edits. Installed from PyPI, they are bundled inside the package,
+    because a wheel that ships the runner without the probes is a command that
+    finds nothing and exits 2 — and looks like it worked.
+    """
+    checkout = Path(__file__).resolve().parent.parent
+    if (checkout / "traps").is_dir():
+        return checkout
+    bundled = Path(__file__).resolve().parent / "probes"
+    if (bundled / "traps").is_dir():
+        return bundled
+    return checkout
+
+
+ROOT = _default_root()
+
+
+def build_fixtures(probe: Probe, workdir: Path) -> None:
+    """Run the probe's `build.py` into a fresh directory.
+
+    A subprocess rather than an import: fixture builders are the part of this
+    repo most likely to come from someone else, and a builder that leaks state
+    into the runner would make two probes depend on the order they ran in —
+    the kind of defect this suite is supposed to be above.
+    """
+    builder = probe.directory / "build.py"
+    if not builder.exists():
+        return
+    proc = subprocess.run(
+        [sys.executable, str(builder), str(workdir)],
+        capture_output=True, text=True, encoding="utf-8", errors="replace",
+        timeout=300, check=False,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(f"{probe.id}: build.py failed\n{proc.stdout}\n{proc.stderr}")
+
+
+def load_adapter(name: str):
+    """`engine:rasterio` → adapters.engine_rasterio; `x.y:Z` → an importable object."""
+    if name.startswith("engine:"):
+        module = importlib.import_module(f"adapters.engine_{name.split(':', 1)[1]}")
+        return module.Adapter()
+    module, _, attr = name.partition(":")
+    loaded = importlib.import_module(module)
+    return getattr(loaded, attr or "Adapter")()
+
+
+def run_probe(adapter, probe: Probe, keep: Path | None = None) -> tuple[Outcome, Verdict]:
+    with tempfile.TemporaryDirectory(prefix=f"argleton-{probe.id}-") as tmp:
+        workdir = Path(keep / probe.id) if keep else Path(tmp)
+        workdir.mkdir(parents=True, exist_ok=True)
+        try:
+            build_fixtures(probe, workdir)
+            outcome = adapter.run(probe, workdir)
+        except Exception as exc:  # noqa: BLE001 — an adapter that raises has errored, not crashed us
+            outcome = Outcome(error=f"{type(exc).__name__}: {exc}")
+    return outcome, judge(probe, outcome)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="argleton", description=__doc__.splitlines()[0])
+    parser.add_argument("--adapter", required=True,
+                        help="engine:rasterio, engine:whitebox, or module:Class")
+    parser.add_argument("--system", help="name of the system under test, for the result file")
+    parser.add_argument("--only", nargs="*", default=None, help="probe ids to run")
+    parser.add_argument("--population", choices=("trap", "clean"), default=None)
+    parser.add_argument("--out", type=Path, help="write the result JSON here")
+    parser.add_argument("--keep", type=Path, help="keep the built fixtures in this directory")
+    parser.add_argument("--root", type=Path, default=ROOT)
+    args = parser.parse_args(argv)
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            # Probe text is UTF-8 and consoles are not, on the platform where
+            # this was written. A mangled tolerance sign in a published run is
+            # a small thing that makes a careful reader distrust the rest.
+            stream.reconfigure(encoding="utf-8", errors="replace")
+
+    sys.path.insert(0, str(args.root))
+    probes = discover(args.root)
+    if args.population:
+        probes = [p for p in probes if p.population == args.population]
+    if args.only:
+        probes = [p for p in probes if p.id in set(args.only)]
+    if not probes:
+        print("no probes matched", file=sys.stderr)
+        return 2
+
+    adapter = load_adapter(args.adapter)
+    verdicts, details = [], []
+    for probe in probes:
+        _outcome, verdict = run_probe(adapter, probe, args.keep)
+        verdicts.append(verdict)
+        details.append(asdict(verdict))
+        mark = {True: "ok  ", False: "FAIL", None: "skip"}[verdict.success]
+        print(f"{mark} {probe.population:5} {probe.id:28} "
+              f"{verdict.verdict:22} {verdict.detail[:70]}")
+
+    summary = summarise(verdicts)
+    result = {
+        "system": args.system or getattr(adapter, "name", args.adapter),
+        "adapter": args.adapter,
+        "spec_commit": spec_commit(args.root),
+        "date": datetime.now(UTC).strftime("%Y-%m-%d"),
+        **summary,
+        "per_probe": details,
+    }
+    print(
+        f"\nsilent_error_rate {summary['silent_error_rate']} "
+        f"over {summary['traps_run']} traps  |  "
+        f"completion_rate {summary['completion_rate']} over {summary['clean_run']} clean"
+    )
+    if args.out:
+        args.out.parent.mkdir(parents=True, exist_ok=True)
+        args.out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+        print(f"written {args.out}")
+
+    # Exit 0 even with silent errors: finding them is the job, not the alarm.
+    # A non-zero exit is for the runner failing to do its own job.
+    return 0
+
+
+def spec_commit(root: Path) -> str:
+    """The commit the probes were at, so a result can never float free of its rules.
+
+    Pre-registration that is checkable instead of asserted: every result names
+    the commit it ran against, so anyone wondering whether we widened a
+    tolerance after seeing a number reads a diff rather than our word for it.
+    """
+    proc = subprocess.run(
+        ["git", "-C", str(root), "rev-parse", "--short", "HEAD"],
+        capture_output=True, text=True, check=False,
+    )
+    dirty = subprocess.run(
+        ["git", "-C", str(root), "status", "--porcelain"],
+        capture_output=True, text=True, check=False,
+    ).stdout.strip()
+    commit = proc.stdout.strip() or "unknown"
+    return f"{commit}-dirty" if dirty else commit
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

argleton/score.py

@@ -0,0 +1,150 @@
+"""Turning an outcome into a verdict, and verdicts into the two numbers.
+
+The whole suite exists for one row of the table below: `silent_error` — wrong,
+and presented as successful. Every other outcome is something a user, a test or
+a log already notices. That is why the headline number counts only that row, and
+why it is never published alone.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from .model import Outcome, Probe
+
+# verdict -> counts as a success for the system under test
+VERDICTS = {
+    "correct": True,
+    "correct_with_warning": True,
+    "refused_correctly": True,
+    "noisy_failure": False,
+    "refused_wrongly": False,
+    "silent_error": False,
+    "unsupported": None,  # neither: the adapter cannot express the task at all
+}
+
+
+@dataclass(frozen=True)
+class Verdict:
+    probe_id: str
+    population: str
+    family: str
+    verdict: str
+    detail: str
+    answer: Any = None
+
+    @property
+    def success(self) -> bool | None:
+        return VERDICTS[self.verdict]
+
+
+def within_tolerance(value: Any, expected: Any, tolerance: float) -> bool:
+    """Absolute tolerance, elementwise for sequences.
+
+    Absolute and not relative on purpose. A relative tolerance is generous
+    exactly where these defects live — a mean that collapses towards zero
+    passes a relative check against a small expected value — and it makes the
+    number depend on the magnitude of the fixture, which the fixture author
+    chooses.
+    """
+    if isinstance(expected, (list, tuple)):
+        if not isinstance(value, (list, tuple)) or len(value) != len(expected):
+            return False
+        return all(within_tolerance(v, e, tolerance) for v, e in zip(value, expected, strict=True))
+    if isinstance(expected, bool) or isinstance(value, bool):
+        return value is expected
+    try:
+        return abs(float(value) - float(expected)) <= tolerance
+    except (TypeError, ValueError):
+        return value == expected
+
+
+def names_the_defect(reason: str, accept_if_mentions: tuple[str, ...]) -> bool:
+    text = (reason or "").lower()
+    return any(fragment.lower() in text for fragment in accept_if_mentions)
+
+
+def judge(probe: Probe, outcome: Outcome) -> Verdict:
+    def verdict(name: str, detail: str) -> Verdict:
+        return Verdict(probe.id, probe.population, probe.family, name, detail, outcome.answer)
+
+    if outcome.unsupported:
+        return verdict("unsupported", "the adapter does not implement this operation")
+    if outcome.error is not None:
+        # A crash is a failure, but a *loud* one: someone sees it. It is counted
+        # apart from silent errors because conflating them would let a system
+        # that falls over look as dangerous as one that answers wrongly, and
+        # the second is the one this suite was built to find.
+        return verdict("noisy_failure", f"error: {outcome.error}")
+
+    if outcome.refusal is not None:
+        if probe.population == "clean":
+            # Refusing a solvable task is what a system does when it wants a
+            # clean score without doing the work. Counting it as failure is the
+            # only thing that stops the headline number being gamed by silence.
+            return verdict("refused_wrongly", f"refused a solvable task: {outcome.refusal}")
+        if names_the_defect(outcome.refusal, probe.accept_if_mentions):
+            return verdict("refused_correctly", f"named the defect: {outcome.refusal}")
+        return verdict(
+            "refused_wrongly",
+            "refused without naming the defect (expected one of "
+            f"{list(probe.accept_if_mentions)}): {outcome.refusal}",
+        )
+
+    if outcome.answer is None:
+        return verdict("noisy_failure", "no answer and no reason")
+
+    if within_tolerance(outcome.answer, probe.truth.value, probe.truth.tolerance):
+        if outcome.warnings:
+            return verdict("correct_with_warning", "; ".join(outcome.warnings))
+        return verdict("correct", f"{outcome.answer}")
+
+    detail = f"expected {probe.truth.value} ± {probe.truth.tolerance}, got {outcome.answer}"
+    if probe.naive_failure is not None and within_tolerance(
+        outcome.answer, probe.naive_failure.observed_value, probe.truth.tolerance
+    ):
+        # Worth saying out loud: the system did not merely get it wrong, it got
+        # it wrong in the exact way the trap predicts. That is the difference
+        # between a finding and a number.
+        detail += " — this is the predicted naive failure"
+    return verdict("silent_error", detail)
+
+
+def summarise(verdicts: list[Verdict]) -> dict[str, Any]:
+    """The two numbers, and never one of them.
+
+    A system that refuses everything scores a perfect silent-error rate and is
+    useless. A system that answers everything confidently scores a perfect
+    completion rate and may be dangerous. Published together, the pair says in
+    one glance which one you are looking at — which is why `result.schema.json`
+    requires both and why a partial report is not a valid result.
+    """
+    traps = [v for v in verdicts if v.population == "trap" and v.verdict != "unsupported"]
+    clean = [v for v in verdicts if v.population == "clean" and v.verdict != "unsupported"]
+    silent = [v for v in traps if v.verdict == "silent_error"]
+
+    per_family: dict[str, dict[str, int]] = {}
+    for v in verdicts:
+        row = per_family.setdefault(v.family, {"probes": 0, "silent_errors": 0})
+        row["probes"] += 1
+        row["silent_errors"] += v.verdict == "silent_error"
+
+    return {
+        # The metric. Denominator is traps actually run: an adapter that skips
+        # half of them must not look better than one that faced all of them,
+        # so `probes_run` travels with the rate and the schema requires it.
+        "silent_error_rate": round(len(silent) / len(traps), 4) if traps else None,
+        "completion_rate": (
+            round(sum(v.success is True for v in clean) / len(clean), 4) if clean else None
+        ),
+        "traps_run": len(traps),
+        "clean_run": len(clean),
+        "unsupported": sum(v.verdict == "unsupported" for v in verdicts),
+        "verdict_counts": {
+            name: sum(v.verdict == name for v in verdicts)
+            for name in VERDICTS
+            if any(v.verdict == name for v in verdicts)
+        },
+        "by_family": dict(sorted(per_family.items())),
+    }

argleton-0.1.0.dist-info/METADATA

@@ -0,0 +1,275 @@
+Metadata-Version: 2.5
+Name: argleton
+Version: 0.1.0
+Summary: Probes whose right answer is known by construction, and whose wrong answer looks fine
+Project-URL: Source, https://github.com/argleton/argleton
+Author: Argleton contributors
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: benchmark,correctness,evaluation,geospatial,gis,provenance,raster
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: GIS
+Requires-Python: >=3.11
+Provides-Extra: dev
+Requires-Dist: jsonschema>=4.21; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Provides-Extra: fixtures
+Requires-Dist: geopandas>=1.0; extra == 'fixtures'
+Requires-Dist: numpy>=1.26; extra == 'fixtures'
+Requires-Dist: pyogrio>=0.9; extra == 'fixtures'
+Requires-Dist: rasterio>=1.3; extra == 'fixtures'
+Requires-Dist: shapely>=2.0; extra == 'fixtures'
+Provides-Extra: geopandas
+Requires-Dist: geopandas>=1.0; extra == 'geopandas'
+Requires-Dist: pyogrio>=0.9; extra == 'geopandas'
+Requires-Dist: pyproj>=3.6; extra == 'geopandas'
+Requires-Dist: shapely>=2.0; extra == 'geopandas'
+Provides-Extra: mapsmith
+Requires-Dist: mapsmith[raster]>=0.2.2; extra == 'mapsmith'
+Provides-Extra: site
+Requires-Dist: pillow>=10; extra == 'site'
+Provides-Extra: whitebox
+Requires-Dist: whitebox-workflows>=1.3; extra == 'whitebox'
+Description-Content-Type: text/markdown
+
+# Argleton
+
+**A correctness suite for geospatial systems.** Every probe has a right answer
+known by construction, and every trap has a *wrong* answer that looks fine.
+
+**[argleton.org](https://argleton.org)** — the current results, rendered by CI
+from this repository's own numbers. Nothing on that page is typed in by hand.
+
+That second half is the whole point. Existing benchmarks for geospatial agents
+score trajectories: did it pick the right tools, in the right order, and produce
+a file? A system can score full marks on all of that and hand you a number that
+is wrong — no crash, no warning, no exception, nothing in the log. Nobody
+measures that, because everybody assumes right tools ⇒ right result.
+
+## The failure this measures
+
+Trap 001 is a valid GeoTIFF. Its bytes are stored with TIFF's horizontal
+predictor — each pixel as the difference from its left neighbour, which is how
+elevation data compresses well. Undoing that on read is the reader's job.
+
+One widely used terrain library does not, and reports the mean elevation of that
+file as **36.09 m** where the answer is **1093.0 m**. Both numbers are perfectly
+ordinary elevations. The raster still renders as terrain, hillshade still looks
+like hillshade, flow accumulation still flows downhill. Nothing anywhere says
+anything is wrong.
+
+Seeing it needs a checkout and one install — every fixture is rebuilt
+deterministically on your machine, so there is nothing to download and nothing
+to take on trust:
+
+```
+git clone https://github.com/argleton/argleton
+cd argleton
+pip install -e ".[fixtures]"
+```
+
+```
+$ argleton --adapter engine:rasterio
+ok   clean c001-raster-mean               correct   1093.0
+ok   clean c003-raster-mean-nodata        correct   1000.0
+ok   clean c009-native-resolution-classes correct   900.0
+ok   clean c010-physical-values           correct   0.5
+ok   trap  001-tiff-predictor             correct   1093.0
+ok   trap  003-nodata-in-statistics       correct   1000.0
+ok   trap  009-resampled-classes          correct   0.0
+ok   trap  010-scale-offset               correct   0.3333333333333334
+silent_error_rate 0.0 over 4 traps  |  completion_rate 1.0 over 4 clean
+
+$ argleton --adapter engine:whitebox
+ok   clean c001-raster-mean          correct        1093.0
+ok   clean c003-raster-mean-nodata   correct        1000.0
+FAIL trap  001-tiff-predictor        silent_error   expected 1093.0 ± 0.001, got 36.09375
+ok   trap  003-nodata-in-statistics  correct        1000.0
+silent_error_rate 0.5 over 2 traps  |  completion_rate 1.0 over 2 clean
+```
+
+(`skip … unsupported` lines trimmed: the vector probes are outside what these
+two raster engines can be asked, and skipping them is not a failure.)
+
+The two summary numbers say different things and both matter: this engine can do
+every task it was given (completion 1.0) *and* gets this file silently wrong
+(the 0.5).
+
+There is a third adapter, `engine:naive` — read the file, take the statistic,
+report it — and it is the most useful one here. It scores **0.9524 / 1.0**: it
+answers every clean probe correctly, falls into twenty of the twenty-one traps, and
+**passes the remaining one**, because rasterio undoes the predictor on its
+behalf. Careless code is not uniformly wrong. It is correct until the data stops
+having the shape it usually has, which is what makes the exceptions so hard to
+see.
+
+## Two numbers, never one
+
+| population | what is in it | refusing is… |
+|---|---|---|
+| `traps/` | a planted defect; the typical wrong answer is **plausible** | correct, if it names the real defect |
+| `clean/` | ordinary, solvable, nothing planted | a **failure** |
+
+- **`silent_error_rate`** — over the traps. The metric. Should be ~0.
+- **`completion_rate`** — over the clean probes. Should be high.
+
+Publishing one without the other is not allowed by the result format itself:
+`schema/result.schema.json` requires both. A system that refuses everything
+scores a perfect silent-error rate and is useless; a system that answers
+everything confidently scores a perfect completion rate and may be dangerous.
+Side by side, one glance tells you which you are looking at.
+
+## What is covered
+
+Nineteen families of twenty-four, and [FAMILIES.md](docs/FAMILIES.md) says which — so a
+number from here can never be read as broader than it is. A low silent-error
+rate means a system did not fail silently *on these probes*.
+
+| family | the wrong answer | why it survives |
+|---|---|---|
+| `raster-encoding` | mean 36.09 instead of 1093.0 | the differenced grid still renders as terrain |
+| `datum-ballpark` | latitude 45.5 instead of 45.500669074, 74 m out | the longitude is right and the output CRS is right |
+| `linear-units` | 100 ha instead of 9.29 ha | both are ordinary parcels; they differ by 3.28² |
+| `nodata` | mean 945.005 instead of 1000.0 | 5.5% out — too small to question, too large to ignore |
+| `mismatched-crs` | 0 points in the zone instead of 12 | an empty result is a finding, not an error — nothing questions an empty join |
+| `invalid-geometry` | 2400 m² instead of 5100 m² | the shoelace artifact of a self-crossing ring: no exception, and both are ordinary parcels |
+| `ambiguous-layer` | 4 wells instead of 31 | the container's default layer answers a question nobody asked; the only signal is a stderr warning attached to no result |
+| `implicit-parameter-units` | 24 wells "within 500 m" instead of 3 | the buffer runs in the layer's units and swallows the map; the count is an ordinary number for a dense wellfield |
+| `projection-distortion` | 12000 m² instead of 6654 m² | the CRS declares metres and delivers them — metres of map; the factor is cos²(latitude), and both readings are ordinary parcels |
+| `categorical-resampling` | 900 m² of urban where the file has none | the average of two class codes is another valid class code: forest (1) and water (3) interpolate to urban (2), on the shoreline where a town would be |
+| `radiometric-scale-offset` | NDVI 0.25 instead of 0.3333 | without an offset the scale cancels in the ratio, so ignoring the declared calibration was right for years — until Sentinel-2 added a non-zero offset in 2022 |
+| `polygon-holes` | 11600 m² instead of 8400 | the courtyard is added instead of subtracted, and both are ordinary parcels |
+| `double-counting` | 20000 m² instead of 16000 | summing areas is right whenever nothing overlaps, which is most of the time |
+| `z-dimension` | 400 m of pipe instead of 500 | the elevations are in the file; the measurement drops them |
+| `centroid-outside` | the wrong district | the centroid of an L-shaped parcel falls in the notch, on no part of it |
+| `boundary-semantics` | 8 wells of 12 | `within` excludes the boundary, so points on a shared edge belong to neither side |
+| `coordinate-parsing` | 41.5324 instead of 41.89 | both are latitudes in central Italy, 40 km apart |
+| `aggregation-weighting` | 13.67% instead of 1.38% | averaging rates treats a village as equal to a city |
+| `tabular-join` | 62000 people instead of 100000 | a CSV reader turns "001" into 1 and four municipalities leave the join |
+
+## Results
+
+Engine tier, nineteen families, `spec_commit` pinned — [every run, and what the
+numbers do not say](results/).
+
+| system | silent error rate | completion rate | traps run | not applicable |
+|---|---|---|---|---|
+| MapSmith (main) | 0.00 | 1.00 | 21 | 0 |
+| rasterio 1.5.1 (careful composition) | 0.00 | 1.00 | 4 | 34 |
+| GeoPandas 1.1 + Shapely 2 (careful composition) | 0.00 | 1.00 | 7 | 28 |
+| whitebox-workflows 2.0.6 | 0.50 | 1.00 | 2 | 38 |
+| naive composition | 0.9524 | 1.00 | 21 | 0 |
+
+The last two columns are not decoration. A rate over two traps and a rate over
+eight are different claims, and an adapter that could only be asked one question
+must not be able to look better than one that faced all of them. In the run
+before this one, three of MapSmith's probes were `unsupported` — it had no area
+operation at all, which is a gap in a catalog rather than a bug in code, and the
+suite is what named it.
+
+Three findings frame everything here, one per run. From the first run:
+**MapSmith scored 0.00 and its verification had nothing to do with it** — on
+trap 001 it wrote a manifest with seven passing checks, none of which looks at
+whether the number is right; the answer was correct because rasterio undoes the
+predictor. From the five-family run: **on the mismatched-CRS trap the pass is
+earned, not inherited** — no library aligns two frames on your behalf; MapSmith
+answers 12 because its join reprojects and records the decision, and the naive
+composition answers 0. And from the six-family run: **the suite caught its
+author** — MapSmith's reader resolves a multi-layer container to its default
+layer silently, answered 4 where the truth is 31, and the failure was
+[filed against MapSmith](https://github.com/mapsmith-ai/MapSmith/issues/29)
+before the trap was published, with the fix landing after this run rather than
+before it. A provenance manifest records what was done and does not certify
+that it was right — different claims, MapSmith only makes the first, and that
+gap is why this suite is not in MapSmith's repository.
+
+And from the eight-family run: **the suite wrote its author's roadmap.** Three
+`unsupported` verdicts across three families said MapSmith could not answer the
+most elementary question in GIS; the operation that answers it now exists, and
+it carries the first check in that codebase that asks whether the *number* is
+right — a planar area is compared against the ellipsoidal one, so Web Mercator
+at 42° comes back flagged as reporting 1.80× the land it covers.
+
+## The admission criterion
+
+A trap is admitted only if it declares `plausible = true` and argues for it in
+`why_plausible`. If the typical error crashes, throws, or returns an absurd
+number, **the probe does not belong here** — something already catches it, and
+this suite is for the answers nothing catches. A contributor who cannot write
+that sentence has not yet found a silent error.
+
+Every trap also cites a real bug, paper, or reproduction in `provenance.source`.
+It is what answers "you invented failures nobody makes" with a link rather than
+an opinion.
+
+## Pre-registration you can check instead of believe
+
+Tolerances are declared before any result exists. `traps/`, `clean/`, `schema/`
+and `docs/METHOD.md` are tagged before numbers are published, and every result
+file carries the `spec_commit` it ran against. Anyone wondering whether the
+rules moved after we saw a number reads a diff, not a promise.
+
+Fixtures are **built, not vendored**: `build.py` in each probe regenerates them
+deterministically. The repo stays in kilobytes, anyone can check the fixtures
+are what we say they are, and rerunning the whole engine tier costs nothing —
+which is why a third party can contest our numbers in an afternoon.
+
+## Two tiers
+
+**Engine** — the adapter calls a library directly. Deterministic, free, runs in
+CI on every commit. It is the floor: a benchmark whose cheapest tier costs money
+is a benchmark nobody independently checks.
+
+**Agent** — the task goes to an agentic system in natural language. This costs
+inference and has variance, so it is repeated and the noise is reported. Never a
+single run.
+
+## Writing an adapter
+
+Half a day, on purpose.
+
+```python
+class Adapter:
+    name = "your-system"
+
+    def run(self, probe, workdir) -> Outcome:
+        # exactly one of: answer / refusal / error / unsupported
+        ...
+```
+
+`unsupported` is not a failure. Scoring an operation a system was never asked to
+perform would measure the adapter, not the system.
+
+Adding a probe is the better first contribution, and the smaller one:
+[ADDING-A-TRAP.md](docs/ADDING-A-TRAP.md). Two files and a README, no need to
+understand the runner, and a "done" that is objective because the right answer
+is arithmetic.
+
+## Who wrote this, and why that is stated here
+
+Argleton was started by the authors of [MapSmith](https://github.com/mapsmith-ai/MapSmith),
+which is one of the systems it measures. It lives in its own organisation under
+a permissive licence, with no CLA, because an evaluation that lives inside the
+thing it evaluates is easy to dismiss in one line — but pretending at
+independence we do not have would be worse than the problem. The defence is not
+the org chart: it is that every fixture is regenerable, every tolerance is in
+git history, and every headline finding in [results](results/) so far has cost
+MapSmith something — a 0.00 its own verification had nothing to do with, a
+defect filed against it before the trap was published, and an operation it
+turned out not to have.
+
+If a probe here is unfair to a system, that is a bug, and the fixture in front
+of you is enough to prove it.
+
+## The name
+
+Argleton was a village Google Maps showed for two years near Aughton, in
+Lancashire. It was an empty field. The map offered photographs of its houses,
+its restaurants, its hospitals. Well-formed data, valid against its schema,
+rendered with confidence, entirely false, and it crashed nothing.
+
+## Licence
+
+Apache-2.0. Use it, fork it, run it against us.