Spatial-Gym 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/.github/workflows/publish.yml +1 -1
- spatial_gym-0.1.6/CITATION.cff +60 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/NOTES.md +25 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/PKG-INFO +37 -2
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/README.md +35 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/environments/spatial_gym/pyproject.toml +1 -1
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/environments/spatial_gym/spatial_gym_env/prompts.py +17 -17
- spatial_gym-0.1.6/environments/spatial_gym_openenv/.dockerignore +5 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/.gitignore +4 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/README.md +133 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/__init__.py +17 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/assets/spatial-gym-playground.png +0 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/assets/spatial-gym-solve.gif +0 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/client.py +49 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/models.py +112 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/openenv.yaml +22 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/pyproject.toml +40 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/Dockerfile +87 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/__init__.py +5 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/app.py +54 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/board_svg.py +157 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/prompts.py +216 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/requirements.txt +6 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/server/spatial_gym_environment.py +277 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/tests/test_environment.py +123 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/tests/test_server.py +56 -0
- spatial_gym-0.1.6/environments/spatial_gym_openenv/uv.lock +4175 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/pyproject.toml +1 -1
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/.github/workflows/tests.yml +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/.gitignore +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Final_Product.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/LICENCE +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/MIGRATION_NOTES.md +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Spatial_Gym/Spatial_Gym.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Spatial_Gym/__init__.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Spatial_Gym/register_env.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Spatial_Gym/render/__init__.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Spatial_Gym/render/human_renderer.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/Spatial_Gym/render/llm_renderer.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/TEST_SUMMARY.md +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/environments/spatial_gym/README.md +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/environments/spatial_gym/spatial_gym_env/__init__.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/environments/spatial_gym/spatial_gym_env/taskset.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/human_play.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/llm_testing/llm_host.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/llm_testing/parse_logs.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/pytest.ini +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/tests/README.md +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/tests/__init__.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/tests/test_environment.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/tests/test_predefined_paths.py +0 -0
- {spatial_gym-0.1.4 → spatial_gym-0.1.6}/tests/test_random_agent.py +0 -0
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use Spatial-Gym, please cite the paper below."
|
|
3
|
+
title: "Spatial-Gym"
|
|
4
|
+
type: software
|
|
5
|
+
license: MIT
|
|
6
|
+
authors:
|
|
7
|
+
- family-names: Kaesberg
|
|
8
|
+
given-names: Lars Benedikt
|
|
9
|
+
- family-names: Yang
|
|
10
|
+
given-names: Tianyu
|
|
11
|
+
- family-names: Bauer
|
|
12
|
+
given-names: Niklas
|
|
13
|
+
- family-names: Ruas
|
|
14
|
+
given-names: Terry
|
|
15
|
+
- family-names: Wahle
|
|
16
|
+
given-names: Jan Philip
|
|
17
|
+
- family-names: Gipp
|
|
18
|
+
given-names: Bela
|
|
19
|
+
repository-code: "https://github.com/lkaesberg/Spatial-Gym"
|
|
20
|
+
url: "https://sparc.gipplab.org/"
|
|
21
|
+
keywords:
|
|
22
|
+
- spatial reasoning
|
|
23
|
+
- gymnasium
|
|
24
|
+
- reinforcement learning
|
|
25
|
+
- large language models
|
|
26
|
+
preferred-citation:
|
|
27
|
+
type: article
|
|
28
|
+
title: "Mind the Gap Between Spatial Reasoning and Acting! Step-by-Step Evaluation of Agents With Spatial-Gym"
|
|
29
|
+
authors:
|
|
30
|
+
- family-names: Kaesberg
|
|
31
|
+
given-names: Lars Benedikt
|
|
32
|
+
- family-names: Yang
|
|
33
|
+
given-names: Tianyu
|
|
34
|
+
- family-names: Bauer
|
|
35
|
+
given-names: Niklas
|
|
36
|
+
- family-names: Ruas
|
|
37
|
+
given-names: Terry
|
|
38
|
+
- family-names: Wahle
|
|
39
|
+
given-names: Jan Philip
|
|
40
|
+
- family-names: Gipp
|
|
41
|
+
given-names: Bela
|
|
42
|
+
year: 2026
|
|
43
|
+
journal: "arXiv preprint"
|
|
44
|
+
doi: "10.48550/arXiv.2604.09338"
|
|
45
|
+
url: "https://arxiv.org/abs/2604.09338"
|
|
46
|
+
references:
|
|
47
|
+
- type: conference-paper
|
|
48
|
+
title: "SPaRC: A Spatial Pathfinding Reasoning Challenge"
|
|
49
|
+
authors:
|
|
50
|
+
- family-names: Kaesberg
|
|
51
|
+
given-names: Lars Benedikt
|
|
52
|
+
- family-names: Wahle
|
|
53
|
+
given-names: Jan Philip
|
|
54
|
+
- family-names: Ruas
|
|
55
|
+
given-names: Terry
|
|
56
|
+
- family-names: Gipp
|
|
57
|
+
given-names: Bela
|
|
58
|
+
collection-title: "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing"
|
|
59
|
+
year: 2025
|
|
60
|
+
doi: "10.18653/v1/2025.emnlp-main.526"
|
|
@@ -54,6 +54,31 @@ Both branches are pushed to `origin`. No PRs were opened.
|
|
|
54
54
|
`SpatialGymTaskset(SpatialGymConfig(backtracking=...))`, and both build 500 tasks. The env also works with the
|
|
55
55
|
PyPI release `Spatial-Gym==0.1.1` (checked in a fresh venv), which is what Hub users will install.
|
|
56
56
|
|
|
57
|
+
## OpenEnv environment (2026-10-14)
|
|
58
|
+
|
|
59
|
+
`environments/spatial_gym_openenv/` is a hand-written OpenEnv environment, scaffolded with `openenv init`
|
|
60
|
+
(openenv 0.8.0). `openenv import` can wrap verifiers environments now, but it turns them into MCP-style tool
|
|
61
|
+
actions. A native environment keeps Spatial-Gym's own action space: one move per step.
|
|
62
|
+
|
|
63
|
+
- Same prompts and parser as the verifiers environment (`server/prompts.py` copies its step prompts; the move
|
|
64
|
+
parser sits in `models.py`, because clients use it too), same
|
|
65
|
+
deep-copied env templates, same reward (1.0 only when `outcome_reward == 1`).
|
|
66
|
+
- `reset()` options: `puzzle_id`, `difficulty`, `seed`, `split` (`test` or `train`), `backtracking`,
|
|
67
|
+
`max_steps` (default 100). Unknown options raise, because the OpenEnv server would otherwise drop them quietly.
|
|
68
|
+
- The package is `spatial_gym_openenv`, not `spatial_gym`: on macOS's case-insensitive file system a
|
|
69
|
+
`spatial_gym` folder would clash with `Spatial_Gym` in site-packages. `env_name` in `server/app.py` is the
|
|
70
|
+
same name, because the web UI builds its Quick Start import line from it.
|
|
71
|
+
- The dataset is not pinned to a revision, because `Spatial_Gym.__init__` loads the latest version itself.
|
|
72
|
+
- Checks: 9 pytest tests (gold solutions solve one puzzle per level, wrong lines score 0, backtracking, reset
|
|
73
|
+
options, independent sessions, a WebSocket end-to-end run). `openenv validate` passes at the static level and
|
|
74
|
+
against a running server. `openenv build` makes a 1.44 GB image; the container solves a level-5 puzzle in 46
|
|
75
|
+
moves for reward 1.0 and passes `openenv validate --url`.
|
|
76
|
+
- Hugging Face needs a PRO plan for Docker Spaces on a personal account (402 on `openenv push`), so the env is
|
|
77
|
+
also proposed to OpenEnv itself as `envs/spatial_gym_env` (openenv/OpenEnv#1430, merged 2026-11-12, typed
|
|
78
|
+
actions only; maintainers want an issue first before MCP tools are added); merged envs get deployed
|
|
79
|
+
under the `openenv` org. Both copies have the same code layout: the server's `main()` parses its own
|
|
80
|
+
arguments, as OpenEnv's `tests/envs/test_server_main_args.py` requires.
|
|
81
|
+
|
|
57
82
|
## Reproduced numbers
|
|
58
83
|
|
|
59
84
|
`vf-eval` runs on 2026-10-13 with `openai/gpt-oss-120b` through OpenRouter, step-by-step mode, level-1 puzzles
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: Spatial-Gym
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: A Gymnasium environment for benchmarking spatial reasoning capabilities of AI agents on grid-based puzzles
|
|
5
5
|
Project-URL: Repository, https://github.com/lkaesberg/Spatial-Gym
|
|
6
6
|
Project-URL: Homepage, https://github.com/lkaesberg/Spatial-Gym
|
|
@@ -66,6 +66,7 @@ Description-Content-Type: text/markdown
|
|
|
66
66
|
|
|
67
67
|
[](https://arxiv.org/abs/2604.09338)
|
|
68
68
|
[](https://pypi.org/project/Spatial-Gym/)
|
|
69
|
+
[](https://towardsdatascience.com/spatial-reasoning-in-ai-models-taken-apart/)
|
|
69
70
|
[](https://github.com/lkaesberg/Spatial-Gym/actions/workflows/tests.yml)
|
|
70
71
|
[](https://opensource.org/licenses/MIT)
|
|
71
72
|
[](https://www.python.org/downloads/)
|
|
@@ -282,6 +283,24 @@ uv pip install -e environments/spatial_gym
|
|
|
282
283
|
vf-eval spatial-gym-env -m <model> --client.base-url <url> --client.api-key-var <ENV_VAR> -n 20 --no-push
|
|
283
284
|
```
|
|
284
285
|
|
|
286
|
+
### OpenEnv environment
|
|
287
|
+
|
|
288
|
+
Spatial-Gym is part of [OpenEnv](https://github.com/openenv/OpenEnv), the environment standard from Meta and
|
|
289
|
+
Hugging Face, as [`envs/spatial_gym_env`](https://github.com/openenv/OpenEnv/tree/main/envs/spatial_gym_env).
|
|
290
|
+
One episode is one puzzle, solved one move per step with the paper's prompts, and the reward is 1.0 for a correct
|
|
291
|
+
path and 0.0 otherwise. Install it from OpenEnv:
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
pip install "git+https://github.com/openenv/OpenEnv.git#subdirectory=envs/spatial_gym_env"
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
This repository also keeps a copy in `environments/spatial_gym_openenv/`, which runs as a FastAPI server or a
|
|
298
|
+
Hugging Face Space. See [environments/spatial_gym_openenv/README.md](environments/spatial_gym_openenv/README.md).
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
cd environments/spatial_gym_openenv && uv run --project . server
|
|
302
|
+
```
|
|
303
|
+
|
|
285
304
|
## Repository Structure
|
|
286
305
|
|
|
287
306
|
```
|
|
@@ -293,6 +312,9 @@ Spatial-Gym/
|
|
|
293
312
|
│ └── render/ # Rendering modules
|
|
294
313
|
│ ├── human_renderer.py
|
|
295
314
|
│ └── llm_renderer.py
|
|
315
|
+
├── environments/
|
|
316
|
+
│ ├── spatial_gym/ # Prime Intellect verifiers environment
|
|
317
|
+
│ └── spatial_gym_openenv/ # OpenEnv environment (server, client, Dockerfile)
|
|
296
318
|
├── llm_testing/ # LLM evaluation utilities
|
|
297
319
|
│ ├── llm_host.py # LLM interaction wrapper
|
|
298
320
|
│ └── parse_logs.py # Result analysis tools
|
|
@@ -375,6 +397,19 @@ Spatial-Gym is built on the SPaRC puzzles. Please also cite:
|
|
|
375
397
|
}
|
|
376
398
|
```
|
|
377
399
|
|
|
400
|
+
If you use the visual scaffolds on the same puzzles, please also cite:
|
|
401
|
+
|
|
402
|
+
```bibtex
|
|
403
|
+
@misc{kaesberg2026visualprompting,
|
|
404
|
+
title = {Is Visual Prompting All You Need? Studying VLM Spatial Reasoning under Progressive Visual Scaffolds},
|
|
405
|
+
author = {Kaesberg, Lars Benedikt and Yang, Tianyu and Wunderlich, Florian Valentin and Ruas, Terry and Kurzawe, Daniel and Wahle, Jan Philip and Gipp, Bela},
|
|
406
|
+
year = {2026},
|
|
407
|
+
eprint = {2608.21170},
|
|
408
|
+
archivePrefix = {arXiv},
|
|
409
|
+
note = {Accepted at Findings of EMNLP 2026}
|
|
410
|
+
}
|
|
411
|
+
```
|
|
412
|
+
|
|
378
413
|
## Related work from our group
|
|
379
414
|
|
|
380
415
|
| Paper | Venue | Links |
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
[](https://arxiv.org/abs/2604.09338)
|
|
8
8
|
[](https://pypi.org/project/Spatial-Gym/)
|
|
9
|
+
[](https://towardsdatascience.com/spatial-reasoning-in-ai-models-taken-apart/)
|
|
9
10
|
[](https://github.com/lkaesberg/Spatial-Gym/actions/workflows/tests.yml)
|
|
10
11
|
[](https://opensource.org/licenses/MIT)
|
|
11
12
|
[](https://www.python.org/downloads/)
|
|
@@ -222,6 +223,24 @@ uv pip install -e environments/spatial_gym
|
|
|
222
223
|
vf-eval spatial-gym-env -m <model> --client.base-url <url> --client.api-key-var <ENV_VAR> -n 20 --no-push
|
|
223
224
|
```
|
|
224
225
|
|
|
226
|
+
### OpenEnv environment
|
|
227
|
+
|
|
228
|
+
Spatial-Gym is part of [OpenEnv](https://github.com/openenv/OpenEnv), the environment standard from Meta and
|
|
229
|
+
Hugging Face, as [`envs/spatial_gym_env`](https://github.com/openenv/OpenEnv/tree/main/envs/spatial_gym_env).
|
|
230
|
+
One episode is one puzzle, solved one move per step with the paper's prompts, and the reward is 1.0 for a correct
|
|
231
|
+
path and 0.0 otherwise. Install it from OpenEnv:
|
|
232
|
+
|
|
233
|
+
```bash
|
|
234
|
+
pip install "git+https://github.com/openenv/OpenEnv.git#subdirectory=envs/spatial_gym_env"
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
This repository also keeps a copy in `environments/spatial_gym_openenv/`, which runs as a FastAPI server or a
|
|
238
|
+
Hugging Face Space. See [environments/spatial_gym_openenv/README.md](environments/spatial_gym_openenv/README.md).
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
cd environments/spatial_gym_openenv && uv run --project . server
|
|
242
|
+
```
|
|
243
|
+
|
|
225
244
|
## Repository Structure
|
|
226
245
|
|
|
227
246
|
```
|
|
@@ -233,6 +252,9 @@ Spatial-Gym/
|
|
|
233
252
|
│ └── render/ # Rendering modules
|
|
234
253
|
│ ├── human_renderer.py
|
|
235
254
|
│ └── llm_renderer.py
|
|
255
|
+
├── environments/
|
|
256
|
+
│ ├── spatial_gym/ # Prime Intellect verifiers environment
|
|
257
|
+
│ └── spatial_gym_openenv/ # OpenEnv environment (server, client, Dockerfile)
|
|
236
258
|
├── llm_testing/ # LLM evaluation utilities
|
|
237
259
|
│ ├── llm_host.py # LLM interaction wrapper
|
|
238
260
|
│ └── parse_logs.py # Result analysis tools
|
|
@@ -315,6 +337,19 @@ Spatial-Gym is built on the SPaRC puzzles. Please also cite:
|
|
|
315
337
|
}
|
|
316
338
|
```
|
|
317
339
|
|
|
340
|
+
If you use the visual scaffolds on the same puzzles, please also cite:
|
|
341
|
+
|
|
342
|
+
```bibtex
|
|
343
|
+
@misc{kaesberg2026visualprompting,
|
|
344
|
+
title = {Is Visual Prompting All You Need? Studying VLM Spatial Reasoning under Progressive Visual Scaffolds},
|
|
345
|
+
author = {Kaesberg, Lars Benedikt and Yang, Tianyu and Wunderlich, Florian Valentin and Ruas, Terry and Kurzawe, Daniel and Wahle, Jan Philip and Gipp, Bela},
|
|
346
|
+
year = {2026},
|
|
347
|
+
eprint = {2608.21170},
|
|
348
|
+
archivePrefix = {arXiv},
|
|
349
|
+
note = {Accepted at Findings of EMNLP 2026}
|
|
350
|
+
}
|
|
351
|
+
```
|
|
352
|
+
|
|
318
353
|
## Related work from our group
|
|
319
354
|
|
|
320
355
|
| Paper | Venue | Links |
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Prompts and answer parsers for the Spatial-Gym verifiers environment.
|
|
2
2
|
|
|
3
3
|
Copied from the SPaRC reference code, so the environment asks the same
|
|
4
|
-
questions the papers did (https://github.com/lkaesberg/SPaRC/tree/
|
|
4
|
+
questions the papers did (https://github.com/lkaesberg/SPaRC/tree/v0.7.0):
|
|
5
5
|
|
|
6
6
|
* sparc/prompts/gym_step.py -> step_prompt
|
|
7
7
|
* sparc/prompts/gym_step_traceback.py -> step_backtrack_prompt
|
|
@@ -9,7 +9,9 @@ questions the papers did (https://github.com/lkaesberg/SPaRC/tree/9f2a3d6):
|
|
|
9
9
|
* sparc/process_puzzle.py -> STEP_USER_TEMPLATE, parse_action
|
|
10
10
|
* sparc/validation.py -> extract_solution_path
|
|
11
11
|
|
|
12
|
-
The text is unchanged
|
|
12
|
+
The text is unchanged, except that the star, polyshape and negative polyshape rules are
|
|
13
|
+
stated the way the puzzles were built, as in SPaRC 0.7.0. The papers' wording is kept in
|
|
14
|
+
SPaRC's *_paper prompt modules.
|
|
13
15
|
"""
|
|
14
16
|
|
|
15
17
|
import json
|
|
@@ -90,13 +92,12 @@ def step_prompt(puzzle_data: Dict) -> Dict:
|
|
|
90
92
|
Squares: All squares within a single region MUST be the same color. Squares of different colors MUST be separated into different regions by the path.
|
|
91
93
|
Stars: Within a single region, each star symbol MUST be paired with exactly ONE other element of the same color. Other colors within the region are irrelevant to this specific star's rule.
|
|
92
94
|
|
|
93
|
-
Polyshapes(poly): The region containing this symbol MUST
|
|
94
|
-
If multiple positive polyshapes are in one region,
|
|
95
|
+
Polyshapes(poly): The region containing this symbol MUST have exactly the specified shape (defined in Polyshape Definitions): the shape fills the whole region, with no cells left over.
|
|
96
|
+
If multiple positive polyshapes are in one region, their shapes together must fill the region exactly, without overlapping. Rotation of polyshapes is NOT allowed. They must fit in their given orientation. The color of a polyshape does not matter for this rule.
|
|
95
97
|
|
|
96
|
-
Negative Polyshapes(ylop): These subtract
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
Negative shapes can sometimes rationalize apparent overlaps or boundary violations of positive shapes if interpreted as cancellations.
|
|
98
|
+
Negative Polyshapes(ylop): These subtract area from the positive polyshapes in the same region.
|
|
99
|
+
The positive shapes, minus the cells covered by the negative shapes, must fill the region exactly, so the region's size is the total size of the positive shapes minus the total size of the negative shapes.
|
|
100
|
+
A negative shape does not have to match the shape or color of any positive shape. If the negative shapes cancel the positive shapes completely, the region has no shape constraint.
|
|
100
101
|
|
|
101
102
|
3. Path-Based Rules (Edge Touching):
|
|
102
103
|
Triangles: The path MUST touch a specific number of edges of the cell containing the triangle symbol.
|
|
@@ -184,13 +185,12 @@ def step_backtrack_prompt(puzzle_data: Dict) -> Dict:
|
|
|
184
185
|
Squares: All squares within a single region MUST be the same color. Squares of different colors MUST be separated into different regions by the path.
|
|
185
186
|
Stars: Within a single region, each star symbol MUST be paired with exactly ONE other element of the same color. Other colors within the region are irrelevant to this specific star's rule.
|
|
186
187
|
|
|
187
|
-
Polyshapes(poly): The region containing this symbol MUST
|
|
188
|
-
If multiple positive polyshapes are in one region,
|
|
188
|
+
Polyshapes(poly): The region containing this symbol MUST have exactly the specified shape (defined in Polyshape Definitions): the shape fills the whole region, with no cells left over.
|
|
189
|
+
If multiple positive polyshapes are in one region, their shapes together must fill the region exactly, without overlapping. Rotation of polyshapes is NOT allowed. They must fit in their given orientation. The color of a polyshape does not matter for this rule.
|
|
189
190
|
|
|
190
|
-
Negative Polyshapes(ylop): These subtract
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
Negative shapes can sometimes rationalize apparent overlaps or boundary violations of positive shapes if interpreted as cancellations.
|
|
191
|
+
Negative Polyshapes(ylop): These subtract area from the positive polyshapes in the same region.
|
|
192
|
+
The positive shapes, minus the cells covered by the negative shapes, must fill the region exactly, so the region's size is the total size of the positive shapes minus the total size of the negative shapes.
|
|
193
|
+
A negative shape does not have to match the shape or color of any positive shape. If the negative shapes cancel the positive shapes completely, the region has no shape constraint.
|
|
194
194
|
|
|
195
195
|
3. Path-Based Rules (Edge Touching):
|
|
196
196
|
Triangles: The path MUST touch a specific number of edges of the cell containing the triangle symbol.
|
|
@@ -282,9 +282,9 @@ def one_shot_prompt(puzzle_data: Dict) -> Dict:
|
|
|
282
282
|
|
|
283
283
|
2. **Region-Based Rules** (Apply to areas enclosed by the path):
|
|
284
284
|
* **Squares (`o-X`):** All squares within a single region **MUST** be the same color. Squares of different colors **MUST** be separated into different regions by the path.
|
|
285
|
-
* **Stars (`*-X`):** Within a single region, each star symbol **MUST** be paired with exactly **ONE** other
|
|
286
|
-
* **Polyshapes (`P-X-Y`):** The region containing this symbol **MUST**
|
|
287
|
-
* **Negative Polyshapes (`Y-X-Y`):** These
|
|
285
|
+
* **Stars (`*-X`):** Within a single region, each star symbol **MUST** be paired with exactly **ONE** other symbol *of the same color*. Any rule symbol can be the partner: a star, square, triangle, polyshape or negative polyshape. Other colors within the region are irrelevant to this specific star's rule.
|
|
286
|
+
* **Polyshapes (`P-X-Y`):** The region containing this symbol **MUST** have exactly the specified shape (defined in Polyshape Definitions): the shape fills the whole region, with no cells left over. If multiple positive polyshapes are in one region, their shapes together must fill the region exactly, without overlapping. Rotation of polyshapes is NOT allowed. They must fit in their given orientation. The color of a polyshape does not matter for this rule.
|
|
287
|
+
* **Negative Polyshapes (`Y-X-Y`):** These subtract area from the positive polyshapes in the same region. The positive shapes, minus the cells covered by the negative shapes, must fill the region exactly, so the region's size is the total size of the positive shapes minus the total size of the negative shapes. A negative shape does not have to match the shape or color of any positive shape. If the negative shapes cancel the positive shapes completely, the region has no shape constraint.
|
|
288
288
|
|
|
289
289
|
3. **Path-Based Rules (Edge Touching):**
|
|
290
290
|
* **Triangles (`A-X`, `B-X`, `C-X`, `D-X`):** The path **MUST** touch a specific number of edges of the cell containing the triangle symbol.
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Spatial-Gym
|
|
3
|
+
emoji: 🧩
|
|
4
|
+
colorFrom: green
|
|
5
|
+
colorTo: gray
|
|
6
|
+
sdk: docker
|
|
7
|
+
pinned: false
|
|
8
|
+
app_port: 8000
|
|
9
|
+
base_path: /web
|
|
10
|
+
license: mit
|
|
11
|
+
short_description: SPaRC grid pathfinding puzzles, one move per step
|
|
12
|
+
tags:
|
|
13
|
+
- openenv
|
|
14
|
+
- reinforcement-learning
|
|
15
|
+
- spatial-reasoning
|
|
16
|
+
- puzzles
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
# Spatial-Gym for OpenEnv
|
|
20
|
+
|
|
21
|
+
[Spatial-Gym](https://github.com/lkaesberg/Spatial-Gym) turns the [SPaRC](https://sparc.gipplab.org/) grid puzzles into a step-by-step environment. The agent draws one line from the start node to the exit, one move at a time, and the line has to follow every rule on the board. In the [Spatial-Gym paper](https://arxiv.org/abs/2604.09338), humans solve 98.0% of the test puzzles, and the best model, GPT-OSS 120B, solves 16.0%.
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
*A level-5 puzzle solved move by move: the line starts at the big circle and has to reach the exit at the bottom left while following every rule on the board.*
|
|
26
|
+
|
|
27
|
+
This folder serves Spatial-Gym through [OpenEnv](https://github.com/openenv/OpenEnv), so it works with any OpenEnv client or trainer. The same environment is also part of OpenEnv itself, as [`envs/spatial_gym_env`](https://github.com/openenv/OpenEnv/tree/main/envs/spatial_gym_env). It uses the paper's prompts and scoring. The polyshape and negative polyshape rules in the prompts are stated the way the puzzles were built, which the paper's prompts worded differently.
|
|
28
|
+
|
|
29
|
+
## One episode
|
|
30
|
+
|
|
31
|
+
- `reset()` loads one puzzle. The observation holds the board, the legal moves, the paper's rules prompt (`system_prompt`) and the paper's message for the first move (`prompt`).
|
|
32
|
+
- `step(SpatialGymAction(action=...))` makes one move: `0`=RIGHT, `1`=UP, `2`=LEFT, `3`=DOWN. A move that is not legal leaves the line where it is but still counts as a step.
|
|
33
|
+
- The episode ends when the line reaches the exit, when no legal move is left, or after `max_steps` moves.
|
|
34
|
+
- The reward is `1.0` when the episode ends with a line that exactly matches one of the puzzle's stored solutions, and `0.0` otherwise. `metadata["rule_status"]` shows which rules the line meets so far.
|
|
35
|
+
|
|
36
|
+

|
|
37
|
+
|
|
38
|
+
*The `/web` playground mid-episode: the board with the line so far, and one button per legal move.*
|
|
39
|
+
|
|
40
|
+
## Quick start
|
|
41
|
+
|
|
42
|
+
Install the client:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install "git+https://github.com/lkaesberg/Spatial-Gym.git#subdirectory=environments/spatial_gym_openenv"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Connect to a running server, then let a model play one puzzle:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from spatial_gym_openenv import SpatialGymAction, SpatialGymEnv
|
|
52
|
+
|
|
53
|
+
with SpatialGymEnv(base_url="http://localhost:8000").sync() as env:
|
|
54
|
+
result = env.reset(difficulty=2, seed=0)
|
|
55
|
+
messages = [
|
|
56
|
+
{"role": "system", "content": result.observation.system_prompt},
|
|
57
|
+
{"role": "user", "content": result.observation.prompt},
|
|
58
|
+
]
|
|
59
|
+
while not result.done:
|
|
60
|
+
reply = my_model(messages) # your LLM call; the reply ends with "Final: <action>"
|
|
61
|
+
messages.append({"role": "assistant", "content": reply})
|
|
62
|
+
result = env.step(SpatialGymAction.from_reply(reply))
|
|
63
|
+
if not result.done:
|
|
64
|
+
messages.append({"role": "user", "content": result.observation.prompt})
|
|
65
|
+
print("solved" if result.reward == 1.0 else "not solved")
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`SpatialGymAction.from_reply` reads the move the way the paper did and raises `ValueError` when the reply has none. To run the environment from its Hugging Face Space image with Docker, use `SpatialGymEnv.from_env("<space-id>")` instead of `base_url`.
|
|
69
|
+
|
|
70
|
+
## Reset options
|
|
71
|
+
|
|
72
|
+
| Option | Default | Meaning |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| `puzzle_id` | none | Play this SPaRC puzzle |
|
|
75
|
+
| `difficulty` | all levels | A level from 1 to 5, or a list of levels, to draw the puzzle from |
|
|
76
|
+
| `seed` | none | Makes the draw repeatable |
|
|
77
|
+
| `split` | `"test"` | `"test"` holds the 500 benchmark puzzles, `"train"` 500 more for training |
|
|
78
|
+
| `backtracking` | `False` | Allow stepping back onto the previous node to undo a move |
|
|
79
|
+
| `max_steps` | `100` | The episode ends after this many moves, as in the paper |
|
|
80
|
+
|
|
81
|
+
## Observation
|
|
82
|
+
|
|
83
|
+
| Field | Meaning |
|
|
84
|
+
|---|---|
|
|
85
|
+
| `puzzle_id`, `difficulty` | The puzzle and its level |
|
|
86
|
+
| `step` | Moves taken so far |
|
|
87
|
+
| `position` | Current node as `[x, y]` |
|
|
88
|
+
| `path` | The line so far, as `[x, y]` nodes from the start |
|
|
89
|
+
| `legal_actions` | Moves allowed from here |
|
|
90
|
+
| `grid` | Board rows in SPaRC notation; `V` marks visited nodes and `L` the current one |
|
|
91
|
+
| `system_prompt` | Rules and symbol legend from the paper, sent on reset only |
|
|
92
|
+
| `prompt` | The paper's message for the next move, empty once the episode is over |
|
|
93
|
+
| `solved`, `reached_end` | The line matches a stored solution; the line ended on the exit |
|
|
94
|
+
| `done`, `reward` | Episode over; 1.0 for a solved puzzle, else 0.0 |
|
|
95
|
+
|
|
96
|
+
`state()` returns the episode's puzzle, options, step count and the full path.
|
|
97
|
+
|
|
98
|
+
## Run it yourself
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
cd environments/spatial_gym_openenv
|
|
102
|
+
uv run --project . server # http://localhost:8000, web UI at /web with ENABLE_WEB_INTERFACE=true
|
|
103
|
+
uv run --extra dev pytest # tests; they download the SPaRC dataset
|
|
104
|
+
openenv build # Docker image
|
|
105
|
+
openenv push --repo-id <user>/spatial_gym --exclude .gitignore # Hugging Face Space
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The server keeps one environment per WebSocket session, up to `MAX_CONCURRENT_ENVS` (default 8). It downloads the SPaRC dataset from Hugging Face on the first reset of each split.
|
|
109
|
+
|
|
110
|
+
## Citation
|
|
111
|
+
|
|
112
|
+
```bibtex
|
|
113
|
+
@misc{kaesberg2026spatialgym,
|
|
114
|
+
title = {Mind the Gap Between Spatial Reasoning and Acting! Step-by-Step Evaluation of Agents With Spatial-Gym},
|
|
115
|
+
author = {Kaesberg, Lars Benedikt and Yang, Tianyu and Bauer, Niklas and Ruas, Terry and Wahle, Jan Philip and Gipp, Bela},
|
|
116
|
+
year = {2026},
|
|
117
|
+
eprint = {2604.09338},
|
|
118
|
+
archivePrefix = {arXiv}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
@inproceedings{kaesberg-etal-2025-sparc,
|
|
122
|
+
title = "{SP}a{RC}: A Spatial Pathfinding Reasoning Challenge",
|
|
123
|
+
author = "Kaesberg, Lars Benedikt and Wahle, Jan Philip and Ruas, Terry and Gipp, Bela",
|
|
124
|
+
booktitle = "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing",
|
|
125
|
+
month = nov,
|
|
126
|
+
year = "2025",
|
|
127
|
+
address = "Suzhou, China",
|
|
128
|
+
publisher = "Association for Computational Linguistics",
|
|
129
|
+
url = "https://aclanthology.org/2025.emnlp-main.526/",
|
|
130
|
+
doi = "10.18653/v1/2025.emnlp-main.526",
|
|
131
|
+
pages = "10359--10390"
|
|
132
|
+
}
|
|
133
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Spatial-Gym for OpenEnv: SPaRC pathfinding puzzles, solved one move per step."""
|
|
2
|
+
|
|
3
|
+
from .client import SpatialGymEnv
|
|
4
|
+
from .models import (
|
|
5
|
+
SpatialGymAction,
|
|
6
|
+
SpatialGymObservation,
|
|
7
|
+
SpatialGymState,
|
|
8
|
+
parse_action,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"SpatialGymAction",
|
|
13
|
+
"SpatialGymEnv",
|
|
14
|
+
"SpatialGymObservation",
|
|
15
|
+
"SpatialGymState",
|
|
16
|
+
"parse_action",
|
|
17
|
+
]
|
|
Binary file
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Client for the Spatial-Gym OpenEnv server (one WebSocket session per client)."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
|
|
5
|
+
from openenv.core.client_types import StepResult
|
|
6
|
+
from openenv.core.env_client import EnvClient
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
from .models import SpatialGymAction, SpatialGymObservation, SpatialGymState
|
|
10
|
+
except ImportError:
|
|
11
|
+
from models import SpatialGymAction, SpatialGymObservation, SpatialGymState
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class SpatialGymEnv(
|
|
15
|
+
EnvClient[SpatialGymAction, SpatialGymObservation, SpatialGymState]
|
|
16
|
+
):
|
|
17
|
+
"""Connect to a running server, a Docker image or a Hugging Face Space.
|
|
18
|
+
|
|
19
|
+
>>> with SpatialGymEnv(base_url="http://localhost:8000").sync() as env:
|
|
20
|
+
... result = env.reset(difficulty=1, seed=0)
|
|
21
|
+
... result = env.step(SpatialGymAction(action=0))
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def _step_payload(self, action: SpatialGymAction) -> Dict[str, Any]:
|
|
25
|
+
return {"action": action.action}
|
|
26
|
+
|
|
27
|
+
def _parse_result(
|
|
28
|
+
self, payload: Dict[str, Any]
|
|
29
|
+
) -> StepResult[SpatialGymObservation]:
|
|
30
|
+
fields = dict(payload.get("observation", {}))
|
|
31
|
+
fields["done"] = payload.get("done", False)
|
|
32
|
+
fields["reward"] = payload.get("reward")
|
|
33
|
+
fields["metadata"] = fields.get("metadata") or payload.get("metadata") or {}
|
|
34
|
+
observation = SpatialGymObservation(
|
|
35
|
+
**{
|
|
36
|
+
k: v
|
|
37
|
+
for k, v in fields.items()
|
|
38
|
+
if k in SpatialGymObservation.model_fields
|
|
39
|
+
}
|
|
40
|
+
)
|
|
41
|
+
return StepResult(
|
|
42
|
+
observation=observation,
|
|
43
|
+
reward=payload.get("reward"),
|
|
44
|
+
done=payload.get("done", False),
|
|
45
|
+
metadata=payload.get("metadata"),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
def _parse_state(self, payload: Dict[str, Any]) -> SpatialGymState:
|
|
49
|
+
return SpatialGymState(**payload)
|