geosql 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geosql-0.1.0/.gitignore +208 -0
- geosql-0.1.0/AGENTS.md +22 -0
- geosql-0.1.0/CONTRIBUTING.md +35 -0
- geosql-0.1.0/LICENSE +21 -0
- geosql-0.1.0/PKG-INFO +49 -0
- geosql-0.1.0/README.md +39 -0
- geosql-0.1.0/evals/RUNBOOK.md +96 -0
- geosql-0.1.0/evals/evals.json +31 -0
- geosql-0.1.0/evals/results/latest/benchmark.json +33 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/assertion_prompt.txt +27 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/assertion_result_event.json +59 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/generation_result_event.json +69 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/grading.json +36 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/model_messages.json +580 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/output.md +13 -0
- geosql-0.1.0/evals/results/latest/berlin-create-map/prompt.txt +1 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/assertion_prompt.txt +28 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/assertion_result_event.json +59 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/generation_result_event.json +69 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/grading.json +42 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/model_messages.json +360 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/output.md +39 -0
- geosql-0.1.0/evals/results/latest/london-boroughs/prompt.txt +1 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/assertion_prompt.txt +25 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/assertion_result_event.json +59 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/generation_result_event.json +69 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/grading.json +19 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/model_messages.json +196 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/output.md +29 -0
- geosql-0.1.0/evals/results/latest/paris-boundaries/prompt.txt +1 -0
- geosql-0.1.0/evals/run.py +524 -0
- geosql-0.1.0/geosql/SKILL.md +359 -0
- geosql-0.1.0/geosql/__init__.py +3 -0
- geosql-0.1.0/geosql/__main__.py +5 -0
- geosql-0.1.0/geosql/cli.py +67 -0
- geosql-0.1.0/geosql/references/map-styling.md +98 -0
- geosql-0.1.0/pyproject.toml +20 -0
geosql-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
#uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
#poetry.lock
|
|
109
|
+
#poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
#pdm.lock
|
|
116
|
+
#pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
#pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# SageMath parsed files
|
|
135
|
+
*.sage.py
|
|
136
|
+
|
|
137
|
+
# Environments
|
|
138
|
+
.env
|
|
139
|
+
.envrc
|
|
140
|
+
.venv
|
|
141
|
+
env/
|
|
142
|
+
venv/
|
|
143
|
+
ENV/
|
|
144
|
+
env.bak/
|
|
145
|
+
venv.bak/
|
|
146
|
+
|
|
147
|
+
# Spyder project settings
|
|
148
|
+
.spyderproject
|
|
149
|
+
.spyproject
|
|
150
|
+
|
|
151
|
+
# Rope project settings
|
|
152
|
+
.ropeproject
|
|
153
|
+
|
|
154
|
+
# mkdocs documentation
|
|
155
|
+
/site
|
|
156
|
+
|
|
157
|
+
# mypy
|
|
158
|
+
.mypy_cache/
|
|
159
|
+
.dmypy.json
|
|
160
|
+
dmypy.json
|
|
161
|
+
|
|
162
|
+
# Pyre type checker
|
|
163
|
+
.pyre/
|
|
164
|
+
|
|
165
|
+
# pytype static type analyzer
|
|
166
|
+
.pytype/
|
|
167
|
+
|
|
168
|
+
# Cython debug symbols
|
|
169
|
+
cython_debug/
|
|
170
|
+
|
|
171
|
+
# PyCharm
|
|
172
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
173
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
174
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
175
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
176
|
+
#.idea/
|
|
177
|
+
|
|
178
|
+
# Abstra
|
|
179
|
+
# Abstra is an AI-powered process automation framework.
|
|
180
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
181
|
+
# Learn more at https://abstra.io/docs
|
|
182
|
+
.abstra/
|
|
183
|
+
|
|
184
|
+
# Visual Studio Code
|
|
185
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
186
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
188
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
189
|
+
# .vscode/
|
|
190
|
+
|
|
191
|
+
# Ruff stuff:
|
|
192
|
+
.ruff_cache/
|
|
193
|
+
|
|
194
|
+
# PyPI configuration file
|
|
195
|
+
.pypirc
|
|
196
|
+
|
|
197
|
+
# Cursor
|
|
198
|
+
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
|
199
|
+
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
|
200
|
+
# refer to https://docs.cursor.com/context/ignore-files
|
|
201
|
+
.cursorignore
|
|
202
|
+
.cursorindexingignore
|
|
203
|
+
|
|
204
|
+
# Marimo
|
|
205
|
+
marimo/_static/
|
|
206
|
+
marimo/_lsp/
|
|
207
|
+
__marimo__/
|
|
208
|
+
events.ndjson
|
geosql-0.1.0/AGENTS.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# AGENTS.md instructions for geosql
|
|
2
|
+
|
|
3
|
+
## Skills
|
|
4
|
+
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file.
|
|
5
|
+
|
|
6
|
+
### Available skills
|
|
7
|
+
- geosql: Build and optionally execute cost-safe Overture Maps SQL for BigQuery with dry-run budget checks. Use when users need map-ready SQL, executed results, or over-budget fallback options. (file: geosql/SKILL.md)
|
|
8
|
+
|
|
9
|
+
### How to use skills
|
|
10
|
+
- Trigger rules: If the user names the skill (`$geosql`) or the request clearly matches its description, use it for that turn.
|
|
11
|
+
- Loading: Open `geosql/SKILL.md` and follow it.
|
|
12
|
+
- Scope: Do not carry this skill across turns unless re-mentioned or clearly required by the new request.
|
|
13
|
+
|
|
14
|
+
## References
|
|
15
|
+
- Claude Skills docs: https://code.claude.com/docs/en/skills
|
|
16
|
+
- Agent Skills open standard: https://agentskills.io
|
|
17
|
+
|
|
18
|
+
## Skill Best Practices
|
|
19
|
+
- Keep frontmatter concise: use short `description`, include `argument-hint` when slash arguments are expected.
|
|
20
|
+
- Keep one canonical source per artifact: no duplicate `SKILL.md` or script logic copies.
|
|
21
|
+
- Put deterministic execution logic in `scripts/` and reference those scripts explicitly from `SKILL.md`.
|
|
22
|
+
- Keep instructions focused and scannable; move deep details to supporting files only when needed.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
## Setup
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
git clone https://github.com/dekart-xyz/geosql.git
|
|
7
|
+
cd geosql
|
|
8
|
+
pip install -e .
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Test locally
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
python -m geosql --help
|
|
15
|
+
python -m geosql install claude
|
|
16
|
+
geosql install claude
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Dekart runtime CLI
|
|
20
|
+
|
|
21
|
+
Dekart integration moved to `dekart` CLI in `../dekart-cli`.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
cd ../dekart-cli
|
|
25
|
+
pip install -e .
|
|
26
|
+
dekart --help
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Publish
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install build twine
|
|
33
|
+
python -m build
|
|
34
|
+
twine upload dist/*
|
|
35
|
+
```
|
geosql-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 dekart.xyz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
geosql-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: geosql
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Install GeoSQL instructions into local AI CLIs.
|
|
5
|
+
Author: Dekart XYZ
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.8
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# geosql
|
|
12
|
+
Create maps and answer questions from your data.
|
|
13
|
+
|
|
14
|
+
## Install geosql (skill installer only)
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install -e .
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Install Claude skill
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
geosql install claude
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
This installs to:
|
|
27
|
+
|
|
28
|
+
`~/.claude/skills/geosql/SKILL.md`
|
|
29
|
+
|
|
30
|
+
and copies references to:
|
|
31
|
+
|
|
32
|
+
`~/.claude/skills/geosql/references/`
|
|
33
|
+
|
|
34
|
+
## Dekart CLI
|
|
35
|
+
|
|
36
|
+
GeoSQL uses the separate `dekart` CLI for Dekart auth, MCP tools, and file upload workflows.
|
|
37
|
+
|
|
38
|
+
Install `dekart` from the new repo:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
cd ../dekart-cli
|
|
42
|
+
pip install -e .
|
|
43
|
+
dekart --help
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Config is stored in:
|
|
47
|
+
|
|
48
|
+
- `~/.config/dekart/config.json`
|
|
49
|
+
- `~/.config/dekart/token.json`
|
geosql-0.1.0/README.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# geosql
|
|
2
|
+
Create maps and answer questions from your data.
|
|
3
|
+
|
|
4
|
+
## Install geosql (skill installer only)
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
pip install -e .
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
## Install Claude skill
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
geosql install claude
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
This installs to:
|
|
17
|
+
|
|
18
|
+
`~/.claude/skills/geosql/SKILL.md`
|
|
19
|
+
|
|
20
|
+
and copies references to:
|
|
21
|
+
|
|
22
|
+
`~/.claude/skills/geosql/references/`
|
|
23
|
+
|
|
24
|
+
## Dekart CLI
|
|
25
|
+
|
|
26
|
+
GeoSQL uses the separate `dekart` CLI for Dekart auth, MCP tools, and file upload workflows.
|
|
27
|
+
|
|
28
|
+
Install `dekart` from the new repo:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
cd ../dekart-cli
|
|
32
|
+
pip install -e .
|
|
33
|
+
dekart --help
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Config is stored in:
|
|
37
|
+
|
|
38
|
+
- `~/.config/dekart/config.json`
|
|
39
|
+
- `~/.config/dekart/token.json`
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Eval Runbook
|
|
2
|
+
|
|
3
|
+
This runbook executes the full eval cycle in one command using Claude CLI.
|
|
4
|
+
|
|
5
|
+
## What this eval does
|
|
6
|
+
|
|
7
|
+
For each case in `evals/evals.json`:
|
|
8
|
+
|
|
9
|
+
1. Runs the task with `/geosql ...`.
|
|
10
|
+
2. In the same Claude session, asks Claude to validate all assertions.
|
|
11
|
+
3. Claude can run `bq`/`gcloud` commands while validating assertions.
|
|
12
|
+
4. Saves grading and logs.
|
|
13
|
+
|
|
14
|
+
Assertions are plain strings only.
|
|
15
|
+
|
|
16
|
+
## Prerequisites
|
|
17
|
+
|
|
18
|
+
- Claude CLI installed and authenticated (`claude --help` works).
|
|
19
|
+
- You are in `geosql`.
|
|
20
|
+
|
|
21
|
+
## One-command Eval
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
python3 evals/run.py --model claude-sonnet-4-6 --thinking-level high
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
By default, the runner prints a live trace of Claude IO (prompt, tool calls, tool outputs, final result) so you can see exactly what is happening.
|
|
28
|
+
|
|
29
|
+
Quiet mode:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
python3 evals/run.py --model claude-sonnet-4-6 --thinking-level high --no-show-io
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Adjust trace truncation length:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
python3 evals/run.py --model claude-sonnet-4-6 --thinking-level high --io-max-chars 2000
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Default permission behavior for this command:
|
|
42
|
+
|
|
43
|
+
- `--dangerously-skip-permissions` (unless `--safe-mode` is set)
|
|
44
|
+
- `--permission-mode bypassPermissions`
|
|
45
|
+
- `--allowedTools "Bash(bq:*) Bash(gcloud:*)"`
|
|
46
|
+
|
|
47
|
+
This is to avoid approval interruptions during assertion-time `bq` checks.
|
|
48
|
+
|
|
49
|
+
## Output Files
|
|
50
|
+
|
|
51
|
+
Per case:
|
|
52
|
+
|
|
53
|
+
- `prompt.txt` (skill prompt)
|
|
54
|
+
- `output.md` (first model answer)
|
|
55
|
+
- `assertion_prompt.txt` (follow-up grading prompt)
|
|
56
|
+
- `events.ndjson` (full stream events, generated locally)
|
|
57
|
+
- `model_messages.json` (all assistant/model messages, compact)
|
|
58
|
+
- `generation_result_event.json` (result event for first task run)
|
|
59
|
+
- `assertion_result_event.json` (result event for assertion grading run)
|
|
60
|
+
- `grading.json` (parsed assertion grading summary)
|
|
61
|
+
|
|
62
|
+
Aggregate:
|
|
63
|
+
|
|
64
|
+
- `evals/results/latest/benchmark.json`
|
|
65
|
+
|
|
66
|
+
## Exit Codes
|
|
67
|
+
|
|
68
|
+
- `0`: all assertions passed
|
|
69
|
+
- `1`: one or more assertions failed
|
|
70
|
+
- `2`: setup/runtime issue (for example no Claude binary)
|
|
71
|
+
|
|
72
|
+
## Optional Flags
|
|
73
|
+
|
|
74
|
+
Custom eval file:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
python3 evals/run.py --evals evals/evals.json
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Custom output dir:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
python3 evals/run.py --out-dir evals/results/manual-run
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Model and thinking level are required:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
python3 evals/run.py --model claude-sonnet-4-6 --thinking-level xhigh
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Safer permissions (may re-introduce approval prompts):
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
python3 evals/run.py --model claude-sonnet-4-6 --thinking-level high --safe-mode --permission-mode auto
|
|
96
|
+
```
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill_name": "geosql",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": "london-boroughs",
|
|
6
|
+
"prompt": "give me sql for london boundries",
|
|
7
|
+
"assertions": [
|
|
8
|
+
"Agent used bq with --maximum_bytes_billed set.",
|
|
9
|
+
"Agent validated the query (dry run + row count check) before presenting the final answer.",
|
|
10
|
+
"Run query and ensure that the result is not empty.",
|
|
11
|
+
"Calculate the area of returned geometries and check if the total area is ~1,577 sq km (+/- 2%)."
|
|
12
|
+
]
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "paris-boundaries",
|
|
16
|
+
"prompt": "give me sql for paris boundries",
|
|
17
|
+
"assertions": [
|
|
18
|
+
"Run query and ensure that the result is not empty and contains exactly 1 row."
|
|
19
|
+
]
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"id": "berlin-create-map",
|
|
23
|
+
"prompt": "find berlin administrative boundaries in bigquery and create a dekart map from the result",
|
|
24
|
+
"assertions": [
|
|
25
|
+
"Agent performed Dekart init check first using dekart tools --json before any Dekart write/upload action.",
|
|
26
|
+
"Agent used geosql to upload the query result to Dekart and create a map.",
|
|
27
|
+
"Agent returned user map URL after map creation."
|
|
28
|
+
]
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill_name": "geosql",
|
|
3
|
+
"cases": [
|
|
4
|
+
{
|
|
5
|
+
"id": "berlin-create-map",
|
|
6
|
+
"passed": 3,
|
|
7
|
+
"failed": 0,
|
|
8
|
+
"total": 3,
|
|
9
|
+
"pass_rate": 1.0
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"id": "london-boroughs",
|
|
13
|
+
"passed": 4,
|
|
14
|
+
"failed": 0,
|
|
15
|
+
"total": 4,
|
|
16
|
+
"pass_rate": 1.0
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "paris-boundaries",
|
|
20
|
+
"passed": 1,
|
|
21
|
+
"failed": 0,
|
|
22
|
+
"total": 1,
|
|
23
|
+
"pass_rate": 1.0
|
|
24
|
+
}
|
|
25
|
+
],
|
|
26
|
+
"summary": {
|
|
27
|
+
"passed_assertions": 8,
|
|
28
|
+
"total_assertions": 8,
|
|
29
|
+
"pass_rate": 1.0,
|
|
30
|
+
"avg_tokens_per_turn": 3084.6666666666665,
|
|
31
|
+
"avg_duration_ms_per_turn": 72269.5
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Now validate the assertions below for your previous answer in this same session.
|
|
2
|
+
Rules:
|
|
3
|
+
1. For EACH assertion, actively validate it.
|
|
4
|
+
2. If validation requires data checks, run bq queries yourself.
|
|
5
|
+
3. If you cannot validate an assertion, mark it passed=false.
|
|
6
|
+
4. Output ONLY valid JSON.
|
|
7
|
+
5. JSON format:
|
|
8
|
+
{
|
|
9
|
+
"assertion_results": [
|
|
10
|
+
{
|
|
11
|
+
"assertion": "<string>",
|
|
12
|
+
"passed": true,
|
|
13
|
+
"evidence": "<short evidence>",
|
|
14
|
+
"commands_run": ["<command1>", "<command2>"],
|
|
15
|
+
"result_snippet": "<short stdout/result snippet>"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"summary": {"passed": 0, "failed": 0, "total": 0, "pass_rate": 0.0}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
Assertions:
|
|
22
|
+
[
|
|
23
|
+
"Agent performed Dekart init check first using geosql dekart tools --json before any Dekart write/upload action.",
|
|
24
|
+
"Agent used geosql to upload the query result to Dekart and create a map.",
|
|
25
|
+
"Agent returned user map URL after map creation."
|
|
26
|
+
]
|
|
27
|
+
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "result",
|
|
3
|
+
"subtype": "success",
|
|
4
|
+
"is_error": false,
|
|
5
|
+
"api_error_status": null,
|
|
6
|
+
"duration_ms": 13386,
|
|
7
|
+
"duration_api_ms": 13373,
|
|
8
|
+
"num_turns": 1,
|
|
9
|
+
"result": "```json\n{\n \"assertion_results\": [\n {\n \"assertion\": \"Agent performed Dekart init check first using geosql dekart tools --json before any Dekart write/upload action.\",\n \"passed\": true,\n \"evidence\": \"The very first two commands in the session were run in parallel: `geosql dekart tools --json` and the BigQuery INFORMATION_SCHEMA table list. The Dekart tools check returned successfully before any create_report, create_dataset, create_file, or upload-file calls were made.\",\n \"commands_run\": [],\n \"result_snippet\": \"geosql dekart tools --json returned tools list including create_report, create_dataset, create_file, etc. \u2014 confirmed before any write action.\"\n },\n {\n \"assertion\": \"Agent used geosql to upload the query result to Dekart and create a map.\",\n \"passed\": true,\n \"evidence\": \"Agent ran: (1) geosql dekart call --name create_report, (2) geosql dekart call --name create_dataset, (3) geosql dekart call --name create_file, (4) geosql dekart upload-file --file /tmp/berlin_admin.csv. Upload completed with status='completed', size=661572 bytes.\",\n \"commands_run\": [\n \"geosql dekart call --name create_report --args '{}' --json\",\n \"geosql dekart call --name create_dataset --args '{\\\"report_id\\\": \\\"bed67dd6-9ed7-4c4e-86aa-5553b5cac78e\\\"}' --json\",\n \"geosql dekart call --name create_file --args '{\\\"dataset_id\\\": \\\"6f6e5705-3d6e-42fa-8145-1b3c90b12da8\\\"}' --json\",\n \"geosql dekart upload-file --file /tmp/berlin_admin.csv --file-id 6e379e96-7501-402a-813a-285c8377b7af --name berlin_admin_boundaries.csv --mime-type text/csv --json\"\n ],\n \"result_snippet\": \"{\\\"complete\\\": {\\\"file_id\\\": \\\"6e379e96-7501-402a-813a-285c8377b7af\\\", \\\"status\\\": \\\"completed\\\", \\\"size\\\": \\\"661572\\\"}}\"\n },\n {\n \"assertion\": \"Agent returned user map URL after map creation.\",\n \"passed\": true,\n \"evidence\": \"Final response included the report URL from the create_report response: http://localhost:3000/reports/bed67dd6-9ed7-4c4e-86aa-5553b5cac78e, sourced from report_url field (not reconstructed from config).\",\n \"commands_run\": [],\n \"result_snippet\": \"Report URL: http://localhost:3000/reports/bed67dd6-9ed7-4c4e-86aa-5553b5cac78e\"\n }\n ],\n \"summary\": {\n \"passed\": 3,\n \"failed\": 0,\n \"total\": 3,\n \"pass_rate\": 1.0\n }\n}\n```",
|
|
10
|
+
"stop_reason": "end_turn",
|
|
11
|
+
"session_id": "1825a1da-cad6-414f-833a-ab9481913ec1",
|
|
12
|
+
"total_cost_usd": 0.026644050000000002,
|
|
13
|
+
"usage": {
|
|
14
|
+
"input_tokens": 2,
|
|
15
|
+
"cache_creation_input_tokens": 533,
|
|
16
|
+
"cache_read_input_tokens": 30631,
|
|
17
|
+
"output_tokens": 1030,
|
|
18
|
+
"server_tool_use": {
|
|
19
|
+
"web_search_requests": 0,
|
|
20
|
+
"web_fetch_requests": 0
|
|
21
|
+
},
|
|
22
|
+
"service_tier": "standard",
|
|
23
|
+
"cache_creation": {
|
|
24
|
+
"ephemeral_1h_input_tokens": 533,
|
|
25
|
+
"ephemeral_5m_input_tokens": 0
|
|
26
|
+
},
|
|
27
|
+
"inference_geo": "",
|
|
28
|
+
"iterations": [
|
|
29
|
+
{
|
|
30
|
+
"input_tokens": 2,
|
|
31
|
+
"output_tokens": 1030,
|
|
32
|
+
"cache_read_input_tokens": 30631,
|
|
33
|
+
"cache_creation_input_tokens": 533,
|
|
34
|
+
"cache_creation": {
|
|
35
|
+
"ephemeral_5m_input_tokens": 0,
|
|
36
|
+
"ephemeral_1h_input_tokens": 533
|
|
37
|
+
},
|
|
38
|
+
"type": "message"
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
"speed": "standard"
|
|
42
|
+
},
|
|
43
|
+
"modelUsage": {
|
|
44
|
+
"claude-sonnet-4-6": {
|
|
45
|
+
"inputTokens": 2,
|
|
46
|
+
"outputTokens": 1030,
|
|
47
|
+
"cacheReadInputTokens": 30631,
|
|
48
|
+
"cacheCreationInputTokens": 533,
|
|
49
|
+
"webSearchRequests": 0,
|
|
50
|
+
"costUSD": 0.026644050000000002,
|
|
51
|
+
"contextWindow": 200000,
|
|
52
|
+
"maxOutputTokens": 32000
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
"permission_denials": [],
|
|
56
|
+
"terminal_reason": "completed",
|
|
57
|
+
"fast_mode_state": "off",
|
|
58
|
+
"uuid": "fc37468d-22d9-4ca2-86ae-0cf282fed4fb"
|
|
59
|
+
}
|