inspect-briq 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inspect_briq-0.1.0/.gitignore +7 -0
- inspect_briq-0.1.0/LICENSE +21 -0
- inspect_briq-0.1.0/PKG-INFO +136 -0
- inspect_briq-0.1.0/README.md +113 -0
- inspect_briq-0.1.0/pyproject.toml +62 -0
- inspect_briq-0.1.0/src/inspect_briq/__init__.py +7 -0
- inspect_briq-0.1.0/src/inspect_briq/_config.py +94 -0
- inspect_briq-0.1.0/src/inspect_briq/_environment.py +239 -0
- inspect_briq-0.1.0/src/inspect_briq/_registry.py +5 -0
- inspect_briq-0.1.0/src/inspect_briq/_version.py +1 -0
- inspect_briq-0.1.0/src/inspect_briq/py.typed +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 briq
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: inspect-briq
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Run Inspect AI sandboxes on briq: one microVM per sample.
|
|
5
|
+
Project-URL: Homepage, https://briq.cloud
|
|
6
|
+
Project-URL: Repository, https://github.com/briq-cloud/inspect-briq
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: agents,briq,evals,inspect,inspect-ai,sandbox
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: briq-cloud<0.2,>=0.1.1
|
|
19
|
+
Requires-Dist: inspect-ai>=0.3.287
|
|
20
|
+
Requires-Dist: pydantic>=2
|
|
21
|
+
Requires-Dist: typing-extensions>=4.5
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# inspect-briq
|
|
25
|
+
|
|
26
|
+
[](https://github.com/briq-cloud/inspect-briq/actions/workflows/ci.yml) [](https://pypi.org/project/inspect-briq/) [](LICENSE)
|
|
27
|
+
|
|
28
|
+
Run [Inspect AI](https://inspect.aisi.org.uk/) sandboxes on [briq](https://briq.cloud). Each sample gets its own microVM, destroyed when the sample ends.
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from inspect_ai import Task, task
|
|
32
|
+
from inspect_ai.dataset import Sample
|
|
33
|
+
from inspect_ai.scorer import includes
|
|
34
|
+
from inspect_ai.solver import generate, use_tools
|
|
35
|
+
from inspect_ai.tool import bash
|
|
36
|
+
from inspect_ai.util import SandboxEnvironmentSpec
|
|
37
|
+
from inspect_briq import BriqSandboxEnvironmentConfig
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@task
|
|
41
|
+
def kernel():
|
|
42
|
+
return Task(
|
|
43
|
+
dataset=[
|
|
44
|
+
Sample(
|
|
45
|
+
input="Run `uname -s` with the bash tool and tell me its output.",
|
|
46
|
+
target="Linux",
|
|
47
|
+
files={"notes.txt": "one file per sample, copied in before the first message"},
|
|
48
|
+
)
|
|
49
|
+
],
|
|
50
|
+
solver=[use_tools(bash(timeout=30)), generate()],
|
|
51
|
+
scorer=includes(),
|
|
52
|
+
sandbox=SandboxEnvironmentSpec("briq", BriqSandboxEnvironmentConfig(size="medium")),
|
|
53
|
+
)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
pip install inspect-briq
|
|
58
|
+
export BRIQ_API_KEY=...
|
|
59
|
+
inspect eval kernel.py --model anthropic/claude-sonnet-4-5
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`sandbox="briq"` uses the defaults. Python 3.11 or later (what Inspect needs), asyncio backend.
|
|
63
|
+
|
|
64
|
+
## Configure
|
|
65
|
+
|
|
66
|
+
`BriqSandboxEnvironmentConfig`, or a JSON file with the same fields: `sandbox=("briq", "briq.json")`.
|
|
67
|
+
|
|
68
|
+
| Field | Default | Means |
|
|
69
|
+
| ---------------------- | ------------------------ | ------------------------------------------------------------------------- |
|
|
70
|
+
| `size` | the API's default | `small`, `medium`, `large` or `xlarge`. Anything else fails `unsupported`. |
|
|
71
|
+
| `image` | briq's sandbox image | An OCI image. |
|
|
72
|
+
| `sleep` | `30m` | Idle time before the sandbox sleeps: `30m`, `3h`, `7d`, `never`. |
|
|
73
|
+
| `working_dir` | `/work` | Where commands run and relative paths resolve. |
|
|
74
|
+
| `env` | none | Variables of the sandbox. They show in its status and in the eval log. |
|
|
75
|
+
| `secret_env` | none | Names of variables of the eval process, sent write-only. Needs API support. |
|
|
76
|
+
| `default_exec_timeout` | `600` | Seconds for a command that has no timeout. It raises `TimeoutError`. |
|
|
77
|
+
| `wait_timeout` | `120` | Seconds to wait for a sandbox to start. |
|
|
78
|
+
| `base_url` | `BRIQ_API_URL`, or the public API | The briq API. |
|
|
79
|
+
|
|
80
|
+
The API key is read from `BRIQ_API_KEY` only: a key in the configuration would end up in the eval log.
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
BriqSandboxEnvironmentConfig(
|
|
84
|
+
size="large",
|
|
85
|
+
env={"MODE": "test"},
|
|
86
|
+
secret_env=["GITHUB_TOKEN"], # read from os.environ, never stored in the log
|
|
87
|
+
)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`secret_env` fails with `unsupported` when the API cannot keep a value write-only. It never falls back to a plain variable.
|
|
91
|
+
|
|
92
|
+
## What is supported
|
|
93
|
+
|
|
94
|
+
| Inspect | On briq |
|
|
95
|
+
| -------------------------------------- | ------------------------------------------------------------------------- |
|
|
96
|
+
| `exec` with `cwd`, `env`, `timeout`, `input` | Yes. `input` is text: bytes that are not UTF-8 fail `unsupported`. |
|
|
97
|
+
| `exec(user=...)`, `read_file(user=...)`, `write_file(user=...)` | No: `SandboxUserUnsupportedError`. Commands run as the sandbox's own user. |
|
|
98
|
+
| `read_file`, `write_file` | Yes. Missing directories are created. A file over 100 MiB fails. |
|
|
99
|
+
| Sample `files` and `setup` | Yes, through `write_file` and `exec`. |
|
|
100
|
+
| One sandbox per sample | Yes, named `default`. |
|
|
101
|
+
| Several named sandboxes per sample | No. |
|
|
102
|
+
| `connection()` (`inspect sandbox shell`) | No. |
|
|
103
|
+
| Docker `compose.yaml` and `Dockerfile` | No. Use `image`. |
|
|
104
|
+
|
|
105
|
+
`timeout_retry` and `concurrency` have no effect: a command that outlives its timeout is stopped and raises `TimeoutError` at once. Limit parallel sandboxes with `--max-sandboxes`.
|
|
106
|
+
|
|
107
|
+
Errors:
|
|
108
|
+
|
|
109
|
+
| Raised | When |
|
|
110
|
+
| ---------------------------- | ------------------------------------------------------------ |
|
|
111
|
+
| `FileNotFoundError` | The file does not exist. |
|
|
112
|
+
| `TimeoutError` | A command or a call outlived its timeout. |
|
|
113
|
+
| `SandboxUnavailableError` | The sandbox is gone or briq could not take the call. |
|
|
114
|
+
| `OutputLimitExceededError` | Output or a file is over Inspect's limit. |
|
|
115
|
+
| `briq_cloud.BriqError` | Anything else: quota, credits, plan, a wrong key. It has a stable `code`. |
|
|
116
|
+
|
|
117
|
+
A command whose end is not known raises `BriqError` with `code="outcome_unknown"`. It is never repeated.
|
|
118
|
+
|
|
119
|
+
## Clean up
|
|
120
|
+
|
|
121
|
+
A sample's sandbox is destroyed when the sample ends, also when it fails or is cancelled. If the evaluation is killed, its sandboxes stay: they sleep after `sleep` and are not billed while asleep. Destroy them from the dashboard, or with `briq_cloud`: `AsyncBriq().sandbox(id).destroy()`.
|
|
122
|
+
|
|
123
|
+
## Develop
|
|
124
|
+
|
|
125
|
+
Needs [uv](https://docs.astral.sh/uv/).
|
|
126
|
+
|
|
127
|
+
```sh
|
|
128
|
+
uv sync
|
|
129
|
+
uv run pytest # offline, the briq client is faked
|
|
130
|
+
uv run ruff check . && uv run ruff format --check .
|
|
131
|
+
uv run mypy
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## License
|
|
135
|
+
|
|
136
|
+
MIT
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# inspect-briq
|
|
2
|
+
|
|
3
|
+
[](https://github.com/briq-cloud/inspect-briq/actions/workflows/ci.yml) [](https://pypi.org/project/inspect-briq/) [](LICENSE)
|
|
4
|
+
|
|
5
|
+
Run [Inspect AI](https://inspect.aisi.org.uk/) sandboxes on [briq](https://briq.cloud). Each sample gets its own microVM, destroyed when the sample ends.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from inspect_ai import Task, task
|
|
9
|
+
from inspect_ai.dataset import Sample
|
|
10
|
+
from inspect_ai.scorer import includes
|
|
11
|
+
from inspect_ai.solver import generate, use_tools
|
|
12
|
+
from inspect_ai.tool import bash
|
|
13
|
+
from inspect_ai.util import SandboxEnvironmentSpec
|
|
14
|
+
from inspect_briq import BriqSandboxEnvironmentConfig
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@task
|
|
18
|
+
def kernel():
|
|
19
|
+
return Task(
|
|
20
|
+
dataset=[
|
|
21
|
+
Sample(
|
|
22
|
+
input="Run `uname -s` with the bash tool and tell me its output.",
|
|
23
|
+
target="Linux",
|
|
24
|
+
files={"notes.txt": "one file per sample, copied in before the first message"},
|
|
25
|
+
)
|
|
26
|
+
],
|
|
27
|
+
solver=[use_tools(bash(timeout=30)), generate()],
|
|
28
|
+
scorer=includes(),
|
|
29
|
+
sandbox=SandboxEnvironmentSpec("briq", BriqSandboxEnvironmentConfig(size="medium")),
|
|
30
|
+
)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
pip install inspect-briq
|
|
35
|
+
export BRIQ_API_KEY=...
|
|
36
|
+
inspect eval kernel.py --model anthropic/claude-sonnet-4-5
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`sandbox="briq"` uses the defaults. Python 3.11 or later (what Inspect needs), asyncio backend.
|
|
40
|
+
|
|
41
|
+
## Configure
|
|
42
|
+
|
|
43
|
+
`BriqSandboxEnvironmentConfig`, or a JSON file with the same fields: `sandbox=("briq", "briq.json")`.
|
|
44
|
+
|
|
45
|
+
| Field | Default | Means |
|
|
46
|
+
| ---------------------- | ------------------------ | ------------------------------------------------------------------------- |
|
|
47
|
+
| `size` | the API's default | `small`, `medium`, `large` or `xlarge`. Anything else fails `unsupported`. |
|
|
48
|
+
| `image` | briq's sandbox image | An OCI image. |
|
|
49
|
+
| `sleep` | `30m` | Idle time before the sandbox sleeps: `30m`, `3h`, `7d`, `never`. |
|
|
50
|
+
| `working_dir` | `/work` | Where commands run and relative paths resolve. |
|
|
51
|
+
| `env` | none | Variables of the sandbox. They show in its status and in the eval log. |
|
|
52
|
+
| `secret_env` | none | Names of variables of the eval process, sent write-only. Needs API support. |
|
|
53
|
+
| `default_exec_timeout` | `600` | Seconds for a command that has no timeout. It raises `TimeoutError`. |
|
|
54
|
+
| `wait_timeout` | `120` | Seconds to wait for a sandbox to start. |
|
|
55
|
+
| `base_url` | `BRIQ_API_URL`, or the public API | The briq API. |
|
|
56
|
+
|
|
57
|
+
The API key is read from `BRIQ_API_KEY` only: a key in the configuration would end up in the eval log.
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
BriqSandboxEnvironmentConfig(
|
|
61
|
+
size="large",
|
|
62
|
+
env={"MODE": "test"},
|
|
63
|
+
secret_env=["GITHUB_TOKEN"], # read from os.environ, never stored in the log
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`secret_env` fails with `unsupported` when the API cannot keep a value write-only. It never falls back to a plain variable.
|
|
68
|
+
|
|
69
|
+
## What is supported
|
|
70
|
+
|
|
71
|
+
| Inspect | On briq |
|
|
72
|
+
| -------------------------------------- | ------------------------------------------------------------------------- |
|
|
73
|
+
| `exec` with `cwd`, `env`, `timeout`, `input` | Yes. `input` is text: bytes that are not UTF-8 fail `unsupported`. |
|
|
74
|
+
| `exec(user=...)`, `read_file(user=...)`, `write_file(user=...)` | No: `SandboxUserUnsupportedError`. Commands run as the sandbox's own user. |
|
|
75
|
+
| `read_file`, `write_file` | Yes. Missing directories are created. A file over 100 MiB fails. |
|
|
76
|
+
| Sample `files` and `setup` | Yes, through `write_file` and `exec`. |
|
|
77
|
+
| One sandbox per sample | Yes, named `default`. |
|
|
78
|
+
| Several named sandboxes per sample | No. |
|
|
79
|
+
| `connection()` (`inspect sandbox shell`) | No. |
|
|
80
|
+
| Docker `compose.yaml` and `Dockerfile` | No. Use `image`. |
|
|
81
|
+
|
|
82
|
+
`timeout_retry` and `concurrency` have no effect: a command that outlives its timeout is stopped and raises `TimeoutError` at once. Limit parallel sandboxes with `--max-sandboxes`.
|
|
83
|
+
|
|
84
|
+
Errors:
|
|
85
|
+
|
|
86
|
+
| Raised | When |
|
|
87
|
+
| ---------------------------- | ------------------------------------------------------------ |
|
|
88
|
+
| `FileNotFoundError` | The file does not exist. |
|
|
89
|
+
| `TimeoutError` | A command or a call outlived its timeout. |
|
|
90
|
+
| `SandboxUnavailableError` | The sandbox is gone or briq could not take the call. |
|
|
91
|
+
| `OutputLimitExceededError` | Output or a file is over Inspect's limit. |
|
|
92
|
+
| `briq_cloud.BriqError` | Anything else: quota, credits, plan, a wrong key. It has a stable `code`. |
|
|
93
|
+
|
|
94
|
+
A command whose end is not known raises `BriqError` with `code="outcome_unknown"`. It is never repeated.
|
|
95
|
+
|
|
96
|
+
## Clean up
|
|
97
|
+
|
|
98
|
+
A sample's sandbox is destroyed when the sample ends, also when it fails or is cancelled. If the evaluation is killed, its sandboxes stay: they sleep after `sleep` and are not billed while asleep. Destroy them from the dashboard, or with `briq_cloud`: `AsyncBriq().sandbox(id).destroy()`.
|
|
99
|
+
|
|
100
|
+
## Develop
|
|
101
|
+
|
|
102
|
+
Needs [uv](https://docs.astral.sh/uv/).
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
uv sync
|
|
106
|
+
uv run pytest # offline, the briq client is faked
|
|
107
|
+
uv run ruff check . && uv run ruff format --check .
|
|
108
|
+
uv run mypy
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## License
|
|
112
|
+
|
|
113
|
+
MIT
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "inspect-briq"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Run Inspect AI sandboxes on briq: one microVM per sample."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
keywords = ["inspect", "inspect-ai", "evals", "sandbox", "briq", "agents"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Typing :: Typed",
|
|
22
|
+
]
|
|
23
|
+
dependencies = ["inspect-ai>=0.3.287", "briq-cloud>=0.1.1,<0.2", "pydantic>=2", "typing-extensions>=4.5"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://briq.cloud"
|
|
27
|
+
Repository = "https://github.com/briq-cloud/inspect-briq"
|
|
28
|
+
|
|
29
|
+
# Inspect loads this module at startup, which registers the "briq" sandbox type.
|
|
30
|
+
[project.entry-points.inspect_ai]
|
|
31
|
+
inspect_briq = "inspect_briq._registry"
|
|
32
|
+
|
|
33
|
+
[dependency-groups]
|
|
34
|
+
dev = ["pytest>=8", "pytest-asyncio>=0.24", "mypy>=1.11", "ruff>=0.6"]
|
|
35
|
+
|
|
36
|
+
[tool.hatch.version]
|
|
37
|
+
path = "src/inspect_briq/_version.py"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.wheel]
|
|
40
|
+
packages = ["src/inspect_briq"]
|
|
41
|
+
|
|
42
|
+
[tool.hatch.build.targets.sdist]
|
|
43
|
+
include = ["src", "README.md", "LICENSE", "pyproject.toml"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
addopts = "-p no:cacheprovider"
|
|
48
|
+
asyncio_mode = "auto"
|
|
49
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
line-length = 100
|
|
53
|
+
target-version = "py311"
|
|
54
|
+
src = ["src", "tests"]
|
|
55
|
+
|
|
56
|
+
[tool.ruff.lint]
|
|
57
|
+
select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF", "C4", "PT"]
|
|
58
|
+
|
|
59
|
+
[tool.mypy]
|
|
60
|
+
strict = true
|
|
61
|
+
python_version = "3.11"
|
|
62
|
+
files = ["src", "tests"]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Inspect AI sandbox environment on briq: one microVM per sample."""
|
|
2
|
+
|
|
3
|
+
from ._config import BriqSandboxEnvironmentConfig
|
|
4
|
+
from ._environment import BriqSandboxEnvironment
|
|
5
|
+
from ._version import __version__
|
|
6
|
+
|
|
7
|
+
__all__ = ["BriqSandboxEnvironment", "BriqSandboxEnvironmentConfig", "__version__"]
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""The configuration of a ``briq`` sandbox: a typed object, or a JSON file that holds one."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from briq_cloud import BriqError
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
11
|
+
|
|
12
|
+
SIZES = ("small", "medium", "large", "xlarge")
|
|
13
|
+
|
|
14
|
+
_SLEEP = re.compile(r"\d+[mhd]|never")
|
|
15
|
+
_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class BriqSandboxEnvironmentConfig(BaseModel):
|
|
19
|
+
"""How each sample's sandbox is made.
|
|
20
|
+
|
|
21
|
+
The API key is never part of the configuration (it would end up in the eval log): set
|
|
22
|
+
``BRIQ_API_KEY``.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
26
|
+
|
|
27
|
+
base_url: str | None = None
|
|
28
|
+
"""The briq API. Default: ``BRIQ_API_URL``, then ``https://api.briq.cloud``."""
|
|
29
|
+
|
|
30
|
+
size: str | None = None
|
|
31
|
+
"""``small``, ``medium``, ``large`` or ``xlarge``. Default: the API's default (``small``)."""
|
|
32
|
+
|
|
33
|
+
image: str | None = None
|
|
34
|
+
"""An OCI image for the sandbox. Default: briq's sandbox image."""
|
|
35
|
+
|
|
36
|
+
sleep: str = "30m"
|
|
37
|
+
"""Idle time before the sandbox sleeps (``30m``, ``3h``, ``7d`` or ``never``). A killed
|
|
38
|
+
evaluation leaves its sandboxes to sleep, which is not billed."""
|
|
39
|
+
|
|
40
|
+
working_dir: str = "/work"
|
|
41
|
+
"""Where commands run and relative paths resolve."""
|
|
42
|
+
|
|
43
|
+
env: dict[str, str] = Field(default_factory=dict)
|
|
44
|
+
"""Environment variables of the sandbox. They show in the sandbox's status and in the eval
|
|
45
|
+
log: no secrets here."""
|
|
46
|
+
|
|
47
|
+
secret_env: list[str] = Field(default_factory=list)
|
|
48
|
+
"""Names of variables of *this* process whose values go to the sandbox write-only. The values
|
|
49
|
+
are not stored in the configuration or the log. The API must support it: if it does not,
|
|
50
|
+
creating the sandbox fails with ``unsupported``."""
|
|
51
|
+
|
|
52
|
+
default_exec_timeout: int = 600
|
|
53
|
+
"""Seconds a command may run when ``exec`` is given no timeout. Longer commands are stopped
|
|
54
|
+
and raise ``TimeoutError``. 1 second to 24 hours."""
|
|
55
|
+
|
|
56
|
+
wait_timeout: int = 120
|
|
57
|
+
"""Seconds to wait for a sandbox to start."""
|
|
58
|
+
|
|
59
|
+
def __hash__(self) -> int:
|
|
60
|
+
# Inspect keeps configurations in dicts, and the dict and list fields are not hashable.
|
|
61
|
+
return hash(self.model_dump_json())
|
|
62
|
+
|
|
63
|
+
def check(self) -> None:
|
|
64
|
+
"""Fail with ``unsupported`` or ``invalid_argument`` before any request is made."""
|
|
65
|
+
if self.size is not None and self.size not in SIZES:
|
|
66
|
+
raise BriqError("unsupported", f"size is one of {', '.join(SIZES)}, not {self.size!r}")
|
|
67
|
+
if not _SLEEP.fullmatch(self.sleep):
|
|
68
|
+
raise BriqError("invalid_argument", "sleep is like '30m', '3h', '7d' or 'never'")
|
|
69
|
+
if not self.working_dir.startswith("/"):
|
|
70
|
+
raise BriqError("invalid_argument", "working_dir is an absolute path")
|
|
71
|
+
if not 1 <= self.default_exec_timeout <= 86_400:
|
|
72
|
+
raise BriqError("invalid_argument", "default_exec_timeout is 1 second to 24 hours")
|
|
73
|
+
for name in self.secret_env:
|
|
74
|
+
if not _ENV_NAME.fullmatch(name):
|
|
75
|
+
raise BriqError("invalid_argument", f"{name!r} is not a variable name")
|
|
76
|
+
if set(self.env) & set(self.secret_env):
|
|
77
|
+
raise BriqError("invalid_argument", "A variable is in both env and secret_env")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def resolve_config(config: object) -> BriqSandboxEnvironmentConfig:
|
|
81
|
+
"""What Inspect passes as ``config``: nothing, our object, or the path of a JSON file."""
|
|
82
|
+
if config is None:
|
|
83
|
+
resolved = BriqSandboxEnvironmentConfig()
|
|
84
|
+
elif isinstance(config, BriqSandboxEnvironmentConfig):
|
|
85
|
+
resolved = config
|
|
86
|
+
elif isinstance(config, str):
|
|
87
|
+
resolved = BriqSandboxEnvironmentConfig.model_validate(json.loads(Path(config).read_text()))
|
|
88
|
+
else:
|
|
89
|
+
raise TypeError(
|
|
90
|
+
"The briq sandbox takes a BriqSandboxEnvironmentConfig or the path of a JSON file, "
|
|
91
|
+
f"not {type(config).__name__}"
|
|
92
|
+
)
|
|
93
|
+
resolved.check()
|
|
94
|
+
return resolved
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""The ``briq`` sandbox environment: one briq sandbox per sample."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import errno
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import posixpath
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from typing import Any, Literal, overload
|
|
12
|
+
|
|
13
|
+
from briq_cloud import AsyncBriq, AsyncSandbox, BriqError
|
|
14
|
+
from inspect_ai.util import (
|
|
15
|
+
ExecResult,
|
|
16
|
+
OutputLimitExceededError,
|
|
17
|
+
SandboxEnvironment,
|
|
18
|
+
SandboxEnvironmentConfigType,
|
|
19
|
+
SandboxEnvironmentLimits,
|
|
20
|
+
SandboxUnavailableError,
|
|
21
|
+
SandboxUserUnsupportedError,
|
|
22
|
+
sandboxenv,
|
|
23
|
+
)
|
|
24
|
+
from pydantic import BaseModel
|
|
25
|
+
from typing_extensions import override
|
|
26
|
+
|
|
27
|
+
from ._config import BriqSandboxEnvironmentConfig, resolve_config
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@sandboxenv(name="briq")
|
|
33
|
+
class BriqSandboxEnvironment(SandboxEnvironment):
|
|
34
|
+
"""A sample's sandbox, a briq microVM that is destroyed when the sample ends.
|
|
35
|
+
|
|
36
|
+
Not supported, and refused instead of ignored: ``user`` (commands run as the sandbox's own
|
|
37
|
+
user), ``input`` that is not UTF-8, several sandboxes per sample, ``connection()``.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self, client: AsyncBriq, sandbox: AsyncSandbox, config: BriqSandboxEnvironmentConfig
|
|
42
|
+
) -> None:
|
|
43
|
+
super().__init__()
|
|
44
|
+
self._client = client
|
|
45
|
+
self._sandbox = sandbox
|
|
46
|
+
self._config = config
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def sandbox_id(self) -> str:
|
|
50
|
+
return self._sandbox.id
|
|
51
|
+
|
|
52
|
+
# --- the sandbox ---
|
|
53
|
+
|
|
54
|
+
@override
|
|
55
|
+
async def exec(
|
|
56
|
+
self,
|
|
57
|
+
cmd: list[str],
|
|
58
|
+
input: str | bytes | None = None,
|
|
59
|
+
cwd: str | None = None,
|
|
60
|
+
env: dict[str, str] | None = None,
|
|
61
|
+
user: str | None = None,
|
|
62
|
+
timeout: int | None = None,
|
|
63
|
+
timeout_retry: bool = True,
|
|
64
|
+
concurrency: bool = True,
|
|
65
|
+
) -> ExecResult[str]:
|
|
66
|
+
_refuse_user(user)
|
|
67
|
+
stdin = _text(input) if input is not None else None
|
|
68
|
+
with _inspect_errors():
|
|
69
|
+
result = await self._sandbox.exec(
|
|
70
|
+
cmd,
|
|
71
|
+
cwd=self._path(cwd) if cwd is not None else self._config.working_dir,
|
|
72
|
+
env=env or None,
|
|
73
|
+
stdin=stdin,
|
|
74
|
+
timeout=float(timeout or self._config.default_exec_timeout),
|
|
75
|
+
)
|
|
76
|
+
if result.timed_out:
|
|
77
|
+
raise TimeoutError(
|
|
78
|
+
f"The command did not end within {timeout or self._config.default_exec_timeout} s"
|
|
79
|
+
)
|
|
80
|
+
if result.exit_code is None:
|
|
81
|
+
raise RuntimeError("briq did not report how the command ended")
|
|
82
|
+
for stream in (result.stdout, result.stderr):
|
|
83
|
+
if len(stream.encode()) > SandboxEnvironmentLimits.MAX_EXEC_OUTPUT_SIZE:
|
|
84
|
+
raise OutputLimitExceededError(
|
|
85
|
+
SandboxEnvironmentLimits.MAX_EXEC_OUTPUT_SIZE_STR, stream[-4096:]
|
|
86
|
+
)
|
|
87
|
+
return ExecResult(
|
|
88
|
+
success=result.exit_code == 0,
|
|
89
|
+
returncode=result.exit_code,
|
|
90
|
+
stdout=result.stdout,
|
|
91
|
+
stderr=result.stderr,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
@override
|
|
95
|
+
async def write_file(
|
|
96
|
+
self, file: str, contents: str | bytes, *, user: str | None = None
|
|
97
|
+
) -> None:
|
|
98
|
+
_refuse_user(user)
|
|
99
|
+
data = contents.encode() if isinstance(contents, str) else contents
|
|
100
|
+
with _inspect_errors(file):
|
|
101
|
+
await self._sandbox.write_file(self._path(file), data)
|
|
102
|
+
|
|
103
|
+
@overload
|
|
104
|
+
async def read_file(
|
|
105
|
+
self, file: str, text: Literal[True] = True, *, user: str | None = None
|
|
106
|
+
) -> str: ...
|
|
107
|
+
|
|
108
|
+
@overload
|
|
109
|
+
async def read_file(
|
|
110
|
+
self, file: str, text: Literal[False], *, user: str | None = None
|
|
111
|
+
) -> bytes: ...
|
|
112
|
+
|
|
113
|
+
@override
|
|
114
|
+
async def read_file(
|
|
115
|
+
self, file: str, text: bool = True, *, user: str | None = None
|
|
116
|
+
) -> str | bytes:
|
|
117
|
+
_refuse_user(user)
|
|
118
|
+
with _inspect_errors(file):
|
|
119
|
+
data = await self._sandbox.read_file(self._path(file))
|
|
120
|
+
if len(data) > SandboxEnvironmentLimits.MAX_READ_FILE_SIZE:
|
|
121
|
+
raise OutputLimitExceededError(SandboxEnvironmentLimits.MAX_READ_FILE_SIZE_STR, None)
|
|
122
|
+
return data.decode("utf-8") if text else data
|
|
123
|
+
|
|
124
|
+
def _path(self, path: str) -> str:
|
|
125
|
+
"""Relative paths resolve against ``working_dir``, as the Inspect contract says."""
|
|
126
|
+
return posixpath.join(self._config.working_dir, path)
|
|
127
|
+
|
|
128
|
+
# --- the lifecycle ---
|
|
129
|
+
|
|
130
|
+
@override
|
|
131
|
+
@classmethod
|
|
132
|
+
async def task_init(cls, task_name: str, config: SandboxEnvironmentConfigType | None) -> None:
|
|
133
|
+
"""Fail before the first sample when the configuration or the API key is wrong."""
|
|
134
|
+
resolved = resolve_config(config)
|
|
135
|
+
client = AsyncBriq(base_url=resolved.base_url) # no key, or a bad one: raises here
|
|
136
|
+
await client.close()
|
|
137
|
+
_secrets(resolved)
|
|
138
|
+
|
|
139
|
+
@override
|
|
140
|
+
@classmethod
|
|
141
|
+
async def sample_init(
|
|
142
|
+
cls,
|
|
143
|
+
task_name: str,
|
|
144
|
+
config: SandboxEnvironmentConfigType | None,
|
|
145
|
+
metadata: dict[str, str],
|
|
146
|
+
) -> dict[str, SandboxEnvironment]:
|
|
147
|
+
resolved = resolve_config(config)
|
|
148
|
+
secrets = _secrets(resolved)
|
|
149
|
+
client = AsyncBriq(base_url=resolved.base_url)
|
|
150
|
+
try:
|
|
151
|
+
sandbox = await client.create_sandbox(
|
|
152
|
+
size=resolved.size,
|
|
153
|
+
image=resolved.image,
|
|
154
|
+
env=resolved.env or None,
|
|
155
|
+
secret_env=secrets or None,
|
|
156
|
+
sleep=resolved.sleep,
|
|
157
|
+
wait_timeout=resolved.wait_timeout,
|
|
158
|
+
)
|
|
159
|
+
except BaseException:
|
|
160
|
+
await _close(client)
|
|
161
|
+
raise
|
|
162
|
+
return {"default": cls(client, sandbox, resolved)}
|
|
163
|
+
|
|
164
|
+
@override
|
|
165
|
+
@classmethod
|
|
166
|
+
async def sample_cleanup(
|
|
167
|
+
cls,
|
|
168
|
+
task_name: str,
|
|
169
|
+
config: SandboxEnvironmentConfigType | None,
|
|
170
|
+
environments: dict[str, SandboxEnvironment],
|
|
171
|
+
interrupted: bool,
|
|
172
|
+
) -> None:
|
|
173
|
+
for environment in environments.values():
|
|
174
|
+
if not isinstance(environment, BriqSandboxEnvironment):
|
|
175
|
+
continue
|
|
176
|
+
try:
|
|
177
|
+
await environment._sandbox.destroy()
|
|
178
|
+
except Exception as error:
|
|
179
|
+
# A failed cleanup must not fail the sample. An idle sandbox sleeps, unbilled.
|
|
180
|
+
logger.warning(
|
|
181
|
+
"Could not destroy briq sandbox %s: %s", environment.sandbox_id, error
|
|
182
|
+
)
|
|
183
|
+
finally:
|
|
184
|
+
await _close(environment._client)
|
|
185
|
+
|
|
186
|
+
@override
|
|
187
|
+
@classmethod
|
|
188
|
+
def config_deserialize(cls, config: dict[str, Any]) -> BaseModel:
|
|
189
|
+
return BriqSandboxEnvironmentConfig.model_validate(config)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _secrets(config: BriqSandboxEnvironmentConfig) -> dict[str, str]:
|
|
193
|
+
"""The values of ``secret_env``, read from this process."""
|
|
194
|
+
missing = [name for name in config.secret_env if name not in os.environ]
|
|
195
|
+
if missing:
|
|
196
|
+
raise BriqError(
|
|
197
|
+
"invalid_argument", f"secret_env names variables that are not set: {', '.join(missing)}"
|
|
198
|
+
)
|
|
199
|
+
return {name: os.environ[name] for name in config.secret_env}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _refuse_user(user: str | None) -> None:
|
|
203
|
+
if user is not None:
|
|
204
|
+
raise SandboxUserUnsupportedError(
|
|
205
|
+
"briq runs commands as the sandbox's own user: it cannot run them as another"
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _text(value: str | bytes) -> str:
|
|
210
|
+
if isinstance(value, str):
|
|
211
|
+
return value
|
|
212
|
+
try:
|
|
213
|
+
return value.decode("utf-8")
|
|
214
|
+
except UnicodeDecodeError:
|
|
215
|
+
raise BriqError(
|
|
216
|
+
"unsupported", "briq takes text for stdin: the input is not UTF-8"
|
|
217
|
+
) from None
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
async def _close(client: AsyncBriq) -> None:
|
|
221
|
+
try:
|
|
222
|
+
await client.close()
|
|
223
|
+
except Exception:
|
|
224
|
+
logger.debug("Could not close the briq client", exc_info=True)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@contextmanager
|
|
228
|
+
def _inspect_errors(file: str | None = None) -> Iterator[None]:
|
|
229
|
+
"""A ``BriqError`` as the exception Inspect's contract names, where there is one."""
|
|
230
|
+
try:
|
|
231
|
+
yield
|
|
232
|
+
except BriqError as error:
|
|
233
|
+
if error.code == "file_not_found":
|
|
234
|
+
raise FileNotFoundError(errno.ENOENT, "No such file or directory.", file) from error
|
|
235
|
+
if error.code in ("unavailable", "not_found"):
|
|
236
|
+
raise SandboxUnavailableError(str(error)) from error
|
|
237
|
+
if error.code == "timeout":
|
|
238
|
+
raise TimeoutError(str(error)) from error
|
|
239
|
+
raise
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
File without changes
|