inspect-drex 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inspect_drex-0.1.0/.github/workflows/ci.yaml +61 -0
- inspect_drex-0.1.0/.github/workflows/release.yaml +96 -0
- inspect_drex-0.1.0/.gitignore +11 -0
- inspect_drex-0.1.0/CHANGELOG.md +15 -0
- inspect_drex-0.1.0/LICENSE +21 -0
- inspect_drex-0.1.0/PKG-INFO +206 -0
- inspect_drex-0.1.0/README.md +176 -0
- inspect_drex-0.1.0/examples/support_eval.py +102 -0
- inspect_drex-0.1.0/inspect_drex/__init__.py +35 -0
- inspect_drex-0.1.0/inspect_drex/_client.py +279 -0
- inspect_drex-0.1.0/inspect_drex/_images.py +84 -0
- inspect_drex-0.1.0/inspect_drex/_registry.py +5 -0
- inspect_drex-0.1.0/inspect_drex/_version.py +1 -0
- inspect_drex-0.1.0/inspect_drex/criteria.py +153 -0
- inspect_drex-0.1.0/inspect_drex/errors.py +116 -0
- inspect_drex-0.1.0/inspect_drex/scorer.py +225 -0
- inspect_drex-0.1.0/pyproject.toml +82 -0
- inspect_drex-0.1.0/tests/__init__.py +0 -0
- inspect_drex-0.1.0/tests/fakes.py +132 -0
- inspect_drex-0.1.0/tests/fixtures/blue.png +0 -0
- inspect_drex-0.1.0/tests/fixtures/red.png +0 -0
- inspect_drex-0.1.0/tests/integration_tests/__init__.py +0 -0
- inspect_drex-0.1.0/tests/integration_tests/conftest.py +41 -0
- inspect_drex-0.1.0/tests/integration_tests/test_live.py +119 -0
- inspect_drex-0.1.0/tests/integration_tests/test_live_openrouter.py +81 -0
- inspect_drex-0.1.0/tests/unit_tests/__init__.py +0 -0
- inspect_drex-0.1.0/tests/unit_tests/conftest.py +84 -0
- inspect_drex-0.1.0/tests/unit_tests/test_client.py +436 -0
- inspect_drex-0.1.0/tests/unit_tests/test_helpers.py +123 -0
- inspect_drex-0.1.0/tests/unit_tests/test_scorer.py +463 -0
- inspect_drex-0.1.0/tests/unit_tests/test_version.py +31 -0
- inspect_drex-0.1.0/uv.lock +3001 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ci-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
lint:
|
|
17
|
+
name: Lint
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
timeout-minutes: 10
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
|
|
23
|
+
- name: Install uv
|
|
24
|
+
uses: astral-sh/setup-uv@v4
|
|
25
|
+
with:
|
|
26
|
+
enable-cache: true
|
|
27
|
+
cache-dependency-glob: "uv.lock"
|
|
28
|
+
|
|
29
|
+
- name: Install dependencies
|
|
30
|
+
run: uv sync --all-groups --locked --python 3.12
|
|
31
|
+
|
|
32
|
+
- name: Ruff check
|
|
33
|
+
run: uv run ruff check .
|
|
34
|
+
|
|
35
|
+
- name: Ruff format
|
|
36
|
+
run: uv run ruff format --check .
|
|
37
|
+
|
|
38
|
+
test:
|
|
39
|
+
name: Offline tests (Python ${{ matrix.python-version }})
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
timeout-minutes: 15
|
|
42
|
+
strategy:
|
|
43
|
+
fail-fast: false
|
|
44
|
+
matrix:
|
|
45
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
46
|
+
steps:
|
|
47
|
+
- uses: actions/checkout@v4
|
|
48
|
+
|
|
49
|
+
- name: Install uv
|
|
50
|
+
uses: astral-sh/setup-uv@v4
|
|
51
|
+
with:
|
|
52
|
+
enable-cache: true
|
|
53
|
+
cache-dependency-glob: "uv.lock"
|
|
54
|
+
|
|
55
|
+
- name: Install dependencies
|
|
56
|
+
run: uv sync --all-groups --locked --python ${{ matrix.python-version }}
|
|
57
|
+
|
|
58
|
+
# The live tests need keys and make billed requests, so only the offline
|
|
59
|
+
# suite runs here: `tests/unit_tests` leaves them out of collection.
|
|
60
|
+
- name: Offline tests
|
|
61
|
+
run: uv run pytest tests/unit_tests --cov
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Publishes inspect-drex to PyPI when an `inspect-drex-v*` tag is pushed.
|
|
2
|
+
#
|
|
3
|
+
# Releasing is deliberate: `git tag inspect-drex-v0.1.0 && git push origin inspect-drex-v0.1.0`.
|
|
4
|
+
# A PyPI upload cannot be undone or replaced, so everything that can be checked runs
|
|
5
|
+
# before the upload, and `workflow_dispatch` runs the same checks plus the build while
|
|
6
|
+
# skipping the publish, so the pipeline can be rehearsed.
|
|
7
|
+
name: Release
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
push:
|
|
11
|
+
tags: ["inspect-drex-v*"]
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
17
|
+
concurrency:
|
|
18
|
+
group: release-${{ github.ref }}
|
|
19
|
+
# Never cancel: a half-finished release is worse than a queued one.
|
|
20
|
+
cancel-in-progress: false
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
verify:
|
|
24
|
+
name: Verify and build
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
timeout-minutes: 15
|
|
27
|
+
steps:
|
|
28
|
+
- uses: actions/checkout@v4
|
|
29
|
+
|
|
30
|
+
- name: Install uv
|
|
31
|
+
uses: astral-sh/setup-uv@v4
|
|
32
|
+
with:
|
|
33
|
+
enable-cache: true
|
|
34
|
+
cache-dependency-glob: "uv.lock"
|
|
35
|
+
|
|
36
|
+
- name: Verify lockfile and install dependencies
|
|
37
|
+
run: uv sync --all-groups --locked --python 3.12
|
|
38
|
+
|
|
39
|
+
# The tag names the version users will get, so a tag that disagrees with the
|
|
40
|
+
# package is caught here rather than after an unrepeatable upload.
|
|
41
|
+
- name: Tag matches package version
|
|
42
|
+
if: github.event_name == 'push'
|
|
43
|
+
run: |
|
|
44
|
+
tag_version="${GITHUB_REF_NAME#inspect-drex-v}"
|
|
45
|
+
pkg_version="$(uv run python -c "from inspect_drex import __version__; print(__version__)")"
|
|
46
|
+
if [ "$tag_version" != "$pkg_version" ]; then
|
|
47
|
+
echo "Tag ${GITHUB_REF_NAME} would publish $pkg_version, not $tag_version." >&2
|
|
48
|
+
echo "Bump inspect_drex/_version.py or retag before releasing." >&2
|
|
49
|
+
exit 1
|
|
50
|
+
fi
|
|
51
|
+
echo "Releasing inspect-drex $pkg_version"
|
|
52
|
+
|
|
53
|
+
- name: Ruff
|
|
54
|
+
run: uv run ruff check .
|
|
55
|
+
|
|
56
|
+
- name: Offline tests
|
|
57
|
+
run: uv run pytest tests/unit_tests --cov
|
|
58
|
+
|
|
59
|
+
- name: Build distributions
|
|
60
|
+
run: uv build --out-dir dist
|
|
61
|
+
|
|
62
|
+
# The README becomes the PyPI project page; twine rejects markup PyPI cannot
|
|
63
|
+
# render, which is otherwise only discovered at upload time.
|
|
64
|
+
- name: Check distribution metadata
|
|
65
|
+
run: uvx twine check dist/*
|
|
66
|
+
|
|
67
|
+
- uses: actions/upload-artifact@v4
|
|
68
|
+
with:
|
|
69
|
+
name: inspect-drex-dist
|
|
70
|
+
path: dist/
|
|
71
|
+
if-no-files-found: error
|
|
72
|
+
|
|
73
|
+
publish:
|
|
74
|
+
name: Publish to PyPI
|
|
75
|
+
needs: verify
|
|
76
|
+
# Only a tag publishes; workflow_dispatch stops after verify.
|
|
77
|
+
if: github.event_name == 'push'
|
|
78
|
+
runs-on: ubuntu-latest
|
|
79
|
+
timeout-minutes: 15
|
|
80
|
+
# Half of the Trusted Publishing identity, and where release reviewers get
|
|
81
|
+
# added later without touching this file.
|
|
82
|
+
environment: pypi
|
|
83
|
+
permissions:
|
|
84
|
+
# Trusted Publishing mints a short-lived token from this; no PyPI
|
|
85
|
+
# credential is stored in the repository.
|
|
86
|
+
id-token: write
|
|
87
|
+
steps:
|
|
88
|
+
- uses: actions/download-artifact@v4
|
|
89
|
+
with:
|
|
90
|
+
name: inspect-drex-dist
|
|
91
|
+
path: dist
|
|
92
|
+
|
|
93
|
+
# Pinned to a commit: anything running in this job can mint the PyPI token
|
|
94
|
+
# for inspect-drex. Update it deliberately when a new release is reviewed.
|
|
95
|
+
- name: Publish
|
|
96
|
+
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- `drex_scorer`: an Inspect scorer that asks Drex yes/no (`noul`), `choice` and `score`
|
|
8
|
+
questions about a sample's output and returns the pass probability as `Score.value`, with
|
|
9
|
+
Drex's full probabilities kept in `Score.metadata["drex"]`.
|
|
10
|
+
- Registered with Inspect through the `inspect_ai` entry point, so
|
|
11
|
+
`inspect score --scorer inspect_drex/drex_scorer` works.
|
|
12
|
+
- Image input for Drex Max, refused before sending for text-only models.
|
|
13
|
+
- Drex through OpenRouter (`nace-ai/<model>` ids) with the key following the host.
|
|
14
|
+
- Self-hosted Drex deployments through `NACE_BASE_URL` or `base_url=`.
|
|
15
|
+
- Errors raise by default; `on_error="unscored"` leaves the sample out of the metrics.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nace AI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: inspect-drex
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An Inspect (UK AISI) scorer backed by Drex decision models from Nace
|
|
5
|
+
Project-URL: Homepage, https://console.nace.ai/docs
|
|
6
|
+
Project-URL: Repository, https://github.com/nace-ai/inspect-drex
|
|
7
|
+
Project-URL: Issues, https://github.com/nace-ai/inspect-drex/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/nace-ai/inspect-drex/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Nace AI <hello@nace.ai>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: drex,evals,inspect,inspect-ai,llm-evaluation,nace,scorer
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Framework :: Pytest
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
25
|
+
Requires-Python: <4.0,>=3.11
|
|
26
|
+
Requires-Dist: httpx<1.0.0,>=0.27.0
|
|
27
|
+
Requires-Dist: inspect-ai>=0.3.245
|
|
28
|
+
Requires-Dist: nace-sdk<1.0.0,>=0.4.0
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# inspect-drex
|
|
32
|
+
|
|
33
|
+
A scorer for [Inspect](https://inspect.aisi.org.uk) (the UK AI Security Institute's evaluation
|
|
34
|
+
framework) that asks [Drex](https://console.nace.ai/docs) decision models from Nace whether a
|
|
35
|
+
sample's output meets your criteria. Drex answers typed questions and returns
|
|
36
|
+
**probabilities**, so you can choose the pass threshold after the run instead of before it.
|
|
37
|
+
|
|
38
|
+
- Drex Max reads text, JSON and images. `drex-v2.0` reads text and JSON only.
|
|
39
|
+
- Any other model id that Drex serves is passed through unchanged; there is no closed list.
|
|
40
|
+
- The default model is `drex-max`, which moves to the newest flagship with each Drex release.
|
|
41
|
+
Pin `model=` (or set `NACE_DEFAULT_DECISION_MODEL`) if you need results that never change.
|
|
42
|
+
See the Drex docs for each model's request limits.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install inspect-drex
|
|
48
|
+
export NACE_API_KEY=... # an API key from the Drex console
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Python 3.11 or newer (the floor of `inspect-ai`).
|
|
52
|
+
|
|
53
|
+
`NACE_BASE_URL` is optional. Set it, or pass `base_url=`, for a self-hosted Drex deployment on
|
|
54
|
+
your own domain. Any host that is not OpenRouter is treated as Drex itself: requests go to
|
|
55
|
+
`<base_url>/v1/systemone` and use your `NACE_API_KEY`.
|
|
56
|
+
|
|
57
|
+
## Use
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from inspect_ai import Task, task
|
|
61
|
+
from inspect_ai.dataset import Sample
|
|
62
|
+
from inspect_ai.solver import generate
|
|
63
|
+
from inspect_drex import Criterion, drex_scorer
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@task
|
|
67
|
+
def support():
|
|
68
|
+
return Task(
|
|
69
|
+
dataset=[Sample(input="How long do refunds take?", target="5 business days")],
|
|
70
|
+
solver=generate(),
|
|
71
|
+
scorer=drex_scorer(
|
|
72
|
+
[
|
|
73
|
+
Criterion.noul(
|
|
74
|
+
"grounded", "Does the answer state the reference fact?", threshold=0.7
|
|
75
|
+
),
|
|
76
|
+
Criterion.noul("filler", "Is the answer padded with filler?", want=False),
|
|
77
|
+
],
|
|
78
|
+
model="drex-max",
|
|
79
|
+
),
|
|
80
|
+
)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Save it as `support.py` and run it with any model Inspect supports for the solver:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
inspect eval support.py --model openai/gpt-4o-mini
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
A rubric string is a single yes/no criterion: `drex_scorer("The answer is helpful.")`.
|
|
90
|
+
The package registers itself with Inspect through an entry point, so a finished log can also be
|
|
91
|
+
scored from the CLI:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
inspect score logs/run.eval --scorer inspect_drex/drex_scorer -S criteria="The answer is helpful."
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`examples/support_eval.py` is an offline example (a mock model and a fake Drex server).
|
|
98
|
+
|
|
99
|
+
### What you get back
|
|
100
|
+
|
|
101
|
+
- `Score.value` is the 0-1 **pass probability** (weighted mean of the criteria, or `min` with
|
|
102
|
+
`aggregate="min"`). `mean()` and `stderr()` report it. With `output="grade"` it is `CORRECT`
|
|
103
|
+
or `INCORRECT` by each criterion's own `threshold`, and `mean()` is the pass rate.
|
|
104
|
+
- `Score.metadata["drex"]` keeps what Drex returned: `request_id`, `model`, `input_tokens`,
|
|
105
|
+
`image_tokens`, `evaluation_time_ms`, and per criterion its kind, pass score, threshold,
|
|
106
|
+
verdict and full probabilities (noul: `yes` / `no`; choice and score: per label).
|
|
107
|
+
Input tokens are counted once per request, not once per question.
|
|
108
|
+
- For an unscored sample, `Score.metadata["drex"]` holds only `error` (the exception class
|
|
109
|
+
name); `Score.explanation` has the message, including the status and `request_id`.
|
|
110
|
+
- `Score.explanation` lists the criteria that failed, with their probabilities.
|
|
111
|
+
|
|
112
|
+
### Criteria and score direction
|
|
113
|
+
|
|
114
|
+
`Criterion.noul` asks a yes/no question, `Criterion.choice` picks among labelled options and
|
|
115
|
+
`Criterion.score` places the output on an ordered scale. A criterion's pass score always means
|
|
116
|
+
"how well it holds": higher is better. A *defect* question ("is the reply rude?") needs
|
|
117
|
+
`want=False`, which scores `1 - P(yes)`. Choice criteria sum the probability of the labels you
|
|
118
|
+
mark `good`. Score criteria use the expected level over `levels - 1` (or its complement with
|
|
119
|
+
`invert=True`); an expected level is a fraction that can fall between two levels, and the raw
|
|
120
|
+
level is kept in the detail text. See Drex's guides on
|
|
121
|
+
[questions](https://console.nace.ai/docs/drex/guides/questions) and
|
|
122
|
+
[reading answers](https://console.nace.ai/docs/drex/guides/reading-answers).
|
|
123
|
+
|
|
124
|
+
### What Drex sees
|
|
125
|
+
|
|
126
|
+
By default `{"input", "output", "reference"}`: the user messages, the model output and the
|
|
127
|
+
sample target. Give Drex only what the criteria need, or pass your own
|
|
128
|
+
`state=lambda state, target: ...` (and `images=` for Drex Max).
|
|
129
|
+
|
|
130
|
+
Images in the user's messages are sent in order, and the input text names each one
|
|
131
|
+
`[Image 1]`, `[Image 2]`, so a criterion can say "Image 1". Images are never dropped: with a
|
|
132
|
+
text-only model (`drex-v2.0`, `drex-v1.5`, `drex-latest`) the scorer raises before sending
|
|
133
|
+
(with `on_error="unscored"` the sample is left unscored instead). Drex never fetches URLs, so
|
|
134
|
+
remote image URLs are refused; use data URLs or local files. Keep each request within Drex's
|
|
135
|
+
size limit (inline images count, base64 included). For larger images, upload them with
|
|
136
|
+
`nace-sdk` and pass the uploaded file through `images=`.
|
|
137
|
+
|
|
138
|
+
### Errors are not failed criteria
|
|
139
|
+
|
|
140
|
+
A 401 (bad key), 402 (no funds; the message links the billing page), 422 (each issue's
|
|
141
|
+
`path: message` is shown) or an outage raises, and Inspect marks the sample as errored. With
|
|
142
|
+
`on_error="unscored"` the sample stays in the log as `Score.unscored(reason="scoring_failed")`
|
|
143
|
+
and is left out of the metrics. 429 and 529 are retried by `nace-sdk`, honouring
|
|
144
|
+
`retry-after-ms`. The client timeout is at least 60 s, and the client caps concurrent requests
|
|
145
|
+
(`max_in_flight=`, default 8).
|
|
146
|
+
|
|
147
|
+
### OpenRouter
|
|
148
|
+
|
|
149
|
+
Drex is also served through OpenRouter. Pass `base_url="https://openrouter.ai/api/v1"` and use
|
|
150
|
+
model ids such as `nace-ai/<model>`; a bare id (`drex-max`) is sent as `nace-ai/drex-max`, and ids
|
|
151
|
+
with a provider prefix are sent as given. `drex-latest` is refused because OpenRouter has no id
|
|
152
|
+
for it.
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
export OPENROUTER_API_KEY=...
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
drex_scorer(
|
|
160
|
+
"The answer is helpful.",
|
|
161
|
+
base_url="https://openrouter.ai/api/v1",
|
|
162
|
+
model="nace-ai/drex-max",
|
|
163
|
+
)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The API key follows the host: for OpenRouter it is read from `OPENROUTER_API_KEY`, never from
|
|
167
|
+
`NACE_API_KEY`, so a Nace key is not sent to OpenRouter and an OpenRouter key is not sent to
|
|
168
|
+
Nace. Through OpenRouter, images work only inline (in `state` content parts); uploaded and
|
|
169
|
+
workspace-file images are refused.
|
|
170
|
+
|
|
171
|
+
### Secrets and the eval log
|
|
172
|
+
|
|
173
|
+
Inspect writes scorer arguments to the eval log in plain text, so `drex_scorer` refuses
|
|
174
|
+
`api_key=` and `extra_headers=`. Put the key in the environment, or build the client yourself
|
|
175
|
+
and pass `client=AsyncDrexClient(base_url=..., api_key=..., model=...)`.
|
|
176
|
+
|
|
177
|
+
### Attribution
|
|
178
|
+
|
|
179
|
+
Every request sends `User-Agent` and `X-Nace-Client` as `inspect-drex/<version>`, so Nace can
|
|
180
|
+
count usage per integration.
|
|
181
|
+
|
|
182
|
+
## What is verified
|
|
183
|
+
|
|
184
|
+
- The offline tests (`tests/unit_tests`) run against a fake Drex server and are the CI gate
|
|
185
|
+
on Python 3.11, 3.12 and 3.13.
|
|
186
|
+
- The live tests passed against production Drex and against OpenRouter on 2026-10-29. They are
|
|
187
|
+
not run in CI, because they need keys and make billed requests.
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
uv sync --all-groups
|
|
191
|
+
uv run ruff check . && uv run ruff format --check .
|
|
192
|
+
uv run pytest tests/unit_tests --cov # offline
|
|
193
|
+
NACE_API_KEY=... uv run pytest tests/integration_tests -m live -s -rs
|
|
194
|
+
OPENROUTER_API_KEY=... uv run pytest tests/integration_tests -m openrouter -s -rs
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Limits
|
|
198
|
+
|
|
199
|
+
- Images need a model that reads them (Drex Max); text-only models raise before sending.
|
|
200
|
+
- Each request must fit Drex's size and question limits; one request is made per sample, however
|
|
201
|
+
many criteria it holds.
|
|
202
|
+
- The scorer judges what you give it. It does not fetch URLs or files for the judge.
|
|
203
|
+
|
|
204
|
+
## License
|
|
205
|
+
|
|
206
|
+
MIT. See [LICENSE](LICENSE). Issues: <https://github.com/nace-ai/inspect-drex/issues>.
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# inspect-drex
|
|
2
|
+
|
|
3
|
+
A scorer for [Inspect](https://inspect.aisi.org.uk) (the UK AI Security Institute's evaluation
|
|
4
|
+
framework) that asks [Drex](https://console.nace.ai/docs) decision models from Nace whether a
|
|
5
|
+
sample's output meets your criteria. Drex answers typed questions and returns
|
|
6
|
+
**probabilities**, so you can choose the pass threshold after the run instead of before it.
|
|
7
|
+
|
|
8
|
+
- Drex Max reads text, JSON and images. `drex-v2.0` reads text and JSON only.
|
|
9
|
+
- Any other model id that Drex serves is passed through unchanged; there is no closed list.
|
|
10
|
+
- The default model is `drex-max`, which moves to the newest flagship with each Drex release.
|
|
11
|
+
Pin `model=` (or set `NACE_DEFAULT_DECISION_MODEL`) if you need results that never change.
|
|
12
|
+
See the Drex docs for each model's request limits.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install inspect-drex
|
|
18
|
+
export NACE_API_KEY=... # an API key from the Drex console
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Python 3.11 or newer (the floor of `inspect-ai`).
|
|
22
|
+
|
|
23
|
+
`NACE_BASE_URL` is optional. Set it, or pass `base_url=`, for a self-hosted Drex deployment on
|
|
24
|
+
your own domain. Any host that is not OpenRouter is treated as Drex itself: requests go to
|
|
25
|
+
`<base_url>/v1/systemone` and use your `NACE_API_KEY`.
|
|
26
|
+
|
|
27
|
+
## Use
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from inspect_ai import Task, task
|
|
31
|
+
from inspect_ai.dataset import Sample
|
|
32
|
+
from inspect_ai.solver import generate
|
|
33
|
+
from inspect_drex import Criterion, drex_scorer
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@task
|
|
37
|
+
def support():
|
|
38
|
+
return Task(
|
|
39
|
+
dataset=[Sample(input="How long do refunds take?", target="5 business days")],
|
|
40
|
+
solver=generate(),
|
|
41
|
+
scorer=drex_scorer(
|
|
42
|
+
[
|
|
43
|
+
Criterion.noul(
|
|
44
|
+
"grounded", "Does the answer state the reference fact?", threshold=0.7
|
|
45
|
+
),
|
|
46
|
+
Criterion.noul("filler", "Is the answer padded with filler?", want=False),
|
|
47
|
+
],
|
|
48
|
+
model="drex-max",
|
|
49
|
+
),
|
|
50
|
+
)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Save it as `support.py` and run it with any model Inspect supports for the solver:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
inspect eval support.py --model openai/gpt-4o-mini
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
A rubric string is a single yes/no criterion: `drex_scorer("The answer is helpful.")`.
|
|
60
|
+
The package registers itself with Inspect through an entry point, so a finished log can also be
|
|
61
|
+
scored from the CLI:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
inspect score logs/run.eval --scorer inspect_drex/drex_scorer -S criteria="The answer is helpful."
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`examples/support_eval.py` is an offline example (a mock model and a fake Drex server).
|
|
68
|
+
|
|
69
|
+
### What you get back
|
|
70
|
+
|
|
71
|
+
- `Score.value` is the 0-1 **pass probability** (weighted mean of the criteria, or `min` with
|
|
72
|
+
`aggregate="min"`). `mean()` and `stderr()` report it. With `output="grade"` it is `CORRECT`
|
|
73
|
+
or `INCORRECT` by each criterion's own `threshold`, and `mean()` is the pass rate.
|
|
74
|
+
- `Score.metadata["drex"]` keeps what Drex returned: `request_id`, `model`, `input_tokens`,
|
|
75
|
+
`image_tokens`, `evaluation_time_ms`, and per criterion its kind, pass score, threshold,
|
|
76
|
+
verdict and full probabilities (noul: `yes` / `no`; choice and score: per label).
|
|
77
|
+
Input tokens are counted once per request, not once per question.
|
|
78
|
+
- For an unscored sample, `Score.metadata["drex"]` holds only `error` (the exception class
|
|
79
|
+
name); `Score.explanation` has the message, including the status and `request_id`.
|
|
80
|
+
- `Score.explanation` lists the criteria that failed, with their probabilities.
|
|
81
|
+
|
|
82
|
+
### Criteria and score direction
|
|
83
|
+
|
|
84
|
+
`Criterion.noul` asks a yes/no question, `Criterion.choice` picks among labelled options and
|
|
85
|
+
`Criterion.score` places the output on an ordered scale. A criterion's pass score always means
|
|
86
|
+
"how well it holds": higher is better. A *defect* question ("is the reply rude?") needs
|
|
87
|
+
`want=False`, which scores `1 - P(yes)`. Choice criteria sum the probability of the labels you
|
|
88
|
+
mark `good`. Score criteria use the expected level over `levels - 1` (or its complement with
|
|
89
|
+
`invert=True`); an expected level is a fraction that can fall between two levels, and the raw
|
|
90
|
+
level is kept in the detail text. See Drex's guides on
|
|
91
|
+
[questions](https://console.nace.ai/docs/drex/guides/questions) and
|
|
92
|
+
[reading answers](https://console.nace.ai/docs/drex/guides/reading-answers).
|
|
93
|
+
|
|
94
|
+
### What Drex sees
|
|
95
|
+
|
|
96
|
+
By default `{"input", "output", "reference"}`: the user messages, the model output and the
|
|
97
|
+
sample target. Give Drex only what the criteria need, or pass your own
|
|
98
|
+
`state=lambda state, target: ...` (and `images=` for Drex Max).
|
|
99
|
+
|
|
100
|
+
Images in the user's messages are sent in order, and the input text names each one
|
|
101
|
+
`[Image 1]`, `[Image 2]`, so a criterion can say "Image 1". Images are never dropped: with a
|
|
102
|
+
text-only model (`drex-v2.0`, `drex-v1.5`, `drex-latest`) the scorer raises before sending
|
|
103
|
+
(with `on_error="unscored"` the sample is left unscored instead). Drex never fetches URLs, so
|
|
104
|
+
remote image URLs are refused; use data URLs or local files. Keep each request within Drex's
|
|
105
|
+
size limit (inline images count, base64 included). For larger images, upload them with
|
|
106
|
+
`nace-sdk` and pass the uploaded file through `images=`.
|
|
107
|
+
|
|
108
|
+
### Errors are not failed criteria
|
|
109
|
+
|
|
110
|
+
A 401 (bad key), 402 (no funds; the message links the billing page), 422 (each issue's
|
|
111
|
+
`path: message` is shown) or an outage raises, and Inspect marks the sample as errored. With
|
|
112
|
+
`on_error="unscored"` the sample stays in the log as `Score.unscored(reason="scoring_failed")`
|
|
113
|
+
and is left out of the metrics. 429 and 529 are retried by `nace-sdk`, honouring
|
|
114
|
+
`retry-after-ms`. The client timeout is at least 60 s, and the client caps concurrent requests
|
|
115
|
+
(`max_in_flight=`, default 8).
|
|
116
|
+
|
|
117
|
+
### OpenRouter
|
|
118
|
+
|
|
119
|
+
Drex is also served through OpenRouter. Pass `base_url="https://openrouter.ai/api/v1"` and use
|
|
120
|
+
model ids such as `nace-ai/<model>`; a bare id (`drex-max`) is sent as `nace-ai/drex-max`, and ids
|
|
121
|
+
with a provider prefix are sent as given. `drex-latest` is refused because OpenRouter has no id
|
|
122
|
+
for it.
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
export OPENROUTER_API_KEY=...
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
drex_scorer(
|
|
130
|
+
"The answer is helpful.",
|
|
131
|
+
base_url="https://openrouter.ai/api/v1",
|
|
132
|
+
model="nace-ai/drex-max",
|
|
133
|
+
)
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
The API key follows the host: for OpenRouter it is read from `OPENROUTER_API_KEY`, never from
|
|
137
|
+
`NACE_API_KEY`, so a Nace key is not sent to OpenRouter and an OpenRouter key is not sent to
|
|
138
|
+
Nace. Through OpenRouter, images work only inline (in `state` content parts); uploaded and
|
|
139
|
+
workspace-file images are refused.
|
|
140
|
+
|
|
141
|
+
### Secrets and the eval log
|
|
142
|
+
|
|
143
|
+
Inspect writes scorer arguments to the eval log in plain text, so `drex_scorer` refuses
|
|
144
|
+
`api_key=` and `extra_headers=`. Put the key in the environment, or build the client yourself
|
|
145
|
+
and pass `client=AsyncDrexClient(base_url=..., api_key=..., model=...)`.
|
|
146
|
+
|
|
147
|
+
### Attribution
|
|
148
|
+
|
|
149
|
+
Every request sends `User-Agent` and `X-Nace-Client` as `inspect-drex/<version>`, so Nace can
|
|
150
|
+
count usage per integration.
|
|
151
|
+
|
|
152
|
+
## What is verified
|
|
153
|
+
|
|
154
|
+
- The offline tests (`tests/unit_tests`) run against a fake Drex server and are the CI gate
|
|
155
|
+
on Python 3.11, 3.12 and 3.13.
|
|
156
|
+
- The live tests passed against production Drex and against OpenRouter on 2026-10-29. They are
|
|
157
|
+
not run in CI, because they need keys and make billed requests.
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
uv sync --all-groups
|
|
161
|
+
uv run ruff check . && uv run ruff format --check .
|
|
162
|
+
uv run pytest tests/unit_tests --cov # offline
|
|
163
|
+
NACE_API_KEY=... uv run pytest tests/integration_tests -m live -s -rs
|
|
164
|
+
OPENROUTER_API_KEY=... uv run pytest tests/integration_tests -m openrouter -s -rs
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Limits
|
|
168
|
+
|
|
169
|
+
- Images need a model that reads them (Drex Max); text-only models raise before sending.
|
|
170
|
+
- Each request must fit Drex's size and question limits; one request is made per sample, however
|
|
171
|
+
many criteria it holds.
|
|
172
|
+
- The scorer judges what you give it. It does not fetch URLs or files for the judge.
|
|
173
|
+
|
|
174
|
+
## License
|
|
175
|
+
|
|
176
|
+
MIT. See [LICENSE](LICENSE). Issues: <https://github.com/nace-ai/inspect-drex/issues>.
|