inspect-drex 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. inspect_drex-0.1.0/.github/workflows/ci.yaml +61 -0
  2. inspect_drex-0.1.0/.github/workflows/release.yaml +96 -0
  3. inspect_drex-0.1.0/.gitignore +11 -0
  4. inspect_drex-0.1.0/CHANGELOG.md +15 -0
  5. inspect_drex-0.1.0/LICENSE +21 -0
  6. inspect_drex-0.1.0/PKG-INFO +206 -0
  7. inspect_drex-0.1.0/README.md +176 -0
  8. inspect_drex-0.1.0/examples/support_eval.py +102 -0
  9. inspect_drex-0.1.0/inspect_drex/__init__.py +35 -0
  10. inspect_drex-0.1.0/inspect_drex/_client.py +279 -0
  11. inspect_drex-0.1.0/inspect_drex/_images.py +84 -0
  12. inspect_drex-0.1.0/inspect_drex/_registry.py +5 -0
  13. inspect_drex-0.1.0/inspect_drex/_version.py +1 -0
  14. inspect_drex-0.1.0/inspect_drex/criteria.py +153 -0
  15. inspect_drex-0.1.0/inspect_drex/errors.py +116 -0
  16. inspect_drex-0.1.0/inspect_drex/scorer.py +225 -0
  17. inspect_drex-0.1.0/pyproject.toml +82 -0
  18. inspect_drex-0.1.0/tests/__init__.py +0 -0
  19. inspect_drex-0.1.0/tests/fakes.py +132 -0
  20. inspect_drex-0.1.0/tests/fixtures/blue.png +0 -0
  21. inspect_drex-0.1.0/tests/fixtures/red.png +0 -0
  22. inspect_drex-0.1.0/tests/integration_tests/__init__.py +0 -0
  23. inspect_drex-0.1.0/tests/integration_tests/conftest.py +41 -0
  24. inspect_drex-0.1.0/tests/integration_tests/test_live.py +119 -0
  25. inspect_drex-0.1.0/tests/integration_tests/test_live_openrouter.py +81 -0
  26. inspect_drex-0.1.0/tests/unit_tests/__init__.py +0 -0
  27. inspect_drex-0.1.0/tests/unit_tests/conftest.py +84 -0
  28. inspect_drex-0.1.0/tests/unit_tests/test_client.py +436 -0
  29. inspect_drex-0.1.0/tests/unit_tests/test_helpers.py +123 -0
  30. inspect_drex-0.1.0/tests/unit_tests/test_scorer.py +463 -0
  31. inspect_drex-0.1.0/tests/unit_tests/test_version.py +31 -0
  32. inspect_drex-0.1.0/uv.lock +3001 -0
@@ -0,0 +1,61 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ concurrency:
12
+ group: ci-${{ github.ref }}
13
+ cancel-in-progress: true
14
+
15
+ jobs:
16
+ lint:
17
+ name: Lint
18
+ runs-on: ubuntu-latest
19
+ timeout-minutes: 10
20
+ steps:
21
+ - uses: actions/checkout@v4
22
+
23
+ - name: Install uv
24
+ uses: astral-sh/setup-uv@v4
25
+ with:
26
+ enable-cache: true
27
+ cache-dependency-glob: "uv.lock"
28
+
29
+ - name: Install dependencies
30
+ run: uv sync --all-groups --locked --python 3.12
31
+
32
+ - name: Ruff check
33
+ run: uv run ruff check .
34
+
35
+ - name: Ruff format
36
+ run: uv run ruff format --check .
37
+
38
+ test:
39
+ name: Offline tests (Python ${{ matrix.python-version }})
40
+ runs-on: ubuntu-latest
41
+ timeout-minutes: 15
42
+ strategy:
43
+ fail-fast: false
44
+ matrix:
45
+ python-version: ["3.11", "3.12", "3.13"]
46
+ steps:
47
+ - uses: actions/checkout@v4
48
+
49
+ - name: Install uv
50
+ uses: astral-sh/setup-uv@v4
51
+ with:
52
+ enable-cache: true
53
+ cache-dependency-glob: "uv.lock"
54
+
55
+ - name: Install dependencies
56
+ run: uv sync --all-groups --locked --python ${{ matrix.python-version }}
57
+
58
+ # The live tests need keys and make billed requests, so only the offline
59
+ # suite runs here: `tests/unit_tests` leaves them out of collection.
60
+ - name: Offline tests
61
+ run: uv run pytest tests/unit_tests --cov
@@ -0,0 +1,96 @@
1
+ # Publishes inspect-drex to PyPI when an `inspect-drex-v*` tag is pushed.
2
+ #
3
+ # Releasing is deliberate: `git tag inspect-drex-v0.1.0 && git push origin inspect-drex-v0.1.0`.
4
+ # A PyPI upload cannot be undone or replaced, so everything that can be checked runs
5
+ # before the upload, and `workflow_dispatch` runs the same checks plus the build while
6
+ # skipping the publish, so the pipeline can be rehearsed.
7
+ name: Release
8
+
9
+ on:
10
+ push:
11
+ tags: ["inspect-drex-v*"]
12
+ workflow_dispatch:
13
+
14
+ permissions:
15
+ contents: read
16
+
17
+ concurrency:
18
+ group: release-${{ github.ref }}
19
+ # Never cancel: a half-finished release is worse than a queued one.
20
+ cancel-in-progress: false
21
+
22
+ jobs:
23
+ verify:
24
+ name: Verify and build
25
+ runs-on: ubuntu-latest
26
+ timeout-minutes: 15
27
+ steps:
28
+ - uses: actions/checkout@v4
29
+
30
+ - name: Install uv
31
+ uses: astral-sh/setup-uv@v4
32
+ with:
33
+ enable-cache: true
34
+ cache-dependency-glob: "uv.lock"
35
+
36
+ - name: Verify lockfile and install dependencies
37
+ run: uv sync --all-groups --locked --python 3.12
38
+
39
+ # The tag names the version users will get, so a tag that disagrees with the
40
+ # package is caught here rather than after an unrepeatable upload.
41
+ - name: Tag matches package version
42
+ if: github.event_name == 'push'
43
+ run: |
44
+ tag_version="${GITHUB_REF_NAME#inspect-drex-v}"
45
+ pkg_version="$(uv run python -c "from inspect_drex import __version__; print(__version__)")"
46
+ if [ "$tag_version" != "$pkg_version" ]; then
47
+ echo "Tag ${GITHUB_REF_NAME} would publish $pkg_version, not $tag_version." >&2
48
+ echo "Bump inspect_drex/_version.py or retag before releasing." >&2
49
+ exit 1
50
+ fi
51
+ echo "Releasing inspect-drex $pkg_version"
52
+
53
+ - name: Ruff
54
+ run: uv run ruff check .
55
+
56
+ - name: Offline tests
57
+ run: uv run pytest tests/unit_tests --cov
58
+
59
+ - name: Build distributions
60
+ run: uv build --out-dir dist
61
+
62
+ # The README becomes the PyPI project page; twine rejects markup PyPI cannot
63
+ # render, which is otherwise only discovered at upload time.
64
+ - name: Check distribution metadata
65
+ run: uvx twine check dist/*
66
+
67
+ - uses: actions/upload-artifact@v4
68
+ with:
69
+ name: inspect-drex-dist
70
+ path: dist/
71
+ if-no-files-found: error
72
+
73
+ publish:
74
+ name: Publish to PyPI
75
+ needs: verify
76
+ # Only a tag publishes; workflow_dispatch stops after verify.
77
+ if: github.event_name == 'push'
78
+ runs-on: ubuntu-latest
79
+ timeout-minutes: 15
80
+ # Half of the Trusted Publishing identity, and where release reviewers get
81
+ # added later without touching this file.
82
+ environment: pypi
83
+ permissions:
84
+ # Trusted Publishing mints a short-lived token from this; no PyPI
85
+ # credential is stored in the repository.
86
+ id-token: write
87
+ steps:
88
+ - uses: actions/download-artifact@v4
89
+ with:
90
+ name: inspect-drex-dist
91
+ path: dist
92
+
93
+ # Pinned to a commit: anything running in this job can mint the PyPI token
94
+ # for inspect-drex. Update it deliberately when a new release is reviewed.
95
+ - name: Publish
96
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
@@ -0,0 +1,11 @@
1
+ .venv/
2
+ dist/
3
+ build/
4
+ *.egg-info/
5
+ .coverage
6
+ .pytest_cache/
7
+ .ruff_cache/
8
+ __pycache__/
9
+ .DS_Store
10
+ .env
11
+ logs/
@@ -0,0 +1,15 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0
4
+
5
+ First release.
6
+
7
+ - `drex_scorer`: an Inspect scorer that asks Drex yes/no (`noul`), `choice` and `score`
8
+ questions about a sample's output and returns the pass probability as `Score.value`, with
9
+ Drex's full probabilities kept in `Score.metadata["drex"]`.
10
+ - Registered with Inspect through the `inspect_ai` entry point, so
11
+ `inspect score --scorer inspect_drex/drex_scorer` works.
12
+ - Image input for Drex Max, refused before sending for text-only models.
13
+ - Drex through OpenRouter (`nace-ai/<model>` ids) with the key following the host.
14
+ - Self-hosted Drex deployments through `NACE_BASE_URL` or `base_url=`.
15
+ - Errors raise by default; `on_error="unscored"` leaves the sample out of the metrics.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Nace AI
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,206 @@
1
+ Metadata-Version: 2.5
2
+ Name: inspect-drex
3
+ Version: 0.1.0
4
+ Summary: An Inspect (UK AISI) scorer backed by Drex decision models from Nace
5
+ Project-URL: Homepage, https://console.nace.ai/docs
6
+ Project-URL: Repository, https://github.com/nace-ai/inspect-drex
7
+ Project-URL: Issues, https://github.com/nace-ai/inspect-drex/issues
8
+ Project-URL: Changelog, https://github.com/nace-ai/inspect-drex/blob/main/CHANGELOG.md
9
+ Author-email: Nace AI <hello@nace.ai>
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: drex,evals,inspect,inspect-ai,llm-evaluation,nace,scorer
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Framework :: Pytest
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Intended Audience :: Science/Research
17
+ Classifier: License :: OSI Approved :: MIT License
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
25
+ Requires-Python: <4.0,>=3.11
26
+ Requires-Dist: httpx<1.0.0,>=0.27.0
27
+ Requires-Dist: inspect-ai>=0.3.245
28
+ Requires-Dist: nace-sdk<1.0.0,>=0.4.0
29
+ Description-Content-Type: text/markdown
30
+
31
+ # inspect-drex
32
+
33
+ A scorer for [Inspect](https://inspect.aisi.org.uk) (the UK AI Security Institute's evaluation
34
+ framework) that asks [Drex](https://console.nace.ai/docs) decision models from Nace whether a
35
+ sample's output meets your criteria. Drex answers typed questions and returns
36
+ **probabilities**, so you can choose the pass threshold after the run instead of before it.
37
+
38
+ - Drex Max reads text, JSON and images. `drex-v2.0` reads text and JSON only.
39
+ - Any other model id that Drex serves is passed through unchanged; there is no closed list.
40
+ - The default model is `drex-max`, which moves to the newest flagship with each Drex release.
41
+ Pin `model=` (or set `NACE_DEFAULT_DECISION_MODEL`) if you need results that never change.
42
+ See the Drex docs for each model's request limits.
43
+
44
+ ## Install
45
+
46
+ ```bash
47
+ pip install inspect-drex
48
+ export NACE_API_KEY=... # an API key from the Drex console
49
+ ```
50
+
51
+ Python 3.11 or newer (the floor of `inspect-ai`).
52
+
53
+ `NACE_BASE_URL` is optional. Set it, or pass `base_url=`, for a self-hosted Drex deployment on
54
+ your own domain. Any host that is not OpenRouter is treated as Drex itself: requests go to
55
+ `<base_url>/v1/systemone` and use your `NACE_API_KEY`.
56
+
57
+ ## Use
58
+
59
+ ```python
60
+ from inspect_ai import Task, task
61
+ from inspect_ai.dataset import Sample
62
+ from inspect_ai.solver import generate
63
+ from inspect_drex import Criterion, drex_scorer
64
+
65
+
66
+ @task
67
+ def support():
68
+ return Task(
69
+ dataset=[Sample(input="How long do refunds take?", target="5 business days")],
70
+ solver=generate(),
71
+ scorer=drex_scorer(
72
+ [
73
+ Criterion.noul(
74
+ "grounded", "Does the answer state the reference fact?", threshold=0.7
75
+ ),
76
+ Criterion.noul("filler", "Is the answer padded with filler?", want=False),
77
+ ],
78
+ model="drex-max",
79
+ ),
80
+ )
81
+ ```
82
+
83
+ Save it as `support.py` and run it with any model Inspect supports for the solver:
84
+
85
+ ```bash
86
+ inspect eval support.py --model openai/gpt-4o-mini
87
+ ```
88
+
89
+ A rubric string is a single yes/no criterion: `drex_scorer("The answer is helpful.")`.
90
+ The package registers itself with Inspect through an entry point, so a finished log can also be
91
+ scored from the CLI:
92
+
93
+ ```bash
94
+ inspect score logs/run.eval --scorer inspect_drex/drex_scorer -S criteria="The answer is helpful."
95
+ ```
96
+
97
+ `examples/support_eval.py` is an offline example (a mock model and a fake Drex server).
98
+
99
+ ### What you get back
100
+
101
+ - `Score.value` is the 0-1 **pass probability** (weighted mean of the criteria, or `min` with
102
+ `aggregate="min"`). `mean()` and `stderr()` report it. With `output="grade"` it is `CORRECT`
103
+ or `INCORRECT` by each criterion's own `threshold`, and `mean()` is the pass rate.
104
+ - `Score.metadata["drex"]` keeps what Drex returned: `request_id`, `model`, `input_tokens`,
105
+ `image_tokens`, `evaluation_time_ms`, and per criterion its kind, pass score, threshold,
106
+ verdict and full probabilities (noul: `yes` / `no`; choice and score: per label).
107
+ Input tokens are counted once per request, not once per question.
108
+ - For an unscored sample, `Score.metadata["drex"]` holds only `error` (the exception class
109
+ name); `Score.explanation` has the message, including the status and `request_id`.
110
+ - `Score.explanation` lists the criteria that failed, with their probabilities.
111
+
112
+ ### Criteria and score direction
113
+
114
+ `Criterion.noul` asks a yes/no question, `Criterion.choice` picks among labelled options and
115
+ `Criterion.score` places the output on an ordered scale. A criterion's pass score always means
116
+ "how well it holds": higher is better. A *defect* question ("is the reply rude?") needs
117
+ `want=False`, which scores `1 - P(yes)`. Choice criteria sum the probability of the labels you
118
+ mark `good`. Score criteria use the expected level over `levels - 1` (or its complement with
119
+ `invert=True`); an expected level is a fraction that can fall between two levels, and the raw
120
+ level is kept in the detail text. See Drex's guides on
121
+ [questions](https://console.nace.ai/docs/drex/guides/questions) and
122
+ [reading answers](https://console.nace.ai/docs/drex/guides/reading-answers).
123
+
124
+ ### What Drex sees
125
+
126
+ By default `{"input", "output", "reference"}`: the user messages, the model output and the
127
+ sample target. Give Drex only what the criteria need, or pass your own
128
+ `state=lambda state, target: ...` (and `images=` for Drex Max).
129
+
130
+ Images in the user's messages are sent in order, and the input text names each one
131
+ `[Image 1]`, `[Image 2]`, so a criterion can say "Image 1". Images are never dropped: with a
132
+ text-only model (`drex-v2.0`, `drex-v1.5`, `drex-latest`) the scorer raises before sending
133
+ (with `on_error="unscored"` the sample is left unscored instead). Drex never fetches URLs, so
134
+ remote image URLs are refused; use data URLs or local files. Keep each request within Drex's
135
+ size limit (inline images count, base64 included). For larger images, upload them with
136
+ `nace-sdk` and pass the uploaded file through `images=`.
137
+
138
+ ### Errors are not failed criteria
139
+
140
+ A 401 (bad key), 402 (no funds; the message links the billing page), 422 (each issue's
141
+ `path: message` is shown) or an outage raises, and Inspect marks the sample as errored. With
142
+ `on_error="unscored"` the sample stays in the log as `Score.unscored(reason="scoring_failed")`
143
+ and is left out of the metrics. 429 and 529 are retried by `nace-sdk`, honouring
144
+ `retry-after-ms`. The client timeout is at least 60 s, and the client caps concurrent requests
145
+ (`max_in_flight=`, default 8).
146
+
147
+ ### OpenRouter
148
+
149
+ Drex is also served through OpenRouter. Pass `base_url="https://openrouter.ai/api/v1"` and use
150
+ model ids such as `nace-ai/<model>`; a bare id (`drex-max`) is sent as `nace-ai/drex-max`, and ids
151
+ with a provider prefix are sent as given. `drex-latest` is refused because OpenRouter has no id
152
+ for it.
153
+
154
+ ```bash
155
+ export OPENROUTER_API_KEY=...
156
+ ```
157
+
158
+ ```python
159
+ drex_scorer(
160
+ "The answer is helpful.",
161
+ base_url="https://openrouter.ai/api/v1",
162
+ model="nace-ai/drex-max",
163
+ )
164
+ ```
165
+
166
+ The API key follows the host: for OpenRouter it is read from `OPENROUTER_API_KEY`, never from
167
+ `NACE_API_KEY`, so a Nace key is not sent to OpenRouter and an OpenRouter key is not sent to
168
+ Nace. Through OpenRouter, images work only inline (in `state` content parts); uploaded and
169
+ workspace-file images are refused.
170
+
171
+ ### Secrets and the eval log
172
+
173
+ Inspect writes scorer arguments to the eval log in plain text, so `drex_scorer` refuses
174
+ `api_key=` and `extra_headers=`. Put the key in the environment, or build the client yourself
175
+ and pass `client=AsyncDrexClient(base_url=..., api_key=..., model=...)`.
176
+
177
+ ### Attribution
178
+
179
+ Every request sends `User-Agent` and `X-Nace-Client` as `inspect-drex/<version>`, so Nace can
180
+ count usage per integration.
181
+
182
+ ## What is verified
183
+
184
+ - The offline tests (`tests/unit_tests`) run against a fake Drex server and are the CI gate
185
+ on Python 3.11, 3.12 and 3.13.
186
+ - The live tests passed against production Drex and against OpenRouter on 2026-10-29. They are
187
+ not run in CI, because they need keys and make billed requests.
188
+
189
+ ```bash
190
+ uv sync --all-groups
191
+ uv run ruff check . && uv run ruff format --check .
192
+ uv run pytest tests/unit_tests --cov # offline
193
+ NACE_API_KEY=... uv run pytest tests/integration_tests -m live -s -rs
194
+ OPENROUTER_API_KEY=... uv run pytest tests/integration_tests -m openrouter -s -rs
195
+ ```
196
+
197
+ ## Limits
198
+
199
+ - Images need a model that reads them (Drex Max); text-only models raise before sending.
200
+ - Each request must fit Drex's size and question limits; one request is made per sample, however
201
+ many criteria it holds.
202
+ - The scorer judges what you give it. It does not fetch URLs or files for the judge.
203
+
204
+ ## License
205
+
206
+ MIT. See [LICENSE](LICENSE). Issues: <https://github.com/nace-ai/inspect-drex/issues>.
@@ -0,0 +1,176 @@
1
+ # inspect-drex
2
+
3
+ A scorer for [Inspect](https://inspect.aisi.org.uk) (the UK AI Security Institute's evaluation
4
+ framework) that asks [Drex](https://console.nace.ai/docs) decision models from Nace whether a
5
+ sample's output meets your criteria. Drex answers typed questions and returns
6
+ **probabilities**, so you can choose the pass threshold after the run instead of before it.
7
+
8
+ - Drex Max reads text, JSON and images. `drex-v2.0` reads text and JSON only.
9
+ - Any other model id that Drex serves is passed through unchanged; there is no closed list.
10
+ - The default model is `drex-max`, which moves to the newest flagship with each Drex release.
11
+ Pin `model=` (or set `NACE_DEFAULT_DECISION_MODEL`) if you need results that never change.
12
+ See the Drex docs for each model's request limits.
13
+
14
+ ## Install
15
+
16
+ ```bash
17
+ pip install inspect-drex
18
+ export NACE_API_KEY=... # an API key from the Drex console
19
+ ```
20
+
21
+ Python 3.11 or newer (the floor of `inspect-ai`).
22
+
23
+ `NACE_BASE_URL` is optional. Set it, or pass `base_url=`, for a self-hosted Drex deployment on
24
+ your own domain. Any host that is not OpenRouter is treated as Drex itself: requests go to
25
+ `<base_url>/v1/systemone` and use your `NACE_API_KEY`.
26
+
27
+ ## Use
28
+
29
+ ```python
30
+ from inspect_ai import Task, task
31
+ from inspect_ai.dataset import Sample
32
+ from inspect_ai.solver import generate
33
+ from inspect_drex import Criterion, drex_scorer
34
+
35
+
36
+ @task
37
+ def support():
38
+ return Task(
39
+ dataset=[Sample(input="How long do refunds take?", target="5 business days")],
40
+ solver=generate(),
41
+ scorer=drex_scorer(
42
+ [
43
+ Criterion.noul(
44
+ "grounded", "Does the answer state the reference fact?", threshold=0.7
45
+ ),
46
+ Criterion.noul("filler", "Is the answer padded with filler?", want=False),
47
+ ],
48
+ model="drex-max",
49
+ ),
50
+ )
51
+ ```
52
+
53
+ Save it as `support.py` and run it with any model Inspect supports for the solver:
54
+
55
+ ```bash
56
+ inspect eval support.py --model openai/gpt-4o-mini
57
+ ```
58
+
59
+ A rubric string is a single yes/no criterion: `drex_scorer("The answer is helpful.")`.
60
+ The package registers itself with Inspect through an entry point, so a finished log can also be
61
+ scored from the CLI:
62
+
63
+ ```bash
64
+ inspect score logs/run.eval --scorer inspect_drex/drex_scorer -S criteria="The answer is helpful."
65
+ ```
66
+
67
+ `examples/support_eval.py` is an offline example (a mock model and a fake Drex server).
68
+
69
+ ### What you get back
70
+
71
+ - `Score.value` is the 0-1 **pass probability** (weighted mean of the criteria, or `min` with
72
+ `aggregate="min"`). `mean()` and `stderr()` report it. With `output="grade"` it is `CORRECT`
73
+ or `INCORRECT` by each criterion's own `threshold`, and `mean()` is the pass rate.
74
+ - `Score.metadata["drex"]` keeps what Drex returned: `request_id`, `model`, `input_tokens`,
75
+ `image_tokens`, `evaluation_time_ms`, and per criterion its kind, pass score, threshold,
76
+ verdict and full probabilities (noul: `yes` / `no`; choice and score: per label).
77
+ Input tokens are counted once per request, not once per question.
78
+ - For an unscored sample, `Score.metadata["drex"]` holds only `error` (the exception class
79
+ name); `Score.explanation` has the message, including the status and `request_id`.
80
+ - `Score.explanation` lists the criteria that failed, with their probabilities.
81
+
82
+ ### Criteria and score direction
83
+
84
+ `Criterion.noul` asks a yes/no question, `Criterion.choice` picks among labelled options and
85
+ `Criterion.score` places the output on an ordered scale. A criterion's pass score always means
86
+ "how well it holds": higher is better. A *defect* question ("is the reply rude?") needs
87
+ `want=False`, which scores `1 - P(yes)`. Choice criteria sum the probability of the labels you
88
+ mark `good`. Score criteria use the expected level over `levels - 1` (or its complement with
89
+ `invert=True`); an expected level is a fraction that can fall between two levels, and the raw
90
+ level is kept in the detail text. See Drex's guides on
91
+ [questions](https://console.nace.ai/docs/drex/guides/questions) and
92
+ [reading answers](https://console.nace.ai/docs/drex/guides/reading-answers).
93
+
94
+ ### What Drex sees
95
+
96
+ By default `{"input", "output", "reference"}`: the user messages, the model output and the
97
+ sample target. Give Drex only what the criteria need, or pass your own
98
+ `state=lambda state, target: ...` (and `images=` for Drex Max).
99
+
100
+ Images in the user's messages are sent in order, and the input text names each one
101
+ `[Image 1]`, `[Image 2]`, so a criterion can say "Image 1". Images are never dropped: with a
102
+ text-only model (`drex-v2.0`, `drex-v1.5`, `drex-latest`) the scorer raises before sending
103
+ (with `on_error="unscored"` the sample is left unscored instead). Drex never fetches URLs, so
104
+ remote image URLs are refused; use data URLs or local files. Keep each request within Drex's
105
+ size limit (inline images count, base64 included). For larger images, upload them with
106
+ `nace-sdk` and pass the uploaded file through `images=`.
107
+
108
+ ### Errors are not failed criteria
109
+
110
+ A 401 (bad key), 402 (no funds; the message links the billing page), 422 (each issue's
111
+ `path: message` is shown) or an outage raises, and Inspect marks the sample as errored. With
112
+ `on_error="unscored"` the sample stays in the log as `Score.unscored(reason="scoring_failed")`
113
+ and is left out of the metrics. 429 and 529 are retried by `nace-sdk`, honouring
114
+ `retry-after-ms`. The client timeout is at least 60 s, and the client caps concurrent requests
115
+ (`max_in_flight=`, default 8).
116
+
117
+ ### OpenRouter
118
+
119
+ Drex is also served through OpenRouter. Pass `base_url="https://openrouter.ai/api/v1"` and use
120
+ model ids such as `nace-ai/<model>`; a bare id (`drex-max`) is sent as `nace-ai/drex-max`, and ids
121
+ with a provider prefix are sent as given. `drex-latest` is refused because OpenRouter has no id
122
+ for it.
123
+
124
+ ```bash
125
+ export OPENROUTER_API_KEY=...
126
+ ```
127
+
128
+ ```python
129
+ drex_scorer(
130
+ "The answer is helpful.",
131
+ base_url="https://openrouter.ai/api/v1",
132
+ model="nace-ai/drex-max",
133
+ )
134
+ ```
135
+
136
+ The API key follows the host: for OpenRouter it is read from `OPENROUTER_API_KEY`, never from
137
+ `NACE_API_KEY`, so a Nace key is not sent to OpenRouter and an OpenRouter key is not sent to
138
+ Nace. Through OpenRouter, images work only inline (in `state` content parts); uploaded and
139
+ workspace-file images are refused.
140
+
141
+ ### Secrets and the eval log
142
+
143
+ Inspect writes scorer arguments to the eval log in plain text, so `drex_scorer` refuses
144
+ `api_key=` and `extra_headers=`. Put the key in the environment, or build the client yourself
145
+ and pass `client=AsyncDrexClient(base_url=..., api_key=..., model=...)`.
146
+
147
+ ### Attribution
148
+
149
+ Every request sends `User-Agent` and `X-Nace-Client` as `inspect-drex/<version>`, so Nace can
150
+ count usage per integration.
151
+
152
+ ## What is verified
153
+
154
+ - The offline tests (`tests/unit_tests`) run against a fake Drex server and are the CI gate
155
+ on Python 3.11, 3.12 and 3.13.
156
+ - The live tests passed against production Drex and against OpenRouter on 2026-10-29. They are
157
+ not run in CI, because they need keys and make billed requests.
158
+
159
+ ```bash
160
+ uv sync --all-groups
161
+ uv run ruff check . && uv run ruff format --check .
162
+ uv run pytest tests/unit_tests --cov # offline
163
+ NACE_API_KEY=... uv run pytest tests/integration_tests -m live -s -rs
164
+ OPENROUTER_API_KEY=... uv run pytest tests/integration_tests -m openrouter -s -rs
165
+ ```
166
+
167
+ ## Limits
168
+
169
+ - Images need a model that reads them (Drex Max); text-only models raise before sending.
170
+ - Each request must fit Drex's size and question limits; one request is made per sample, however
171
+ many criteria it holds.
172
+ - The scorer judges what you give it. It does not fetch URLs or files for the judge.
173
+
174
+ ## License
175
+
176
+ MIT. See [LICENSE](LICENSE). Issues: <https://github.com/nace-ai/inspect-drex/issues>.