fairmind-mcp 0.3.1a0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fairmind_mcp-0.3.1a0/.gitignore +366 -0
- fairmind_mcp-0.3.1a0/PKG-INFO +103 -0
- fairmind_mcp-0.3.1a0/README.md +92 -0
- fairmind_mcp-0.3.1a0/pyproject.toml +30 -0
- fairmind_mcp-0.3.1a0/src/fairmind_mcp/__init__.py +1 -0
- fairmind_mcp-0.3.1a0/src/fairmind_mcp/__main__.py +3 -0
- fairmind_mcp-0.3.1a0/src/fairmind_mcp/server.py +246 -0
- fairmind_mcp-0.3.1a0/tests/test_server.py +149 -0
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
# Internal planning docs (local only)
|
|
2
|
+
TODO_DEEP_ANALYSIS.md
|
|
3
|
+
|
|
4
|
+
# Dependencies
|
|
5
|
+
node_modules/
|
|
6
|
+
junk/
|
|
7
|
+
.pnp
|
|
8
|
+
.pnp.js
|
|
9
|
+
|
|
10
|
+
# Testing
|
|
11
|
+
coverage/
|
|
12
|
+
.nyc_output
|
|
13
|
+
|
|
14
|
+
# Next.js
|
|
15
|
+
.next/
|
|
16
|
+
out/
|
|
17
|
+
build/
|
|
18
|
+
dist/
|
|
19
|
+
|
|
20
|
+
# Production
|
|
21
|
+
build/
|
|
22
|
+
|
|
23
|
+
# Misc
|
|
24
|
+
.DS_Store
|
|
25
|
+
*.pem
|
|
26
|
+
*.p12
|
|
27
|
+
*.key
|
|
28
|
+
*.pfx
|
|
29
|
+
|
|
30
|
+
# Debug
|
|
31
|
+
npm-debug.log*
|
|
32
|
+
yarn-debug.log*
|
|
33
|
+
yarn-error.log*
|
|
34
|
+
|
|
35
|
+
# Local env files
|
|
36
|
+
.env
|
|
37
|
+
.env*.local
|
|
38
|
+
.env.development.local
|
|
39
|
+
.env.test.local
|
|
40
|
+
.env.production.local
|
|
41
|
+
.env.prisma
|
|
42
|
+
|
|
43
|
+
# Vercel
|
|
44
|
+
.vercel
|
|
45
|
+
|
|
46
|
+
# TypeScript
|
|
47
|
+
*.tsbuildinfo
|
|
48
|
+
next-env.d.ts
|
|
49
|
+
|
|
50
|
+
# Python
|
|
51
|
+
__pycache__/
|
|
52
|
+
*.py[cod]
|
|
53
|
+
*$py.class
|
|
54
|
+
*.so
|
|
55
|
+
.Python
|
|
56
|
+
build/
|
|
57
|
+
develop-eggs/
|
|
58
|
+
dist/
|
|
59
|
+
downloads/
|
|
60
|
+
eggs/
|
|
61
|
+
.eggs/
|
|
62
|
+
# Python lib directories (but not frontend src/lib)
|
|
63
|
+
/lib/
|
|
64
|
+
/lib64/
|
|
65
|
+
apps/backend/lib/
|
|
66
|
+
apps/backend/lib64/
|
|
67
|
+
parts/
|
|
68
|
+
sdist/
|
|
69
|
+
var/
|
|
70
|
+
wheels/
|
|
71
|
+
*.egg-info/
|
|
72
|
+
.installed.cfg
|
|
73
|
+
*.egg
|
|
74
|
+
MANIFEST
|
|
75
|
+
|
|
76
|
+
# PyInstaller
|
|
77
|
+
*.manifest
|
|
78
|
+
*.spec
|
|
79
|
+
|
|
80
|
+
# Installer logs
|
|
81
|
+
pip-log.txt
|
|
82
|
+
pip-delete-this-directory.txt
|
|
83
|
+
|
|
84
|
+
# Unit test / coverage reports
|
|
85
|
+
htmlcov/
|
|
86
|
+
.tox/
|
|
87
|
+
.coverage
|
|
88
|
+
.coverage.*
|
|
89
|
+
.cache
|
|
90
|
+
nosetests.xml
|
|
91
|
+
coverage.xml
|
|
92
|
+
*.cover
|
|
93
|
+
.hypothesis/
|
|
94
|
+
.pytest_cache/
|
|
95
|
+
|
|
96
|
+
# Translations
|
|
97
|
+
*.mo
|
|
98
|
+
*.pot
|
|
99
|
+
|
|
100
|
+
# Django stuff:
|
|
101
|
+
*.log
|
|
102
|
+
local_settings.py
|
|
103
|
+
db.sqlite3
|
|
104
|
+
|
|
105
|
+
# Flask stuff:
|
|
106
|
+
instance/
|
|
107
|
+
.webassets-cache
|
|
108
|
+
|
|
109
|
+
# Scrapy stuff:
|
|
110
|
+
.scrapy
|
|
111
|
+
|
|
112
|
+
# Sphinx documentation
|
|
113
|
+
docs/_build/
|
|
114
|
+
|
|
115
|
+
# PyBuilder
|
|
116
|
+
target/
|
|
117
|
+
|
|
118
|
+
# Jupyter Notebook
|
|
119
|
+
.ipynb_checkpoints
|
|
120
|
+
|
|
121
|
+
# pyenv
|
|
122
|
+
.python-version
|
|
123
|
+
|
|
124
|
+
# celery beat schedule file
|
|
125
|
+
celerybeat-schedule
|
|
126
|
+
|
|
127
|
+
# SageMath parsed files
|
|
128
|
+
*.sage.py
|
|
129
|
+
|
|
130
|
+
# Environments
|
|
131
|
+
.env
|
|
132
|
+
.venv
|
|
133
|
+
env/
|
|
134
|
+
venv/
|
|
135
|
+
ENV/
|
|
136
|
+
env.bak/
|
|
137
|
+
venv.bak/
|
|
138
|
+
|
|
139
|
+
# Spyder project settings
|
|
140
|
+
.spyderproject
|
|
141
|
+
.spyproject
|
|
142
|
+
|
|
143
|
+
# Rope project settings
|
|
144
|
+
.ropeproject
|
|
145
|
+
|
|
146
|
+
# mkdocs documentation
|
|
147
|
+
/site
|
|
148
|
+
/WORKSPACE_CLEANUP_SUMMARY.md
|
|
149
|
+
# mypy
|
|
150
|
+
.mypy_cache/
|
|
151
|
+
.dmypy.json
|
|
152
|
+
dmypy.json
|
|
153
|
+
|
|
154
|
+
# Pyre type checker
|
|
155
|
+
.pyre/
|
|
156
|
+
|
|
157
|
+
# Security & Secrets
|
|
158
|
+
*.pem
|
|
159
|
+
*.key
|
|
160
|
+
*.p12
|
|
161
|
+
*.pfx
|
|
162
|
+
*.crt
|
|
163
|
+
*.csr
|
|
164
|
+
*.pem
|
|
165
|
+
private_key*
|
|
166
|
+
public_key*
|
|
167
|
+
id_rsa*
|
|
168
|
+
id_dsa*
|
|
169
|
+
id_ecdsa*
|
|
170
|
+
id_ed25519*
|
|
171
|
+
*.env
|
|
172
|
+
.env*
|
|
173
|
+
secrets*
|
|
174
|
+
config/secrets*
|
|
175
|
+
credentials*
|
|
176
|
+
|
|
177
|
+
# JSON files (but allow important config files)
|
|
178
|
+
*.json
|
|
179
|
+
# fumadocs section indexes order the docs navigation; they are source, not data.
|
|
180
|
+
!apps/docs/content/**/meta.json
|
|
181
|
+
!package.json
|
|
182
|
+
!package-lock.json
|
|
183
|
+
!research/aibom-fairness-evidence/**/*.json
|
|
184
|
+
!apps/backend/src/domain/assurance/evidence-passport-v2.schema.json
|
|
185
|
+
!apps/backend/src/api/routers/pack_catalogue_data/*.json
|
|
186
|
+
!apps/backend/reference_data/frameworks/*.json
|
|
187
|
+
!apps/eval-runner/seccomp/sandbox.json
|
|
188
|
+
!tsconfig.json
|
|
189
|
+
!next.config.js
|
|
190
|
+
!tailwind.config.ts
|
|
191
|
+
!postcss.config.js
|
|
192
|
+
!eslint.config.js
|
|
193
|
+
!babel.config.js
|
|
194
|
+
!jest.config.js
|
|
195
|
+
!vite.config.js
|
|
196
|
+
!rollup.config.js
|
|
197
|
+
!webpack.config.js
|
|
198
|
+
!metro.config.js
|
|
199
|
+
!bunfig.toml
|
|
200
|
+
!pyproject.toml
|
|
201
|
+
!setup.py
|
|
202
|
+
!requirements.txt
|
|
203
|
+
!Pipfile
|
|
204
|
+
!Pipfile.lock
|
|
205
|
+
!poetry.lock
|
|
206
|
+
!Cargo.toml
|
|
207
|
+
!Cargo.lock
|
|
208
|
+
!go.mod
|
|
209
|
+
!go.sum
|
|
210
|
+
!composer.json
|
|
211
|
+
!composer.lock
|
|
212
|
+
!Gemfile
|
|
213
|
+
!Gemfile.lock
|
|
214
|
+
!yarn.lock
|
|
215
|
+
!pnpm-lock.yaml
|
|
216
|
+
!bun.lockb
|
|
217
|
+
|
|
218
|
+
# Database
|
|
219
|
+
*.db
|
|
220
|
+
*.sqlite
|
|
221
|
+
*.sqlite3
|
|
222
|
+
*.db-journal
|
|
223
|
+
|
|
224
|
+
# Logs
|
|
225
|
+
*.log
|
|
226
|
+
logs/
|
|
227
|
+
|
|
228
|
+
# IDE
|
|
229
|
+
.vscode/
|
|
230
|
+
.idea/
|
|
231
|
+
|
|
232
|
+
# Local/external nested repositories (never track in this repo)
|
|
233
|
+
.claude/worktrees/
|
|
234
|
+
.wiki/
|
|
235
|
+
_external/
|
|
236
|
+
apps/backend/src/domain/bias/bias_bench_repo/
|
|
237
|
+
apps/backend/uploads/
|
|
238
|
+
*.swp
|
|
239
|
+
*.swo
|
|
240
|
+
*~
|
|
241
|
+
|
|
242
|
+
# OS
|
|
243
|
+
.DS_Store
|
|
244
|
+
.DS_Store?
|
|
245
|
+
._*
|
|
246
|
+
.Spotlight-V100
|
|
247
|
+
.Trashes
|
|
248
|
+
ehthumbs.db
|
|
249
|
+
Thumbs.db
|
|
250
|
+
|
|
251
|
+
# Temporary files
|
|
252
|
+
*.tmp
|
|
253
|
+
*.temp
|
|
254
|
+
*.bak
|
|
255
|
+
*.backup
|
|
256
|
+
*.orig
|
|
257
|
+
|
|
258
|
+
# Large files
|
|
259
|
+
*.zip
|
|
260
|
+
*.tar.gz
|
|
261
|
+
*.rar
|
|
262
|
+
*.7z
|
|
263
|
+
*.iso
|
|
264
|
+
*.dmg
|
|
265
|
+
*.pkg
|
|
266
|
+
*.deb
|
|
267
|
+
*.rpm
|
|
268
|
+
*.msi
|
|
269
|
+
*.exe
|
|
270
|
+
*.dll
|
|
271
|
+
*.so
|
|
272
|
+
*.dylib
|
|
273
|
+
*.bin
|
|
274
|
+
|
|
275
|
+
# Model files (large ML models)
|
|
276
|
+
*.pkl
|
|
277
|
+
*.joblib
|
|
278
|
+
*.h5
|
|
279
|
+
*.hdf5
|
|
280
|
+
*.onnx
|
|
281
|
+
*.pb
|
|
282
|
+
*.pt
|
|
283
|
+
*.pth
|
|
284
|
+
*.safetensors
|
|
285
|
+
*.tflite
|
|
286
|
+
*.mlmodel
|
|
287
|
+
models/
|
|
288
|
+
checkpoints/
|
|
289
|
+
|
|
290
|
+
# Data files
|
|
291
|
+
*.csv
|
|
292
|
+
!research/aibom-fairness-evidence/**/*.csv
|
|
293
|
+
*.xml
|
|
294
|
+
*.yaml
|
|
295
|
+
*.yml
|
|
296
|
+
# GitHub workflows and composite actions are source.
|
|
297
|
+
!.github/**/*.yml
|
|
298
|
+
!.github/**/*.yaml
|
|
299
|
+
data/
|
|
300
|
+
datasets/
|
|
301
|
+
*.parquet
|
|
302
|
+
*.feather
|
|
303
|
+
*.hdf
|
|
304
|
+
*.h5
|
|
305
|
+
*.nc
|
|
306
|
+
*.netcdf
|
|
307
|
+
|
|
308
|
+
# Cache
|
|
309
|
+
.cache/
|
|
310
|
+
cache/
|
|
311
|
+
tmp/
|
|
312
|
+
temp/
|
|
313
|
+
|
|
314
|
+
# Specific security-sensitive files
|
|
315
|
+
localhost.har
|
|
316
|
+
*.har
|
|
317
|
+
apps/backend/public_key.pem
|
|
318
|
+
|
|
319
|
+
# The broad ML artifact rule ``models/`` must not hide active API source.
|
|
320
|
+
# Re-including the directory still leaves *.pt, *.pkl etc. ignored inside it.
|
|
321
|
+
!apps/backend/api/models/
|
|
322
|
+
!apps/backend/api/models/**/*.py
|
|
323
|
+
!apps/backend/src/**/models/
|
|
324
|
+
apps/backend/private_key.pem
|
|
325
|
+
apps/backend/api/services/real_bias_detection_service.py
|
|
326
|
+
apps/backend/email_service.py
|
|
327
|
+
apps/backend/neo4j_integration.py
|
|
328
|
+
apps/frontend/src/app/\(auth\)/signup/page.tsx
|
|
329
|
+
apps/website/bun.lock
|
|
330
|
+
apps/website/src/content/docs/installation.mdx
|
|
331
|
+
|
|
332
|
+
# Local Netlify folder
|
|
333
|
+
.netlify
|
|
334
|
+
|
|
335
|
+
# Archive directory
|
|
336
|
+
archive/
|
|
337
|
+
neura-dash-backup/
|
|
338
|
+
|
|
339
|
+
.qoder/
|
|
340
|
+
|
|
341
|
+
# Playwright test artifacts
|
|
342
|
+
.playwright-mcp/
|
|
343
|
+
playwright-report/
|
|
344
|
+
test-results/
|
|
345
|
+
|
|
346
|
+
# Screenshots and images (except logo and issue screenshots)
|
|
347
|
+
*.png
|
|
348
|
+
!**/public/logo.png
|
|
349
|
+
!**/logo.png
|
|
350
|
+
!docs/assets/issues/*.png
|
|
351
|
+
!assets/logo/*.png
|
|
352
|
+
!assets/diagrams/*.png
|
|
353
|
+
|
|
354
|
+
# Temporary documentation files
|
|
355
|
+
.temp-docs/
|
|
356
|
+
|
|
357
|
+
# Frontend archive (preserved for reference)
|
|
358
|
+
apps/frontend-archive/
|
|
359
|
+
apps/frontend-archive/node_modules/
|
|
360
|
+
apps/frontend-archive/.next/
|
|
361
|
+
apps/frontend-archive/out/
|
|
362
|
+
apps/frontend-archive/build/
|
|
363
|
+
apps/frontend-archive/dist/
|
|
364
|
+
|
|
365
|
+
# graft's local graph cache: regenerable, not committed (run `graft build`).
|
|
366
|
+
/graft/
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fairmind-mcp
|
|
3
|
+
Version: 0.3.1a0
|
|
4
|
+
Summary: MCP server that lets a coding agent ask FairMind whether an AI system may ship and run fairness tests.
|
|
5
|
+
Author-email: Adhithya Rajasekaran <adhithya@axonome.xyz>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Requires-Dist: fairmind
|
|
9
|
+
Requires-Dist: mcp<2,>=1.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# fairmind-mcp
|
|
13
|
+
|
|
14
|
+
<!-- mcp-name: io.github.adhit-r/fairmind -->
|
|
15
|
+
|
|
16
|
+
An MCP server that lets a coding agent ask FairMind whether an AI system may ship, and run fairness tests on CSVs from your machine. It is a thin layer over the [`fairmind` Python SDK](../sdk/README.md) and talks to the server over stdio.
|
|
17
|
+
|
|
18
|
+
## Tools
|
|
19
|
+
|
|
20
|
+
| Tool | What it does |
|
|
21
|
+
| --- | --- |
|
|
22
|
+
| `list_systems` | AI systems your key can see, with id, risk tier and lifecycle stage |
|
|
23
|
+
| `release_gate` | Verdict (`go`, `conditional_go`, `no_go`), confidence, the five checks and the reasons |
|
|
24
|
+
| `clause_map` | The gate read per regulatory clause; optional `frameworks` such as `EUAIA`, `ISO42001` |
|
|
25
|
+
| `run_fairness_test` | Uploads a local CSV, stores the measured result as evidence, returns metrics and the new gate |
|
|
26
|
+
| `record_monitoring_snapshot` | Compares recent decisions in a local CSV with the latest fairness run (drift, impact ratio, PSI) |
|
|
27
|
+
| `attest_check` | Records a person's statement that `data_governance`, `human_oversight`, `monitoring` or `environmental` is satisfied |
|
|
28
|
+
| `list_evaluations` | Searches the evaluation catalogue (drift, leakage, prompt injection, jailbreaks, counterfactual flip rate, agent configuration and more) and says which FairMind can run |
|
|
29
|
+
| `describe_evaluation` | The files and params an evaluation takes, and whether it also runs as a behaviour test against the live system |
|
|
30
|
+
| `run_evaluation` | Runs an evaluation on local files, against the live system (`live`) or on the connected agent platform (`use_connection`); stores the result as measured evidence |
|
|
31
|
+
|
|
32
|
+
Each tool returns a compact text summary and the full API response as structured content. `run_evaluation` waits for behaviour tests, which run as background jobs and can take minutes. The evaluation tools act in the key's first organisation; set `FAIRMIND_ORG_ID` to pick another. API failures come back as tool errors carrying the HTTP status and the server's message. `attest_check` writes manual evidence, so the tool tells the agent to use only statements the user gave or confirmed.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
# from a checkout, with uv (the SDK is resolved from ../sdk)
|
|
38
|
+
cd apps/mcp && uv sync
|
|
39
|
+
|
|
40
|
+
# or with pip: SDK first, then this package
|
|
41
|
+
pip install -e apps/sdk -e apps/mcp
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The server targets the `mcp` 1.x `FastMCP` API (`mcp>=1.10,<2`); in `mcp` 2.x it was renamed `MCPServer`.
|
|
45
|
+
|
|
46
|
+
## Configure
|
|
47
|
+
|
|
48
|
+
Create a key once with `POST /api/v1/auth/api-keys`, export it, and let the client pass it through. The config below never contains the key itself.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
export FAIRMIND_API_URL=https://fairmind.example.com
|
|
52
|
+
export FAIRMIND_API_KEY=...
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### Claude Code
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
claude mcp add fairmind -- uv run --project /path/to/fairmind/apps/mcp fairmind-mcp
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The server inherits `FAIRMIND_API_URL` and `FAIRMIND_API_KEY` from the shell that starts Claude Code. To scope it to a repository, commit a `.mcp.json` that expands the variables from each developer's environment:
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"mcpServers": {
|
|
66
|
+
"fairmind": {
|
|
67
|
+
"command": "uv",
|
|
68
|
+
"args": ["run", "--project", "/path/to/fairmind/apps/mcp", "fairmind-mcp"],
|
|
69
|
+
"env": {
|
|
70
|
+
"FAIRMIND_API_URL": "${FAIRMIND_API_URL}",
|
|
71
|
+
"FAIRMIND_API_KEY": "${FAIRMIND_API_KEY}"
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Claude Desktop
|
|
79
|
+
|
|
80
|
+
Desktop does not expand variables, so launch through a shell that reads them from your environment. In `claude_desktop_config.json`:
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{
|
|
84
|
+
"mcpServers": {
|
|
85
|
+
"fairmind": {
|
|
86
|
+
"command": "/bin/sh",
|
|
87
|
+
"args": ["-c", "exec uv run --project /path/to/fairmind/apps/mcp fairmind-mcp"]
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Set `FAIRMIND_API_URL` and `FAIRMIND_API_KEY` in the environment that starts Claude Desktop (for example with `launchctl setenv` on macOS), or keep them in a file outside the repository that you source in the `-c` command.
|
|
94
|
+
|
|
95
|
+
## Try it
|
|
96
|
+
|
|
97
|
+
Ask the agent: "May the credit-scoring system ship?" It calls `list_systems`, then `release_gate`. Ask it to "run a fairness test on `holdout.csv` for gender and region, outcome `repaid`, prediction `approved`" and it calls `run_fairness_test` with the file path on your machine.
|
|
98
|
+
|
|
99
|
+
## Tests
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
cd apps/mcp && uv run --with pytest pytest -q -p no:cacheprovider tests
|
|
103
|
+
```
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# fairmind-mcp
|
|
2
|
+
|
|
3
|
+
<!-- mcp-name: io.github.adhit-r/fairmind -->
|
|
4
|
+
|
|
5
|
+
An MCP server that lets a coding agent ask FairMind whether an AI system may ship, and run fairness tests on CSVs from your machine. It is a thin layer over the [`fairmind` Python SDK](../sdk/README.md) and talks to the server over stdio.
|
|
6
|
+
|
|
7
|
+
## Tools
|
|
8
|
+
|
|
9
|
+
| Tool | What it does |
|
|
10
|
+
| --- | --- |
|
|
11
|
+
| `list_systems` | AI systems your key can see, with id, risk tier and lifecycle stage |
|
|
12
|
+
| `release_gate` | Verdict (`go`, `conditional_go`, `no_go`), confidence, the five checks and the reasons |
|
|
13
|
+
| `clause_map` | The gate read per regulatory clause; optional `frameworks` such as `EUAIA`, `ISO42001` |
|
|
14
|
+
| `run_fairness_test` | Uploads a local CSV, stores the measured result as evidence, returns metrics and the new gate |
|
|
15
|
+
| `record_monitoring_snapshot` | Compares recent decisions in a local CSV with the latest fairness run (drift, impact ratio, PSI) |
|
|
16
|
+
| `attest_check` | Records a person's statement that `data_governance`, `human_oversight`, `monitoring` or `environmental` is satisfied |
|
|
17
|
+
| `list_evaluations` | Searches the evaluation catalogue (drift, leakage, prompt injection, jailbreaks, counterfactual flip rate, agent configuration and more) and says which FairMind can run |
|
|
18
|
+
| `describe_evaluation` | The files and params an evaluation takes, and whether it also runs as a behaviour test against the live system |
|
|
19
|
+
| `run_evaluation` | Runs an evaluation on local files, against the live system (`live`) or on the connected agent platform (`use_connection`); stores the result as measured evidence |
|
|
20
|
+
|
|
21
|
+
Each tool returns a compact text summary and the full API response as structured content. `run_evaluation` waits for behaviour tests, which run as background jobs and can take minutes. The evaluation tools act in the key's first organisation; set `FAIRMIND_ORG_ID` to pick another. API failures come back as tool errors carrying the HTTP status and the server's message. `attest_check` writes manual evidence, so the tool tells the agent to use only statements the user gave or confirmed.
|
|
22
|
+
|
|
23
|
+
## Install
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# from a checkout, with uv (the SDK is resolved from ../sdk)
|
|
27
|
+
cd apps/mcp && uv sync
|
|
28
|
+
|
|
29
|
+
# or with pip: SDK first, then this package
|
|
30
|
+
pip install -e apps/sdk -e apps/mcp
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The server targets the `mcp` 1.x `FastMCP` API (`mcp>=1.10,<2`); in `mcp` 2.x it was renamed `MCPServer`.
|
|
34
|
+
|
|
35
|
+
## Configure
|
|
36
|
+
|
|
37
|
+
Create a key once with `POST /api/v1/auth/api-keys`, export it, and let the client pass it through. The config below never contains the key itself.
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
export FAIRMIND_API_URL=https://fairmind.example.com
|
|
41
|
+
export FAIRMIND_API_KEY=...
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Claude Code
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
claude mcp add fairmind -- uv run --project /path/to/fairmind/apps/mcp fairmind-mcp
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The server inherits `FAIRMIND_API_URL` and `FAIRMIND_API_KEY` from the shell that starts Claude Code. To scope it to a repository, commit a `.mcp.json` that expands the variables from each developer's environment:
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{
|
|
54
|
+
"mcpServers": {
|
|
55
|
+
"fairmind": {
|
|
56
|
+
"command": "uv",
|
|
57
|
+
"args": ["run", "--project", "/path/to/fairmind/apps/mcp", "fairmind-mcp"],
|
|
58
|
+
"env": {
|
|
59
|
+
"FAIRMIND_API_URL": "${FAIRMIND_API_URL}",
|
|
60
|
+
"FAIRMIND_API_KEY": "${FAIRMIND_API_KEY}"
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### Claude Desktop
|
|
68
|
+
|
|
69
|
+
Desktop does not expand variables, so launch through a shell that reads them from your environment. In `claude_desktop_config.json`:
|
|
70
|
+
|
|
71
|
+
```json
|
|
72
|
+
{
|
|
73
|
+
"mcpServers": {
|
|
74
|
+
"fairmind": {
|
|
75
|
+
"command": "/bin/sh",
|
|
76
|
+
"args": ["-c", "exec uv run --project /path/to/fairmind/apps/mcp fairmind-mcp"]
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Set `FAIRMIND_API_URL` and `FAIRMIND_API_KEY` in the environment that starts Claude Desktop (for example with `launchctl setenv` on macOS), or keep them in a file outside the repository that you source in the `-c` command.
|
|
83
|
+
|
|
84
|
+
## Try it
|
|
85
|
+
|
|
86
|
+
Ask the agent: "May the credit-scoring system ship?" It calls `list_systems`, then `release_gate`. Ask it to "run a fairness test on `holdout.csv` for gender and region, outcome `repaid`, prediction `approved`" and it calls `run_fairness_test` with the file path on your machine.
|
|
87
|
+
|
|
88
|
+
## Tests
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
cd apps/mcp && uv run --with pytest pytest -q -p no:cacheprovider tests
|
|
92
|
+
```
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "fairmind-mcp"
|
|
3
|
+
version = "0.3.1a0"
|
|
4
|
+
description = "MCP server that lets a coding agent ask FairMind whether an AI system may ship and run fairness tests."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = { text = "Apache-2.0" }
|
|
8
|
+
authors = [{ name = "Adhithya Rajasekaran", email = "adhithya@axonome.xyz" }]
|
|
9
|
+
# mcp 2.x renames FastMCP to MCPServer; this server targets the 1.x FastMCP API.
|
|
10
|
+
dependencies = ["fairmind", "mcp>=1.10,<2"]
|
|
11
|
+
|
|
12
|
+
[project.scripts]
|
|
13
|
+
fairmind-mcp = "fairmind_mcp.server:main"
|
|
14
|
+
|
|
15
|
+
[dependency-groups]
|
|
16
|
+
dev = ["pytest>=7.4"]
|
|
17
|
+
|
|
18
|
+
# uv resolves the SDK from this checkout; with pip, install ../sdk first (see README).
|
|
19
|
+
[tool.uv.sources]
|
|
20
|
+
fairmind = { path = "../sdk", editable = true }
|
|
21
|
+
|
|
22
|
+
[build-system]
|
|
23
|
+
requires = ["hatchling"]
|
|
24
|
+
build-backend = "hatchling.build"
|
|
25
|
+
|
|
26
|
+
[tool.hatch.build.targets.wheel]
|
|
27
|
+
packages = ["src/fairmind_mcp"]
|
|
28
|
+
|
|
29
|
+
[tool.pytest.ini_options]
|
|
30
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""MCP server for FairMind."""
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""FairMind MCP server: tools a coding agent uses to ask "may this ship?" and to run tests.
|
|
2
|
+
|
|
3
|
+
Configuration is the SDK's: FAIRMIND_API_URL and FAIRMIND_API_KEY. Every tool returns a compact
|
|
4
|
+
text summary and the full API response as structured content.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import functools
|
|
10
|
+
from typing import Any, Dict, List, Literal, Optional
|
|
11
|
+
|
|
12
|
+
from fairmind import FairMind, FairMindError
|
|
13
|
+
from mcp.server.fastmcp import FastMCP
|
|
14
|
+
from mcp.server.fastmcp.exceptions import ToolError
|
|
15
|
+
from mcp.types import CallToolResult, TextContent
|
|
16
|
+
|
|
17
|
+
VERDICT_WORD = {"go": "GO", "conditional_go": "CONDITIONAL GO", "no_go": "NO GO"}
|
|
18
|
+
STATE_MARK = {"proven": "+", "attested": "~", "open": "-"}
|
|
19
|
+
|
|
20
|
+
INSTRUCTIONS = (
|
|
21
|
+
"FairMind grades the evidence behind an AI system and answers whether it may ship. "
|
|
22
|
+
"Call list_systems to find a system id, release_gate to read the verdict, run_fairness_test or "
|
|
23
|
+
"record_monitoring_snapshot with a CSV on this machine to add measured evidence, and attest_check "
|
|
24
|
+
"only for a statement the user has confirmed. For the wider catalogue (drift, leakage, prompt injection, "
|
|
25
|
+
"jailbreaks, counterfactual flip rate, agent configuration and more), call list_evaluations, then "
|
|
26
|
+
"describe_evaluation for the files and params one takes, then run_evaluation."
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _result(text: str, structured: Dict[str, Any]) -> CallToolResult:
|
|
31
|
+
return CallToolResult(content=[TextContent(type="text", text=text)], structuredContent=structured)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def render_gate(gate: Dict[str, Any]) -> str:
|
|
35
|
+
lines = [
|
|
36
|
+
f"{VERDICT_WORD.get(gate['verdict'], gate['verdict'])} confidence {gate['confidence']:.2f} ({gate.get('band', 'n/a')}) "
|
|
37
|
+
f"{gate.get('proven', 0)} proven, {gate.get('attested', 0)} attested, {gate.get('open', 0)} open"
|
|
38
|
+
]
|
|
39
|
+
for check in gate.get("checks", []):
|
|
40
|
+
conf = f" {check['confidence']:.2f}" if check.get("confidence") else ""
|
|
41
|
+
lines.append(f" {STATE_MARK.get(check['state'], '?')} {check['label']}: {check['state']}{conf}")
|
|
42
|
+
if "approvalStatus" in gate:
|
|
43
|
+
lines.append(f" Release decision: {gate['approvalStatus'].replace('_', ' ')}")
|
|
44
|
+
lines.extend(f" Reason: {reason}" for reason in gate.get("reasons", []))
|
|
45
|
+
return "\n".join(lines)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _fmt(value: Optional[float], ok: Optional[bool]) -> str:
|
|
49
|
+
return "n/a" if value is None else f"{value:.3f}{'' if ok in (None, True) else '!'}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def render_fairness(run: Dict[str, Any]) -> str:
|
|
53
|
+
result = run["result"]
|
|
54
|
+
head = "All checks within thresholds" if result["all_checks_pass"] else "At least one check outside thresholds"
|
|
55
|
+
lines = [f"{head}, {result['rows']} rows, evidence {run['evidenceId']}"]
|
|
56
|
+
for attr in result["attributes"]:
|
|
57
|
+
checks = attr["checks"]
|
|
58
|
+
cal = attr.get("calibration")
|
|
59
|
+
lines.append(
|
|
60
|
+
f" {attr['attribute']}: impact ratio {_fmt(attr['disparate_impact_ratio'], checks.get('disparate_impact'))}, "
|
|
61
|
+
f"equal opportunity gap {_fmt(attr['equal_opportunity']['value'], checks.get('equal_opportunity'))}, "
|
|
62
|
+
f"equalized odds gap {_fmt(attr['equalized_odds']['value'], checks.get('equalized_odds'))}, "
|
|
63
|
+
f"calibration {_fmt(cal['value'] if cal else None, checks.get('calibration'))}"
|
|
64
|
+
)
|
|
65
|
+
if attr.get("small_groups"):
|
|
66
|
+
lines.append(f" small groups: {', '.join(attr['small_groups'])}")
|
|
67
|
+
lines.append(" ! marks a value outside its threshold (impact ratio below 0.80, gap above 0.05)")
|
|
68
|
+
return "\n".join(lines + ["", render_gate(run["gate"])])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def render_monitoring(run: Dict[str, Any]) -> str:
|
|
72
|
+
result = run["result"]
|
|
73
|
+
psi = result.get("overall_psi")
|
|
74
|
+
head = "Stable against the baseline" if result["all_checks_pass"] else "Drift or disparity outside thresholds"
|
|
75
|
+
lines = [
|
|
76
|
+
f"{head}, {result['rows']} rows, overall score PSI {'n/a' if psi is None else f'{psi:.3f}'}, "
|
|
77
|
+
f"baseline {'present' if result.get('baseline', {}).get('present') else 'none (run a fairness test first)'}, evidence {run['evidenceId']}"
|
|
78
|
+
]
|
|
79
|
+
for attr in result["attributes"]:
|
|
80
|
+
checks = attr["checks"]
|
|
81
|
+
lines.append(
|
|
82
|
+
f" {attr['attribute']}: max rate drift {_fmt(attr.get('max_rate_drift'), checks.get('selection_rate_stable'))}, "
|
|
83
|
+
f"impact ratio {_fmt(attr['disparate_impact_ratio'], checks.get('disparate_impact'))}, "
|
|
84
|
+
f"worst PSI {_fmt(attr.get('worst_psi'), checks.get('score_stable'))}"
|
|
85
|
+
)
|
|
86
|
+
lines.append(" ! marks a value outside its threshold (rate drift above 0.05, impact ratio below 0.80, PSI at or above 0.10)")
|
|
87
|
+
return "\n".join(lines + ["", render_gate(run["gate"])])
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def render_evaluation_run(run: Dict[str, Any]) -> str:
|
|
91
|
+
lines = [f"{run.get('evaluationId')}: {str(run.get('status', '')).upper()}, evidence {run.get('id')}"]
|
|
92
|
+
if run.get("summary"):
|
|
93
|
+
lines.append(f" {run['summary']}")
|
|
94
|
+
for key, value in (run.get("metrics") or {}).items():
|
|
95
|
+
lines.append(f" {key}: {value:.3f}" if isinstance(value, float) else f" {key}: {value}")
|
|
96
|
+
return "\n".join(lines)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def render_engine(spec: Dict[str, Any]) -> str:
|
|
100
|
+
def part(engine: Dict[str, Any], title: str) -> List[str]:
|
|
101
|
+
out = [f"{title}: {engine['method']}"]
|
|
102
|
+
out += [f" file {s['name']} ({s['kind']}{'' if s['required'] else ', optional'}): {s['help']}" for s in engine["slots"]]
|
|
103
|
+
out += [f" param {p['name']} ({p['kind']}{', required' if p['required'] else ''}): {p['help']}" for p in engine["params"]]
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
lines = part(spec, "Behaviour test against the live system" if spec.get("live") else "On files")
|
|
107
|
+
if spec.get("platform"):
|
|
108
|
+
lines.append(" can also read the connected agent platform instead of files (source=connection)")
|
|
109
|
+
if spec.get("alternate"):
|
|
110
|
+
lines += part(spec["alternate"], "Also as a behaviour test (live=true)")
|
|
111
|
+
return "\n".join(lines)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def render_clause_map(cmap: Dict[str, Any]) -> str:
|
|
115
|
+
lines = [f"Gate {VERDICT_WORD.get(cmap['verdict'], cmap['verdict'])}, confidence {cmap['confidence']:.2f}"]
|
|
116
|
+
for fw in cmap["frameworks"]:
|
|
117
|
+
c = fw["counts"]
|
|
118
|
+
lines.append(f"{fw['name']} ({fw['key']}): {c['proven']} proven, {c['attested']} attested, {c['open']} open of {c['total']} clauses")
|
|
119
|
+
open_clauses = [cl["control"] for cl in fw["clauses"] if cl["state"] == "open"]
|
|
120
|
+
if open_clauses:
|
|
121
|
+
lines.append(f" open: {', '.join(open_clauses)}")
|
|
122
|
+
return "\n".join(lines)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def create_server(client: Optional[FairMind] = None) -> FastMCP:
|
|
126
|
+
"""Build the server. `client` defaults to FairMind() from the environment, created on first use."""
|
|
127
|
+
holder: Dict[str, FairMind] = {"client": client} if client else {}
|
|
128
|
+
|
|
129
|
+
def fm() -> FairMind:
|
|
130
|
+
if "client" not in holder:
|
|
131
|
+
holder["client"] = FairMind()
|
|
132
|
+
return holder["client"]
|
|
133
|
+
|
|
134
|
+
def guarded(fn): # surface API failures as tool errors with the server's message
|
|
135
|
+
@functools.wraps(fn)
|
|
136
|
+
def wrapper(*args: Any, **kwargs: Any) -> CallToolResult:
|
|
137
|
+
try:
|
|
138
|
+
return fn(*args, **kwargs)
|
|
139
|
+
except FairMindError as exc:
|
|
140
|
+
raise ToolError(str(exc)) from exc
|
|
141
|
+
|
|
142
|
+
return wrapper
|
|
143
|
+
|
|
144
|
+
server = FastMCP("fairmind", instructions=INSTRUCTIONS)
|
|
145
|
+
|
|
146
|
+
@server.tool()
|
|
147
|
+
@guarded
|
|
148
|
+
def list_systems() -> CallToolResult:
|
|
149
|
+
"""List the AI systems the configured FairMind key can see, with id, risk tier and lifecycle stage."""
|
|
150
|
+
systems = fm().systems()
|
|
151
|
+
if not systems:
|
|
152
|
+
return _result("No systems in this organisation yet.", {"systems": []})
|
|
153
|
+
lines = [f"{s.get('name', '')} tier {s.get('riskTier', '')} stage {s.get('lifecycleStage', '')} id {s.get('id', '')}" for s in systems]
|
|
154
|
+
return _result("\n".join(lines), {"systems": systems})
|
|
155
|
+
|
|
156
|
+
@server.tool()
|
|
157
|
+
@guarded
|
|
158
|
+
def release_gate(system_id: str) -> CallToolResult:
|
|
159
|
+
"""May this AI system ship? Returns the verdict (go, conditional_go, no_go), confidence, the five checks (each proven, attested or open), the reasons and the release decision status (not_requested, pending, approved, rejected). A release that needs sign-off may ship only when that status is approved."""
|
|
160
|
+
gate = fm().gate(system_id)
|
|
161
|
+
return _result(render_gate(gate), dict(gate))
|
|
162
|
+
|
|
163
|
+
@server.tool()
|
|
164
|
+
@guarded
|
|
165
|
+
def clause_map(system_id: str, frameworks: Optional[List[str]] = None) -> CallToolResult:
|
|
166
|
+
"""Read the release gate per regulatory clause (EU AI Act, ISO 42001, NIST AI RMF, ...). `frameworks` optionally limits it to catalogue keys such as EUAIA or ISO42001."""
|
|
167
|
+
cmap = fm().clause_map(system_id, frameworks)
|
|
168
|
+
return _result(render_clause_map(cmap), dict(cmap))
|
|
169
|
+
|
|
170
|
+
@server.tool()
|
|
171
|
+
@guarded
|
|
172
|
+
def run_fairness_test(
|
|
173
|
+
system_id: str,
|
|
174
|
+
csv_path: str,
|
|
175
|
+
outcome: str,
|
|
176
|
+
attributes: List[str],
|
|
177
|
+
prediction: Optional[str] = None,
|
|
178
|
+
score: Optional[str] = None,
|
|
179
|
+
favorable_label: Optional[str] = None,
|
|
180
|
+
) -> CallToolResult:
|
|
181
|
+
"""Run a fairness test on a CSV on this machine (one row per decision) and store it as measured evidence. `outcome` is the column with what happened, `attributes` the protected attribute columns, and `prediction` (0/1) or `score` (0 to 1) the model's output column. `favorable_label` names the favourable value when a column is not 0/1. Returns the metrics and the updated gate."""
|
|
182
|
+
run = fm().run_fairness(system_id, csv_path, outcome=outcome, attributes=attributes, prediction=prediction, score=score, favorable_label=favorable_label)
|
|
183
|
+
return _result(render_fairness(run), dict(run))
|
|
184
|
+
|
|
185
|
+
@server.tool()
|
|
186
|
+
@guarded
|
|
187
|
+
def record_monitoring_snapshot(
|
|
188
|
+
system_id: str,
|
|
189
|
+
csv_path: str,
|
|
190
|
+
attributes: List[str],
|
|
191
|
+
prediction: Optional[str] = None,
|
|
192
|
+
score: Optional[str] = None,
|
|
193
|
+
favorable_label: Optional[str] = None,
|
|
194
|
+
) -> CallToolResult:
|
|
195
|
+
"""Compare recent production decisions in a CSV on this machine against the system's latest fairness run (selection-rate drift, impact ratio, score PSI) and store the snapshot as measured evidence. Needs `prediction` (0/1) or `score` (0 to 1)."""
|
|
196
|
+
run = fm().run_monitoring(system_id, csv_path, attributes=attributes, prediction=prediction, score=score, favorable_label=favorable_label)
|
|
197
|
+
return _result(render_monitoring(run), dict(run))
|
|
198
|
+
|
|
199
|
+
@server.tool()
|
|
200
|
+
@guarded
|
|
201
|
+
def attest_check(
|
|
202
|
+
system_id: str,
|
|
203
|
+
check: Literal["data_governance", "human_oversight", "monitoring", "environmental"],
|
|
204
|
+
statement: str,
|
|
205
|
+
reference: Optional[str] = None,
|
|
206
|
+
) -> CallToolResult:
|
|
207
|
+
"""Record that a person states a gate check is satisfied (manual evidence: it attests, it never proves). Only call this with a statement the user has given or confirmed; never invent one. `statement` is at least 12 characters; `reference` optionally points at a document or ticket."""
|
|
208
|
+
done = fm().attest(system_id, check, statement, reference)
|
|
209
|
+
return _result(f"Attested {check}, evidence {done['evidenceId']}\n\n{render_gate(done['gate'])}", dict(done))
|
|
210
|
+
|
|
211
|
+
@server.tool()
|
|
212
|
+
@guarded
|
|
213
|
+
def list_evaluations(query: Optional[str] = None) -> CallToolResult:
|
|
214
|
+
"""Search FairMind's evaluation catalogue (for example "drift", "injection", "leakage", "counterfactual"). Each entry says what it measures, its pass threshold and whether FairMind can run it."""
|
|
215
|
+
evals = fm().evaluations(query)
|
|
216
|
+
if not evals:
|
|
217
|
+
return _result("No evaluation matches.", {"evaluations": []})
|
|
218
|
+
lines = [f"{e['id']} {'runnable' if e.get('runnable') else 'evidence only'} {e['name']}: {e.get('measures') or ''}" for e in evals]
|
|
219
|
+
return _result("\n".join(lines), {"evaluations": evals})
|
|
220
|
+
|
|
221
|
+
@server.tool()
|
|
222
|
+
@guarded
|
|
223
|
+
def describe_evaluation(evaluation_id: str) -> CallToolResult:
|
|
224
|
+
"""The files and params an evaluation takes before run_evaluation, and whether it can run as a behaviour test against the live system."""
|
|
225
|
+
spec = fm().evaluation_engine(evaluation_id)
|
|
226
|
+
return _result(render_engine(spec), dict(spec))
|
|
227
|
+
|
|
228
|
+
@server.tool()
|
|
229
|
+
@guarded
|
|
230
|
+
def run_evaluation(
|
|
231
|
+
system_id: str,
|
|
232
|
+
evaluation_id: str,
|
|
233
|
+
files: Optional[Dict[str, str]] = None,
|
|
234
|
+
params: Optional[Dict[str, Any]] = None,
|
|
235
|
+
live: bool = False,
|
|
236
|
+
use_connection: bool = False,
|
|
237
|
+
) -> CallToolResult:
|
|
238
|
+
"""Run a catalogue evaluation and store the result as measured evidence. `files` maps the file names from describe_evaluation to paths on this machine. `live=true` runs its behaviour test against the system's connected endpoint, judged by the organisation's model (this can take minutes). `use_connection=true` reads the organisation's connected agent platform instead of files. Returns passed, failed or insufficient with the metrics."""
|
|
239
|
+
run = fm().run_evaluation(system_id, evaluation_id, files, params, live=live, source="connection" if use_connection else "upload")
|
|
240
|
+
return _result(render_evaluation_run(run), dict(run))
|
|
241
|
+
|
|
242
|
+
return server
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def main() -> None:
|
|
246
|
+
create_server().run() # stdio
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from unittest.mock import Mock
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from fairmind import FairMind, FairMindError
|
|
6
|
+
from mcp.shared.memory import create_connected_server_and_client_session
|
|
7
|
+
|
|
8
|
+
from fairmind_mcp.server import create_server
|
|
9
|
+
|
|
10
|
+
GATE = {
|
|
11
|
+
"verdict": "conditional_go",
|
|
12
|
+
"confidence": 0.62,
|
|
13
|
+
"band": "medium",
|
|
14
|
+
"proven": 1,
|
|
15
|
+
"attested": 1,
|
|
16
|
+
"open": 1,
|
|
17
|
+
"reasons": ["Nothing on file for: Monitoring"],
|
|
18
|
+
"approvalStatus": "not_requested",
|
|
19
|
+
"checks": [
|
|
20
|
+
{"key": "fairness", "label": "Fairness measured", "state": "proven", "confidence": 0.88, "evidenceId": "e1", "detail": None},
|
|
21
|
+
{"key": "data_governance", "label": "Data governance", "state": "attested", "confidence": 0.3, "evidenceId": "e2", "detail": None},
|
|
22
|
+
{"key": "monitoring", "label": "Monitoring", "state": "open", "confidence": 0.0, "evidenceId": None, "detail": None},
|
|
23
|
+
],
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def call(client, tool, args):
|
|
28
|
+
async def go():
|
|
29
|
+
async with create_connected_server_and_client_session(create_server(client)._mcp_server) as session:
|
|
30
|
+
return await session.call_tool(tool, args)
|
|
31
|
+
|
|
32
|
+
return asyncio.run(go())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@pytest.fixture
|
|
36
|
+
def client():
|
|
37
|
+
return Mock(spec=FairMind)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_tools_are_registered(client):
|
|
41
|
+
async def go():
|
|
42
|
+
async with create_connected_server_and_client_session(create_server(client)._mcp_server) as session:
|
|
43
|
+
return {t.name for t in (await session.list_tools()).tools}
|
|
44
|
+
|
|
45
|
+
assert asyncio.run(go()) == {
|
|
46
|
+
"list_systems", "release_gate", "clause_map", "run_fairness_test", "record_monitoring_snapshot", "attest_check",
|
|
47
|
+
"list_evaluations", "describe_evaluation", "run_evaluation",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_release_gate_text_and_structured(client):
|
|
52
|
+
client.gate.return_value = GATE
|
|
53
|
+
result = call(client, "release_gate", {"system_id": "s1"})
|
|
54
|
+
client.gate.assert_called_once_with("s1")
|
|
55
|
+
text = result.content[0].text
|
|
56
|
+
assert text.startswith("CONDITIONAL GO confidence 0.62") and "- Monitoring: open" in text and "Reason: Nothing on file" in text
|
|
57
|
+
assert "Release decision: not requested" in text
|
|
58
|
+
assert result.structuredContent["verdict"] == "conditional_go" and not result.isError
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_list_systems(client):
|
|
62
|
+
client.systems.return_value = [{"id": "s1", "name": "Credit model", "riskTier": "high", "lifecycleStage": "staging"}]
|
|
63
|
+
result = call(client, "list_systems", {})
|
|
64
|
+
assert "Credit model" in result.content[0].text and "id s1" in result.content[0].text
|
|
65
|
+
assert result.structuredContent == {"systems": client.systems.return_value}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_clause_map_passes_frameworks(client):
|
|
69
|
+
client.clause_map.return_value = {
|
|
70
|
+
"verdict": "go", "confidence": 0.9,
|
|
71
|
+
"frameworks": [{"key": "EUAIA", "name": "EU AI Act", "counts": {"proven": 1, "attested": 0, "open": 1, "total": 2},
|
|
72
|
+
"clauses": [{"control": "Art 10", "state": "proven"}, {"control": "Art 14", "state": "open"}]}],
|
|
73
|
+
}
|
|
74
|
+
result = call(client, "clause_map", {"system_id": "s1", "frameworks": ["EUAIA"]})
|
|
75
|
+
client.clause_map.assert_called_once_with("s1", ["EUAIA"])
|
|
76
|
+
assert "EU AI Act (EUAIA): 1 proven, 0 attested, 1 open of 2 clauses" in result.content[0].text and "open: Art 14" in result.content[0].text
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_run_fairness_test(client):
|
|
80
|
+
client.run_fairness.return_value = {
|
|
81
|
+
"runId": "r1", "evidenceId": "ev9", "gate": GATE,
|
|
82
|
+
"result": {"all_checks_pass": False, "rows": 1200, "attributes": [{
|
|
83
|
+
"attribute": "gender", "disparate_impact_ratio": 0.71, "small_groups": [],
|
|
84
|
+
"checks": {"disparate_impact": False, "equal_opportunity": True},
|
|
85
|
+
"equal_opportunity": {"value": 0.02}, "equalized_odds": {"value": 0.03}, "calibration": None}]},
|
|
86
|
+
}
|
|
87
|
+
result = call(client, "run_fairness_test", {"system_id": "s1", "csv_path": "p.csv", "outcome": "repaid", "attributes": ["gender"], "prediction": "approved"})
|
|
88
|
+
client.run_fairness.assert_called_once_with("s1", "p.csv", outcome="repaid", attributes=["gender"], prediction="approved", score=None, favorable_label=None)
|
|
89
|
+
text = result.content[0].text
|
|
90
|
+
assert "At least one check outside thresholds, 1200 rows, evidence ev9" in text and "impact ratio 0.710!" in text and "calibration n/a" in text
|
|
91
|
+
assert result.structuredContent["runId"] == "r1"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_record_monitoring_snapshot(client):
|
|
95
|
+
client.run_monitoring.return_value = {
|
|
96
|
+
"evidenceId": "m1", "gate": GATE,
|
|
97
|
+
"result": {"all_checks_pass": True, "rows": 50, "overall_psi": 0.04, "baseline": {"present": True}, "attributes": [{
|
|
98
|
+
"attribute": "region", "disparate_impact_ratio": 0.93, "max_rate_drift": 0.01, "worst_psi": 0.04, "checks": {}}]},
|
|
99
|
+
}
|
|
100
|
+
result = call(client, "record_monitoring_snapshot", {"system_id": "s1", "csv_path": "d.csv", "attributes": ["region"], "score": "score"})
|
|
101
|
+
client.run_monitoring.assert_called_once_with("s1", "d.csv", attributes=["region"], prediction=None, score="score", favorable_label=None)
|
|
102
|
+
assert "Stable against the baseline, 50 rows" in result.content[0].text and "baseline present" in result.content[0].text
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_attest_check(client):
|
|
106
|
+
client.attest.return_value = {"evidenceId": "a1", "check": "monitoring", "gate": GATE}
|
|
107
|
+
result = call(client, "attest_check", {"system_id": "s1", "check": "monitoring", "statement": "Weekly dashboard review", "reference": "OPS-4"})
|
|
108
|
+
client.attest.assert_called_once_with("s1", "monitoring", "Weekly dashboard review", "OPS-4")
|
|
109
|
+
assert result.content[0].text.startswith("Attested monitoring, evidence a1")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_invalid_check_is_rejected(client):
|
|
113
|
+
assert call(client, "attest_check", {"system_id": "s1", "check": "fairness", "statement": "Weekly dashboard review"}).isError
|
|
114
|
+
client.attest.assert_not_called()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_api_error_becomes_tool_error_with_server_message(client):
|
|
118
|
+
client.gate.side_effect = FairMindError("GET /gate failed: AI system not found", 404)
|
|
119
|
+
result = call(client, "release_gate", {"system_id": "nope"})
|
|
120
|
+
assert result.isError and "404" in result.content[0].text and "AI system not found" in result.content[0].text
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_list_and_describe_evaluations(client):
|
|
124
|
+
client.evaluations.return_value = [{"id": "EV-DAT-01", "name": "Data drift", "measures": "PSI", "runnable": True}]
|
|
125
|
+
client.evaluation_engine.return_value = {
|
|
126
|
+
"method": "PSI per column", "live": False, "platform": False,
|
|
127
|
+
"slots": [{"name": "baseline", "kind": "csv", "required": True, "help": "reference data"}],
|
|
128
|
+
"params": [{"name": "threshold", "kind": "number", "required": False, "help": "PSI limit"}],
|
|
129
|
+
"alternate": None,
|
|
130
|
+
}
|
|
131
|
+
listed = call(client, "list_evaluations", {"query": "drift"})
|
|
132
|
+
assert "EV-DAT-01 runnable Data drift" in listed.content[0].text
|
|
133
|
+
client.evaluations.assert_called_once_with("drift")
|
|
134
|
+
text = call(client, "describe_evaluation", {"evaluation_id": "EV-DAT-01"}).content[0].text
|
|
135
|
+
assert "file baseline (csv): reference data" in text and "param threshold (number)" in text
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_run_evaluation_passes_files_params_and_mode(client):
|
|
139
|
+
client.run_evaluation.return_value = {"id": "ev1", "evaluationId": "EV-SEC-01", "status": "failed", "summary": "3 of 20 probes leaked", "metrics": {"leak_rate": 0.15}}
|
|
140
|
+
result = call(client, "run_evaluation", {"system_id": "s1", "evaluation_id": "EV-SEC-01", "params": {"probes": 20}, "live": True})
|
|
141
|
+
assert not result.isError
|
|
142
|
+
assert "EV-SEC-01: FAILED, evidence ev1" in result.content[0].text and "leak_rate: 0.150" in result.content[0].text
|
|
143
|
+
client.run_evaluation.assert_called_once_with("s1", "EV-SEC-01", None, {"probes": 20}, live=True, source="upload")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_run_evaluation_error_is_a_tool_error(client):
|
|
147
|
+
client.run_evaluation.side_effect = FairMindError("Evaluation EV-SEC-01 failed: no judge")
|
|
148
|
+
result = call(client, "run_evaluation", {"system_id": "s1", "evaluation_id": "EV-SEC-01", "use_connection": True})
|
|
149
|
+
assert result.isError and "no judge" in result.content[0].text
|