fairmind-cli 0.3.1a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fairmind_cli/__init__.py
ADDED
fairmind_cli/main.py
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""fairmind: gate, test and systems.
|
|
2
|
+
|
|
3
|
+
fairmind gate --system <id> exit 0 go, 2 conditional go, 1 no go
|
|
4
|
+
fairmind gate --system <id> --require-approval exit 1 unless the release decision is approved; approved, a conditional go exits 0
|
|
5
|
+
fairmind test --system <id> --file p.csv \
|
|
6
|
+
--outcome repaid --attributes gender,region [--prediction approved] [--score score]
|
|
7
|
+
fairmind monitor --system <id> --file decisions.csv --attributes gender [--prediction approved] [--score score]
|
|
8
|
+
fairmind systems
|
|
9
|
+
fairmind verify pack.json offline: recompute the pack hash and chain heads; exit 0 intact, 1 not
|
|
10
|
+
|
|
11
|
+
Configuration comes from flags or the environment: FAIRMIND_API_URL (default
|
|
12
|
+
http://localhost:8010) and FAIRMIND_API_KEY, a key minted at POST /api/v1/auth/api-keys.
|
|
13
|
+
Exit code 3 means the command itself failed (network, auth, bad input).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import hashlib
|
|
20
|
+
import hmac
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import sys
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any, Callable, Dict, Optional, Sequence
|
|
26
|
+
|
|
27
|
+
import httpx
|
|
28
|
+
|
|
29
|
+
from . import __version__
|
|
30
|
+
|
|
31
|
+
GOV = "/api/v1/ai-governance"
|
|
32
|
+
EXIT_GO, EXIT_NO_GO, EXIT_CONDITIONAL, EXIT_ERROR = 0, 1, 2, 3
|
|
33
|
+
|
|
34
|
+
ClientFactory = Callable[[str, Dict[str, str]], httpx.Client]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class CliError(Exception):
|
|
38
|
+
"""Reported to stderr with exit code 3."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _default_client(base_url: str, headers: Dict[str, str]) -> httpx.Client:
|
|
42
|
+
return httpx.Client(base_url=base_url, headers=headers, timeout=httpx.Timeout(180.0, connect=10.0))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _client(args: argparse.Namespace, factory: ClientFactory) -> httpx.Client:
|
|
46
|
+
base_url = (args.api or os.environ.get("FAIRMIND_API_URL") or "http://localhost:8010").rstrip("/")
|
|
47
|
+
key = args.api_key or os.environ.get("FAIRMIND_API_KEY") or ""
|
|
48
|
+
headers = {"Accept": "application/json", "User-Agent": f"fairmind-cli/{__version__}"}
|
|
49
|
+
if key:
|
|
50
|
+
headers["Authorization"] = f"Bearer {key}"
|
|
51
|
+
return factory(base_url, headers)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> Any:
|
|
55
|
+
try:
|
|
56
|
+
response = client.request(method, path, **kwargs)
|
|
57
|
+
except httpx.HTTPError as exc:
|
|
58
|
+
raise CliError(f"Could not reach FairMind at {client.base_url}: {exc}") from exc
|
|
59
|
+
if response.status_code == 401:
|
|
60
|
+
raise CliError("Not signed in. Set FAIRMIND_API_KEY to a key from POST /api/v1/auth/api-keys.")
|
|
61
|
+
if response.status_code == 404:
|
|
62
|
+
raise CliError(f"Not found: {path}. Check the system id and that your key belongs to its organisation.")
|
|
63
|
+
if response.status_code >= 400:
|
|
64
|
+
try:
|
|
65
|
+
detail = response.json().get("detail") or response.json().get("message") or response.text
|
|
66
|
+
except ValueError:
|
|
67
|
+
detail = response.text
|
|
68
|
+
raise CliError(f"{method} {path} failed with {response.status_code}: {detail}")
|
|
69
|
+
try:
|
|
70
|
+
return response.json()
|
|
71
|
+
except ValueError as exc:
|
|
72
|
+
raise CliError(f"{method} {path} returned something that is not JSON") from exc
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
# gate
|
|
77
|
+
# ---------------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
VERDICT_WORD = {"go": "GO", "conditional_go": "CONDITIONAL GO", "no_go": "NO GO"}
|
|
80
|
+
STATE_MARK = {"proven": "+", "attested": "~", "open": "-"}
|
|
81
|
+
APPROVAL_WORD = {"not_requested": "not requested", "pending": "pending", "approved": "approved", "rejected": "rejected"}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def render_gate(gate: Dict[str, Any]) -> str:
|
|
85
|
+
lines = [
|
|
86
|
+
f"{VERDICT_WORD.get(gate['verdict'], gate['verdict'])} confidence {gate['confidence']:.2f} ({gate['band']}) "
|
|
87
|
+
f"{gate['proven']} proven, {gate['attested']} attested, {gate['open']} open",
|
|
88
|
+
"",
|
|
89
|
+
]
|
|
90
|
+
for check in gate.get("checks", []):
|
|
91
|
+
conf = f" {check['confidence']:.2f}" if check.get("confidence") else ""
|
|
92
|
+
lines.append(f" {STATE_MARK.get(check['state'], '?')} {check['label']:<22} {check['state']}{conf}")
|
|
93
|
+
if "approvalStatus" in gate:
|
|
94
|
+
lines.append(f" {'Release decision':<22} {APPROVAL_WORD.get(gate['approvalStatus'], gate['approvalStatus'])}")
|
|
95
|
+
if gate.get("reasons"):
|
|
96
|
+
lines.append("")
|
|
97
|
+
lines.extend(f" {reason}" for reason in gate["reasons"])
|
|
98
|
+
return "\n".join(lines)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def cmd_gate(args: argparse.Namespace, factory: ClientFactory, out: Callable[[str], None]) -> int:
|
|
102
|
+
with _client(args, factory) as client:
|
|
103
|
+
gate = _call(client, "GET", f"{GOV}/systems/{args.system}/gate")
|
|
104
|
+
out(json.dumps(gate, indent=2) if args.json else render_gate(gate))
|
|
105
|
+
if args.require_approval and gate.get("approvalStatus") != "approved":
|
|
106
|
+
status = APPROVAL_WORD.get(gate.get("approvalStatus"), "not reported by this server")
|
|
107
|
+
print(f"fairmind: --require-approval: the release decision is {status}", file=sys.stderr)
|
|
108
|
+
return EXIT_NO_GO
|
|
109
|
+
verdict = gate.get("verdict")
|
|
110
|
+
if verdict == "go":
|
|
111
|
+
return EXIT_GO
|
|
112
|
+
if verdict == "conditional_go":
|
|
113
|
+
# Past --require-approval the decision is approved, and approving a conditional go records the accepted conditions.
|
|
114
|
+
return EXIT_GO if args.allow_conditional or args.require_approval else EXIT_CONDITIONAL
|
|
115
|
+
return EXIT_NO_GO
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ---------------------------------------------------------------------------
|
|
119
|
+
# test
|
|
120
|
+
# ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
def render_run(run: Dict[str, Any]) -> str:
|
|
123
|
+
result = run["result"]
|
|
124
|
+
lines = [
|
|
125
|
+
f"{'All checks within thresholds' if result['all_checks_pass'] else 'At least one check outside thresholds'} "
|
|
126
|
+
f"{result['rows']} rows evidence confidence {result['evidence_confidence']:.2f} (measured) evidence {run['evidenceId']}",
|
|
127
|
+
"",
|
|
128
|
+
f" {'attribute':<14}{'impact ratio':>13}{'eq. opp. gap':>14}{'eq. odds gap':>14}{'calibration':>13}",
|
|
129
|
+
]
|
|
130
|
+
for attr in result["attributes"]:
|
|
131
|
+
checks = attr["checks"]
|
|
132
|
+
def cell(value: Optional[float], ok: Optional[bool], width: int) -> str:
|
|
133
|
+
if value is None:
|
|
134
|
+
return f"{'n/a':>{width}}"
|
|
135
|
+
mark = "" if ok else "!"
|
|
136
|
+
return f"{value:.3f}{mark}".rjust(width)
|
|
137
|
+
cal = attr.get("calibration")
|
|
138
|
+
lines.append(
|
|
139
|
+
f" {attr['attribute']:<14}"
|
|
140
|
+
+ cell(attr["disparate_impact_ratio"], checks.get("disparate_impact"), 13)
|
|
141
|
+
+ cell(attr["equal_opportunity"]["value"], checks.get("equal_opportunity"), 14)
|
|
142
|
+
+ cell(attr["equalized_odds"]["value"], checks.get("equalized_odds"), 14)
|
|
143
|
+
+ cell(cal["value"] if cal else None, checks.get("calibration"), 13)
|
|
144
|
+
)
|
|
145
|
+
if attr.get("small_groups"):
|
|
146
|
+
lines.append(f" {'':<14}small groups: {', '.join(attr['small_groups'])}")
|
|
147
|
+
lines.append("")
|
|
148
|
+
lines.append(" ! marks a check outside its threshold (impact ratio below 0.80, gap above 0.05)")
|
|
149
|
+
lines.append("")
|
|
150
|
+
lines.append(render_gate(run["gate"]))
|
|
151
|
+
return "\n".join(lines)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def cmd_test(args: argparse.Namespace, factory: ClientFactory, out: Callable[[str], None]) -> int:
|
|
155
|
+
path = Path(args.file)
|
|
156
|
+
if not path.is_file():
|
|
157
|
+
raise CliError(f"No such file: {path}")
|
|
158
|
+
data = {"outcome": args.outcome, "attributes": args.attributes}
|
|
159
|
+
if args.prediction:
|
|
160
|
+
data["prediction"] = args.prediction
|
|
161
|
+
if args.score:
|
|
162
|
+
data["score"] = args.score
|
|
163
|
+
if args.favorable_label:
|
|
164
|
+
data["favorable_label"] = args.favorable_label
|
|
165
|
+
if not args.prediction and not args.score:
|
|
166
|
+
raise CliError("Give --prediction (a 0/1 column) or --score (a 0 to 1 column)")
|
|
167
|
+
with _client(args, factory) as client, path.open("rb") as handle:
|
|
168
|
+
run = _call(client, "POST", f"{GOV}/systems/{args.system}/fairness-runs", data=data, files={"file": (path.name, handle, "text/csv")})
|
|
169
|
+
out(json.dumps(run, indent=2) if args.json else render_run(run))
|
|
170
|
+
if args.gate:
|
|
171
|
+
verdict = run["gate"]["verdict"]
|
|
172
|
+
if verdict == "go":
|
|
173
|
+
return EXIT_GO
|
|
174
|
+
if verdict == "conditional_go":
|
|
175
|
+
return EXIT_GO if args.allow_conditional else EXIT_CONDITIONAL
|
|
176
|
+
return EXIT_NO_GO
|
|
177
|
+
return EXIT_GO if run["result"]["all_checks_pass"] else EXIT_NO_GO
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# ---------------------------------------------------------------------------
|
|
181
|
+
# monitor
|
|
182
|
+
# ---------------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
def render_monitoring(run: Dict[str, Any]) -> str:
|
|
185
|
+
result = run["result"]
|
|
186
|
+
base = result["baseline"]
|
|
187
|
+
lines = [
|
|
188
|
+
f"{'Stable against the baseline' if result['all_checks_pass'] else 'Drift or disparity outside thresholds'} "
|
|
189
|
+
f"{result['rows']} rows overall score PSI {('%.3f' % result['overall_psi']) if result.get('overall_psi') is not None else 'n/a'} "
|
|
190
|
+
f"baseline {'present' if base.get('present') else 'none (run a fairness test first)'} evidence {run['evidenceId']}",
|
|
191
|
+
"",
|
|
192
|
+
f" {'attribute':<14}{'max rate drift':>15}{'impact ratio':>13}{'worst PSI':>11}",
|
|
193
|
+
]
|
|
194
|
+
for attr in result["attributes"]:
|
|
195
|
+
checks = attr["checks"]
|
|
196
|
+
def cell(value: Optional[float], ok: Optional[bool], width: int) -> str:
|
|
197
|
+
if value is None:
|
|
198
|
+
return f"{'n/a':>{width}}"
|
|
199
|
+
return f"{value:.3f}{'' if ok in (None, True) else '!'}".rjust(width)
|
|
200
|
+
lines.append(f" {attr['attribute']:<14}" + cell(attr.get("max_rate_drift"), checks.get("selection_rate_stable"), 15) + cell(attr["disparate_impact_ratio"], checks.get("disparate_impact"), 13) + cell(attr.get("worst_psi"), checks.get("score_stable"), 11))
|
|
201
|
+
lines += ["", " ! marks a check outside its threshold (rate drift above 0.05, impact ratio below 0.80, PSI at or above 0.10)", "", render_gate(run["gate"])]
|
|
202
|
+
return "\n".join(lines)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def cmd_monitor(args: argparse.Namespace, factory: ClientFactory, out: Callable[[str], None]) -> int:
|
|
206
|
+
path = Path(args.file)
|
|
207
|
+
if not path.is_file():
|
|
208
|
+
raise CliError(f"No such file: {path}")
|
|
209
|
+
if not args.prediction and not args.score:
|
|
210
|
+
raise CliError("Give --prediction (a 0/1 column) or --score (a 0 to 1 column)")
|
|
211
|
+
data = {"attributes": args.attributes}
|
|
212
|
+
if args.prediction:
|
|
213
|
+
data["prediction"] = args.prediction
|
|
214
|
+
if args.score:
|
|
215
|
+
data["score"] = args.score
|
|
216
|
+
with _client(args, factory) as client, path.open("rb") as handle:
|
|
217
|
+
run = _call(client, "POST", f"{GOV}/systems/{args.system}/monitoring-runs", data=data, files={"file": (path.name, handle, "text/csv")})
|
|
218
|
+
out(json.dumps(run, indent=2) if args.json else render_monitoring(run))
|
|
219
|
+
return EXIT_GO if run["result"]["all_checks_pass"] else EXIT_NO_GO
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ---------------------------------------------------------------------------
|
|
223
|
+
# systems
|
|
224
|
+
# ---------------------------------------------------------------------------
|
|
225
|
+
|
|
226
|
+
def cmd_systems(args: argparse.Namespace, factory: ClientFactory, out: Callable[[str], None]) -> int:
|
|
227
|
+
with _client(args, factory) as client:
|
|
228
|
+
systems = _call(client, "GET", f"{GOV}/systems")
|
|
229
|
+
if args.json:
|
|
230
|
+
out(json.dumps(systems, indent=2))
|
|
231
|
+
return EXIT_GO
|
|
232
|
+
if not systems:
|
|
233
|
+
out("No systems in your organisation yet. Connect a source or register one in FairMind.")
|
|
234
|
+
return EXIT_GO
|
|
235
|
+
width = max(len(s.get("name", "")) for s in systems)
|
|
236
|
+
out(f" {'name':<{width}} {'tier':<9}{'stage':<10}id")
|
|
237
|
+
for s in systems:
|
|
238
|
+
out(f" {s.get('name', ''):<{width}} {s.get('riskTier', ''):<9}{s.get('lifecycleStage', ''):<10}{s.get('id', '')}")
|
|
239
|
+
return EXIT_GO
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# ---------------------------------------------------------------------------
|
|
243
|
+
# verify (offline)
|
|
244
|
+
# ---------------------------------------------------------------------------
|
|
245
|
+
|
|
246
|
+
CANONICAL_FORM = (
|
|
247
|
+
'Canonical form: sha256(json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()). '
|
|
248
|
+
'Chain head per system: h = sha256("fairmind-evidence-chain-v1"); for each evidence entry in order, '
|
|
249
|
+
'h = sha256(f"{h}:{entry.hash}"). The file is {"manifest": {...}, "manifestHash": "..."}.'
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _chain_head(entries: Sequence[Dict[str, Any]]) -> str:
|
|
254
|
+
head = hashlib.sha256(b"fairmind-evidence-chain-v1").hexdigest()
|
|
255
|
+
for entry in entries:
|
|
256
|
+
head = hashlib.sha256(f"{head}:{entry['hash']}".encode()).hexdigest()
|
|
257
|
+
return head
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def verify_pack(document: Dict[str, Any]) -> Dict[str, Any]:
|
|
261
|
+
"""Pure check of a downloaded pack; no network."""
|
|
262
|
+
manifest, expected = document.get("manifest"), document.get("manifestHash")
|
|
263
|
+
if not isinstance(manifest, dict) or not isinstance(expected, str):
|
|
264
|
+
raise CliError('Not an inspection pack: expected {"manifest": {...}, "manifestHash": "..."}')
|
|
265
|
+
actual = hashlib.sha256(json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest()
|
|
266
|
+
systems = []
|
|
267
|
+
for item in manifest.get("systems", []):
|
|
268
|
+
try:
|
|
269
|
+
recomputed = _chain_head(item.get("evidence", []))
|
|
270
|
+
except (KeyError, TypeError) as exc:
|
|
271
|
+
raise CliError(f"Malformed evidence entry: {exc!r}") from exc
|
|
272
|
+
system = item.get("system") or {}
|
|
273
|
+
systems.append({"name": system.get("name") or system.get("id"), "evidence": len(item.get("evidence", [])), "intact": hmac.compare_digest(str(item.get("chainHead")), recomputed)})
|
|
274
|
+
hash_ok = hmac.compare_digest(expected, actual)
|
|
275
|
+
return {"title": manifest.get("title"), "hashOk": hash_ok, "expected": expected, "actual": actual, "systems": systems, "intact": hash_ok and all(s["intact"] for s in systems)}
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def cmd_verify(args: argparse.Namespace, factory: ClientFactory, out: Callable[[str], None]) -> int:
|
|
279
|
+
path = Path(args.pack)
|
|
280
|
+
if not path.is_file():
|
|
281
|
+
raise CliError(f"No such file: {path}")
|
|
282
|
+
try:
|
|
283
|
+
document = json.loads(path.read_text(encoding="utf-8"))
|
|
284
|
+
except ValueError as exc:
|
|
285
|
+
raise CliError(f"{path} is not valid JSON: {exc}") from exc
|
|
286
|
+
result = verify_pack(document)
|
|
287
|
+
if args.json:
|
|
288
|
+
out(json.dumps(result, indent=2))
|
|
289
|
+
else:
|
|
290
|
+
out(f"{result['title']}")
|
|
291
|
+
out(f" manifest hash {'ok' if result['hashOk'] else 'MISMATCH'} {result['actual']}")
|
|
292
|
+
if not result["hashOk"]:
|
|
293
|
+
out(f" recorded {result['expected']}")
|
|
294
|
+
for s in result["systems"]:
|
|
295
|
+
out(f" {'ok' if s['intact'] else 'BROKEN':<9}{s['name']} {s['evidence']} evidence records, chain head {'matches' if s['intact'] else 'does not match'}")
|
|
296
|
+
out("")
|
|
297
|
+
out("Pack intact" if result["intact"] else "Pack does NOT verify: it was changed after it was built")
|
|
298
|
+
return EXIT_GO if result["intact"] else EXIT_NO_GO
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
# ---------------------------------------------------------------------------
|
|
302
|
+
# entry point
|
|
303
|
+
# ---------------------------------------------------------------------------
|
|
304
|
+
|
|
305
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
306
|
+
parser = argparse.ArgumentParser(prog="fairmind", description="FairMind release gate and fairness tests.")
|
|
307
|
+
parser.add_argument("--version", action="version", version=f"fairmind-cli {__version__}")
|
|
308
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
309
|
+
common.add_argument("--api", help="FairMind API base URL (default: FAIRMIND_API_URL or http://localhost:8010)")
|
|
310
|
+
common.add_argument("--api-key", help="API key (default: FAIRMIND_API_KEY)")
|
|
311
|
+
common.add_argument("--json", action="store_true", help="print the raw response")
|
|
312
|
+
|
|
313
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
314
|
+
|
|
315
|
+
gate = sub.add_parser("gate", parents=[common], help="read the release gate; exit 0 go, 2 conditional, 1 no go")
|
|
316
|
+
gate.add_argument("--system", required=True, help="AI system id")
|
|
317
|
+
gate.add_argument("--allow-conditional", action="store_true", help="exit 0 on a conditional go")
|
|
318
|
+
gate.add_argument("--require-approval", action="store_true", help="exit 1 unless the release decision is approved (with or without conditions)")
|
|
319
|
+
gate.set_defaults(func=cmd_gate)
|
|
320
|
+
|
|
321
|
+
test = sub.add_parser("test", parents=[common], help="run a fairness test from a predictions CSV and store it as evidence")
|
|
322
|
+
test.add_argument("--system", required=True, help="AI system id")
|
|
323
|
+
test.add_argument("--file", required=True, help="CSV with one row per decision")
|
|
324
|
+
test.add_argument("--outcome", required=True, help="column with what happened (0/1)")
|
|
325
|
+
test.add_argument("--attributes", required=True, help="comma-separated protected attribute columns")
|
|
326
|
+
test.add_argument("--prediction", help="column with the model's decision (0/1)")
|
|
327
|
+
test.add_argument("--score", help="column with the model's score (0 to 1)")
|
|
328
|
+
test.add_argument("--favorable-label", help="value that counts as the favourable outcome when columns are not 0/1")
|
|
329
|
+
test.add_argument("--gate", action="store_true", help="exit by the release gate's verdict instead of the test's checks")
|
|
330
|
+
test.add_argument("--allow-conditional", action="store_true", help="with --gate: exit 0 on a conditional go")
|
|
331
|
+
test.set_defaults(func=cmd_test)
|
|
332
|
+
|
|
333
|
+
monitor = sub.add_parser("monitor", parents=[common], help="record a monitoring snapshot from recent decisions; exit 1 on drift or disparity")
|
|
334
|
+
monitor.add_argument("--system", required=True, help="AI system id")
|
|
335
|
+
monitor.add_argument("--file", required=True, help="CSV of recent decisions")
|
|
336
|
+
monitor.add_argument("--attributes", required=True, help="comma-separated protected attribute columns")
|
|
337
|
+
monitor.add_argument("--prediction", help="column with the model's decision (0/1)")
|
|
338
|
+
monitor.add_argument("--score", help="column with the model's score (0 to 1)")
|
|
339
|
+
monitor.set_defaults(func=cmd_monitor)
|
|
340
|
+
|
|
341
|
+
systems = sub.add_parser("systems", parents=[common], help="list the AI systems your key can see")
|
|
342
|
+
systems.set_defaults(func=cmd_systems)
|
|
343
|
+
|
|
344
|
+
verify = sub.add_parser("verify", help="verify a downloaded inspection pack offline (no network); exit 0 intact, 1 not", description=CANONICAL_FORM)
|
|
345
|
+
verify.add_argument("pack", help="pack JSON downloaded from FairMind or an auditor link")
|
|
346
|
+
verify.add_argument("--json", action="store_true", help="print the result as JSON")
|
|
347
|
+
verify.set_defaults(func=cmd_verify)
|
|
348
|
+
return parser
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def main(argv: Optional[Sequence[str]] = None, *, client_factory: ClientFactory = _default_client, out: Callable[[str], None] = print) -> int:
|
|
352
|
+
args = build_parser().parse_args(argv)
|
|
353
|
+
try:
|
|
354
|
+
return int(args.func(args, client_factory, out))
|
|
355
|
+
except CliError as exc:
|
|
356
|
+
print(f"fairmind: {exc}", file=sys.stderr)
|
|
357
|
+
return EXIT_ERROR
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
if __name__ == "__main__": # pragma: no cover
|
|
361
|
+
sys.exit(main())
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fairmind-cli
|
|
3
|
+
Version: 0.3.1a0
|
|
4
|
+
Summary: Run FairMind's release gate and fairness tests from a terminal or CI.
|
|
5
|
+
Author-email: Adhithya Rajasekaran <adhithya@axonome.xyz>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: httpx>=0.27
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
12
|
+
|
|
13
|
+
# fairmind CLI
|
|
14
|
+
|
|
15
|
+
The release gate and fairness tests from a terminal or a CI job. One dependency, `httpx`.
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pipx install "git+https://github.com/adhit-r/fairmind.git#subdirectory=apps/cli"
|
|
19
|
+
# from a checkout of this repository: pipx install ./apps/cli
|
|
20
|
+
export FAIRMIND_API_URL=https://fairmind.example.com
|
|
21
|
+
export FAIRMIND_API_KEY=... # minted at POST /api/v1/auth/api-keys, valid 365 days
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Commands
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
fairmind systems
|
|
28
|
+
fairmind gate --system <id> # exit 0 go, 2 conditional go, 1 no go
|
|
29
|
+
fairmind gate --system <id> --allow-conditional # exit 0 on conditional go
|
|
30
|
+
fairmind gate --system <id> --require-approval # exit 1 unless the release decision is approved
|
|
31
|
+
fairmind test --system <id> --file predictions.csv \
|
|
32
|
+
--outcome repaid --prediction approved --score score --attributes gender,region
|
|
33
|
+
fairmind test ... --gate # exit by the gate's verdict after the run
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`test` uploads the CSV, runs disparate impact, demographic parity, equal opportunity, equalized odds and calibration per attribute with bootstrap intervals, and stores the result as measured evidence on the system. `gate` reads the release gate: five checks, each proven, attested or open, a confidence, the verdict from the impact-by-confidence matrix, and the latest release decision (`approvalStatus`: not requested, pending, approved or rejected). A rejected decision makes the verdict no go until a reviewer approves a new request. With `--require-approval` an approval with conditions counts as approved: an approved conditional go exits 0.
|
|
37
|
+
|
|
38
|
+
Exit code 3 means the command itself failed: no network, no key, wrong system id, a column that is not in the file. Add `--json` to any command for the raw response.
|
|
39
|
+
|
|
40
|
+
## In GitHub Actions
|
|
41
|
+
|
|
42
|
+
```yaml
|
|
43
|
+
- uses: adhit-r/fairmind/.github/actions/fairmind-gate@main # in this repository: ./.github/actions/fairmind-gate
|
|
44
|
+
with:
|
|
45
|
+
system: ${{ vars.FAIRMIND_SYSTEM_ID }}
|
|
46
|
+
api-url: ${{ vars.FAIRMIND_API_URL }}
|
|
47
|
+
api-key: ${{ secrets.FAIRMIND_API_KEY }}
|
|
48
|
+
predictions: artifacts/predictions.csv # optional: run the test first
|
|
49
|
+
outcome: repaid
|
|
50
|
+
prediction: approved
|
|
51
|
+
attributes: gender,region
|
|
52
|
+
require-approval: "true" # optional: fail unless the release decision is approved
|
|
53
|
+
```
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
fairmind_cli/__init__.py,sha256=tQ0HzAR1O06SseCZr_gvK2RwIwE-nBZg7GYBh2MZ2j8,103
|
|
2
|
+
fairmind_cli/main.py,sha256=FlZTZvz5q8famlb3ZEayS9XPSfxbq7z49Zzw80iisC8,18468
|
|
3
|
+
fairmind_cli-0.3.1a0.dist-info/METADATA,sha256=4kwb8vX_AV2fRDoQroc4t14iH6CXaVVyBMikOF1Awq8,2679
|
|
4
|
+
fairmind_cli-0.3.1a0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
5
|
+
fairmind_cli-0.3.1a0.dist-info/entry_points.txt,sha256=lBeS3RQFR6aHFD7y15CEGW5rbwGjKqMxmlF4U53tWBU,52
|
|
6
|
+
fairmind_cli-0.3.1a0.dist-info/top_level.txt,sha256=bscKW967wl_1HTOUmWIeABLG7VFllXwebfVZTQxr4iY,13
|
|
7
|
+
fairmind_cli-0.3.1a0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fairmind_cli
|