mcp-ops-kit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcp_ops_kit-0.1.0/LICENSE +21 -0
- mcp_ops_kit-0.1.0/PKG-INFO +127 -0
- mcp_ops_kit-0.1.0/README.md +100 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/__init__.py +16 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/audit.py +315 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/auth.py +572 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/crypto.py +141 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/errors.py +27 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/evals/__init__.py +0 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/evals/agents.py +54 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/evals/grader.py +275 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/evals/harness.py +256 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/evals/openai_agent.py +159 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/evals/run.py +445 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/limits.py +34 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/oauth.py +381 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/py.typed +0 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/server.py +356 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/smoke.py +83 -0
- mcp_ops_kit-0.1.0/mcp_ops_kit/sqlite.py +285 -0
- mcp_ops_kit-0.1.0/pyproject.toml +56 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nikhil
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: mcp-ops-kit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Hardening and trajectory evals for MCP servers that let agents act on booking systems: scoped, audience-bound tokens, OAuth JWTs, an encrypted audit log, encryption at rest, rate limits and a CI eval gate.
|
|
5
|
+
Project-URL: Homepage, https://github.com/nikhilll30/mcp-ops-server/tree/main/mcp_ops_kit
|
|
6
|
+
Project-URL: Source, https://github.com/nikhilll30/mcp-ops-server
|
|
7
|
+
Project-URL: Changelog, https://github.com/nikhilll30/mcp-ops-server/blob/main/FAILURES.md
|
|
8
|
+
Project-URL: Security, https://github.com/nikhilll30/mcp-ops-server/blob/main/SECURITY.md
|
|
9
|
+
Author: Nikhil
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,audit-log,evals,mcp,model-context-protocol,oauth
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Requires-Dist: cryptography>=42
|
|
23
|
+
Requires-Dist: mcp<3,>=2.3
|
|
24
|
+
Provides-Extra: live
|
|
25
|
+
Requires-Dist: openai<4,>=3.35; extra == 'live'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# mcp-ops-kit
|
|
29
|
+
|
|
30
|
+
The domain-free half of this repo: what an MCP server needs to let agents act on a booking
|
|
31
|
+
system safely, plus the trajectory evals that gate it in CI. It was **extracted from two working
|
|
32
|
+
domains**, not designed up front. The synthetic clinic (`mcp_ops_server`) came first. Then a
|
|
33
|
+
synthetic salon (`salon_ops`) was built by copying the clinic's code. The kit is what the two
|
|
34
|
+
copies turned out to share (Part 8, see [`FAILURES.md`](https://github.com/nikhilll30/mcp-ops-server/blob/kit-v0.1.0/FAILURES.md)).
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install mcp-ops-kit # add [live] for the OpenAI agent used in live eval runs
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
(Once the first release is tagged; until then, `pip install -e .` from this repo.)
|
|
41
|
+
|
|
42
|
+
The source lives in this repo, next to the two domains built on it. Releases are published to
|
|
43
|
+
PyPI from a `kit-vX.Y.Z` tag on `main` by trusted publishing, with attestations, after CI has
|
|
44
|
+
built the wheel and used it from a fresh venv (`release-kit.yml`).
|
|
45
|
+
|
|
46
|
+
## What the kit owns, and what a domain supplies
|
|
47
|
+
|
|
48
|
+
| Kit module | Gives you | Your domain supplies |
|
|
49
|
+
|---|---|---|
|
|
50
|
+
| `auth` | Hashed bearer tokens in a versioned file: each names its audience and expiry and can be revoked; the file is re-read on change, so revocation needs no restart. A token CLI (`issue`, `revoke`, `list`, `migrate`), scopes and a subject binding, all checked in server code (`Guard`) | An `AuthConfig`: scope names, the subject claim (`patient_id`, `customer_id`), its ID pattern, the default audience |
|
|
51
|
+
| `oauth` | OAuth 2.1 resource-server tokens: JWT access tokens (RFC 9068, RS256/ES256) checked against your authorization server's JWKS (file or URL), with a `jti` revocation file | Your issuer and JWKS location (settings) |
|
|
52
|
+
| `audit` | One encrypted, HMAC-chained record per tool call (who, token, tool, scope, subject ID, outcome, time), never the arguments; `verify` and `show` CLIs | Nothing |
|
|
53
|
+
| `crypto` | The data key (`new-key`), per-purpose subkeys, AES-256-GCM, atomic writes | The key, from your secret store |
|
|
54
|
+
| `limits` | A per-client token-bucket rate limit, applied before argument validation | Nothing |
|
|
55
|
+
| `server` | The audit-and-rate-limit-first `OpsServer`, which also rebuilds every result's text from the validated output, so an agent sees only declared fields. A `health` tool, `/mcp` with bearer auth, `/healthz`, env settings, a fail-closed `serve` (no token source, audit log or data key: no start) | Your tools (`register(server, guard)`), your settings fields, your env prefix |
|
|
56
|
+
| `sqlite` | One deadline per call: lock wait, busy wait and execution, with a late COMMIT rolled back. With a data key, the file on disk is encrypted (the database lives in memory; each committed write re-encrypts it atomically) | Your schema, seed and queries (subclass `SqliteStore`) |
|
|
57
|
+
| `errors` | `StoreError` and its kinds. Tools turn them into messages an agent can act on (`store_errors`) | Your own rule errors, as subclasses |
|
|
58
|
+
| `evals` | The harness (fresh data and a real server per task), the trajectory grader, the replay and OpenAI agents, the report and the CI gate | An `EvalDomain`, golden tasks and negative controls |
|
|
59
|
+
| `smoke` | `/healthz`, 401 without a token, then an MCP session | Your tool round trip, with each write read back |
|
|
60
|
+
|
|
61
|
+
## Build a domain on it: the salon as the worked example
|
|
62
|
+
|
|
63
|
+
1. **Data and rules.** Write a deterministic synthetic seed (`salon_ops/seed.py`) and a store
|
|
64
|
+
that subclasses `SqliteStore` (`salon_ops/store.py`). Enforce every rule in the store, and
|
|
65
|
+
make the critical one a database constraint too. The salon keeps one booked visit per
|
|
66
|
+
opening with a unique index.
|
|
67
|
+
2. **Auth.** One `AuthConfig` (`salon_ops/auth.py`): three scopes, tokens bound to
|
|
68
|
+
`customer_id` by the pattern `CUS-[0-9]{4}`, and the audience `salon-ops`. The token CLI
|
|
69
|
+
is `kit.main(AUTH)`.
|
|
70
|
+
3. **Tools.** Thin wrappers (`salon_ops/tools.py`). Each calls `guard.check(scope, subject_id)`
|
|
71
|
+
and then the store, inside `store_errors()`, and returns a `TypedDict` so it gets an output
|
|
72
|
+
schema.
|
|
73
|
+
4. **Server.** `kit.build_server(name, version, AUTH, register, audit=...)`, with
|
|
74
|
+
`kit.build_verifier(settings, AUTH)` and `kit.open_audit(settings)` in your
|
|
75
|
+
`build_http_app`, then `kit.build_http_app`, and `serve` in `__main__`
|
|
76
|
+
(`salon_ops/server.py`, `salon_ops/__main__.py`). A database file goes through
|
|
77
|
+
`require_key_for_file`, so it is never plaintext.
|
|
78
|
+
5. **Evals.** An `EvalDomain` (`salon_ops/evals/run.py`) says who the principals are, which
|
|
79
|
+
argument names the subject (`SubjectRule`), the default clock, and how to get fresh data
|
|
80
|
+
per task. Then write golden tasks and **negative controls for every grader check**
|
|
81
|
+
(`salon_ops/evals/golden/make_tasks.py`) before you trust the gate.
|
|
82
|
+
6. **Smoke test.** Your `exercise(session)` for `kit.smoke.main` (`salon_ops/smoke.py`).
|
|
83
|
+
|
|
84
|
+
The salon example has the following parts:
|
|
85
|
+
- 6 tools;
|
|
86
|
+
- 3 rules the clinic does not have: a stylist must offer the service, a 24-hour late-cancel
|
|
87
|
+
window, and at most 3 upcoming visits per customer;
|
|
88
|
+
- 18 golden tasks and 14 negative controls, gated in CI next to the clinic's.
|
|
89
|
+
|
|
90
|
+
## Guarantees the kit enforces, with tests
|
|
91
|
+
|
|
92
|
+
- **The kit never imports a domain.** `tests/test_kit.py` walks every import in `mcp_ops_kit/`.
|
|
93
|
+
- **A bound token never loads unbound.** Unknown keys are refused, so a clinic token bound to
|
|
94
|
+
a patient does not load in the salon as an unbound token with its `patient_id` silently
|
|
95
|
+
dropped. Before the kit, the clinic's own loader ignored unknown keys in exactly this way.
|
|
96
|
+
A token whose scopes another domain does not declare is refused too.
|
|
97
|
+
- **A token is for one server.** Every token names its audience (static tokens in the file,
|
|
98
|
+
JWTs in `aud`). A tokens file holding another audience's token does not load, and the
|
|
99
|
+
verifier checks the audience of each token again. This closes the Part 8 limit, where an
|
|
100
|
+
unbound token loaded in any domain declaring the same scope names (`tests/test_kit.py`).
|
|
101
|
+
- **Tokens expire and can be revoked on a running server.** Static tokens by `revoke`, JWTs
|
|
102
|
+
by `jti`; both files are re-read on change. If either file becomes invalid, every token it
|
|
103
|
+
governs is refused until it is fixed, so a revocation is never skipped
|
|
104
|
+
(`tests/test_tokens.py`, `tests/test_oauth.py`).
|
|
105
|
+
- **Every tool call is audited, or no call runs.** A record that cannot be written stops the
|
|
106
|
+
server from acting (the call already made still returns its real result). An edited,
|
|
107
|
+
deleted or reordered record fails `verify` at its line (`tests/test_audit.py`).
|
|
108
|
+
- **No plaintext database on disk.** A server refuses a database file without a key, and a
|
|
109
|
+
plaintext SQLite file with one (`tests/test_encryption.py`).
|
|
110
|
+
- **An agent sees only declared fields** (`tests/test_minimisation.py`), each pinned with a reason.
|
|
111
|
+
- **The state check sees every change.** The record diff grades the booking lifecycle (booked,
|
|
112
|
+
cancelled, booked-then-cancelled), and reports anything else (an edited, removed or
|
|
113
|
+
odd-status record) as `unexpected`. That fails the task unless the task expects it.
|
|
114
|
+
- **A refusal can be pinned to its error.** A step pattern may carry `"error": {"$icontains": ...}`,
|
|
115
|
+
so a refusal task fails when the agent hits some other error.
|
|
116
|
+
- **A scope a tool names but the domain never declared** raises an error. It is not quietly
|
|
117
|
+
treated as "forbidden".
|
|
118
|
+
- Everything else is held to both domains' full suites and eval gates: the clinic's (on SQLite
|
|
119
|
+
and on FHIR) and the salon's.
|
|
120
|
+
|
|
121
|
+
## Not in the kit (yet)
|
|
122
|
+
|
|
123
|
+
- The FHIR backend is clinic-specific by nature, and stays in `mcp_ops_server`. Encryption at
|
|
124
|
+
rest for FHIR data is the FHIR server's storage, not this code.
|
|
125
|
+
- Key rotation: there is no re-encrypt command yet. A new data key means a fresh audit log
|
|
126
|
+
and database file (keep the old key to read the old ones).
|
|
127
|
+
- Token introspection (RFC 7662) and a shared rate limit across replicas.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# mcp-ops-kit
|
|
2
|
+
|
|
3
|
+
The domain-free half of this repo: what an MCP server needs to let agents act on a booking
|
|
4
|
+
system safely, plus the trajectory evals that gate it in CI. It was **extracted from two working
|
|
5
|
+
domains**, not designed up front. The synthetic clinic (`mcp_ops_server`) came first. Then a
|
|
6
|
+
synthetic salon (`salon_ops`) was built by copying the clinic's code. The kit is what the two
|
|
7
|
+
copies turned out to share (Part 8, see [`FAILURES.md`](https://github.com/nikhilll30/mcp-ops-server/blob/kit-v0.1.0/FAILURES.md)).
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install mcp-ops-kit # add [live] for the OpenAI agent used in live eval runs
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
(Once the first release is tagged; until then, `pip install -e .` from this repo.)
|
|
14
|
+
|
|
15
|
+
The source lives in this repo, next to the two domains built on it. Releases are published to
|
|
16
|
+
PyPI from a `kit-vX.Y.Z` tag on `main` by trusted publishing, with attestations, after CI has
|
|
17
|
+
built the wheel and used it from a fresh venv (`release-kit.yml`).
|
|
18
|
+
|
|
19
|
+
## What the kit owns, and what a domain supplies
|
|
20
|
+
|
|
21
|
+
| Kit module | Gives you | Your domain supplies |
|
|
22
|
+
|---|---|---|
|
|
23
|
+
| `auth` | Hashed bearer tokens in a versioned file: each names its audience and expiry and can be revoked; the file is re-read on change, so revocation needs no restart. A token CLI (`issue`, `revoke`, `list`, `migrate`), scopes and a subject binding, all checked in server code (`Guard`) | An `AuthConfig`: scope names, the subject claim (`patient_id`, `customer_id`), its ID pattern, the default audience |
|
|
24
|
+
| `oauth` | OAuth 2.1 resource-server tokens: JWT access tokens (RFC 9068, RS256/ES256) checked against your authorization server's JWKS (file or URL), with a `jti` revocation file | Your issuer and JWKS location (settings) |
|
|
25
|
+
| `audit` | One encrypted, HMAC-chained record per tool call (who, token, tool, scope, subject ID, outcome, time), never the arguments; `verify` and `show` CLIs | Nothing |
|
|
26
|
+
| `crypto` | The data key (`new-key`), per-purpose subkeys, AES-256-GCM, atomic writes | The key, from your secret store |
|
|
27
|
+
| `limits` | A per-client token-bucket rate limit, applied before argument validation | Nothing |
|
|
28
|
+
| `server` | The audit-and-rate-limit-first `OpsServer`, which also rebuilds every result's text from the validated output, so an agent sees only declared fields. A `health` tool, `/mcp` with bearer auth, `/healthz`, env settings, a fail-closed `serve` (no token source, audit log or data key: no start) | Your tools (`register(server, guard)`), your settings fields, your env prefix |
|
|
29
|
+
| `sqlite` | One deadline per call: lock wait, busy wait and execution, with a late COMMIT rolled back. With a data key, the file on disk is encrypted (the database lives in memory; each committed write re-encrypts it atomically) | Your schema, seed and queries (subclass `SqliteStore`) |
|
|
30
|
+
| `errors` | `StoreError` and its kinds. Tools turn them into messages an agent can act on (`store_errors`) | Your own rule errors, as subclasses |
|
|
31
|
+
| `evals` | The harness (fresh data and a real server per task), the trajectory grader, the replay and OpenAI agents, the report and the CI gate | An `EvalDomain`, golden tasks and negative controls |
|
|
32
|
+
| `smoke` | `/healthz`, 401 without a token, then an MCP session | Your tool round trip, with each write read back |
|
|
33
|
+
|
|
34
|
+
## Build a domain on it: the salon as the worked example
|
|
35
|
+
|
|
36
|
+
1. **Data and rules.** Write a deterministic synthetic seed (`salon_ops/seed.py`) and a store
|
|
37
|
+
that subclasses `SqliteStore` (`salon_ops/store.py`). Enforce every rule in the store, and
|
|
38
|
+
make the critical one a database constraint too. The salon keeps one booked visit per
|
|
39
|
+
opening with a unique index.
|
|
40
|
+
2. **Auth.** One `AuthConfig` (`salon_ops/auth.py`): three scopes, tokens bound to
|
|
41
|
+
`customer_id` by the pattern `CUS-[0-9]{4}`, and the audience `salon-ops`. The token CLI
|
|
42
|
+
is `kit.main(AUTH)`.
|
|
43
|
+
3. **Tools.** Thin wrappers (`salon_ops/tools.py`). Each calls `guard.check(scope, subject_id)`
|
|
44
|
+
and then the store, inside `store_errors()`, and returns a `TypedDict` so it gets an output
|
|
45
|
+
schema.
|
|
46
|
+
4. **Server.** `kit.build_server(name, version, AUTH, register, audit=...)`, with
|
|
47
|
+
`kit.build_verifier(settings, AUTH)` and `kit.open_audit(settings)` in your
|
|
48
|
+
`build_http_app`, then `kit.build_http_app`, and `serve` in `__main__`
|
|
49
|
+
(`salon_ops/server.py`, `salon_ops/__main__.py`). A database file goes through
|
|
50
|
+
`require_key_for_file`, so it is never plaintext.
|
|
51
|
+
5. **Evals.** An `EvalDomain` (`salon_ops/evals/run.py`) says who the principals are, which
|
|
52
|
+
argument names the subject (`SubjectRule`), the default clock, and how to get fresh data
|
|
53
|
+
per task. Then write golden tasks and **negative controls for every grader check**
|
|
54
|
+
(`salon_ops/evals/golden/make_tasks.py`) before you trust the gate.
|
|
55
|
+
6. **Smoke test.** Your `exercise(session)` for `kit.smoke.main` (`salon_ops/smoke.py`).
|
|
56
|
+
|
|
57
|
+
The salon example has the following parts:
|
|
58
|
+
- 6 tools;
|
|
59
|
+
- 3 rules the clinic does not have: a stylist must offer the service, a 24-hour late-cancel
|
|
60
|
+
window, and at most 3 upcoming visits per customer;
|
|
61
|
+
- 18 golden tasks and 14 negative controls, gated in CI next to the clinic's.
|
|
62
|
+
|
|
63
|
+
## Guarantees the kit enforces, with tests
|
|
64
|
+
|
|
65
|
+
- **The kit never imports a domain.** `tests/test_kit.py` walks every import in `mcp_ops_kit/`.
|
|
66
|
+
- **A bound token never loads unbound.** Unknown keys are refused, so a clinic token bound to
|
|
67
|
+
a patient does not load in the salon as an unbound token with its `patient_id` silently
|
|
68
|
+
dropped. Before the kit, the clinic's own loader ignored unknown keys in exactly this way.
|
|
69
|
+
A token whose scopes another domain does not declare is refused too.
|
|
70
|
+
- **A token is for one server.** Every token names its audience (static tokens in the file,
|
|
71
|
+
JWTs in `aud`). A tokens file holding another audience's token does not load, and the
|
|
72
|
+
verifier checks the audience of each token again. This closes the Part 8 limit, where an
|
|
73
|
+
unbound token loaded in any domain declaring the same scope names (`tests/test_kit.py`).
|
|
74
|
+
- **Tokens expire and can be revoked on a running server.** Static tokens by `revoke`, JWTs
|
|
75
|
+
by `jti`; both files are re-read on change. If either file becomes invalid, every token it
|
|
76
|
+
governs is refused until it is fixed, so a revocation is never skipped
|
|
77
|
+
(`tests/test_tokens.py`, `tests/test_oauth.py`).
|
|
78
|
+
- **Every tool call is audited, or no call runs.** A record that cannot be written stops the
|
|
79
|
+
server from acting (the call already made still returns its real result). An edited,
|
|
80
|
+
deleted or reordered record fails `verify` at its line (`tests/test_audit.py`).
|
|
81
|
+
- **No plaintext database on disk.** A server refuses a database file without a key, and a
|
|
82
|
+
plaintext SQLite file with one (`tests/test_encryption.py`).
|
|
83
|
+
- **An agent sees only declared fields** (`tests/test_minimisation.py`), each pinned with a reason.
|
|
84
|
+
- **The state check sees every change.** The record diff grades the booking lifecycle (booked,
|
|
85
|
+
cancelled, booked-then-cancelled), and reports anything else (an edited, removed or
|
|
86
|
+
odd-status record) as `unexpected`. That fails the task unless the task expects it.
|
|
87
|
+
- **A refusal can be pinned to its error.** A step pattern may carry `"error": {"$icontains": ...}`,
|
|
88
|
+
so a refusal task fails when the agent hits some other error.
|
|
89
|
+
- **A scope a tool names but the domain never declared** raises an error. It is not quietly
|
|
90
|
+
treated as "forbidden".
|
|
91
|
+
- Everything else is held to both domains' full suites and eval gates: the clinic's (on SQLite
|
|
92
|
+
and on FHIR) and the salon's.
|
|
93
|
+
|
|
94
|
+
## Not in the kit (yet)
|
|
95
|
+
|
|
96
|
+
- The FHIR backend is clinic-specific by nature, and stays in `mcp_ops_server`. Encryption at
|
|
97
|
+
rest for FHIR data is the FHIR server's storage, not this code.
|
|
98
|
+
- Key rotation: there is no re-encrypt command yet. A new data key means a fresh audit log
|
|
99
|
+
and database file (keep the old key to read the old ones).
|
|
100
|
+
- Token introspection (RFC 7662) and a shared rate limit across replicas.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""mcp-ops-kit: the domain-free parts of an ops MCP server and its trajectory evals.
|
|
2
|
+
|
|
3
|
+
Extracted from two working domains in this repo (the clinic in ``mcp_ops_server`` and the
|
|
4
|
+
salon in ``salon_ops``), not designed up front. A domain supplies its tools, its store and
|
|
5
|
+
an ``AuthConfig``; the kit supplies:
|
|
6
|
+
|
|
7
|
+
- ``auth``: hashed bearer tokens, scopes and a subject binding, enforced in server code;
|
|
8
|
+
- ``limits``: a per-client rate limit, applied before argument validation;
|
|
9
|
+
- ``server``: the MCP server wiring, the authenticated HTTP app, ``/healthz``, a
|
|
10
|
+
fail-closed entry point;
|
|
11
|
+
- ``sqlite``: one deadline per call over SQLite (lock wait, busy wait, execution);
|
|
12
|
+
- ``evals``: the harness, the trajectory grader, the replay and OpenAI agents, the runner
|
|
13
|
+
with its report and CI gate, and an end-to-end smoke test.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
"""Audit log (45 CFR 164.312(b)): one record per tool call, encrypted and hash-chained.
|
|
2
|
+
|
|
3
|
+
Each record says who called (client ID and token ID), which tool, which scope and subject
|
|
4
|
+
it was checked against, the outcome and when, and never the call's arguments or results.
|
|
5
|
+
The subject is recorded only if it is a well-formed ID of the domain's subject (e.g.
|
|
6
|
+
``PAT-0008``); anything else is dropped, so free text cannot reach the log.
|
|
7
|
+
|
|
8
|
+
Integrity: record *n* carries ``prev``, the chain value of record *n-1*, and its own chain
|
|
9
|
+
value is ``HMAC-SHA256(audit-mac key, record n)``. Editing, deleting, inserting or reordering
|
|
10
|
+
a record breaks the chain from that point, and without the key it cannot be recomputed.
|
|
11
|
+
Confidentiality: each record is stored as AES-256-GCM ciphertext bound to its sequence
|
|
12
|
+
number. One line per record::
|
|
13
|
+
|
|
14
|
+
{"seq": 7, "ct": "<base64url of nonce || ciphertext || tag>"}
|
|
15
|
+
|
|
16
|
+
What it cannot show by itself: records cut off the *end* of the file. ``verify`` prints the
|
|
17
|
+
last sequence number and chain value; record them elsewhere (a ticket, a second log) to
|
|
18
|
+
detect truncation later. One process writes a log at a time (an exclusive lock).
|
|
19
|
+
|
|
20
|
+
python -m mcp_ops_kit.audit verify --file audit.log --key-env MCP_OPS_DATA_KEY
|
|
21
|
+
python -m mcp_ops_kit.audit show --file audit.log --key-env MCP_OPS_DATA_KEY
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import base64
|
|
26
|
+
import fcntl
|
|
27
|
+
import hashlib
|
|
28
|
+
import hmac
|
|
29
|
+
import json
|
|
30
|
+
import logging
|
|
31
|
+
import os
|
|
32
|
+
import sys
|
|
33
|
+
import threading
|
|
34
|
+
from collections.abc import Callable, Iterator
|
|
35
|
+
from contextvars import ContextVar
|
|
36
|
+
from dataclasses import dataclass, fields
|
|
37
|
+
from datetime import UTC, datetime
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
from mcp.server.auth.middleware.auth_context import get_access_token
|
|
41
|
+
from mcp.server.mcpserver.exceptions import ToolError
|
|
42
|
+
|
|
43
|
+
from mcp_ops_kit.crypto import DataKey, DecryptionError, seal, unseal
|
|
44
|
+
from mcp_ops_kit.errors import ConfigError
|
|
45
|
+
|
|
46
|
+
logger = logging.getLogger(__name__)
|
|
47
|
+
|
|
48
|
+
GENESIS = "0" * 64
|
|
49
|
+
OUTCOMES = ("ok", "error", "denied", "rate_limited", "invalid")
|
|
50
|
+
# Every key a record has, in the order ``show`` prints them. Nothing else is ever written.
|
|
51
|
+
RECORD_KEYS = (
|
|
52
|
+
"seq",
|
|
53
|
+
"time",
|
|
54
|
+
"client_id",
|
|
55
|
+
"token_id",
|
|
56
|
+
"tool",
|
|
57
|
+
"scope",
|
|
58
|
+
"subject",
|
|
59
|
+
"outcome",
|
|
60
|
+
"error",
|
|
61
|
+
"prev",
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class AuditTamperedError(Exception):
|
|
66
|
+
"""The log failed verification at ``line`` (1-based)."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, line: int, reason: str) -> None:
|
|
69
|
+
super().__init__(f"line {line}: {reason}")
|
|
70
|
+
self.line = line
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class Call:
|
|
75
|
+
"""What one tool call has told the audit log so far. The guard fills ``scope``,
|
|
76
|
+
``subject`` and ``checked``; the server fills ``outcome`` and ``error``."""
|
|
77
|
+
|
|
78
|
+
tool: str
|
|
79
|
+
client_id: str | None
|
|
80
|
+
token_id: str | None
|
|
81
|
+
scope: str | None = None
|
|
82
|
+
subject: str | None = None
|
|
83
|
+
checked: bool = False
|
|
84
|
+
outcome: str | None = None
|
|
85
|
+
error: str | None = None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
_CURRENT: ContextVar[Call | None] = ContextVar("mcp_ops_kit_audit_call", default=None)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def mark(**updates: object) -> None:
|
|
92
|
+
"""Note facts about the call being audited (a no-op when nothing is audited). The
|
|
93
|
+
first ``outcome`` set wins: a refusal stays a refusal whatever is raised after it."""
|
|
94
|
+
call = _CURRENT.get()
|
|
95
|
+
if call is None:
|
|
96
|
+
return
|
|
97
|
+
for name, value in updates.items():
|
|
98
|
+
if name == "outcome" and call.outcome is not None:
|
|
99
|
+
continue
|
|
100
|
+
setattr(call, name, value)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _canonical(record: dict[str, Any]) -> bytes:
|
|
104
|
+
return json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _aad(seq: int) -> bytes:
|
|
108
|
+
return f"mcp-ops-kit/audit/v1/{seq}".encode()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class _Keys:
|
|
112
|
+
def __init__(self, key: DataKey) -> None:
|
|
113
|
+
self.enc = key.subkey("audit-enc")
|
|
114
|
+
self.mac = key.subkey("audit-mac")
|
|
115
|
+
|
|
116
|
+
def chain(self, record: dict[str, Any]) -> str:
|
|
117
|
+
return hmac.new(self.mac, _canonical(record), hashlib.sha256).hexdigest()
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _decode_line(keys: _Keys, line: bytes, number: int) -> dict[str, Any]:
|
|
121
|
+
try:
|
|
122
|
+
outer = json.loads(line)
|
|
123
|
+
seq, ct = outer["seq"], outer["ct"]
|
|
124
|
+
if set(outer) != {"seq", "ct"} or not isinstance(seq, int) or not isinstance(ct, str):
|
|
125
|
+
raise ValueError("unexpected fields")
|
|
126
|
+
blob = base64.urlsafe_b64decode(ct + "=" * (-len(ct) % 4))
|
|
127
|
+
record = json.loads(unseal(keys.enc, blob, _aad(seq)))
|
|
128
|
+
except DecryptionError as exc:
|
|
129
|
+
raise AuditTamperedError(number, f"does not decrypt ({exc})") from exc
|
|
130
|
+
except (ValueError, KeyError, TypeError) as exc:
|
|
131
|
+
raise AuditTamperedError(number, f"not an audit record ({exc})") from exc
|
|
132
|
+
if not isinstance(record, dict) or tuple(sorted(record)) != tuple(sorted(RECORD_KEYS)):
|
|
133
|
+
raise AuditTamperedError(number, "record has unexpected fields")
|
|
134
|
+
if record["seq"] != seq:
|
|
135
|
+
raise AuditTamperedError(number, "sequence number inside and outside differ")
|
|
136
|
+
return record
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def records(path: str, key: DataKey) -> Iterator[dict[str, Any]]:
|
|
140
|
+
"""Yield every record, checking the whole chain; raises ``AuditTamperedError`` at the
|
|
141
|
+
first line that breaks it. A missing file is an empty log."""
|
|
142
|
+
keys = _Keys(key)
|
|
143
|
+
prev, expected = GENESIS, 1
|
|
144
|
+
try:
|
|
145
|
+
f = open(path, "rb") # noqa: SIM115 - closed below; a generator holds it open
|
|
146
|
+
except FileNotFoundError:
|
|
147
|
+
return
|
|
148
|
+
with f:
|
|
149
|
+
for number, line in enumerate(f, start=1):
|
|
150
|
+
if not line.endswith(b"\n"):
|
|
151
|
+
raise AuditTamperedError(number, "incomplete last line")
|
|
152
|
+
record = _decode_line(keys, line, number)
|
|
153
|
+
if record["seq"] != expected:
|
|
154
|
+
raise AuditTamperedError(
|
|
155
|
+
number, f"sequence {record['seq']} where {expected} was expected"
|
|
156
|
+
)
|
|
157
|
+
if not hmac.compare_digest(record["prev"], prev):
|
|
158
|
+
raise AuditTamperedError(number, "chain broken: prev does not match")
|
|
159
|
+
prev, expected = keys.chain(record), expected + 1
|
|
160
|
+
yield record
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def verify(path: str, key: DataKey) -> tuple[int, str]:
|
|
164
|
+
"""Check the whole log. Returns (records, chain value of the last one)."""
|
|
165
|
+
keys, count, head = _Keys(key), 0, GENESIS
|
|
166
|
+
for record in records(path, key):
|
|
167
|
+
count, head = record["seq"], keys.chain(record)
|
|
168
|
+
return count, head
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class AuditLog:
|
|
172
|
+
"""The append side. Opened once per server process; refuses to start on a log whose
|
|
173
|
+
last record does not decrypt (wrong key, or tampered), or that another process holds.
|
|
174
|
+
|
|
175
|
+
``begin`` / ``end`` bracket one tool call (``server.OpsServer`` calls them). If a record
|
|
176
|
+
cannot be written (disk full, file gone), the call that was just made still returns its
|
|
177
|
+
real result, since it has already happened, but the log is marked failed and every
|
|
178
|
+
later call is refused: the server does not act without an audit trail.
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
def __init__(
|
|
182
|
+
self,
|
|
183
|
+
path: str,
|
|
184
|
+
key: DataKey,
|
|
185
|
+
*,
|
|
186
|
+
fsync: bool = True,
|
|
187
|
+
clock: Callable[[], datetime] = lambda: datetime.now(UTC),
|
|
188
|
+
) -> None:
|
|
189
|
+
self.path = path
|
|
190
|
+
self._keys = _Keys(key)
|
|
191
|
+
self._fsync = fsync
|
|
192
|
+
self._clock = clock
|
|
193
|
+
self._lock = threading.Lock()
|
|
194
|
+
self.failed: str | None = None
|
|
195
|
+
try:
|
|
196
|
+
self._fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600)
|
|
197
|
+
except OSError as exc:
|
|
198
|
+
raise ConfigError(f"Cannot open audit log {path}: {exc.strerror}") from exc
|
|
199
|
+
try:
|
|
200
|
+
fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
201
|
+
except OSError as exc:
|
|
202
|
+
os.close(self._fd)
|
|
203
|
+
raise ConfigError(f"Audit log {path} is in use by another process") from exc
|
|
204
|
+
try:
|
|
205
|
+
self._seq, self._prev = self._tail()
|
|
206
|
+
except BaseException:
|
|
207
|
+
self.close()
|
|
208
|
+
raise
|
|
209
|
+
|
|
210
|
+
def _tail(self) -> tuple[int, str]:
|
|
211
|
+
with open(self._fd, "rb", closefd=False) as f:
|
|
212
|
+
lines = f.read().splitlines(keepends=True)
|
|
213
|
+
if not lines:
|
|
214
|
+
return 0, GENESIS
|
|
215
|
+
if not lines[-1].endswith(b"\n"):
|
|
216
|
+
raise ConfigError(
|
|
217
|
+
f"Audit log {self.path}: the last line is incomplete (a crash during a write?). "
|
|
218
|
+
"Run verify on it before doing anything else."
|
|
219
|
+
)
|
|
220
|
+
try:
|
|
221
|
+
last = _decode_line(self._keys, lines[-1], len(lines))
|
|
222
|
+
except AuditTamperedError as exc:
|
|
223
|
+
raise ConfigError(
|
|
224
|
+
f"Audit log {self.path} {exc}: the wrong data key, or the log was changed. "
|
|
225
|
+
"Refusing to append to it."
|
|
226
|
+
) from exc
|
|
227
|
+
if last["seq"] != len(lines):
|
|
228
|
+
raise ConfigError(
|
|
229
|
+
f"Audit log {self.path}: {len(lines)} lines but last seq {last['seq']}"
|
|
230
|
+
)
|
|
231
|
+
return last["seq"], self._keys.chain(last)
|
|
232
|
+
|
|
233
|
+
def append(self, **values: Any) -> int:
|
|
234
|
+
"""Write one record; returns its sequence number."""
|
|
235
|
+
with self._lock:
|
|
236
|
+
seq = self._seq + 1
|
|
237
|
+
record = {k: values.get(k) for k in RECORD_KEYS}
|
|
238
|
+
stamp = self._clock().astimezone(UTC).isoformat(timespec="milliseconds")
|
|
239
|
+
record |= {"seq": seq, "time": stamp.replace("+00:00", "Z"), "prev": self._prev}
|
|
240
|
+
blob = seal(self._keys.enc, _canonical(record), _aad(seq))
|
|
241
|
+
ct = base64.urlsafe_b64encode(blob).decode().rstrip("=")
|
|
242
|
+
line = json.dumps({"seq": seq, "ct": ct}, separators=(",", ":")).encode() + b"\n"
|
|
243
|
+
self._check_still_the_file()
|
|
244
|
+
written = os.write(self._fd, line)
|
|
245
|
+
if written != len(line):
|
|
246
|
+
raise OSError(f"short write to audit log ({written} of {len(line)} bytes)")
|
|
247
|
+
if self._fsync:
|
|
248
|
+
os.fsync(self._fd)
|
|
249
|
+
self._seq, self._prev = seq, self._keys.chain(record)
|
|
250
|
+
return seq
|
|
251
|
+
|
|
252
|
+
def _check_still_the_file(self) -> None:
|
|
253
|
+
"""Writes go to the open file descriptor. If the log was deleted or replaced while
|
|
254
|
+
open, they would land in a file nobody can read, and another process could start a
|
|
255
|
+
new log at the path (the lock is on the old file). So refuse, and the server stops
|
|
256
|
+
acting (Codex review, PR #20)."""
|
|
257
|
+
open_file = os.fstat(self._fd)
|
|
258
|
+
try:
|
|
259
|
+
at_path = os.stat(self.path)
|
|
260
|
+
except FileNotFoundError as exc:
|
|
261
|
+
raise OSError(f"audit log {self.path} was deleted while in use") from exc
|
|
262
|
+
if (open_file.st_dev, open_file.st_ino) != (at_path.st_dev, at_path.st_ino):
|
|
263
|
+
raise OSError(f"audit log {self.path} was replaced while in use")
|
|
264
|
+
|
|
265
|
+
# ---- one tool call ---------------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
def begin(self, tool: str) -> tuple[Call, object]:
|
|
268
|
+
"""Start auditing a call. Refuses it outright if an earlier record failed to write."""
|
|
269
|
+
if self.failed is not None:
|
|
270
|
+
raise ToolError("The audit log is unavailable, so this server is refusing calls.")
|
|
271
|
+
token = get_access_token()
|
|
272
|
+
token_id = hashlib.sha256(token.token.encode()).hexdigest()[:12] if token else None
|
|
273
|
+
call = Call(tool, token.client_id if token else None, token_id)
|
|
274
|
+
return call, _CURRENT.set(call)
|
|
275
|
+
|
|
276
|
+
def end(self, call: Call, reset: object) -> None:
|
|
277
|
+
_CURRENT.reset(reset) # type: ignore[arg-type] - the token begin() returned
|
|
278
|
+
values = {f.name: getattr(call, f.name) for f in fields(Call) if f.name != "checked"}
|
|
279
|
+
try:
|
|
280
|
+
self.append(**values)
|
|
281
|
+
except OSError as exc:
|
|
282
|
+
self.failed = str(exc)
|
|
283
|
+
logger.critical("Audit log %s write failed; refusing further calls: %s", self.path, exc)
|
|
284
|
+
|
|
285
|
+
def close(self) -> None:
|
|
286
|
+
if self._fd >= 0:
|
|
287
|
+
os.close(self._fd) # releases the lock
|
|
288
|
+
self._fd = -1
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def main(argv: list[str] | None = None) -> None:
|
|
292
|
+
parser = argparse.ArgumentParser(prog="python -m mcp_ops_kit.audit")
|
|
293
|
+
parser.add_argument("command", choices=["verify", "show"])
|
|
294
|
+
parser.add_argument("--file", required=True)
|
|
295
|
+
parser.add_argument(
|
|
296
|
+
"--key-env", required=True, help="the variable holding the data key, e.g. MCP_OPS_DATA_KEY"
|
|
297
|
+
)
|
|
298
|
+
args = parser.parse_args(argv)
|
|
299
|
+
raw = os.environ.get(args.key_env)
|
|
300
|
+
if not raw:
|
|
301
|
+
sys.exit(f"{args.key_env} is not set")
|
|
302
|
+
key = DataKey.from_text(raw, args.key_env)
|
|
303
|
+
try:
|
|
304
|
+
if args.command == "verify":
|
|
305
|
+
count, head = verify(args.file, key)
|
|
306
|
+
print(f"ok: {count} records, chain intact; last seq {count}, head {head}") # noqa: T201
|
|
307
|
+
else:
|
|
308
|
+
for record in records(args.file, key):
|
|
309
|
+
print(json.dumps({k: record[k] for k in RECORD_KEYS})) # noqa: T201
|
|
310
|
+
except AuditTamperedError as exc:
|
|
311
|
+
sys.exit(f"TAMPERED: {args.file} {exc}")
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
if __name__ == "__main__":
|
|
315
|
+
main()
|