verify-outcomes 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verify_outcomes-0.4.1/LICENSE +21 -0
- verify_outcomes-0.4.1/PKG-INFO +103 -0
- verify_outcomes-0.4.1/README.md +84 -0
- verify_outcomes-0.4.1/pyproject.toml +31 -0
- verify_outcomes-0.4.1/setup.cfg +4 -0
- verify_outcomes-0.4.1/src/verify/__init__.py +5 -0
- verify_outcomes-0.4.1/src/verify/__main__.py +3 -0
- verify_outcomes-0.4.1/src/verify/cli.py +135 -0
- verify_outcomes-0.4.1/src/verify/contract.py +33 -0
- verify_outcomes-0.4.1/src/verify/core.py +152 -0
- verify_outcomes-0.4.1/src/verify/hubspot.py +72 -0
- verify_outcomes-0.4.1/src/verify/intercom.py +82 -0
- verify_outcomes-0.4.1/src/verify/local.py +103 -0
- verify_outcomes-0.4.1/src/verify/mcp_server.py +117 -0
- verify_outcomes-0.4.1/src/verify/push.py +32 -0
- verify_outcomes-0.4.1/src/verify/shopify.py +118 -0
- verify_outcomes-0.4.1/src/verify/stripe.py +119 -0
- verify_outcomes-0.4.1/src/verify/zendesk.py +118 -0
- verify_outcomes-0.4.1/src/verify_outcomes.egg-info/PKG-INFO +103 -0
- verify_outcomes-0.4.1/src/verify_outcomes.egg-info/SOURCES.txt +23 -0
- verify_outcomes-0.4.1/src/verify_outcomes.egg-info/dependency_links.txt +1 -0
- verify_outcomes-0.4.1/src/verify_outcomes.egg-info/entry_points.txt +2 -0
- verify_outcomes-0.4.1/src/verify_outcomes.egg-info/top_level.txt +1 -0
- verify_outcomes-0.4.1/tests/test_extras.py +34 -0
- verify_outcomes-0.4.1/tests/test_verify.py +134 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Abhishek Shankar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: verify-outcomes
|
|
3
|
+
Version: 0.4.1
|
|
4
|
+
Summary: Independent readback verification for AI agents and automations. Unknown fails closed.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/abhishekshankar/verify
|
|
7
|
+
Project-URL: Issues, https://github.com/abhishekshankar/verify/issues
|
|
8
|
+
Keywords: ai-agents,verification,audit,stripe,zendesk,hubspot,llmops
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Software Development :: Testing
|
|
14
|
+
Classifier: Topic :: System :: Monitoring
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# Verify
|
|
21
|
+
|
|
22
|
+
Your agent says "done". Verify checks the system of record.
|
|
23
|
+
|
|
24
|
+
AI agents and automations report success from their own logs. Verify ignores the claim, reads the
|
|
25
|
+
system of record back with a **separate read-only credential**, and returns one of five verdicts.
|
|
26
|
+
Anything it can't confirm fails closed.
|
|
27
|
+
|
|
28
|
+

|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
Agent says: 3/3 "Refund issued ✅"
|
|
32
|
+
Verify (Stripe test mode, independence=full):
|
|
33
|
+
PASS L2
|
|
34
|
+
FAIL L2 refund.amount 2500 ≠ 5000
|
|
35
|
+
NOT_OBSERVED L1 refund never reached Stripe
|
|
36
|
+
Verified success rate: 1/3 hash chain intact: True
|
|
37
|
+
```
|
|
38
|
+
Real output, Stripe test mode, 2026-11-07. The same run on Zendesk: 1/3.
|
|
39
|
+
|
|
40
|
+
## Verdicts
|
|
41
|
+
| Verdict | Meaning |
|
|
42
|
+
|---|---|
|
|
43
|
+
| `pass` | every postcondition observed in the system of record |
|
|
44
|
+
| `fail` | a postcondition observed false, or a forbidden side effect observed |
|
|
45
|
+
| `inapplicable` | preconditions not met; the action should never have run |
|
|
46
|
+
| `not_observed` | the subject record is absent after the observe window |
|
|
47
|
+
| `blocked` | readback impossible (auth, outage, rate limit) |
|
|
48
|
+
|
|
49
|
+
Only `pass` counts as success.
|
|
50
|
+
|
|
51
|
+
## Quick start
|
|
52
|
+
Python 3.10+, no dependencies.
|
|
53
|
+
```bash
|
|
54
|
+
pip install verify-outcomes # or: pip install git+https://github.com/abhishekshankar/verify
|
|
55
|
+
verify --version
|
|
56
|
+
verify init stripe # writes contracts/ and an example claim
|
|
57
|
+
verify selfcheck # all verdicts, tamper detection, every adapter offline
|
|
58
|
+
verify demo local # SQLite stands in for Stripe + Zendesk
|
|
59
|
+
verify ledger demo_out/ledger.db
|
|
60
|
+
verify validate contracts/*.json # reject bad contracts before any action runs
|
|
61
|
+
verify ledger export demo_out/ledger.db audit.jsonl && verify ledger verify audit.jsonl
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Live adapters (test accounts only; keys come from env vars, never from code):
|
|
65
|
+
```bash
|
|
66
|
+
STRIPE_AGENT_KEY=sk_test_... STRIPE_VERIFY_KEY=rk_test_... verify demo stripe
|
|
67
|
+
ZENDESK_SUBDOMAIN=... ZENDESK_AGENT_OAUTH=... ZENDESK_VERIFY_TOKEN=... verify demo zendesk
|
|
68
|
+
```
|
|
69
|
+
`STRIPE_VERIFY_KEY` should be a restricted key with Charges and Refunds set to Read.
|
|
70
|
+
`ZENDESK_VERIFY_TOKEN` should be an OAuth token with scope `read`. `HUBSPOT_VERIFY_TOKEN` should be a private-app token with only `crm.objects.deals.read`. `INTERCOM_VERIFY_TOKEN` should be an Intercom app access token with only the Read conversations permission. `SHOPIFY_SHOP` (the `*.myshopify.com` subdomain) and `SHOPIFY_VERIFY_TOKEN` should come from a custom app with only `read_orders`. Live Stripe keys are refused.
|
|
71
|
+
|
|
72
|
+
## Check one real action
|
|
73
|
+
Write a contract once, then check each claim the agent makes. The exit code is 0 only on `pass`, so it can gate a pipeline. A claim missing a field the contract needs is rejected (exit 2) before anything is read.
|
|
74
|
+
```bash
|
|
75
|
+
STRIPE_VERIFY_KEY=rk_test_... verify check contracts/refund.full.stripe.json claim.json --ledger verify.db
|
|
76
|
+
```
|
|
77
|
+
`claim.json` holds what the agent says it did, e.g. `{"actor": "support-agent", "charge_id": "ch_...", "refund_id": "re_...", "amount": 5000}` (see `examples/`).
|
|
78
|
+
|
|
79
|
+
Many claims at once (one JSON object per line; exit 0 only if every claim passes):
|
|
80
|
+
```bash
|
|
81
|
+
verify check-batch contracts/refund.full.stripe.json claims.jsonl --ledger verify.db
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Hosted ledger (optional)
|
|
85
|
+
`hosted/` is an append-only, multi-tenant ledger API (SQLite or Postgres). Push a local ledger to it with
|
|
86
|
+
`VERIFY_HOSTED_KEY=... verify push verify.db --to https://ledger.example.com`. Deployment to AWS us-east-2 is in `hosted/deploy/` (Terraform; not applied).
|
|
87
|
+
|
|
88
|
+
## Use from an agent
|
|
89
|
+
`python -m verify.mcp_server` is an MCP server with `verify_check` and `verify_validate` (see `docs/MCP.md`). Framework examples are in `examples/`.
|
|
90
|
+
|
|
91
|
+
## How it works
|
|
92
|
+
1. **Contract**, written before the action: subject, preconditions, postconditions, forbidden effects, windows, scope.
|
|
93
|
+
2. **Claim**: the actor's "done" is recorded, never used as evidence.
|
|
94
|
+
3. **Readback**: an adapter reads only the fields in scope, with its own credential.
|
|
95
|
+
4. **Verdict**: evaluated, then written to an append-only SQLite ledger (`synchronous=FULL`) where each event hashes the one before it.
|
|
96
|
+
|
|
97
|
+
## Layout
|
|
98
|
+
Python API: `verify.check(contract, claim, reader)`; see `docs/ADAPTERS.md`. Contracts: `docs/CONTRACTS.md`.
|
|
99
|
+
|
|
100
|
+
`src/verify/core.py` holds evaluation, the ledger and the verify loop; `contract.py` loads and validates contracts (`contracts/*.json`). The adapters are `stripe.py`, `zendesk.py`, `hubspot.py` (deals), `intercom.py` (conversations), `shopify.py` (refunds, Admin GraphQL) (the last three offline-tested, live run pending) and `local.py`.
|
|
101
|
+
|
|
102
|
+
## Status
|
|
103
|
+
v0.4, pre-production. Not yet a stable API. MIT licence.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Verify
|
|
2
|
+
|
|
3
|
+
Your agent says "done". Verify checks the system of record.
|
|
4
|
+
|
|
5
|
+
AI agents and automations report success from their own logs. Verify ignores the claim, reads the
|
|
6
|
+
system of record back with a **separate read-only credential**, and returns one of five verdicts.
|
|
7
|
+
Anything it can't confirm fails closed.
|
|
8
|
+
|
|
9
|
+

|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
Agent says: 3/3 "Refund issued ✅"
|
|
13
|
+
Verify (Stripe test mode, independence=full):
|
|
14
|
+
PASS L2
|
|
15
|
+
FAIL L2 refund.amount 2500 ≠ 5000
|
|
16
|
+
NOT_OBSERVED L1 refund never reached Stripe
|
|
17
|
+
Verified success rate: 1/3 hash chain intact: True
|
|
18
|
+
```
|
|
19
|
+
Real output, Stripe test mode, 2026-11-07. The same run on Zendesk: 1/3.
|
|
20
|
+
|
|
21
|
+
## Verdicts
|
|
22
|
+
| Verdict | Meaning |
|
|
23
|
+
|---|---|
|
|
24
|
+
| `pass` | every postcondition observed in the system of record |
|
|
25
|
+
| `fail` | a postcondition observed false, or a forbidden side effect observed |
|
|
26
|
+
| `inapplicable` | preconditions not met; the action should never have run |
|
|
27
|
+
| `not_observed` | the subject record is absent after the observe window |
|
|
28
|
+
| `blocked` | readback impossible (auth, outage, rate limit) |
|
|
29
|
+
|
|
30
|
+
Only `pass` counts as success.
|
|
31
|
+
|
|
32
|
+
## Quick start
|
|
33
|
+
Python 3.10+, no dependencies.
|
|
34
|
+
```bash
|
|
35
|
+
pip install verify-outcomes # or: pip install git+https://github.com/abhishekshankar/verify
|
|
36
|
+
verify --version
|
|
37
|
+
verify init stripe # writes contracts/ and an example claim
|
|
38
|
+
verify selfcheck # all verdicts, tamper detection, every adapter offline
|
|
39
|
+
verify demo local # SQLite stands in for Stripe + Zendesk
|
|
40
|
+
verify ledger demo_out/ledger.db
|
|
41
|
+
verify validate contracts/*.json # reject bad contracts before any action runs
|
|
42
|
+
verify ledger export demo_out/ledger.db audit.jsonl && verify ledger verify audit.jsonl
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Live adapters (test accounts only; keys come from env vars, never from code):
|
|
46
|
+
```bash
|
|
47
|
+
STRIPE_AGENT_KEY=sk_test_... STRIPE_VERIFY_KEY=rk_test_... verify demo stripe
|
|
48
|
+
ZENDESK_SUBDOMAIN=... ZENDESK_AGENT_OAUTH=... ZENDESK_VERIFY_TOKEN=... verify demo zendesk
|
|
49
|
+
```
|
|
50
|
+
`STRIPE_VERIFY_KEY` should be a restricted key with Charges and Refunds set to Read.
|
|
51
|
+
`ZENDESK_VERIFY_TOKEN` should be an OAuth token with scope `read`. `HUBSPOT_VERIFY_TOKEN` should be a private-app token with only `crm.objects.deals.read`. `INTERCOM_VERIFY_TOKEN` should be an Intercom app access token with only the Read conversations permission. `SHOPIFY_SHOP` (the `*.myshopify.com` subdomain) and `SHOPIFY_VERIFY_TOKEN` should come from a custom app with only `read_orders`. Live Stripe keys are refused.
|
|
52
|
+
|
|
53
|
+
## Check one real action
|
|
54
|
+
Write a contract once, then check each claim the agent makes. The exit code is 0 only on `pass`, so it can gate a pipeline. A claim missing a field the contract needs is rejected (exit 2) before anything is read.
|
|
55
|
+
```bash
|
|
56
|
+
STRIPE_VERIFY_KEY=rk_test_... verify check contracts/refund.full.stripe.json claim.json --ledger verify.db
|
|
57
|
+
```
|
|
58
|
+
`claim.json` holds what the agent says it did, e.g. `{"actor": "support-agent", "charge_id": "ch_...", "refund_id": "re_...", "amount": 5000}` (see `examples/`).
|
|
59
|
+
|
|
60
|
+
Many claims at once (one JSON object per line; exit 0 only if every claim passes):
|
|
61
|
+
```bash
|
|
62
|
+
verify check-batch contracts/refund.full.stripe.json claims.jsonl --ledger verify.db
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Hosted ledger (optional)
|
|
66
|
+
`hosted/` is an append-only, multi-tenant ledger API (SQLite or Postgres). Push a local ledger to it with
|
|
67
|
+
`VERIFY_HOSTED_KEY=... verify push verify.db --to https://ledger.example.com`. Deployment to AWS us-east-2 is in `hosted/deploy/` (Terraform; not applied).
|
|
68
|
+
|
|
69
|
+
## Use from an agent
|
|
70
|
+
`python -m verify.mcp_server` is an MCP server with `verify_check` and `verify_validate` (see `docs/MCP.md`). Framework examples are in `examples/`.
|
|
71
|
+
|
|
72
|
+
## How it works
|
|
73
|
+
1. **Contract**, written before the action: subject, preconditions, postconditions, forbidden effects, windows, scope.
|
|
74
|
+
2. **Claim**: the actor's "done" is recorded, never used as evidence.
|
|
75
|
+
3. **Readback**: an adapter reads only the fields in scope, with its own credential.
|
|
76
|
+
4. **Verdict**: evaluated, then written to an append-only SQLite ledger (`synchronous=FULL`) where each event hashes the one before it.
|
|
77
|
+
|
|
78
|
+
## Layout
|
|
79
|
+
Python API: `verify.check(contract, claim, reader)`; see `docs/ADAPTERS.md`. Contracts: `docs/CONTRACTS.md`.
|
|
80
|
+
|
|
81
|
+
`src/verify/core.py` holds evaluation, the ledger and the verify loop; `contract.py` loads and validates contracts (`contracts/*.json`). The adapters are `stripe.py`, `zendesk.py`, `hubspot.py` (deals), `intercom.py` (conversations), `shopify.py` (refunds, Admin GraphQL) (the last three offline-tested, live run pending) and `local.py`.
|
|
82
|
+
|
|
83
|
+
## Status
|
|
84
|
+
v0.4, pre-production. Not yet a stable API. MIT licence.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "verify-outcomes"
|
|
7
|
+
version = "0.4.1"
|
|
8
|
+
description = "Independent readback verification for AI agents and automations. Unknown fails closed."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
dependencies = []
|
|
13
|
+
keywords = ["ai-agents", "verification", "audit", "stripe", "zendesk", "hubspot", "llmops"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Topic :: Software Development :: Testing",
|
|
20
|
+
"Topic :: System :: Monitoring",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/abhishekshankar/verify"
|
|
25
|
+
Issues = "https://github.com/abhishekshankar/verify/issues"
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
verify = "verify.cli:main"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
where = ["src"]
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""usage:
|
|
2
|
+
verify check <contract.json> <claim.json> [--ledger verify.db] exit 0 only on pass
|
|
3
|
+
verify check-batch <contract.json> <claims.jsonl> [--ledger verify.db] one line per claim; exit 0 only if all pass
|
|
4
|
+
verify push <ledger.db> --to <url> upload to the hosted ledger (key: env VERIFY_HOSTED_KEY)
|
|
5
|
+
verify validate <contract.json>...
|
|
6
|
+
verify ledger <db>
|
|
7
|
+
verify ledger export <db> [out.jsonl] hash-chained JSONL + chain head
|
|
8
|
+
verify ledger verify <export.jsonl> check an export offline
|
|
9
|
+
verify init {stripe,zendesk,hubspot,intercom,shopify} [dir]
|
|
10
|
+
verify --version
|
|
11
|
+
verify demo {local,stripe,zendesk}
|
|
12
|
+
verify selfcheck
|
|
13
|
+
"""
|
|
14
|
+
import json, os, sys
|
|
15
|
+
from . import contract, hubspot, intercom, local, shopify, stripe, zendesk
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .core import BadClaim, Ledger, verify, verify_export
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def reader_for(c):
|
|
21
|
+
"""Pick the read-only adapter from the contract's target_system. Credentials only from env."""
|
|
22
|
+
t = c["target_system"]
|
|
23
|
+
if t == "stripe":
|
|
24
|
+
key = stripe.test_key("STRIPE_VERIFY_KEY")
|
|
25
|
+
if not key: sys.exit("Set STRIPE_VERIFY_KEY (restricted rk_test_ key, Charges+Refunds Read).")
|
|
26
|
+
return stripe.make_reader(stripe.stripe_http, key)
|
|
27
|
+
if t == "zendesk":
|
|
28
|
+
sub, tok = os.environ.get("ZENDESK_SUBDOMAIN"), os.environ.get("ZENDESK_VERIFY_TOKEN")
|
|
29
|
+
if not (sub and tok): sys.exit("Set ZENDESK_SUBDOMAIN and ZENDESK_VERIFY_TOKEN (OAuth, scope read).")
|
|
30
|
+
return zendesk.make_reader(zendesk.zd_http(sub, f"Bearer {tok}"))
|
|
31
|
+
if t == "hubspot":
|
|
32
|
+
tok = os.environ.get("HUBSPOT_VERIFY_TOKEN")
|
|
33
|
+
if not tok: sys.exit("Set HUBSPOT_VERIFY_TOKEN (private app, crm.objects.deals.read only).")
|
|
34
|
+
return hubspot.make_reader(hubspot.hs_http(tok))
|
|
35
|
+
if t == "intercom":
|
|
36
|
+
tok = os.environ.get("INTERCOM_VERIFY_TOKEN")
|
|
37
|
+
if not tok: sys.exit("Set INTERCOM_VERIFY_TOKEN (Intercom app access token, Read conversations permission only).")
|
|
38
|
+
return intercom.make_reader(intercom.ic_http(tok))
|
|
39
|
+
if t == "shopify":
|
|
40
|
+
shop, tok = os.environ.get("SHOPIFY_SHOP"), os.environ.get("SHOPIFY_VERIFY_TOKEN")
|
|
41
|
+
if not (shop and tok): sys.exit("Set SHOPIFY_SHOP (subdomain of .myshopify.com) and SHOPIFY_VERIFY_TOKEN (custom app Admin API token, read_orders scope only).")
|
|
42
|
+
return shopify.make_reader(shopify.sh_http(shop, tok))
|
|
43
|
+
sys.exit(f"no adapter for target_system {t!r} (have: stripe, zendesk, hubspot, intercom, shopify)")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def check(contract_path, claim_path, ledger_path="verify.db"):
|
|
47
|
+
c = contract.load(contract_path)
|
|
48
|
+
with open(claim_path) as f: claim = json.load(f)
|
|
49
|
+
try: run_id, v, level, reasons = verify(Ledger(ledger_path), None, claim, c, reader_for(c), actor=claim.get("actor", "unknown"))
|
|
50
|
+
except BadClaim as e: print(f"error: {e}", file=sys.stderr); return 2
|
|
51
|
+
print(json.dumps({"run_id": run_id, "verdict": v, "level": level, "reasons": reasons, "ledger": ledger_path}, indent=1))
|
|
52
|
+
return 0 if v == "pass" else 1
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def check_batch(contract_path, claims_path, ledger_path="verify.db", reader=None):
|
|
56
|
+
c = contract.load(contract_path)
|
|
57
|
+
led, reader = Ledger(ledger_path), reader or reader_for(c)
|
|
58
|
+
counts = {}
|
|
59
|
+
with open(claims_path) as f:
|
|
60
|
+
for n, line in enumerate(f, 1):
|
|
61
|
+
if not line.strip(): continue
|
|
62
|
+
claim = json.loads(line)
|
|
63
|
+
try: run_id, v, level, reasons = verify(led, None, claim, c, reader, actor=claim.get("actor", "unknown"))
|
|
64
|
+
except BadClaim as e: run_id, v, level, reasons = None, "bad_claim", "-", [str(e)]
|
|
65
|
+
counts[v] = counts.get(v, 0) + 1
|
|
66
|
+
print(json.dumps({"line": n, "run_id": run_id, "verdict": v, "level": level, "reasons": reasons}))
|
|
67
|
+
total = sum(counts.values())
|
|
68
|
+
print(f"# {counts.get('pass', 0)}/{total} pass " + " ".join(f"{k}={v}" for k, v in sorted(counts.items())), file=sys.stderr)
|
|
69
|
+
return 0 if total and counts.get("pass", 0) == total else 1
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
CLAIMS = {"stripe": {"actor": "support-agent", "charge_id": "ch_...", "refund_id": "re_...", "amount": 5000},
|
|
73
|
+
"zendesk": {"actor": "support-agent", "ticket": 123},
|
|
74
|
+
"hubspot": {"actor": "revops-agent", "deal": "123", "amount": 4200.0, "company": "456"},
|
|
75
|
+
"intercom": {"actor": "support-agent", "conversation": "123"},
|
|
76
|
+
"shopify": {"actor": "support-agent", "order": "123", "refund_id": "456", "amount": 50.0}}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def init(system, d="."):
|
|
80
|
+
mod = {"stripe": stripe, "zendesk": zendesk, "hubspot": hubspot, "intercom": intercom, "shopify": shopify}[system]
|
|
81
|
+
os.makedirs(os.path.join(d, "contracts"), exist_ok=True)
|
|
82
|
+
paths = [os.path.join(d, "contracts", mod.CONTRACT["id"] + ".json"), os.path.join(d, f"claim.{system}.json")]
|
|
83
|
+
for p, body in zip(paths, (mod.CONTRACT, CLAIMS[system])):
|
|
84
|
+
if os.path.exists(p): print(f"exists, kept: {p}"); continue
|
|
85
|
+
with open(p, "w") as f: json.dump(body, f, indent=2); f.write("\n")
|
|
86
|
+
print(f"wrote {p}")
|
|
87
|
+
print(f"next: edit {paths[1]}, then verify check {paths[0]} {paths[1]}")
|
|
88
|
+
return 0
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main(argv=None):
|
|
92
|
+
a = sys.argv[1:] if argv is None else argv
|
|
93
|
+
if a[:1] in (["--version"], ["-V"]): print(f"verify {__version__}"); return 0
|
|
94
|
+
if a[:1] == ["init"] and len(a) in (2, 3) and a[1] in CLAIMS: return init(*a[1:])
|
|
95
|
+
if a[:2] == ["ledger", "export"] and len(a) in (3, 4):
|
|
96
|
+
out = open(a[3], "w") if len(a) == 4 else sys.stdout
|
|
97
|
+
head = Ledger(a[2]).export(out)
|
|
98
|
+
if out is not sys.stdout: out.close(); print(f"chain head {head}")
|
|
99
|
+
return 0
|
|
100
|
+
if a[:2] == ["ledger", "verify"] and len(a) == 3:
|
|
101
|
+
with open(a[2]) as f: ok, n, head = verify_export(f)
|
|
102
|
+
print(f"{n} events, chain {'intact' if ok else 'BROKEN'}" + (f", head {head}" if ok else ""))
|
|
103
|
+
return 0 if ok else 1
|
|
104
|
+
if a[:1] == ["check"] and len(a) in (3, 5) and (len(a) == 3 or a[3] == "--ledger"):
|
|
105
|
+
return check(a[1], a[2], *a[4:5])
|
|
106
|
+
if a[:1] == ["check-batch"] and len(a) in (3, 5) and (len(a) == 3 or a[3] == "--ledger"):
|
|
107
|
+
return check_batch(a[1], a[2], *a[4:5])
|
|
108
|
+
if a[:1] == ["push"] and len(a) == 4 and a[2] == "--to":
|
|
109
|
+
from .push import push
|
|
110
|
+
key = os.environ.get("VERIFY_HOSTED_KEY")
|
|
111
|
+
if not key: print("error: set VERIFY_HOSTED_KEY", file=sys.stderr); return 2
|
|
112
|
+
r = push(a[1], a[3], key); print(f"pushed {r['pushed']} events, head {r['head']}"); return 0
|
|
113
|
+
if a[:1] == ["validate"] and len(a) > 1:
|
|
114
|
+
bad = 0
|
|
115
|
+
for p in a[1:]:
|
|
116
|
+
errs = contract.validate(json.load(open(p)))
|
|
117
|
+
print(f"{p}: " + ("ok" if not errs else "INVALID\n " + "\n ".join(errs)))
|
|
118
|
+
bad += bool(errs)
|
|
119
|
+
return 1 if bad else 0
|
|
120
|
+
if a[:1] == ["ledger"] and len(a) == 2:
|
|
121
|
+
led = Ledger(a[1])
|
|
122
|
+
for row in led.db.execute("SELECT run_id, verdict, level_reached, independence, reasons_json FROM verdict"):
|
|
123
|
+
print(*row, sep=" ")
|
|
124
|
+
ok = led.verify_chain()
|
|
125
|
+
print("hash chain intact:", ok)
|
|
126
|
+
return 0 if ok else 1
|
|
127
|
+
if a[:2] == ["demo", "local"]: local.demo(); return 0
|
|
128
|
+
if a[:2] == ["demo", "stripe"]: stripe.main(); return 0
|
|
129
|
+
if a[:2] == ["demo", "zendesk"]: zendesk.main(); return 0
|
|
130
|
+
if a[:1] == ["selfcheck"]: local.check(); stripe.check(); zendesk.check(); hubspot.check(); intercom.check(); shopify.check(); return 0
|
|
131
|
+
print(__doc__); return 2
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__":
|
|
135
|
+
sys.exit(main())
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Load and validate outcome contracts (JSON). Rules from the outcome contract schema v0.1."""
|
|
2
|
+
import json
|
|
3
|
+
from .core import OPS
|
|
4
|
+
|
|
5
|
+
REQUIRED = ("id", "version", "owner", "target_system", "subject_ref", "postconditions",
|
|
6
|
+
"required_level", "settle_window_s", "max_attempts", "scope")
|
|
7
|
+
LEVELS = ("L1", "L2", "L3", "L4")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate(c):
|
|
11
|
+
"""Return a list of problems; empty means valid. Rejected at registration, never at runtime."""
|
|
12
|
+
errs = [f"missing field: {f}" for f in REQUIRED if f not in c]
|
|
13
|
+
if errs: return errs
|
|
14
|
+
if not c["postconditions"]: errs.append("postconditions must not be empty")
|
|
15
|
+
if c["required_level"] not in LEVELS: errs.append(f"required_level must be one of {LEVELS}")
|
|
16
|
+
if not isinstance(c["max_attempts"], int) or c["max_attempts"] < 1: errs.append("max_attempts must be an int >= 1")
|
|
17
|
+
if not isinstance(c["settle_window_s"], (int, float)) or c["settle_window_s"] < 0: errs.append("settle_window_s must be >= 0")
|
|
18
|
+
scope = set(c["scope"])
|
|
19
|
+
for group in ("preconditions", "postconditions", "forbidden"):
|
|
20
|
+
for i, chk in enumerate(c.get(group, [])):
|
|
21
|
+
where = f"{group}[{i}]"
|
|
22
|
+
if chk.get("op") not in OPS: errs.append(f"{where}: unknown op {chk.get('op')!r}")
|
|
23
|
+
if chk.get("path") not in scope: errs.append(f"{where}: path {chk.get('path')!r} not in scope")
|
|
24
|
+
sp = c.get("subject_present")
|
|
25
|
+
if sp and sp not in scope: errs.append(f"subject_present {sp!r} not in scope")
|
|
26
|
+
return errs
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load(path):
|
|
30
|
+
with open(path) as f: c = json.load(f)
|
|
31
|
+
errs = validate(c)
|
|
32
|
+
if errs: raise ValueError(f"{path}: invalid contract:\n " + "\n ".join(errs))
|
|
33
|
+
return c
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Verify core: contracts -> independent readback -> 5 verdicts -> hash-chained ledger."""
|
|
2
|
+
import sqlite3, json, hashlib, time, uuid
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
|
|
5
|
+
def now(): return datetime.now(timezone.utc).isoformat()
|
|
6
|
+
|
|
7
|
+
class Blocked(Exception):
|
|
8
|
+
"""Readback impossible (auth, outage, rate limit). retry_after: seconds the system asked us to wait."""
|
|
9
|
+
def __init__(self, msg, retry_after=None):
|
|
10
|
+
super().__init__(msg); self.retry_after = retry_after
|
|
11
|
+
|
|
12
|
+
# ---------------- evaluation ----------------
|
|
13
|
+
OPS = {"eq": lambda a, b: a == b, "ne": lambda a, b: a != b, "gt": lambda a, b: a is not None and a > b,
|
|
14
|
+
"lt": lambda a, b: a is not None and a < b, "exists": lambda a, b: a is not None}
|
|
15
|
+
|
|
16
|
+
def resolve(v, claim):
|
|
17
|
+
if isinstance(v, str) and v.startswith("{") and v.endswith("}"): return claim[v[1:-1]]
|
|
18
|
+
return v
|
|
19
|
+
|
|
20
|
+
def evaluate(contract, fields, claim):
|
|
21
|
+
"""Returns (verdict, level, reasons). Unknown fails closed."""
|
|
22
|
+
def run(checks): return [(c, fields.get(c["path"]), OPS[c["op"]](fields.get(c["path"]), resolve(c["value"], claim)))
|
|
23
|
+
for c in checks]
|
|
24
|
+
pre = run(contract["preconditions"])
|
|
25
|
+
if not all(ok for *_, ok in pre):
|
|
26
|
+
return "inapplicable", "L0", [f"precondition {c['path']}={got!r}" for c, got, ok in pre if not ok]
|
|
27
|
+
if fields.get(contract.get("subject_present", "refund.amount")) in (None, False): # subject record absent
|
|
28
|
+
return "not_observed", "L1", [f"{contract['subject_ref'].format(**claim)} not found in system of record"]
|
|
29
|
+
bad = [f"forbidden {c['path']}={got!r}" for c, got, hit in run(contract["forbidden"]) if hit]
|
|
30
|
+
bad += [f"{c['path']} = {got!r}, expected {c['op']} {resolve(c['value'], claim)!r}"
|
|
31
|
+
for c, got, ok in run(contract["postconditions"]) if not ok]
|
|
32
|
+
return ("fail", "L2", bad) if bad else ("pass", contract["required_level"], [])
|
|
33
|
+
|
|
34
|
+
# ---------------- ledger (doc 03) ----------------
|
|
35
|
+
LEDGER_SQL = """
|
|
36
|
+
CREATE TABLE IF NOT EXISTS contract(id TEXT, version TEXT, body_json TEXT NOT NULL, body_hash TEXT NOT NULL,
|
|
37
|
+
registered_at TEXT NOT NULL, owner TEXT NOT NULL, PRIMARY KEY(id, version));
|
|
38
|
+
CREATE TABLE IF NOT EXISTS run(run_id TEXT PRIMARY KEY, contract_id TEXT NOT NULL, contract_version TEXT NOT NULL,
|
|
39
|
+
subject_ref TEXT NOT NULL, actor TEXT NOT NULL, started_at TEXT NOT NULL);
|
|
40
|
+
CREATE TABLE IF NOT EXISTS event(seq INTEGER PRIMARY KEY AUTOINCREMENT, run_id TEXT NOT NULL, type TEXT NOT NULL,
|
|
41
|
+
at TEXT NOT NULL, clock_source TEXT NOT NULL, payload_json TEXT NOT NULL, evidence_hash TEXT, prev_hash TEXT, hash TEXT NOT NULL);
|
|
42
|
+
CREATE TABLE IF NOT EXISTS verdict(run_id TEXT PRIMARY KEY, verdict TEXT NOT NULL
|
|
43
|
+
CHECK (verdict IN ('pass','fail','inapplicable','not_observed','blocked')),
|
|
44
|
+
level_reached TEXT NOT NULL, independence TEXT NOT NULL, reasons_json TEXT NOT NULL, final_event_seq INTEGER NOT NULL);
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
class Ledger:
|
|
48
|
+
def __init__(self, path):
|
|
49
|
+
self.db = sqlite3.connect(path)
|
|
50
|
+
self.db.execute("PRAGMA journal_mode=WAL"); self.db.execute("PRAGMA synchronous=FULL")
|
|
51
|
+
self.db.executescript(LEDGER_SQL)
|
|
52
|
+
|
|
53
|
+
def event(self, run_id, type_, payload, evidence=None):
|
|
54
|
+
prev = (self.db.execute("SELECT hash FROM event ORDER BY seq DESC LIMIT 1").fetchone() or [""])[0]
|
|
55
|
+
row = dict(run_id=run_id, type=type_, at=now(), clock_source="system", payload_json=json.dumps(payload, sort_keys=True),
|
|
56
|
+
evidence_hash=hashlib.sha256(evidence.encode()).hexdigest() if evidence else None)
|
|
57
|
+
h = hashlib.sha256((prev + json.dumps(row, sort_keys=True)).encode()).hexdigest()
|
|
58
|
+
cur = self.db.execute("INSERT INTO event(run_id,type,at,clock_source,payload_json,evidence_hash,prev_hash,hash) VALUES (?,?,?,?,?,?,?,?)",
|
|
59
|
+
(*row.values(), prev, h))
|
|
60
|
+
self.db.commit(); return cur.lastrowid
|
|
61
|
+
|
|
62
|
+
EVENT_KEYS = ["run_id", "type", "at", "clock_source", "payload_json", "evidence_hash"]
|
|
63
|
+
|
|
64
|
+
def export(self, out):
|
|
65
|
+
"""Write every event as JSONL, then a final line with the chain head. Returns the head hash."""
|
|
66
|
+
head = ""
|
|
67
|
+
for seq, *row, p, h in self.db.execute("SELECT seq,run_id,type,at,clock_source,payload_json,evidence_hash,prev_hash,hash FROM event ORDER BY seq"):
|
|
68
|
+
out.write(json.dumps(dict(seq=seq, **dict(zip(self.EVENT_KEYS, row)), prev_hash=p, hash=h), sort_keys=True) + "\n")
|
|
69
|
+
head = h
|
|
70
|
+
out.write(json.dumps({"chain_head": head}) + "\n")
|
|
71
|
+
return head
|
|
72
|
+
|
|
73
|
+
def verify_chain(self):
|
|
74
|
+
prev = ""
|
|
75
|
+
for *row, p, h in self.db.execute("SELECT run_id,type,at,clock_source,payload_json,evidence_hash,prev_hash,hash FROM event ORDER BY seq"):
|
|
76
|
+
keys = ["run_id", "type", "at", "clock_source", "payload_json", "evidence_hash"]
|
|
77
|
+
if p != prev or h != hashlib.sha256((prev + json.dumps(dict(zip(keys, row)), sort_keys=True)).encode()).hexdigest():
|
|
78
|
+
return False
|
|
79
|
+
prev = h
|
|
80
|
+
return True
|
|
81
|
+
|
|
82
|
+
def verify_export(lines):
|
|
83
|
+
"""Check an exported JSONL ledger offline. Returns (ok, events, head)."""
|
|
84
|
+
prev, n, head = "", 0, None
|
|
85
|
+
for line in lines:
|
|
86
|
+
e = json.loads(line)
|
|
87
|
+
if "chain_head" in e: head = e["chain_head"]; break
|
|
88
|
+
row = {k: e[k] for k in Ledger.EVENT_KEYS}
|
|
89
|
+
if e["prev_hash"] != prev or e["hash"] != hashlib.sha256((prev + json.dumps(row, sort_keys=True)).encode()).hexdigest():
|
|
90
|
+
return False, n, None
|
|
91
|
+
prev, n = e["hash"], n + 1
|
|
92
|
+
return head == prev, n, head
|
|
93
|
+
|
|
94
|
+
# ---------------- Verify ----------------
|
|
95
|
+
def claim_fields(contract):
|
|
96
|
+
"""Claim keys the contract needs: {placeholders} in subject_ref and in check values."""
|
|
97
|
+
import string
|
|
98
|
+
need = {f for _, f, _, _ in string.Formatter().parse(contract["subject_ref"]) if f}
|
|
99
|
+
for g in ("preconditions", "postconditions", "forbidden"):
|
|
100
|
+
for c in contract.get(g, []):
|
|
101
|
+
v = c.get("value")
|
|
102
|
+
if isinstance(v, str) and v.startswith("{") and v.endswith("}"): need.add(v[1:-1])
|
|
103
|
+
return need
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class BadClaim(ValueError): pass
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def verify(ledger, sor_path, claim, contract, reader, actor="support-agent", independence="full"):
|
|
110
|
+
missing = sorted(claim_fields(contract) - set(claim))
|
|
111
|
+
if missing: raise BadClaim(f"claim is missing {', '.join(missing)} (needed by contract {contract['id']})") # before any ledger write
|
|
112
|
+
body = json.dumps(contract, sort_keys=True)
|
|
113
|
+
ledger.db.execute("INSERT OR IGNORE INTO contract VALUES (?,?,?,?,?,?)", (contract["id"], contract["version"], body,
|
|
114
|
+
hashlib.sha256(body.encode()).hexdigest(), now(), contract["owner"]))
|
|
115
|
+
run_id = uuid.uuid4().hex[:8]
|
|
116
|
+
ledger.db.execute("INSERT INTO run VALUES (?,?,?,?,?,?)", (run_id, contract["id"], contract["version"],
|
|
117
|
+
contract["subject_ref"].format(**claim), actor, now()))
|
|
118
|
+
ledger.event(run_id, "action_claimed", claim) # claim recorded, never used as evidence
|
|
119
|
+
verdict, level, reasons = "not_observed", "L1", []
|
|
120
|
+
wait = contract["settle_window_s"]
|
|
121
|
+
for attempt in range(1, contract["max_attempts"] + 1):
|
|
122
|
+
time.sleep(wait)
|
|
123
|
+
wait = contract["settle_window_s"]
|
|
124
|
+
ledger.event(run_id, "readback_started", {"attempt": attempt})
|
|
125
|
+
try:
|
|
126
|
+
fields, raw = reader(sor_path, claim)
|
|
127
|
+
except Blocked as e:
|
|
128
|
+
ledger.event(run_id, "readback_blocked", {"error": str(e), "retry_after": e.retry_after})
|
|
129
|
+
wait = min(e.retry_after if e.retry_after is not None else max(1, wait) * 2 ** attempt, 60) # backoff, capped
|
|
130
|
+
verdict, level, reasons = "blocked", "L0", [str(e)]
|
|
131
|
+
continue
|
|
132
|
+
ledger.event(run_id, "readback_observed", fields, evidence=raw)
|
|
133
|
+
verdict, level, reasons = evaluate(contract, fields, claim)
|
|
134
|
+
if verdict != "not_observed": break # retry only missing evidence
|
|
135
|
+
seq = ledger.event(run_id, "verdict_final", {"verdict": verdict, "reasons": reasons})
|
|
136
|
+
ledger.db.execute("INSERT INTO verdict VALUES (?,?,?,?,?,?)", (run_id, verdict, level, independence, json.dumps(reasons), seq))
|
|
137
|
+
ledger.db.commit()
|
|
138
|
+
return run_id, verdict, level, reasons
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def check(contract, claim, reader, ledger="verify.db", actor=None, independence="full"):
|
|
143
|
+
"""Python API. contract: dict or path; claim: dict; reader(sor, claim) -> (fields, raw_json).
|
|
144
|
+
Returns dict with verdict, level, reasons, run_id. Only verdict == "pass" means success."""
|
|
145
|
+
from . import contract as _c
|
|
146
|
+
c = _c.load(contract) if isinstance(contract, str) else contract
|
|
147
|
+
if not isinstance(contract, str):
|
|
148
|
+
errs = _c.validate(c)
|
|
149
|
+
if errs: raise ValueError("invalid contract: " + "; ".join(errs))
|
|
150
|
+
led = ledger if isinstance(ledger, Ledger) else Ledger(ledger)
|
|
151
|
+
run_id, v, level, reasons = verify(led, None, claim, c, reader, actor=actor or claim.get("actor", "unknown"), independence=independence)
|
|
152
|
+
return {"run_id": run_id, "verdict": v, "level": level, "reasons": reasons, "ok": v == "pass"}
|