trustmodel 3.2.0__tar.gz → 3.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trustmodel-3.2.0 → trustmodel-3.3.0}/PKG-INFO +94 -1
- {trustmodel-3.2.0 → trustmodel-3.3.0}/README.md +93 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/pyproject.toml +4 -1
- trustmodel-3.3.0/src/trustmodel/cli.py +141 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/client.py +100 -16
- trustmodel-3.3.0/src/trustmodel/endpoints/agentcert.py +150 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/agentic.py +23 -1
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/evaluations.py +23 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/guardrails.py +88 -2
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/__init__.py +6 -0
- trustmodel-3.3.0/src/trustmodel/models/agentcert.py +118 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/agentic.py +32 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/evaluation.py +9 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/telemetry/auto_init.py +74 -10
- {trustmodel-3.2.0 → trustmodel-3.3.0}/.gitignore +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/LICENSE +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/__init__.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/compat.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/__init__.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/_runtime_transport.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/batch_jobs.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/config.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/credits.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/frameworks.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/galileo.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/mcp_scanner.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/endpoints/models.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/exceptions.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/batch_job.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/credits.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/frameworks.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/galileo.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/guardrails.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/mcp_scanner.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/models.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/models/oauth.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/oauth/__init__.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/oauth/auth.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/oauth/scopes.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/telemetry/__init__.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/telemetry/_constants.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/telemetry/_instrumentors.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/utils/__init__.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/utils/validation.py +0 -0
- {trustmodel-3.2.0 → trustmodel-3.3.0}/src/trustmodel/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: trustmodel
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.3.0
|
|
4
4
|
Summary: Official Python SDK for TrustModel AI evaluation platform
|
|
5
5
|
Project-URL: Homepage, https://trustmodel.ai
|
|
6
6
|
Author-email: TrustModel <info@predixtions.com>
|
|
@@ -118,6 +118,7 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
|
|
|
118
118
|
- [Agentic Trace Evaluation](#agentic-trace-evaluation)
|
|
119
119
|
- [Galileo Integration](#galileo-integration)
|
|
120
120
|
- [MCP Scanner](#mcp-scanner)
|
|
121
|
+
- [AgentCert](#agentcert)
|
|
121
122
|
- [Requirements](#requirements)
|
|
122
123
|
- [Support](#support)
|
|
123
124
|
- [License](#license)
|
|
@@ -131,6 +132,7 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
|
|
|
131
132
|
- 📊 **Comprehensive**: Support for all evaluation types and configurations
|
|
132
133
|
- 🌍 **Framework Agnostic**: Works with any Python framework or standalone scripts
|
|
133
134
|
- 🛰️ **Zero-Config Auto-Capture**: Add two lines (`auto_init`) and every OpenAI / Anthropic / LangChain / CrewAI / etc. call is automatically traced and evaluated
|
|
135
|
+
- 🔖 **AgentCert**: Issue and verify cryptographic certificates for AI agents — `client.agentcert.issue()` / `.verify()` and the `trustmodel certify` CLI
|
|
134
136
|
|
|
135
137
|
## Installation
|
|
136
138
|
|
|
@@ -1375,6 +1377,8 @@ JSONL files are also supported (one JSON object per line).
|
|
|
1375
1377
|
| `expected_outcome` | No | What should have happened |
|
|
1376
1378
|
| `actual_outcome` | No | What actually happened |
|
|
1377
1379
|
| `goal_achieved` | No | Whether the agent achieved its goal |
|
|
1380
|
+
| `frameworks` | No | List of compliance framework slugs to evaluate against (e.g., `["owasp-asi", "nist-ai-rmf"]`) |
|
|
1381
|
+
| `control_ids` | No | List of control IDs to scope the compliance evaluation. Defaults to all controls of the selected frameworks |
|
|
1378
1382
|
|
|
1379
1383
|
### File Validation
|
|
1380
1384
|
|
|
@@ -1416,6 +1420,33 @@ for score in detail.scores:
|
|
|
1416
1420
|
}
|
|
1417
1421
|
```
|
|
1418
1422
|
|
|
1423
|
+
### Compliance Frameworks
|
|
1424
|
+
|
|
1425
|
+
Pass one or more compliance framework slugs to `frameworks` to evaluate the trace against a regulatory or security standard. Optionally scope the controls with `control_ids` (defaults to all controls of the selected frameworks).
|
|
1426
|
+
|
|
1427
|
+
```python
|
|
1428
|
+
result = client.agentic.evaluate(
|
|
1429
|
+
file_path="traces/agent_run.json",
|
|
1430
|
+
goal="Resolve customer billing inquiry",
|
|
1431
|
+
name="Support Bot Evaluation",
|
|
1432
|
+
agent_framework="langchain",
|
|
1433
|
+
frameworks=["owasp-asi", "nist-ai-rmf"],
|
|
1434
|
+
)
|
|
1435
|
+
```
|
|
1436
|
+
|
|
1437
|
+
The detail response surfaces a per-framework compliance roll-up once evaluation completes:
|
|
1438
|
+
|
|
1439
|
+
```python
|
|
1440
|
+
detail = client.agentic.get(result.evaluation_run_id)
|
|
1441
|
+
|
|
1442
|
+
if detail.compliance_report_url:
|
|
1443
|
+
print(f"Compliance report: {detail.compliance_report_url}")
|
|
1444
|
+
|
|
1445
|
+
for fw in detail.compliance_frameworks:
|
|
1446
|
+
print(f" {fw.slug}: {fw.overall_status} ({fw.compliance_percentage}%)")
|
|
1447
|
+
print(f" passed={fw.passed_controls} failed={fw.failed_controls}")
|
|
1448
|
+
```
|
|
1449
|
+
|
|
1419
1450
|
### Listing Evaluations
|
|
1420
1451
|
|
|
1421
1452
|
```python
|
|
@@ -1712,6 +1743,68 @@ When `edge_url` is not configured and Aurora is the only path, a missing `agent_
|
|
|
1712
1743
|
|
|
1713
1744
|
Edge currently accepts unauthenticated requests in-cluster. The SDK uses a `RuntimeTransport` abstraction (see `trustmodel.endpoints._runtime_transport`) so future Edge auth — SVID, JWT, mTLS, service tokens — can be added without changing the public `client.guardrails.decide()` signature.
|
|
1714
1745
|
|
|
1746
|
+
## AgentCert
|
|
1747
|
+
|
|
1748
|
+
AgentCert is TrustModel's Certificate Authority for AI agents — the "padlock for the agentic web". A certificate is minted off a completed evaluation run and binds the agent's identity, behavior, and liveness into a cryptographically verifiable, continuously-revalidated credential. Anyone can verify an agent via the public verify endpoint; a cert whose live TrustScore drifts (or which is revoked) stops verifying even though its signature is intact.
|
|
1749
|
+
|
|
1750
|
+
Issuing a cert is org-authenticated (any API key); verifying is public.
|
|
1751
|
+
|
|
1752
|
+
### Issue and verify (SDK)
|
|
1753
|
+
|
|
1754
|
+
```python
|
|
1755
|
+
import trustmodel
|
|
1756
|
+
|
|
1757
|
+
client = trustmodel.TrustModelClient(api_key="tm-...")
|
|
1758
|
+
|
|
1759
|
+
# Mint a cert for an agent off a completed evaluation run.
|
|
1760
|
+
cert = client.agentcert.issue(
|
|
1761
|
+
ans_name="helpbot.cisco.com", # Agent Naming Service binding
|
|
1762
|
+
evaluation_run_id=12345, # source eval run (must belong to your org)
|
|
1763
|
+
subject_agent_id="helpbot", # optional
|
|
1764
|
+
validity_days=90, # optional (1–825; default 90)
|
|
1765
|
+
)
|
|
1766
|
+
print(cert.id, cert.serial, cert.status) # e.g. "...", "TM-2026-...", "active"
|
|
1767
|
+
print(cert.validation_level) # "DV" (DV / OV / EV)
|
|
1768
|
+
print(cert.trust_score_snapshot) # 87.0 (0–100, same scale as overall_score)
|
|
1769
|
+
print(cert.dimension_scores) # {"safety": 92.0, "fairness": 85.0, ...}
|
|
1770
|
+
# cert.vc_jsonld holds the W3C Verifiable Credential; cert.x509_pem the X.509 PEM.
|
|
1771
|
+
|
|
1772
|
+
# Verify an agent (public — no auth required). Accepts an ANS name or agent id.
|
|
1773
|
+
result = client.agentcert.verify("helpbot.cisco.com")
|
|
1774
|
+
if result.verified:
|
|
1775
|
+
print("trusted — live score:", result.trust_score)
|
|
1776
|
+
else:
|
|
1777
|
+
print("not trusted:", result.status, result.revocation_reason)
|
|
1778
|
+
```
|
|
1779
|
+
|
|
1780
|
+
A **revoked** cert returns a normal result with `verified=False, status="revoked"`. A **missing** cert returns `found=False` (rather than raising), so verification is always a value check.
|
|
1781
|
+
|
|
1782
|
+
### Certify (issue + verify in one call)
|
|
1783
|
+
|
|
1784
|
+
```python
|
|
1785
|
+
cert, verification = client.agentcert.certify(
|
|
1786
|
+
ans_name="helpbot.cisco.com",
|
|
1787
|
+
evaluation_run_id=12345,
|
|
1788
|
+
)
|
|
1789
|
+
print(cert.serial, "->", "VALID" if verification.verified else "INVALID")
|
|
1790
|
+
```
|
|
1791
|
+
|
|
1792
|
+
### CLI
|
|
1793
|
+
|
|
1794
|
+
Installing the SDK also installs the `trustmodel` command:
|
|
1795
|
+
|
|
1796
|
+
```bash
|
|
1797
|
+
export TRUSTMODEL_API_KEY=tm-...
|
|
1798
|
+
|
|
1799
|
+
# Mint a cert for an agent off an evaluation run, then verify it:
|
|
1800
|
+
trustmodel certify helpbot.cisco.com --evaluation-run-id 12345
|
|
1801
|
+
|
|
1802
|
+
# Just verify an existing agent's cert (public):
|
|
1803
|
+
trustmodel certify helpbot.cisco.com --verify-only
|
|
1804
|
+
```
|
|
1805
|
+
|
|
1806
|
+
`certify` prints the certificate + verification as JSON and exits non-zero if the agent does not verify, so it drops cleanly into CI gates.
|
|
1807
|
+
|
|
1715
1808
|
## Requirements
|
|
1716
1809
|
|
|
1717
1810
|
- Python 3.9 or higher
|
|
@@ -50,6 +50,7 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
|
|
|
50
50
|
- [Agentic Trace Evaluation](#agentic-trace-evaluation)
|
|
51
51
|
- [Galileo Integration](#galileo-integration)
|
|
52
52
|
- [MCP Scanner](#mcp-scanner)
|
|
53
|
+
- [AgentCert](#agentcert)
|
|
53
54
|
- [Requirements](#requirements)
|
|
54
55
|
- [Support](#support)
|
|
55
56
|
- [License](#license)
|
|
@@ -63,6 +64,7 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
|
|
|
63
64
|
- 📊 **Comprehensive**: Support for all evaluation types and configurations
|
|
64
65
|
- 🌍 **Framework Agnostic**: Works with any Python framework or standalone scripts
|
|
65
66
|
- 🛰️ **Zero-Config Auto-Capture**: Add two lines (`auto_init`) and every OpenAI / Anthropic / LangChain / CrewAI / etc. call is automatically traced and evaluated
|
|
67
|
+
- 🔖 **AgentCert**: Issue and verify cryptographic certificates for AI agents — `client.agentcert.issue()` / `.verify()` and the `trustmodel certify` CLI
|
|
66
68
|
|
|
67
69
|
## Installation
|
|
68
70
|
|
|
@@ -1307,6 +1309,8 @@ JSONL files are also supported (one JSON object per line).
|
|
|
1307
1309
|
| `expected_outcome` | No | What should have happened |
|
|
1308
1310
|
| `actual_outcome` | No | What actually happened |
|
|
1309
1311
|
| `goal_achieved` | No | Whether the agent achieved its goal |
|
|
1312
|
+
| `frameworks` | No | List of compliance framework slugs to evaluate against (e.g., `["owasp-asi", "nist-ai-rmf"]`) |
|
|
1313
|
+
| `control_ids` | No | List of control IDs to scope the compliance evaluation. Defaults to all controls of the selected frameworks |
|
|
1310
1314
|
|
|
1311
1315
|
### File Validation
|
|
1312
1316
|
|
|
@@ -1348,6 +1352,33 @@ for score in detail.scores:
|
|
|
1348
1352
|
}
|
|
1349
1353
|
```
|
|
1350
1354
|
|
|
1355
|
+
### Compliance Frameworks
|
|
1356
|
+
|
|
1357
|
+
Pass one or more compliance framework slugs to `frameworks` to evaluate the trace against a regulatory or security standard. Optionally scope the controls with `control_ids` (defaults to all controls of the selected frameworks).
|
|
1358
|
+
|
|
1359
|
+
```python
|
|
1360
|
+
result = client.agentic.evaluate(
|
|
1361
|
+
file_path="traces/agent_run.json",
|
|
1362
|
+
goal="Resolve customer billing inquiry",
|
|
1363
|
+
name="Support Bot Evaluation",
|
|
1364
|
+
agent_framework="langchain",
|
|
1365
|
+
frameworks=["owasp-asi", "nist-ai-rmf"],
|
|
1366
|
+
)
|
|
1367
|
+
```
|
|
1368
|
+
|
|
1369
|
+
The detail response surfaces a per-framework compliance roll-up once evaluation completes:
|
|
1370
|
+
|
|
1371
|
+
```python
|
|
1372
|
+
detail = client.agentic.get(result.evaluation_run_id)
|
|
1373
|
+
|
|
1374
|
+
if detail.compliance_report_url:
|
|
1375
|
+
print(f"Compliance report: {detail.compliance_report_url}")
|
|
1376
|
+
|
|
1377
|
+
for fw in detail.compliance_frameworks:
|
|
1378
|
+
print(f" {fw.slug}: {fw.overall_status} ({fw.compliance_percentage}%)")
|
|
1379
|
+
print(f" passed={fw.passed_controls} failed={fw.failed_controls}")
|
|
1380
|
+
```
|
|
1381
|
+
|
|
1351
1382
|
### Listing Evaluations
|
|
1352
1383
|
|
|
1353
1384
|
```python
|
|
@@ -1644,6 +1675,68 @@ When `edge_url` is not configured and Aurora is the only path, a missing `agent_
|
|
|
1644
1675
|
|
|
1645
1676
|
Edge currently accepts unauthenticated requests in-cluster. The SDK uses a `RuntimeTransport` abstraction (see `trustmodel.endpoints._runtime_transport`) so future Edge auth — SVID, JWT, mTLS, service tokens — can be added without changing the public `client.guardrails.decide()` signature.
|
|
1646
1677
|
|
|
1678
|
+
## AgentCert
|
|
1679
|
+
|
|
1680
|
+
AgentCert is TrustModel's Certificate Authority for AI agents — the "padlock for the agentic web". A certificate is minted off a completed evaluation run and binds the agent's identity, behavior, and liveness into a cryptographically verifiable, continuously-revalidated credential. Anyone can verify an agent via the public verify endpoint; a cert whose live TrustScore drifts (or which is revoked) stops verifying even though its signature is intact.
|
|
1681
|
+
|
|
1682
|
+
Issuing a cert is org-authenticated (any API key); verifying is public.
|
|
1683
|
+
|
|
1684
|
+
### Issue and verify (SDK)
|
|
1685
|
+
|
|
1686
|
+
```python
|
|
1687
|
+
import trustmodel
|
|
1688
|
+
|
|
1689
|
+
client = trustmodel.TrustModelClient(api_key="tm-...")
|
|
1690
|
+
|
|
1691
|
+
# Mint a cert for an agent off a completed evaluation run.
|
|
1692
|
+
cert = client.agentcert.issue(
|
|
1693
|
+
ans_name="helpbot.cisco.com", # Agent Naming Service binding
|
|
1694
|
+
evaluation_run_id=12345, # source eval run (must belong to your org)
|
|
1695
|
+
subject_agent_id="helpbot", # optional
|
|
1696
|
+
validity_days=90, # optional (1–825; default 90)
|
|
1697
|
+
)
|
|
1698
|
+
print(cert.id, cert.serial, cert.status) # e.g. "...", "TM-2026-...", "active"
|
|
1699
|
+
print(cert.validation_level) # "DV" (DV / OV / EV)
|
|
1700
|
+
print(cert.trust_score_snapshot) # 87.0 (0–100, same scale as overall_score)
|
|
1701
|
+
print(cert.dimension_scores) # {"safety": 92.0, "fairness": 85.0, ...}
|
|
1702
|
+
# cert.vc_jsonld holds the W3C Verifiable Credential; cert.x509_pem the X.509 PEM.
|
|
1703
|
+
|
|
1704
|
+
# Verify an agent (public — no auth required). Accepts an ANS name or agent id.
|
|
1705
|
+
result = client.agentcert.verify("helpbot.cisco.com")
|
|
1706
|
+
if result.verified:
|
|
1707
|
+
print("trusted — live score:", result.trust_score)
|
|
1708
|
+
else:
|
|
1709
|
+
print("not trusted:", result.status, result.revocation_reason)
|
|
1710
|
+
```
|
|
1711
|
+
|
|
1712
|
+
A **revoked** cert returns a normal result with `verified=False, status="revoked"`. A **missing** cert returns `found=False` (rather than raising), so verification is always a value check.
|
|
1713
|
+
|
|
1714
|
+
### Certify (issue + verify in one call)
|
|
1715
|
+
|
|
1716
|
+
```python
|
|
1717
|
+
cert, verification = client.agentcert.certify(
|
|
1718
|
+
ans_name="helpbot.cisco.com",
|
|
1719
|
+
evaluation_run_id=12345,
|
|
1720
|
+
)
|
|
1721
|
+
print(cert.serial, "->", "VALID" if verification.verified else "INVALID")
|
|
1722
|
+
```
|
|
1723
|
+
|
|
1724
|
+
### CLI
|
|
1725
|
+
|
|
1726
|
+
Installing the SDK also installs the `trustmodel` command:
|
|
1727
|
+
|
|
1728
|
+
```bash
|
|
1729
|
+
export TRUSTMODEL_API_KEY=tm-...
|
|
1730
|
+
|
|
1731
|
+
# Mint a cert for an agent off an evaluation run, then verify it:
|
|
1732
|
+
trustmodel certify helpbot.cisco.com --evaluation-run-id 12345
|
|
1733
|
+
|
|
1734
|
+
# Just verify an existing agent's cert (public):
|
|
1735
|
+
trustmodel certify helpbot.cisco.com --verify-only
|
|
1736
|
+
```
|
|
1737
|
+
|
|
1738
|
+
`certify` prints the certificate + verification as JSON and exits non-zero if the agent does not verify, so it drops cleanly into CI gates.
|
|
1739
|
+
|
|
1647
1740
|
## Requirements
|
|
1648
1741
|
|
|
1649
1742
|
- Python 3.9 or higher
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "trustmodel"
|
|
7
|
-
version = "3.
|
|
7
|
+
version = "3.3.0"
|
|
8
8
|
authors = [
|
|
9
9
|
{name = "TrustModel", email = "info@predixtions.com"},
|
|
10
10
|
]
|
|
@@ -39,6 +39,9 @@ dependencies = [
|
|
|
39
39
|
"importlib-metadata>=4.0.0; python_version<'3.8'",
|
|
40
40
|
]
|
|
41
41
|
|
|
42
|
+
[project.scripts]
|
|
43
|
+
trustmodel = "trustmodel.cli:main"
|
|
44
|
+
|
|
42
45
|
[project.optional-dependencies]
|
|
43
46
|
# Auto-capture AI agent traces and stream them to TrustModel.
|
|
44
47
|
# Install with: pip install "trustmodel[telemetry]"
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""``trustmodel`` command-line interface.
|
|
2
|
+
|
|
3
|
+
Entry point ``trustmodel`` (wired via ``[project.scripts]`` in
|
|
4
|
+
pyproject.toml). argparse only — no extra dependencies.
|
|
5
|
+
|
|
6
|
+
Commands:
|
|
7
|
+
|
|
8
|
+
* ``trustmodel certify <agent>`` — mint a verifiable AgentCert for an agent
|
|
9
|
+
off a completed evaluation run, then verify it (signature + revocation +
|
|
10
|
+
live score). Use ``--verify-only`` to skip issuance and just verify.
|
|
11
|
+
|
|
12
|
+
Auth is an API key (``--api-key`` or ``TRUSTMODEL_API_KEY``) — the issue
|
|
13
|
+
endpoint accepts any org-scoped key; verify is public.
|
|
14
|
+
|
|
15
|
+
Exit codes: ``0`` issued/verified ok · ``1`` not verified / API error ·
|
|
16
|
+
``2`` missing key or usage error.
|
|
17
|
+
|
|
18
|
+
TRUS-1228 (EPIC TRUS-1219)
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import sys
|
|
27
|
+
from typing import List, Optional
|
|
28
|
+
|
|
29
|
+
from .client import TrustModelClient
|
|
30
|
+
from .exceptions import TrustModelError
|
|
31
|
+
|
|
32
|
+
API_KEY_ENV_VAR = "TRUSTMODEL_API_KEY"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
36
|
+
parser = argparse.ArgumentParser(
|
|
37
|
+
prog="trustmodel",
|
|
38
|
+
description="TrustModel CLI — AI assurance from the command line.",
|
|
39
|
+
)
|
|
40
|
+
sub = parser.add_subparsers(dest="command")
|
|
41
|
+
|
|
42
|
+
certify = sub.add_parser(
|
|
43
|
+
"certify",
|
|
44
|
+
help="Issue and verify an AgentCert for an AI agent (TRUS-1228).",
|
|
45
|
+
description=(
|
|
46
|
+
"Mint a verifiable certificate for an agent off a completed "
|
|
47
|
+
"evaluation run, then verify it (signature + revocation + live "
|
|
48
|
+
"score)."
|
|
49
|
+
),
|
|
50
|
+
)
|
|
51
|
+
certify.add_argument("agent", help="Agent to certify — its ANS name (e.g. helpbot.cisco.com).")
|
|
52
|
+
certify.add_argument(
|
|
53
|
+
"--evaluation-run-id",
|
|
54
|
+
type=int,
|
|
55
|
+
help="Evaluation run id to mint the cert off (required unless --verify-only).",
|
|
56
|
+
)
|
|
57
|
+
certify.add_argument("--subject-agent-id", help="Optional agent identifier to bind.")
|
|
58
|
+
certify.add_argument("--auditor-slug", help="Optional auditor slug to link the cert to.")
|
|
59
|
+
certify.add_argument(
|
|
60
|
+
"--validity-days", type=int, help="Cert lifetime in days (1–825; backend default 90)."
|
|
61
|
+
)
|
|
62
|
+
certify.add_argument(
|
|
63
|
+
"--verify-only", action="store_true", help="Skip issuance; only verify the agent's cert."
|
|
64
|
+
)
|
|
65
|
+
certify.add_argument("--api-key", help=f"TrustModel API key. Defaults to ${API_KEY_ENV_VAR}.")
|
|
66
|
+
certify.add_argument(
|
|
67
|
+
"--environment",
|
|
68
|
+
default="production",
|
|
69
|
+
choices=["local", "qa", "production"],
|
|
70
|
+
help="API environment (default: production). Ignored if --base-url is set.",
|
|
71
|
+
)
|
|
72
|
+
certify.add_argument("--base-url", help="Explicit API root; overrides --environment.")
|
|
73
|
+
certify.set_defaults(func=_cmd_certify)
|
|
74
|
+
|
|
75
|
+
return parser
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _make_client(args: argparse.Namespace) -> TrustModelClient:
|
|
79
|
+
api_key = args.api_key or os.environ.get(API_KEY_ENV_VAR)
|
|
80
|
+
if not api_key:
|
|
81
|
+
raise _UsageError(f"no API key. Pass --api-key or set ${API_KEY_ENV_VAR}.")
|
|
82
|
+
if args.base_url:
|
|
83
|
+
return TrustModelClient(api_key=api_key, base_url=args.base_url)
|
|
84
|
+
return TrustModelClient(api_key=api_key, environment=args.environment)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class _UsageError(Exception):
|
|
88
|
+
"""Missing creds / bad invocation — maps to exit code 2."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _cmd_certify(args: argparse.Namespace) -> int:
|
|
92
|
+
client = _make_client(args)
|
|
93
|
+
try:
|
|
94
|
+
if args.verify_only:
|
|
95
|
+
verification = client.agentcert.verify(args.agent)
|
|
96
|
+
print(json.dumps(verification.model_dump(), indent=2, default=str))
|
|
97
|
+
return 0 if verification.verified else 1
|
|
98
|
+
|
|
99
|
+
if args.evaluation_run_id is None:
|
|
100
|
+
raise _UsageError("provide --evaluation-run-id (or use --verify-only).")
|
|
101
|
+
|
|
102
|
+
cert, verification = client.agentcert.certify(
|
|
103
|
+
ans_name=args.agent,
|
|
104
|
+
evaluation_run_id=args.evaluation_run_id,
|
|
105
|
+
subject_agent_id=args.subject_agent_id,
|
|
106
|
+
auditor_slug=args.auditor_slug,
|
|
107
|
+
validity_days=args.validity_days,
|
|
108
|
+
)
|
|
109
|
+
print(
|
|
110
|
+
json.dumps(
|
|
111
|
+
{
|
|
112
|
+
"certificate": cert.model_dump(),
|
|
113
|
+
"verification": verification.model_dump(),
|
|
114
|
+
},
|
|
115
|
+
indent=2,
|
|
116
|
+
default=str,
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
return 0 if verification.verified else 1
|
|
120
|
+
finally:
|
|
121
|
+
client.close()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
125
|
+
parser = _build_parser()
|
|
126
|
+
args = parser.parse_args(argv)
|
|
127
|
+
if not args.command:
|
|
128
|
+
parser.print_help()
|
|
129
|
+
return 1
|
|
130
|
+
try:
|
|
131
|
+
return int(args.func(args))
|
|
132
|
+
except _UsageError as exc:
|
|
133
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
134
|
+
return 2
|
|
135
|
+
except TrustModelError as exc:
|
|
136
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
137
|
+
return 1
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
if __name__ == "__main__": # pragma: no cover
|
|
141
|
+
raise SystemExit(main())
|
|
@@ -7,13 +7,14 @@ from __future__ import annotations
|
|
|
7
7
|
import json
|
|
8
8
|
import logging
|
|
9
9
|
import os
|
|
10
|
-
from typing import Any, Dict, Optional
|
|
10
|
+
from typing import Any, Dict, List, Optional, Union, cast
|
|
11
11
|
from urllib.parse import urljoin
|
|
12
12
|
|
|
13
13
|
import requests
|
|
14
14
|
from requests.adapters import HTTPAdapter
|
|
15
15
|
from urllib3.util.retry import Retry
|
|
16
16
|
|
|
17
|
+
from .endpoints.agentcert import AgentCertEndpoint
|
|
17
18
|
from .endpoints.agentic import AgenticEndpoint
|
|
18
19
|
from .endpoints.batch_jobs import BatchJobsEndpoint
|
|
19
20
|
from .endpoints.config import ConfigEndpoint
|
|
@@ -135,6 +136,7 @@ class TrustModelClient:
|
|
|
135
136
|
auth: Optional[OAuthClientCredentials] = None,
|
|
136
137
|
edge_url: Optional[str] = None,
|
|
137
138
|
fail_mode: str = "closed",
|
|
139
|
+
organization_id: Optional[str] = None,
|
|
138
140
|
) -> None:
|
|
139
141
|
# api_key and auth are mutually exclusive. Either / both / neither
|
|
140
142
|
# may be combined with edge_url (the at-least-one check happens
|
|
@@ -219,6 +221,16 @@ class TrustModelClient:
|
|
|
219
221
|
if self._api_key is not None:
|
|
220
222
|
self._session.headers["Authorization"] = f"Bearer {self._api_key}"
|
|
221
223
|
|
|
224
|
+
# Aurora requires ``X-Organization-ID`` on every request to resolve
|
|
225
|
+
# the caller's org context (post-TRUS-1395 tenant scoping). Prefer
|
|
226
|
+
# the explicit constructor arg; fall back to ``TRUSTMODEL_ORGANIZATION_ID``
|
|
227
|
+
# env var so callers can set it once at process start without
|
|
228
|
+
# threading it through every construction site.
|
|
229
|
+
org_id = organization_id or os.environ.get("TRUSTMODEL_ORGANIZATION_ID")
|
|
230
|
+
self._organization_id: Optional[str] = org_id
|
|
231
|
+
if org_id:
|
|
232
|
+
self._session.headers["X-Organization-ID"] = org_id
|
|
233
|
+
|
|
222
234
|
# Initialize endpoint handlers
|
|
223
235
|
self.models = ModelsEndpoint(self)
|
|
224
236
|
self.evaluations = EvaluationsEndpoint(self)
|
|
@@ -231,6 +243,9 @@ class TrustModelClient:
|
|
|
231
243
|
# Runtime guardrails (P4 / v2.3.0). Edge primary with Aurora
|
|
232
244
|
# fallback — see ``client.guardrails.decide()``.
|
|
233
245
|
self.guardrails = GuardrailsEndpoint(self)
|
|
246
|
+
# AgentCert (TRUS-1228): issue + verify certificates for AI agents.
|
|
247
|
+
# issue is org-authenticated (any API key); verify is public.
|
|
248
|
+
self.agentcert = AgentCertEndpoint(self)
|
|
234
249
|
# MCP Scanner (TRUS-847): CRUD over AGT scan reports produced by
|
|
235
250
|
# trustmodel-mcp-server and persisted by the aurora-gateway
|
|
236
251
|
# `mcp_scanner` app. Not gated behind an extra — pure HTTP, no
|
|
@@ -300,7 +315,7 @@ class TrustModelClient:
|
|
|
300
315
|
endpoint: str,
|
|
301
316
|
data: Optional[Dict[str, Any]] = None,
|
|
302
317
|
params: Optional[Dict[str, Any]] = None,
|
|
303
|
-
) -> Dict[str, Any]:
|
|
318
|
+
) -> Union[Dict[str, Any], List[Any]]:
|
|
304
319
|
"""
|
|
305
320
|
Make an HTTP request to the TrustModel API.
|
|
306
321
|
|
|
@@ -322,9 +337,14 @@ class TrustModelClient:
|
|
|
322
337
|
|
|
323
338
|
# For OAuth auth, attach a fresh bearer token per request. The
|
|
324
339
|
# provider returns a cached token and only re-fetches near expiry.
|
|
325
|
-
headers:
|
|
340
|
+
headers: Dict[str, str] = {}
|
|
326
341
|
if self._auth is not None:
|
|
327
|
-
headers
|
|
342
|
+
headers["Authorization"] = self._auth.authorization_header()
|
|
343
|
+
# TRUS-1556 P4c: force Accept: application/json on every request
|
|
344
|
+
# so a session-header monkeypatch or middleware can't cause
|
|
345
|
+
# browsable-HTML content negotiation to win. Session default
|
|
346
|
+
# already sets this; this is belt-and-suspenders.
|
|
347
|
+
headers["Accept"] = "application/json"
|
|
328
348
|
|
|
329
349
|
try:
|
|
330
350
|
response = self._session.request(
|
|
@@ -344,7 +364,7 @@ class TrustModelClient:
|
|
|
344
364
|
|
|
345
365
|
return self._handle_response(response)
|
|
346
366
|
|
|
347
|
-
def _handle_response(self, response: requests.Response) -> Dict[str, Any]:
|
|
367
|
+
def _handle_response(self, response: requests.Response) -> Union[Dict[str, Any], List[Any]]:
|
|
348
368
|
"""
|
|
349
369
|
Handle the API response and raise appropriate exceptions for errors.
|
|
350
370
|
|
|
@@ -352,19 +372,73 @@ class TrustModelClient:
|
|
|
352
372
|
response: The HTTP response object
|
|
353
373
|
|
|
354
374
|
Returns:
|
|
355
|
-
Parsed response data
|
|
375
|
+
Parsed response data. Endpoints that return JSON objects yield a
|
|
376
|
+
``dict``; endpoints that return JSON arrays (e.g. paginated list
|
|
377
|
+
endpoints without an envelope, such as ``/api/agent-evaluation/
|
|
378
|
+
frameworks/``) yield a ``list``. The previous ``dict(response_data)``
|
|
379
|
+
coercion silently broke array responses with ``ValueError``.
|
|
356
380
|
|
|
357
381
|
Raises:
|
|
358
382
|
Various TrustModelError subclasses based on the error type
|
|
359
383
|
"""
|
|
384
|
+
# TRUS-1556 P4c: detect HTML responses (e.g. Django default 404 /
|
|
385
|
+
# 500 error pages, or a browsable-API renderer picked because the
|
|
386
|
+
# session ``Accept`` header got stripped by a middleware). Instead
|
|
387
|
+
# of returning a confusing ``ValueError`` on json.loads, surface a
|
|
388
|
+
# clear APIError so the caller knows the server sent the wrong
|
|
389
|
+
# content-type.
|
|
390
|
+
# ``headers`` may be absent on Mock responses used by tests — guard
|
|
391
|
+
# every step so this diagnostic never masks the real assertion.
|
|
392
|
+
raw_headers = getattr(response, "headers", None) or {}
|
|
393
|
+
try:
|
|
394
|
+
content_type_raw = (
|
|
395
|
+
raw_headers.get("Content-Type") if hasattr(raw_headers, "get") else ""
|
|
396
|
+
)
|
|
397
|
+
except Exception:
|
|
398
|
+
content_type_raw = ""
|
|
399
|
+
content_type = str(content_type_raw or "").lower()
|
|
400
|
+
if content_type.startswith("text/html") or content_type.startswith("application/xhtml"):
|
|
401
|
+
raw_text = getattr(response, "text", "") or ""
|
|
402
|
+
snippet = str(raw_text)[:300] if isinstance(raw_text, (str, bytes)) else ""
|
|
403
|
+
logger.error(
|
|
404
|
+
"trustmodel.client.html_response",
|
|
405
|
+
extra={
|
|
406
|
+
"status_code": response.status_code,
|
|
407
|
+
"url": getattr(getattr(response, "request", None), "url", ""),
|
|
408
|
+
"content_type": content_type,
|
|
409
|
+
},
|
|
410
|
+
)
|
|
411
|
+
raise APIError(
|
|
412
|
+
(
|
|
413
|
+
"Server returned HTML instead of JSON — the SDK cannot "
|
|
414
|
+
f"parse this. HTTP {response.status_code}, "
|
|
415
|
+
f"Content-Type={content_type!r}. First 300 chars: {snippet!r}"
|
|
416
|
+
),
|
|
417
|
+
status_code=response.status_code,
|
|
418
|
+
response={"content_type": content_type, "snippet": snippet},
|
|
419
|
+
error_code="html_response",
|
|
420
|
+
)
|
|
421
|
+
|
|
360
422
|
try:
|
|
361
423
|
response_data = response.json()
|
|
362
424
|
except json.JSONDecodeError:
|
|
363
425
|
response_data = {"detail": response.text or "Unknown error"}
|
|
364
426
|
|
|
365
|
-
# Handle successful responses
|
|
427
|
+
# Handle successful responses — pass through the shape the server sent.
|
|
366
428
|
if 200 <= response.status_code < 300:
|
|
367
|
-
|
|
429
|
+
if isinstance(response_data, list):
|
|
430
|
+
return list(response_data)
|
|
431
|
+
if isinstance(response_data, dict):
|
|
432
|
+
return dict(response_data)
|
|
433
|
+
# Primitives (bare bool / int / str) are rare but valid JSON; wrap
|
|
434
|
+
# in a stable envelope so callers can still `.get(...)` safely.
|
|
435
|
+
return {"data": response_data}
|
|
436
|
+
|
|
437
|
+
# Error paths below assume a dict body (server contract for errors).
|
|
438
|
+
# A non-dict error body is degenerate — coerce so downstream .get()
|
|
439
|
+
# calls don't AttributeError before we raise APIError.
|
|
440
|
+
if not isinstance(response_data, dict):
|
|
441
|
+
response_data = {"detail": str(response_data)}
|
|
368
442
|
|
|
369
443
|
# Handle error responses
|
|
370
444
|
error_message = response_data.get("detail", f"HTTP {response.status_code}")
|
|
@@ -432,21 +506,31 @@ class TrustModelClient:
|
|
|
432
506
|
error_code=error_code,
|
|
433
507
|
)
|
|
434
508
|
|
|
509
|
+
# ── public HTTP wrappers ──────────────────────────────────────────────
|
|
510
|
+
#
|
|
511
|
+
# ``_request`` / ``_handle_response`` return ``Union[Dict, List]`` so
|
|
512
|
+
# top-level array bodies (fleet listings, decisions feed) survive
|
|
513
|
+
# deserialization. Every caller of the four wrappers below (see
|
|
514
|
+
# ``endpoints/*.py``) still assumes a dict — they use ``**client.get(...)``
|
|
515
|
+
# or ``.get(field)`` chains — so we keep the wrapper signature as
|
|
516
|
+
# ``Dict[str, Any]`` and cast at this boundary. Endpoints that need
|
|
517
|
+
# array shapes should call ``_request`` directly (or a future
|
|
518
|
+
# ``.get_list()`` helper) instead of round-tripping through here.
|
|
435
519
|
def get(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
436
|
-
"""Make a GET request."""
|
|
437
|
-
return self._request("GET", endpoint, params=params)
|
|
520
|
+
"""Make a GET request. Returns the JSON body as a dict."""
|
|
521
|
+
return cast(Dict[str, Any], self._request("GET", endpoint, params=params))
|
|
438
522
|
|
|
439
523
|
def post(self, endpoint: str, data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
440
|
-
"""Make a POST request."""
|
|
441
|
-
return self._request("POST", endpoint, data=data)
|
|
524
|
+
"""Make a POST request. Returns the JSON body as a dict."""
|
|
525
|
+
return cast(Dict[str, Any], self._request("POST", endpoint, data=data))
|
|
442
526
|
|
|
443
527
|
def put(self, endpoint: str, data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
444
|
-
"""Make a PUT request."""
|
|
445
|
-
return self._request("PUT", endpoint, data=data)
|
|
528
|
+
"""Make a PUT request. Returns the JSON body as a dict."""
|
|
529
|
+
return cast(Dict[str, Any], self._request("PUT", endpoint, data=data))
|
|
446
530
|
|
|
447
531
|
def delete(self, endpoint: str) -> Dict[str, Any]:
|
|
448
|
-
"""Make a DELETE request."""
|
|
449
|
-
return self._request("DELETE", endpoint)
|
|
532
|
+
"""Make a DELETE request. Returns the JSON body as a dict."""
|
|
533
|
+
return cast(Dict[str, Any], self._request("DELETE", endpoint))
|
|
450
534
|
|
|
451
535
|
def ping(self) -> bool:
|
|
452
536
|
"""
|