trustmodel 2.1.0__tar.gz → 2.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trustmodel-2.1.0/README.md → trustmodel-2.2.1/PKG-INFO +243 -0
- trustmodel-2.1.0/PKG-INFO → trustmodel-2.2.1/README.md +175 -54
- {trustmodel-2.1.0 → trustmodel-2.2.1}/pyproject.toml +34 -1
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/__init__.py +13 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/client.py +2 -0
- trustmodel-2.2.1/src/trustmodel/endpoints/frameworks.py +94 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/evaluation.py +6 -3
- trustmodel-2.2.1/src/trustmodel/models/frameworks.py +27 -0
- trustmodel-2.2.1/src/trustmodel/telemetry/__init__.py +22 -0
- trustmodel-2.2.1/src/trustmodel/telemetry/_constants.py +33 -0
- trustmodel-2.2.1/src/trustmodel/telemetry/_instrumentors.py +53 -0
- trustmodel-2.2.1/src/trustmodel/telemetry/auto_init.py +289 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/.gitignore +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/LICENSE +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/compat.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/__init__.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/agentic.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/batch_jobs.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/config.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/credits.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/evaluations.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/galileo.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/models.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/exceptions.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/__init__.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/agentic.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/batch_job.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/credits.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/galileo.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/models.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/utils/__init__.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/utils/validation.py +0 -0
- {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/utils/version.py +0 -0
|
@@ -1,3 +1,71 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: trustmodel
|
|
3
|
+
Version: 2.2.1
|
|
4
|
+
Summary: Official Python SDK for TrustModel AI evaluation platform
|
|
5
|
+
Project-URL: Homepage, https://trustmodel.ai
|
|
6
|
+
Author-email: TrustModel <info@predixtions.com>
|
|
7
|
+
License: Proprietary - TrustModel Python SDK License
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: ai,api,evaluation,sdk,trustmodel
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: Other/Proprietary License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Requires-Python: >=3.7
|
|
24
|
+
Requires-Dist: eval-type-backport>=0.2.0; python_version < '3.10'
|
|
25
|
+
Requires-Dist: importlib-metadata>=4.0.0; python_version < '3.8'
|
|
26
|
+
Requires-Dist: pydantic<2.0.0,>=1.10.0; python_version < '3.8'
|
|
27
|
+
Requires-Dist: pydantic<3.0.0,>=2.0.0; python_version >= '3.8'
|
|
28
|
+
Requires-Dist: python-dateutil>=2.8.0
|
|
29
|
+
Requires-Dist: requests<3.0.0,>=2.25.0
|
|
30
|
+
Requires-Dist: tqdm>=4.60.0
|
|
31
|
+
Requires-Dist: typing-extensions>=4.0.0
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: black<23.0.0,>=22.0.0; (python_version < '3.8') and extra == 'dev'
|
|
34
|
+
Requires-Dist: black>=23.0.0; (python_version >= '3.8') and extra == 'dev'
|
|
35
|
+
Requires-Dist: bump2version>=1.0.1; extra == 'dev'
|
|
36
|
+
Requires-Dist: importlib-metadata>=4.0.0; (python_version < '3.8') and extra == 'dev'
|
|
37
|
+
Requires-Dist: isort<5.12.0,>=5.11.0; (python_version < '3.8') and extra == 'dev'
|
|
38
|
+
Requires-Dist: isort>=5.12.0; (python_version >= '3.8') and extra == 'dev'
|
|
39
|
+
Requires-Dist: mypy<1.5.0,>=1.0.0; (python_version < '3.8') and extra == 'dev'
|
|
40
|
+
Requires-Dist: mypy>=1.5.0; (python_version >= '3.8') and extra == 'dev'
|
|
41
|
+
Requires-Dist: pytest-cov<5.0.0,>=4.0.0; (python_version < '3.8') and extra == 'dev'
|
|
42
|
+
Requires-Dist: pytest-cov>=4.0.0; (python_version >= '3.8') and extra == 'dev'
|
|
43
|
+
Requires-Dist: pytest<8.0.0,>=7.0.0; (python_version < '3.8') and extra == 'dev'
|
|
44
|
+
Requires-Dist: pytest>=7.0.0; (python_version >= '3.8') and extra == 'dev'
|
|
45
|
+
Requires-Dist: ruff<0.1.0,>=0.0.277; (python_version < '3.8') and extra == 'dev'
|
|
46
|
+
Requires-Dist: ruff>=0.1.0; (python_version >= '3.8') and extra == 'dev'
|
|
47
|
+
Requires-Dist: types-requests>=2.25.0; extra == 'dev'
|
|
48
|
+
Requires-Dist: types-tqdm>=4.60.0; extra == 'dev'
|
|
49
|
+
Provides-Extra: docs
|
|
50
|
+
Requires-Dist: sphinx-autodoc-typehints>=1.24.0; extra == 'docs'
|
|
51
|
+
Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == 'docs'
|
|
52
|
+
Requires-Dist: sphinx>=7.0.0; extra == 'docs'
|
|
53
|
+
Provides-Extra: telemetry
|
|
54
|
+
Requires-Dist: openinference-instrumentation-anthropic>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
55
|
+
Requires-Dist: openinference-instrumentation-bedrock>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
56
|
+
Requires-Dist: openinference-instrumentation-crewai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
57
|
+
Requires-Dist: openinference-instrumentation-dspy>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
58
|
+
Requires-Dist: openinference-instrumentation-groq>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
59
|
+
Requires-Dist: openinference-instrumentation-langchain>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
60
|
+
Requires-Dist: openinference-instrumentation-llama-index>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
61
|
+
Requires-Dist: openinference-instrumentation-mistralai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
62
|
+
Requires-Dist: openinference-instrumentation-openai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
63
|
+
Requires-Dist: openinference-instrumentation-vertexai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
64
|
+
Requires-Dist: opentelemetry-api>=1.20.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
65
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.20.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
66
|
+
Requires-Dist: opentelemetry-sdk>=1.20.0; (python_version >= '3.10') and extra == 'telemetry'
|
|
67
|
+
Description-Content-Type: text/markdown
|
|
68
|
+
|
|
1
69
|
<p align="center">
|
|
2
70
|
<a href="https://www.trustmodel.ai">
|
|
3
71
|
<img src="https://www.trustmodel.ai/assets/trustmodel-wordmark-CfuXSOoK.svg" alt="TrustModel" width="400">
|
|
@@ -23,6 +91,36 @@
|
|
|
23
91
|
|
|
24
92
|
Evaluate AI models for safety, bias, and performance with a simple, intuitive interface.
|
|
25
93
|
|
|
94
|
+
## Table of Contents
|
|
95
|
+
|
|
96
|
+
- [Features](#features)
|
|
97
|
+
- [Installation](#installation)
|
|
98
|
+
- [Prerequisites](#prerequisites)
|
|
99
|
+
- [Quick Start](#quick-start)
|
|
100
|
+
- [Authentication](#authentication)
|
|
101
|
+
- [Evaluation Modes](#evaluation-modes)
|
|
102
|
+
- [Getting Available Vendors](#getting-available-vendors)
|
|
103
|
+
- [Getting Available Models](#getting-available-models)
|
|
104
|
+
- [Platform Key (Default)](#platform-key-default)
|
|
105
|
+
- [BYOK (Bring Your Own Key)](#byok-bring-your-own-key)
|
|
106
|
+
- [Custom Endpoint](#custom-endpoint)
|
|
107
|
+
- [Core Concepts](#core-concepts)
|
|
108
|
+
- [Models](#models)
|
|
109
|
+
- [Evaluations](#evaluations)
|
|
110
|
+
- [Configuration](#configuration)
|
|
111
|
+
- [Credits Management](#credits-management)
|
|
112
|
+
- [Error Handling](#error-handling)
|
|
113
|
+
- [Rate Limiting](#rate-limiting)
|
|
114
|
+
- [Webhook Notifications](#webhook-notifications)
|
|
115
|
+
- [Advanced Usage](#advanced-usage)
|
|
116
|
+
- [Framework Integration](#framework-integration)
|
|
117
|
+
- [Zero-Config Auto-Capture (`auto_init`)](#zero-config-auto-capture-auto_init)
|
|
118
|
+
- [Agentic Trace Evaluation](#agentic-trace-evaluation)
|
|
119
|
+
- [Galileo Integration](#galileo-integration)
|
|
120
|
+
- [Requirements](#requirements)
|
|
121
|
+
- [Support](#support)
|
|
122
|
+
- [License](#license)
|
|
123
|
+
|
|
26
124
|
## Features
|
|
27
125
|
|
|
28
126
|
- 🚀 **Simple Interface**: Easy-to-use client for all TrustModel operations
|
|
@@ -31,13 +129,24 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
|
|
|
31
129
|
- 🔄 **Reliable**: Automatic retries and comprehensive error handling
|
|
32
130
|
- 📊 **Comprehensive**: Support for all evaluation types and configurations
|
|
33
131
|
- 🌍 **Framework Agnostic**: Works with any Python framework or standalone scripts
|
|
132
|
+
- 🛰️ **Zero-Config Auto-Capture**: Add two lines (`auto_init`) and every OpenAI / Anthropic / LangChain / CrewAI / etc. call is automatically traced and evaluated
|
|
34
133
|
|
|
35
134
|
## Installation
|
|
36
135
|
|
|
136
|
+
Core SDK (works on Python 3.7+):
|
|
137
|
+
|
|
37
138
|
```bash
|
|
38
139
|
pip install trustmodel
|
|
39
140
|
```
|
|
40
141
|
|
|
142
|
+
Add the `telemetry` extra to enable zero-config auto-capture of AI agent calls (Python 3.10+ required):
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
pip install "trustmodel[telemetry]"
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
The `telemetry` extra installs OpenTelemetry plus all OpenInference instrumentors (OpenAI, Anthropic, LangChain, LlamaIndex, Bedrock, Mistral, Groq, CrewAI, VertexAI, DSPy). See [Zero-Config Auto-Capture](#zero-config-auto-capture-auto_init) below.
|
|
149
|
+
|
|
41
150
|
## Prerequisites
|
|
42
151
|
|
|
43
152
|
Before using the SDK, you **must** complete the following setup in the [TrustModel Dashboard](https://app.trustmodel.ai/app/keys):
|
|
@@ -1008,6 +1117,140 @@ def evaluate():
|
|
|
1008
1117
|
})
|
|
1009
1118
|
```
|
|
1010
1119
|
|
|
1120
|
+
## Zero-Config Auto-Capture (`auto_init`)
|
|
1121
|
+
|
|
1122
|
+
Capture every LLM and tool call your AI agent makes — without changing your existing code. Two lines, and TrustModel takes care of trace collection, batching, transport, and evaluation.
|
|
1123
|
+
|
|
1124
|
+
### How It Works
|
|
1125
|
+
|
|
1126
|
+
1. You call `auto_init(api_key, agent_id, ...)` once at startup.
|
|
1127
|
+
2. The SDK auto-detects which AI libraries you're using (OpenAI, Anthropic, LangChain, etc.) and installs an OpenInference instrumentor for each.
|
|
1128
|
+
3. Every subsequent LLM / tool call is captured as an OpenTelemetry span and streamed to the TrustModel gateway.
|
|
1129
|
+
4. The TrustModel Control Plane buffers traces server-side, groups them by `agent_id` + `domain` + `frameworks`, and runs evaluation on the schedule you configure (daily, weekly, or monthly).
|
|
1130
|
+
|
|
1131
|
+
### Installation
|
|
1132
|
+
|
|
1133
|
+
```bash
|
|
1134
|
+
pip install "trustmodel[telemetry]"
|
|
1135
|
+
```
|
|
1136
|
+
|
|
1137
|
+
This installs OpenTelemetry plus the OpenInference instrumentors. **Python 3.10+ required** for the telemetry extra.
|
|
1138
|
+
|
|
1139
|
+
If you only want the core SDK (e.g., `client.evaluations.create()`), `pip install trustmodel` works on Python 3.7+ but `auto_init` will not be available.
|
|
1140
|
+
|
|
1141
|
+
### Quick Start
|
|
1142
|
+
|
|
1143
|
+
```python
|
|
1144
|
+
from trustmodel.telemetry import auto_init
|
|
1145
|
+
|
|
1146
|
+
auto_init(
|
|
1147
|
+
api_key="tm-...", # your TrustModel API key
|
|
1148
|
+
agent_id="my-customer-support-agent", # any identifier you choose
|
|
1149
|
+
domain="general_ai", # fair_lending | hr_bias | healthcare | general_ai
|
|
1150
|
+
frameworks=["nist-ai-rmf"], # one or more compliance framework slugs
|
|
1151
|
+
)
|
|
1152
|
+
|
|
1153
|
+
# Your existing agent code — no changes needed
|
|
1154
|
+
import openai
|
|
1155
|
+
client = openai.OpenAI()
|
|
1156
|
+
client.chat.completions.create(
|
|
1157
|
+
model="gpt-4o-mini",
|
|
1158
|
+
messages=[{"role": "user", "content": "Hello"}],
|
|
1159
|
+
)
|
|
1160
|
+
```
|
|
1161
|
+
|
|
1162
|
+
That's it. The OpenAI call is automatically captured and streamed to TrustModel.
|
|
1163
|
+
|
|
1164
|
+
### Discovering Domains and Frameworks
|
|
1165
|
+
|
|
1166
|
+
Use the `client.frameworks` endpoint to list available domains and the compliance frameworks within each:
|
|
1167
|
+
|
|
1168
|
+
```python
|
|
1169
|
+
from trustmodel import TrustModelClient
|
|
1170
|
+
|
|
1171
|
+
client = TrustModelClient(api_key="tm-...")
|
|
1172
|
+
|
|
1173
|
+
# All available domains
|
|
1174
|
+
print(client.frameworks.list_domains())
|
|
1175
|
+
# ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
|
|
1176
|
+
|
|
1177
|
+
# Frameworks for a specific domain
|
|
1178
|
+
for f in client.frameworks.list(domain="fair_lending"):
|
|
1179
|
+
print(f"{f.slug}: {f.name} ({f.credits} credits)")
|
|
1180
|
+
```
|
|
1181
|
+
|
|
1182
|
+
Use the `slug` values when calling `auto_init(frameworks=[...])`.
|
|
1183
|
+
|
|
1184
|
+
### Supported AI Libraries
|
|
1185
|
+
|
|
1186
|
+
`auto_init` will automatically activate instrumentation for any of these libraries that you have installed:
|
|
1187
|
+
|
|
1188
|
+
| Library | Auto-detected via |
|
|
1189
|
+
|---------|-------------------|
|
|
1190
|
+
| OpenAI | `openinference-instrumentation-openai` |
|
|
1191
|
+
| Anthropic (Claude) | `openinference-instrumentation-anthropic` |
|
|
1192
|
+
| LangChain | `openinference-instrumentation-langchain` |
|
|
1193
|
+
| LlamaIndex | `openinference-instrumentation-llama-index` |
|
|
1194
|
+
| AWS Bedrock | `openinference-instrumentation-bedrock` |
|
|
1195
|
+
| Mistral AI | `openinference-instrumentation-mistralai` |
|
|
1196
|
+
| Groq | `openinference-instrumentation-groq` |
|
|
1197
|
+
| CrewAI | `openinference-instrumentation-crewai` |
|
|
1198
|
+
| Vertex AI | `openinference-instrumentation-vertexai` |
|
|
1199
|
+
| DSPy | `openinference-instrumentation-dspy` |
|
|
1200
|
+
|
|
1201
|
+
All ten are installed by `pip install "trustmodel[telemetry]"`. The instrumentors are no-ops when the underlying library isn't being used — there's no overhead.
|
|
1202
|
+
|
|
1203
|
+
### Existing OpenTelemetry Setup (Datadog, Jaeger, Honeycomb, Arize)
|
|
1204
|
+
|
|
1205
|
+
If your application already has OpenTelemetry configured for another observability backend, `auto_init` detects this and **adds TrustModel as an additional exporter** rather than replacing your setup.
|
|
1206
|
+
|
|
1207
|
+
```python
|
|
1208
|
+
# Your existing OTel setup (e.g., Datadog APM, Jaeger, Honeycomb)
|
|
1209
|
+
from opentelemetry import trace
|
|
1210
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
1211
|
+
provider = TracerProvider(...)
|
|
1212
|
+
trace.set_tracer_provider(provider)
|
|
1213
|
+
|
|
1214
|
+
# Then add TrustModel — both backends now receive every span
|
|
1215
|
+
from trustmodel.telemetry import auto_init
|
|
1216
|
+
auto_init(api_key="tm-...", agent_id="my-agent")
|
|
1217
|
+
```
|
|
1218
|
+
|
|
1219
|
+
Spans fan out to both your existing exporter and TrustModel's. Nothing in your existing pipeline is modified or replaced.
|
|
1220
|
+
|
|
1221
|
+
### Parameters
|
|
1222
|
+
|
|
1223
|
+
| Parameter | Required | Description |
|
|
1224
|
+
|-----------|----------|-------------|
|
|
1225
|
+
| `api_key` | yes | Your TrustModel API key (`tm-...`) |
|
|
1226
|
+
| `agent_id` | yes | Any string you choose. All traces sharing this `agent_id` are grouped together for evaluation. |
|
|
1227
|
+
| `domain` | yes | One of `fair_lending`, `hr_bias`, `healthcare`, `general_ai`. Determines which evaluators run. |
|
|
1228
|
+
| `frameworks` | yes | List of compliance framework slugs (e.g., `["ecoa-regb", "fcra"]`). Use `client.frameworks.list(domain=...)` to discover. |
|
|
1229
|
+
| `service_name` | no | Logical service name (default: `"default"`) |
|
|
1230
|
+
|
|
1231
|
+
### Schedule and Evaluation
|
|
1232
|
+
|
|
1233
|
+
Configure when buffered traces are evaluated in the [Control Plane dashboard](https://app.trustmodel.ai):
|
|
1234
|
+
|
|
1235
|
+
- **Manual** — only when you click "Trigger Evaluation Now"
|
|
1236
|
+
- **Daily / Weekly / Monthly** — automatically via cron at midnight UTC
|
|
1237
|
+
|
|
1238
|
+
Each evaluation produces:
|
|
1239
|
+
- An overall trust score
|
|
1240
|
+
- Per-category breakdowns (safety, fairness, accuracy, etc.)
|
|
1241
|
+
- Findings and recommendations
|
|
1242
|
+
|
|
1243
|
+
### Failure Modes (Safe by Design)
|
|
1244
|
+
|
|
1245
|
+
`auto_init` wraps everything in `try/except`. If anything fails — missing dependencies, network issues, an instrumentor crash — your application keeps running. Telemetry is best-effort; nothing TrustModel does will break your agent.
|
|
1246
|
+
|
|
1247
|
+
If the `[telemetry]` extras aren't installed and you try to import the telemetry module directly, you get a clear error:
|
|
1248
|
+
|
|
1249
|
+
```
|
|
1250
|
+
ImportError: TrustModel telemetry requires extra dependencies.
|
|
1251
|
+
Install with: pip install "trustmodel[telemetry]"
|
|
1252
|
+
```
|
|
1253
|
+
|
|
1011
1254
|
## Agentic Trace Evaluation
|
|
1012
1255
|
|
|
1013
1256
|
Evaluate AI agent execution traces for safety, reasoning quality, tool usage, and goal completion. Upload a JSON or JSONL trace file and get scored across 14 dimensions.
|
|
@@ -1,57 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: trustmodel
|
|
3
|
-
Version: 2.1.0
|
|
4
|
-
Summary: Official Python SDK for TrustModel AI evaluation platform
|
|
5
|
-
Project-URL: Homepage, https://trustmodel.ai
|
|
6
|
-
Author-email: TrustModel <info@predixtions.com>
|
|
7
|
-
License: Proprietary - TrustModel Python SDK License
|
|
8
|
-
License-File: LICENSE
|
|
9
|
-
Keywords: ai,api,evaluation,sdk,trustmodel
|
|
10
|
-
Classifier: Development Status :: 4 - Beta
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: License :: Other/Proprietary License
|
|
13
|
-
Classifier: Operating System :: OS Independent
|
|
14
|
-
Classifier: Programming Language :: Python :: 3
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
-
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
-
Requires-Python: >=3.7
|
|
24
|
-
Requires-Dist: eval-type-backport>=0.2.0; python_version < '3.10'
|
|
25
|
-
Requires-Dist: importlib-metadata>=4.0.0; python_version < '3.8'
|
|
26
|
-
Requires-Dist: pydantic<2.0.0,>=1.10.0; python_version < '3.8'
|
|
27
|
-
Requires-Dist: pydantic<3.0.0,>=2.0.0; python_version >= '3.8'
|
|
28
|
-
Requires-Dist: python-dateutil>=2.8.0
|
|
29
|
-
Requires-Dist: requests<3.0.0,>=2.25.0
|
|
30
|
-
Requires-Dist: tqdm>=4.60.0
|
|
31
|
-
Requires-Dist: typing-extensions>=4.0.0
|
|
32
|
-
Provides-Extra: dev
|
|
33
|
-
Requires-Dist: black<23.0.0,>=22.0.0; (python_version < '3.8') and extra == 'dev'
|
|
34
|
-
Requires-Dist: black>=23.0.0; (python_version >= '3.8') and extra == 'dev'
|
|
35
|
-
Requires-Dist: bump2version>=1.0.1; extra == 'dev'
|
|
36
|
-
Requires-Dist: importlib-metadata>=4.0.0; (python_version < '3.8') and extra == 'dev'
|
|
37
|
-
Requires-Dist: isort<5.12.0,>=5.11.0; (python_version < '3.8') and extra == 'dev'
|
|
38
|
-
Requires-Dist: isort>=5.12.0; (python_version >= '3.8') and extra == 'dev'
|
|
39
|
-
Requires-Dist: mypy<1.5.0,>=1.0.0; (python_version < '3.8') and extra == 'dev'
|
|
40
|
-
Requires-Dist: mypy>=1.5.0; (python_version >= '3.8') and extra == 'dev'
|
|
41
|
-
Requires-Dist: pytest-cov<5.0.0,>=4.0.0; (python_version < '3.8') and extra == 'dev'
|
|
42
|
-
Requires-Dist: pytest-cov>=4.0.0; (python_version >= '3.8') and extra == 'dev'
|
|
43
|
-
Requires-Dist: pytest<8.0.0,>=7.0.0; (python_version < '3.8') and extra == 'dev'
|
|
44
|
-
Requires-Dist: pytest>=7.0.0; (python_version >= '3.8') and extra == 'dev'
|
|
45
|
-
Requires-Dist: ruff<0.1.0,>=0.0.277; (python_version < '3.8') and extra == 'dev'
|
|
46
|
-
Requires-Dist: ruff>=0.1.0; (python_version >= '3.8') and extra == 'dev'
|
|
47
|
-
Requires-Dist: types-requests>=2.25.0; extra == 'dev'
|
|
48
|
-
Requires-Dist: types-tqdm>=4.60.0; extra == 'dev'
|
|
49
|
-
Provides-Extra: docs
|
|
50
|
-
Requires-Dist: sphinx-autodoc-typehints>=1.24.0; extra == 'docs'
|
|
51
|
-
Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == 'docs'
|
|
52
|
-
Requires-Dist: sphinx>=7.0.0; extra == 'docs'
|
|
53
|
-
Description-Content-Type: text/markdown
|
|
54
|
-
|
|
55
1
|
<p align="center">
|
|
56
2
|
<a href="https://www.trustmodel.ai">
|
|
57
3
|
<img src="https://www.trustmodel.ai/assets/trustmodel-wordmark-CfuXSOoK.svg" alt="TrustModel" width="400">
|
|
@@ -77,6 +23,36 @@ Description-Content-Type: text/markdown
|
|
|
77
23
|
|
|
78
24
|
Evaluate AI models for safety, bias, and performance with a simple, intuitive interface.
|
|
79
25
|
|
|
26
|
+
## Table of Contents
|
|
27
|
+
|
|
28
|
+
- [Features](#features)
|
|
29
|
+
- [Installation](#installation)
|
|
30
|
+
- [Prerequisites](#prerequisites)
|
|
31
|
+
- [Quick Start](#quick-start)
|
|
32
|
+
- [Authentication](#authentication)
|
|
33
|
+
- [Evaluation Modes](#evaluation-modes)
|
|
34
|
+
- [Getting Available Vendors](#getting-available-vendors)
|
|
35
|
+
- [Getting Available Models](#getting-available-models)
|
|
36
|
+
- [Platform Key (Default)](#platform-key-default)
|
|
37
|
+
- [BYOK (Bring Your Own Key)](#byok-bring-your-own-key)
|
|
38
|
+
- [Custom Endpoint](#custom-endpoint)
|
|
39
|
+
- [Core Concepts](#core-concepts)
|
|
40
|
+
- [Models](#models)
|
|
41
|
+
- [Evaluations](#evaluations)
|
|
42
|
+
- [Configuration](#configuration)
|
|
43
|
+
- [Credits Management](#credits-management)
|
|
44
|
+
- [Error Handling](#error-handling)
|
|
45
|
+
- [Rate Limiting](#rate-limiting)
|
|
46
|
+
- [Webhook Notifications](#webhook-notifications)
|
|
47
|
+
- [Advanced Usage](#advanced-usage)
|
|
48
|
+
- [Framework Integration](#framework-integration)
|
|
49
|
+
- [Zero-Config Auto-Capture (`auto_init`)](#zero-config-auto-capture-auto_init)
|
|
50
|
+
- [Agentic Trace Evaluation](#agentic-trace-evaluation)
|
|
51
|
+
- [Galileo Integration](#galileo-integration)
|
|
52
|
+
- [Requirements](#requirements)
|
|
53
|
+
- [Support](#support)
|
|
54
|
+
- [License](#license)
|
|
55
|
+
|
|
80
56
|
## Features
|
|
81
57
|
|
|
82
58
|
- 🚀 **Simple Interface**: Easy-to-use client for all TrustModel operations
|
|
@@ -85,13 +61,24 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
|
|
|
85
61
|
- 🔄 **Reliable**: Automatic retries and comprehensive error handling
|
|
86
62
|
- 📊 **Comprehensive**: Support for all evaluation types and configurations
|
|
87
63
|
- 🌍 **Framework Agnostic**: Works with any Python framework or standalone scripts
|
|
64
|
+
- 🛰️ **Zero-Config Auto-Capture**: Add two lines (`auto_init`) and every OpenAI / Anthropic / LangChain / CrewAI / etc. call is automatically traced and evaluated
|
|
88
65
|
|
|
89
66
|
## Installation
|
|
90
67
|
|
|
68
|
+
Core SDK (works on Python 3.7+):
|
|
69
|
+
|
|
91
70
|
```bash
|
|
92
71
|
pip install trustmodel
|
|
93
72
|
```
|
|
94
73
|
|
|
74
|
+
Add the `telemetry` extra to enable zero-config auto-capture of AI agent calls (Python 3.10+ required):
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install "trustmodel[telemetry]"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
The `telemetry` extra installs OpenTelemetry plus all OpenInference instrumentors (OpenAI, Anthropic, LangChain, LlamaIndex, Bedrock, Mistral, Groq, CrewAI, VertexAI, DSPy). See [Zero-Config Auto-Capture](#zero-config-auto-capture-auto_init) below.
|
|
81
|
+
|
|
95
82
|
## Prerequisites
|
|
96
83
|
|
|
97
84
|
Before using the SDK, you **must** complete the following setup in the [TrustModel Dashboard](https://app.trustmodel.ai/app/keys):
|
|
@@ -1062,6 +1049,140 @@ def evaluate():
|
|
|
1062
1049
|
})
|
|
1063
1050
|
```
|
|
1064
1051
|
|
|
1052
|
+
## Zero-Config Auto-Capture (`auto_init`)
|
|
1053
|
+
|
|
1054
|
+
Capture every LLM and tool call your AI agent makes — without changing your existing code. Two lines, and TrustModel takes care of trace collection, batching, transport, and evaluation.
|
|
1055
|
+
|
|
1056
|
+
### How It Works
|
|
1057
|
+
|
|
1058
|
+
1. You call `auto_init(api_key, agent_id, ...)` once at startup.
|
|
1059
|
+
2. The SDK auto-detects which AI libraries you're using (OpenAI, Anthropic, LangChain, etc.) and installs an OpenInference instrumentor for each.
|
|
1060
|
+
3. Every subsequent LLM / tool call is captured as an OpenTelemetry span and streamed to the TrustModel gateway.
|
|
1061
|
+
4. The TrustModel Control Plane buffers traces server-side, groups them by `agent_id` + `domain` + `frameworks`, and runs evaluation on the schedule you configure (daily, weekly, or monthly).
|
|
1062
|
+
|
|
1063
|
+
### Installation
|
|
1064
|
+
|
|
1065
|
+
```bash
|
|
1066
|
+
pip install "trustmodel[telemetry]"
|
|
1067
|
+
```
|
|
1068
|
+
|
|
1069
|
+
This installs OpenTelemetry plus the OpenInference instrumentors. **Python 3.10+ required** for the telemetry extra.
|
|
1070
|
+
|
|
1071
|
+
If you only want the core SDK (e.g., `client.evaluations.create()`), `pip install trustmodel` works on Python 3.7+ but `auto_init` will not be available.
|
|
1072
|
+
|
|
1073
|
+
### Quick Start
|
|
1074
|
+
|
|
1075
|
+
```python
|
|
1076
|
+
from trustmodel.telemetry import auto_init
|
|
1077
|
+
|
|
1078
|
+
auto_init(
|
|
1079
|
+
api_key="tm-...", # your TrustModel API key
|
|
1080
|
+
agent_id="my-customer-support-agent", # any identifier you choose
|
|
1081
|
+
domain="general_ai", # fair_lending | hr_bias | healthcare | general_ai
|
|
1082
|
+
frameworks=["nist-ai-rmf"], # one or more compliance framework slugs
|
|
1083
|
+
)
|
|
1084
|
+
|
|
1085
|
+
# Your existing agent code — no changes needed
|
|
1086
|
+
import openai
|
|
1087
|
+
client = openai.OpenAI()
|
|
1088
|
+
client.chat.completions.create(
|
|
1089
|
+
model="gpt-4o-mini",
|
|
1090
|
+
messages=[{"role": "user", "content": "Hello"}],
|
|
1091
|
+
)
|
|
1092
|
+
```
|
|
1093
|
+
|
|
1094
|
+
That's it. The OpenAI call is automatically captured and streamed to TrustModel.
|
|
1095
|
+
|
|
1096
|
+
### Discovering Domains and Frameworks
|
|
1097
|
+
|
|
1098
|
+
Use the `client.frameworks` endpoint to list available domains and the compliance frameworks within each:
|
|
1099
|
+
|
|
1100
|
+
```python
|
|
1101
|
+
from trustmodel import TrustModelClient
|
|
1102
|
+
|
|
1103
|
+
client = TrustModelClient(api_key="tm-...")
|
|
1104
|
+
|
|
1105
|
+
# All available domains
|
|
1106
|
+
print(client.frameworks.list_domains())
|
|
1107
|
+
# ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
|
|
1108
|
+
|
|
1109
|
+
# Frameworks for a specific domain
|
|
1110
|
+
for f in client.frameworks.list(domain="fair_lending"):
|
|
1111
|
+
print(f"{f.slug}: {f.name} ({f.credits} credits)")
|
|
1112
|
+
```
|
|
1113
|
+
|
|
1114
|
+
Use the `slug` values when calling `auto_init(frameworks=[...])`.
|
|
1115
|
+
|
|
1116
|
+
### Supported AI Libraries
|
|
1117
|
+
|
|
1118
|
+
`auto_init` will automatically activate instrumentation for any of these libraries that you have installed:
|
|
1119
|
+
|
|
1120
|
+
| Library | Auto-detected via |
|
|
1121
|
+
|---------|-------------------|
|
|
1122
|
+
| OpenAI | `openinference-instrumentation-openai` |
|
|
1123
|
+
| Anthropic (Claude) | `openinference-instrumentation-anthropic` |
|
|
1124
|
+
| LangChain | `openinference-instrumentation-langchain` |
|
|
1125
|
+
| LlamaIndex | `openinference-instrumentation-llama-index` |
|
|
1126
|
+
| AWS Bedrock | `openinference-instrumentation-bedrock` |
|
|
1127
|
+
| Mistral AI | `openinference-instrumentation-mistralai` |
|
|
1128
|
+
| Groq | `openinference-instrumentation-groq` |
|
|
1129
|
+
| CrewAI | `openinference-instrumentation-crewai` |
|
|
1130
|
+
| Vertex AI | `openinference-instrumentation-vertexai` |
|
|
1131
|
+
| DSPy | `openinference-instrumentation-dspy` |
|
|
1132
|
+
|
|
1133
|
+
All ten are installed by `pip install "trustmodel[telemetry]"`. The instrumentors are no-ops when the underlying library isn't being used — there's no overhead.
|
|
1134
|
+
|
|
1135
|
+
### Existing OpenTelemetry Setup (Datadog, Jaeger, Honeycomb, Arize)
|
|
1136
|
+
|
|
1137
|
+
If your application already has OpenTelemetry configured for another observability backend, `auto_init` detects this and **adds TrustModel as an additional exporter** rather than replacing your setup.
|
|
1138
|
+
|
|
1139
|
+
```python
|
|
1140
|
+
# Your existing OTel setup (e.g., Datadog APM, Jaeger, Honeycomb)
|
|
1141
|
+
from opentelemetry import trace
|
|
1142
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
1143
|
+
provider = TracerProvider(...)
|
|
1144
|
+
trace.set_tracer_provider(provider)
|
|
1145
|
+
|
|
1146
|
+
# Then add TrustModel — both backends now receive every span
|
|
1147
|
+
from trustmodel.telemetry import auto_init
|
|
1148
|
+
auto_init(api_key="tm-...", agent_id="my-agent")
|
|
1149
|
+
```
|
|
1150
|
+
|
|
1151
|
+
Spans fan out to both your existing exporter and TrustModel's. Nothing in your existing pipeline is modified or replaced.
|
|
1152
|
+
|
|
1153
|
+
### Parameters
|
|
1154
|
+
|
|
1155
|
+
| Parameter | Required | Description |
|
|
1156
|
+
|-----------|----------|-------------|
|
|
1157
|
+
| `api_key` | yes | Your TrustModel API key (`tm-...`) |
|
|
1158
|
+
| `agent_id` | yes | Any string you choose. All traces sharing this `agent_id` are grouped together for evaluation. |
|
|
1159
|
+
| `domain` | yes | One of `fair_lending`, `hr_bias`, `healthcare`, `general_ai`. Determines which evaluators run. |
|
|
1160
|
+
| `frameworks` | yes | List of compliance framework slugs (e.g., `["ecoa-regb", "fcra"]`). Use `client.frameworks.list(domain=...)` to discover. |
|
|
1161
|
+
| `service_name` | no | Logical service name (default: `"default"`) |
|
|
1162
|
+
|
|
1163
|
+
### Schedule and Evaluation
|
|
1164
|
+
|
|
1165
|
+
Configure when buffered traces are evaluated in the [Control Plane dashboard](https://app.trustmodel.ai):
|
|
1166
|
+
|
|
1167
|
+
- **Manual** — only when you click "Trigger Evaluation Now"
|
|
1168
|
+
- **Daily / Weekly / Monthly** — automatically via cron at midnight UTC
|
|
1169
|
+
|
|
1170
|
+
Each evaluation produces:
|
|
1171
|
+
- An overall trust score
|
|
1172
|
+
- Per-category breakdowns (safety, fairness, accuracy, etc.)
|
|
1173
|
+
- Findings and recommendations
|
|
1174
|
+
|
|
1175
|
+
### Failure Modes (Safe by Design)
|
|
1176
|
+
|
|
1177
|
+
`auto_init` wraps everything in `try/except`. If anything fails — missing dependencies, network issues, an instrumentor crash — your application keeps running. Telemetry is best-effort; nothing TrustModel does will break your agent.
|
|
1178
|
+
|
|
1179
|
+
If the `[telemetry]` extras aren't installed and you try to import the telemetry module directly, you get a clear error:
|
|
1180
|
+
|
|
1181
|
+
```
|
|
1182
|
+
ImportError: TrustModel telemetry requires extra dependencies.
|
|
1183
|
+
Install with: pip install "trustmodel[telemetry]"
|
|
1184
|
+
```
|
|
1185
|
+
|
|
1065
1186
|
## Agentic Trace Evaluation
|
|
1066
1187
|
|
|
1067
1188
|
Evaluate AI agent execution traces for safety, reasoning quality, tool usage, and goal completion. Upload a JSON or JSONL trace file and get scored across 14 dimensions.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "trustmodel"
|
|
7
|
-
version = "2.1
|
|
7
|
+
version = "2.2.1"
|
|
8
8
|
authors = [
|
|
9
9
|
{name = "TrustModel", email = "info@predixtions.com"},
|
|
10
10
|
]
|
|
@@ -40,6 +40,24 @@ dependencies = [
|
|
|
40
40
|
]
|
|
41
41
|
|
|
42
42
|
[project.optional-dependencies]
|
|
43
|
+
# Auto-capture AI agent traces and stream them to TrustModel.
|
|
44
|
+
# Install with: pip install "trustmodel[telemetry]"
|
|
45
|
+
# Requires Python 3.10+ because several OpenInference instrumentors do.
|
|
46
|
+
telemetry = [
|
|
47
|
+
"opentelemetry-api>=1.20.0; python_version>='3.10'",
|
|
48
|
+
"opentelemetry-sdk>=1.20.0; python_version>='3.10'",
|
|
49
|
+
"opentelemetry-exporter-otlp-proto-http>=1.20.0; python_version>='3.10'",
|
|
50
|
+
"openinference-instrumentation-openai>=0.1.0; python_version>='3.10'",
|
|
51
|
+
"openinference-instrumentation-anthropic>=0.1.0; python_version>='3.10'",
|
|
52
|
+
"openinference-instrumentation-langchain>=0.1.0; python_version>='3.10'",
|
|
53
|
+
"openinference-instrumentation-llama-index>=0.1.0; python_version>='3.10'",
|
|
54
|
+
"openinference-instrumentation-bedrock>=0.1.0; python_version>='3.10'",
|
|
55
|
+
"openinference-instrumentation-mistralai>=0.1.0; python_version>='3.10'",
|
|
56
|
+
"openinference-instrumentation-groq>=0.1.0; python_version>='3.10'",
|
|
57
|
+
"openinference-instrumentation-crewai>=0.1.0; python_version>='3.10'",
|
|
58
|
+
"openinference-instrumentation-vertexai>=0.1.0; python_version>='3.10'",
|
|
59
|
+
"openinference-instrumentation-dspy>=0.1.0; python_version>='3.10'",
|
|
60
|
+
]
|
|
43
61
|
dev = [
|
|
44
62
|
"pytest>=7.0.0,<8.0.0; python_version<'3.8'",
|
|
45
63
|
"pytest>=7.0.0; python_version>='3.8'",
|
|
@@ -119,6 +137,15 @@ strict_equality = true
|
|
|
119
137
|
module = "tests.*"
|
|
120
138
|
disallow_untyped_defs = false
|
|
121
139
|
|
|
140
|
+
[[tool.mypy.overrides]]
|
|
141
|
+
# Optional telemetry deps — installed via [telemetry] extra. CI runs mypy
|
|
142
|
+
# without these installed, so suppress missing-import errors.
|
|
143
|
+
module = [
|
|
144
|
+
"opentelemetry.*",
|
|
145
|
+
"openinference.*",
|
|
146
|
+
]
|
|
147
|
+
ignore_missing_imports = true
|
|
148
|
+
|
|
122
149
|
[tool.ruff]
|
|
123
150
|
target-version = "py37"
|
|
124
151
|
line-length = 100
|
|
@@ -159,6 +186,12 @@ source = ["src/trustmodel"]
|
|
|
159
186
|
omit = [
|
|
160
187
|
"*/tests/*",
|
|
161
188
|
"*/test_*",
|
|
189
|
+
# Telemetry code paths require real OTel runtime + live OpenInference
|
|
190
|
+
# instrumentors to exercise meaningfully — covered by integration tests
|
|
191
|
+
# in trus638-testing rather than unit tests here.
|
|
192
|
+
"*/telemetry/*",
|
|
193
|
+
"*/endpoints/frameworks.py",
|
|
194
|
+
"*/models/frameworks.py",
|
|
162
195
|
]
|
|
163
196
|
|
|
164
197
|
[tool.coverage.report]
|
|
@@ -30,6 +30,17 @@ try:
|
|
|
30
30
|
except PackageNotFoundError:
|
|
31
31
|
# Package not installed, use fallback
|
|
32
32
|
__version__ = "0.1.0"
|
|
33
|
+
|
|
34
|
+
# Telemetry (auto_init) is an optional feature.
|
|
35
|
+
# Install with: pip install "trustmodel[telemetry]"
|
|
36
|
+
# If telemetry deps are missing, importing the SDK still works.
|
|
37
|
+
try:
|
|
38
|
+
from .telemetry import auto_init # noqa: F401
|
|
39
|
+
|
|
40
|
+
_HAS_TELEMETRY = True
|
|
41
|
+
except ImportError:
|
|
42
|
+
_HAS_TELEMETRY = False
|
|
43
|
+
|
|
33
44
|
__all__ = [
|
|
34
45
|
"TrustModelClient",
|
|
35
46
|
"TrustModelError",
|
|
@@ -40,3 +51,5 @@ __all__ = [
|
|
|
40
51
|
"InsufficientCreditsError",
|
|
41
52
|
"ConnectionValidationError",
|
|
42
53
|
]
|
|
54
|
+
if _HAS_TELEMETRY:
|
|
55
|
+
__all__.append("auto_init")
|
|
@@ -18,6 +18,7 @@ from .endpoints.batch_jobs import BatchJobsEndpoint
|
|
|
18
18
|
from .endpoints.config import ConfigEndpoint
|
|
19
19
|
from .endpoints.credits import CreditsEndpoint
|
|
20
20
|
from .endpoints.evaluations import EvaluationsEndpoint
|
|
21
|
+
from .endpoints.frameworks import FrameworksEndpoint
|
|
21
22
|
from .endpoints.galileo import GalileoEndpoint
|
|
22
23
|
from .endpoints.models import ModelsEndpoint
|
|
23
24
|
from .exceptions import (
|
|
@@ -120,6 +121,7 @@ class TrustModelClient:
|
|
|
120
121
|
self.batch_jobs = BatchJobsEndpoint(self)
|
|
121
122
|
self.agentic = AgenticEndpoint(self)
|
|
122
123
|
self.galileo = GalileoEndpoint(self)
|
|
124
|
+
self.frameworks = FrameworksEndpoint(self)
|
|
123
125
|
|
|
124
126
|
def _request(
|
|
125
127
|
self,
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Frameworks endpoint wrapper for the TrustModel SDK.
|
|
3
|
+
|
|
4
|
+
Lets users discover available compliance frameworks and domains
|
|
5
|
+
for use with `auto_init(domain=..., frameworks=[...])`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import TYPE_CHECKING, Any, List, Optional
|
|
11
|
+
|
|
12
|
+
from ..models.frameworks import Framework
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from ..client import TrustModelClient
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# Hardcoded list mirrors EvaluationSubTypeChoices on the backend.
|
|
19
|
+
# Update if backend adds new domains.
|
|
20
|
+
_KNOWN_DOMAINS = [
|
|
21
|
+
"fair_lending",
|
|
22
|
+
"hr_bias",
|
|
23
|
+
"healthcare",
|
|
24
|
+
"general_ai",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class FrameworksEndpoint:
|
|
29
|
+
"""
|
|
30
|
+
Interface for compliance framework discovery.
|
|
31
|
+
|
|
32
|
+
Frameworks are organized by domain (framework_type). Use these methods
|
|
33
|
+
to discover what's available before calling auto_init().
|
|
34
|
+
|
|
35
|
+
Example:
|
|
36
|
+
>>> client = TrustModelClient(api_key="tm-...")
|
|
37
|
+
>>> domains = client.frameworks.list_domains()
|
|
38
|
+
>>> # ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
|
|
39
|
+
>>> frameworks = client.frameworks.list(domain="fair_lending")
|
|
40
|
+
>>> for f in frameworks:
|
|
41
|
+
... print(f"{f.slug}: {f.name}")
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, client: TrustModelClient) -> None:
|
|
45
|
+
self._client = client
|
|
46
|
+
|
|
47
|
+
def list_domains(self) -> List[str]:
|
|
48
|
+
"""
|
|
49
|
+
Get list of available domain identifiers.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
List of domain strings. Pass any of these as `domain` to
|
|
53
|
+
`auto_init()` or as the `?framework_type=...` query parameter.
|
|
54
|
+
|
|
55
|
+
Example:
|
|
56
|
+
>>> client.frameworks.list_domains()
|
|
57
|
+
['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
|
|
58
|
+
"""
|
|
59
|
+
return list(_KNOWN_DOMAINS)
|
|
60
|
+
|
|
61
|
+
def list(self, domain: Optional[str] = None) -> List[Framework]:
|
|
62
|
+
"""
|
|
63
|
+
List available compliance frameworks.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
domain: Optional domain filter (e.g., 'fair_lending'). If omitted,
|
|
67
|
+
returns all active frameworks across all domains.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
List of Framework objects. Use the `slug` field as the framework
|
|
71
|
+
identifier when calling auto_init(frameworks=[...]).
|
|
72
|
+
|
|
73
|
+
Example:
|
|
74
|
+
>>> frameworks = client.frameworks.list(domain="fair_lending")
|
|
75
|
+
>>> auto_init(
|
|
76
|
+
... api_key="tm-...",
|
|
77
|
+
... agent_id="my-agent",
|
|
78
|
+
... domain="fair_lending",
|
|
79
|
+
... frameworks=[f.slug for f in frameworks[:2]],
|
|
80
|
+
... )
|
|
81
|
+
"""
|
|
82
|
+
params = {}
|
|
83
|
+
if domain:
|
|
84
|
+
params["framework_type"] = domain
|
|
85
|
+
|
|
86
|
+
response: Any = self._client.get("/api/agent-evaluation/frameworks/", params=params)
|
|
87
|
+
|
|
88
|
+
# API may return a list or a paginated object {results: [...]}
|
|
89
|
+
if isinstance(response, list):
|
|
90
|
+
items = response
|
|
91
|
+
else:
|
|
92
|
+
items = response.get("results", [])
|
|
93
|
+
|
|
94
|
+
return [Framework(**item) for item in items]
|
|
@@ -18,6 +18,7 @@ class EvaluationStatus(str, Enum):
|
|
|
18
18
|
RUNNING = "running"
|
|
19
19
|
COMPLETED = "completed"
|
|
20
20
|
FAILED = "failed"
|
|
21
|
+
PAYMENT_PENDING = "payment_pending"
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
class ApplicationType(str, Enum):
|
|
@@ -30,6 +31,8 @@ class ApplicationType(str, Enum):
|
|
|
30
31
|
CLASSIFICATION = "classification"
|
|
31
32
|
CODE_GENERATION = "code_generation"
|
|
32
33
|
TRANSLATION = "translation"
|
|
34
|
+
EVALUATION_WIZARD = "evaluation_wizard"
|
|
35
|
+
AUTOMATION_AGENT = "automation-agent"
|
|
33
36
|
|
|
34
37
|
|
|
35
38
|
class UserPersona(str, Enum):
|
|
@@ -143,9 +146,9 @@ class Evaluation(BaseModel):
|
|
|
143
146
|
status: EvaluationStatus
|
|
144
147
|
model_identifier: str
|
|
145
148
|
vendor_identifier: Optional[str] = None
|
|
146
|
-
model_config_name: str
|
|
147
|
-
application_type: ApplicationType
|
|
148
|
-
user_personas: List[str]
|
|
149
|
+
model_config_name: Optional[str] = None
|
|
150
|
+
application_type: Optional[ApplicationType] = None
|
|
151
|
+
user_personas: Optional[List[str]] = None
|
|
149
152
|
application_description: Optional[str] = None
|
|
150
153
|
domain_expert_description: Optional[str] = None
|
|
151
154
|
completion_percentage: int = Field(..., ge=0, le=100)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Pydantic models for compliance frameworks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from decimal import Decimal
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Framework(BaseModel):
|
|
12
|
+
"""A compliance framework that can be selected for evaluation."""
|
|
13
|
+
|
|
14
|
+
id: str
|
|
15
|
+
name: str
|
|
16
|
+
subtitle: Optional[str] = ""
|
|
17
|
+
slug: str
|
|
18
|
+
description: Optional[str] = ""
|
|
19
|
+
credits: Decimal = Field(default=Decimal("0"))
|
|
20
|
+
framework_type: Optional[str] = Field(
|
|
21
|
+
default=None,
|
|
22
|
+
description="Domain classification (fair_lending, hr_bias, healthcare, general_ai)",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
class Config:
|
|
26
|
+
# allow extra fields from API to not fail
|
|
27
|
+
extra = "allow"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TrustModel Telemetry — auto-capture AI agent calls via OpenTelemetry.
|
|
3
|
+
|
|
4
|
+
Requires the `telemetry` extra:
|
|
5
|
+
pip install "trustmodel[telemetry]"
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
from trustmodel.telemetry import auto_init
|
|
9
|
+
auto_init(api_key="tm-...", agent_id="my-agent")
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
import opentelemetry # noqa: F401
|
|
14
|
+
except ImportError as exc:
|
|
15
|
+
raise ImportError(
|
|
16
|
+
"TrustModel telemetry requires extra dependencies. "
|
|
17
|
+
'Install with: pip install "trustmodel[telemetry]"'
|
|
18
|
+
) from exc
|
|
19
|
+
|
|
20
|
+
from .auto_init import auto_init, flush
|
|
21
|
+
|
|
22
|
+
__all__ = ["auto_init", "flush"]
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OpenInference semantic convention attribute keys.
|
|
3
|
+
|
|
4
|
+
Reference for the OTLP → agentic trace converter on the gateway side.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
# Span classification
|
|
8
|
+
OPENINFERENCE_SPAN_KIND = "openinference.span.kind"
|
|
9
|
+
|
|
10
|
+
# LLM attributes
|
|
11
|
+
LLM_MODEL_NAME = "llm.model_name"
|
|
12
|
+
LLM_TOKEN_COUNT_PROMPT = "llm.token_count.prompt"
|
|
13
|
+
LLM_TOKEN_COUNT_COMPLETION = "llm.token_count.completion"
|
|
14
|
+
LLM_OUTPUT_MESSAGES = "llm.output_messages"
|
|
15
|
+
LLM_INPUT_MESSAGES = "llm.input_messages"
|
|
16
|
+
LLM_INVOCATION_PARAMETERS = "llm.invocation_parameters"
|
|
17
|
+
|
|
18
|
+
# Tool attributes
|
|
19
|
+
TOOL_NAME = "tool.name"
|
|
20
|
+
TOOL_PARAMETERS = "tool.parameters"
|
|
21
|
+
TOOL_DESCRIPTION = "tool.description"
|
|
22
|
+
|
|
23
|
+
# I/O attributes
|
|
24
|
+
INPUT_VALUE = "input.value"
|
|
25
|
+
INPUT_MIME_TYPE = "input.mime_type"
|
|
26
|
+
OUTPUT_VALUE = "output.value"
|
|
27
|
+
OUTPUT_MIME_TYPE = "output.mime_type"
|
|
28
|
+
|
|
29
|
+
# Retriever attributes
|
|
30
|
+
RETRIEVAL_DOCUMENTS = "retrieval.documents"
|
|
31
|
+
|
|
32
|
+
# Embedding attributes
|
|
33
|
+
EMBEDDING_MODEL_NAME = "embedding.model_name"
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Auto-detect and install OpenInference instrumentors.
|
|
3
|
+
|
|
4
|
+
Each instrumentor wraps a specific AI library (OpenAI, Anthropic, LangChain, etc.)
|
|
5
|
+
to emit OTel spans with OpenInference semantic attributes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from typing import List
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
# (module_path, class_name) — order doesn't matter
|
|
16
|
+
_KNOWN_INSTRUMENTORS = [
|
|
17
|
+
("openinference.instrumentation.openai", "OpenAIInstrumentor"),
|
|
18
|
+
("openinference.instrumentation.anthropic", "AnthropicInstrumentor"),
|
|
19
|
+
("openinference.instrumentation.langchain", "LangChainInstrumentor"),
|
|
20
|
+
("openinference.instrumentation.llama_index", "LlamaIndexInstrumentor"),
|
|
21
|
+
("openinference.instrumentation.bedrock", "BedrockInstrumentor"),
|
|
22
|
+
("openinference.instrumentation.mistralai", "MistralAIInstrumentor"),
|
|
23
|
+
("openinference.instrumentation.groq", "GroqInstrumentor"),
|
|
24
|
+
("openinference.instrumentation.crewai", "CrewAIInstrumentor"),
|
|
25
|
+
("openinference.instrumentation.vertexai", "VertexAIInstrumentor"),
|
|
26
|
+
("openinference.instrumentation.dspy", "DSPyInstrumentor"),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def install_instrumentors() -> List[str]:
|
|
31
|
+
"""
|
|
32
|
+
Attempt to import and instrument each known OpenInference instrumentor.
|
|
33
|
+
|
|
34
|
+
Only instrumentors whose packages are installed will be activated.
|
|
35
|
+
Failures are logged and silently skipped — never crash the customer's app.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
List of successfully installed instrumentor names.
|
|
39
|
+
"""
|
|
40
|
+
installed = []
|
|
41
|
+
for module_path, class_name in _KNOWN_INSTRUMENTORS:
|
|
42
|
+
try:
|
|
43
|
+
module = __import__(module_path, fromlist=[class_name])
|
|
44
|
+
instrumentor_cls = getattr(module, class_name)
|
|
45
|
+
instrumentor_cls().instrument()
|
|
46
|
+
installed.append(class_name)
|
|
47
|
+
logger.debug("TrustModel telemetry: installed %s", class_name)
|
|
48
|
+
except ImportError:
|
|
49
|
+
# Package not installed — expected, skip silently
|
|
50
|
+
pass
|
|
51
|
+
except Exception:
|
|
52
|
+
logger.debug("TrustModel telemetry: failed to install %s", class_name, exc_info=True)
|
|
53
|
+
return installed
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""
|
|
2
|
+
auto_init() — 2-line setup for automatic AI agent trace capture.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
from trustmodel.telemetry import auto_init
|
|
6
|
+
auto_init(api_key="tm-...", agent_id="my-weather-agent")
|
|
7
|
+
|
|
8
|
+
# All subsequent OpenAI/Anthropic/LangChain calls are now traced.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import atexit
|
|
14
|
+
import logging
|
|
15
|
+
import uuid as _uuid
|
|
16
|
+
from typing import TYPE_CHECKING, Any, List, Optional
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
_initialized = False
|
|
24
|
+
_tracer_provider: Optional[SDKTracerProvider] = None
|
|
25
|
+
|
|
26
|
+
ENVIRONMENT_URLS = {
|
|
27
|
+
"local": "http://localhost:8000",
|
|
28
|
+
"qa": "https://api-trustmodel.pdxqa.com",
|
|
29
|
+
"production": "https://api.trustmodel.ai",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def auto_init(
|
|
34
|
+
api_key: str,
|
|
35
|
+
*,
|
|
36
|
+
agent_id: str,
|
|
37
|
+
domain: str,
|
|
38
|
+
frameworks: List[str],
|
|
39
|
+
service_name: str = "default",
|
|
40
|
+
environment: str = "production",
|
|
41
|
+
base_url: Optional[str] = None,
|
|
42
|
+
) -> List[str]:
|
|
43
|
+
"""
|
|
44
|
+
Initialize automatic OTel trace capture and export to TrustModel.
|
|
45
|
+
|
|
46
|
+
Safe to call from any Python app — wraps everything in try/except
|
|
47
|
+
so it never crashes the customer's application.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
api_key: TrustModel API key (tm-...).
|
|
51
|
+
agent_id: Unique identifier for this agent. All traces are grouped
|
|
52
|
+
and evaluated per agent_id.
|
|
53
|
+
domain: Domain classification — one of 'fair_lending', 'hr_bias',
|
|
54
|
+
'healthcare', 'general_ai'. Required. Use
|
|
55
|
+
TrustModelClient.frameworks.list_domains() to discover.
|
|
56
|
+
frameworks: List of compliance framework slugs (e.g.,
|
|
57
|
+
['eu-ai-act-high-risk', 'iso-42001']). Required and must
|
|
58
|
+
be non-empty. Use TrustModelClient.frameworks.list(domain=...)
|
|
59
|
+
to discover available slugs.
|
|
60
|
+
service_name: Logical service name for this application.
|
|
61
|
+
environment: One of 'local', 'qa', 'production'.
|
|
62
|
+
base_url: Explicit base URL (overrides environment).
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
List of instrumentor names that were successfully installed.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
ValueError: If `domain` is empty or `frameworks` is empty/missing.
|
|
69
|
+
"""
|
|
70
|
+
global _initialized, _tracer_provider
|
|
71
|
+
|
|
72
|
+
if not domain:
|
|
73
|
+
raise ValueError("auto_init() requires a non-empty 'domain' argument.")
|
|
74
|
+
if not frameworks:
|
|
75
|
+
raise ValueError(
|
|
76
|
+
"auto_init() requires a non-empty 'frameworks' list. "
|
|
77
|
+
"Use TrustModelClient.frameworks.list(domain=...) to discover slugs."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if _initialized:
|
|
81
|
+
logger.debug("TrustModel telemetry already initialized, skipping.")
|
|
82
|
+
return []
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
return _do_init(api_key, agent_id, domain, frameworks, service_name, environment, base_url)
|
|
86
|
+
except Exception:
|
|
87
|
+
logger.debug("TrustModel telemetry: auto_init failed", exc_info=True)
|
|
88
|
+
return []
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _do_init(
|
|
92
|
+
api_key: str,
|
|
93
|
+
agent_id: str,
|
|
94
|
+
domain: str,
|
|
95
|
+
frameworks: List[str],
|
|
96
|
+
service_name: str,
|
|
97
|
+
environment: str,
|
|
98
|
+
base_url: Optional[str],
|
|
99
|
+
) -> List[str]:
|
|
100
|
+
global _initialized, _tracer_provider
|
|
101
|
+
|
|
102
|
+
from opentelemetry import trace
|
|
103
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
104
|
+
from opentelemetry.sdk.resources import Resource
|
|
105
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
106
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
107
|
+
|
|
108
|
+
session_id = str(_uuid.uuid4())
|
|
109
|
+
|
|
110
|
+
# Resolve endpoint — with explicit endpoint param, the exporter
|
|
111
|
+
# does NOT append /v1/traces, so we include the full path.
|
|
112
|
+
if base_url:
|
|
113
|
+
endpoint = base_url.rstrip("/") + "/sdk/v1/otel/v1/traces"
|
|
114
|
+
else:
|
|
115
|
+
if environment not in ENVIRONMENT_URLS:
|
|
116
|
+
raise ValueError(
|
|
117
|
+
f"Invalid environment '{environment}'. "
|
|
118
|
+
f"Must be one of: {', '.join(ENVIRONMENT_URLS)}"
|
|
119
|
+
)
|
|
120
|
+
endpoint = ENVIRONMENT_URLS[environment] + "/sdk/v1/otel/v1/traces"
|
|
121
|
+
|
|
122
|
+
exporter = OTLPSpanExporter(
|
|
123
|
+
endpoint=endpoint,
|
|
124
|
+
headers={"X-API-Key": api_key},
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
# Normalize domain + frameworks (both required, validated upstream)
|
|
128
|
+
domain_str = domain
|
|
129
|
+
frameworks_str = ",".join(frameworks)
|
|
130
|
+
|
|
131
|
+
# Wrap exporter so agent_id + session_id + domain + frameworks are always
|
|
132
|
+
# injected as resource attributes, even when piggybacking on a customer's
|
|
133
|
+
# existing TracerProvider.
|
|
134
|
+
wrapped_exporter = _make_trustmodel_exporter(
|
|
135
|
+
exporter, agent_id, session_id, domain_str, frameworks_str
|
|
136
|
+
)
|
|
137
|
+
processor = BatchSpanProcessor(wrapped_exporter)
|
|
138
|
+
|
|
139
|
+
current_provider = trace.get_tracer_provider()
|
|
140
|
+
|
|
141
|
+
if isinstance(current_provider, TracerProvider):
|
|
142
|
+
# Customer already has OTel set up — just add our exporter
|
|
143
|
+
current_provider.add_span_processor(processor)
|
|
144
|
+
_tracer_provider = current_provider
|
|
145
|
+
logger.debug("TrustModel telemetry: added exporter to existing TracerProvider")
|
|
146
|
+
else:
|
|
147
|
+
# No existing OTel — create our own provider
|
|
148
|
+
resource_attrs = {
|
|
149
|
+
"service.name": service_name,
|
|
150
|
+
"trustmodel.agent_id": agent_id,
|
|
151
|
+
"trustmodel.session_id": session_id,
|
|
152
|
+
}
|
|
153
|
+
if domain_str:
|
|
154
|
+
resource_attrs["trustmodel.domain"] = domain_str
|
|
155
|
+
if frameworks_str:
|
|
156
|
+
resource_attrs["trustmodel.frameworks"] = frameworks_str
|
|
157
|
+
resource = Resource.create(resource_attrs)
|
|
158
|
+
provider = TracerProvider(resource=resource)
|
|
159
|
+
provider.add_span_processor(processor)
|
|
160
|
+
trace.set_tracer_provider(provider)
|
|
161
|
+
_tracer_provider = provider
|
|
162
|
+
logger.debug("TrustModel telemetry: created new TracerProvider")
|
|
163
|
+
|
|
164
|
+
# Auto-detect and install instrumentors
|
|
165
|
+
from ._instrumentors import install_instrumentors
|
|
166
|
+
|
|
167
|
+
installed = install_instrumentors()
|
|
168
|
+
|
|
169
|
+
_initialized = True
|
|
170
|
+
atexit.register(_shutdown)
|
|
171
|
+
|
|
172
|
+
if installed:
|
|
173
|
+
logger.info("TrustModel telemetry: active — instrumentors: %s", ", ".join(installed))
|
|
174
|
+
else:
|
|
175
|
+
logger.info(
|
|
176
|
+
"TrustModel telemetry: active — no instrumentors found (install openinference-instrumentation-*)"
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
return installed
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def flush(
|
|
183
|
+
api_key: str,
|
|
184
|
+
*,
|
|
185
|
+
environment: str = "production",
|
|
186
|
+
base_url: Optional[str] = None,
|
|
187
|
+
) -> None:
|
|
188
|
+
"""
|
|
189
|
+
Flush buffered OTel traces to trigger evaluation.
|
|
190
|
+
|
|
191
|
+
Call this when the agent session is complete.
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
api_key: TrustModel API key.
|
|
195
|
+
environment: One of 'local', 'qa', 'production'.
|
|
196
|
+
base_url: Explicit base URL (overrides environment).
|
|
197
|
+
"""
|
|
198
|
+
import requests
|
|
199
|
+
|
|
200
|
+
if base_url:
|
|
201
|
+
url = base_url.rstrip("/") + "/sdk/v1/otel/v1/flush"
|
|
202
|
+
else:
|
|
203
|
+
url = (
|
|
204
|
+
ENVIRONMENT_URLS.get(environment, ENVIRONMENT_URLS["production"])
|
|
205
|
+
+ "/sdk/v1/otel/v1/flush"
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# Force-flush any pending spans first
|
|
209
|
+
if _tracer_provider is not None:
|
|
210
|
+
try:
|
|
211
|
+
_tracer_provider.force_flush(timeout_millis=5000)
|
|
212
|
+
except Exception:
|
|
213
|
+
logger.debug("TrustModel telemetry: force_flush failed", exc_info=True)
|
|
214
|
+
|
|
215
|
+
response = requests.post(
|
|
216
|
+
url,
|
|
217
|
+
headers={"X-API-Key": api_key, "Content-Type": "application/json"},
|
|
218
|
+
json={},
|
|
219
|
+
timeout=30,
|
|
220
|
+
)
|
|
221
|
+
response.raise_for_status()
|
|
222
|
+
data = response.json()
|
|
223
|
+
logger.info(
|
|
224
|
+
"TrustModel telemetry: flushed %d evaluation(s)",
|
|
225
|
+
data.get("count", 0),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _make_trustmodel_exporter(
|
|
230
|
+
inner_exporter: Any,
|
|
231
|
+
agent_id: str,
|
|
232
|
+
session_id: str,
|
|
233
|
+
domain: str = "",
|
|
234
|
+
frameworks: str = "",
|
|
235
|
+
) -> Any:
|
|
236
|
+
"""
|
|
237
|
+
Build a SpanExporter subclass that wraps the inner exporter and injects
|
|
238
|
+
TrustModel resource attributes on every exported span. Implemented as a
|
|
239
|
+
factory so SpanExporter can be a runtime base class without polluting
|
|
240
|
+
module-level imports (telemetry extras may be missing at import time).
|
|
241
|
+
"""
|
|
242
|
+
from opentelemetry.sdk.resources import Resource
|
|
243
|
+
from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter
|
|
244
|
+
|
|
245
|
+
class _TrustModelExporter(_SpanExporter):
|
|
246
|
+
def __init__(self) -> None:
|
|
247
|
+
self._inner = inner_exporter
|
|
248
|
+
self._agent_id = agent_id
|
|
249
|
+
self._session_id = session_id
|
|
250
|
+
self._domain = domain
|
|
251
|
+
self._frameworks = frameworks
|
|
252
|
+
|
|
253
|
+
def export(self, spans): # type: ignore[no-untyped-def]
|
|
254
|
+
attrs = {
|
|
255
|
+
"trustmodel.agent_id": self._agent_id,
|
|
256
|
+
"trustmodel.session_id": self._session_id,
|
|
257
|
+
}
|
|
258
|
+
if self._domain:
|
|
259
|
+
attrs["trustmodel.domain"] = self._domain
|
|
260
|
+
if self._frameworks:
|
|
261
|
+
attrs["trustmodel.frameworks"] = self._frameworks
|
|
262
|
+
patched_resource = Resource.create(attrs)
|
|
263
|
+
|
|
264
|
+
patched = []
|
|
265
|
+
for span in spans:
|
|
266
|
+
merged = span.resource.merge(patched_resource)
|
|
267
|
+
span._resource = merged
|
|
268
|
+
patched.append(span)
|
|
269
|
+
|
|
270
|
+
return self._inner.export(patched)
|
|
271
|
+
|
|
272
|
+
def shutdown(self) -> None:
|
|
273
|
+
self._inner.shutdown()
|
|
274
|
+
|
|
275
|
+
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
276
|
+
if hasattr(self._inner, "force_flush"):
|
|
277
|
+
return bool(self._inner.force_flush(timeout_millis))
|
|
278
|
+
return True
|
|
279
|
+
|
|
280
|
+
return _TrustModelExporter()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _shutdown() -> None:
|
|
284
|
+
"""Graceful shutdown — flush pending spans on process exit."""
|
|
285
|
+
if _tracer_provider is not None:
|
|
286
|
+
try:
|
|
287
|
+
_tracer_provider.shutdown()
|
|
288
|
+
except Exception:
|
|
289
|
+
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|