trustmodel 2.1.0__tar.gz → 2.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. trustmodel-2.1.0/README.md → trustmodel-2.2.1/PKG-INFO +243 -0
  2. trustmodel-2.1.0/PKG-INFO → trustmodel-2.2.1/README.md +175 -54
  3. {trustmodel-2.1.0 → trustmodel-2.2.1}/pyproject.toml +34 -1
  4. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/__init__.py +13 -0
  5. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/client.py +2 -0
  6. trustmodel-2.2.1/src/trustmodel/endpoints/frameworks.py +94 -0
  7. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/evaluation.py +6 -3
  8. trustmodel-2.2.1/src/trustmodel/models/frameworks.py +27 -0
  9. trustmodel-2.2.1/src/trustmodel/telemetry/__init__.py +22 -0
  10. trustmodel-2.2.1/src/trustmodel/telemetry/_constants.py +33 -0
  11. trustmodel-2.2.1/src/trustmodel/telemetry/_instrumentors.py +53 -0
  12. trustmodel-2.2.1/src/trustmodel/telemetry/auto_init.py +289 -0
  13. {trustmodel-2.1.0 → trustmodel-2.2.1}/.gitignore +0 -0
  14. {trustmodel-2.1.0 → trustmodel-2.2.1}/LICENSE +0 -0
  15. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/compat.py +0 -0
  16. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/__init__.py +0 -0
  17. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/agentic.py +0 -0
  18. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/batch_jobs.py +0 -0
  19. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/config.py +0 -0
  20. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/credits.py +0 -0
  21. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/evaluations.py +0 -0
  22. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/galileo.py +0 -0
  23. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/endpoints/models.py +0 -0
  24. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/exceptions.py +0 -0
  25. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/__init__.py +0 -0
  26. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/agentic.py +0 -0
  27. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/batch_job.py +0 -0
  28. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/credits.py +0 -0
  29. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/galileo.py +0 -0
  30. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/models/models.py +0 -0
  31. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/utils/__init__.py +0 -0
  32. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/utils/validation.py +0 -0
  33. {trustmodel-2.1.0 → trustmodel-2.2.1}/src/trustmodel/utils/version.py +0 -0
@@ -1,3 +1,71 @@
1
+ Metadata-Version: 2.4
2
+ Name: trustmodel
3
+ Version: 2.2.1
4
+ Summary: Official Python SDK for TrustModel AI evaluation platform
5
+ Project-URL: Homepage, https://trustmodel.ai
6
+ Author-email: TrustModel <info@predixtions.com>
7
+ License: Proprietary - TrustModel Python SDK License
8
+ License-File: LICENSE
9
+ Keywords: ai,api,evaluation,sdk,trustmodel
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: Other/Proprietary License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.7
16
+ Classifier: Programming Language :: Python :: 3.8
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
+ Requires-Python: >=3.7
24
+ Requires-Dist: eval-type-backport>=0.2.0; python_version < '3.10'
25
+ Requires-Dist: importlib-metadata>=4.0.0; python_version < '3.8'
26
+ Requires-Dist: pydantic<2.0.0,>=1.10.0; python_version < '3.8'
27
+ Requires-Dist: pydantic<3.0.0,>=2.0.0; python_version >= '3.8'
28
+ Requires-Dist: python-dateutil>=2.8.0
29
+ Requires-Dist: requests<3.0.0,>=2.25.0
30
+ Requires-Dist: tqdm>=4.60.0
31
+ Requires-Dist: typing-extensions>=4.0.0
32
+ Provides-Extra: dev
33
+ Requires-Dist: black<23.0.0,>=22.0.0; (python_version < '3.8') and extra == 'dev'
34
+ Requires-Dist: black>=23.0.0; (python_version >= '3.8') and extra == 'dev'
35
+ Requires-Dist: bump2version>=1.0.1; extra == 'dev'
36
+ Requires-Dist: importlib-metadata>=4.0.0; (python_version < '3.8') and extra == 'dev'
37
+ Requires-Dist: isort<5.12.0,>=5.11.0; (python_version < '3.8') and extra == 'dev'
38
+ Requires-Dist: isort>=5.12.0; (python_version >= '3.8') and extra == 'dev'
39
+ Requires-Dist: mypy<1.5.0,>=1.0.0; (python_version < '3.8') and extra == 'dev'
40
+ Requires-Dist: mypy>=1.5.0; (python_version >= '3.8') and extra == 'dev'
41
+ Requires-Dist: pytest-cov<5.0.0,>=4.0.0; (python_version < '3.8') and extra == 'dev'
42
+ Requires-Dist: pytest-cov>=4.0.0; (python_version >= '3.8') and extra == 'dev'
43
+ Requires-Dist: pytest<8.0.0,>=7.0.0; (python_version < '3.8') and extra == 'dev'
44
+ Requires-Dist: pytest>=7.0.0; (python_version >= '3.8') and extra == 'dev'
45
+ Requires-Dist: ruff<0.1.0,>=0.0.277; (python_version < '3.8') and extra == 'dev'
46
+ Requires-Dist: ruff>=0.1.0; (python_version >= '3.8') and extra == 'dev'
47
+ Requires-Dist: types-requests>=2.25.0; extra == 'dev'
48
+ Requires-Dist: types-tqdm>=4.60.0; extra == 'dev'
49
+ Provides-Extra: docs
50
+ Requires-Dist: sphinx-autodoc-typehints>=1.24.0; extra == 'docs'
51
+ Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == 'docs'
52
+ Requires-Dist: sphinx>=7.0.0; extra == 'docs'
53
+ Provides-Extra: telemetry
54
+ Requires-Dist: openinference-instrumentation-anthropic>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
55
+ Requires-Dist: openinference-instrumentation-bedrock>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
56
+ Requires-Dist: openinference-instrumentation-crewai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
57
+ Requires-Dist: openinference-instrumentation-dspy>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
58
+ Requires-Dist: openinference-instrumentation-groq>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
59
+ Requires-Dist: openinference-instrumentation-langchain>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
60
+ Requires-Dist: openinference-instrumentation-llama-index>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
61
+ Requires-Dist: openinference-instrumentation-mistralai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
62
+ Requires-Dist: openinference-instrumentation-openai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
63
+ Requires-Dist: openinference-instrumentation-vertexai>=0.1.0; (python_version >= '3.10') and extra == 'telemetry'
64
+ Requires-Dist: opentelemetry-api>=1.20.0; (python_version >= '3.10') and extra == 'telemetry'
65
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.20.0; (python_version >= '3.10') and extra == 'telemetry'
66
+ Requires-Dist: opentelemetry-sdk>=1.20.0; (python_version >= '3.10') and extra == 'telemetry'
67
+ Description-Content-Type: text/markdown
68
+
1
69
  <p align="center">
2
70
  <a href="https://www.trustmodel.ai">
3
71
  <img src="https://www.trustmodel.ai/assets/trustmodel-wordmark-CfuXSOoK.svg" alt="TrustModel" width="400">
@@ -23,6 +91,36 @@
23
91
 
24
92
  Evaluate AI models for safety, bias, and performance with a simple, intuitive interface.
25
93
 
94
+ ## Table of Contents
95
+
96
+ - [Features](#features)
97
+ - [Installation](#installation)
98
+ - [Prerequisites](#prerequisites)
99
+ - [Quick Start](#quick-start)
100
+ - [Authentication](#authentication)
101
+ - [Evaluation Modes](#evaluation-modes)
102
+ - [Getting Available Vendors](#getting-available-vendors)
103
+ - [Getting Available Models](#getting-available-models)
104
+ - [Platform Key (Default)](#platform-key-default)
105
+ - [BYOK (Bring Your Own Key)](#byok-bring-your-own-key)
106
+ - [Custom Endpoint](#custom-endpoint)
107
+ - [Core Concepts](#core-concepts)
108
+ - [Models](#models)
109
+ - [Evaluations](#evaluations)
110
+ - [Configuration](#configuration)
111
+ - [Credits Management](#credits-management)
112
+ - [Error Handling](#error-handling)
113
+ - [Rate Limiting](#rate-limiting)
114
+ - [Webhook Notifications](#webhook-notifications)
115
+ - [Advanced Usage](#advanced-usage)
116
+ - [Framework Integration](#framework-integration)
117
+ - [Zero-Config Auto-Capture (`auto_init`)](#zero-config-auto-capture-auto_init)
118
+ - [Agentic Trace Evaluation](#agentic-trace-evaluation)
119
+ - [Galileo Integration](#galileo-integration)
120
+ - [Requirements](#requirements)
121
+ - [Support](#support)
122
+ - [License](#license)
123
+
26
124
  ## Features
27
125
 
28
126
  - 🚀 **Simple Interface**: Easy-to-use client for all TrustModel operations
@@ -31,13 +129,24 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
31
129
  - 🔄 **Reliable**: Automatic retries and comprehensive error handling
32
130
  - 📊 **Comprehensive**: Support for all evaluation types and configurations
33
131
  - 🌍 **Framework Agnostic**: Works with any Python framework or standalone scripts
132
+ - 🛰️ **Zero-Config Auto-Capture**: Add two lines (`auto_init`) and every OpenAI / Anthropic / LangChain / CrewAI / etc. call is automatically traced and evaluated
34
133
 
35
134
  ## Installation
36
135
 
136
+ Core SDK (works on Python 3.7+):
137
+
37
138
  ```bash
38
139
  pip install trustmodel
39
140
  ```
40
141
 
142
+ Add the `telemetry` extra to enable zero-config auto-capture of AI agent calls (Python 3.10+ required):
143
+
144
+ ```bash
145
+ pip install "trustmodel[telemetry]"
146
+ ```
147
+
148
+ The `telemetry` extra installs OpenTelemetry plus all OpenInference instrumentors (OpenAI, Anthropic, LangChain, LlamaIndex, Bedrock, Mistral, Groq, CrewAI, VertexAI, DSPy). See [Zero-Config Auto-Capture](#zero-config-auto-capture-auto_init) below.
149
+
41
150
  ## Prerequisites
42
151
 
43
152
  Before using the SDK, you **must** complete the following setup in the [TrustModel Dashboard](https://app.trustmodel.ai/app/keys):
@@ -1008,6 +1117,140 @@ def evaluate():
1008
1117
  })
1009
1118
  ```
1010
1119
 
1120
+ ## Zero-Config Auto-Capture (`auto_init`)
1121
+
1122
+ Capture every LLM and tool call your AI agent makes — without changing your existing code. Two lines, and TrustModel takes care of trace collection, batching, transport, and evaluation.
1123
+
1124
+ ### How It Works
1125
+
1126
+ 1. You call `auto_init(api_key, agent_id, ...)` once at startup.
1127
+ 2. The SDK auto-detects which AI libraries you're using (OpenAI, Anthropic, LangChain, etc.) and installs an OpenInference instrumentor for each.
1128
+ 3. Every subsequent LLM / tool call is captured as an OpenTelemetry span and streamed to the TrustModel gateway.
1129
+ 4. The TrustModel Control Plane buffers traces server-side, groups them by `agent_id` + `domain` + `frameworks`, and runs evaluation on the schedule you configure (daily, weekly, or monthly).
1130
+
1131
+ ### Installation
1132
+
1133
+ ```bash
1134
+ pip install "trustmodel[telemetry]"
1135
+ ```
1136
+
1137
+ This installs OpenTelemetry plus the OpenInference instrumentors. **Python 3.10+ required** for the telemetry extra.
1138
+
1139
+ If you only want the core SDK (e.g., `client.evaluations.create()`), `pip install trustmodel` works on Python 3.7+ but `auto_init` will not be available.
1140
+
1141
+ ### Quick Start
1142
+
1143
+ ```python
1144
+ from trustmodel.telemetry import auto_init
1145
+
1146
+ auto_init(
1147
+ api_key="tm-...", # your TrustModel API key
1148
+ agent_id="my-customer-support-agent", # any identifier you choose
1149
+ domain="general_ai", # fair_lending | hr_bias | healthcare | general_ai
1150
+ frameworks=["nist-ai-rmf"], # one or more compliance framework slugs
1151
+ )
1152
+
1153
+ # Your existing agent code — no changes needed
1154
+ import openai
1155
+ client = openai.OpenAI()
1156
+ client.chat.completions.create(
1157
+ model="gpt-4o-mini",
1158
+ messages=[{"role": "user", "content": "Hello"}],
1159
+ )
1160
+ ```
1161
+
1162
+ That's it. The OpenAI call is automatically captured and streamed to TrustModel.
1163
+
1164
+ ### Discovering Domains and Frameworks
1165
+
1166
+ Use the `client.frameworks` endpoint to list available domains and the compliance frameworks within each:
1167
+
1168
+ ```python
1169
+ from trustmodel import TrustModelClient
1170
+
1171
+ client = TrustModelClient(api_key="tm-...")
1172
+
1173
+ # All available domains
1174
+ print(client.frameworks.list_domains())
1175
+ # ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
1176
+
1177
+ # Frameworks for a specific domain
1178
+ for f in client.frameworks.list(domain="fair_lending"):
1179
+ print(f"{f.slug}: {f.name} ({f.credits} credits)")
1180
+ ```
1181
+
1182
+ Use the `slug` values when calling `auto_init(frameworks=[...])`.
1183
+
1184
+ ### Supported AI Libraries
1185
+
1186
+ `auto_init` will automatically activate instrumentation for any of these libraries that you have installed:
1187
+
1188
+ | Library | Auto-detected via |
1189
+ |---------|-------------------|
1190
+ | OpenAI | `openinference-instrumentation-openai` |
1191
+ | Anthropic (Claude) | `openinference-instrumentation-anthropic` |
1192
+ | LangChain | `openinference-instrumentation-langchain` |
1193
+ | LlamaIndex | `openinference-instrumentation-llama-index` |
1194
+ | AWS Bedrock | `openinference-instrumentation-bedrock` |
1195
+ | Mistral AI | `openinference-instrumentation-mistralai` |
1196
+ | Groq | `openinference-instrumentation-groq` |
1197
+ | CrewAI | `openinference-instrumentation-crewai` |
1198
+ | Vertex AI | `openinference-instrumentation-vertexai` |
1199
+ | DSPy | `openinference-instrumentation-dspy` |
1200
+
1201
+ All ten are installed by `pip install "trustmodel[telemetry]"`. The instrumentors are no-ops when the underlying library isn't being used — there's no overhead.
1202
+
1203
+ ### Existing OpenTelemetry Setup (Datadog, Jaeger, Honeycomb, Arize)
1204
+
1205
+ If your application already has OpenTelemetry configured for another observability backend, `auto_init` detects this and **adds TrustModel as an additional exporter** rather than replacing your setup.
1206
+
1207
+ ```python
1208
+ # Your existing OTel setup (e.g., Datadog APM, Jaeger, Honeycomb)
1209
+ from opentelemetry import trace
1210
+ from opentelemetry.sdk.trace import TracerProvider
1211
+ provider = TracerProvider(...)
1212
+ trace.set_tracer_provider(provider)
1213
+
1214
+ # Then add TrustModel — both backends now receive every span
1215
+ from trustmodel.telemetry import auto_init
1216
+ auto_init(api_key="tm-...", agent_id="my-agent")
1217
+ ```
1218
+
1219
+ Spans fan out to both your existing exporter and TrustModel's. Nothing in your existing pipeline is modified or replaced.
1220
+
1221
+ ### Parameters
1222
+
1223
+ | Parameter | Required | Description |
1224
+ |-----------|----------|-------------|
1225
+ | `api_key` | yes | Your TrustModel API key (`tm-...`) |
1226
+ | `agent_id` | yes | Any string you choose. All traces sharing this `agent_id` are grouped together for evaluation. |
1227
+ | `domain` | yes | One of `fair_lending`, `hr_bias`, `healthcare`, `general_ai`. Determines which evaluators run. |
1228
+ | `frameworks` | yes | List of compliance framework slugs (e.g., `["ecoa-regb", "fcra"]`). Use `client.frameworks.list(domain=...)` to discover. |
1229
+ | `service_name` | no | Logical service name (default: `"default"`) |
1230
+
1231
+ ### Schedule and Evaluation
1232
+
1233
+ Configure when buffered traces are evaluated in the [Control Plane dashboard](https://app.trustmodel.ai):
1234
+
1235
+ - **Manual** — only when you click "Trigger Evaluation Now"
1236
+ - **Daily / Weekly / Monthly** — automatically via cron at midnight UTC
1237
+
1238
+ Each evaluation produces:
1239
+ - An overall trust score
1240
+ - Per-category breakdowns (safety, fairness, accuracy, etc.)
1241
+ - Findings and recommendations
1242
+
1243
+ ### Failure Modes (Safe by Design)
1244
+
1245
+ `auto_init` wraps everything in `try/except`. If anything fails — missing dependencies, network issues, an instrumentor crash — your application keeps running. Telemetry is best-effort; nothing TrustModel does will break your agent.
1246
+
1247
+ If the `[telemetry]` extras aren't installed and you try to import the telemetry module directly, you get a clear error:
1248
+
1249
+ ```
1250
+ ImportError: TrustModel telemetry requires extra dependencies.
1251
+ Install with: pip install "trustmodel[telemetry]"
1252
+ ```
1253
+
1011
1254
  ## Agentic Trace Evaluation
1012
1255
 
1013
1256
  Evaluate AI agent execution traces for safety, reasoning quality, tool usage, and goal completion. Upload a JSON or JSONL trace file and get scored across 14 dimensions.
@@ -1,57 +1,3 @@
1
- Metadata-Version: 2.4
2
- Name: trustmodel
3
- Version: 2.1.0
4
- Summary: Official Python SDK for TrustModel AI evaluation platform
5
- Project-URL: Homepage, https://trustmodel.ai
6
- Author-email: TrustModel <info@predixtions.com>
7
- License: Proprietary - TrustModel Python SDK License
8
- License-File: LICENSE
9
- Keywords: ai,api,evaluation,sdk,trustmodel
10
- Classifier: Development Status :: 4 - Beta
11
- Classifier: Intended Audience :: Developers
12
- Classifier: License :: Other/Proprietary License
13
- Classifier: Operating System :: OS Independent
14
- Classifier: Programming Language :: Python :: 3
15
- Classifier: Programming Language :: Python :: 3.7
16
- Classifier: Programming Language :: Python :: 3.8
17
- Classifier: Programming Language :: Python :: 3.9
18
- Classifier: Programming Language :: Python :: 3.10
19
- Classifier: Programming Language :: Python :: 3.11
20
- Classifier: Programming Language :: Python :: 3.12
21
- Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
- Requires-Python: >=3.7
24
- Requires-Dist: eval-type-backport>=0.2.0; python_version < '3.10'
25
- Requires-Dist: importlib-metadata>=4.0.0; python_version < '3.8'
26
- Requires-Dist: pydantic<2.0.0,>=1.10.0; python_version < '3.8'
27
- Requires-Dist: pydantic<3.0.0,>=2.0.0; python_version >= '3.8'
28
- Requires-Dist: python-dateutil>=2.8.0
29
- Requires-Dist: requests<3.0.0,>=2.25.0
30
- Requires-Dist: tqdm>=4.60.0
31
- Requires-Dist: typing-extensions>=4.0.0
32
- Provides-Extra: dev
33
- Requires-Dist: black<23.0.0,>=22.0.0; (python_version < '3.8') and extra == 'dev'
34
- Requires-Dist: black>=23.0.0; (python_version >= '3.8') and extra == 'dev'
35
- Requires-Dist: bump2version>=1.0.1; extra == 'dev'
36
- Requires-Dist: importlib-metadata>=4.0.0; (python_version < '3.8') and extra == 'dev'
37
- Requires-Dist: isort<5.12.0,>=5.11.0; (python_version < '3.8') and extra == 'dev'
38
- Requires-Dist: isort>=5.12.0; (python_version >= '3.8') and extra == 'dev'
39
- Requires-Dist: mypy<1.5.0,>=1.0.0; (python_version < '3.8') and extra == 'dev'
40
- Requires-Dist: mypy>=1.5.0; (python_version >= '3.8') and extra == 'dev'
41
- Requires-Dist: pytest-cov<5.0.0,>=4.0.0; (python_version < '3.8') and extra == 'dev'
42
- Requires-Dist: pytest-cov>=4.0.0; (python_version >= '3.8') and extra == 'dev'
43
- Requires-Dist: pytest<8.0.0,>=7.0.0; (python_version < '3.8') and extra == 'dev'
44
- Requires-Dist: pytest>=7.0.0; (python_version >= '3.8') and extra == 'dev'
45
- Requires-Dist: ruff<0.1.0,>=0.0.277; (python_version < '3.8') and extra == 'dev'
46
- Requires-Dist: ruff>=0.1.0; (python_version >= '3.8') and extra == 'dev'
47
- Requires-Dist: types-requests>=2.25.0; extra == 'dev'
48
- Requires-Dist: types-tqdm>=4.60.0; extra == 'dev'
49
- Provides-Extra: docs
50
- Requires-Dist: sphinx-autodoc-typehints>=1.24.0; extra == 'docs'
51
- Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == 'docs'
52
- Requires-Dist: sphinx>=7.0.0; extra == 'docs'
53
- Description-Content-Type: text/markdown
54
-
55
1
  <p align="center">
56
2
  <a href="https://www.trustmodel.ai">
57
3
  <img src="https://www.trustmodel.ai/assets/trustmodel-wordmark-CfuXSOoK.svg" alt="TrustModel" width="400">
@@ -77,6 +23,36 @@ Description-Content-Type: text/markdown
77
23
 
78
24
  Evaluate AI models for safety, bias, and performance with a simple, intuitive interface.
79
25
 
26
+ ## Table of Contents
27
+
28
+ - [Features](#features)
29
+ - [Installation](#installation)
30
+ - [Prerequisites](#prerequisites)
31
+ - [Quick Start](#quick-start)
32
+ - [Authentication](#authentication)
33
+ - [Evaluation Modes](#evaluation-modes)
34
+ - [Getting Available Vendors](#getting-available-vendors)
35
+ - [Getting Available Models](#getting-available-models)
36
+ - [Platform Key (Default)](#platform-key-default)
37
+ - [BYOK (Bring Your Own Key)](#byok-bring-your-own-key)
38
+ - [Custom Endpoint](#custom-endpoint)
39
+ - [Core Concepts](#core-concepts)
40
+ - [Models](#models)
41
+ - [Evaluations](#evaluations)
42
+ - [Configuration](#configuration)
43
+ - [Credits Management](#credits-management)
44
+ - [Error Handling](#error-handling)
45
+ - [Rate Limiting](#rate-limiting)
46
+ - [Webhook Notifications](#webhook-notifications)
47
+ - [Advanced Usage](#advanced-usage)
48
+ - [Framework Integration](#framework-integration)
49
+ - [Zero-Config Auto-Capture (`auto_init`)](#zero-config-auto-capture-auto_init)
50
+ - [Agentic Trace Evaluation](#agentic-trace-evaluation)
51
+ - [Galileo Integration](#galileo-integration)
52
+ - [Requirements](#requirements)
53
+ - [Support](#support)
54
+ - [License](#license)
55
+
80
56
  ## Features
81
57
 
82
58
  - 🚀 **Simple Interface**: Easy-to-use client for all TrustModel operations
@@ -85,13 +61,24 @@ Evaluate AI models for safety, bias, and performance with a simple, intuitive in
85
61
  - 🔄 **Reliable**: Automatic retries and comprehensive error handling
86
62
  - 📊 **Comprehensive**: Support for all evaluation types and configurations
87
63
  - 🌍 **Framework Agnostic**: Works with any Python framework or standalone scripts
64
+ - 🛰️ **Zero-Config Auto-Capture**: Add two lines (`auto_init`) and every OpenAI / Anthropic / LangChain / CrewAI / etc. call is automatically traced and evaluated
88
65
 
89
66
  ## Installation
90
67
 
68
+ Core SDK (works on Python 3.7+):
69
+
91
70
  ```bash
92
71
  pip install trustmodel
93
72
  ```
94
73
 
74
+ Add the `telemetry` extra to enable zero-config auto-capture of AI agent calls (Python 3.10+ required):
75
+
76
+ ```bash
77
+ pip install "trustmodel[telemetry]"
78
+ ```
79
+
80
+ The `telemetry` extra installs OpenTelemetry plus all OpenInference instrumentors (OpenAI, Anthropic, LangChain, LlamaIndex, Bedrock, Mistral, Groq, CrewAI, VertexAI, DSPy). See [Zero-Config Auto-Capture](#zero-config-auto-capture-auto_init) below.
81
+
95
82
  ## Prerequisites
96
83
 
97
84
  Before using the SDK, you **must** complete the following setup in the [TrustModel Dashboard](https://app.trustmodel.ai/app/keys):
@@ -1062,6 +1049,140 @@ def evaluate():
1062
1049
  })
1063
1050
  ```
1064
1051
 
1052
+ ## Zero-Config Auto-Capture (`auto_init`)
1053
+
1054
+ Capture every LLM and tool call your AI agent makes — without changing your existing code. Two lines, and TrustModel takes care of trace collection, batching, transport, and evaluation.
1055
+
1056
+ ### How It Works
1057
+
1058
+ 1. You call `auto_init(api_key, agent_id, ...)` once at startup.
1059
+ 2. The SDK auto-detects which AI libraries you're using (OpenAI, Anthropic, LangChain, etc.) and installs an OpenInference instrumentor for each.
1060
+ 3. Every subsequent LLM / tool call is captured as an OpenTelemetry span and streamed to the TrustModel gateway.
1061
+ 4. The TrustModel Control Plane buffers traces server-side, groups them by `agent_id` + `domain` + `frameworks`, and runs evaluation on the schedule you configure (daily, weekly, or monthly).
1062
+
1063
+ ### Installation
1064
+
1065
+ ```bash
1066
+ pip install "trustmodel[telemetry]"
1067
+ ```
1068
+
1069
+ This installs OpenTelemetry plus the OpenInference instrumentors. **Python 3.10+ required** for the telemetry extra.
1070
+
1071
+ If you only want the core SDK (e.g., `client.evaluations.create()`), `pip install trustmodel` works on Python 3.7+ but `auto_init` will not be available.
1072
+
1073
+ ### Quick Start
1074
+
1075
+ ```python
1076
+ from trustmodel.telemetry import auto_init
1077
+
1078
+ auto_init(
1079
+ api_key="tm-...", # your TrustModel API key
1080
+ agent_id="my-customer-support-agent", # any identifier you choose
1081
+ domain="general_ai", # fair_lending | hr_bias | healthcare | general_ai
1082
+ frameworks=["nist-ai-rmf"], # one or more compliance framework slugs
1083
+ )
1084
+
1085
+ # Your existing agent code — no changes needed
1086
+ import openai
1087
+ client = openai.OpenAI()
1088
+ client.chat.completions.create(
1089
+ model="gpt-4o-mini",
1090
+ messages=[{"role": "user", "content": "Hello"}],
1091
+ )
1092
+ ```
1093
+
1094
+ That's it. The OpenAI call is automatically captured and streamed to TrustModel.
1095
+
1096
+ ### Discovering Domains and Frameworks
1097
+
1098
+ Use the `client.frameworks` endpoint to list available domains and the compliance frameworks within each:
1099
+
1100
+ ```python
1101
+ from trustmodel import TrustModelClient
1102
+
1103
+ client = TrustModelClient(api_key="tm-...")
1104
+
1105
+ # All available domains
1106
+ print(client.frameworks.list_domains())
1107
+ # ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
1108
+
1109
+ # Frameworks for a specific domain
1110
+ for f in client.frameworks.list(domain="fair_lending"):
1111
+ print(f"{f.slug}: {f.name} ({f.credits} credits)")
1112
+ ```
1113
+
1114
+ Use the `slug` values when calling `auto_init(frameworks=[...])`.
1115
+
1116
+ ### Supported AI Libraries
1117
+
1118
+ `auto_init` will automatically activate instrumentation for any of these libraries that you have installed:
1119
+
1120
+ | Library | Auto-detected via |
1121
+ |---------|-------------------|
1122
+ | OpenAI | `openinference-instrumentation-openai` |
1123
+ | Anthropic (Claude) | `openinference-instrumentation-anthropic` |
1124
+ | LangChain | `openinference-instrumentation-langchain` |
1125
+ | LlamaIndex | `openinference-instrumentation-llama-index` |
1126
+ | AWS Bedrock | `openinference-instrumentation-bedrock` |
1127
+ | Mistral AI | `openinference-instrumentation-mistralai` |
1128
+ | Groq | `openinference-instrumentation-groq` |
1129
+ | CrewAI | `openinference-instrumentation-crewai` |
1130
+ | Vertex AI | `openinference-instrumentation-vertexai` |
1131
+ | DSPy | `openinference-instrumentation-dspy` |
1132
+
1133
+ All ten are installed by `pip install "trustmodel[telemetry]"`. The instrumentors are no-ops when the underlying library isn't being used — there's no overhead.
1134
+
1135
+ ### Existing OpenTelemetry Setup (Datadog, Jaeger, Honeycomb, Arize)
1136
+
1137
+ If your application already has OpenTelemetry configured for another observability backend, `auto_init` detects this and **adds TrustModel as an additional exporter** rather than replacing your setup.
1138
+
1139
+ ```python
1140
+ # Your existing OTel setup (e.g., Datadog APM, Jaeger, Honeycomb)
1141
+ from opentelemetry import trace
1142
+ from opentelemetry.sdk.trace import TracerProvider
1143
+ provider = TracerProvider(...)
1144
+ trace.set_tracer_provider(provider)
1145
+
1146
+ # Then add TrustModel — both backends now receive every span
1147
+ from trustmodel.telemetry import auto_init
1148
+ auto_init(api_key="tm-...", agent_id="my-agent")
1149
+ ```
1150
+
1151
+ Spans fan out to both your existing exporter and TrustModel's. Nothing in your existing pipeline is modified or replaced.
1152
+
1153
+ ### Parameters
1154
+
1155
+ | Parameter | Required | Description |
1156
+ |-----------|----------|-------------|
1157
+ | `api_key` | yes | Your TrustModel API key (`tm-...`) |
1158
+ | `agent_id` | yes | Any string you choose. All traces sharing this `agent_id` are grouped together for evaluation. |
1159
+ | `domain` | yes | One of `fair_lending`, `hr_bias`, `healthcare`, `general_ai`. Determines which evaluators run. |
1160
+ | `frameworks` | yes | List of compliance framework slugs (e.g., `["ecoa-regb", "fcra"]`). Use `client.frameworks.list(domain=...)` to discover. |
1161
+ | `service_name` | no | Logical service name (default: `"default"`) |
1162
+
1163
+ ### Schedule and Evaluation
1164
+
1165
+ Configure when buffered traces are evaluated in the [Control Plane dashboard](https://app.trustmodel.ai):
1166
+
1167
+ - **Manual** — only when you click "Trigger Evaluation Now"
1168
+ - **Daily / Weekly / Monthly** — automatically via cron at midnight UTC
1169
+
1170
+ Each evaluation produces:
1171
+ - An overall trust score
1172
+ - Per-category breakdowns (safety, fairness, accuracy, etc.)
1173
+ - Findings and recommendations
1174
+
1175
+ ### Failure Modes (Safe by Design)
1176
+
1177
+ `auto_init` wraps everything in `try/except`. If anything fails — missing dependencies, network issues, an instrumentor crash — your application keeps running. Telemetry is best-effort; nothing TrustModel does will break your agent.
1178
+
1179
+ If the `[telemetry]` extras aren't installed and you try to import the telemetry module directly, you get a clear error:
1180
+
1181
+ ```
1182
+ ImportError: TrustModel telemetry requires extra dependencies.
1183
+ Install with: pip install "trustmodel[telemetry]"
1184
+ ```
1185
+
1065
1186
  ## Agentic Trace Evaluation
1066
1187
 
1067
1188
  Evaluate AI agent execution traces for safety, reasoning quality, tool usage, and goal completion. Upload a JSON or JSONL trace file and get scored across 14 dimensions.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "trustmodel"
7
- version = "2.1.0"
7
+ version = "2.2.1"
8
8
  authors = [
9
9
  {name = "TrustModel", email = "info@predixtions.com"},
10
10
  ]
@@ -40,6 +40,24 @@ dependencies = [
40
40
  ]
41
41
 
42
42
  [project.optional-dependencies]
43
+ # Auto-capture AI agent traces and stream them to TrustModel.
44
+ # Install with: pip install "trustmodel[telemetry]"
45
+ # Requires Python 3.10+ because several OpenInference instrumentors do.
46
+ telemetry = [
47
+ "opentelemetry-api>=1.20.0; python_version>='3.10'",
48
+ "opentelemetry-sdk>=1.20.0; python_version>='3.10'",
49
+ "opentelemetry-exporter-otlp-proto-http>=1.20.0; python_version>='3.10'",
50
+ "openinference-instrumentation-openai>=0.1.0; python_version>='3.10'",
51
+ "openinference-instrumentation-anthropic>=0.1.0; python_version>='3.10'",
52
+ "openinference-instrumentation-langchain>=0.1.0; python_version>='3.10'",
53
+ "openinference-instrumentation-llama-index>=0.1.0; python_version>='3.10'",
54
+ "openinference-instrumentation-bedrock>=0.1.0; python_version>='3.10'",
55
+ "openinference-instrumentation-mistralai>=0.1.0; python_version>='3.10'",
56
+ "openinference-instrumentation-groq>=0.1.0; python_version>='3.10'",
57
+ "openinference-instrumentation-crewai>=0.1.0; python_version>='3.10'",
58
+ "openinference-instrumentation-vertexai>=0.1.0; python_version>='3.10'",
59
+ "openinference-instrumentation-dspy>=0.1.0; python_version>='3.10'",
60
+ ]
43
61
  dev = [
44
62
  "pytest>=7.0.0,<8.0.0; python_version<'3.8'",
45
63
  "pytest>=7.0.0; python_version>='3.8'",
@@ -119,6 +137,15 @@ strict_equality = true
119
137
  module = "tests.*"
120
138
  disallow_untyped_defs = false
121
139
 
140
+ [[tool.mypy.overrides]]
141
+ # Optional telemetry deps — installed via [telemetry] extra. CI runs mypy
142
+ # without these installed, so suppress missing-import errors.
143
+ module = [
144
+ "opentelemetry.*",
145
+ "openinference.*",
146
+ ]
147
+ ignore_missing_imports = true
148
+
122
149
  [tool.ruff]
123
150
  target-version = "py37"
124
151
  line-length = 100
@@ -159,6 +186,12 @@ source = ["src/trustmodel"]
159
186
  omit = [
160
187
  "*/tests/*",
161
188
  "*/test_*",
189
+ # Telemetry code paths require real OTel runtime + live OpenInference
190
+ # instrumentors to exercise meaningfully — covered by integration tests
191
+ # in trus638-testing rather than unit tests here.
192
+ "*/telemetry/*",
193
+ "*/endpoints/frameworks.py",
194
+ "*/models/frameworks.py",
162
195
  ]
163
196
 
164
197
  [tool.coverage.report]
@@ -30,6 +30,17 @@ try:
30
30
  except PackageNotFoundError:
31
31
  # Package not installed, use fallback
32
32
  __version__ = "0.1.0"
33
+
34
+ # Telemetry (auto_init) is an optional feature.
35
+ # Install with: pip install "trustmodel[telemetry]"
36
+ # If telemetry deps are missing, importing the SDK still works.
37
+ try:
38
+ from .telemetry import auto_init # noqa: F401
39
+
40
+ _HAS_TELEMETRY = True
41
+ except ImportError:
42
+ _HAS_TELEMETRY = False
43
+
33
44
  __all__ = [
34
45
  "TrustModelClient",
35
46
  "TrustModelError",
@@ -40,3 +51,5 @@ __all__ = [
40
51
  "InsufficientCreditsError",
41
52
  "ConnectionValidationError",
42
53
  ]
54
+ if _HAS_TELEMETRY:
55
+ __all__.append("auto_init")
@@ -18,6 +18,7 @@ from .endpoints.batch_jobs import BatchJobsEndpoint
18
18
  from .endpoints.config import ConfigEndpoint
19
19
  from .endpoints.credits import CreditsEndpoint
20
20
  from .endpoints.evaluations import EvaluationsEndpoint
21
+ from .endpoints.frameworks import FrameworksEndpoint
21
22
  from .endpoints.galileo import GalileoEndpoint
22
23
  from .endpoints.models import ModelsEndpoint
23
24
  from .exceptions import (
@@ -120,6 +121,7 @@ class TrustModelClient:
120
121
  self.batch_jobs = BatchJobsEndpoint(self)
121
122
  self.agentic = AgenticEndpoint(self)
122
123
  self.galileo = GalileoEndpoint(self)
124
+ self.frameworks = FrameworksEndpoint(self)
123
125
 
124
126
  def _request(
125
127
  self,
@@ -0,0 +1,94 @@
1
+ """
2
+ Frameworks endpoint wrapper for the TrustModel SDK.
3
+
4
+ Lets users discover available compliance frameworks and domains
5
+ for use with `auto_init(domain=..., frameworks=[...])`.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import TYPE_CHECKING, Any, List, Optional
11
+
12
+ from ..models.frameworks import Framework
13
+
14
+ if TYPE_CHECKING:
15
+ from ..client import TrustModelClient
16
+
17
+
18
+ # Hardcoded list mirrors EvaluationSubTypeChoices on the backend.
19
+ # Update if backend adds new domains.
20
+ _KNOWN_DOMAINS = [
21
+ "fair_lending",
22
+ "hr_bias",
23
+ "healthcare",
24
+ "general_ai",
25
+ ]
26
+
27
+
28
+ class FrameworksEndpoint:
29
+ """
30
+ Interface for compliance framework discovery.
31
+
32
+ Frameworks are organized by domain (framework_type). Use these methods
33
+ to discover what's available before calling auto_init().
34
+
35
+ Example:
36
+ >>> client = TrustModelClient(api_key="tm-...")
37
+ >>> domains = client.frameworks.list_domains()
38
+ >>> # ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
39
+ >>> frameworks = client.frameworks.list(domain="fair_lending")
40
+ >>> for f in frameworks:
41
+ ... print(f"{f.slug}: {f.name}")
42
+ """
43
+
44
+ def __init__(self, client: TrustModelClient) -> None:
45
+ self._client = client
46
+
47
+ def list_domains(self) -> List[str]:
48
+ """
49
+ Get list of available domain identifiers.
50
+
51
+ Returns:
52
+ List of domain strings. Pass any of these as `domain` to
53
+ `auto_init()` or as the `?framework_type=...` query parameter.
54
+
55
+ Example:
56
+ >>> client.frameworks.list_domains()
57
+ ['fair_lending', 'hr_bias', 'healthcare', 'general_ai']
58
+ """
59
+ return list(_KNOWN_DOMAINS)
60
+
61
+ def list(self, domain: Optional[str] = None) -> List[Framework]:
62
+ """
63
+ List available compliance frameworks.
64
+
65
+ Args:
66
+ domain: Optional domain filter (e.g., 'fair_lending'). If omitted,
67
+ returns all active frameworks across all domains.
68
+
69
+ Returns:
70
+ List of Framework objects. Use the `slug` field as the framework
71
+ identifier when calling auto_init(frameworks=[...]).
72
+
73
+ Example:
74
+ >>> frameworks = client.frameworks.list(domain="fair_lending")
75
+ >>> auto_init(
76
+ ... api_key="tm-...",
77
+ ... agent_id="my-agent",
78
+ ... domain="fair_lending",
79
+ ... frameworks=[f.slug for f in frameworks[:2]],
80
+ ... )
81
+ """
82
+ params = {}
83
+ if domain:
84
+ params["framework_type"] = domain
85
+
86
+ response: Any = self._client.get("/api/agent-evaluation/frameworks/", params=params)
87
+
88
+ # API may return a list or a paginated object {results: [...]}
89
+ if isinstance(response, list):
90
+ items = response
91
+ else:
92
+ items = response.get("results", [])
93
+
94
+ return [Framework(**item) for item in items]
@@ -18,6 +18,7 @@ class EvaluationStatus(str, Enum):
18
18
  RUNNING = "running"
19
19
  COMPLETED = "completed"
20
20
  FAILED = "failed"
21
+ PAYMENT_PENDING = "payment_pending"
21
22
 
22
23
 
23
24
  class ApplicationType(str, Enum):
@@ -30,6 +31,8 @@ class ApplicationType(str, Enum):
30
31
  CLASSIFICATION = "classification"
31
32
  CODE_GENERATION = "code_generation"
32
33
  TRANSLATION = "translation"
34
+ EVALUATION_WIZARD = "evaluation_wizard"
35
+ AUTOMATION_AGENT = "automation-agent"
33
36
 
34
37
 
35
38
  class UserPersona(str, Enum):
@@ -143,9 +146,9 @@ class Evaluation(BaseModel):
143
146
  status: EvaluationStatus
144
147
  model_identifier: str
145
148
  vendor_identifier: Optional[str] = None
146
- model_config_name: str
147
- application_type: ApplicationType
148
- user_personas: List[str]
149
+ model_config_name: Optional[str] = None
150
+ application_type: Optional[ApplicationType] = None
151
+ user_personas: Optional[List[str]] = None
149
152
  application_description: Optional[str] = None
150
153
  domain_expert_description: Optional[str] = None
151
154
  completion_percentage: int = Field(..., ge=0, le=100)
@@ -0,0 +1,27 @@
1
+ """Pydantic models for compliance frameworks."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from decimal import Decimal
6
+ from typing import Optional
7
+
8
+ from pydantic import BaseModel, Field
9
+
10
+
11
+ class Framework(BaseModel):
12
+ """A compliance framework that can be selected for evaluation."""
13
+
14
+ id: str
15
+ name: str
16
+ subtitle: Optional[str] = ""
17
+ slug: str
18
+ description: Optional[str] = ""
19
+ credits: Decimal = Field(default=Decimal("0"))
20
+ framework_type: Optional[str] = Field(
21
+ default=None,
22
+ description="Domain classification (fair_lending, hr_bias, healthcare, general_ai)",
23
+ )
24
+
25
+ class Config:
26
+ # allow extra fields from API to not fail
27
+ extra = "allow"
@@ -0,0 +1,22 @@
1
+ """
2
+ TrustModel Telemetry — auto-capture AI agent calls via OpenTelemetry.
3
+
4
+ Requires the `telemetry` extra:
5
+ pip install "trustmodel[telemetry]"
6
+
7
+ Usage:
8
+ from trustmodel.telemetry import auto_init
9
+ auto_init(api_key="tm-...", agent_id="my-agent")
10
+ """
11
+
12
+ try:
13
+ import opentelemetry # noqa: F401
14
+ except ImportError as exc:
15
+ raise ImportError(
16
+ "TrustModel telemetry requires extra dependencies. "
17
+ 'Install with: pip install "trustmodel[telemetry]"'
18
+ ) from exc
19
+
20
+ from .auto_init import auto_init, flush
21
+
22
+ __all__ = ["auto_init", "flush"]
@@ -0,0 +1,33 @@
1
+ """
2
+ OpenInference semantic convention attribute keys.
3
+
4
+ Reference for the OTLP → agentic trace converter on the gateway side.
5
+ """
6
+
7
+ # Span classification
8
+ OPENINFERENCE_SPAN_KIND = "openinference.span.kind"
9
+
10
+ # LLM attributes
11
+ LLM_MODEL_NAME = "llm.model_name"
12
+ LLM_TOKEN_COUNT_PROMPT = "llm.token_count.prompt"
13
+ LLM_TOKEN_COUNT_COMPLETION = "llm.token_count.completion"
14
+ LLM_OUTPUT_MESSAGES = "llm.output_messages"
15
+ LLM_INPUT_MESSAGES = "llm.input_messages"
16
+ LLM_INVOCATION_PARAMETERS = "llm.invocation_parameters"
17
+
18
+ # Tool attributes
19
+ TOOL_NAME = "tool.name"
20
+ TOOL_PARAMETERS = "tool.parameters"
21
+ TOOL_DESCRIPTION = "tool.description"
22
+
23
+ # I/O attributes
24
+ INPUT_VALUE = "input.value"
25
+ INPUT_MIME_TYPE = "input.mime_type"
26
+ OUTPUT_VALUE = "output.value"
27
+ OUTPUT_MIME_TYPE = "output.mime_type"
28
+
29
+ # Retriever attributes
30
+ RETRIEVAL_DOCUMENTS = "retrieval.documents"
31
+
32
+ # Embedding attributes
33
+ EMBEDDING_MODEL_NAME = "embedding.model_name"
@@ -0,0 +1,53 @@
1
+ """
2
+ Auto-detect and install OpenInference instrumentors.
3
+
4
+ Each instrumentor wraps a specific AI library (OpenAI, Anthropic, LangChain, etc.)
5
+ to emit OTel spans with OpenInference semantic attributes.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from typing import List
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ # (module_path, class_name) — order doesn't matter
16
+ _KNOWN_INSTRUMENTORS = [
17
+ ("openinference.instrumentation.openai", "OpenAIInstrumentor"),
18
+ ("openinference.instrumentation.anthropic", "AnthropicInstrumentor"),
19
+ ("openinference.instrumentation.langchain", "LangChainInstrumentor"),
20
+ ("openinference.instrumentation.llama_index", "LlamaIndexInstrumentor"),
21
+ ("openinference.instrumentation.bedrock", "BedrockInstrumentor"),
22
+ ("openinference.instrumentation.mistralai", "MistralAIInstrumentor"),
23
+ ("openinference.instrumentation.groq", "GroqInstrumentor"),
24
+ ("openinference.instrumentation.crewai", "CrewAIInstrumentor"),
25
+ ("openinference.instrumentation.vertexai", "VertexAIInstrumentor"),
26
+ ("openinference.instrumentation.dspy", "DSPyInstrumentor"),
27
+ ]
28
+
29
+
30
+ def install_instrumentors() -> List[str]:
31
+ """
32
+ Attempt to import and instrument each known OpenInference instrumentor.
33
+
34
+ Only instrumentors whose packages are installed will be activated.
35
+ Failures are logged and silently skipped — never crash the customer's app.
36
+
37
+ Returns:
38
+ List of successfully installed instrumentor names.
39
+ """
40
+ installed = []
41
+ for module_path, class_name in _KNOWN_INSTRUMENTORS:
42
+ try:
43
+ module = __import__(module_path, fromlist=[class_name])
44
+ instrumentor_cls = getattr(module, class_name)
45
+ instrumentor_cls().instrument()
46
+ installed.append(class_name)
47
+ logger.debug("TrustModel telemetry: installed %s", class_name)
48
+ except ImportError:
49
+ # Package not installed — expected, skip silently
50
+ pass
51
+ except Exception:
52
+ logger.debug("TrustModel telemetry: failed to install %s", class_name, exc_info=True)
53
+ return installed
@@ -0,0 +1,289 @@
1
+ """
2
+ auto_init() — 2-line setup for automatic AI agent trace capture.
3
+
4
+ Usage:
5
+ from trustmodel.telemetry import auto_init
6
+ auto_init(api_key="tm-...", agent_id="my-weather-agent")
7
+
8
+ # All subsequent OpenAI/Anthropic/LangChain calls are now traced.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import atexit
14
+ import logging
15
+ import uuid as _uuid
16
+ from typing import TYPE_CHECKING, Any, List, Optional
17
+
18
+ if TYPE_CHECKING:
19
+ from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ _initialized = False
24
+ _tracer_provider: Optional[SDKTracerProvider] = None
25
+
26
+ ENVIRONMENT_URLS = {
27
+ "local": "http://localhost:8000",
28
+ "qa": "https://api-trustmodel.pdxqa.com",
29
+ "production": "https://api.trustmodel.ai",
30
+ }
31
+
32
+
33
+ def auto_init(
34
+ api_key: str,
35
+ *,
36
+ agent_id: str,
37
+ domain: str,
38
+ frameworks: List[str],
39
+ service_name: str = "default",
40
+ environment: str = "production",
41
+ base_url: Optional[str] = None,
42
+ ) -> List[str]:
43
+ """
44
+ Initialize automatic OTel trace capture and export to TrustModel.
45
+
46
+ Safe to call from any Python app — wraps everything in try/except
47
+ so it never crashes the customer's application.
48
+
49
+ Args:
50
+ api_key: TrustModel API key (tm-...).
51
+ agent_id: Unique identifier for this agent. All traces are grouped
52
+ and evaluated per agent_id.
53
+ domain: Domain classification — one of 'fair_lending', 'hr_bias',
54
+ 'healthcare', 'general_ai'. Required. Use
55
+ TrustModelClient.frameworks.list_domains() to discover.
56
+ frameworks: List of compliance framework slugs (e.g.,
57
+ ['eu-ai-act-high-risk', 'iso-42001']). Required and must
58
+ be non-empty. Use TrustModelClient.frameworks.list(domain=...)
59
+ to discover available slugs.
60
+ service_name: Logical service name for this application.
61
+ environment: One of 'local', 'qa', 'production'.
62
+ base_url: Explicit base URL (overrides environment).
63
+
64
+ Returns:
65
+ List of instrumentor names that were successfully installed.
66
+
67
+ Raises:
68
+ ValueError: If `domain` is empty or `frameworks` is empty/missing.
69
+ """
70
+ global _initialized, _tracer_provider
71
+
72
+ if not domain:
73
+ raise ValueError("auto_init() requires a non-empty 'domain' argument.")
74
+ if not frameworks:
75
+ raise ValueError(
76
+ "auto_init() requires a non-empty 'frameworks' list. "
77
+ "Use TrustModelClient.frameworks.list(domain=...) to discover slugs."
78
+ )
79
+
80
+ if _initialized:
81
+ logger.debug("TrustModel telemetry already initialized, skipping.")
82
+ return []
83
+
84
+ try:
85
+ return _do_init(api_key, agent_id, domain, frameworks, service_name, environment, base_url)
86
+ except Exception:
87
+ logger.debug("TrustModel telemetry: auto_init failed", exc_info=True)
88
+ return []
89
+
90
+
91
+ def _do_init(
92
+ api_key: str,
93
+ agent_id: str,
94
+ domain: str,
95
+ frameworks: List[str],
96
+ service_name: str,
97
+ environment: str,
98
+ base_url: Optional[str],
99
+ ) -> List[str]:
100
+ global _initialized, _tracer_provider
101
+
102
+ from opentelemetry import trace
103
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
104
+ from opentelemetry.sdk.resources import Resource
105
+ from opentelemetry.sdk.trace import TracerProvider
106
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
107
+
108
+ session_id = str(_uuid.uuid4())
109
+
110
+ # Resolve endpoint — with explicit endpoint param, the exporter
111
+ # does NOT append /v1/traces, so we include the full path.
112
+ if base_url:
113
+ endpoint = base_url.rstrip("/") + "/sdk/v1/otel/v1/traces"
114
+ else:
115
+ if environment not in ENVIRONMENT_URLS:
116
+ raise ValueError(
117
+ f"Invalid environment '{environment}'. "
118
+ f"Must be one of: {', '.join(ENVIRONMENT_URLS)}"
119
+ )
120
+ endpoint = ENVIRONMENT_URLS[environment] + "/sdk/v1/otel/v1/traces"
121
+
122
+ exporter = OTLPSpanExporter(
123
+ endpoint=endpoint,
124
+ headers={"X-API-Key": api_key},
125
+ )
126
+
127
+ # Normalize domain + frameworks (both required, validated upstream)
128
+ domain_str = domain
129
+ frameworks_str = ",".join(frameworks)
130
+
131
+ # Wrap exporter so agent_id + session_id + domain + frameworks are always
132
+ # injected as resource attributes, even when piggybacking on a customer's
133
+ # existing TracerProvider.
134
+ wrapped_exporter = _make_trustmodel_exporter(
135
+ exporter, agent_id, session_id, domain_str, frameworks_str
136
+ )
137
+ processor = BatchSpanProcessor(wrapped_exporter)
138
+
139
+ current_provider = trace.get_tracer_provider()
140
+
141
+ if isinstance(current_provider, TracerProvider):
142
+ # Customer already has OTel set up — just add our exporter
143
+ current_provider.add_span_processor(processor)
144
+ _tracer_provider = current_provider
145
+ logger.debug("TrustModel telemetry: added exporter to existing TracerProvider")
146
+ else:
147
+ # No existing OTel — create our own provider
148
+ resource_attrs = {
149
+ "service.name": service_name,
150
+ "trustmodel.agent_id": agent_id,
151
+ "trustmodel.session_id": session_id,
152
+ }
153
+ if domain_str:
154
+ resource_attrs["trustmodel.domain"] = domain_str
155
+ if frameworks_str:
156
+ resource_attrs["trustmodel.frameworks"] = frameworks_str
157
+ resource = Resource.create(resource_attrs)
158
+ provider = TracerProvider(resource=resource)
159
+ provider.add_span_processor(processor)
160
+ trace.set_tracer_provider(provider)
161
+ _tracer_provider = provider
162
+ logger.debug("TrustModel telemetry: created new TracerProvider")
163
+
164
+ # Auto-detect and install instrumentors
165
+ from ._instrumentors import install_instrumentors
166
+
167
+ installed = install_instrumentors()
168
+
169
+ _initialized = True
170
+ atexit.register(_shutdown)
171
+
172
+ if installed:
173
+ logger.info("TrustModel telemetry: active — instrumentors: %s", ", ".join(installed))
174
+ else:
175
+ logger.info(
176
+ "TrustModel telemetry: active — no instrumentors found (install openinference-instrumentation-*)"
177
+ )
178
+
179
+ return installed
180
+
181
+
182
+ def flush(
183
+ api_key: str,
184
+ *,
185
+ environment: str = "production",
186
+ base_url: Optional[str] = None,
187
+ ) -> None:
188
+ """
189
+ Flush buffered OTel traces to trigger evaluation.
190
+
191
+ Call this when the agent session is complete.
192
+
193
+ Args:
194
+ api_key: TrustModel API key.
195
+ environment: One of 'local', 'qa', 'production'.
196
+ base_url: Explicit base URL (overrides environment).
197
+ """
198
+ import requests
199
+
200
+ if base_url:
201
+ url = base_url.rstrip("/") + "/sdk/v1/otel/v1/flush"
202
+ else:
203
+ url = (
204
+ ENVIRONMENT_URLS.get(environment, ENVIRONMENT_URLS["production"])
205
+ + "/sdk/v1/otel/v1/flush"
206
+ )
207
+
208
+ # Force-flush any pending spans first
209
+ if _tracer_provider is not None:
210
+ try:
211
+ _tracer_provider.force_flush(timeout_millis=5000)
212
+ except Exception:
213
+ logger.debug("TrustModel telemetry: force_flush failed", exc_info=True)
214
+
215
+ response = requests.post(
216
+ url,
217
+ headers={"X-API-Key": api_key, "Content-Type": "application/json"},
218
+ json={},
219
+ timeout=30,
220
+ )
221
+ response.raise_for_status()
222
+ data = response.json()
223
+ logger.info(
224
+ "TrustModel telemetry: flushed %d evaluation(s)",
225
+ data.get("count", 0),
226
+ )
227
+
228
+
229
+ def _make_trustmodel_exporter(
230
+ inner_exporter: Any,
231
+ agent_id: str,
232
+ session_id: str,
233
+ domain: str = "",
234
+ frameworks: str = "",
235
+ ) -> Any:
236
+ """
237
+ Build a SpanExporter subclass that wraps the inner exporter and injects
238
+ TrustModel resource attributes on every exported span. Implemented as a
239
+ factory so SpanExporter can be a runtime base class without polluting
240
+ module-level imports (telemetry extras may be missing at import time).
241
+ """
242
+ from opentelemetry.sdk.resources import Resource
243
+ from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter
244
+
245
+ class _TrustModelExporter(_SpanExporter):
246
+ def __init__(self) -> None:
247
+ self._inner = inner_exporter
248
+ self._agent_id = agent_id
249
+ self._session_id = session_id
250
+ self._domain = domain
251
+ self._frameworks = frameworks
252
+
253
+ def export(self, spans): # type: ignore[no-untyped-def]
254
+ attrs = {
255
+ "trustmodel.agent_id": self._agent_id,
256
+ "trustmodel.session_id": self._session_id,
257
+ }
258
+ if self._domain:
259
+ attrs["trustmodel.domain"] = self._domain
260
+ if self._frameworks:
261
+ attrs["trustmodel.frameworks"] = self._frameworks
262
+ patched_resource = Resource.create(attrs)
263
+
264
+ patched = []
265
+ for span in spans:
266
+ merged = span.resource.merge(patched_resource)
267
+ span._resource = merged
268
+ patched.append(span)
269
+
270
+ return self._inner.export(patched)
271
+
272
+ def shutdown(self) -> None:
273
+ self._inner.shutdown()
274
+
275
+ def force_flush(self, timeout_millis: int = 30000) -> bool:
276
+ if hasattr(self._inner, "force_flush"):
277
+ return bool(self._inner.force_flush(timeout_millis))
278
+ return True
279
+
280
+ return _TrustModelExporter()
281
+
282
+
283
+ def _shutdown() -> None:
284
+ """Graceful shutdown — flush pending spans on process exit."""
285
+ if _tracer_provider is not None:
286
+ try:
287
+ _tracer_provider.shutdown()
288
+ except Exception:
289
+ pass
File without changes
File without changes