openbox-temporal-sdk-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openbox/__init__.py +107 -0
- openbox/activities.py +163 -0
- openbox/activity_interceptor.py +755 -0
- openbox/config.py +274 -0
- openbox/otel_setup.py +969 -0
- openbox/py.typed +0 -0
- openbox/span_processor.py +361 -0
- openbox/tracing.py +228 -0
- openbox/types.py +166 -0
- openbox/worker.py +257 -0
- openbox/workflow_interceptor.py +264 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/METADATA +1214 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/RECORD +15 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/WHEEL +4 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1214 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: openbox-temporal-sdk-python
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: OpenBox SDK - Governance and observability for Temporal workflows
|
|
5
|
+
Project-URL: Homepage, https://github.com/OpenBox-AI/temporal-sdk-python
|
|
6
|
+
Project-URL: Documentation, https://github.com/OpenBox-AI/temporal-sdk-python#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/OpenBox-AI/temporal-sdk-python.git
|
|
8
|
+
Project-URL: Issues, https://github.com/OpenBox-AI/temporal-sdk-python/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/OpenBox-AI/temporal-sdk-python/blob/main/CHANGELOG.md
|
|
10
|
+
Author-email: OpenBox Team <tino@openbox.ai>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: governance,hitl,human-in-the-loop,observability,opentelemetry,temporal,workflow
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: System :: Monitoring
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Requires-Dist: asyncpg>=0.29.0
|
|
27
|
+
Requires-Dist: httpx<1,>=0.28.0
|
|
28
|
+
Requires-Dist: mysql-connector-python>=8.0.0
|
|
29
|
+
Requires-Dist: opentelemetry-api>=1.38.0
|
|
30
|
+
Requires-Dist: opentelemetry-instrumentation-asyncpg>=0.59b0
|
|
31
|
+
Requires-Dist: opentelemetry-instrumentation-httpx>=0.59b0
|
|
32
|
+
Requires-Dist: opentelemetry-instrumentation-mysql>=0.59b0
|
|
33
|
+
Requires-Dist: opentelemetry-instrumentation-psycopg2>=0.59b0
|
|
34
|
+
Requires-Dist: opentelemetry-instrumentation-pymongo>=0.59b0
|
|
35
|
+
Requires-Dist: opentelemetry-instrumentation-pymysql>=0.59b0
|
|
36
|
+
Requires-Dist: opentelemetry-instrumentation-redis>=0.59b0
|
|
37
|
+
Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
|
|
38
|
+
Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
|
|
39
|
+
Requires-Dist: opentelemetry-instrumentation-urllib3>=0.59b0
|
|
40
|
+
Requires-Dist: opentelemetry-instrumentation-urllib>=0.59b0
|
|
41
|
+
Requires-Dist: opentelemetry-sdk>=1.38.0
|
|
42
|
+
Requires-Dist: psycopg2-binary>=2.9.10
|
|
43
|
+
Requires-Dist: pymongo>=4.0.0
|
|
44
|
+
Requires-Dist: pymysql>=1.0.0
|
|
45
|
+
Requires-Dist: redis>=5.0.0
|
|
46
|
+
Requires-Dist: sqlalchemy>=2.0.0
|
|
47
|
+
Requires-Dist: temporalio<2,>=1.8.0
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# OpenBox SDK for Temporal Workflows
|
|
51
|
+
|
|
52
|
+
OpenBox SDK provides governance and observability for Temporal workflows by capturing workflow/activity lifecycle events and HTTP telemetry, then sending them to OpenBox Core for policy evaluation.
|
|
53
|
+
|
|
54
|
+
## Architecture
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
┌─────────────────────────────────────────────────────────────────────────┐
|
|
58
|
+
│ Temporal Worker │
|
|
59
|
+
│ │
|
|
60
|
+
│ ┌────────────────────────┐ ┌────────────────────────────────────┐ │
|
|
61
|
+
│ │ Workflow Interceptor │ │ Activity Interceptor │ │
|
|
62
|
+
│ │ ──────────────────── │ │ ──────────────────────────────── │ │
|
|
63
|
+
│ │ - WorkflowStarted │ │ - ActivityStarted (+ input) │ │
|
|
64
|
+
│ │ - WorkflowCompleted │ │ - ActivityCompleted (+ output) │ │
|
|
65
|
+
│ │ - WorkflowFailed │ │ │ │
|
|
66
|
+
│ │ - SignalReceived │ │ Guardrails: Redact/modify input │ │
|
|
67
|
+
│ │ │ │ before execution, output after │ │
|
|
68
|
+
│ │ Sends via activity │ │ │ │
|
|
69
|
+
│ │ (determinism) │ │ Collects all spans (see below) │ │
|
|
70
|
+
│ └────────────────────────┘ └────────────────────────────────────┘ │
|
|
71
|
+
│ │ │ │
|
|
72
|
+
│ │ ▼ │
|
|
73
|
+
│ │ ┌──────────────────────────────────────────────┐ │
|
|
74
|
+
│ │ │ WorkflowSpanProcessor │ │
|
|
75
|
+
│ │ │ ────────────────────────────────────────── │ │
|
|
76
|
+
│ │ │ - Buffers spans per workflow │ │
|
|
77
|
+
│ │ │ - Merges body/header data from HTTP hooks │ │
|
|
78
|
+
│ │ │ - Maps trace_id → workflow_id │ │
|
|
79
|
+
│ │ │ - Ignores OpenBox Core URLs │ │
|
|
80
|
+
│ │ └──────────────────────────────────────────────┘ │
|
|
81
|
+
│ │ │ │
|
|
82
|
+
│ ▼ ▼ │
|
|
83
|
+
│ ┌──────────────────────────────────────────────────────────────────────────┐│
|
|
84
|
+
│ │ OTel Instrumentation Layer ││
|
|
85
|
+
│ │ ────────────────────────────────────────────────────────────────────── ││
|
|
86
|
+
│ │ HTTP: httpx, requests, urllib3 (headers + bodies) ││
|
|
87
|
+
│ │ Database: PostgreSQL, MySQL, MongoDB, Redis, SQLAlchemy (db.statement) ││
|
|
88
|
+
│ │ File I/O: open(), read(), write() (path, bytes, mode) ││
|
|
89
|
+
│ │ Functions: @traced decorator, create_span() (args + results) ││
|
|
90
|
+
│ └──────────────────────────────────────────────────────────────────────────┘│
|
|
91
|
+
└──────────────────────────────────────────────────────────────────────────────┘
|
|
92
|
+
│
|
|
93
|
+
▼
|
|
94
|
+
┌─────────────────────────┐
|
|
95
|
+
│ OpenBox Core │
|
|
96
|
+
│ ─────────────────── │
|
|
97
|
+
│ POST /governance/ │
|
|
98
|
+
│ evaluate │
|
|
99
|
+
│ │
|
|
100
|
+
│ Returns: │
|
|
101
|
+
│ - verdict: allow/halt │
|
|
102
|
+
│ - verdict: block │
|
|
103
|
+
│ - guardrails_result │
|
|
104
|
+
│ (redacted input/ │
|
|
105
|
+
│ output) │
|
|
106
|
+
└─────────────────────────┘
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Event Types (6 events)
|
|
110
|
+
| Event | Trigger | Key Fields |
|
|
111
|
+
|-------|---------|------------|
|
|
112
|
+
| `WorkflowStarted` | Workflow begins | workflow_id, run_id, workflow_type, task_queue |
|
|
113
|
+
| `WorkflowCompleted` | Workflow succeeds | workflow_id, run_id, workflow_type |
|
|
114
|
+
| `WorkflowFailed` | Workflow fails | workflow_id, run_id, workflow_type, **error** |
|
|
115
|
+
| `SignalReceived` | Signal received | workflow_id, signal_name, signal_args |
|
|
116
|
+
| `ActivityStarted` | Activity begins | activity_id, activity_type, **activity_input** |
|
|
117
|
+
| `ActivityCompleted` | Activity ends | activity_id, activity_type, status, **activity_input**, **activity_output**, spans, error |
|
|
118
|
+
|
|
119
|
+
## Governance Verdicts
|
|
120
|
+
|
|
121
|
+
OpenBox Core returns a verdict indicating what action the SDK should take.
|
|
122
|
+
|
|
123
|
+
### v1.1 Verdict Enum (5-tier graduated response)
|
|
124
|
+
|
|
125
|
+
| Verdict | Value | SDK Behavior |
|
|
126
|
+
|---------|-------|--------------|
|
|
127
|
+
| `ALLOW` | `"allow"` | Continue execution normally |
|
|
128
|
+
| `CONSTRAIN` | `"constrain"` | Log constraints, continue (sandbox enforcement future) |
|
|
129
|
+
| `REQUIRE_APPROVAL` | `"require_approval"` | Pause, poll for human approval |
|
|
130
|
+
| `BLOCK` | `"block"` | Raise non-retryable error |
|
|
131
|
+
| `HALT` | `"halt"` | Raise non-retryable error, terminate workflow |
|
|
132
|
+
|
|
133
|
+
### Backward Compatibility (v1.0)
|
|
134
|
+
|
|
135
|
+
The SDK automatically maps v1.0 action strings to v1.1 verdicts:
|
|
136
|
+
|
|
137
|
+
| v1.0 Action | v1.1 Verdict |
|
|
138
|
+
|-------------|--------------|
|
|
139
|
+
| `"continue"` | `ALLOW` |
|
|
140
|
+
| `"stop"` | `HALT` |
|
|
141
|
+
| `"require-approval"` | `REQUIRE_APPROVAL` |
|
|
142
|
+
|
|
143
|
+
### Verdict Priority
|
|
144
|
+
|
|
145
|
+
When aggregating multiple verdicts (e.g., from multiple policies), the highest priority wins:
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
HALT (5) > BLOCK (4) > REQUIRE_APPROVAL (3) > CONSTRAIN (2) > ALLOW (1)
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### v1.1 Response Fields
|
|
152
|
+
|
|
153
|
+
| Field | Type | Description |
|
|
154
|
+
|-------|------|-------------|
|
|
155
|
+
| `verdict` | `string` | v1.1 verdict value (see table above) |
|
|
156
|
+
| `action` | `string` | v1.0 action (for backward compat) |
|
|
157
|
+
| `reason` | `string` | Human-readable explanation |
|
|
158
|
+
| `policy_id` | `string` | Policy that triggered the verdict |
|
|
159
|
+
| `risk_score` | `float` | Risk score (0.0 - 1.0) |
|
|
160
|
+
| `trust_tier` | `string` | Trust tier (v1.1) |
|
|
161
|
+
| `alignment_score` | `float` | Alignment score (v1.1) |
|
|
162
|
+
| `behavioral_violations` | `array` | List of violations (v1.1) |
|
|
163
|
+
| `approval_id` | `string` | Approval tracking ID (v1.1) |
|
|
164
|
+
| `constraints` | `array` | Constraints to apply (v1.1) |
|
|
165
|
+
| `guardrails_result` | `object` | Guardrails redaction result |
|
|
166
|
+
|
|
167
|
+
## Guardrails (Input/Output Validation & Redaction)
|
|
168
|
+
|
|
169
|
+
OpenBox Core can return `guardrails_result` to validate and modify activity input before execution or output after completion:
|
|
170
|
+
|
|
171
|
+
```json
|
|
172
|
+
{
|
|
173
|
+
"verdict": "allow",
|
|
174
|
+
"guardrails_result": {
|
|
175
|
+
"input_type": "activity_input",
|
|
176
|
+
"redacted_input": {"prompt": "[REDACTED]", "user_id": "123"},
|
|
177
|
+
"raw_logs": {"evaluation_id": "eval-123", "model": "gpt-4"},
|
|
178
|
+
"validation_passed": true,
|
|
179
|
+
"reasons": []
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### Guardrails Response Fields
|
|
185
|
+
|
|
186
|
+
| Field | Type | Description |
|
|
187
|
+
|-------|------|-------------|
|
|
188
|
+
| `input_type` | `string` | `"activity_input"` or `"activity_output"` |
|
|
189
|
+
| `redacted_input` | `any` | Redacted/modified data to replace original |
|
|
190
|
+
| `raw_logs` | `object` | Raw logs from guardrails evaluation |
|
|
191
|
+
| `validation_passed` | `bool` | If `false`, workflow is terminated |
|
|
192
|
+
| `reasons` | `array` | Validation failure reasons (see below) |
|
|
193
|
+
|
|
194
|
+
### Validation Failure
|
|
195
|
+
|
|
196
|
+
When `validation_passed` is `false`, the workflow is terminated with a non-retryable `ApplicationError` of type `GuardrailsValidationFailed`. The `reasons` array contains structured failure details:
|
|
197
|
+
|
|
198
|
+
```json
|
|
199
|
+
{
|
|
200
|
+
"verdict": "allow",
|
|
201
|
+
"guardrails_result": {
|
|
202
|
+
"input_type": "activity_input",
|
|
203
|
+
"validation_passed": false,
|
|
204
|
+
"reasons": [
|
|
205
|
+
{"type": "pii", "field": "email", "reason": "Contains PII data"},
|
|
206
|
+
{"type": "sensitive", "field": "ssn", "reason": "SSN detected in input"}
|
|
207
|
+
]
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Each reason object:
|
|
213
|
+
| Field | Type | Description |
|
|
214
|
+
|-------|------|-------------|
|
|
215
|
+
| `type` | `string` | Category of validation failure |
|
|
216
|
+
| `field` | `string` | Field that triggered the failure |
|
|
217
|
+
| `reason` | `string` | Human-readable explanation |
|
|
218
|
+
|
|
219
|
+
### Redaction Behavior
|
|
220
|
+
|
|
221
|
+
| `input_type` | When Applied | Effect |
|
|
222
|
+
|--------------|--------------|--------|
|
|
223
|
+
| `activity_input` | Before activity executes | Replaces activity input with redacted version |
|
|
224
|
+
| `activity_output` | After activity completes | Replaces activity output with redacted version |
|
|
225
|
+
|
|
226
|
+
This allows governance to:
|
|
227
|
+
1. **Validate** - Block workflows that violate policies (PII, sensitive data, etc.)
|
|
228
|
+
2. **Redact** - Sanitize sensitive data without stopping the workflow
|
|
229
|
+
|
|
230
|
+
## Error Handling Policy
|
|
231
|
+
|
|
232
|
+
Configure via `on_api_error` in `GovernanceConfig`:
|
|
233
|
+
|
|
234
|
+
| Policy | Behavior |
|
|
235
|
+
|--------|----------|
|
|
236
|
+
| `fail_open` (default) | If governance API fails, allow workflow to continue |
|
|
237
|
+
| `fail_closed` | If governance API fails, terminate workflow |
|
|
238
|
+
|
|
239
|
+
## SDK Components
|
|
240
|
+
|
|
241
|
+
| File | Purpose |
|
|
242
|
+
|------|---------|
|
|
243
|
+
| `worker.py` | `create_openbox_worker()` - **Recommended** factory for worker-side governance |
|
|
244
|
+
| `workflow_interceptor.py` | `GovernanceInterceptor` - workflow lifecycle events (via activity for determinism) |
|
|
245
|
+
| `activity_interceptor.py` | `ActivityGovernanceInterceptor` - activity lifecycle with input/output capture, guardrails, and span collection |
|
|
246
|
+
| `activities.py` | `send_governance_event` activity for workflow-level HTTP calls |
|
|
247
|
+
| `span_processor.py` | `WorkflowSpanProcessor` - buffers spans per workflow_id, merges body/header data |
|
|
248
|
+
| `otel_setup.py` | HTTP instrumentation with body/header capture hooks |
|
|
249
|
+
| `config.py` | `GovernanceConfig` configuration options |
|
|
250
|
+
| `types.py` | `WorkflowEventType` enum, `WorkflowSpanBuffer`, `GuardrailsCheckResult` |
|
|
251
|
+
|
|
252
|
+
## Quick Start (Recommended)
|
|
253
|
+
|
|
254
|
+
Use the `create_openbox_worker()` factory function for simple integration:
|
|
255
|
+
|
|
256
|
+
```python
|
|
257
|
+
import os
|
|
258
|
+
from openbox import create_openbox_worker
|
|
259
|
+
|
|
260
|
+
worker = create_openbox_worker(
|
|
261
|
+
client=client,
|
|
262
|
+
task_queue="my-task-queue",
|
|
263
|
+
workflows=[MyWorkflow],
|
|
264
|
+
activities=[my_activity],
|
|
265
|
+
# OpenBox config
|
|
266
|
+
openbox_url=os.getenv("OPENBOX_URL"),
|
|
267
|
+
openbox_api_key=os.getenv("OPENBOX_API_KEY"),
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
await worker.run()
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
The factory function automatically:
|
|
274
|
+
1. Validates the API key
|
|
275
|
+
2. Creates WorkflowSpanProcessor
|
|
276
|
+
3. Sets up OpenTelemetry HTTP instrumentation
|
|
277
|
+
4. Creates governance interceptors (workflow + activity)
|
|
278
|
+
5. Adds `send_governance_event` activity
|
|
279
|
+
6. Returns a fully configured Worker
|
|
280
|
+
|
|
281
|
+
### All Parameters
|
|
282
|
+
|
|
283
|
+
```python
|
|
284
|
+
worker = create_openbox_worker(
|
|
285
|
+
client=client,
|
|
286
|
+
task_queue="my-task-queue",
|
|
287
|
+
workflows=[MyWorkflow],
|
|
288
|
+
activities=[my_activity],
|
|
289
|
+
|
|
290
|
+
# OpenBox config (required for governance)
|
|
291
|
+
openbox_url="http://localhost:8086",
|
|
292
|
+
openbox_api_key="obx_test_key_1",
|
|
293
|
+
governance_timeout=30.0, # default: 30.0
|
|
294
|
+
governance_policy="fail_closed", # default: "fail_open"
|
|
295
|
+
|
|
296
|
+
# Event filtering
|
|
297
|
+
send_start_event=True,
|
|
298
|
+
send_activity_start_event=True,
|
|
299
|
+
skip_workflow_types={"InternalWorkflow"},
|
|
300
|
+
skip_activity_types={"send_governance_event"},
|
|
301
|
+
skip_signals={"heartbeat"},
|
|
302
|
+
|
|
303
|
+
# Database instrumentation
|
|
304
|
+
instrument_databases=True, # default: True
|
|
305
|
+
db_libraries={"psycopg2", "redis"}, # default: None (all available)
|
|
306
|
+
|
|
307
|
+
# File I/O instrumentation
|
|
308
|
+
instrument_file_io=True, # default: False
|
|
309
|
+
|
|
310
|
+
# Standard Worker options (all supported)
|
|
311
|
+
activity_executor=my_executor,
|
|
312
|
+
max_concurrent_activities=10,
|
|
313
|
+
# ... any other Worker parameter
|
|
314
|
+
)
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
## Advanced Usage
|
|
318
|
+
|
|
319
|
+
For fine-grained control, you can configure components manually:
|
|
320
|
+
|
|
321
|
+
```python
|
|
322
|
+
from temporalio.worker import Worker
|
|
323
|
+
from openbox import (
|
|
324
|
+
initialize,
|
|
325
|
+
WorkflowSpanProcessor,
|
|
326
|
+
GovernanceInterceptor,
|
|
327
|
+
GovernanceConfig,
|
|
328
|
+
)
|
|
329
|
+
from openbox.otel_setup import setup_opentelemetry_for_governance
|
|
330
|
+
from openbox.activity_interceptor import ActivityGovernanceInterceptor
|
|
331
|
+
from openbox.activities import send_governance_event
|
|
332
|
+
|
|
333
|
+
# Configuration
|
|
334
|
+
openbox_url = "http://localhost:8086"
|
|
335
|
+
openbox_key = "obx_test_key_1"
|
|
336
|
+
|
|
337
|
+
# 1. Initialize SDK (validates API key)
|
|
338
|
+
initialize(api_url=openbox_url, api_key=openbox_key)
|
|
339
|
+
|
|
340
|
+
# 2. Create span processor (ignore OpenBox API calls)
|
|
341
|
+
span_processor = WorkflowSpanProcessor(ignored_url_prefixes=[openbox_url])
|
|
342
|
+
|
|
343
|
+
# 3. Setup OTel instrumentation with body/header capture
|
|
344
|
+
setup_opentelemetry_for_governance(span_processor, ignored_urls=[openbox_url])
|
|
345
|
+
|
|
346
|
+
# 4. Create governance config
|
|
347
|
+
config = GovernanceConfig(
|
|
348
|
+
on_api_error="fail_closed", # or "fail_open"
|
|
349
|
+
api_timeout=30.0,
|
|
350
|
+
send_start_event=True,
|
|
351
|
+
send_activity_start_event=True,
|
|
352
|
+
skip_workflow_types={"InternalWorkflow"},
|
|
353
|
+
skip_activity_types={"send_governance_event"}, # Default
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
# 5. Create interceptors
|
|
357
|
+
workflow_interceptor = GovernanceInterceptor(
|
|
358
|
+
api_url=openbox_url,
|
|
359
|
+
api_key=openbox_key,
|
|
360
|
+
span_processor=span_processor,
|
|
361
|
+
config=config,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
activity_interceptor = ActivityGovernanceInterceptor(
|
|
365
|
+
api_url=openbox_url,
|
|
366
|
+
api_key=openbox_key,
|
|
367
|
+
span_processor=span_processor,
|
|
368
|
+
config=config,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
# 6. Create worker with both interceptors
|
|
372
|
+
worker = Worker(
|
|
373
|
+
client=client,
|
|
374
|
+
task_queue="my-task-queue",
|
|
375
|
+
workflows=[MyWorkflow],
|
|
376
|
+
activities=[my_activity, send_governance_event], # Include governance activity!
|
|
377
|
+
interceptors=[workflow_interceptor, activity_interceptor],
|
|
378
|
+
)
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
## Event Payloads
|
|
382
|
+
|
|
383
|
+
### WorkflowStarted
|
|
384
|
+
```json
|
|
385
|
+
{
|
|
386
|
+
"source": "workflow-telemetry",
|
|
387
|
+
"event_type": "WorkflowStarted",
|
|
388
|
+
"workflow_id": "my-workflow-123",
|
|
389
|
+
"run_id": "abc-123",
|
|
390
|
+
"workflow_type": "MyWorkflow",
|
|
391
|
+
"task_queue": "my-task-queue",
|
|
392
|
+
"timestamp": "2024-01-01T00:00:00.000Z"
|
|
393
|
+
}
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
### WorkflowFailed
|
|
397
|
+
```json
|
|
398
|
+
{
|
|
399
|
+
"source": "workflow-telemetry",
|
|
400
|
+
"event_type": "WorkflowFailed",
|
|
401
|
+
"workflow_id": "my-workflow-123",
|
|
402
|
+
"run_id": "abc-123",
|
|
403
|
+
"workflow_type": "MyWorkflow",
|
|
404
|
+
"error": {
|
|
405
|
+
"type": "ApplicationError",
|
|
406
|
+
"message": "Governance blocked: Policy violation detected"
|
|
407
|
+
},
|
|
408
|
+
"timestamp": "2024-01-01T00:00:00.000Z"
|
|
409
|
+
}
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
### ActivityCompleted (with input/output and spans)
|
|
413
|
+
```json
|
|
414
|
+
{
|
|
415
|
+
"source": "workflow-telemetry",
|
|
416
|
+
"event_type": "ActivityCompleted",
|
|
417
|
+
"workflow_id": "my-workflow-123",
|
|
418
|
+
"activity_id": "1",
|
|
419
|
+
"activity_type": "call_llm",
|
|
420
|
+
"status": "completed",
|
|
421
|
+
"duration_ms": 1234.56,
|
|
422
|
+
"activity_input": [{"prompt": "Hello, how are you?"}],
|
|
423
|
+
"activity_output": {"response": "I'm doing well, thank you!"},
|
|
424
|
+
"span_count": 1,
|
|
425
|
+
"spans": [
|
|
426
|
+
{
|
|
427
|
+
"span_id": "abc123",
|
|
428
|
+
"trace_id": "def456",
|
|
429
|
+
"name": "POST",
|
|
430
|
+
"kind": "CLIENT",
|
|
431
|
+
"attributes": {
|
|
432
|
+
"http.method": "POST",
|
|
433
|
+
"http.url": "https://api.openai.com/v1/chat/completions",
|
|
434
|
+
"http.status_code": 200
|
|
435
|
+
},
|
|
436
|
+
"request_body": "{\"model\":\"gpt-4\",\"messages\":[...]}",
|
|
437
|
+
"response_body": "{\"choices\":[...]}"
|
|
438
|
+
}
|
|
439
|
+
],
|
|
440
|
+
"timestamp": "2024-01-01T00:00:00.000Z"
|
|
441
|
+
}
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
### Governance Stop Response
|
|
445
|
+
When OpenBox Core returns `verdict: "block"` or `verdict: "halt"` (or v1.0 `action: "stop"`):
|
|
446
|
+
```json
|
|
447
|
+
{
|
|
448
|
+
"verdict": "halt",
|
|
449
|
+
"reason": "Policy violation: unauthorized API call detected",
|
|
450
|
+
"policy_id": "policy-123",
|
|
451
|
+
"risk_score": 0.95,
|
|
452
|
+
"trust_tier": "untrusted",
|
|
453
|
+
"behavioral_violations": ["unauthorized_api_call"]
|
|
454
|
+
}
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
The workflow/activity will be terminated with a non-retryable `ApplicationError`.
|
|
458
|
+
|
|
459
|
+
### Error Types
|
|
460
|
+
|
|
461
|
+
| Error Type | Trigger | Description |
|
|
462
|
+
|------------|---------|-------------|
|
|
463
|
+
| `GovernanceStop` | `verdict: BLOCK/HALT` | Governance policy blocked the workflow |
|
|
464
|
+
| `GuardrailsValidationFailed` | `validation_passed: false` | Guardrails validation failed (PII, sensitive data, etc.) |
|
|
465
|
+
|
|
466
|
+
## Span Data Structures
|
|
467
|
+
|
|
468
|
+
OpenBox captures different types of spans, each with specific attributes. All spans share a common base structure.
|
|
469
|
+
|
|
470
|
+
### Base Span Structure
|
|
471
|
+
|
|
472
|
+
All spans include these core fields:
|
|
473
|
+
|
|
474
|
+
```json
|
|
475
|
+
{
|
|
476
|
+
"span_id": "1a2b3c4d5e6f7890",
|
|
477
|
+
"trace_id": "1a2b3c4d5e6f78901a2b3c4d5e6f7890",
|
|
478
|
+
"parent_span_id": "0987654321fedcba",
|
|
479
|
+
"name": "POST",
|
|
480
|
+
"kind": "CLIENT",
|
|
481
|
+
"start_time": 1704067200000000000,
|
|
482
|
+
"end_time": 1704067201000000000,
|
|
483
|
+
"duration_ns": 1000000000,
|
|
484
|
+
"status": {
|
|
485
|
+
"code": "OK",
|
|
486
|
+
"description": null
|
|
487
|
+
},
|
|
488
|
+
"events": [],
|
|
489
|
+
"attributes": {},
|
|
490
|
+
"activity_id": "1"
|
|
491
|
+
}
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
| Field | Type | Description |
|
|
495
|
+
|-------|------|-------------|
|
|
496
|
+
| `span_id` | `string` | 16-char hex span identifier |
|
|
497
|
+
| `trace_id` | `string` | 32-char hex trace identifier |
|
|
498
|
+
| `parent_span_id` | `string?` | Parent span ID (null for root spans) |
|
|
499
|
+
| `name` | `string` | Span name (e.g., "POST", "SELECT", "file.read") |
|
|
500
|
+
| `kind` | `string` | Span kind: `CLIENT`, `SERVER`, `INTERNAL`, `PRODUCER`, `CONSUMER` |
|
|
501
|
+
| `start_time` | `int64` | Start time in nanoseconds (Unix epoch) |
|
|
502
|
+
| `end_time` | `int64` | End time in nanoseconds |
|
|
503
|
+
| `duration_ns` | `int64` | Duration in nanoseconds |
|
|
504
|
+
| `status.code` | `string` | `OK`, `ERROR`, or `UNSET` |
|
|
505
|
+
| `status.description` | `string?` | Error description if status is ERROR |
|
|
506
|
+
| `events` | `array` | Span events (exceptions, logs) |
|
|
507
|
+
| `attributes` | `object` | Type-specific attributes (see below) |
|
|
508
|
+
| `activity_id` | `string?` | Temporal activity ID (for filtering) |
|
|
509
|
+
|
|
510
|
+
### HTTP Span Attributes
|
|
511
|
+
|
|
512
|
+
HTTP spans include request/response bodies and headers in addition to standard OTel attributes:
|
|
513
|
+
|
|
514
|
+
```json
|
|
515
|
+
{
|
|
516
|
+
"attributes": {
|
|
517
|
+
"http.method": "POST",
|
|
518
|
+
"http.url": "https://api.openai.com/v1/chat/completions",
|
|
519
|
+
"http.host": "api.openai.com",
|
|
520
|
+
"http.scheme": "https",
|
|
521
|
+
"http.status_code": 200,
|
|
522
|
+
"http.target": "/v1/chat/completions",
|
|
523
|
+
"net.peer.name": "api.openai.com",
|
|
524
|
+
"net.peer.port": 443
|
|
525
|
+
},
|
|
526
|
+
"request_body": "{\"model\":\"gpt-4\",\"messages\":[...]}",
|
|
527
|
+
"response_body": "{\"choices\":[{\"message\":{...}}]}",
|
|
528
|
+
"request_headers": {
|
|
529
|
+
"content-type": "application/json",
|
|
530
|
+
"authorization": "Bearer sk-..."
|
|
531
|
+
},
|
|
532
|
+
"response_headers": {
|
|
533
|
+
"content-type": "application/json",
|
|
534
|
+
"x-request-id": "req-123"
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
```
|
|
538
|
+
|
|
539
|
+
| Field | Type | Description |
|
|
540
|
+
|-------|------|-------------|
|
|
541
|
+
| `http.method` | `string` | HTTP method (GET, POST, PUT, DELETE, etc.) |
|
|
542
|
+
| `http.url` | `string` | Full request URL |
|
|
543
|
+
| `http.status_code` | `int` | HTTP response status code |
|
|
544
|
+
| `http.target` | `string` | Request path and query string |
|
|
545
|
+
| `net.peer.name` | `string` | Remote host name |
|
|
546
|
+
| `net.peer.port` | `int` | Remote port |
|
|
547
|
+
| `request_body` | `string?` | HTTP request body (text content types only) |
|
|
548
|
+
| `response_body` | `string?` | HTTP response body (text content types only) |
|
|
549
|
+
| `request_headers` | `object?` | HTTP request headers |
|
|
550
|
+
| `response_headers` | `object?` | HTTP response headers |
|
|
551
|
+
|
|
552
|
+
**Note:** Binary content types are not captured. Only text-based content types are included: `text/*`, `application/json`, `application/xml`, `application/javascript`, `application/x-www-form-urlencoded`.
|
|
553
|
+
|
|
554
|
+
### Database Span Attributes
|
|
555
|
+
|
|
556
|
+
Database spans capture query information:
|
|
557
|
+
|
|
558
|
+
```json
|
|
559
|
+
{
|
|
560
|
+
"name": "SELECT",
|
|
561
|
+
"attributes": {
|
|
562
|
+
"db.system": "postgresql",
|
|
563
|
+
"db.name": "mydb",
|
|
564
|
+
"db.user": "postgres",
|
|
565
|
+
"db.statement": "SELECT * FROM users WHERE id = $1",
|
|
566
|
+
"db.operation": "SELECT",
|
|
567
|
+
"net.peer.name": "localhost",
|
|
568
|
+
"net.peer.port": 5432
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
| Attribute | Type | Description |
|
|
574
|
+
|-----------|------|-------------|
|
|
575
|
+
| `db.system` | `string` | Database type: `postgresql`, `mysql`, `mongodb`, `redis` |
|
|
576
|
+
| `db.name` | `string` | Database name |
|
|
577
|
+
| `db.user` | `string` | Database user |
|
|
578
|
+
| `db.statement` | `string` | SQL query or command |
|
|
579
|
+
| `db.operation` | `string` | Operation type: `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `GET`, `SET` |
|
|
580
|
+
| `net.peer.name` | `string` | Database host |
|
|
581
|
+
| `net.peer.port` | `int` | Database port |
|
|
582
|
+
|
|
583
|
+
**MongoDB-specific:**
|
|
584
|
+
| Attribute | Description |
|
|
585
|
+
|-----------|-------------|
|
|
586
|
+
| `db.mongodb.collection` | Collection name |
|
|
587
|
+
|
|
588
|
+
**Redis-specific:**
|
|
589
|
+
| Attribute | Description |
|
|
590
|
+
|-----------|-------------|
|
|
591
|
+
| `db.redis.database_index` | Redis database index |
|
|
592
|
+
|
|
593
|
+
### File I/O Span Attributes
|
|
594
|
+
|
|
595
|
+
File operations are captured as nested spans:
|
|
596
|
+
|
|
597
|
+
```json
|
|
598
|
+
{
|
|
599
|
+
"name": "file.open",
|
|
600
|
+
"attributes": {
|
|
601
|
+
"file.path": "/app/data/config.json",
|
|
602
|
+
"file.mode": "r",
|
|
603
|
+
"file.total_bytes_read": 1024,
|
|
604
|
+
"file.total_bytes_written": 0
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
```
|
|
608
|
+
|
|
609
|
+
**file.open span:**
|
|
610
|
+
| Attribute | Type | Description |
|
|
611
|
+
|-----------|------|-------------|
|
|
612
|
+
| `file.path` | `string` | Absolute file path |
|
|
613
|
+
| `file.mode` | `string` | Open mode: `r`, `w`, `a`, `rb`, `wb`, etc. |
|
|
614
|
+
| `file.total_bytes_read` | `int` | Total bytes read (set on close) |
|
|
615
|
+
| `file.total_bytes_written` | `int` | Total bytes written (set on close) |
|
|
616
|
+
|
|
617
|
+
**file.read / file.write child spans:**
|
|
618
|
+
```json
|
|
619
|
+
{
|
|
620
|
+
"name": "file.read",
|
|
621
|
+
"attributes": {
|
|
622
|
+
"file.path": "/app/data/config.json",
|
|
623
|
+
"file.operation": "read",
|
|
624
|
+
"file.bytes": 512
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
```
|
|
628
|
+
|
|
629
|
+
| Attribute | Type | Description |
|
|
630
|
+
|-----------|------|-------------|
|
|
631
|
+
| `file.path` | `string` | File path |
|
|
632
|
+
| `file.operation` | `string` | `read`, `readline`, `readlines`, `write`, `writelines` |
|
|
633
|
+
| `file.bytes` | `int` | Bytes read/written in this operation |
|
|
634
|
+
| `file.lines` | `int` | Line count (for `readlines`/`writelines`) |
|
|
635
|
+
|
|
636
|
+
**Error attributes (on failure):**
|
|
637
|
+
| Attribute | Description |
|
|
638
|
+
|-----------|-------------|
|
|
639
|
+
| `error` | `true` if operation failed |
|
|
640
|
+
| `error.type` | Exception class name (e.g., `FileNotFoundError`) |
|
|
641
|
+
| `error.message` | Exception message |
|
|
642
|
+
|
|
643
|
+
### Internal Function Call Attributes (`@traced`)
|
|
644
|
+
|
|
645
|
+
Functions decorated with `@traced` create spans with:
|
|
646
|
+
|
|
647
|
+
```json
|
|
648
|
+
{
|
|
649
|
+
"name": "process_data",
|
|
650
|
+
"attributes": {
|
|
651
|
+
"code.function": "process_data",
|
|
652
|
+
"code.namespace": "myapp.processing",
|
|
653
|
+
"function.arg.0": "{\"input\": \"data\"}",
|
|
654
|
+
"function.kwarg.verbose": "true",
|
|
655
|
+
"function.result": "{\"output\": \"processed\"}"
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
```
|
|
659
|
+
|
|
660
|
+
| Attribute | Type | Description |
|
|
661
|
+
|-----------|------|-------------|
|
|
662
|
+
| `code.function` | `string` | Function name |
|
|
663
|
+
| `code.namespace` | `string` | Module path |
|
|
664
|
+
| `function.arg.N` | `string` | Positional argument at index N (JSON serialized) |
|
|
665
|
+
| `function.kwarg.X` | `string` | Keyword argument named X (JSON serialized) |
|
|
666
|
+
| `function.result` | `string` | Return value (JSON serialized, if `capture_result=True`) |
|
|
667
|
+
|
|
668
|
+
**Error attributes (on exception):**
|
|
669
|
+
| Attribute | Description |
|
|
670
|
+
|-----------|-------------|
|
|
671
|
+
| `error` | `true` |
|
|
672
|
+
| `error.type` | Exception class name |
|
|
673
|
+
| `error.message` | Exception message |
|
|
674
|
+
|
|
675
|
+
**Note:** Arguments and results are truncated at 2000 characters by default. Configure with `max_arg_length` parameter.
|
|
676
|
+
|
|
677
|
+
## Instrumentation Setup
|
|
678
|
+
|
|
679
|
+
This section explains how to enable each type of span capture with the OpenBox SDK.
|
|
680
|
+
|
|
681
|
+
### Quick Setup (All Instrumentation)
|
|
682
|
+
|
|
683
|
+
The simplest way to enable all instrumentation:
|
|
684
|
+
|
|
685
|
+
```python
|
|
686
|
+
from openbox import create_openbox_worker
|
|
687
|
+
|
|
688
|
+
worker = create_openbox_worker(
|
|
689
|
+
client=client,
|
|
690
|
+
task_queue="my-queue",
|
|
691
|
+
workflows=[MyWorkflow],
|
|
692
|
+
activities=[my_activity],
|
|
693
|
+
|
|
694
|
+
# OpenBox config (required)
|
|
695
|
+
openbox_url=os.getenv("OPENBOX_URL"),
|
|
696
|
+
openbox_api_key=os.getenv("OPENBOX_API_KEY"),
|
|
697
|
+
|
|
698
|
+
# Instrumentation options
|
|
699
|
+
instrument_databases=True, # Capture database queries (default: True)
|
|
700
|
+
instrument_file_io=True, # Capture file operations (default: False)
|
|
701
|
+
)
|
|
702
|
+
```
|
|
703
|
+
|
|
704
|
+
### HTTP Instrumentation (Auto-enabled)
|
|
705
|
+
|
|
706
|
+
HTTP instrumentation is **automatically enabled** when using `create_openbox_worker()`. No additional setup required.
|
|
707
|
+
|
|
708
|
+
**Supported libraries:**
|
|
709
|
+
| Library | Package | Notes |
|
|
710
|
+
|---------|---------|-------|
|
|
711
|
+
| httpx | `opentelemetry-instrumentation-httpx` | Sync + async, body capture via patching |
|
|
712
|
+
| requests | `opentelemetry-instrumentation-requests` | Full body capture |
|
|
713
|
+
| urllib3 | `opentelemetry-instrumentation-urllib3` | Full body capture |
|
|
714
|
+
| urllib | `opentelemetry-instrumentation-urllib` | Request body only |
|
|
715
|
+
|
|
716
|
+
**Required packages** (install if not present):
|
|
717
|
+
```bash
|
|
718
|
+
uv add opentelemetry-instrumentation-httpx
|
|
719
|
+
uv add opentelemetry-instrumentation-requests
|
|
720
|
+
uv add opentelemetry-instrumentation-urllib3
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
### Database Instrumentation
|
|
724
|
+
|
|
725
|
+
Database instrumentation is **enabled by default** but requires the corresponding OTel instrumentation package.
|
|
726
|
+
|
|
727
|
+
**Supported Databases:**
|
|
728
|
+
|
|
729
|
+
| Database | Driver Library | OTel Instrumentation Package | `db_libraries` key |
|
|
730
|
+
|----------|---------------|------------------------------|-------------------|
|
|
731
|
+
| PostgreSQL | `psycopg2` / `psycopg2-binary` | `opentelemetry-instrumentation-psycopg2` | `"psycopg2"` |
|
|
732
|
+
| PostgreSQL (async) | `asyncpg` | `opentelemetry-instrumentation-asyncpg` | `"asyncpg"` |
|
|
733
|
+
| MySQL | `mysql-connector-python` | `opentelemetry-instrumentation-mysql` | `"mysql"` |
|
|
734
|
+
| MySQL | `pymysql` | `opentelemetry-instrumentation-pymysql` | `"pymysql"` |
|
|
735
|
+
| MongoDB | `pymongo` | `opentelemetry-instrumentation-pymongo` | `"pymongo"` |
|
|
736
|
+
| Redis | `redis` | `opentelemetry-instrumentation-redis` | `"redis"` |
|
|
737
|
+
| SQLAlchemy (ORM) | `sqlalchemy` | `opentelemetry-instrumentation-sqlalchemy` | `"sqlalchemy"` |
|
|
738
|
+
|
|
739
|
+
**Captured Span Attributes by Database:**
|
|
740
|
+
|
|
741
|
+
| Database | `db.system` | `db.statement` | `db.operation` | Extra Attributes |
|
|
742
|
+
|----------|-------------|----------------|----------------|------------------|
|
|
743
|
+
| PostgreSQL | `postgresql` | SQL query | `SELECT`, `INSERT`, etc. | `db.name`, `db.user` |
|
|
744
|
+
| MySQL | `mysql` | SQL query | `SELECT`, `INSERT`, etc. | `db.name`, `db.user` |
|
|
745
|
+
| MongoDB | `mongodb` | Command JSON | `find`, `insert`, etc. | `db.mongodb.collection` |
|
|
746
|
+
| Redis | `redis` | Command | `GET`, `SET`, `HGET`, etc. | `db.redis.database_index` |
|
|
747
|
+
| SQLAlchemy | varies | SQL query | `SELECT`, `INSERT`, etc. | `db.name` |
|
|
748
|
+
|
|
749
|
+
**Step 1: Install the instrumentation package for your database:**
|
|
750
|
+
|
|
751
|
+
```bash
|
|
752
|
+
# PostgreSQL (sync)
|
|
753
|
+
uv add psycopg2-binary opentelemetry-instrumentation-psycopg2
|
|
754
|
+
|
|
755
|
+
# PostgreSQL (async)
|
|
756
|
+
uv add asyncpg opentelemetry-instrumentation-asyncpg
|
|
757
|
+
|
|
758
|
+
# MySQL
|
|
759
|
+
uv add mysql-connector-python opentelemetry-instrumentation-mysql
|
|
760
|
+
|
|
761
|
+
# PyMySQL
|
|
762
|
+
uv add pymysql opentelemetry-instrumentation-pymysql
|
|
763
|
+
|
|
764
|
+
# MongoDB
|
|
765
|
+
uv add pymongo opentelemetry-instrumentation-pymongo
|
|
766
|
+
|
|
767
|
+
# Redis
|
|
768
|
+
uv add redis opentelemetry-instrumentation-redis
|
|
769
|
+
|
|
770
|
+
# SQLAlchemy ORM
|
|
771
|
+
uv add sqlalchemy opentelemetry-instrumentation-sqlalchemy
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
**Step 2: Configure the worker:**
|
|
775
|
+
|
|
776
|
+
```python
|
|
777
|
+
# Option A: Instrument all available databases (default)
|
|
778
|
+
worker = create_openbox_worker(
|
|
779
|
+
...,
|
|
780
|
+
instrument_databases=True, # Default
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
# Option B: Instrument specific databases only
|
|
784
|
+
worker = create_openbox_worker(
|
|
785
|
+
...,
|
|
786
|
+
db_libraries={"psycopg2", "redis"}, # Only these
|
|
787
|
+
)
|
|
788
|
+
|
|
789
|
+
# Option C: Disable database instrumentation
|
|
790
|
+
worker = create_openbox_worker(
|
|
791
|
+
...,
|
|
792
|
+
instrument_databases=False,
|
|
793
|
+
)
|
|
794
|
+
```
|
|
795
|
+
|
|
796
|
+
**Example: Capturing PostgreSQL queries**
|
|
797
|
+
|
|
798
|
+
```python
|
|
799
|
+
import psycopg2
|
|
800
|
+
|
|
801
|
+
# This query will be captured as a span
|
|
802
|
+
conn = psycopg2.connect("postgresql://user:pass@localhost/mydb")
|
|
803
|
+
cursor = conn.cursor()
|
|
804
|
+
cursor.execute("SELECT * FROM users WHERE id = %s", (user_id,))
|
|
805
|
+
row = cursor.fetchone()
|
|
806
|
+
cursor.close()
|
|
807
|
+
conn.close()
|
|
808
|
+
```
|
|
809
|
+
|
|
810
|
+
The span will include:
|
|
811
|
+
```json
|
|
812
|
+
{
|
|
813
|
+
"name": "SELECT",
|
|
814
|
+
"attributes": {
|
|
815
|
+
"db.system": "postgresql",
|
|
816
|
+
"db.name": "mydb",
|
|
817
|
+
"db.statement": "SELECT * FROM users WHERE id = %s",
|
|
818
|
+
"db.operation": "SELECT"
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
```
|
|
822
|
+
|
|
823
|
+
### File I/O Instrumentation
|
|
824
|
+
|
|
825
|
+
File I/O instrumentation is **disabled by default** (can be noisy). Enable it explicitly:
|
|
826
|
+
|
|
827
|
+
```python
|
|
828
|
+
worker = create_openbox_worker(
|
|
829
|
+
...,
|
|
830
|
+
instrument_file_io=True,
|
|
831
|
+
)
|
|
832
|
+
```
|
|
833
|
+
|
|
834
|
+
**Example: Capturing file operations**
|
|
835
|
+
|
|
836
|
+
```python
|
|
837
|
+
# These operations will be captured as spans
|
|
838
|
+
with open("/app/config.json", "r") as f:
|
|
839
|
+
content = f.read() # Creates file.read span
|
|
840
|
+
|
|
841
|
+
with open("/app/output.txt", "w") as f:
|
|
842
|
+
f.write("Hello, World!") # Creates file.write span
|
|
843
|
+
```
|
|
844
|
+
|
|
845
|
+
**Skipped paths:** System paths are automatically ignored to reduce noise:
|
|
846
|
+
- `/dev/`, `/proc/`, `/sys/`
|
|
847
|
+
- `__pycache__`, `.pyc`, `.pyo`, `.so`, `.dylib`
|
|
848
|
+
|
|
849
|
+
### Internal Function Tracing (`@traced`)
|
|
850
|
+
|
|
851
|
+
Use the `@traced` decorator to capture custom function calls:
|
|
852
|
+
|
|
853
|
+
**Step 1: Import the decorator**
|
|
854
|
+
|
|
855
|
+
```python
|
|
856
|
+
from openbox.tracing import traced
|
|
857
|
+
```
|
|
858
|
+
|
|
859
|
+
**Step 2: Decorate functions to trace**
|
|
860
|
+
|
|
861
|
+
```python
|
|
862
|
+
@traced
|
|
863
|
+
def process_payment(order_id: str, amount: float) -> dict:
|
|
864
|
+
# Business logic here
|
|
865
|
+
return {"status": "success", "transaction_id": "txn_123"}
|
|
866
|
+
|
|
867
|
+
@traced
|
|
868
|
+
async def fetch_user_data(user_id: str) -> dict:
|
|
869
|
+
# Async functions work too
|
|
870
|
+
return await db.get_user(user_id)
|
|
871
|
+
```
|
|
872
|
+
|
|
873
|
+
**Step 3: Configure capture options (optional)**
|
|
874
|
+
|
|
875
|
+
```python
|
|
876
|
+
# Capture arguments and results (default)
|
|
877
|
+
@traced(capture_args=True, capture_result=True)
|
|
878
|
+
def my_function(data):
|
|
879
|
+
return process(data)
|
|
880
|
+
|
|
881
|
+
# Don't capture sensitive results
|
|
882
|
+
@traced(capture_result=False)
|
|
883
|
+
def handle_password(password: str) -> bool:
|
|
884
|
+
return verify(password)
|
|
885
|
+
|
|
886
|
+
# Custom span name
|
|
887
|
+
@traced(name="payment-processing")
|
|
888
|
+
def process_payment(order):
|
|
889
|
+
return charge(order)
|
|
890
|
+
|
|
891
|
+
# Limit argument size (default: 2000 chars)
|
|
892
|
+
@traced(max_arg_length=500)
|
|
893
|
+
def handle_large_input(big_data):
|
|
894
|
+
return summarize(big_data)
|
|
895
|
+
```
|
|
896
|
+
|
|
897
|
+
**Manual span creation** (for fine-grained control):
|
|
898
|
+
|
|
899
|
+
```python
|
|
900
|
+
from openbox.tracing import create_span
|
|
901
|
+
|
|
902
|
+
def complex_operation(data):
|
|
903
|
+
with create_span("validate-input", {"data_size": len(data)}) as span:
|
|
904
|
+
validated = validate(data)
|
|
905
|
+
span.set_attribute("validation.passed", True)
|
|
906
|
+
|
|
907
|
+
with create_span("transform-data") as span:
|
|
908
|
+
result = transform(validated)
|
|
909
|
+
span.set_attribute("output_size", len(result))
|
|
910
|
+
|
|
911
|
+
return result
|
|
912
|
+
```
|
|
913
|
+
|
|
914
|
+
### Advanced: Manual Setup
|
|
915
|
+
|
|
916
|
+
For fine-grained control, set up instrumentation manually:
|
|
917
|
+
|
|
918
|
+
```python
|
|
919
|
+
from openbox.span_processor import WorkflowSpanProcessor
|
|
920
|
+
from openbox.otel_setup import setup_opentelemetry_for_governance
|
|
921
|
+
|
|
922
|
+
# Create span processor
|
|
923
|
+
span_processor = WorkflowSpanProcessor(
|
|
924
|
+
ignored_url_prefixes=["http://localhost:8086"] # Ignore OpenBox API
|
|
925
|
+
)
|
|
926
|
+
|
|
927
|
+
# Setup instrumentation
|
|
928
|
+
setup_opentelemetry_for_governance(
|
|
929
|
+
span_processor=span_processor,
|
|
930
|
+
ignored_urls=["http://localhost:8086"],
|
|
931
|
+
|
|
932
|
+
# Database options
|
|
933
|
+
instrument_databases=True,
|
|
934
|
+
db_libraries={"psycopg2", "asyncpg", "redis"}, # Or None for all
|
|
935
|
+
|
|
936
|
+
# File I/O options
|
|
937
|
+
instrument_file_io=True,
|
|
938
|
+
)
|
|
939
|
+
```
|
|
940
|
+
|
|
941
|
+
### Troubleshooting
|
|
942
|
+
|
|
943
|
+
**Database queries not captured?**
|
|
944
|
+
|
|
945
|
+
1. Check if the OTel instrumentation package is installed:
|
|
946
|
+
```bash
|
|
947
|
+
uv pip list | grep instrumentation
|
|
948
|
+
```
|
|
949
|
+
|
|
950
|
+
2. Ensure instrumentation is enabled BEFORE database connections are created:
|
|
951
|
+
```python
|
|
952
|
+
# WRONG: Database imported before worker setup
|
|
953
|
+
import psycopg2 # Module-level import
|
|
954
|
+
|
|
955
|
+
worker = create_openbox_worker(...) # Too late!
|
|
956
|
+
|
|
957
|
+
# RIGHT: Worker setup happens at application start
|
|
958
|
+
# (before any database operations)
|
|
959
|
+
```
|
|
960
|
+
|
|
961
|
+
3. Verify OpenBox is configured:
|
|
962
|
+
```bash
|
|
963
|
+
# These must be set
|
|
964
|
+
echo $OPENBOX_URL
|
|
965
|
+
echo $OPENBOX_API_KEY
|
|
966
|
+
```
|
|
967
|
+
|
|
968
|
+
**File I/O spans missing?**
|
|
969
|
+
|
|
970
|
+
1. Ensure `instrument_file_io=True` is set
|
|
971
|
+
2. Check if the path is in the skip list (system paths are ignored)
|
|
972
|
+
|
|
973
|
+
**HTTP spans missing bodies?**
|
|
974
|
+
|
|
975
|
+
1. Only text content types are captured (not binary)
|
|
976
|
+
2. Check if the URL is in the ignored list
|
|
977
|
+
3. Ensure httpx/requests instrumentation packages are installed
|
|
978
|
+
|
|
979
|
+
## Configuration Options
|
|
980
|
+
|
|
981
|
+
| Option | Default | Description |
|
|
982
|
+
|--------|---------|-------------|
|
|
983
|
+
| `on_api_error` | `"fail_open"` | `"fail_open"` = continue on API error, `"fail_closed"` = stop on API error |
|
|
984
|
+
| `api_timeout` | `30.0` | HTTP timeout for governance API calls (seconds) |
|
|
985
|
+
| `send_start_event` | `True` | Send WorkflowStarted events |
|
|
986
|
+
| `send_activity_start_event` | `True` | Send ActivityStarted events (with input) |
|
|
987
|
+
| `skip_workflow_types` | `set()` | Workflow types to skip |
|
|
988
|
+
| `skip_activity_types` | `{"send_governance_event"}` | Activity types to skip |
|
|
989
|
+
| `skip_signals` | `set()` | Signal names to skip |
|
|
990
|
+
|
|
991
|
+
## Environment Variables (Example)
|
|
992
|
+
|
|
993
|
+
Configure these in your `.env` file and pass to `create_openbox_worker()`:
|
|
994
|
+
|
|
995
|
+
```bash
|
|
996
|
+
OPENBOX_URL=http://localhost:8086
|
|
997
|
+
OPENBOX_API_KEY=obx_test_key_1
|
|
998
|
+
OPENBOX_GOVERNANCE_TIMEOUT=30.0
|
|
999
|
+
OPENBOX_GOVERNANCE_POLICY=fail_closed # fail_open or fail_closed
|
|
1000
|
+
```
|
|
1001
|
+
|
|
1002
|
+
```python
|
|
1003
|
+
# In your worker code
|
|
1004
|
+
worker = create_openbox_worker(
|
|
1005
|
+
...,
|
|
1006
|
+
openbox_url=os.getenv("OPENBOX_URL"),
|
|
1007
|
+
openbox_api_key=os.getenv("OPENBOX_API_KEY"),
|
|
1008
|
+
governance_timeout=float(os.getenv("OPENBOX_GOVERNANCE_TIMEOUT", "30.0")),
|
|
1009
|
+
governance_policy=os.getenv("OPENBOX_GOVERNANCE_POLICY", "fail_open"),
|
|
1010
|
+
)
|
|
1011
|
+
```
|
|
1012
|
+
|
|
1013
|
+
## Key Design Decisions
|
|
1014
|
+
|
|
1015
|
+
1. **Workflow determinism**: Workflow interceptor sends events via `send_governance_event` activity because workflows cannot make HTTP calls directly.
|
|
1016
|
+
|
|
1017
|
+
2. **Activity direct HTTP**: Activity interceptor sends events directly since activities are allowed to make HTTP calls.
|
|
1018
|
+
|
|
1019
|
+
3. **Input/Output capture**: Activity arguments and return values are serialized and included in governance events for policy evaluation.
|
|
1020
|
+
|
|
1021
|
+
4. **Body/header capture**: Stored separately from OTel span attributes to keep sensitive data out of external tracing systems.
|
|
1022
|
+
|
|
1023
|
+
5. **trace_id mapping**: Child HTTP spans are associated with parent activity via trace_id → workflow_id/activity_id mapping.
|
|
1024
|
+
|
|
1025
|
+
6. **Governance stop**: When API returns `verdict: "block"` or `verdict: "halt"`, raises `ApplicationError` with `non_retryable=True` to immediately terminate the workflow.
|
|
1026
|
+
|
|
1027
|
+
7. **Fail-open/closed policy**: Configurable behavior when governance API is unreachable.
|
|
1028
|
+
|
|
1029
|
+
## Function Tracing
|
|
1030
|
+
|
|
1031
|
+
OpenBox SDK provides a `@traced` decorator to capture internal function calls as spans. These spans are automatically included in governance events.
|
|
1032
|
+
|
|
1033
|
+
### Basic Usage
|
|
1034
|
+
|
|
1035
|
+
```python
|
|
1036
|
+
from openbox.tracing import traced
|
|
1037
|
+
|
|
1038
|
+
@traced
|
|
1039
|
+
def process_data(input_data):
|
|
1040
|
+
return transform(input_data)
|
|
1041
|
+
|
|
1042
|
+
@traced
|
|
1043
|
+
async def fetch_external_data(url):
|
|
1044
|
+
return await http_get(url)
|
|
1045
|
+
```
|
|
1046
|
+
|
|
1047
|
+
### With Options
|
|
1048
|
+
|
|
1049
|
+
```python
|
|
1050
|
+
from openbox.tracing import traced
|
|
1051
|
+
|
|
1052
|
+
@traced(name="custom-span-name", capture_args=True, capture_result=True)
|
|
1053
|
+
def my_function(data):
|
|
1054
|
+
return process(data)
|
|
1055
|
+
|
|
1056
|
+
# Don't capture sensitive results
|
|
1057
|
+
@traced(capture_result=False)
|
|
1058
|
+
def handle_credentials(username, password):
|
|
1059
|
+
return authenticate(username, password)
|
|
1060
|
+
```
|
|
1061
|
+
|
|
1062
|
+
### Manual Span Creation
|
|
1063
|
+
|
|
1064
|
+
```python
|
|
1065
|
+
from openbox.tracing import create_span
|
|
1066
|
+
|
|
1067
|
+
def complex_operation(data):
|
|
1068
|
+
with create_span("step-1", {"input": data}) as span:
|
|
1069
|
+
result = do_step_1(data)
|
|
1070
|
+
span.set_attribute("step1.result", result)
|
|
1071
|
+
|
|
1072
|
+
with create_span("step-2") as span:
|
|
1073
|
+
final = do_step_2(result)
|
|
1074
|
+
|
|
1075
|
+
return final
|
|
1076
|
+
```
|
|
1077
|
+
|
|
1078
|
+
### Span Attributes
|
|
1079
|
+
|
|
1080
|
+
Traced functions include these attributes:
|
|
1081
|
+
| Attribute | Description |
|
|
1082
|
+
|-----------|-------------|
|
|
1083
|
+
| `code.function` | Function name |
|
|
1084
|
+
| `code.namespace` | Module name |
|
|
1085
|
+
| `function.arg.N` | Positional arguments (if `capture_args=True`) |
|
|
1086
|
+
| `function.kwarg.X` | Keyword arguments (if `capture_args=True`) |
|
|
1087
|
+
| `function.result` | Return value (if `capture_result=True`) |
|
|
1088
|
+
| `error` | `True` if exception occurred |
|
|
1089
|
+
| `error.type` | Exception class name |
|
|
1090
|
+
| `error.message` | Exception message |
|
|
1091
|
+
|
|
1092
|
+
## File I/O Instrumentation
|
|
1093
|
+
|
|
1094
|
+
OpenBox SDK can capture file read/write operations as spans.
|
|
1095
|
+
|
|
1096
|
+
### Enabling File I/O Instrumentation
|
|
1097
|
+
|
|
1098
|
+
```python
|
|
1099
|
+
worker = create_openbox_worker(
|
|
1100
|
+
...,
|
|
1101
|
+
instrument_file_io=True,
|
|
1102
|
+
)
|
|
1103
|
+
```
|
|
1104
|
+
|
|
1105
|
+
### Captured Operations
|
|
1106
|
+
|
|
1107
|
+
| Operation | Span Name | Attributes |
|
|
1108
|
+
|-----------|-----------|------------|
|
|
1109
|
+
| `open(path, mode)` | `file.open` | `file.path`, `file.mode` |
|
|
1110
|
+
| `file.read()` | `file.read` | `file.path`, `file.bytes` |
|
|
1111
|
+
| `file.write(data)` | `file.write` | `file.path`, `file.bytes` |
|
|
1112
|
+
| `file.readline()` | `file.readline` | `file.path`, `file.bytes` |
|
|
1113
|
+
| `file.readlines()` | `file.readlines` | `file.path`, `file.lines`, `file.bytes` |
|
|
1114
|
+
|
|
1115
|
+
### Skipped Paths
|
|
1116
|
+
|
|
1117
|
+
System paths are automatically skipped: `/dev/`, `/proc/`, `/sys/`, `__pycache__`, `.pyc`, `.so`
|
|
1118
|
+
|
|
1119
|
+
## Database Instrumentation
|
|
1120
|
+
|
|
1121
|
+
OpenBox SDK can capture database queries as spans, enabling governance policies on database operations.
|
|
1122
|
+
|
|
1123
|
+
### Supported Databases
|
|
1124
|
+
|
|
1125
|
+
| Database | Library | OTel Package |
|
|
1126
|
+
|----------|---------|--------------|
|
|
1127
|
+
| PostgreSQL | psycopg2 | `opentelemetry-instrumentation-psycopg2` |
|
|
1128
|
+
| PostgreSQL (async) | asyncpg | `opentelemetry-instrumentation-asyncpg` |
|
|
1129
|
+
| MySQL | mysql-connector-python | `opentelemetry-instrumentation-mysql` |
|
|
1130
|
+
| MySQL | pymysql | `opentelemetry-instrumentation-pymysql` |
|
|
1131
|
+
| MongoDB | pymongo | `opentelemetry-instrumentation-pymongo` |
|
|
1132
|
+
| Redis | redis | `opentelemetry-instrumentation-redis` |
|
|
1133
|
+
| SQLAlchemy | sqlalchemy | `opentelemetry-instrumentation-sqlalchemy` |
|
|
1134
|
+
|
|
1135
|
+
### Enabling Database Instrumentation
|
|
1136
|
+
|
|
1137
|
+
Database instrumentation is **enabled by default**. Install the OTel instrumentation package for your database:
|
|
1138
|
+
|
|
1139
|
+
```bash
|
|
1140
|
+
# PostgreSQL
|
|
1141
|
+
pip install opentelemetry-instrumentation-psycopg2
|
|
1142
|
+
|
|
1143
|
+
# Or for async PostgreSQL
|
|
1144
|
+
pip install opentelemetry-instrumentation-asyncpg
|
|
1145
|
+
|
|
1146
|
+
# MongoDB
|
|
1147
|
+
pip install opentelemetry-instrumentation-pymongo
|
|
1148
|
+
|
|
1149
|
+
# Redis
|
|
1150
|
+
pip install opentelemetry-instrumentation-redis
|
|
1151
|
+
```
|
|
1152
|
+
|
|
1153
|
+
### Configuration
|
|
1154
|
+
|
|
1155
|
+
```python
|
|
1156
|
+
# Default: instrument all available databases
|
|
1157
|
+
worker = create_openbox_worker(
|
|
1158
|
+
client=client,
|
|
1159
|
+
task_queue="my-queue",
|
|
1160
|
+
workflows=[MyWorkflow],
|
|
1161
|
+
activities=[my_activity],
|
|
1162
|
+
openbox_url=os.getenv("OPENBOX_URL"),
|
|
1163
|
+
openbox_api_key=os.getenv("OPENBOX_API_KEY"),
|
|
1164
|
+
# instrument_databases=True (default)
|
|
1165
|
+
)
|
|
1166
|
+
|
|
1167
|
+
# Disable database instrumentation
|
|
1168
|
+
worker = create_openbox_worker(
|
|
1169
|
+
...,
|
|
1170
|
+
instrument_databases=False,
|
|
1171
|
+
)
|
|
1172
|
+
|
|
1173
|
+
# Instrument only specific databases
|
|
1174
|
+
worker = create_openbox_worker(
|
|
1175
|
+
...,
|
|
1176
|
+
db_libraries={"psycopg2", "redis"},
|
|
1177
|
+
)
|
|
1178
|
+
```
|
|
1179
|
+
|
|
1180
|
+
### Database Span Data
|
|
1181
|
+
|
|
1182
|
+
Database spans include:
|
|
1183
|
+
|
|
1184
|
+
| Attribute | Description |
|
|
1185
|
+
|-----------|-------------|
|
|
1186
|
+
| `db.system` | Database type (postgresql, mysql, mongodb, redis) |
|
|
1187
|
+
| `db.name` | Database name |
|
|
1188
|
+
| `db.statement` | SQL query or command |
|
|
1189
|
+
| `db.operation` | Operation type (SELECT, INSERT, GET, etc.) |
|
|
1190
|
+
| `net.peer.name` | Database host |
|
|
1191
|
+
| `net.peer.port` | Database port |
|
|
1192
|
+
|
|
1193
|
+
**Note:** Unlike HTTP spans, database spans do not capture query results (only the query itself).
|
|
1194
|
+
|
|
1195
|
+
## Requirements
|
|
1196
|
+
|
|
1197
|
+
- Python 3.9+
|
|
1198
|
+
- temporalio
|
|
1199
|
+
- opentelemetry-api
|
|
1200
|
+
- opentelemetry-sdk
|
|
1201
|
+
- opentelemetry-instrumentation-httpx
|
|
1202
|
+
- httpx
|
|
1203
|
+
|
|
1204
|
+
### Optional Database Instrumentation Packages
|
|
1205
|
+
|
|
1206
|
+
```bash
|
|
1207
|
+
pip install opentelemetry-instrumentation-psycopg2 # PostgreSQL
|
|
1208
|
+
pip install opentelemetry-instrumentation-asyncpg # PostgreSQL async
|
|
1209
|
+
pip install opentelemetry-instrumentation-mysql # MySQL
|
|
1210
|
+
pip install opentelemetry-instrumentation-pymysql # PyMySQL
|
|
1211
|
+
pip install opentelemetry-instrumentation-pymongo # MongoDB
|
|
1212
|
+
pip install opentelemetry-instrumentation-redis # Redis
|
|
1213
|
+
pip install opentelemetry-instrumentation-sqlalchemy # SQLAlchemy ORM
|
|
1214
|
+
```
|