aws-durable-execution-sdk-python-otel 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aws_durable_execution_sdk_python_otel-0.1.0 → aws_durable_execution_sdk_python_otel-0.3.0}/.gitignore +3 -1
- aws_durable_execution_sdk_python_otel-0.3.0/PKG-INFO +318 -0
- aws_durable_execution_sdk_python_otel-0.3.0/README.md +292 -0
- {aws_durable_execution_sdk_python_otel-0.1.0 → aws_durable_execution_sdk_python_otel-0.3.0}/pyproject.toml +51 -2
- {aws_durable_execution_sdk_python_otel-0.1.0 → aws_durable_execution_sdk_python_otel-0.3.0}/src/aws_durable_execution_sdk_python_otel/__about__.py +1 -1
- aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/__init__.py +30 -0
- aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/context_extractors.py +40 -0
- aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/deterministic_id_generator.py +127 -0
- aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/log_filter.py +104 -0
- aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/plugin.py +548 -0
- aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/py.typed +0 -0
- aws_durable_execution_sdk_python_otel-0.3.0/tests/__init__.py +0 -0
- aws_durable_execution_sdk_python_otel-0.3.0/tests/test_context_extractors.py +66 -0
- aws_durable_execution_sdk_python_otel-0.3.0/tests/test_deterministic_id_generator.py +285 -0
- aws_durable_execution_sdk_python_otel-0.3.0/tests/test_init.py +18 -0
- aws_durable_execution_sdk_python_otel-0.3.0/tests/test_log_filter.py +236 -0
- aws_durable_execution_sdk_python_otel-0.3.0/tests/test_plugin.py +800 -0
- aws_durable_execution_sdk_python_otel-0.1.0/PKG-INFO +0 -30
- aws_durable_execution_sdk_python_otel-0.1.0/README.md +0 -9
- aws_durable_execution_sdk_python_otel-0.1.0/src/aws_durable_execution_sdk_python_otel/__init__.py +0 -6
- {aws_durable_execution_sdk_python_otel-0.1.0 → aws_durable_execution_sdk_python_otel-0.3.0}/LICENSE +0 -0
- {aws_durable_execution_sdk_python_otel-0.1.0 → aws_durable_execution_sdk_python_otel-0.3.0}/NOTICE +0 -0
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aws-durable-execution-sdk-python-otel
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: OpenTelemetry instrumentation for AWS Durable Execution SDK for Python
|
|
5
|
+
Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
|
|
6
|
+
Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
|
|
7
|
+
Project-URL: Source, https://github.com/aws/aws-durable-execution-sdk-python
|
|
8
|
+
Author-email: AWS durable-execution-dev <durable-execution-dev@amazon.com>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
Keywords: durable-execution,observability,opentelemetry,tracing
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: aws-durable-execution-sdk-python>=1.5.0
|
|
21
|
+
Requires-Dist: opentelemetry-api>=1.20.0
|
|
22
|
+
Requires-Dist: opentelemetry-exporter-otlp
|
|
23
|
+
Requires-Dist: opentelemetry-propagator-aws-xray
|
|
24
|
+
Requires-Dist: opentelemetry-sdk>=1.20.0
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# AWS Durable Execution SDK - OpenTelemetry Plugin
|
|
28
|
+
|
|
29
|
+
OpenTelemetry instrumentation plugin for the [AWS Durable Execution SDK for Python](https://github.com/aws/aws-durable-execution-sdk-python). Emits distributed traces that correlate across multiple Lambda invocations of a single durable execution, producing deterministic span and trace IDs so that spans from different invocations are stitched into a single coherent trace.
|
|
30
|
+
|
|
31
|
+
## Features
|
|
32
|
+
|
|
33
|
+
- **Deterministic Trace IDs**: All invocations of the same durable execution share a single trace, derived from the X-Ray trace header or execution ARN
|
|
34
|
+
- **Span-per-Operation**: Each durable operation (step, wait, invoke) gets its own span with accurate timing
|
|
35
|
+
- **Continuation Spans**: Operations completing in a different invocation are linked back to the original span
|
|
36
|
+
- **Log Correlation**: Enrich application logs with trace ID and span ID for end-to-end observability
|
|
37
|
+
- **Configurable Sampling**: Control trace volume via plugin options
|
|
38
|
+
- **Self-Contained Setup**: No manual TracerProvider configuration required
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install aws-durable-execution-sdk-python-otel
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Quick Start using X-Ray/CloudWatch Tracing
|
|
47
|
+
|
|
48
|
+
1. Add the [ADOT Lambda Layer](#1-adot-lambda-layer) to your function and set `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument`
|
|
49
|
+
2. Enable [X-Ray Active Tracing](#2-aws-x-ray-active-tracing) on the function
|
|
50
|
+
3. Pass `OtelPlugin` to your handler's `plugins` list
|
|
51
|
+
4. Add X-Ray write permissions
|
|
52
|
+
|
|
53
|
+
### 1. ADOT Lambda Layer
|
|
54
|
+
|
|
55
|
+
This plugin requires the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) to export traces from your Lambda function.
|
|
56
|
+
|
|
57
|
+
The layer ARN follows the format:
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-<arch>-ver-<version>
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Refer to the [ADOT Lambda Layer ARNs](https://aws-otel.github.io/docs/getting-started/lambda/lambda-python) page for the latest version number, architecture, and supported regions.
|
|
64
|
+
|
|
65
|
+
**AWS CLI:**
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
aws lambda update-function-configuration \
|
|
69
|
+
--function-name your-function-name \
|
|
70
|
+
--layers "arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
You must also set the `AWS_LAMBDA_EXEC_WRAPPER` environment variable:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
aws lambda update-function-configuration \
|
|
77
|
+
--function-name your-function-name \
|
|
78
|
+
--environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument}"
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
> **Note:** Replace `<region>` with your function's region and `<version>`/`<arch>` with the latest layer version and architecture from the ADOT docs.
|
|
82
|
+
|
|
83
|
+
**CloudFormation / SAM:**
|
|
84
|
+
|
|
85
|
+
```yaml
|
|
86
|
+
MyFunction:
|
|
87
|
+
Type: AWS::Serverless::Function
|
|
88
|
+
Properties:
|
|
89
|
+
Layers:
|
|
90
|
+
- !Sub arn:aws:lambda:${AWS::Region}:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>
|
|
91
|
+
Environment:
|
|
92
|
+
Variables:
|
|
93
|
+
AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
**CDK:**
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from aws_cdk import aws_lambda as lambda_
|
|
100
|
+
|
|
101
|
+
adot_layer = lambda_.LayerVersion.from_layer_version_arn(
|
|
102
|
+
self,
|
|
103
|
+
"AdotLayer",
|
|
104
|
+
f"arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>",
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
fn = lambda_.Function(
|
|
108
|
+
self,
|
|
109
|
+
"MyFunction",
|
|
110
|
+
runtime=lambda_.Runtime.PYTHON_3_12,
|
|
111
|
+
handler="index.handler",
|
|
112
|
+
code=lambda_.Code.from_asset("lambda"),
|
|
113
|
+
layers=[adot_layer],
|
|
114
|
+
environment={"AWS_LAMBDA_EXEC_WRAPPER": "/opt/otel-instrument"},
|
|
115
|
+
)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
> **Tip:** Pin the layer version to a specific number in production deployments to avoid unexpected behavior from automatic version changes.
|
|
119
|
+
|
|
120
|
+
### 2. AWS X-Ray Active Tracing
|
|
121
|
+
|
|
122
|
+
Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to derive deterministic trace IDs that remain consistent across all invocations of the same durable execution.
|
|
123
|
+
|
|
124
|
+
**AWS Console:** Lambda → Configuration → Monitoring and operations tools → Active tracing → Enable
|
|
125
|
+
|
|
126
|
+
**AWS CLI:**
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
aws lambda update-function-configuration \
|
|
130
|
+
--function-name your-function-name \
|
|
131
|
+
--tracing-config Mode=Active
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
**CloudFormation / SAM:**
|
|
135
|
+
|
|
136
|
+
```yaml
|
|
137
|
+
MyFunction:
|
|
138
|
+
Type: AWS::Lambda::Function
|
|
139
|
+
Properties:
|
|
140
|
+
TracingConfig:
|
|
141
|
+
Mode: Active
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
**CDK:**
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
lambda_.Function(
|
|
148
|
+
self,
|
|
149
|
+
"MyFunction",
|
|
150
|
+
tracing=lambda_.Tracing.ACTIVE,
|
|
151
|
+
)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### 3. In your Lambda handler (index.py)
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from aws_durable_execution_sdk_python import DurableContext
|
|
158
|
+
from aws_durable_execution_sdk_python.execution import durable_execution
|
|
159
|
+
from aws_durable_execution_sdk_python_otel import OtelPlugin
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@durable_execution(plugins=[OtelPlugin()])
|
|
163
|
+
def handler(event: dict, context: DurableContext) -> dict:
|
|
164
|
+
result = context.step(lambda _: fetch_data(event["id"]), name="fetch-data")
|
|
165
|
+
|
|
166
|
+
context.wait(duration=Duration.from_seconds(5))
|
|
167
|
+
|
|
168
|
+
context.step(lambda _: process(result), name="process")
|
|
169
|
+
|
|
170
|
+
return result
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
That's it. The plugin handles TracerProvider setup, deterministic ID generation, and span lifecycle internally.
|
|
174
|
+
|
|
175
|
+
### 4. Grant Permissions
|
|
176
|
+
|
|
177
|
+
The function's execution role needs the `AWSXRayDaemonWriteAccess` managed policy (or equivalent permissions) if using X-Ray as the tracing backend.
|
|
178
|
+
|
|
179
|
+
### Environment Variables for ADOT layer
|
|
180
|
+
|
|
181
|
+
| Variable | Description | Default |
|
|
182
|
+
| ----------------------------- | --------------------------------------------------------------------------------------------- | ----------------- |
|
|
183
|
+
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Endpoint for the OTLP exporter (e.g., `http://localhost:4318` for the ADOT collector sidecar) | Set by ADOT layer |
|
|
184
|
+
| `AWS_LAMBDA_EXEC_WRAPPER` | Set to `/opt/otel-instrument` for the ADOT layer to instrument your function | — |
|
|
185
|
+
| `OTEL_TRACES_SAMPLER` | Sampler to use (e.g., `traceidratio` for ratio-based sampling) | `always_on` |
|
|
186
|
+
| `OTEL_TRACES_SAMPLER_ARG` | Argument for the sampler (e.g., `0.3` to sample 30% of traces) | — |
|
|
187
|
+
|
|
188
|
+
See the [ADOT sampling configuration](https://aws-otel.github.io/docs/getting-started/lambda#sampling-configuration) for more details.
|
|
189
|
+
|
|
190
|
+
## Configuration
|
|
191
|
+
|
|
192
|
+
### Plugin Options
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
from aws_durable_execution_sdk_python_otel import (
|
|
196
|
+
OtelPlugin,
|
|
197
|
+
xray_context_extractor,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
plugin = OtelPlugin(
|
|
201
|
+
# Provide your own TracerProvider if you already have one configured.
|
|
202
|
+
# Defaults to the globally configured tracer provider.
|
|
203
|
+
trace_provider=None,
|
|
204
|
+
# Use a custom context extractor (default: xray_context_extractor).
|
|
205
|
+
context_extractor=xray_context_extractor,
|
|
206
|
+
# Custom instrumentation scope name
|
|
207
|
+
# (default: "aws-durable-execution-sdk-python").
|
|
208
|
+
instrument_name="my-service",
|
|
209
|
+
# Install a root-logger filter that stamps trace context onto every
|
|
210
|
+
# log record (default: True).
|
|
211
|
+
enrich_logger=True,
|
|
212
|
+
)
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Context Extractors
|
|
216
|
+
|
|
217
|
+
The plugin supports multiple strategies for extracting upstream trace context:
|
|
218
|
+
|
|
219
|
+
```python
|
|
220
|
+
from aws_durable_execution_sdk_python_otel import (
|
|
221
|
+
OtelPlugin,
|
|
222
|
+
w3c_client_context_extractor,
|
|
223
|
+
xray_context_extractor,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# Default: X-Ray trace header (recommended for most Lambda deployments)
|
|
227
|
+
OtelPlugin(context_extractor=xray_context_extractor)
|
|
228
|
+
|
|
229
|
+
# W3C Trace Context via clientContext (requires backend propagation support)
|
|
230
|
+
OtelPlugin(context_extractor=w3c_client_context_extractor)
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
### Log Correlation
|
|
234
|
+
|
|
235
|
+
When `enrich_logger=True` (the default), the plugin installs a logging filter on
|
|
236
|
+
the root logger at invocation start. The filter stamps the active OTel trace
|
|
237
|
+
context onto every emitted log record using these attributes:
|
|
238
|
+
|
|
239
|
+
- `traceId`: 32-char hex trace identifier
|
|
240
|
+
- `spanId`: 16-char hex span identifier
|
|
241
|
+
- `otelTraceSampled`: boolean indicating if the trace is sampled
|
|
242
|
+
|
|
243
|
+
These attributes are only set when a valid span context is active, so any log
|
|
244
|
+
formatter or schema must treat the fields as optional.
|
|
245
|
+
|
|
246
|
+
## Verification
|
|
247
|
+
|
|
248
|
+
After deploying your function with the plugin configured:
|
|
249
|
+
|
|
250
|
+
1. **Invoke your durable function** — trigger at least one execution that includes multiple steps or a wait/resume cycle.
|
|
251
|
+
|
|
252
|
+
2. **Check the CloudWatch console** — Navigate to CloudWatch → Traces in the AWS Console. You should see a trace with:
|
|
253
|
+
- An "invocation" span per invocation
|
|
254
|
+
- Child spans for each durable operation (named after your step names)
|
|
255
|
+
- All invocations of the same execution grouped under one trace ID
|
|
256
|
+
|
|
257
|
+
3. **Check log correlation** — verify that your logs include `traceId` and `spanId` fields matching the spans in X-Ray.
|
|
258
|
+
|
|
259
|
+
4. **Confirm sampling** — If you set `OTEL_TRACES_SAMPLER=traceidratio` and `OTEL_TRACES_SAMPLER_ARG` to a value less than 1.0, verify that only the expected proportion of traces appear.
|
|
260
|
+
|
|
261
|
+
5. **Span links** — For operations that span multiple invocations (e.g., after a wait resumes), though span links are set, they are not visualized within the CloudWatch console.
|
|
262
|
+
|
|
263
|
+
### Troubleshooting
|
|
264
|
+
|
|
265
|
+
| Symptom | Likely Cause |
|
|
266
|
+
| --------------------------------- | --------------------------------------------------------------- |
|
|
267
|
+
| No traces appear | ADOT layer not configured, or `AWS_LAMBDA_EXEC_WRAPPER` not set |
|
|
268
|
+
| Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function |
|
|
269
|
+
| Missing spans for some operations | `OTEL_TRACES_SAMPLER_ARG` set below 1.0 |
|
|
270
|
+
| `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled |
|
|
271
|
+
|
|
272
|
+
## API Reference
|
|
273
|
+
|
|
274
|
+
### `OtelPlugin`
|
|
275
|
+
|
|
276
|
+
The main plugin class. Implements `DurableInstrumentationPlugin` from `aws_durable_execution_sdk_python`.
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
OtelPlugin(
|
|
280
|
+
trace_provider=None,
|
|
281
|
+
context_extractor=None,
|
|
282
|
+
instrument_name="aws-durable-execution-sdk-python",
|
|
283
|
+
enrich_logger=True,
|
|
284
|
+
)
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
### `DeterministicIdGenerator`
|
|
288
|
+
|
|
289
|
+
A custom OpenTelemetry `IdGenerator` that produces reproducible trace and span IDs from execution metadata. Exported for advanced use cases.
|
|
290
|
+
|
|
291
|
+
### `xray_context_extractor`
|
|
292
|
+
|
|
293
|
+
Default context extractor. Reads the `_X_AMZN_TRACE_ID` environment variable to derive trace context.
|
|
294
|
+
|
|
295
|
+
### `w3c_client_context_extractor`
|
|
296
|
+
|
|
297
|
+
Alternative context extractor. Reads W3C `traceparent` from `context.clientContext.custom.traceparent`. Requires backend `clientContext` propagation to be enabled.
|
|
298
|
+
|
|
299
|
+
### `ContextExtractor`
|
|
300
|
+
|
|
301
|
+
Type alias for custom context extractor functions.
|
|
302
|
+
|
|
303
|
+
### `OtelContextLogFilter` / `install_log_filter`
|
|
304
|
+
|
|
305
|
+
The logging filter (and its installer) used to stamp trace context onto log
|
|
306
|
+
records. Installed automatically when `enrich_logger=True`; exported for manual
|
|
307
|
+
setups.
|
|
308
|
+
|
|
309
|
+
## Requirements
|
|
310
|
+
|
|
311
|
+
- Python >= 3.11
|
|
312
|
+
- `aws-durable-execution-sdk-python` >= 1.5.0
|
|
313
|
+
- `opentelemetry-api` >= 1.20.0
|
|
314
|
+
- `opentelemetry-sdk` >= 1.20.0
|
|
315
|
+
|
|
316
|
+
## License
|
|
317
|
+
|
|
318
|
+
Apache-2.0
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# AWS Durable Execution SDK - OpenTelemetry Plugin
|
|
2
|
+
|
|
3
|
+
OpenTelemetry instrumentation plugin for the [AWS Durable Execution SDK for Python](https://github.com/aws/aws-durable-execution-sdk-python). Emits distributed traces that correlate across multiple Lambda invocations of a single durable execution, producing deterministic span and trace IDs so that spans from different invocations are stitched into a single coherent trace.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Deterministic Trace IDs**: All invocations of the same durable execution share a single trace, derived from the X-Ray trace header or execution ARN
|
|
8
|
+
- **Span-per-Operation**: Each durable operation (step, wait, invoke) gets its own span with accurate timing
|
|
9
|
+
- **Continuation Spans**: Operations completing in a different invocation are linked back to the original span
|
|
10
|
+
- **Log Correlation**: Enrich application logs with trace ID and span ID for end-to-end observability
|
|
11
|
+
- **Configurable Sampling**: Control trace volume via plugin options
|
|
12
|
+
- **Self-Contained Setup**: No manual TracerProvider configuration required
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install aws-durable-execution-sdk-python-otel
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick Start using X-Ray/CloudWatch Tracing
|
|
21
|
+
|
|
22
|
+
1. Add the [ADOT Lambda Layer](#1-adot-lambda-layer) to your function and set `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument`
|
|
23
|
+
2. Enable [X-Ray Active Tracing](#2-aws-x-ray-active-tracing) on the function
|
|
24
|
+
3. Pass `OtelPlugin` to your handler's `plugins` list
|
|
25
|
+
4. Add X-Ray write permissions
|
|
26
|
+
|
|
27
|
+
### 1. ADOT Lambda Layer
|
|
28
|
+
|
|
29
|
+
This plugin requires the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) to export traces from your Lambda function.
|
|
30
|
+
|
|
31
|
+
The layer ARN follows the format:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-<arch>-ver-<version>
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Refer to the [ADOT Lambda Layer ARNs](https://aws-otel.github.io/docs/getting-started/lambda/lambda-python) page for the latest version number, architecture, and supported regions.
|
|
38
|
+
|
|
39
|
+
**AWS CLI:**
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
aws lambda update-function-configuration \
|
|
43
|
+
--function-name your-function-name \
|
|
44
|
+
--layers "arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
You must also set the `AWS_LAMBDA_EXEC_WRAPPER` environment variable:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
aws lambda update-function-configuration \
|
|
51
|
+
--function-name your-function-name \
|
|
52
|
+
--environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument}"
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
> **Note:** Replace `<region>` with your function's region and `<version>`/`<arch>` with the latest layer version and architecture from the ADOT docs.
|
|
56
|
+
|
|
57
|
+
**CloudFormation / SAM:**
|
|
58
|
+
|
|
59
|
+
```yaml
|
|
60
|
+
MyFunction:
|
|
61
|
+
Type: AWS::Serverless::Function
|
|
62
|
+
Properties:
|
|
63
|
+
Layers:
|
|
64
|
+
- !Sub arn:aws:lambda:${AWS::Region}:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>
|
|
65
|
+
Environment:
|
|
66
|
+
Variables:
|
|
67
|
+
AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
**CDK:**
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from aws_cdk import aws_lambda as lambda_
|
|
74
|
+
|
|
75
|
+
adot_layer = lambda_.LayerVersion.from_layer_version_arn(
|
|
76
|
+
self,
|
|
77
|
+
"AdotLayer",
|
|
78
|
+
f"arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>",
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
fn = lambda_.Function(
|
|
82
|
+
self,
|
|
83
|
+
"MyFunction",
|
|
84
|
+
runtime=lambda_.Runtime.PYTHON_3_12,
|
|
85
|
+
handler="index.handler",
|
|
86
|
+
code=lambda_.Code.from_asset("lambda"),
|
|
87
|
+
layers=[adot_layer],
|
|
88
|
+
environment={"AWS_LAMBDA_EXEC_WRAPPER": "/opt/otel-instrument"},
|
|
89
|
+
)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
> **Tip:** Pin the layer version to a specific number in production deployments to avoid unexpected behavior from automatic version changes.
|
|
93
|
+
|
|
94
|
+
### 2. AWS X-Ray Active Tracing
|
|
95
|
+
|
|
96
|
+
Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to derive deterministic trace IDs that remain consistent across all invocations of the same durable execution.
|
|
97
|
+
|
|
98
|
+
**AWS Console:** Lambda → Configuration → Monitoring and operations tools → Active tracing → Enable
|
|
99
|
+
|
|
100
|
+
**AWS CLI:**
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
aws lambda update-function-configuration \
|
|
104
|
+
--function-name your-function-name \
|
|
105
|
+
--tracing-config Mode=Active
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
**CloudFormation / SAM:**
|
|
109
|
+
|
|
110
|
+
```yaml
|
|
111
|
+
MyFunction:
|
|
112
|
+
Type: AWS::Lambda::Function
|
|
113
|
+
Properties:
|
|
114
|
+
TracingConfig:
|
|
115
|
+
Mode: Active
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
**CDK:**
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
lambda_.Function(
|
|
122
|
+
self,
|
|
123
|
+
"MyFunction",
|
|
124
|
+
tracing=lambda_.Tracing.ACTIVE,
|
|
125
|
+
)
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### 3. In your Lambda handler (index.py)
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from aws_durable_execution_sdk_python import DurableContext
|
|
132
|
+
from aws_durable_execution_sdk_python.execution import durable_execution
|
|
133
|
+
from aws_durable_execution_sdk_python_otel import OtelPlugin
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@durable_execution(plugins=[OtelPlugin()])
|
|
137
|
+
def handler(event: dict, context: DurableContext) -> dict:
|
|
138
|
+
result = context.step(lambda _: fetch_data(event["id"]), name="fetch-data")
|
|
139
|
+
|
|
140
|
+
context.wait(duration=Duration.from_seconds(5))
|
|
141
|
+
|
|
142
|
+
context.step(lambda _: process(result), name="process")
|
|
143
|
+
|
|
144
|
+
return result
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
That's it. The plugin handles TracerProvider setup, deterministic ID generation, and span lifecycle internally.
|
|
148
|
+
|
|
149
|
+
### 4. Grant Permissions
|
|
150
|
+
|
|
151
|
+
The function's execution role needs the `AWSXRayDaemonWriteAccess` managed policy (or equivalent permissions) if using X-Ray as the tracing backend.
|
|
152
|
+
|
|
153
|
+
### Environment Variables for ADOT layer
|
|
154
|
+
|
|
155
|
+
| Variable | Description | Default |
|
|
156
|
+
| ----------------------------- | --------------------------------------------------------------------------------------------- | ----------------- |
|
|
157
|
+
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Endpoint for the OTLP exporter (e.g., `http://localhost:4318` for the ADOT collector sidecar) | Set by ADOT layer |
|
|
158
|
+
| `AWS_LAMBDA_EXEC_WRAPPER` | Set to `/opt/otel-instrument` for the ADOT layer to instrument your function | — |
|
|
159
|
+
| `OTEL_TRACES_SAMPLER` | Sampler to use (e.g., `traceidratio` for ratio-based sampling) | `always_on` |
|
|
160
|
+
| `OTEL_TRACES_SAMPLER_ARG` | Argument for the sampler (e.g., `0.3` to sample 30% of traces) | — |
|
|
161
|
+
|
|
162
|
+
See the [ADOT sampling configuration](https://aws-otel.github.io/docs/getting-started/lambda#sampling-configuration) for more details.
|
|
163
|
+
|
|
164
|
+
## Configuration
|
|
165
|
+
|
|
166
|
+
### Plugin Options
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from aws_durable_execution_sdk_python_otel import (
|
|
170
|
+
OtelPlugin,
|
|
171
|
+
xray_context_extractor,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
plugin = OtelPlugin(
|
|
175
|
+
# Provide your own TracerProvider if you already have one configured.
|
|
176
|
+
# Defaults to the globally configured tracer provider.
|
|
177
|
+
trace_provider=None,
|
|
178
|
+
# Use a custom context extractor (default: xray_context_extractor).
|
|
179
|
+
context_extractor=xray_context_extractor,
|
|
180
|
+
# Custom instrumentation scope name
|
|
181
|
+
# (default: "aws-durable-execution-sdk-python").
|
|
182
|
+
instrument_name="my-service",
|
|
183
|
+
# Install a root-logger filter that stamps trace context onto every
|
|
184
|
+
# log record (default: True).
|
|
185
|
+
enrich_logger=True,
|
|
186
|
+
)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Context Extractors
|
|
190
|
+
|
|
191
|
+
The plugin supports multiple strategies for extracting upstream trace context:
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
from aws_durable_execution_sdk_python_otel import (
|
|
195
|
+
OtelPlugin,
|
|
196
|
+
w3c_client_context_extractor,
|
|
197
|
+
xray_context_extractor,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
# Default: X-Ray trace header (recommended for most Lambda deployments)
|
|
201
|
+
OtelPlugin(context_extractor=xray_context_extractor)
|
|
202
|
+
|
|
203
|
+
# W3C Trace Context via clientContext (requires backend propagation support)
|
|
204
|
+
OtelPlugin(context_extractor=w3c_client_context_extractor)
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Log Correlation
|
|
208
|
+
|
|
209
|
+
When `enrich_logger=True` (the default), the plugin installs a logging filter on
|
|
210
|
+
the root logger at invocation start. The filter stamps the active OTel trace
|
|
211
|
+
context onto every emitted log record using these attributes:
|
|
212
|
+
|
|
213
|
+
- `traceId`: 32-char hex trace identifier
|
|
214
|
+
- `spanId`: 16-char hex span identifier
|
|
215
|
+
- `otelTraceSampled`: boolean indicating if the trace is sampled
|
|
216
|
+
|
|
217
|
+
These attributes are only set when a valid span context is active, so any log
|
|
218
|
+
formatter or schema must treat the fields as optional.
|
|
219
|
+
|
|
220
|
+
## Verification
|
|
221
|
+
|
|
222
|
+
After deploying your function with the plugin configured:
|
|
223
|
+
|
|
224
|
+
1. **Invoke your durable function** — trigger at least one execution that includes multiple steps or a wait/resume cycle.
|
|
225
|
+
|
|
226
|
+
2. **Check the CloudWatch console** — Navigate to CloudWatch → Traces in the AWS Console. You should see a trace with:
|
|
227
|
+
- An "invocation" span per invocation
|
|
228
|
+
- Child spans for each durable operation (named after your step names)
|
|
229
|
+
- All invocations of the same execution grouped under one trace ID
|
|
230
|
+
|
|
231
|
+
3. **Check log correlation** — verify that your logs include `traceId` and `spanId` fields matching the spans in X-Ray.
|
|
232
|
+
|
|
233
|
+
4. **Confirm sampling** — If you set `OTEL_TRACES_SAMPLER=traceidratio` and `OTEL_TRACES_SAMPLER_ARG` to a value less than 1.0, verify that only the expected proportion of traces appear.
|
|
234
|
+
|
|
235
|
+
5. **Span links** — For operations that span multiple invocations (e.g., after a wait resumes), though span links are set, they are not visualized within the CloudWatch console.
|
|
236
|
+
|
|
237
|
+
### Troubleshooting
|
|
238
|
+
|
|
239
|
+
| Symptom | Likely Cause |
|
|
240
|
+
| --------------------------------- | --------------------------------------------------------------- |
|
|
241
|
+
| No traces appear | ADOT layer not configured, or `AWS_LAMBDA_EXEC_WRAPPER` not set |
|
|
242
|
+
| Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function |
|
|
243
|
+
| Missing spans for some operations | `OTEL_TRACES_SAMPLER_ARG` set below 1.0 |
|
|
244
|
+
| `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled |
|
|
245
|
+
|
|
246
|
+
## API Reference
|
|
247
|
+
|
|
248
|
+
### `OtelPlugin`
|
|
249
|
+
|
|
250
|
+
The main plugin class. Implements `DurableInstrumentationPlugin` from `aws_durable_execution_sdk_python`.
|
|
251
|
+
|
|
252
|
+
```python
|
|
253
|
+
OtelPlugin(
|
|
254
|
+
trace_provider=None,
|
|
255
|
+
context_extractor=None,
|
|
256
|
+
instrument_name="aws-durable-execution-sdk-python",
|
|
257
|
+
enrich_logger=True,
|
|
258
|
+
)
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
### `DeterministicIdGenerator`
|
|
262
|
+
|
|
263
|
+
A custom OpenTelemetry `IdGenerator` that produces reproducible trace and span IDs from execution metadata. Exported for advanced use cases.
|
|
264
|
+
|
|
265
|
+
### `xray_context_extractor`
|
|
266
|
+
|
|
267
|
+
Default context extractor. Reads the `_X_AMZN_TRACE_ID` environment variable to derive trace context.
|
|
268
|
+
|
|
269
|
+
### `w3c_client_context_extractor`
|
|
270
|
+
|
|
271
|
+
Alternative context extractor. Reads W3C `traceparent` from `context.clientContext.custom.traceparent`. Requires backend `clientContext` propagation to be enabled.
|
|
272
|
+
|
|
273
|
+
### `ContextExtractor`
|
|
274
|
+
|
|
275
|
+
Type alias for custom context extractor functions.
|
|
276
|
+
|
|
277
|
+
### `OtelContextLogFilter` / `install_log_filter`
|
|
278
|
+
|
|
279
|
+
The logging filter (and its installer) used to stamp trace context onto log
|
|
280
|
+
records. Installed automatically when `enrich_logger=True`; exported for manual
|
|
281
|
+
setups.
|
|
282
|
+
|
|
283
|
+
## Requirements
|
|
284
|
+
|
|
285
|
+
- Python >= 3.11
|
|
286
|
+
- `aws-durable-execution-sdk-python` >= 1.5.0
|
|
287
|
+
- `opentelemetry-api` >= 1.20.0
|
|
288
|
+
- `opentelemetry-sdk` >= 1.20.0
|
|
289
|
+
|
|
290
|
+
## License
|
|
291
|
+
|
|
292
|
+
Apache-2.0
|
|
@@ -12,7 +12,7 @@ license = "Apache-2.0"
|
|
|
12
12
|
keywords = ["opentelemetry", "tracing", "observability", "durable-execution"]
|
|
13
13
|
authors = [{ name = "AWS durable-execution-dev", email = "durable-execution-dev@amazon.com" }]
|
|
14
14
|
classifiers = [
|
|
15
|
-
"Development Status ::
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
16
|
"Programming Language :: Python",
|
|
17
17
|
"Programming Language :: Python :: 3.11",
|
|
18
18
|
"Programming Language :: Python :: 3.12",
|
|
@@ -21,7 +21,13 @@ classifiers = [
|
|
|
21
21
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
22
22
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
23
23
|
]
|
|
24
|
-
dependencies = [
|
|
24
|
+
dependencies = [
|
|
25
|
+
"aws-durable-execution-sdk-python>=1.5.0",
|
|
26
|
+
"opentelemetry-api>=1.20.0",
|
|
27
|
+
"opentelemetry-sdk>=1.20.0",
|
|
28
|
+
"opentelemetry-exporter-otlp",
|
|
29
|
+
"opentelemetry-propagator-aws-xray",
|
|
30
|
+
]
|
|
25
31
|
|
|
26
32
|
[project.urls]
|
|
27
33
|
Documentation = "https://github.com/aws/aws-durable-execution-sdk-python#readme"
|
|
@@ -41,3 +47,46 @@ packages = ["src/aws_durable_execution_sdk_python_otel"]
|
|
|
41
47
|
|
|
42
48
|
[tool.hatch.version]
|
|
43
49
|
path = "src/aws_durable_execution_sdk_python_otel/__about__.py"
|
|
50
|
+
|
|
51
|
+
[tool.hatch.publish.index]
|
|
52
|
+
disable = true
|
|
53
|
+
|
|
54
|
+
[tool.coverage.run]
|
|
55
|
+
source_pkgs = ["aws_durable_execution_sdk_python_otel"]
|
|
56
|
+
branch = true
|
|
57
|
+
parallel = true
|
|
58
|
+
omit = ["src/aws_durable_execution_sdk_python_otel/__about__.py"]
|
|
59
|
+
|
|
60
|
+
[tool.coverage.paths]
|
|
61
|
+
aws_durable_execution_sdk_python_otel = [
|
|
62
|
+
"src/aws_durable_execution_sdk_python_otel",
|
|
63
|
+
"*/aws-durable-execution-sdk-python-otel/src/aws_durable_execution_sdk_python_otel",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[tool.coverage.report]
|
|
67
|
+
exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
|
|
68
|
+
|
|
69
|
+
[tool.ruff]
|
|
70
|
+
line-length = 88
|
|
71
|
+
target-version = "py311"
|
|
72
|
+
|
|
73
|
+
[tool.ruff.lint]
|
|
74
|
+
preview = true
|
|
75
|
+
select = ["E4", "E7", "E9", "F", "TID252"] # pycodestyle (E4/E7/E9) + Pyflakes + absolute imports
|
|
76
|
+
|
|
77
|
+
[tool.ruff.lint.isort]
|
|
78
|
+
known-first-party = ["aws_durable_execution_sdk_python_otel"]
|
|
79
|
+
force-single-line = false
|
|
80
|
+
lines-after-imports = 2
|
|
81
|
+
|
|
82
|
+
[tool.ruff.lint.per-file-ignores]
|
|
83
|
+
"tests/**" = [
|
|
84
|
+
"ARG001",
|
|
85
|
+
"ARG002",
|
|
86
|
+
"ARG005",
|
|
87
|
+
"S101",
|
|
88
|
+
"PLR2004",
|
|
89
|
+
"PLR6301",
|
|
90
|
+
"SIM117",
|
|
91
|
+
"TRY301",
|
|
92
|
+
]
|