aws-durable-execution-sdk-python-otel 0.3.0__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. aws_durable_execution_sdk_python_otel-1.1.0/PKG-INFO +590 -0
  2. aws_durable_execution_sdk_python_otel-1.1.0/README.md +564 -0
  3. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/pyproject.toml +13 -4
  4. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/__about__.py +1 -1
  5. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/__init__.py +54 -0
  6. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/context_extractors.py +132 -0
  7. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/deterministic_id_generator.py +208 -0
  8. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/durable_parent_span.py +159 -0
  9. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/durable_sampling.py +239 -0
  10. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/execution_plugin.py +988 -0
  11. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/execution_root.py +68 -0
  12. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/execution_trace_context.py +98 -0
  13. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/invocation_plugin.py +1056 -0
  14. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/log_filter.py +26 -4
  15. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/otel_plugin_config.py +50 -0
  16. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/plugin_provider.py +23 -0
  17. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/propagation.py +31 -0
  18. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/provider.py +45 -0
  19. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/timestamps.py +8 -0
  20. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_branch_context_int.py +179 -0
  21. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_fallback_root_lifecycle.py +225 -0
  22. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_invocation_wait_resume_int.py +499 -0
  23. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_invoke_checkpoint_propagation.py +247 -0
  24. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_propagation_metadata.py +349 -0
  25. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_replaced_sampler_invoke.py +143 -0
  26. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_view_registration.py +536 -0
  27. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_context_extractors.py +128 -0
  28. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_deterministic_id_generator.py +395 -0
  29. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_durable_sampling.py +546 -0
  30. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_execution_plugin.py +1694 -0
  31. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_execution_plugin_integration.py +462 -0
  32. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_execution_root.py +584 -0
  33. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_execution_trace_context.py +202 -0
  34. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_instrumentation_ownership.py +48 -0
  35. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_invocation_plugin.py +2302 -0
  36. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_invocation_plugin_integration.py +429 -0
  37. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_log_filter.py +27 -12
  38. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_package_metadata.py +107 -0
  39. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_plugin_provider.py +64 -0
  40. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_propagation_sampling.py +153 -0
  41. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_provider.py +35 -0
  42. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_root_reentry.py +154 -0
  43. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_status_mapping.py +172 -0
  44. aws_durable_execution_sdk_python_otel-0.3.0/PKG-INFO +0 -318
  45. aws_durable_execution_sdk_python_otel-0.3.0/README.md +0 -292
  46. aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/__init__.py +0 -30
  47. aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/context_extractors.py +0 -40
  48. aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/deterministic_id_generator.py +0 -127
  49. aws_durable_execution_sdk_python_otel-0.3.0/src/aws_durable_execution_sdk_python_otel/plugin.py +0 -548
  50. aws_durable_execution_sdk_python_otel-0.3.0/tests/test_context_extractors.py +0 -66
  51. aws_durable_execution_sdk_python_otel-0.3.0/tests/test_deterministic_id_generator.py +0 -285
  52. aws_durable_execution_sdk_python_otel-0.3.0/tests/test_plugin.py +0 -800
  53. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/.gitignore +0 -0
  54. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/LICENSE +0 -0
  55. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/NOTICE +0 -0
  56. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/py.typed +0 -0
  57. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/__init__.py +0 -0
  58. {aws_durable_execution_sdk_python_otel-0.3.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_init.py +0 -0
@@ -0,0 +1,590 @@
1
+ Metadata-Version: 2.5
2
+ Name: aws-durable-execution-sdk-python-otel
3
+ Version: 1.1.0
4
+ Summary: OpenTelemetry instrumentation for AWS Durable Execution SDK for Python
5
+ Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
6
+ Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
7
+ Project-URL: Source, https://github.com/aws/aws-durable-execution-sdk-python
8
+ Author-email: AWS durable-execution-dev <durable-execution-dev@amazon.com>
9
+ License-Expression: Apache-2.0
10
+ Keywords: durable-execution,observability,opentelemetry,tracing
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Programming Language :: Python
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
17
+ Classifier: Programming Language :: Python :: Implementation :: CPython
18
+ Classifier: Programming Language :: Python :: Implementation :: PyPy
19
+ Requires-Python: >=3.11
20
+ Requires-Dist: aws-durable-execution-sdk-python>=2.1.0
21
+ Provides-Extra: standalone
22
+ Requires-Dist: opentelemetry-api>=1.20.0; extra == 'standalone'
23
+ Requires-Dist: opentelemetry-propagator-aws-xray; extra == 'standalone'
24
+ Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'standalone'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # AWS Durable Execution SDK - OpenTelemetry Plugin
28
+
29
+ OpenTelemetry instrumentation plugin for the [AWS Durable Execution SDK for Python](https://github.com/aws/aws-durable-execution-sdk-python). Emits durable execution spans on one execution trace, with deterministic workflow, synthetic-root, and operation span IDs.
30
+
31
+ ## Features
32
+
33
+ - **Shared Execution Trace**: Workflow and Invocation spans share one trace, anchored to a propagated backend parent when available or a deterministic synthetic execution root otherwise
34
+ - **Same-Trace Ambient Parenting**: Invocation spans use the active ambient span only when it already belongs to the execution trace
35
+ - **Span-per-Operation**: Each durable operation (step, wait, invoke) gets its own span with accurate timing
36
+ - **Continuation Spans**: Operations completing in another invocation produce a new correlated span without fabricating an unobserved prior span context
37
+ - **Log Correlation**: Enrich application logs with trace ID and span ID for end-to-end observability
38
+ - **Provider Integration**: Use the global ADOT provider or supply an explicit SDK `TracerProvider`
39
+ - **Execution Sampling**: Resolve sampling once per invocation and apply it consistently to Workflow, Invocation, operation, and attempt spans
40
+
41
+ ## Installation
42
+
43
+ When using an ADOT or community OpenTelemetry Lambda layer:
44
+
45
+ ```bash
46
+ pip install aws-durable-execution-sdk-python-otel
47
+ ```
48
+
49
+ The base package intentionally does not install OpenTelemetry libraries. The
50
+ Lambda layer supplies a version-aligned API, SDK, exporter, and propagators,
51
+ preventing packages in the function artifact from shadowing parts of the layer.
52
+
53
+ The generated `aws-durable-execution-sdk-python-otel-plugin` Lambda layer includes
54
+ Core's runtime dependencies, including its required boto3/botocore service model.
55
+ It still leaves OpenTelemetry provider packages to the ADOT/community layer.
56
+ Dependencies bundled in the function artifact take precedence over layer packages;
57
+ those application-owned boto3/botocore versions must also meet Core's minimum.
58
+
59
+ For an application that configures its own OpenTelemetry provider instead of
60
+ using a Lambda layer:
61
+
62
+ ```bash
63
+ pip install "aws-durable-execution-sdk-python-otel[standalone]"
64
+ ```
65
+
66
+ The `standalone` extra installs the OpenTelemetry API, SDK, and AWS X-Ray
67
+ propagator. The application remains responsible for configuring its provider,
68
+ processors, and exporter.
69
+
70
+ ## Quick Start using X-Ray/CloudWatch Tracing
71
+
72
+ 1. Add the [ADOT Lambda Layer](#1-adot-lambda-layer) to your function and set `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument`
73
+ 2. Enable [X-Ray Active Tracing](#2-aws-x-ray-active-tracing) on the function
74
+ 3. Pass `InvocationOtelPlugin` to your handler's `plugins` list
75
+ 4. Add X-Ray write permissions
76
+
77
+ Alternatively, install this package in the function artifact or a Lambda layer
78
+ and select either OTel plugin by entry-point name:
79
+
80
+ ```text
81
+ DURABLE_EXECUTION_PLUGINS=otel-invocation
82
+ DURABLE_EXECUTION_PLUGINS=otel-execution
83
+ ```
84
+
85
+ `otel-invocation` creates `InvocationOtelPlugin`; `otel-execution` creates
86
+ `ExecutionOtelPlugin`. The SDK discovers the selected package entry point at
87
+ cold start, so the handler does not need to import or explicitly register the
88
+ plugin.
89
+
90
+ OTel 1.1.0 requires core SDK 2.1.0 or later for the invocation-worker lifecycle
91
+ and automatic mutual-exclusion validation. Publish the redesigned core first,
92
+ then the plugin. Core 2.0.x runs invocation hooks on the caller and cannot provide
93
+ the new plugin's handler propagation and host-context isolation; that version
94
+ pair is not supported by OTel 1.1.0.
95
+ Configure only one OTel view on every core version. `InvocationOtelPlugin`
96
+ shows work within each Lambda invocation; `ExecutionOtelPlugin` shows logical
97
+ operations across the whole execution. They emit overlapping telemetry and
98
+ manage competing active contexts, so the coordinated core 2.1+/OTel 1.1+ pair
99
+ rejects both at cold start with `PluginLoadError` naming both views. Keep only one across the combined
100
+ `plugins=[...]` and `DURABLE_EXECUTION_PLUGINS` configuration. Unrelated plugins
101
+ may run alongside the selected view; using no OTel plugin is also supported.
102
+
103
+ The bundled views explicitly declare `__durable_registration_api__ = 1`.
104
+ Custom subclasses inherit their group validation and registration cleanup.
105
+ The closest marker gates registration and selects the callback contract; an
106
+ explicit value other than the integer `1` shadows the inherited opt-in.
107
+ Once enabled, each MRO class explicitly declaring the integer `1` contributes
108
+ its resolved `exclusive_group`. A subclass's own group adds to the inherited
109
+ view group rather than replacing it; `None` cannot remove an inherited group.
110
+ Repeated groups count once per plugin. Unmarked classes' unrelated fields/helpers
111
+ named `exclusive_group` or `on_registration_result` remain inert. Repeat the marker
112
+ to add a group or customize the callback, which is still notified once per plugin.
113
+ Re-accepting a previously rejected bundled instance
114
+ restores its constructor-time logger enrichment immediately.
115
+
116
+ ### 1. ADOT Lambda Layer
117
+
118
+ This plugin requires the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) to export traces from your Lambda function.
119
+
120
+ The layer ARN follows the format:
121
+
122
+ ```
123
+ arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-<arch>-ver-<version>
124
+ ```
125
+
126
+ Refer to the [ADOT Lambda Layer ARNs](https://aws-otel.github.io/docs/getting-started/lambda/lambda-python) page for the latest version number, architecture, and supported regions.
127
+
128
+ **AWS CLI:**
129
+
130
+ ```bash
131
+ aws lambda update-function-configuration \
132
+ --function-name your-function-name \
133
+ --layers "arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>"
134
+ ```
135
+
136
+ You must also set the `AWS_LAMBDA_EXEC_WRAPPER` environment variable:
137
+
138
+ ```bash
139
+ aws lambda update-function-configuration \
140
+ --function-name your-function-name \
141
+ --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument}"
142
+ ```
143
+
144
+ > **Note:** Replace `<region>` with your function's region and `<version>`/`<arch>` with the latest layer version and architecture from the ADOT docs.
145
+
146
+ **CloudFormation / SAM:**
147
+
148
+ ```yaml
149
+ MyFunction:
150
+ Type: AWS::Serverless::Function
151
+ Properties:
152
+ Layers:
153
+ - !Sub arn:aws:lambda:${AWS::Region}:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>
154
+ Environment:
155
+ Variables:
156
+ AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument
157
+ ```
158
+
159
+ **CDK:**
160
+
161
+ ```python
162
+ from aws_cdk import aws_lambda as lambda_
163
+
164
+ adot_layer = lambda_.LayerVersion.from_layer_version_arn(
165
+ self,
166
+ "AdotLayer",
167
+ f"arn:aws:lambda:<region>:<awsAccountId>:layer:aws-otel-python-amd64-ver-<version>",
168
+ )
169
+
170
+ fn = lambda_.Function(
171
+ self,
172
+ "MyFunction",
173
+ runtime=lambda_.Runtime.PYTHON_3_12,
174
+ handler="index.handler",
175
+ code=lambda_.Code.from_asset("lambda"),
176
+ layers=[adot_layer],
177
+ environment={"AWS_LAMBDA_EXEC_WRAPPER": "/opt/otel-instrument"},
178
+ )
179
+ ```
180
+
181
+ > **Tip:** Pin the layer version to a specific number in production deployments to avoid unexpected behavior from automatic version changes.
182
+
183
+ ### 2. AWS X-Ray Active Tracing
184
+
185
+ Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to anchor the execution trace on the propagated X-Ray `Root`/`Parent` when both are valid, and preserves `Sampled=1` or `Sampled=0` as the backend sampling decision.
186
+
187
+ **AWS Console:** Lambda → Configuration → Monitoring and operations tools → Active tracing → Enable
188
+
189
+ **AWS CLI:**
190
+
191
+ ```bash
192
+ aws lambda update-function-configuration \
193
+ --function-name your-function-name \
194
+ --tracing-config Mode=Active
195
+ ```
196
+
197
+ **CloudFormation / SAM:**
198
+
199
+ ```yaml
200
+ MyFunction:
201
+ Type: AWS::Lambda::Function
202
+ Properties:
203
+ TracingConfig:
204
+ Mode: Active
205
+ ```
206
+
207
+ **CDK:**
208
+
209
+ ```python
210
+ lambda_.Function(
211
+ self,
212
+ "MyFunction",
213
+ tracing=lambda_.Tracing.ACTIVE,
214
+ )
215
+ ```
216
+
217
+ ### Handler context propagation
218
+
219
+ The core runs the existing Start hooks, handler, output preparation, resource
220
+ cleanup and End hooks on one invocation worker. Start finishes before checkpoint
221
+ processing begins. End follows registered-branch joins and checkpoint shutdown,
222
+ and output serialization or checkpoint errors retain their normal classification.
223
+ There is no separate handler-context plugin API.
224
+
225
+ During Start, Invocation view preserves a valid active span on the canonical
226
+ execution trace; when it is absent or unrelated, it attaches the Invocation span.
227
+ Execution view attaches the Workflow span, including when an incoming span is on
228
+ the same trace. The Invocation span's own ambient parenting is separate from the
229
+ active context supplied to handler instrumentation. Both views preserve baggage.
230
+
231
+ Start hooks run in registration order. A later successful plugin that deliberately
232
+ sets or clears the active span wins; OTel does not apply a second correction pass.
233
+ Place OTel after a span-replacing plugin when OTel's view-specific context is
234
+ desired. Baggage-only plugins that extend the current context can appear on either
235
+ side. The SDK also copies the coordinator's current bindings for each `map` or
236
+ `parallel` branch admission/resume when plugins are registered. Baggage and
237
+ other successful bindings reach these SDK-managed callbacks, while a branch's
238
+ changes cannot leak into siblings, the coordinator or a reused worker.
239
+ End hooks also retain registration order and reset tokens in their owning
240
+ Context; they do not promise a reverse-stack observation of other plugins' spans.
241
+ Existing registration, factory lifetime and checkpoint formats are unchanged.
242
+
243
+ Upgrade both core to 2.1+ and OTel to 1.1+ for these guarantees. The released
244
+ OTel 1.0 plugin can run on the new core with worker/host isolation, but its
245
+ Invocation view does not attach the new fallback; a core-only upgrade does not
246
+ supply that plugin behavior. CI tests the new pair as installed wheels and tests
247
+ the actual released plugin separately, including before core 2.1 is on PyPI.
248
+
249
+ ### 3. In your Lambda handler (index.py)
250
+
251
+ ```python
252
+ from aws_durable_execution_sdk_python import DurableContext
253
+ from aws_durable_execution_sdk_python.execution import durable_execution
254
+ from aws_durable_execution_sdk_python_otel import InvocationOtelPlugin
255
+
256
+
257
+ @durable_execution(plugins=[InvocationOtelPlugin()])
258
+ def handler(event: dict, context: DurableContext) -> dict:
259
+ result = context.step(lambda _: fetch_data(event["id"]), name="fetch-data")
260
+
261
+ context.wait(duration=Duration.from_seconds(5))
262
+
263
+ context.step(lambda _: process(result), name="process")
264
+
265
+ return result
266
+ ```
267
+
268
+ The ADOT layer supplies the global `TracerProvider`; the plugin handles
269
+ deterministic ID generation and span lifecycle.
270
+
271
+ ### 4. Grant Permissions
272
+
273
+ The function's execution role needs the `AWSXRayDaemonWriteAccess` managed policy (or equivalent permissions) if using X-Ray as the tracing backend.
274
+
275
+ ### Environment Variables for ADOT layer
276
+
277
+ | Variable | Description | Default |
278
+ | ----------------------------- | --------------------------------------------------------------------------------------------- | ----------------- |
279
+ | `OTEL_EXPORTER_OTLP_ENDPOINT` | Endpoint for the OTLP exporter (e.g., `http://localhost:4318` for the ADOT collector sidecar) | Set by ADOT layer |
280
+ | `AWS_LAMBDA_EXEC_WRAPPER` | Set to `/opt/otel-instrument` for the ADOT layer to instrument your function | — |
281
+ | `OTEL_TRACES_SAMPLER` | Sampler to use (e.g., `traceidratio` for ratio-based sampling) | `always_on` |
282
+ | `OTEL_TRACES_SAMPLER_ARG` | Argument for the sampler (e.g., `0.3` to sample 30% of traces) | — |
283
+
284
+ See the [ADOT sampling configuration](https://aws-otel.github.io/docs/getting-started/lambda#sampling-configuration) for more details. When the backend header contains an explicit `Sampled` value, that backend decision takes precedence over local sampler configuration for durable spans.
285
+
286
+ ## Configuration
287
+
288
+ ### Plugin Options
289
+
290
+ ```python
291
+ from aws_durable_execution_sdk_python_otel import (
292
+ InvocationOtelPlugin,
293
+ OtelPluginConfig,
294
+ xray_context_extractor,
295
+ )
296
+
297
+ plugin = InvocationOtelPlugin(
298
+ OtelPluginConfig(
299
+ # Use a custom context extractor (default: xray_context_extractor).
300
+ context_extractor=xray_context_extractor,
301
+ # Custom instrumentation scope name
302
+ # (default: "aws-durable-execution-sdk-python").
303
+ instrument_name="my-service",
304
+ # Install a root-logger filter that stamps trace context onto every
305
+ # log record (default: True).
306
+ enrich_logger=True,
307
+ )
308
+ )
309
+ ```
310
+
311
+ ### Context Extractors
312
+
313
+ Context extractors return an `ExtractedContext` object, or `None` when no
314
+ durable execution trace context is available. The object carries:
315
+
316
+ - `trace_id`: 128-bit OpenTelemetry trace ID
317
+ - `parent_span_id`: 64-bit OpenTelemetry parent span ID
318
+ - `sampling`: `Sampling.SAMPLED`, `Sampling.NOT_SAMPLED`, or `Sampling.UNDECIDED`
319
+
320
+ The plugin supports multiple strategies for extracting durable execution trace
321
+ context:
322
+
323
+ ```python
324
+ from aws_durable_execution_sdk_python_otel import (
325
+ InvocationOtelPlugin,
326
+ OtelPluginConfig,
327
+ w3c_client_context_extractor,
328
+ xray_context_extractor,
329
+ )
330
+
331
+ # Default: X-Ray trace header (recommended for most Lambda deployments).
332
+ InvocationOtelPlugin(OtelPluginConfig(context_extractor=xray_context_extractor))
333
+
334
+ # W3C Trace Context via clientContext (placeholder for backend propagation support).
335
+ InvocationOtelPlugin(OtelPluginConfig(context_extractor=w3c_client_context_extractor))
336
+ ```
337
+
338
+ Custom extractors should return `ExtractedContext`, not an OpenTelemetry
339
+ `Context`.
340
+
341
+ ### Trace Structure
342
+
343
+ Both bundled plugins use the same execution ancestor:
344
+
345
+ - a propagated backend parent when `_X_AMZN_TRACE_ID` contains a valid `Root`
346
+ and `Parent`
347
+ - otherwise a deterministic `DurableExecutionRoot` span derived from the
348
+ durable execution ARN, with `durable.execution.synthetic_root=true`
349
+
350
+ `InvocationOtelPlugin` keeps durable operation spans under the Invocation span
351
+ and links operations to Workflow:
352
+
353
+ ```text
354
+ Execution ancestor
355
+ ├── Workflow
356
+ └── Invocation
357
+ └── operation
358
+ └── operation attempt 1
359
+ ```
360
+
361
+ `ExecutionOtelPlugin` keeps operation spans under Workflow and links operations
362
+ to the current Invocation span:
363
+
364
+ ```text
365
+ Execution ancestor
366
+ ├── Workflow
367
+ │ └── operation
368
+ │ └── operation attempt 1
369
+ └── Invocation
370
+ ```
371
+
372
+ If an ambient Lambda span is active and already has the execution trace ID, the
373
+ Invocation span uses that ambient span as its parent. Ambient spans on a
374
+ different trace are ignored for durable parenting so Invocation remains on the
375
+ execution trace.
376
+
377
+ ### Synthetic fallback anchor
378
+
379
+ Each sampled fallback invocation creates and ends `DurableExecutionRoot` at
380
+ invocation start and attempts to flush it through the normal invocation-end
381
+ lifecycle, including when it returns `PENDING` or `RETRY`. A successfully
382
+ delivered copy supplies ancestry if the execution is later stopped or times out
383
+ without resuming. Export and flush failures, or termination before flushing,
384
+ do not provide a delivery guarantee.
385
+
386
+ The anchor uses the existing execution-scoped span ID and the stable execution
387
+ start timestamp for both its start and end. It carries no terminal outcome;
388
+ `Workflow` remains the span for execution duration and success/failure. Retries
389
+ of the first invocation and later invocations may re-export the same anchor to
390
+ recover lost telemetry; export is not exactly once. Two executions sharing a
391
+ propagated trace ID retain distinct ARN-scoped anchor IDs.
392
+ The anchor is intentionally zero-duration, so its descendants end after it;
393
+ trace viewers may flag that timing relationship even though the ancestry is valid.
394
+
395
+ The anchor uses the same configured tracer, provider resource and processors as
396
+ other durable spans, but carries only the resolved sampling decision. Its SDK
397
+ attributes are `durable.execution.arn` and `durable.execution.synthetic_root`;
398
+ sampler-added attributes and tracestate are omitted from this repeated identity.
399
+ Ordinary durable spans retain the complete resolved sampler metadata. SDK-added
400
+ IDs, timestamps and anchor attributes stay stable across recovery exports;
401
+ provider resources and custom processor enrichment may vary. The SDK does not
402
+ replace configured service identity or freeze detector fields.
403
+
404
+ Complete remote parents are externally owned and are never materialized by the
405
+ SDK. Unsampled invocations emit no anchor. Sampling remains resolved per
406
+ invocation: a later sampled fallback invocation also emits the anchor even if
407
+ an earlier invocation was unsampled. Missing execution start time continues to
408
+ disable telemetry; no wall-clock timestamp is invented for the anchor.
409
+
410
+ ### Sampling
411
+
412
+ Sampling is resolved once per invocation and carried to every durable span in
413
+ that invocation. Each span consumes its private sampling intent before processors
414
+ run. Processor-created spans use their own configured sampling policy and normal
415
+ parent-context semantics, including when a processor forwards the supplied parent
416
+ context. Precedence is:
417
+
418
+ 1. `Sampled=1` or `Sampled=0` from `_X_AMZN_TRACE_ID`
419
+ 2. a same-trace ambient span's recording/sampled state
420
+ 3. the configured OpenTelemetry sampler
421
+
422
+ The resolved decision is applied to Workflow, Invocation, operation, and attempt
423
+ spans. This avoids independently querying stateful or ratio-based samplers for
424
+ each durable span in the same invocation.
425
+
426
+ ### Status attributes
427
+
428
+ `durable.invocation.status` uses `RETRYING` when the core plugin hook reports
429
+ `InvocationStatus.RETRY`. The core enum remains unchanged. Operation spans use
430
+ OTel `OK` only for `SUCCEEDED`, `ERROR` when error details are delivered, and
431
+ `UNSET` for other outcomes without error details, including `FAILED`,
432
+ `CANCELLED`, `TIMED_OUT`, and `STOPPED`. The original durable operation status
433
+ remains in `durable.operation.status`.
434
+
435
+ ### Invocation context isolation
436
+
437
+ With core 2.1+, invocation hooks and the handler run on one worker in an
438
+ invocation-local Context initialized from the host's bindings. Successful Start
439
+ bindings are visible to later hooks, the handler and resource cleanup. The host's
440
+ bindings remain unchanged after return, even if a plugin fails during Start or
441
+ End. A failed Start's bindings are discarded for subsequent work, while End runs
442
+ in that Start's original Context to preserve token ownership. This isolates
443
+ context-variable bindings; it does not undo mutations to shared objects or
444
+ external side effects. The isolation applies when plugins are registered.
445
+
446
+ ### Log Correlation
447
+
448
+ When `enrich_logger=True` (the default), the plugin installs a logging filter on
449
+ the root logger at invocation start. The filter stamps the active OTel trace
450
+ context onto every emitted log record using these attributes:
451
+
452
+ - `traceId`: 32-char hex trace identifier
453
+ - `spanId`: 16-char hex span identifier
454
+ - `otelTraceSampled`: boolean indicating if the trace is sampled
455
+
456
+ These attributes are only set when a valid span context is active, so any log
457
+ formatter or schema must treat the fields as optional.
458
+
459
+ ## Chained-invoke propagation
460
+
461
+ Both views supply the synchronous `provide_propagation_metadata` hook with
462
+ SDK-owned types. It encodes canonical X-Ray `Root`, the calling operation's
463
+ `Parent`, and resolved `Sampled=1` or `Sampled=0`, without creating an extra span.
464
+ An existing operation's actual span ID is used; before span creation, its stable
465
+ ID is derived from the execution ARN and operation ID. Unrelated ambient spans
466
+ cannot replace execution ownership. An operation that has already started uses
467
+ its actual span's sampled flag. A deferred operation contributes metadata only
468
+ while the instrumentation tracer still uses the installed `DurableSampler`:
469
+ replacing that sampler after invocation Start may give the operation a different
470
+ sampling decision, so the producer abstains instead of guessing or querying the
471
+ sampler early. Inactive/mismatched executions contribute nothing. Tracer,
472
+ provider and resource ownership remain unchanged.
473
+
474
+ The core now calls the collector only for a new invoke START and persists the
475
+ contribution in flat `ChainedInvokeOptions.XAmznTraceId`. Pending and terminal
476
+ replay do not recollect metadata; an uncommitted START can be retried. Separate
477
+ invokes carry separate operation parents. Public invoke tests cover both views,
478
+ sampled/unsampled context, parallel branches, preserved tenant/payload and replay.
479
+
480
+ For [#751](https://github.com/aws/aws-durable-execution-sdk-python/issues/751),
481
+ deployed backend delivery and topology still require validation. The core requires
482
+ boto3 1.43.114 or newer, which depends on a botocore model containing the optional
483
+ field. The normal request tests use the published serializer and parameter
484
+ validation. Python has no distributed-map
485
+ model/START path; existing map/parallel APIs are CONTEXT operations.
486
+
487
+ On older supported cores without the propagation contract, plugin loading and
488
+ existing tracing continue, and the optional hook contributes no metadata. The
489
+ new capability requires the coordinated core and its boto3 minimum. The core/OTel
490
+ package floor and provider API version are unchanged; the published model alone
491
+ does not establish deployed downstream topology.
492
+
493
+ ## Verification
494
+
495
+ After deploying your function with the plugin configured:
496
+
497
+ 1. **Invoke your durable function** — trigger at least one execution that includes multiple steps or a wait/resume cycle.
498
+
499
+ 2. **Check the CloudWatch console** — Navigate to CloudWatch → Traces in the AWS Console. You should see an execution trace with:
500
+ - A "Workflow" span exported on the terminal invocation
501
+ - An "Invocation" span per invocation
502
+ - Child spans for each durable operation (named after your step names)
503
+ - All invocations of the same execution grouped under one trace ID
504
+
505
+ 3. **Check log correlation** — verify that your logs include `traceId` and `spanId` fields matching the spans in X-Ray.
506
+
507
+ 4. **Confirm sampling** — If you set `OTEL_TRACES_SAMPLER=traceidratio` and `OTEL_TRACES_SAMPLER_ARG` to a value less than 1.0, verify that only the expected proportion of traces appear.
508
+
509
+ 5. **Span links** — For operations that span multiple invocations (e.g., after a wait resumes), though span links are set, they are not visualized within the CloudWatch console.
510
+
511
+ ### Troubleshooting
512
+
513
+ | Symptom | Likely Cause |
514
+ | --------------------------------- | --------------------------------------------------------------- |
515
+ | No traces appear | ADOT layer not configured, or `AWS_LAMBDA_EXEC_WRAPPER` not set |
516
+ | Traces appear but are fragmented | Backend trace context is not propagated to every invocation |
517
+ | Missing spans for some operations | `OTEL_TRACES_SAMPLER_ARG` set below 1.0 |
518
+ | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled |
519
+
520
+ ## API Reference
521
+
522
+ ### `InvocationOtelPlugin`
523
+
524
+ Invocation-rooted view. Implements `DurableInstrumentationPlugin` from `aws_durable_execution_sdk_python`.
525
+
526
+ ```python
527
+ InvocationOtelPlugin(
528
+ OtelPluginConfig(
529
+ tracer_provider=None,
530
+ context_extractor=None,
531
+ instrument_name="aws-durable-execution-sdk-python",
532
+ enrich_logger=True,
533
+ workflow_span_name="Workflow",
534
+ )
535
+ )
536
+ ```
537
+
538
+ Pass `tracer_provider=...` when the application owns the OpenTelemetry SDK
539
+ provider. When omitted, the globally configured provider is used.
540
+
541
+ ### `ExecutionOtelPlugin`
542
+
543
+ Execution-rooted view. Uses the same execution ancestor and sampling behavior as
544
+ `InvocationOtelPlugin`, but parents operation spans under Workflow and links
545
+ them to Invocation.
546
+
547
+ ### `DeterministicIdGenerator`
548
+
549
+ A custom OpenTelemetry `IdGenerator` that produces reproducible trace and span IDs from execution metadata. Exported for advanced use cases.
550
+
551
+ `use_ids` retains its existing scope-wide trace override by default. The
552
+ synthetic-root implementation uses a separate internal one-span override,
553
+ preventing processor-created roots from inheriting that internal ID. Existing
554
+ `use_ids` signatures and default behavior remain unchanged.
555
+
556
+ ### `xray_context_extractor`
557
+
558
+ Default context extractor. Reads the `_X_AMZN_TRACE_ID` environment variable and
559
+ returns `ExtractedContext` containing parsed `Root`, `Parent`, and `Sampled`
560
+ fields when present.
561
+
562
+ ### `w3c_client_context_extractor`
563
+
564
+ Alternative context extractor placeholder. Returns `None` until backend W3C
565
+ `traceparent` propagation is supported.
566
+
567
+ ### `ContextExtractor`
568
+
569
+ Type alias for custom context extractor functions:
570
+ `Callable[[InvocationStartInfo], ExtractedContext | None]`.
571
+
572
+ ### `ExtractedContext` / `Sampling`
573
+
574
+ Structured trace context and sampling decision returned by context extractors.
575
+
576
+ ### `OtelContextLogFilter` / `install_log_filter`
577
+
578
+ The logging filter (and its installer) used to stamp trace context onto log
579
+ records. Installed automatically when `enrich_logger=True`; exported for manual
580
+ setups.
581
+
582
+ ## Requirements
583
+
584
+ - Python >= 3.11
585
+ - `aws-durable-execution-sdk-python` >= 2.1.0 (release the redesigned core before OTel 1.1.0)
586
+ - An ADOT/community OpenTelemetry Lambda layer, or the `standalone` extra
587
+
588
+ ## License
589
+
590
+ Apache-2.0