aws-durable-execution-sdk-python-otel 1.0.0__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/PKG-INFO +165 -6
  2. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/README.md +163 -4
  3. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/pyproject.toml +2 -2
  4. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/__about__.py +1 -1
  5. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/deterministic_id_generator.py +46 -10
  6. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/durable_sampling.py +31 -4
  7. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/execution_plugin.py +86 -5
  8. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/execution_root.py +68 -0
  9. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/invocation_plugin.py +107 -17
  10. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/log_filter.py +11 -0
  11. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/propagation.py +31 -0
  12. aws_durable_execution_sdk_python_otel-1.1.0/src/aws_durable_execution_sdk_python_otel/timestamps.py +8 -0
  13. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_branch_context_int.py +179 -0
  14. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_fallback_root_lifecycle.py +225 -0
  15. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_invocation_wait_resume_int.py +499 -0
  16. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_invoke_checkpoint_propagation.py +247 -0
  17. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_propagation_metadata.py +349 -0
  18. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_replaced_sampler_invoke.py +143 -0
  19. aws_durable_execution_sdk_python_otel-1.1.0/tests/e2e/test_view_registration.py +536 -0
  20. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_deterministic_id_generator.py +54 -0
  21. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_durable_sampling.py +49 -0
  22. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_execution_plugin.py +78 -6
  23. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_execution_plugin_integration.py +3 -0
  24. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_execution_root.py +584 -0
  25. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_invocation_plugin.py +128 -26
  26. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_invocation_plugin_integration.py +2 -0
  27. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_log_filter.py +16 -3
  28. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_package_metadata.py +32 -7
  29. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_plugin_provider.py +14 -6
  30. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_propagation_sampling.py +153 -0
  31. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_root_reentry.py +154 -0
  32. aws_durable_execution_sdk_python_otel-1.1.0/tests/test_status_mapping.py +172 -0
  33. aws_durable_execution_sdk_python_otel-1.0.0/tests/e2e/test_invocation_wait_resume_int.py +0 -255
  34. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/.gitignore +0 -0
  35. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/LICENSE +0 -0
  36. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/NOTICE +0 -0
  37. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/__init__.py +0 -0
  38. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/context_extractors.py +0 -0
  39. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/durable_parent_span.py +0 -0
  40. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/execution_trace_context.py +0 -0
  41. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/otel_plugin_config.py +0 -0
  42. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/plugin_provider.py +0 -0
  43. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/provider.py +0 -0
  44. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/src/aws_durable_execution_sdk_python_otel/py.typed +0 -0
  45. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/__init__.py +0 -0
  46. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_context_extractors.py +0 -0
  47. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_execution_trace_context.py +0 -0
  48. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_init.py +0 -0
  49. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_instrumentation_ownership.py +0 -0
  50. {aws_durable_execution_sdk_python_otel-1.0.0 → aws_durable_execution_sdk_python_otel-1.1.0}/tests/test_provider.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: aws-durable-execution-sdk-python-otel
3
- Version: 1.0.0
3
+ Version: 1.1.0
4
4
  Summary: OpenTelemetry instrumentation for AWS Durable Execution SDK for Python
5
5
  Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
6
6
  Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.14
17
17
  Classifier: Programming Language :: Python :: Implementation :: CPython
18
18
  Classifier: Programming Language :: Python :: Implementation :: PyPy
19
19
  Requires-Python: >=3.11
20
- Requires-Dist: aws-durable-execution-sdk-python>=2.0.0
20
+ Requires-Dist: aws-durable-execution-sdk-python>=2.1.0
21
21
  Provides-Extra: standalone
22
22
  Requires-Dist: opentelemetry-api>=1.20.0; extra == 'standalone'
23
23
  Requires-Dist: opentelemetry-propagator-aws-xray; extra == 'standalone'
@@ -50,6 +50,12 @@ The base package intentionally does not install OpenTelemetry libraries. The
50
50
  Lambda layer supplies a version-aligned API, SDK, exporter, and propagators,
51
51
  preventing packages in the function artifact from shadowing parts of the layer.
52
52
 
53
+ The generated `aws-durable-execution-sdk-python-otel-plugin` Lambda layer includes
54
+ Core's runtime dependencies, including its required boto3/botocore service model.
55
+ It still leaves OpenTelemetry provider packages to the ADOT/community layer.
56
+ Dependencies bundled in the function artifact take precedence over layer packages;
57
+ those application-owned boto3/botocore versions must also meet Core's minimum.
58
+
53
59
  For an application that configures its own OpenTelemetry provider instead of
54
60
  using a Lambda layer:
55
61
 
@@ -81,6 +87,32 @@ DURABLE_EXECUTION_PLUGINS=otel-execution
81
87
  cold start, so the handler does not need to import or explicitly register the
82
88
  plugin.
83
89
 
90
+ OTel 1.1.0 requires core SDK 2.1.0 or later for the invocation-worker lifecycle
91
+ and automatic mutual-exclusion validation. Publish the redesigned core first,
92
+ then the plugin. Core 2.0.x runs invocation hooks on the caller and cannot provide
93
+ the new plugin's handler propagation and host-context isolation; that version
94
+ pair is not supported by OTel 1.1.0.
95
+ Configure only one OTel view on every core version. `InvocationOtelPlugin`
96
+ shows work within each Lambda invocation; `ExecutionOtelPlugin` shows logical
97
+ operations across the whole execution. They emit overlapping telemetry and
98
+ manage competing active contexts, so the coordinated core 2.1+/OTel 1.1+ pair
99
+ rejects both at cold start with `PluginLoadError` naming both views. Keep only one across the combined
100
+ `plugins=[...]` and `DURABLE_EXECUTION_PLUGINS` configuration. Unrelated plugins
101
+ may run alongside the selected view; using no OTel plugin is also supported.
102
+
103
+ The bundled views explicitly declare `__durable_registration_api__ = 1`.
104
+ Custom subclasses inherit their group validation and registration cleanup.
105
+ The closest marker gates registration and selects the callback contract; an
106
+ explicit value other than the integer `1` shadows the inherited opt-in.
107
+ Once enabled, each MRO class explicitly declaring the integer `1` contributes
108
+ its resolved `exclusive_group`. A subclass's own group adds to the inherited
109
+ view group rather than replacing it; `None` cannot remove an inherited group.
110
+ Repeated groups count once per plugin. Unmarked classes' unrelated fields/helpers
111
+ named `exclusive_group` or `on_registration_result` remain inert. Repeat the marker
112
+ to add a group or customize the callback, which is still notified once per plugin.
113
+ Re-accepting a previously rejected bundled instance
114
+ restores its constructor-time logger enrichment immediately.
115
+
84
116
  ### 1. ADOT Lambda Layer
85
117
 
86
118
  This plugin requires the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) to export traces from your Lambda function.
@@ -182,6 +214,38 @@ lambda_.Function(
182
214
  )
183
215
  ```
184
216
 
217
+ ### Handler context propagation
218
+
219
+ The core runs the existing Start hooks, handler, output preparation, resource
220
+ cleanup and End hooks on one invocation worker. Start finishes before checkpoint
221
+ processing begins. End follows registered-branch joins and checkpoint shutdown,
222
+ and output serialization or checkpoint errors retain their normal classification.
223
+ There is no separate handler-context plugin API.
224
+
225
+ During Start, Invocation view preserves a valid active span on the canonical
226
+ execution trace; when it is absent or unrelated, it attaches the Invocation span.
227
+ Execution view attaches the Workflow span, including when an incoming span is on
228
+ the same trace. The Invocation span's own ambient parenting is separate from the
229
+ active context supplied to handler instrumentation. Both views preserve baggage.
230
+
231
+ Start hooks run in registration order. A later successful plugin that deliberately
232
+ sets or clears the active span wins; OTel does not apply a second correction pass.
233
+ Place OTel after a span-replacing plugin when OTel's view-specific context is
234
+ desired. Baggage-only plugins that extend the current context can appear on either
235
+ side. The SDK also copies the coordinator's current bindings for each `map` or
236
+ `parallel` branch admission/resume when plugins are registered. Baggage and
237
+ other successful bindings reach these SDK-managed callbacks, while a branch's
238
+ changes cannot leak into siblings, the coordinator or a reused worker.
239
+ End hooks also retain registration order and reset tokens in their owning
240
+ Context; they do not promise a reverse-stack observation of other plugins' spans.
241
+ Existing registration, factory lifetime and checkpoint formats are unchanged.
242
+
243
+ Upgrade both core to 2.1+ and OTel to 1.1+ for these guarantees. The released
244
+ OTel 1.0 plugin can run on the new core with worker/host isolation, but its
245
+ Invocation view does not attach the new fallback; a core-only upgrade does not
246
+ supply that plugin behavior. CI tests the new pair as installed wheels and tests
247
+ the actual released plugin separately, including before core 2.1 is on PyPI.
248
+
185
249
  ### 3. In your Lambda handler (index.py)
186
250
 
187
251
  ```python
@@ -280,8 +344,8 @@ Both bundled plugins use the same execution ancestor:
280
344
 
281
345
  - a propagated backend parent when `_X_AMZN_TRACE_ID` contains a valid `Root`
282
346
  and `Parent`
283
- - otherwise a deterministic, non-recording synthetic root derived from the
284
- durable execution ARN
347
+ - otherwise a deterministic `DurableExecutionRoot` span derived from the
348
+ durable execution ARN, with `durable.execution.synthetic_root=true`
285
349
 
286
350
  `InvocationOtelPlugin` keeps durable operation spans under the Invocation span
287
351
  and links operations to Workflow:
@@ -310,10 +374,46 @@ Invocation span uses that ambient span as its parent. Ambient spans on a
310
374
  different trace are ignored for durable parenting so Invocation remains on the
311
375
  execution trace.
312
376
 
377
+ ### Synthetic fallback anchor
378
+
379
+ Each sampled fallback invocation creates and ends `DurableExecutionRoot` at
380
+ invocation start and attempts to flush it through the normal invocation-end
381
+ lifecycle, including when it returns `PENDING` or `RETRY`. A successfully
382
+ delivered copy supplies ancestry if the execution is later stopped or times out
383
+ without resuming. Export and flush failures, or termination before flushing,
384
+ do not provide a delivery guarantee.
385
+
386
+ The anchor uses the existing execution-scoped span ID and the stable execution
387
+ start timestamp for both its start and end. It carries no terminal outcome;
388
+ `Workflow` remains the span for execution duration and success/failure. Retries
389
+ of the first invocation and later invocations may re-export the same anchor to
390
+ recover lost telemetry; export is not exactly once. Two executions sharing a
391
+ propagated trace ID retain distinct ARN-scoped anchor IDs.
392
+ The anchor is intentionally zero-duration, so its descendants end after it;
393
+ trace viewers may flag that timing relationship even though the ancestry is valid.
394
+
395
+ The anchor uses the same configured tracer, provider resource and processors as
396
+ other durable spans, but carries only the resolved sampling decision. Its SDK
397
+ attributes are `durable.execution.arn` and `durable.execution.synthetic_root`;
398
+ sampler-added attributes and tracestate are omitted from this repeated identity.
399
+ Ordinary durable spans retain the complete resolved sampler metadata. SDK-added
400
+ IDs, timestamps and anchor attributes stay stable across recovery exports;
401
+ provider resources and custom processor enrichment may vary. The SDK does not
402
+ replace configured service identity or freeze detector fields.
403
+
404
+ Complete remote parents are externally owned and are never materialized by the
405
+ SDK. Unsampled invocations emit no anchor. Sampling remains resolved per
406
+ invocation: a later sampled fallback invocation also emits the anchor even if
407
+ an earlier invocation was unsampled. Missing execution start time continues to
408
+ disable telemetry; no wall-clock timestamp is invented for the anchor.
409
+
313
410
  ### Sampling
314
411
 
315
412
  Sampling is resolved once per invocation and carried to every durable span in
316
- that invocation. Precedence is:
413
+ that invocation. Each span consumes its private sampling intent before processors
414
+ run. Processor-created spans use their own configured sampling policy and normal
415
+ parent-context semantics, including when a processor forwards the supplied parent
416
+ context. Precedence is:
317
417
 
318
418
  1. `Sampled=1` or `Sampled=0` from `_X_AMZN_TRACE_ID`
319
419
  2. a same-trace ambient span's recording/sampled state
@@ -323,6 +423,26 @@ The resolved decision is applied to Workflow, Invocation, operation, and attempt
323
423
  spans. This avoids independently querying stateful or ratio-based samplers for
324
424
  each durable span in the same invocation.
325
425
 
426
+ ### Status attributes
427
+
428
+ `durable.invocation.status` uses `RETRYING` when the core plugin hook reports
429
+ `InvocationStatus.RETRY`. The core enum remains unchanged. Operation spans use
430
+ OTel `OK` only for `SUCCEEDED`, `ERROR` when error details are delivered, and
431
+ `UNSET` for other outcomes without error details, including `FAILED`,
432
+ `CANCELLED`, `TIMED_OUT`, and `STOPPED`. The original durable operation status
433
+ remains in `durable.operation.status`.
434
+
435
+ ### Invocation context isolation
436
+
437
+ With core 2.1+, invocation hooks and the handler run on one worker in an
438
+ invocation-local Context initialized from the host's bindings. Successful Start
439
+ bindings are visible to later hooks, the handler and resource cleanup. The host's
440
+ bindings remain unchanged after return, even if a plugin fails during Start or
441
+ End. A failed Start's bindings are discarded for subsequent work, while End runs
442
+ in that Start's original Context to preserve token ownership. This isolates
443
+ context-variable bindings; it does not undo mutations to shared objects or
444
+ external side effects. The isolation applies when plugins are registered.
445
+
326
446
  ### Log Correlation
327
447
 
328
448
  When `enrich_logger=True` (the default), the plugin installs a logging filter on
@@ -336,6 +456,40 @@ context onto every emitted log record using these attributes:
336
456
  These attributes are only set when a valid span context is active, so any log
337
457
  formatter or schema must treat the fields as optional.
338
458
 
459
+ ## Chained-invoke propagation
460
+
461
+ Both views supply the synchronous `provide_propagation_metadata` hook with
462
+ SDK-owned types. It encodes canonical X-Ray `Root`, the calling operation's
463
+ `Parent`, and resolved `Sampled=1` or `Sampled=0`, without creating an extra span.
464
+ An existing operation's actual span ID is used; before span creation, its stable
465
+ ID is derived from the execution ARN and operation ID. Unrelated ambient spans
466
+ cannot replace execution ownership. An operation that has already started uses
467
+ its actual span's sampled flag. A deferred operation contributes metadata only
468
+ while the instrumentation tracer still uses the installed `DurableSampler`:
469
+ replacing that sampler after invocation Start may give the operation a different
470
+ sampling decision, so the producer abstains instead of guessing or querying the
471
+ sampler early. Inactive/mismatched executions contribute nothing. Tracer,
472
+ provider and resource ownership remain unchanged.
473
+
474
+ The core now calls the collector only for a new invoke START and persists the
475
+ contribution in flat `ChainedInvokeOptions.XAmznTraceId`. Pending and terminal
476
+ replay do not recollect metadata; an uncommitted START can be retried. Separate
477
+ invokes carry separate operation parents. Public invoke tests cover both views,
478
+ sampled/unsampled context, parallel branches, preserved tenant/payload and replay.
479
+
480
+ For [#751](https://github.com/aws/aws-durable-execution-sdk-python/issues/751),
481
+ deployed backend delivery and topology still require validation. The core requires
482
+ boto3 1.43.114 or newer, which depends on a botocore model containing the optional
483
+ field. The normal request tests use the published serializer and parameter
484
+ validation. Python has no distributed-map
485
+ model/START path; existing map/parallel APIs are CONTEXT operations.
486
+
487
+ On older supported cores without the propagation contract, plugin loading and
488
+ existing tracing continue, and the optional hook contributes no metadata. The
489
+ new capability requires the coordinated core and its boto3 minimum. The core/OTel
490
+ package floor and provider API version are unchanged; the published model alone
491
+ does not establish deployed downstream topology.
492
+
339
493
  ## Verification
340
494
 
341
495
  After deploying your function with the plugin configured:
@@ -394,6 +548,11 @@ them to Invocation.
394
548
 
395
549
  A custom OpenTelemetry `IdGenerator` that produces reproducible trace and span IDs from execution metadata. Exported for advanced use cases.
396
550
 
551
+ `use_ids` retains its existing scope-wide trace override by default. The
552
+ synthetic-root implementation uses a separate internal one-span override,
553
+ preventing processor-created roots from inheriting that internal ID. Existing
554
+ `use_ids` signatures and default behavior remain unchanged.
555
+
397
556
  ### `xray_context_extractor`
398
557
 
399
558
  Default context extractor. Reads the `_X_AMZN_TRACE_ID` environment variable and
@@ -423,7 +582,7 @@ setups.
423
582
  ## Requirements
424
583
 
425
584
  - Python >= 3.11
426
- - `aws-durable-execution-sdk-python` >= 2.0.0
585
+ - `aws-durable-execution-sdk-python` >= 2.1.0 (release the redesigned core before OTel 1.1.0)
427
586
  - An ADOT/community OpenTelemetry Lambda layer, or the `standalone` extra
428
587
 
429
588
  ## License
@@ -24,6 +24,12 @@ The base package intentionally does not install OpenTelemetry libraries. The
24
24
  Lambda layer supplies a version-aligned API, SDK, exporter, and propagators,
25
25
  preventing packages in the function artifact from shadowing parts of the layer.
26
26
 
27
+ The generated `aws-durable-execution-sdk-python-otel-plugin` Lambda layer includes
28
+ Core's runtime dependencies, including its required boto3/botocore service model.
29
+ It still leaves OpenTelemetry provider packages to the ADOT/community layer.
30
+ Dependencies bundled in the function artifact take precedence over layer packages;
31
+ those application-owned boto3/botocore versions must also meet Core's minimum.
32
+
27
33
  For an application that configures its own OpenTelemetry provider instead of
28
34
  using a Lambda layer:
29
35
 
@@ -55,6 +61,32 @@ DURABLE_EXECUTION_PLUGINS=otel-execution
55
61
  cold start, so the handler does not need to import or explicitly register the
56
62
  plugin.
57
63
 
64
+ OTel 1.1.0 requires core SDK 2.1.0 or later for the invocation-worker lifecycle
65
+ and automatic mutual-exclusion validation. Publish the redesigned core first,
66
+ then the plugin. Core 2.0.x runs invocation hooks on the caller and cannot provide
67
+ the new plugin's handler propagation and host-context isolation; that version
68
+ pair is not supported by OTel 1.1.0.
69
+ Configure only one OTel view on every core version. `InvocationOtelPlugin`
70
+ shows work within each Lambda invocation; `ExecutionOtelPlugin` shows logical
71
+ operations across the whole execution. They emit overlapping telemetry and
72
+ manage competing active contexts, so the coordinated core 2.1+/OTel 1.1+ pair
73
+ rejects both at cold start with `PluginLoadError` naming both views. Keep only one across the combined
74
+ `plugins=[...]` and `DURABLE_EXECUTION_PLUGINS` configuration. Unrelated plugins
75
+ may run alongside the selected view; using no OTel plugin is also supported.
76
+
77
+ The bundled views explicitly declare `__durable_registration_api__ = 1`.
78
+ Custom subclasses inherit their group validation and registration cleanup.
79
+ The closest marker gates registration and selects the callback contract; an
80
+ explicit value other than the integer `1` shadows the inherited opt-in.
81
+ Once enabled, each MRO class explicitly declaring the integer `1` contributes
82
+ its resolved `exclusive_group`. A subclass's own group adds to the inherited
83
+ view group rather than replacing it; `None` cannot remove an inherited group.
84
+ Repeated groups count once per plugin. Unmarked classes' unrelated fields/helpers
85
+ named `exclusive_group` or `on_registration_result` remain inert. Repeat the marker
86
+ to add a group or customize the callback, which is still notified once per plugin.
87
+ Re-accepting a previously rejected bundled instance
88
+ restores its constructor-time logger enrichment immediately.
89
+
58
90
  ### 1. ADOT Lambda Layer
59
91
 
60
92
  This plugin requires the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) to export traces from your Lambda function.
@@ -156,6 +188,38 @@ lambda_.Function(
156
188
  )
157
189
  ```
158
190
 
191
+ ### Handler context propagation
192
+
193
+ The core runs the existing Start hooks, handler, output preparation, resource
194
+ cleanup and End hooks on one invocation worker. Start finishes before checkpoint
195
+ processing begins. End follows registered-branch joins and checkpoint shutdown,
196
+ and output serialization or checkpoint errors retain their normal classification.
197
+ There is no separate handler-context plugin API.
198
+
199
+ During Start, Invocation view preserves a valid active span on the canonical
200
+ execution trace; when it is absent or unrelated, it attaches the Invocation span.
201
+ Execution view attaches the Workflow span, including when an incoming span is on
202
+ the same trace. The Invocation span's own ambient parenting is separate from the
203
+ active context supplied to handler instrumentation. Both views preserve baggage.
204
+
205
+ Start hooks run in registration order. A later successful plugin that deliberately
206
+ sets or clears the active span wins; OTel does not apply a second correction pass.
207
+ Place OTel after a span-replacing plugin when OTel's view-specific context is
208
+ desired. Baggage-only plugins that extend the current context can appear on either
209
+ side. The SDK also copies the coordinator's current bindings for each `map` or
210
+ `parallel` branch admission/resume when plugins are registered. Baggage and
211
+ other successful bindings reach these SDK-managed callbacks, while a branch's
212
+ changes cannot leak into siblings, the coordinator or a reused worker.
213
+ End hooks also retain registration order and reset tokens in their owning
214
+ Context; they do not promise a reverse-stack observation of other plugins' spans.
215
+ Existing registration, factory lifetime and checkpoint formats are unchanged.
216
+
217
+ Upgrade both core to 2.1+ and OTel to 1.1+ for these guarantees. The released
218
+ OTel 1.0 plugin can run on the new core with worker/host isolation, but its
219
+ Invocation view does not attach the new fallback; a core-only upgrade does not
220
+ supply that plugin behavior. CI tests the new pair as installed wheels and tests
221
+ the actual released plugin separately, including before core 2.1 is on PyPI.
222
+
159
223
  ### 3. In your Lambda handler (index.py)
160
224
 
161
225
  ```python
@@ -254,8 +318,8 @@ Both bundled plugins use the same execution ancestor:
254
318
 
255
319
  - a propagated backend parent when `_X_AMZN_TRACE_ID` contains a valid `Root`
256
320
  and `Parent`
257
- - otherwise a deterministic, non-recording synthetic root derived from the
258
- durable execution ARN
321
+ - otherwise a deterministic `DurableExecutionRoot` span derived from the
322
+ durable execution ARN, with `durable.execution.synthetic_root=true`
259
323
 
260
324
  `InvocationOtelPlugin` keeps durable operation spans under the Invocation span
261
325
  and links operations to Workflow:
@@ -284,10 +348,46 @@ Invocation span uses that ambient span as its parent. Ambient spans on a
284
348
  different trace are ignored for durable parenting so Invocation remains on the
285
349
  execution trace.
286
350
 
351
+ ### Synthetic fallback anchor
352
+
353
+ Each sampled fallback invocation creates and ends `DurableExecutionRoot` at
354
+ invocation start and attempts to flush it through the normal invocation-end
355
+ lifecycle, including when it returns `PENDING` or `RETRY`. A successfully
356
+ delivered copy supplies ancestry if the execution is later stopped or times out
357
+ without resuming. Export and flush failures, or termination before flushing,
358
+ do not provide a delivery guarantee.
359
+
360
+ The anchor uses the existing execution-scoped span ID and the stable execution
361
+ start timestamp for both its start and end. It carries no terminal outcome;
362
+ `Workflow` remains the span for execution duration and success/failure. Retries
363
+ of the first invocation and later invocations may re-export the same anchor to
364
+ recover lost telemetry; export is not exactly once. Two executions sharing a
365
+ propagated trace ID retain distinct ARN-scoped anchor IDs.
366
+ The anchor is intentionally zero-duration, so its descendants end after it;
367
+ trace viewers may flag that timing relationship even though the ancestry is valid.
368
+
369
+ The anchor uses the same configured tracer, provider resource and processors as
370
+ other durable spans, but carries only the resolved sampling decision. Its SDK
371
+ attributes are `durable.execution.arn` and `durable.execution.synthetic_root`;
372
+ sampler-added attributes and tracestate are omitted from this repeated identity.
373
+ Ordinary durable spans retain the complete resolved sampler metadata. SDK-added
374
+ IDs, timestamps and anchor attributes stay stable across recovery exports;
375
+ provider resources and custom processor enrichment may vary. The SDK does not
376
+ replace configured service identity or freeze detector fields.
377
+
378
+ Complete remote parents are externally owned and are never materialized by the
379
+ SDK. Unsampled invocations emit no anchor. Sampling remains resolved per
380
+ invocation: a later sampled fallback invocation also emits the anchor even if
381
+ an earlier invocation was unsampled. Missing execution start time continues to
382
+ disable telemetry; no wall-clock timestamp is invented for the anchor.
383
+
287
384
  ### Sampling
288
385
 
289
386
  Sampling is resolved once per invocation and carried to every durable span in
290
- that invocation. Precedence is:
387
+ that invocation. Each span consumes its private sampling intent before processors
388
+ run. Processor-created spans use their own configured sampling policy and normal
389
+ parent-context semantics, including when a processor forwards the supplied parent
390
+ context. Precedence is:
291
391
 
292
392
  1. `Sampled=1` or `Sampled=0` from `_X_AMZN_TRACE_ID`
293
393
  2. a same-trace ambient span's recording/sampled state
@@ -297,6 +397,26 @@ The resolved decision is applied to Workflow, Invocation, operation, and attempt
297
397
  spans. This avoids independently querying stateful or ratio-based samplers for
298
398
  each durable span in the same invocation.
299
399
 
400
+ ### Status attributes
401
+
402
+ `durable.invocation.status` uses `RETRYING` when the core plugin hook reports
403
+ `InvocationStatus.RETRY`. The core enum remains unchanged. Operation spans use
404
+ OTel `OK` only for `SUCCEEDED`, `ERROR` when error details are delivered, and
405
+ `UNSET` for other outcomes without error details, including `FAILED`,
406
+ `CANCELLED`, `TIMED_OUT`, and `STOPPED`. The original durable operation status
407
+ remains in `durable.operation.status`.
408
+
409
+ ### Invocation context isolation
410
+
411
+ With core 2.1+, invocation hooks and the handler run on one worker in an
412
+ invocation-local Context initialized from the host's bindings. Successful Start
413
+ bindings are visible to later hooks, the handler and resource cleanup. The host's
414
+ bindings remain unchanged after return, even if a plugin fails during Start or
415
+ End. A failed Start's bindings are discarded for subsequent work, while End runs
416
+ in that Start's original Context to preserve token ownership. This isolates
417
+ context-variable bindings; it does not undo mutations to shared objects or
418
+ external side effects. The isolation applies when plugins are registered.
419
+
300
420
  ### Log Correlation
301
421
 
302
422
  When `enrich_logger=True` (the default), the plugin installs a logging filter on
@@ -310,6 +430,40 @@ context onto every emitted log record using these attributes:
310
430
  These attributes are only set when a valid span context is active, so any log
311
431
  formatter or schema must treat the fields as optional.
312
432
 
433
+ ## Chained-invoke propagation
434
+
435
+ Both views supply the synchronous `provide_propagation_metadata` hook with
436
+ SDK-owned types. It encodes canonical X-Ray `Root`, the calling operation's
437
+ `Parent`, and resolved `Sampled=1` or `Sampled=0`, without creating an extra span.
438
+ An existing operation's actual span ID is used; before span creation, its stable
439
+ ID is derived from the execution ARN and operation ID. Unrelated ambient spans
440
+ cannot replace execution ownership. An operation that has already started uses
441
+ its actual span's sampled flag. A deferred operation contributes metadata only
442
+ while the instrumentation tracer still uses the installed `DurableSampler`:
443
+ replacing that sampler after invocation Start may give the operation a different
444
+ sampling decision, so the producer abstains instead of guessing or querying the
445
+ sampler early. Inactive/mismatched executions contribute nothing. Tracer,
446
+ provider and resource ownership remain unchanged.
447
+
448
+ The core now calls the collector only for a new invoke START and persists the
449
+ contribution in flat `ChainedInvokeOptions.XAmznTraceId`. Pending and terminal
450
+ replay do not recollect metadata; an uncommitted START can be retried. Separate
451
+ invokes carry separate operation parents. Public invoke tests cover both views,
452
+ sampled/unsampled context, parallel branches, preserved tenant/payload and replay.
453
+
454
+ For [#751](https://github.com/aws/aws-durable-execution-sdk-python/issues/751),
455
+ deployed backend delivery and topology still require validation. The core requires
456
+ boto3 1.43.114 or newer, which depends on a botocore model containing the optional
457
+ field. The normal request tests use the published serializer and parameter
458
+ validation. Python has no distributed-map
459
+ model/START path; existing map/parallel APIs are CONTEXT operations.
460
+
461
+ On older supported cores without the propagation contract, plugin loading and
462
+ existing tracing continue, and the optional hook contributes no metadata. The
463
+ new capability requires the coordinated core and its boto3 minimum. The core/OTel
464
+ package floor and provider API version are unchanged; the published model alone
465
+ does not establish deployed downstream topology.
466
+
313
467
  ## Verification
314
468
 
315
469
  After deploying your function with the plugin configured:
@@ -368,6 +522,11 @@ them to Invocation.
368
522
 
369
523
  A custom OpenTelemetry `IdGenerator` that produces reproducible trace and span IDs from execution metadata. Exported for advanced use cases.
370
524
 
525
+ `use_ids` retains its existing scope-wide trace override by default. The
526
+ synthetic-root implementation uses a separate internal one-span override,
527
+ preventing processor-created roots from inheriting that internal ID. Existing
528
+ `use_ids` signatures and default behavior remain unchanged.
529
+
371
530
  ### `xray_context_extractor`
372
531
 
373
532
  Default context extractor. Reads the `_X_AMZN_TRACE_ID` environment variable and
@@ -397,7 +556,7 @@ setups.
397
556
  ## Requirements
398
557
 
399
558
  - Python >= 3.11
400
- - `aws-durable-execution-sdk-python` >= 2.0.0
559
+ - `aws-durable-execution-sdk-python` >= 2.1.0 (release the redesigned core before OTel 1.1.0)
401
560
  - An ADOT/community OpenTelemetry Lambda layer, or the `standalone` extra
402
561
 
403
562
  ## License
@@ -1,5 +1,5 @@
1
1
  [build-system]
2
- requires = ["hatchling"]
2
+ requires = ["hatchling!=1.32.1,<1.33"]
3
3
  build-backend = "hatchling.build"
4
4
 
5
5
  [project]
@@ -22,7 +22,7 @@ classifiers = [
22
22
  "Programming Language :: Python :: Implementation :: PyPy",
23
23
  ]
24
24
  dependencies = [
25
- "aws-durable-execution-sdk-python>=2.0.0",
25
+ "aws-durable-execution-sdk-python>=2.1.0",
26
26
  ]
27
27
 
28
28
  [project.entry-points."aws_durable_execution.plugins"]
@@ -1,4 +1,4 @@
1
1
  # SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates.
2
2
  #
3
3
  # SPDX-License-Identifier: Apache-2.0
4
- __version__ = "1.0.0"
4
+ __version__ = "1.1.0"
@@ -21,6 +21,8 @@ if TYPE_CHECKING:
21
21
  class _IdOverride:
22
22
  trace_id: int | None
23
23
  span_id: int | None
24
+ trace_id_consumed: bool = False
25
+ consume_trace_id: bool = False
24
26
 
25
27
 
26
28
  def _to_otel_trace_id(execution_arn: str, start_timestamp: datetime) -> int:
@@ -83,9 +85,10 @@ def derive_workflow_span_id(durable_execution_arn: str) -> int:
83
85
  def derive_execution_root_span_id(durable_execution_arn: str) -> int:
84
86
  """Derive the deterministic synthetic execution-root span ID.
85
87
 
86
- The synthetic root is a non-recording parent context used when the backend
87
- does not provide a complete remote parent. Its ID is stable across
88
- reinvocations and uses a namespace distinct from Workflow and operation
88
+ The synthetic root anchors the trace when the backend does not provide a
89
+ complete remote parent. Its zero-duration span is exported on each sampled
90
+ fallback invocation. Its ID is stable across reinvocations and uses a
91
+ namespace distinct from Workflow and operation
89
92
  span IDs.
90
93
  """
91
94
  if not durable_execution_arn:
@@ -143,28 +146,61 @@ class DeterministicIdGenerator(RandomIdGenerator):
143
146
  finally:
144
147
  self._id_override.reset(token)
145
148
 
149
+ @contextmanager
150
+ def _use_ids_for_span(
151
+ self,
152
+ *,
153
+ trace_id: int,
154
+ span_id: int,
155
+ ) -> Iterator[None]:
156
+ """Reserve IDs for one SDK-created span without changing public scope semantics."""
157
+ token = self._id_override.set(
158
+ _IdOverride(trace_id, span_id, consume_trace_id=True)
159
+ )
160
+ try:
161
+ yield
162
+ finally:
163
+ self._id_override.reset(token)
164
+
146
165
  def generate_trace_id(self) -> int:
147
166
  """Generate a 128-bit trace ID."""
148
167
  override = self._id_override.get()
149
168
  if override is not None and override.trace_id is not None:
169
+ # Consume the identity once, but keep its non-random classification
170
+ # until the SDK asks for the span ID below.
171
+ if override.consume_trace_id:
172
+ self._id_override.set(_IdOverride(None, override.span_id, True, True))
150
173
  return override.trace_id
174
+ if override is not None and override.trace_id_consumed:
175
+ self._id_override.set(
176
+ _IdOverride(
177
+ None, override.span_id, consume_trace_id=override.consume_trace_id
178
+ )
179
+ )
151
180
  return self._fallback_id_generator.generate_trace_id()
152
181
 
153
182
  def generate_span_id(self) -> int:
154
183
  """Generate a 64-bit span ID."""
155
184
  override = self._id_override.get()
156
- if override is not None and override.span_id is not None:
157
- span_id = override.span_id
158
- # Consume before returning so a re-entrant call in the same span
159
- # creation falls back instead of reusing the deterministic ID.
160
- self._id_override.set(_IdOverride(override.trace_id, None))
161
- return span_id
185
+ if override is not None:
186
+ # Span ID generation completes the ID phase of SDK start_span, before
187
+ # processors run. Release the consumed trace classification too, so
188
+ # a processor creating another root sees only its own generator.
189
+ self._id_override.set(
190
+ _IdOverride(
191
+ override.trace_id, None, consume_trace_id=override.consume_trace_id
192
+ )
193
+ )
194
+ if override.span_id is not None:
195
+ return override.span_id
162
196
  return self._fallback_id_generator.generate_span_id()
163
197
 
164
198
  def is_trace_id_random(self) -> bool:
165
199
  """Report whether the current trace ID is randomly generated."""
166
200
  override = self._id_override.get()
167
- if override is not None and override.trace_id is not None:
201
+ if override is not None and (
202
+ override.trace_id is not None or override.trace_id_consumed
203
+ ):
168
204
  return False
169
205
  fallback_method = getattr(
170
206
  self._fallback_id_generator, "is_trace_id_random", None
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import functools
6
6
  import inspect
7
7
  from dataclasses import dataclass
8
+ from threading import Lock
8
9
  from typing import Any
9
10
 
10
11
  from opentelemetry import context as otel_context
@@ -31,6 +32,20 @@ class DurableSamplingIntent:
31
32
  result: SamplingResult
32
33
 
33
34
 
35
+ class _SamplingIntentDelivery:
36
+ """Give one sampler call the intent, even if its context is forwarded."""
37
+
38
+ def __init__(self, intent: DurableSamplingIntent) -> None:
39
+ self._intent: DurableSamplingIntent | None = intent
40
+ self._lock = Lock()
41
+
42
+ def consume(self) -> DurableSamplingIntent | None:
43
+ with self._lock:
44
+ intent = self._intent
45
+ self._intent = None
46
+ return intent
47
+
48
+
34
49
  class DurableSampler(Sampler):
35
50
  """Sampler that honors a durable sampling intent carried on parent context."""
36
51
 
@@ -56,8 +71,13 @@ class DurableSampler(Sampler):
56
71
  links: Any = None,
57
72
  trace_state: Any = None,
58
73
  ) -> SamplingResult:
59
- intent = otel_context.get_value(_DURABLE_SAMPLING_INTENT_KEY, parent_context)
60
- if isinstance(intent, DurableSamplingIntent):
74
+ delivery = otel_context.get_value(_DURABLE_SAMPLING_INTENT_KEY, parent_context)
75
+ intent = (
76
+ delivery.consume()
77
+ if isinstance(delivery, _SamplingIntentDelivery)
78
+ else None
79
+ )
80
+ if intent is not None:
61
81
  merged_attributes = dict(attributes or {})
62
82
  merged_attributes.update(dict(intent.result.attributes or {}))
63
83
  return SamplingResult(
@@ -84,10 +104,17 @@ def store_sampling_intent(
84
104
  parent_context: Context,
85
105
  intent: DurableSamplingIntent | None,
86
106
  ) -> Context:
87
- """Attach a durable sampling intent to a span parent context."""
107
+ """Attach a fresh single-use delivery of the invocation's sampling result.
108
+
109
+ The sampler consumes it before processors run. A processor forwarding this
110
+ context therefore uses its own sampling policy; another durable span gets a
111
+ new delivery without changing the reusable invocation-level intent.
112
+ """
88
113
  if intent is None:
89
114
  return parent_context
90
- return otel_context.set_value(_DURABLE_SAMPLING_INTENT_KEY, intent, parent_context)
115
+ return otel_context.set_value(
116
+ _DURABLE_SAMPLING_INTENT_KEY, _SamplingIntentDelivery(intent), parent_context
117
+ )
91
118
 
92
119
 
93
120
  def resolve_sampling_result(