atlan-application-sdk-api 3.51.1__tar.gz → 3.52.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/PKG-INFO +5 -3
  2. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/README.md +1 -1
  3. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/api-files.txt +1 -0
  4. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_runtime/offload.py +340 -6
  5. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/_listing.py +6 -3
  6. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/constants.py +48 -22
  7. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/__init__.py +2 -0
  8. atlan_application_sdk_api-3.52.0/application_sdk/handler/_dispatch.py +227 -0
  9. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/asgi.py +21 -7
  10. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/base.py +28 -2
  11. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/context.py +7 -1
  12. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/contracts.py +14 -0
  13. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/metadata_routes.py +82 -20
  14. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/object_routes.py +73 -9
  15. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/routes.py +212 -368
  16. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/version.py +1 -1
  17. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/pyproject.toml +11 -2
  18. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/.gitignore +0 -0
  19. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/__init__.py +0 -0
  20. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_build_info.py +0 -0
  21. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_context_errors.py +0 -0
  22. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_install.py +0 -0
  23. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_logging.py +0 -0
  24. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_runtime/__init__.py +0 -0
  25. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/_runtime/progress.py +0 -0
  26. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/__init__.py +0 -0
  27. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/_interface.py +0 -0
  28. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/_sql_failure_classifier.py +0 -0
  29. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/base.py +0 -0
  30. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/http_errors.py +0 -0
  31. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/models.py +0 -0
  32. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/sql.py +0 -0
  33. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/sql_errors.py +0 -0
  34. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/sql_typecasters.py +0 -0
  35. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/clients/ssl_utils.py +0 -0
  36. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/__init__.py +0 -0
  37. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/_generated_tree.py +0 -0
  38. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/aws_utils.py +0 -0
  39. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/aws_utils_errors.py +0 -0
  40. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/build_identity.py +0 -0
  41. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/concurrency.py +0 -0
  42. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/errors.py +0 -0
  43. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/filter_matching.py +0 -0
  44. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/logger.py +0 -0
  45. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/models.py +0 -0
  46. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/sql_filters.py +0 -0
  47. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/sql_filters_errors.py +0 -0
  48. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/task_queue.py +0 -0
  49. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/types.py +0 -0
  50. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/common/utils.py +0 -0
  51. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/contracts/__init__.py +0 -0
  52. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/contracts/base.py +0 -0
  53. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/contracts/events.py +0 -0
  54. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/contracts/types.py +0 -0
  55. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/contracts/types_errors.py +0 -0
  56. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/__init__.py +0 -0
  57. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/errors.py +0 -0
  58. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/extra.py +0 -0
  59. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/ingress.py +0 -0
  60. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/ref.py +0 -0
  61. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/spec.py +0 -0
  62. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/types.py +0 -0
  63. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/credentials/utils.py +0 -0
  64. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/errors/__init__.py +0 -0
  65. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/errors/base.py +0 -0
  66. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/errors/categories.py +0 -0
  67. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/errors/http.py +0 -0
  68. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/errors/leaves.py +0 -0
  69. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/errors/wire.py +0 -0
  70. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/_warmup.py +0 -0
  71. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/manifest.py +0 -0
  72. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/request_contract.py +0 -0
  73. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/handler/service_errors.py +0 -0
  74. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/server/__init__.py +0 -0
  75. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/server/fastapi/models.py +0 -0
  76. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/templates/__init__.py +0 -0
  77. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/templates/contracts/__init__.py +0 -0
  78. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/templates/contracts/base_metadata_extraction.py +0 -0
  79. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/templates/contracts/incremental_sql.py +0 -0
  80. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/templates/contracts/sql_metadata.py +0 -0
  81. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/application_sdk/templates/contracts/sql_query.py +0 -0
  82. {atlan_application_sdk_api-3.51.1 → atlan_application_sdk_api-3.52.0}/hatch_build.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: atlan-application-sdk-api
3
- Version: 3.51.1
3
+ Version: 3.52.0
4
4
  Summary: Experimental: the handler surface of the Atlan Application SDK, for the consolidated API host
5
5
  Project-URL: Homepage, https://github.com/atlanhq/application-sdk
6
6
  Project-URL: Source, https://github.com/atlanhq/application-sdk/tree/main/packages/api
@@ -15,6 +15,7 @@ Classifier: License :: OSI Approved :: Apache Software License
15
15
  Classifier: Programming Language :: Python :: 3
16
16
  Classifier: Programming Language :: Python :: 3.11
17
17
  Requires-Python: >=3.11
18
+ Requires-Dist: certifi>=2026.10.19
18
19
  Requires-Dist: fastapi<0.144.0,>=0.143.0
19
20
  Requires-Dist: httpx<0.29.0,>=0.27.0
20
21
  Requires-Dist: orjson<4.0.0,>=3.11.6
@@ -27,7 +28,8 @@ Requires-Dist: python-dotenv<2.0.0,>=1.1.0; extra == 'aws'
27
28
  Provides-Extra: pandas
28
29
  Requires-Dist: pandas<3.0.0,>=2.2.3; extra == 'pandas'
29
30
  Provides-Extra: pyatlan
30
- Requires-Dist: pyatlan<12,>=11.3; extra == 'pyatlan'
31
+ Requires-Dist: authlib<2,>=1.9.1; extra == 'pyatlan'
32
+ Requires-Dist: pyatlan<12,>=11.5.0; extra == 'pyatlan'
31
33
  Provides-Extra: sql
32
34
  Requires-Dist: httpx<0.29.0,>=0.27.0; extra == 'sql'
33
35
  Requires-Dist: python-dotenv<2.0.0,>=1.1.0; extra == 'sql'
@@ -158,7 +160,7 @@ an app answers identically from either one:
158
160
  | `POST /workflows/v1/metadata` | `Handler.fetch_metadata` |
159
161
  | `POST /workflows/v1/warmup` | `Handler.warmup` (optional) |
160
162
  | `GET /workflows/v1/configmap/{id}`, `/configmaps`, `/input-contract`, `GET`/`POST /workflows/v1/manifest` | The app's `generated/` folder, if the app package ships one |
161
- | `GET`/`POST /workflows/v1/config/{id}`, `POST /workflows/v1/file` | An S3 bucket named by `ATLAN_S3_BUCKET` (and `ATLAN_S3_REGION`); needs the `aws` extra. These routes return 503 when it isn't set |
163
+ | `GET`/`POST /workflows/v1/config/{id}`, `POST /workflows/v1/file` | An S3 bucket named by `ATLAN_S3_BUCKET` (and `ATLAN_S3_REGION`), or on GCP by `ATLAN_GCP_BUCKET` (and `ATLAN_GCP_REGION`) through GCS's S3-compatible API with the `ATLAN_GCP_HMAC_ACCESS_KEY` / `ATLAN_GCP_HMAC_SECRET` keys; needs the `aws` extra. These routes return 503 when neither is set |
162
164
  | Your own routes | FastAPI routers returned from `Handler.routers()`, served after the SDK's |
163
165
 
164
166
  `build_asgi_app` takes a few keyword options:
@@ -122,7 +122,7 @@ an app answers identically from either one:
122
122
  | `POST /workflows/v1/metadata` | `Handler.fetch_metadata` |
123
123
  | `POST /workflows/v1/warmup` | `Handler.warmup` (optional) |
124
124
  | `GET /workflows/v1/configmap/{id}`, `/configmaps`, `/input-contract`, `GET`/`POST /workflows/v1/manifest` | The app's `generated/` folder, if the app package ships one |
125
- | `GET`/`POST /workflows/v1/config/{id}`, `POST /workflows/v1/file` | An S3 bucket named by `ATLAN_S3_BUCKET` (and `ATLAN_S3_REGION`); needs the `aws` extra. These routes return 503 when it isn't set |
125
+ | `GET`/`POST /workflows/v1/config/{id}`, `POST /workflows/v1/file` | An S3 bucket named by `ATLAN_S3_BUCKET` (and `ATLAN_S3_REGION`), or on GCP by `ATLAN_GCP_BUCKET` (and `ATLAN_GCP_REGION`) through GCS's S3-compatible API with the `ATLAN_GCP_HMAC_ACCESS_KEY` / `ATLAN_GCP_HMAC_SECRET` keys; needs the `aws` extra. These routes return 503 when neither is set |
126
126
  | Your own routes | FastAPI routers returned from `Handler.routers()`, served after the SDK's |
127
127
 
128
128
  `build_asgi_app` takes a few keyword options:
@@ -56,6 +56,7 @@ application_sdk/errors/http.py
56
56
  application_sdk/errors/leaves.py
57
57
  application_sdk/errors/wire.py
58
58
  application_sdk/handler/__init__.py
59
+ application_sdk/handler/_dispatch.py
59
60
  application_sdk/handler/_warmup.py
60
61
  application_sdk/handler/asgi.py
61
62
  application_sdk/handler/base.py
@@ -39,7 +39,7 @@ from application_sdk._runtime.progress import (
39
39
  current_progress_tracker,
40
40
  declared_hold_active,
41
41
  )
42
- from application_sdk.errors import InvalidInputValueError
42
+ from application_sdk.errors import InvalidInputValueError, ResourceExhaustedError
43
43
 
44
44
  logger = get_logger(__name__)
45
45
 
@@ -373,6 +373,314 @@ def _auto_hold(label: str, timeout: float | None) -> Iterator[None]:
373
373
  tracker.exit_hold(hold)
374
374
 
375
375
 
376
+ # ---------------------------------------------------------------------------
377
+ # Per-app containment of the blocking pool (FND-2973)
378
+ # ---------------------------------------------------------------------------
379
+ #
380
+ # A timeout frees the *caller* of a ``run_in_thread`` call but cannot stop its
381
+ # thread, so a driver call hung on a dead host keeps its pool slot until the
382
+ # driver returns on its own. On the worker (one app per pod, the process-wide
383
+ # ``_BLOCKING_EXECUTOR``) that damage stays inside the app that caused it. The
384
+ # consolidated API host gives each app its own pool (``blocking_executor``), so
385
+ # one app's hung calls cannot take another app's slots; within that app they
386
+ # can still take every slot and queue the app's other calls behind them.
387
+ #
388
+ # Two pieces contain it:
389
+ #
390
+ # * **A per-app cap**, opt-in through :data:`OFFLOAD_MAX_THREADS_PER_APP_ENV`.
391
+ # A call that would take an app past its cap fails at once with a retryable
392
+ # ``ResourceExhaustedError`` instead of queueing, so a misbehaving app
393
+ # degrades only itself. On the host, keep the cap at or below the app's pool
394
+ # width (``blocking_threads``): above it, hung calls fill the pool first and
395
+ # the app's next calls queue behind them until the cap is reached. Unset (the
396
+ # default), nothing changes: calls queue
397
+ # exactly as before. A default cap would turn today's queueing into failures
398
+ # for any fan-out wider than the cap, and the pool width it would be derived
399
+ # from is not a reliable number — ``os.cpu_count()`` reports the node's
400
+ # cores, not the pod's CPU limit.
401
+ # * **A per-app stranded count**: calls whose caller stopped waiting while the
402
+ # thread kept running. Exported as the ``offload.stranded_threads`` gauge,
403
+ # labelled by app only, and logged once per strand with the callable's name.
404
+ #
405
+ # A slot is held from submit until the *thread* finishes — released in the
406
+ # executor future's done callback, never when the awaiting coroutine returns.
407
+ # Releasing on the caller's side would let a hung thread hand back its quota
408
+ # while it still holds its pool slot, which is the one thing the cap exists to
409
+ # prevent. The done callback also fires for a queued call cancelled before it
410
+ # started, so that case does not leak a slot either.
411
+
412
+ #: Env var capping how many ``run_in_thread`` calls one app may have in flight
413
+ #: (running or queued) at once. Unset or empty means no cap.
414
+ OFFLOAD_MAX_THREADS_PER_APP_ENV = "ATLAN_OFFLOAD_MAX_THREADS_PER_APP"
415
+
416
+ #: The app a ``run_in_thread`` call is charged to. Bound by the handler layer
417
+ #: (``bind_handler_context``) for each handler invocation, and on the
418
+ #: consolidated host for every request to an app (``handler.asgi``); otherwise
419
+ #: the process's own ``ATLAN_APPLICATION_NAME``. On the host that fallback is
420
+ #: the host's name, which is correct for what reaches it: startup and background
421
+ #: work that belongs to no single app.
422
+ _OFFLOAD_OWNER: contextvars.ContextVar[str | None] = contextvars.ContextVar(
423
+ "_OFFLOAD_OWNER", default=None
424
+ )
425
+
426
+ #: Read at call time, not through ``application_sdk.constants``: that module
427
+ #: loads ``.env`` and is not part of the api distribution's core.
428
+ _APPLICATION_NAME_ENV = "ATLAN_APPLICATION_NAME"
429
+
430
+ #: Owner used when neither a binding nor ``ATLAN_APPLICATION_NAME`` names the app.
431
+ _UNKNOWN_OWNER = "<unknown>"
432
+
433
+ #: Cap values already reported as invalid, so a bad setting warns once.
434
+ _REPORTED_BAD_CAPS: set[str] = set()
435
+
436
+
437
+ @contextlib.contextmanager
438
+ def offload_owner(app_name: str) -> Iterator[None]:
439
+ """Charge every ``run_in_thread`` call made in this block to *app_name*.
440
+
441
+ The binding is a ContextVar, so it is per task and is inherited by tasks
442
+ created inside the block. An empty *app_name* leaves the fallback in place.
443
+ """
444
+ token = _OFFLOAD_OWNER.set(app_name or None)
445
+ try:
446
+ yield
447
+ finally:
448
+ _OFFLOAD_OWNER.reset(token)
449
+
450
+
451
+ def _current_offload_owner() -> str:
452
+ """The app the current ``run_in_thread`` call is charged to."""
453
+ bound = _OFFLOAD_OWNER.get()
454
+ if bound:
455
+ return bound
456
+ return os.environ.get(_APPLICATION_NAME_ENV, "").strip() or _UNKNOWN_OWNER
457
+
458
+
459
+ def _per_app_cap() -> int | None:
460
+ """The configured per-app cap, or ``None`` when no cap applies.
461
+
462
+ Read on every call so a deployment can change it without a code path that
463
+ caches a stale value. An unparseable or non-positive value is logged once
464
+ and ignored rather than raised: raising here would fail every offload in
465
+ the process over a configuration typo.
466
+ """
467
+ raw = os.environ.get(OFFLOAD_MAX_THREADS_PER_APP_ENV, "").strip()
468
+ if not raw:
469
+ return None
470
+ try:
471
+ cap = int(raw)
472
+ except ValueError:
473
+ cap = 0
474
+ if cap >= 1:
475
+ return cap
476
+ if raw not in _REPORTED_BAD_CAPS:
477
+ _REPORTED_BAD_CAPS.add(raw)
478
+ logger.warning(
479
+ "Ignoring %s=%r: expected a positive integer; offloads are not capped "
480
+ "per app",
481
+ OFFLOAD_MAX_THREADS_PER_APP_ENV,
482
+ raw,
483
+ )
484
+ return None
485
+
486
+
487
+ class _OffloadSlot:
488
+ """One ``run_in_thread`` call's claim on its app's share of the pool."""
489
+
490
+ __slots__ = ("owner", "label", "stranded_at", "finished")
491
+
492
+ def __init__(self, owner: str, label: str) -> None:
493
+ self.owner = owner
494
+ self.label = label
495
+ self.stranded_at: float | None = None
496
+ self.finished = False
497
+
498
+
499
+ class _AppOffloadLedger:
500
+ """Per-app in-flight and stranded counts for the blocking pool. Thread-safe.
501
+
502
+ Mutated from the event loop (claim, strand) and from pool threads (the
503
+ executor future's done callback), hence a ``threading.Lock`` rather than
504
+ anything asyncio-side.
505
+ """
506
+
507
+ def __init__(self) -> None:
508
+ self._lock = threading.Lock()
509
+ self._in_flight: dict[str, int] = {}
510
+ self._stranded: dict[str, int] = {}
511
+
512
+ def claim(self, owner: str, label: str, cap: int | None) -> _OffloadSlot | None:
513
+ """Take a slot for *owner*; ``None`` if that would exceed *cap*."""
514
+ with self._lock:
515
+ held = self._in_flight.get(owner, 0)
516
+ if cap is not None and held >= cap:
517
+ return None
518
+ self._in_flight[owner] = held + 1
519
+ return _OffloadSlot(owner, label)
520
+
521
+ def strand(self, slot: _OffloadSlot) -> int | None:
522
+ """Record that *slot*'s caller stopped waiting while its thread runs on.
523
+
524
+ Returns the owner's stranded count after this one, or ``None`` if the
525
+ thread had already finished (nothing is stranded).
526
+ """
527
+ with self._lock:
528
+ if slot.finished or slot.stranded_at is not None:
529
+ return None
530
+ slot.stranded_at = time.monotonic()
531
+ count = self._stranded.get(slot.owner, 0) + 1
532
+ self._stranded[slot.owner] = count
533
+ return count
534
+
535
+ def release(self, slot: _OffloadSlot) -> float | None:
536
+ """Give *slot* back once its thread is done.
537
+
538
+ Returns how long the thread ran after being stranded, or ``None`` if it
539
+ was never stranded.
540
+ """
541
+ with self._lock:
542
+ if slot.finished:
543
+ return None
544
+ slot.finished = True
545
+ self._in_flight[slot.owner] = self._in_flight.get(slot.owner, 1) - 1
546
+ if self._in_flight[slot.owner] <= 0:
547
+ del self._in_flight[slot.owner]
548
+ if slot.stranded_at is None:
549
+ return None
550
+ self._stranded[slot.owner] = self._stranded.get(slot.owner, 1) - 1
551
+ if self._stranded[slot.owner] <= 0:
552
+ del self._stranded[slot.owner]
553
+ return time.monotonic() - slot.stranded_at
554
+
555
+ def in_flight(self, owner: str) -> int:
556
+ with self._lock:
557
+ return self._in_flight.get(owner, 0)
558
+
559
+ def stranded(self) -> dict[str, int]:
560
+ """A snapshot of every app's stranded count, omitting apps at zero."""
561
+ with self._lock:
562
+ return dict(self._stranded)
563
+
564
+
565
+ _LEDGER = _AppOffloadLedger()
566
+
567
+ #: Lazily-created OTel instruments, keyed by name.
568
+ _INSTRUMENTS: dict[str, Any] = {}
569
+ _INSTRUMENTS_LOCK = threading.Lock()
570
+
571
+
572
+ def _observe_stranded(_options: Any) -> Iterator[Any]:
573
+ try:
574
+ from opentelemetry.metrics import ( # noqa: PLC0415 — only reached from the OTel collection callback
575
+ Observation,
576
+ )
577
+ except ImportError:
578
+ # Absent on an api-only install; the gauge is never registered there.
579
+ return
580
+ for owner, count in _LEDGER.stranded().items():
581
+ yield Observation(count, {"app.name": owner})
582
+
583
+
584
+ def _ensure_stranded_gauge() -> None:
585
+ """Register the ``offload.stranded_threads`` gauge, once, on the first strand.
586
+
587
+ Observable rather than pushed: the count only matters while it is non-zero,
588
+ and a callback reads the ledger at collection time instead of every strand
589
+ and release writing a sample. Never raises — telemetry must not turn a
590
+ cancellation into a different failure.
591
+ """
592
+ if "stranded" in _INSTRUMENTS:
593
+ return
594
+ with _INSTRUMENTS_LOCK:
595
+ if "stranded" in _INSTRUMENTS:
596
+ return
597
+ try:
598
+ from opentelemetry import ( # noqa: PLC0415 — cold path: only reached on the first strand
599
+ metrics as _otel_metrics,
600
+ )
601
+
602
+ _INSTRUMENTS["stranded"] = _otel_metrics.get_meter(
603
+ "application_sdk.runtime"
604
+ ).create_observable_gauge(
605
+ "offload.stranded_threads",
606
+ callbacks=[_observe_stranded],
607
+ unit="1",
608
+ description=(
609
+ "run_in_thread calls whose caller stopped waiting (timeout or "
610
+ "cancellation) while the thread is still running and holding a "
611
+ "blocking-pool slot. Labelled by app; a sustained non-zero value "
612
+ "names the app whose blocking calls ignore their own deadlines."
613
+ ),
614
+ )
615
+ except ImportError:
616
+ # An api-only install has no OpenTelemetry: the strand is still
617
+ # logged, there is just no gauge.
618
+ _INSTRUMENTS["stranded"] = None
619
+ except Exception:
620
+ _INSTRUMENTS["stranded"] = None
621
+ logger.debug("Could not register offload.stranded_threads", exc_info=True)
622
+
623
+
624
+ def _on_offload_done(slot: _OffloadSlot) -> None:
625
+ """Executor done callback: release *slot*, reporting a stranded thread's end."""
626
+ stranded_for = _LEDGER.release(slot)
627
+ if stranded_for is not None:
628
+ logger.info(
629
+ "Stranded offloaded call %s for app %s finished %.1fs after its caller "
630
+ "stopped waiting; its blocking-pool slot is free again",
631
+ slot.label,
632
+ slot.owner,
633
+ stranded_for,
634
+ )
635
+
636
+
637
+ def _abandon(slot: _OffloadSlot, future: "concurrent.futures.Future[Any]") -> None:
638
+ """The caller of *future* stopped waiting: cancel it if queued, else record a strand."""
639
+ if future.cancel():
640
+ # Still queued: it will never start, and the done callback has already
641
+ # given the slot back.
642
+ return
643
+ if future.done():
644
+ return
645
+ count = _LEDGER.strand(slot)
646
+ if count is None:
647
+ return
648
+ _ensure_stranded_gauge()
649
+ logger.warning(
650
+ "Offloaded call %s for app %s outlived its caller; the thread keeps a "
651
+ "blocking-pool slot until the call returns on its own (%d stranded for "
652
+ "this app). Give the blocking call a driver-level timeout so it returns "
653
+ "when its caller gives up",
654
+ slot.label,
655
+ slot.owner,
656
+ count,
657
+ )
658
+
659
+
660
+ def _claim_slot(label: str) -> _OffloadSlot:
661
+ """Claim a slot for the current app, or raise if that app is at its cap."""
662
+ owner = _current_offload_owner()
663
+ cap = _per_app_cap()
664
+ slot = _LEDGER.claim(owner, label, cap)
665
+ if slot is not None:
666
+ return slot
667
+ stranded = _LEDGER.stranded().get(owner, 0)
668
+ raise ResourceExhaustedError(
669
+ message=(
670
+ f"App {owner!r} already has {cap} blocking calls in flight "
671
+ f"({stranded} stranded past their caller); refusing {label} rather "
672
+ "than queueing it behind them"
673
+ ),
674
+ resource="offload_threads",
675
+ limit=str(cap),
676
+ observed=str(_LEDGER.in_flight(owner)),
677
+ suggested_action=(
678
+ "Give this app's blocking calls driver-level timeouts so a hung call "
679
+ f"returns its slot, or raise {OFFLOAD_MAX_THREADS_PER_APP_ENV}."
680
+ ),
681
+ )
682
+
683
+
376
684
  async def run_in_thread(
377
685
  func: Callable[..., T],
378
686
  *args: Any,
@@ -429,6 +737,18 @@ async def run_in_thread(
429
737
  isolated from the caller (copy semantics).
430
738
  - Threads run on a dedicated ``sdk-blocking-*`` pool, separate from
431
739
  Temporal's activity pool, to avoid deadlocking the worker.
740
+ - Each call is charged to an app: the handler invocation's app when one is
741
+ bound (see :func:`offload_owner`), else the process's
742
+ ``ATLAN_APPLICATION_NAME``. When ``ATLAN_OFFLOAD_MAX_THREADS_PER_APP`` is set,
743
+ a call that would take its app past that many calls in flight raises a
744
+ retryable ``ResourceExhaustedError`` at once instead of queueing. Unset,
745
+ calls queue as they always have. A call counts against its app until its
746
+ **thread** finishes, not until its caller stops waiting.
747
+ - A call whose caller stops waiting (timeout, cancellation) while its
748
+ thread runs on is *stranded*: logged at WARNING with the callable's name
749
+ and counted per app on the ``offload.stranded_threads`` gauge until the
750
+ thread returns. A call still queued when its caller stops waiting is
751
+ cancelled and never starts.
432
752
  - The offload is automatically wrapped in an **unbounded** progress hold
433
753
  (ADR-0018), so the stall watchdog never accuses a legitimately long
434
754
  blocking call of stalling. Nothing to do at the call site, and nothing
@@ -496,15 +816,29 @@ async def run_in_thread(
496
816
  if scopes:
497
817
  call = _tracked_offload(call, scopes, label)
498
818
  cancel_handle = _bound_handle(func)
819
+ # The app's own pool on the consolidated host (``blocking_executor``), else
820
+ # the process-wide one.
499
821
  executor = _BLOCKING_EXECUTOR_OVERRIDE.get() or _BLOCKING_EXECUTOR
822
+ # Claimed before the hold opens: an app at its cap fails here at once, with
823
+ # nothing to unwind (FND-2973).
824
+ slot = _claim_slot(label)
500
825
  with _auto_hold(_THREAD_HOLD_PREFIX + label, None):
501
- if cancel_handle is None:
502
- return await loop.run_in_executor(executor, call)
503
826
  try:
504
- return await loop.run_in_executor(executor, call)
827
+ future = executor.submit(call)
828
+ except BaseException:
829
+ # Never submitted (pool shut down at interpreter exit): no thread
830
+ # will ever release the slot, so release it here.
831
+ _LEDGER.release(slot)
832
+ raise
833
+ # Released when the *thread* is done — see the containment notes above.
834
+ future.add_done_callback(lambda _done: _on_offload_done(slot))
835
+ try:
836
+ return await asyncio.wrap_future(future, loop=loop)
505
837
  except asyncio.CancelledError:
506
- # Non-blocking: the action is handed to the cancel pool.
507
- cancel_handle.request()
838
+ if cancel_handle is not None:
839
+ # Non-blocking: the action is handed to the cancel pool.
840
+ cancel_handle.request()
841
+ _abandon(slot, future)
508
842
  raise
509
843
 
510
844
 
@@ -29,6 +29,11 @@ import sys
29
29
  from collections.abc import Iterator
30
30
  from pathlib import Path
31
31
 
32
+ try:
33
+ import fcntl
34
+ except ImportError: # Windows has no fcntl; only the darwin barrier below uses it
35
+ fcntl = None # type: ignore[assignment] # — sentinel for "no fcntl on this platform": None is not a module
36
+
32
37
  # Darwin-only fcntl op: flush kernel buffer + drive cache. Value from
33
38
  # <sys/fcntl.h>; not exported by Python's fcntl module.
34
39
  _DARWIN_F_FULLFSYNC = 51
@@ -114,9 +119,7 @@ def _flush_directory_metadata(path: Path) -> None:
114
119
  except OSError:
115
120
  return
116
121
  try:
117
- if sys.platform == "darwin":
118
- import fcntl # noqa: PLC0415
119
-
122
+ if sys.platform == "darwin" and fcntl is not None:
120
123
  try:
121
124
  fcntl.fcntl(fd, _DARWIN_F_FULLFSYNC)
122
125
  return
@@ -117,13 +117,22 @@ from application_sdk._build_info import ( # noqa: E402 — public here as befor
117
117
 
118
118
  _BUILD_INFO = load_build_info()
119
119
 
120
+
121
+ def _load_application_version(build_info: dict[str, str]) -> str:
122
+ return build_info.get("app_version", "") or os.getenv(
123
+ "ATLAN_APPLICATION_VERSION", ""
124
+ )
125
+
126
+
127
+ def _load_commit_sha(build_info: dict[str, str]) -> str:
128
+ return build_info.get("commit_sha", "") or os.getenv("ATLAN_COMMIT_SHA", "")
129
+
130
+
120
131
  #: Version of the app release exactly as Global Marketplace stores it
121
132
  #: (release tag for semver apps, sha7 for CD apps). Baked value first.
122
- APPLICATION_VERSION = _BUILD_INFO.get("app_version", "") or os.getenv(
123
- "ATLAN_APPLICATION_VERSION", ""
124
- )
133
+ APPLICATION_VERSION = _load_application_version(_BUILD_INFO)
125
134
  #: Git commit the image was built from. Baked only — no deployer stamps it.
126
- COMMIT_SHA = _BUILD_INFO.get("commit_sha", "") or os.getenv("ATLAN_COMMIT_SHA", "")
135
+ COMMIT_SHA = _load_commit_sha(_BUILD_INFO)
127
136
  #: Release UUID from Global Marketplace
128
137
  RELEASE_ID = os.getenv("ATLAN_RELEASE_ID", "")
129
138
  #: Release channel (all, beta, staging, specific)
@@ -351,6 +360,7 @@ STORAGE_PROGRESS_LOG_INTERVAL_SECONDS = float(
351
360
  os.getenv("ATLAN_STORAGE_PROGRESS_LOG_INTERVAL_SECONDS", "30")
352
361
  )
353
362
 
363
+
354
364
  #: Interval (seconds) between progress marks while an activity waits on a
355
365
  #: per-destination storage lock (materialise dedupe, chunked-transfer
356
366
  #: exclusion). Must stay under the stall watchdog's no-progress budget, or a
@@ -360,13 +370,17 @@ STORAGE_PROGRESS_LOG_INTERVAL_SECONDS = float(
360
370
  #: ``max_no_progress_seconds`` below that cap is the operator's explicit
361
371
  #: exception and is not defended here). Floored at 1s so a zero cannot turn
362
372
  #: the wait into a busy spin. (CONNECT-1126)
363
- STORAGE_LOCK_WAIT_PROGRESS_SECONDS = max(
364
- 1.0,
365
- min(
366
- float(os.getenv("ATLAN_STORAGE_LOCK_WAIT_PROGRESS_SECONDS", "30")),
367
- float(os.getenv("ATLAN_MAX_NO_PROGRESS_SECONDS", "900")) / 4.0,
368
- ),
369
- )
373
+ def _load_storage_lock_wait_progress_seconds() -> float:
374
+ return max(
375
+ 1.0,
376
+ min(
377
+ float(os.getenv("ATLAN_STORAGE_LOCK_WAIT_PROGRESS_SECONDS", "30")),
378
+ float(os.getenv("ATLAN_MAX_NO_PROGRESS_SECONDS", "900")) / 4.0,
379
+ ),
380
+ )
381
+
382
+
383
+ STORAGE_LOCK_WAIT_PROGRESS_SECONDS = _load_storage_lock_wait_progress_seconds()
370
384
 
371
385
  #: Multipart part size for uploads (default 8 MiB). Raise this per deployment
372
386
  #: when the destination makes *part count* expensive rather than part size.
@@ -454,13 +468,16 @@ STORAGE_WRITE_SIDECARS = (
454
468
  os.getenv("ATLAN_STORAGE_WRITE_SIDECARS", "true").strip().lower() != "false"
455
469
  )
456
470
 
471
+
457
472
  #: Opt-in for the e2e ``sdk:store-assert`` node (FND-3571). The workflow is
458
473
  #: registered on every worker so worker start-up is identical everywhere; with
459
474
  #: this unset the node reads nothing and reports itself disabled. Only e2e
460
475
  #: installs set it, per tenant, through ``deploy.env_overrides``.
461
- STORE_ASSERT_ENABLED = (
462
- os.getenv("ATLAN_STORE_ASSERT_ENABLED", "false").strip().lower() == "true"
463
- )
476
+ def _load_store_assert_enabled() -> bool:
477
+ return os.getenv("ATLAN_STORE_ASSERT_ENABLED", "false").strip().lower() == "true"
478
+
479
+
480
+ STORE_ASSERT_ENABLED = _load_store_assert_enabled()
464
481
 
465
482
  #: Build ID for worker versioning (injected by TWD controller via Kubernetes Downward API).
466
483
  #: When set, workers identify themselves with this build ID so the Temporal server can
@@ -660,9 +677,13 @@ OOM_RESTART_CHECK = _load_oom_restart_check()
660
677
  #: Whether to use server-side cursors for SQL operations.
661
678
  #: Enabled by default; set ATLAN_SQL_USE_SERVER_SIDE_CURSOR to any value other
662
679
  #: than "true" (e.g. "false") to opt out.
663
- USE_SERVER_SIDE_CURSOR = (
664
- os.getenv("ATLAN_SQL_USE_SERVER_SIDE_CURSOR", "true").strip().lower() == "true"
665
- )
680
+ def _load_use_server_side_cursor() -> bool:
681
+ return (
682
+ os.getenv("ATLAN_SQL_USE_SERVER_SIDE_CURSOR", "true").strip().lower() == "true"
683
+ )
684
+
685
+
686
+ USE_SERVER_SIDE_CURSOR = _load_use_server_side_cursor()
666
687
 
667
688
  # DAPR Constants
668
689
  #: Name of the state store component in DAPR
@@ -839,12 +860,17 @@ LOG_LEVEL = (os.getenv("ATLAN_LOG_LEVEL") or os.getenv("LOG_LEVEL", "INFO")).upp
839
860
  #: Used by traces_adaptor.py and metrics_adaptor.py at module level for
840
861
  #: tracer/meter initialization. AppConfig.service_name is the runtime equivalent.
841
862
  SERVICE_NAME: str = os.getenv("OTEL_SERVICE_NAME", "atlan-application-sdk")
842
- #: Service version for OpenTelemetry
843
- SERVICE_VERSION: str = os.getenv("OTEL_SERVICE_VERSION", "")
844
- if not SERVICE_VERSION:
845
- from application_sdk.version import __version__
846
863
 
847
- SERVICE_VERSION = __version__
864
+
865
+ from application_sdk.version import __version__ # noqa: E402
866
+
867
+
868
+ def _load_service_version() -> str:
869
+ return os.getenv("OTEL_SERVICE_VERSION", "") or __version__
870
+
871
+
872
+ #: Service version for OpenTelemetry
873
+ SERVICE_VERSION: str = _load_service_version()
848
874
  #: Additional resource attributes for OpenTelemetry
849
875
  OTEL_RESOURCE_ATTRIBUTES: str = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "")
850
876
  #: Endpoint for the OpenTelemetry collector
@@ -20,6 +20,7 @@ from application_sdk.handler.contracts import (
20
20
  BaseMetadataConfig,
21
21
  CheckTier,
22
22
  HandlerCredential,
23
+ ManifestInput,
23
24
  MetadataInput,
24
25
  MetadataOutput,
25
26
  PreflightCheck,
@@ -69,6 +70,7 @@ __all__ = [
69
70
  "HandlerContext",
70
71
  "HandlerCredential",
71
72
  "HandlerError",
73
+ "ManifestInput",
72
74
  "MetadataInput",
73
75
  "MetadataOutput",
74
76
  "PreflightCheck",