apache-airflow-task-sdk 1.3.1rc2__tar.gz → 1.3.2__tar.gz

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/PKG-INFO

@@ -1,10 +1,10 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: apache-airflow-task-sdk
-Version: 1.3.1rc2
+Version: 1.3.2
 Summary: Python Task SDK for Apache Airflow DAG Authors
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/
-Project-URL: Homepage, https://airflow.staged.apache.org/
+Project-URL: Documentation, https://airflow.apache.org/docs/
+Project-URL: Homepage, https://airflow.apache.org/
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Mastodon, https://fosstodon.org/@airflow
@@ -29,7 +29,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: System :: Monitoring
 Requires-Python: !=3.15,>=3.10
-Requires-Dist: apache-airflow-core<3.4.0,>=3.3.0rc1
+Requires-Dist: apache-airflow-core<3.4.0,>=3.3.0
 Requires-Dist: asgiref>=2.3.0; python_version < '3.14'
 Requires-Dist: asgiref>=3.11.1; python_version >= '3.14'
 Requires-Dist: attrs!=25.2.0,>=24.2.0

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/docs/api.rst

@@ -116,6 +116,10 @@ Deadline Alerts
 
 .. autoclass:: airflow.sdk.DeadlineReference
 
+.. autoclass:: airflow.sdk.BaseDeadlineReference
+
+.. autofunction:: airflow.sdk.deadline_reference
+
 Connections & Variables
 -----------------------
 .. autoapiclass:: airflow.sdk.Connection

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/docs/deferred-vs-async-operators.rst

@@ -234,7 +234,7 @@ This allows multiple paginated requests to be performed efficiently within a sin
 
        @task
        async def get_users():
-           hook = KiotaRequestAdapterHook.get_hook(conn_id="msgraph_default")
+           hook = await KiotaRequestAdapterHook.aget_hook(conn_id="msgraph_default")
 
            return await hook.paginated_run(url="users")
 

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/docs/examples.rst

@@ -18,7 +18,7 @@
 Examples
 ========
 
-.. note:: For a minimal quick start, see the `Getting Started <../index.rst#getting-started>`_ section.
+.. note:: For a minimal quick start, see the :ref:`Getting Started <getting-started>` section.
 
 Key Concepts
 ------------

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/docs/index.rst

@@ -40,6 +40,8 @@ To install the Task SDK, run:
 
    pip install apache-airflow-task-sdk
 
+.. _getting-started:
+
 Getting Started
 ^^^^^^^^^^^^^^^
 Define a basic Dag and task in just a few lines of Python:

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/pyproject.toml

@@ -48,7 +48,7 @@ classifiers = [
     "Topic :: System :: Monitoring",
 ]
 dependencies = [
-    "apache-airflow-core<3.4.0,>=3.3.0rc1",
+    "apache-airflow-core<3.4.0,>=3.3.0",
     "asgiref>=2.3.0; python_version < '3.14'",
     "asgiref>=3.11.1; python_version >= '3.14'",
     "attrs>=24.2.0, !=25.2.0",
@@ -112,8 +112,8 @@ dependencies = [
 
 [project.urls]
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
-Documentation = "https://airflow.staged.apache.org/docs/"
-Homepage = "https://airflow.staged.apache.org/"
+Documentation = "https://airflow.apache.org/docs/"
+Homepage = "https://airflow.apache.org/"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
 "Source Code" = "https://github.com/apache/airflow"
 Mastodon = "https://fosstodon.org/@airflow"
@@ -123,11 +123,12 @@ YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
 
 [build-system]
 requires = [
-    "hatchling==1.30.1",
-    "packaging==26.2",
+    "hatchling==1.32.0",
+    "packaging==26.3",
     "pathspec==1.1.1",
     "pluggy==1.6.0",
     "tomli==2.4.1; python_version < '3.11'",
+    "tomlkit==0.15.1",
     "trove-classifiers==2026.6.1.19",
 ]
 build-backend = "hatchling.build"

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/__init__.py

@@ -31,6 +31,7 @@ __all__ = [
     "AsyncCallback",
     "BaseAsyncOperator",
     "BaseBranchOperator",
+    "BaseDeadlineReference",
     "BaseHook",
     "BaseNotifier",
     "BaseOperator",
@@ -105,6 +106,7 @@ __all__ = [
     "conf",
     "cross_downstream",
     "dag",
+    "deadline_reference",
     "get_current_context",
     "get_parsing_context",
     "literal",
@@ -117,7 +119,7 @@ __all__ = [
     "teardown",
 ]
 
-__version__ = "1.3.1rc2"
+__version__ = "1.3.2"
 
 if TYPE_CHECKING:
     from airflow.sdk.api.datamodels._generated import DagRunState, TaskInstanceState, TriggerRule, WeightRule
@@ -151,7 +153,12 @@ if TYPE_CHECKING:
     from airflow.sdk.definitions.connection import Connection
     from airflow.sdk.definitions.context import Context, get_current_context, get_parsing_context
     from airflow.sdk.definitions.dag import DAG, dag
-    from airflow.sdk.definitions.deadline import DeadlineAlert, DeadlineReference
+    from airflow.sdk.definitions.deadline import (
+        BaseDeadlineReference,
+        DeadlineAlert,
+        DeadlineReference,
+        deadline_reference,
+    )
     from airflow.sdk.definitions.decorators import result, setup, task, teardown
     from airflow.sdk.definitions.decorators.task_group import task_group
     from airflow.sdk.definitions.edges import EdgeModifier, Label
@@ -234,6 +241,7 @@ __lazy_imports: dict[str, str] = {
     "AsyncCallback": ".definitions.callback",
     "BaseAsyncOperator": ".bases.operator",
     "BaseBranchOperator": ".bases.branch",
+    "BaseDeadlineReference": ".definitions.deadline",
     "BaseHook": ".bases.hook",
     "BaseNotifier": ".bases.notifier",
     "BaseOperator": ".bases.operator",
@@ -308,6 +316,7 @@ __lazy_imports: dict[str, str] = {
     "conf": ".configuration",
     "cross_downstream": ".bases.operator",
     "dag": ".definitions.dag",
+    "deadline_reference": ".definitions.deadline",
     "NEVER_EXPIRE": ".execution_time.context",
     "get_current_context": ".definitions.context",
     "get_parsing_context": ".definitions.context",

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/_shared/observability/metrics/metrics_template.yaml

@@ -345,6 +345,24 @@ metrics:
     legacy_name: "-"
     name_variables: []
 
+  - name: "scheduler.dag_bag.cache_hit"
+    description: "Number of cache hits when retrieving SerializedDAG from DBDagBag in the scheduler"
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "scheduler.dag_bag.cache_miss"
+    description: "Number of cache misses when retrieving SerializedDAG from DBDagBag in the scheduler"
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "scheduler.dag_bag.cache_clear"
+    description: "Number of times the DBDagBag cache was cleared in the scheduler"
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
   - name: "connection_test.success"
     description: "Number of worker-dispatched connection tests that completed successfully."
     type: "counter"
@@ -379,6 +397,12 @@ metrics:
     legacy_name: "-"
     name_variables: []
 
+  - name: "scheduler.dag_bag.cache_size"
+    description: "Current number of SerializedDAG objects cached in the scheduler's DBDagBag"
+    type: "gauge"
+    legacy_name: "-"
+    name_variables: []
+
   - name: "connection_test.active"
     description: "Number of connection tests currently in flight (``queued`` + ``running``), sampled by the
     scheduler each tick."

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/_shared/plugins_manager/plugins_manager.py

@@ -28,7 +28,7 @@ import os
 import sys
 import types
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 if TYPE_CHECKING:
     if sys.version_info >= (3, 12):
@@ -177,7 +177,9 @@ def is_valid_plugin(plugin_obj) -> bool:
     )
 
     if is_airflow_plugin and plugin_obj.__name__ != "AirflowPlugin":
-        plugin_obj.validate()
+        # The MRO check above establishes this by name; `inspect.isclass` only narrows to
+        # `type[object]`, which does not carry `validate`.
+        cast("type[AirflowPlugin]", plugin_obj).validate()
         return True
     return False
 

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/_shared/secrets_masker/secrets_masker.py

@@ -219,14 +219,20 @@ class SecretsMasker(logging.Filter):
                     break
             else:
                 # Block only runs if no break above.
-
-                f = cls._redact
-
-                @functools.wraps(f)
-                def _redact(*args, replacement: str = "***", **kwargs):
-                    return f(*args, **kwargs)
-
-                cls._redact = _redact
+                # Use a factory so the captured subclass method is held in the factory's
+                # closure rather than as a default argument on `_redact`. A default arg
+                # would leak the name into `_redact`'s keyword signature (visible via
+                # `inspect.signature(..., follow_wrapped=False)` and callable-visible via
+                # `**kwargs`), which would let a caller silently substitute a different
+                # function inside the secrets-masking path.
+                def _make_redact(f):
+                    @functools.wraps(f)
+                    def _redact(*args, replacement: str = "***", **kwargs):
+                        return f(*args, **kwargs)
+
+                    return _redact
+
+                cls._redact = _make_redact(cls._redact)
                 ...
 
     @classmethod

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/_shared/timezones/timezone.py

@@ -234,6 +234,17 @@ def td_format(td_object: None | dt.timedelta | float | int) -> str | None:
     """
     if td_object is None:
         return None
+
+    prefix = ""
+    if isinstance(td_object, dt.timedelta):
+        if td_object < dt.timedelta(0):
+            prefix = "-"
+            td_object = -td_object
+    elif isinstance(td_object, (int, float)):
+        if td_object < 0:
+            prefix = "-"
+            td_object = -td_object
+
     if isinstance(td_object, dt.timedelta):
         delta = relativedelta() + td_object
     else:
@@ -258,7 +269,7 @@ def td_format(td_object: None | dt.timedelta | float | int) -> str | None:
     joined = ":".join(part for part in parts if part)
     if not joined:
         return "<1s"
-    return joined
+    return f"{prefix}{joined}"
 
 
 def parse_timezone(name: str | int) -> FixedTimezone | Timezone:

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/api/client.py

@@ -211,6 +211,12 @@ def raise_on_4xx_5xx_with_note(response: httpx.Response):
         e.add_note(
             f"Correlation-id={response.headers.get('correlation-id', None) or response.request.headers.get('correlation-id', 'no-correlation-id')}"
         )
+        # .detail sits behind the generic message and only reaches logs where a handler
+        # logs it by hand. Add it as a note too, so uncaught paths (e.g. the executor) keep it.
+
+        detail = getattr(e, "detail", None)
+        if detail is not None:
+            e.add_note(f"Server error detail: {detail!r}")
         raise
 
 

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/api/datamodels/_generated.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  http://0.0.0.0:8080/execution/openapi.json
-#   version:   0.71.0
+#   version:   0.77.0
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/bases/decorator.py

@@ -253,6 +253,38 @@ def determine_kwargs(
     return KeywordParameters.determine(func, args, kwargs).unpacking()
 
 
+_TASK_DECORATOR_CALL_HINT = (
+    "This can happen when a @task-decorated function shadows another callable and the decorated task "
+    "object is called like a regular function. Rename the task function or call the original callable instead."
+)
+
+
+def _is_python_callable_already_executing(python_callable: Callable) -> bool:
+    target_code = getattr(python_callable, "__code__", None)
+    if target_code is None:
+        return False
+
+    frame = inspect.currentframe()
+    try:
+        while frame is not None:
+            if frame.f_code is target_code:
+                return True
+            frame = frame.f_back
+        return False
+    finally:
+        del frame
+
+
+def _should_add_task_decorator_call_hint(
+    err: TypeError, python_callable: Callable, op_args: Collection[Any]
+) -> bool:
+    return (
+        bool(op_args)
+        and "too many positional arguments" in str(err)
+        and _is_python_callable_already_executing(python_callable)
+    )
+
+
 class DecoratedOperator(BaseOperator):
     """
     Wraps a Python callable and captures args/kwargs when called for execution.
@@ -365,10 +397,15 @@ class DecoratedOperator(BaseOperator):
         # check all the arguments we know are valid. Whether these are enough
         # can only be known at execution time, when unmapping happens, and this
         # is called without the _airflow_mapped_validation_only flag.
-        if kwargs.get("_airflow_mapped_validation_only"):
-            signature.bind_partial(*op_args, **op_kwargs)
-        else:
-            signature.bind(*op_args, **op_kwargs)
+        try:
+            if kwargs.get("_airflow_mapped_validation_only"):
+                signature.bind_partial(*op_args, **op_kwargs)
+            else:
+                signature.bind(*op_args, **op_kwargs)
+        except TypeError as err:
+            if _should_add_task_decorator_call_hint(err, python_callable, op_args):
+                raise TypeError(f"{err}. {_TASK_DECORATOR_CALL_HINT}") from err
+            raise
 
         # Params in injected_for_ordering are semantically required even though they received a
         # None default to satisfy Python's ordering constraint. Verify they are actually provided.

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/bases/operator.py

@@ -270,6 +270,21 @@ OPERATOR_DEFAULTS: dict[str, Any] = {
 }
 
 
+def _get_operator_defaults(operator_class: type[BaseOperator]) -> dict[str, Any]:
+    """Get default values for an operator class's BaseOperator arguments."""
+    operator_defaults = OPERATOR_DEFAULTS.copy()
+    operator_classes = operator_class.mro()
+    for operator_base in reversed(operator_classes[: operator_classes.index(BaseOperator)]):
+        if (init := operator_base.__dict__.get("__init__")) is None:
+            continue
+        operator_defaults.update(
+            (name, parameter.default)
+            for name, parameter in inspect.signature(init).parameters.items()
+            if name in OPERATOR_DEFAULTS and parameter.default is not inspect.Parameter.empty
+        )
+    return operator_defaults
+
+
 # This is what handles the actual mapping.
 
 if TYPE_CHECKING:
@@ -372,7 +387,9 @@ else:
         )
 
         # Fill fields not provided by the user with default values.
-        partial_kwargs.update((k, v) for k, v in OPERATOR_DEFAULTS.items() if k not in partial_kwargs)
+        partial_kwargs.update(
+            (k, v) for k, v in _get_operator_defaults(operator_class).items() if k not in partial_kwargs
+        )
 
         # Post-process arguments. Should be kept in sync with _TaskDecorator.expand().
         if "task_concurrency" in kwargs:  # Reject deprecated option.

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/configuration.py

@@ -304,22 +304,32 @@ def initialize_secrets_backends(
     return backend_list
 
 
+_secrets_backend_cache: dict[tuple[str, ...], list] = {}
+
+
+def clear_secrets_backends_cache() -> None:
+    """Drop the memoised backends so the next load rebuilds them from the current config."""
+    _secrets_backend_cache.clear()
+
+
 def ensure_secrets_loaded(
     default_backends: list[str] = _SERVER_DEFAULT_SECRETS_SEARCH_PATH,
 ) -> list:
     """
-    Ensure that all secrets backends are loaded.
+    Return the secrets backends for the given search path, building them once per process.
 
-    If the secrets_backend_list contains only 2 default backends, reload it.
+    A backend holds an authenticated client, so rebuilding one per lookup makes every secret
+    fetch authenticate again against the remote store. Nothing is memoised until a custom
+    backend is configured, so one appearing after the first lookup is still picked up.
     """
-    # Check if the secrets_backend_list contains only 2 default backends.
-
-    # Check if we are loading the backends for worker too by checking if the default_backends is equal
-    # to _SERVER_DEFAULT_SECRETS_SEARCH_PATH.
-    secrets_backend_list = initialize_secrets_backends()
-    if len(secrets_backend_list) == 2 or default_backends != _SERVER_DEFAULT_SECRETS_SEARCH_PATH:
-        return initialize_secrets_backends(default_backends=default_backends)
-    return secrets_backend_list
+    key = tuple(default_backends)
+    if key not in _secrets_backend_cache:
+        backends = initialize_secrets_backends(default_backends=default_backends)
+        # Equal lengths mean nothing was prepended, so no custom backend is configured yet.
+        if len(backends) == len(default_backends):
+            return backends
+        _secrets_backend_cache[key] = backends
+    return _secrets_backend_cache[key]
 
 
 def initialize_config() -> AirflowSDKConfigParser:

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/definitions/connection.py

@@ -153,10 +153,13 @@ class Connection:
         else:
             self.__dict__.update(attrs.asdict(self.from_uri(uri, conn_id=conn_id), recurse=False))
 
-    def get_uri(self) -> str:
-        """Generate and return connection in URI format."""
-        from urllib.parse import parse_qsl
+    def _build_uri(self, extra_dejson: dict) -> str:
+        """
+        Build the connection URI given a pre-resolved extra_dejson dict.
 
+        Shared by ``get_uri`` (sync) and ``aget_uri`` (async) so the
+        URI-assembly logic lives in exactly one place.
+        """
         if self.conn_type:
             uri = f"{self.conn_type.lower().replace('_', '-')}://"
         else:
@@ -202,7 +205,6 @@ class Connection:
 
         if self.extra:
             try:
-                extra_dejson = self.extra_dejson
                 query: str | None = urlencode(extra_dejson)
             except TypeError:
                 query = None
@@ -213,6 +215,20 @@ class Connection:
 
         return uri
 
+    def get_uri(self) -> str:
+        """Generate and return connection in URI format."""
+        return self._build_uri(self.extra_dejson)
+
+    async def aget_uri(self) -> str:
+        """
+        Async version of ``get_uri``, safe for use inside an async task.
+
+        Calls ``aextra_dejson`` so that secret masking uses ``asend()``
+        instead of the synchronous ``send()``, preventing
+        ``DeadlockImminentError`` when invoked from within an async context.
+        """
+        return self._build_uri(await self.aextra_dejson())
+
     def get_hook(self, *, hook_params=None):
         """Return hook based on conn_type."""
         from airflow.sdk._shared.module_loading import import_string
@@ -306,6 +322,25 @@ class Connection:
 
         return extra
 
+    async def aextra_dejson(self) -> dict:
+        """
+        Async version of ``extra_dejson``, safe for use inside an async task.
+
+        Uses ``amask_secret`` instead of the synchronous ``mask_secret``, so calling
+        this from within an async context does not trigger ``DeadlockImminentError``.
+        """
+        from airflow.sdk.log import amask_secret
+
+        extra: dict = {}
+        if self.extra:
+            try:
+                extra = json.loads(self.extra)
+            except JSONDecodeError:
+                log.exception("Failed to deserialize extra property `extra`, returning empty dictionary")
+            else:
+                await amask_secret(extra)
+        return extra
+
     def get_extra_dejson(self) -> dict:
         """Deserialize extra property to JSON."""
         import warnings

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/definitions/deadline.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from abc import ABC
 from dataclasses import dataclass
 from datetime import datetime, timedelta
@@ -42,11 +43,11 @@ class BaseDeadlineReference(ABC):
     """
     Base class for all Deadline Reference implementations.
 
-    This is a lightweight SDK class for DAG authoring. It only handles serialization.
-    The actual evaluation logic (_evaluate_with) is in Core's SerializedReferenceModels.
+    This is a lightweight SDK class for Dag authoring. It only handles serialization.
+    The actual evaluation logic (``_evaluate_with``) is in Core's ``SerializedReferenceModels``.
 
     For custom deadline references, users should inherit from this class and implement
-    _evaluate_with() with deferred Core imports (imports inside the method body).
+    ``_evaluate_with()`` with deferred Core imports (imports inside the method body).
     """
 
     @property
@@ -419,6 +420,13 @@ class VariableInterval:
     key: str
 
     def resolve(self) -> timedelta:
+        warnings.warn(
+            "VariableInterval.resolve() is deprecated and will be removed in a future release. "
+            "Deadline interval resolution is handled internally during deadline evaluation.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
         try:
             value = Variable.get(self.key)
         except AirflowRuntimeError as e:
@@ -431,7 +439,4 @@ class VariableInterval:
                 f"VariableInterval '{self.key}' must be an integer (seconds), got: {value!r}"
             ) from e
 
-        if seconds <= 0:
-            raise ValueError(f"VariableInterval '{self.key}' must be > 0, got: {seconds}")
-
         return timedelta(seconds=seconds)

{apache_airflow_task_sdk-1.3.1rc2 → apache_airflow_task_sdk-1.3.2}/src/airflow/sdk/execution_time/comms.py

@@ -331,10 +331,18 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
         else:
             len_bytes = self.socket.recv(4)
 
-        if len_bytes == b"":
+        if not len_bytes:
             raise EOFError("Request socket closed before length")
 
-        length = int.from_bytes(len_bytes, byteorder="big")
+        # Stream sockets may return fewer bytes than requested; accumulate the header.
+        len_buf = bytearray(len_bytes)
+        while len(len_buf) < 4:
+            chunk = self.socket.recv(4 - len(len_buf))
+            if not chunk:
+                raise EOFError(f"Request socket closed mid-length after {len(len_buf)} of 4 bytes")
+            len_buf.extend(chunk)
+
+        length = int.from_bytes(len_buf, byteorder="big")
 
         buffer = bytearray(length)
         mv = memoryview(buffer)