apache-airflow-task-sdk 1.4.0b2__tar.gz → 1.4.0rc2__tar.gz

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/.pre-commit-config.yaml

@@ -26,6 +26,8 @@ repos:
     hooks:
       - id: check-core-imports
         name: Check for core imports in task-sdk files
+        env:
+          PYTHONPYCACHEPREFIX: .build/pycache
         entry: ../scripts/ci/prek/check_core_imports_in_sdk.py
         language: python
         types: [python]
@@ -54,12 +56,16 @@ repos:
           ^src/airflow/sdk/types.py$
       - id: check-init-decorator-arguments
         name: Sync model __init__ and decorator arguments
+        env:
+          PYTHONPYCACHEPREFIX: .build/pycache
         language: python
         entry: ../scripts/ci/prek/check_init_decorator_arguments.py
         pass_filenames: false
         files: ^src/airflow/sdk/definitions/dag\.py$|^src/airflow/sdk/definitions/decorators/task_group\.py$
       - id: mypy-task-sdk
         name: Run mypy for task-sdk
+        env:
+          PYTHONPYCACHEPREFIX: .build/pycache
         language: python
         entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk
         pass_filenames: false

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apache-airflow-task-sdk
-Version: 1.4.0b2
+Version: 1.4.0rc2
 Summary: Python Task SDK for Apache Airflow DAG Authors
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Documentation, https://airflow.staged.apache.org/docs/
@@ -28,12 +28,12 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: System :: Monitoring
 Requires-Python: !=3.15,>=3.11
-Requires-Dist: apache-airflow-core<3.5.0,>=3.4.0b1
+Requires-Dist: apache-airflow-core<3.5.0,>=3.4.0rc1
 Requires-Dist: asgiref>=2.3.0; python_version < '3.14'
 Requires-Dist: asgiref>=3.11.1; python_version >= '3.14'
 Requires-Dist: attrs!=25.2.0,>=24.2.0
 Requires-Dist: babel>=2.17.0
-Requires-Dist: cadwyn>=6.1.1
+Requires-Dist: cadwyn>=8.1.0
 Requires-Dist: colorlog>=6.8.2
 Requires-Dist: fsspec>=2023.10.0
 Requires-Dist: greenback>=1.2.1

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/docs/ts-bundle-spec.rst

@@ -127,7 +127,7 @@ The ``airflowMetadata`` payload is compact UTF-8 JSON with this logical shape:
       "airflow_bundle_metadata_version": "1.0",
       "sdk": {
         "language": "typescript",
-        "version": "0.1.0-beta1",
+        "version": "1.0.0-beta1",
         "supervisor_schema_version": "2026-06-16"
       },
       "source": "main.ts",

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/pyproject.toml

@@ -47,12 +47,12 @@ classifiers = [
     "Topic :: System :: Monitoring",
 ]
 dependencies = [
-    "apache-airflow-core<3.5.0,>=3.4.0b1",
+    "apache-airflow-core<3.5.0,>=3.4.0rc1",
     "asgiref>=2.3.0; python_version < '3.14'",
     "asgiref>=3.11.1; python_version >= '3.14'",
     "attrs>=24.2.0, !=25.2.0",
     "babel>=2.17.0",
-    "cadwyn>=6.1.1",
+    "cadwyn>=8.1.0",
     "fsspec>=2023.10.0",
     "greenback>=1.2.1",
     "httpx2>=2.0.0",

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/__init__.py

@@ -123,7 +123,7 @@ __all__ = [
     "teardown",
 ]
 
-__version__ = "1.4.0b2"
+__version__ = "1.4.0rc2"
 
 if TYPE_CHECKING:
     from airflow.sdk.api.datamodels._generated import DagRunState, TaskInstanceState, TriggerRule, WeightRule

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/logging/structlog.py

@@ -20,6 +20,7 @@ from __future__ import annotations
 import codecs
 import io
 import itertools
+import linecache
 import logging
 import os
 import re
@@ -363,6 +364,16 @@ def _make_safe_enc_hook(default):
     return safe_enc_hook
 
 
+def _add_source_lines(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Add each frame's source code ``line``, which structlog's ``ExceptionDictTransformer`` omits."""
+    for stack in stacks:
+        for frame in stack["frames"]:
+            linecache.checkcache(frame["filename"])
+            if line := linecache.getline(frame["filename"], frame["lineno"]).strip():
+                frame["line"] = line
+    return stacks
+
+
 @cache
 def structlog_processors(
     json_output: bool,
@@ -436,7 +447,9 @@ def structlog_processors(
             use_rich=False, show_locals=False, suppress=suppress
         )
 
-        dict_tracebacks = structlog.processors.ExceptionRenderer(dict_exc_formatter)
+        dict_tracebacks = structlog.processors.ExceptionRenderer(
+            lambda exc_info: _add_source_lines(dict_exc_formatter(exc_info))
+        )
 
         import msgspec
 

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/module_loading/file_discovery.py

@@ -147,11 +147,13 @@ def _find_path_from_directory(
 
     :return: a generator of file paths which should not be ignored.
     """
-    # A Dict of patterns, keyed using resolved, absolute paths
+    # A Dict of patterns, keyed using the walked (not resolved) paths, so that a symlink switched
+    # during the walk (e.g. by git-sync) does not drop the patterns inherited by subdirectories
     patterns_by_dir: dict[Path, list[_IgnoreRule]] = {}
+    visited_dirs: set[Path] = set()
 
     for root, dirs, files in os.walk(base_dir_path, followlinks=True):
-        patterns: list[_IgnoreRule] = patterns_by_dir.get(Path(root).resolve(), [])
+        patterns: list[_IgnoreRule] = patterns_by_dir.get(Path(root), [])
 
         ignore_file_path = Path(root) / ignore_file_name
         if ignore_file_path.is_file():
@@ -176,12 +178,13 @@ def _find_path_from_directory(
         # explicit loop for infinite recursion detection since we are following symlinks in this walk
         for sd in dirs:
             dirpath = (Path(root) / sd).resolve()
-            if dirpath in patterns_by_dir:
+            if dirpath in visited_dirs:
                 raise RuntimeError(
                     "Detected recursive loop when walking DAG directory "
                     f"{base_dir_path}: {dirpath} has appeared more than once."
                 )
-            patterns_by_dir.update({dirpath: patterns.copy()})
+            visited_dirs.add(dirpath)
+            patterns_by_dir[Path(root) / sd] = patterns.copy()
 
         for file in files:
             if file != ignore_file_name:

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/observability/metrics/metrics_registry.py

@@ -81,7 +81,7 @@ def convert_to_rst_tables(metrics: list) -> str:
     # Counters section
     if counters:
         output.append("Counters")
-        output.append("--------")
+        output.append("^^^^^^^^")
         output.append("")
         output.append(tabulate(counters, headers=headers, tablefmt="rst"))
         output.append("")
@@ -89,7 +89,7 @@ def convert_to_rst_tables(metrics: list) -> str:
     # Gauges section
     if gauges:
         output.append("Gauges")
-        output.append("------")
+        output.append("^^^^^^")
         output.append("")
         output.append(tabulate(gauges, headers=headers, tablefmt="rst"))
         output.append("")
@@ -97,7 +97,7 @@ def convert_to_rst_tables(metrics: list) -> str:
     # Timers section
     if timers:
         output.append("Timers")
-        output.append("------")
+        output.append("^^^^^^")
         output.append("")
         output.append(tabulate(timers, headers=headers, tablefmt="rst"))
         output.append("")

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/observability/metrics/metrics_template.yaml

@@ -53,35 +53,109 @@ metrics:
 
   - name: "resumable_job.fresh_submit"
     description: "Number of times a ResumableJobMixin operator submitted a fresh job
-    with no prior run ID stored (first run). Metric with operator tagging."
+    with no prior run ID stored (first run). Metric with operator, dag_id and task_id tagging."
     type: "counter"
     legacy_name: "-"
     name_variables: []
 
   - name: "resumable_job.already_succeeded"
     description: "Number of times a ResumableJobMixin operator found a stored run ID
-    whose job had already completed successfully and skipped resubmission. Metric with operator tagging."
+    whose job had already completed successfully and skipped resubmission. Metric with operator,
+    dag_id and task_id tagging."
     type: "counter"
     legacy_name: "-"
     name_variables: []
 
   - name: "resumable_job.terminal_resubmit"
     description: "Number of times a ResumableJobMixin operator found a stored run ID
-    whose job was in a terminal (failed) state and submitted a fresh job. Metric with operator tagging."
+    whose job was in a terminal (failed) state and submitted a fresh job. Metric with operator,
+    dag_id and task_id tagging."
     type: "counter"
     legacy_name: "-"
     name_variables: []
 
   - name: "resumable_job.reconnect_attempt"
     description: "Number of times a ResumableJobMixin operator found a stored run ID
-    on retry and attempted to reconnect. Metric with operator tagging."
+    on retry and attempted to reconnect. Metric with operator, dag_id and task_id tagging."
     type: "counter"
     legacy_name: "-"
     name_variables: []
 
   - name: "resumable_job.reconnect_success"
     description: "Number of times a ResumableJobMixin operator successfully reconnected
-    to an active job on retry. Metric with operator tagging."
+    to an active job on retry. Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "task_state_store.read_found"
+    description: "Number of times a task read a key from the task state store and found a value.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "task_state_store.read_absent"
+    description: "Number of times a task read a key from the task state store and found nothing.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "task_state_store.write"
+    description: "Number of times a task wrote a key to the task state store.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "task_state_store.delete"
+    description: "Number of times a task deleted a single key from the task state store.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "task_state_store.clear"
+    description: "Number of times a task cleared every key in its task state store scope.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "asset_state_store.read_found"
+    description: "Number of times a task read a key from the asset state store and found a value.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "asset_state_store.read_absent"
+    description: "Number of times a task read a key from the asset state store and found nothing.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "asset_state_store.write"
+    description: "Number of times a task wrote a key to the asset state store.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
+  - name: "asset_state_store.delete"
+    description: "Number of times a task deleted a single key from the asset state store.
+    Counts each call, so one task may contribute several.
+    Metric with operator, dag_id and task_id tagging."
     type: "counter"
     legacy_name: "-"
     name_variables: []

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/observability/metrics/stats.py

@@ -67,7 +67,8 @@ def _reset_backend_after_fork() -> None:
     _backend = None
 
 
-os.register_at_fork(after_in_child=_reset_backend_after_fork)
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_reset_backend_after_fork)
 
 
 def normalize_name_for_stats(name: str, log_warning: bool = True) -> str:

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/plugins_manager/__init__.py

@@ -26,6 +26,8 @@ from .plugins_manager import (
     ExternalViewDict as ExternalViewDict,
     FastAPIAppDict as FastAPIAppDict,
     FastAPIRootMiddlewareDict as FastAPIRootMiddlewareDict,
+    PluginActionDestinationLiteral as PluginActionDestinationLiteral,
+    PluginActionDict as PluginActionDict,
     PluginsDirectorySource as PluginsDirectorySource,
     ReactAppDict as ReactAppDict,
     _load_entrypoint_plugins as _load_entrypoint_plugins,

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/plugins_manager/plugins_manager.py

@@ -121,6 +121,36 @@ class ReactAppDict(_BaseUIDict):
     destination: NotRequired[Literal[BaseDestinationLiteral, "dashboard", "dag_overview", "task_overview"]]
 
 
+PluginActionDestinationLiteral = Literal["dag", "dag_run", "task_instance"]
+
+
+class _BasePluginActionDict(TypedDict):
+    """Shared fields mirroring ``BasePluginActionResponse``."""
+
+    name: str
+    destination: PluginActionDestinationLiteral
+    icon: NotRequired[str | None]
+    icon_dark_mode: NotRequired[str | None]
+    applies_to: NotRequired[AppliesToDict | None]
+
+
+class PluginLinkActionDict(_BasePluginActionDict):
+    """An ``AirflowPlugin.ui_actions`` entry that opens a URL."""
+
+    action_type: Literal["link"]
+    href: str
+
+
+class PluginReactActionDict(_BasePluginActionDict):
+    """An ``AirflowPlugin.ui_actions`` entry that mounts a React component behind a button."""
+
+    action_type: Literal["react"]
+    bundle_url: str
+
+
+PluginActionDict = PluginLinkActionDict | PluginReactActionDict
+
+
 class FastAPIAppDict(TypedDict):
     """Dictionary structure for entries in AirflowPlugin.fastapi_apps."""
 
@@ -158,6 +188,7 @@ class AirflowPlugin:
     fastapi_root_middlewares: list[FastAPIRootMiddlewareDict] = []
     external_views: list[ExternalViewDict] = []
     react_apps: list[ReactAppDict] = []
+    ui_actions: list[PluginActionDict] = []
     ui_translations: list[Any] = []
     menu_links: list[Any] = []
     appbuilder_views: list[Any] = []

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/_shared/secrets_masker/secrets_masker.py

@@ -719,4 +719,4 @@ class RedactedIO(TextIO):
         return self.target.write(s)
 
     def writelines(self, lines) -> None:
-        self.target.writelines(lines)
+        self.target.writelines(str(redact(line)) for line in lines)

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/api/datamodels/_generated.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  http://0.0.0.0:8080/execution/openapi.json
-#   version:   0.85.0
+#   version:   0.86.0
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -239,10 +239,14 @@ class LoopContext(BaseModel):
     """
 
     node_id: Annotated[str, Field(title="Node Id")]
-    index: Annotated[int, Field(ge=0, title="Index")]
+    index: Annotated[int, Field(ge=1, title="Index")]
     max_iterations: Annotated[int, Field(gt=0, title="Max Iterations")]
     terminal_task_id: Annotated[str, Field(title="Terminal Task Id")]
     terminal_is_mapped: Annotated[bool, Field(title="Terminal Is Mapped")]
+    region_id: Annotated[UUID, Field(title="Region Id")]
+    trajectory: Annotated[int, Field(ge=1, title="Trajectory")]
+    forked_from_region_id: Annotated[UUID | None, Field(title="Forked From Region Id")] = None
+    resumes_from_index: Annotated[int | None, Field(title="Resumes From Index")] = None
 
 
 class LoopDecisionPayload(BaseModel):
@@ -869,6 +873,7 @@ class TIRunContext(BaseModel):
 
     dag_run: DagRun
     loop: LoopContext | None = None
+    expanded_ti_count: Annotated[int | None, Field(title="Expanded Ti Count")] = None
     task_reschedule_count: Annotated[int | None, Field(title="Task Reschedule Count")] = 0
     first_task_reschedule_start_date: Annotated[
         AwareDatetime | None, Field(title="First Task Reschedule Start Date")

{apache_airflow_task_sdk-1.4.0b2 → apache_airflow_task_sdk-1.4.0rc2}/src/airflow/sdk/bases/resumablejobmixin.py

@@ -122,13 +122,8 @@ class ResumableJobMixin(ABC):
             self.poll_until_complete(external_id, context)
             return self.get_job_result(external_id, context)
 
-        stats_tags = {"operator": type(self).__name__}
-        # The task is team-scoped in multi-team deployments; surface team_name on the
-        # resumable_job metrics via the running task instance's stats tags (omitted when
-        # not multi-team or the task has no team).
         ti = context.get("ti")
-        if ti is not None and (team_name := ti.stats_tags.get("team_name")):
-            stats_tags["team_name"] = team_name
+        stats_tags = {**(ti.stats_tags if ti is not None else {}), "operator": type(self).__name__}
 
         reconnect_to: Any = None
         already_succeeded_id: Any = None