apache-airflow-providers-standard 1.9.0__py3-none-any.whl → 1.9.2rc1__py3-none-any.whl

airflow/providers/standard/triggers/hitl.py

@@ -30,6 +30,9 @@ from uuid import UUID
 
 from asgiref.sync import sync_to_async
 
+from airflow.exceptions import ParamValidationError
+from airflow.sdk import Param
+from airflow.sdk.definitions.param import ParamsDict
 from airflow.sdk.execution_time.hitl import (
     HITLUser,
     get_hitl_detail_content_detail,
@@ -43,7 +46,7 @@ class HITLTriggerEventSuccessPayload(TypedDict, total=False):
     """Minimum required keys for a success Human-in-the-loop TriggerEvent."""
 
     chosen_options: list[str]
-    params_input: dict[str, Any]
+    params_input: dict[str, dict[str, Any]]
     responded_by_user: HITLUser | None
     responded_at: datetime
     timedout: bool
@@ -53,7 +56,7 @@ class HITLTriggerEventFailurePayload(TypedDict):
     """Minimum required keys for a failed Human-in-the-loop TriggerEvent."""
 
     error: str
-    error_type: Literal["timeout", "unknown"]
+    error_type: Literal["timeout", "unknown", "validation"]
 
 
 class HITLTrigger(BaseTrigger):
@@ -64,7 +67,7 @@ class HITLTrigger(BaseTrigger):
         *,
         ti_id: UUID,
         options: list[str],
-        params: dict[str, Any],
+        params: dict[str, dict[str, Any]],
         defaults: list[str] | None = None,
         multiple: bool = False,
         timeout_datetime: datetime | None,
@@ -80,7 +83,21 @@ class HITLTrigger(BaseTrigger):
         self.defaults = defaults
         self.timeout_datetime = timeout_datetime
 
-        self.params = params
+        self.params = ParamsDict(
+            {
+                k: Param(
+                    v.pop("value"),
+                    **v,
+                )
+                if HITLTrigger._is_param(v)
+                else Param(v)
+                for k, v in params.items()
+            },
+        )
+
+    @staticmethod
+    def _is_param(value: Any) -> bool:
+        return isinstance(value, dict) and all(key in value for key in ("description", "schema", "value"))
 
     def serialize(self) -> tuple[str, dict[str, Any]]:
         """Serialize HITLTrigger arguments and classpath."""
@@ -90,99 +107,131 @@ class HITLTrigger(BaseTrigger):
                 "ti_id": self.ti_id,
                 "options": self.options,
                 "defaults": self.defaults,
-                "params": self.params,
+                "params": {k: self.params.get_param(k).serialize() for k in self.params},
                 "multiple": self.multiple,
                 "timeout_datetime": self.timeout_datetime,
                 "poke_interval": self.poke_interval,
             },
         )
 
-    async def run(self) -> AsyncIterator[TriggerEvent]:
-        """Loop until the Human-in-the-loop response received or timeout reached."""
-        while True:
-            if self.timeout_datetime and self.timeout_datetime < utcnow():
-                # Fetch latest HITL detail before fallback
-                resp = await sync_to_async(get_hitl_detail_content_detail)(ti_id=self.ti_id)
-                # Response already received, yield success and exit
-                if resp.response_received and resp.chosen_options:
-                    if TYPE_CHECKING:
-                        assert resp.responded_by_user is not None
-                        assert resp.responded_at is not None
-
-                    self.log.info(
-                        "[HITL] responded_by=%s (id=%s) options=%s at %s (timeout fallback skipped)",
-                        resp.responded_by_user.name,
-                        resp.responded_by_user.id,
-                        resp.chosen_options,
-                        resp.responded_at,
-                    )
-                    yield TriggerEvent(
-                        HITLTriggerEventSuccessPayload(
-                            chosen_options=resp.chosen_options,
-                            params_input=resp.params_input or {},
-                            responded_at=resp.responded_at,
-                            responded_by_user=HITLUser(
-                                id=resp.responded_by_user.id,
-                                name=resp.responded_by_user.name,
-                            ),
-                            timedout=False,
-                        )
-                    )
-                    return
-
-                if self.defaults is None:
-                    yield TriggerEvent(
-                        HITLTriggerEventFailurePayload(
-                            error="The timeout has passed, and the response has not yet been received.",
-                            error_type="timeout",
-                        )
-                    )
-                    return
-
-                resp = await sync_to_async(update_hitl_detail_response)(
-                    ti_id=self.ti_id,
-                    chosen_options=self.defaults,
-                    params_input=self.params,
+    async def _handle_timeout(self) -> TriggerEvent:
+        """Handle HITL timeout logic and yield appropriate event."""
+        resp = await sync_to_async(get_hitl_detail_content_detail)(ti_id=self.ti_id)
+
+        # Case 1: Response arrived just before timeout
+        if resp.response_received and resp.chosen_options:
+            if TYPE_CHECKING:
+                assert resp.responded_by_user is not None
+                assert resp.responded_at is not None
+
+            chosen_options_list = list(resp.chosen_options or [])
+            self.log.info(
+                "[HITL] responded_by=%s (id=%s) options=%s at %s (timeout fallback skipped)",
+                resp.responded_by_user.name,
+                resp.responded_by_user.id,
+                chosen_options_list,
+                resp.responded_at,
+            )
+            return TriggerEvent(
+                HITLTriggerEventSuccessPayload(
+                    chosen_options=chosen_options_list,
+                    params_input=resp.params_input or {},
+                    responded_at=resp.responded_at,
+                    responded_by_user=HITLUser(
+                        id=resp.responded_by_user.id,
+                        name=resp.responded_by_user.name,
+                    ),
+                    timedout=False,
                 )
-                if TYPE_CHECKING:
-                    assert resp.responded_at is not None
-                self.log.info(
-                    "[HITL] timeout reached before receiving response, fallback to default %s", self.defaults
+            )
+
+        # Case 2: No defaults defined → failure
+        if self.defaults is None:
+            return TriggerEvent(
+                HITLTriggerEventFailurePayload(
+                    error="The timeout has passed, and the response has not yet been received.",
+                    error_type="timeout",
                 )
-                yield TriggerEvent(
-                    HITLTriggerEventSuccessPayload(
-                        chosen_options=self.defaults,
-                        params_input=self.params,
-                        responded_by_user=None,
-                        responded_at=resp.responded_at,
-                        timedout=True,
+            )
+
+        # Case 3: Timeout fallback to default
+        resp = await sync_to_async(update_hitl_detail_response)(
+            ti_id=self.ti_id,
+            chosen_options=self.defaults,
+            params_input=self.params.dump(),
+        )
+        if TYPE_CHECKING:
+            assert resp.responded_at is not None
+
+        self.log.info(
+            "[HITL] timeout reached before receiving response, fallback to default %s",
+            self.defaults,
+        )
+        return TriggerEvent(
+            HITLTriggerEventSuccessPayload(
+                chosen_options=self.defaults,
+                params_input=self.params.dump(),
+                responded_by_user=None,
+                responded_at=resp.responded_at,
+                timedout=True,
+            )
+        )
+
+    async def _handle_response(self):
+        """Check if HITL response is ready and yield success if so."""
+        resp = await sync_to_async(get_hitl_detail_content_detail)(ti_id=self.ti_id)
+        if TYPE_CHECKING:
+            assert resp.responded_by_user is not None
+            assert resp.responded_at is not None
+
+        if not (resp.response_received and resp.chosen_options):
+            return None
+
+        # validate input
+        if params_input := resp.params_input:
+            try:
+                for key, value in params_input.items():
+                    self.params[key] = value
+            except ParamValidationError as err:
+                return TriggerEvent(
+                    HITLTriggerEventFailurePayload(
+                        error=str(err),
+                        error_type="validation",
                     )
                 )
+
+        chosen_options_list = list(resp.chosen_options or [])
+        self.log.info(
+            "[HITL] responded_by=%s (id=%s) options=%s at %s",
+            resp.responded_by_user.name,
+            resp.responded_by_user.id,
+            chosen_options_list,
+            resp.responded_at,
+        )
+        return TriggerEvent(
+            HITLTriggerEventSuccessPayload(
+                chosen_options=chosen_options_list,
+                params_input=params_input or {},
+                responded_at=resp.responded_at,
+                responded_by_user=HITLUser(
+                    id=resp.responded_by_user.id,
+                    name=resp.responded_by_user.name,
+                ),
+                timedout=False,
+            )
+        )
+
+    async def run(self) -> AsyncIterator[TriggerEvent]:
+        """Loop until the Human-in-the-loop response received or timeout reached."""
+        while True:
+            if self.timeout_datetime and self.timeout_datetime < utcnow():
+                event = await self._handle_timeout()
+                yield event
                 return
 
-            resp = await sync_to_async(get_hitl_detail_content_detail)(ti_id=self.ti_id)
-            if resp.response_received and resp.chosen_options:
-                if TYPE_CHECKING:
-                    assert resp.responded_by_user is not None
-                    assert resp.responded_at is not None
-                self.log.info(
-                    "[HITL] responded_by=%s (id=%s) options=%s at %s",
-                    resp.responded_by_user.name,
-                    resp.responded_by_user.id,
-                    resp.chosen_options,
-                    resp.responded_at,
-                )
-                yield TriggerEvent(
-                    HITLTriggerEventSuccessPayload(
-                        chosen_options=resp.chosen_options,
-                        params_input=resp.params_input or {},
-                        responded_at=resp.responded_at,
-                        responded_by_user=HITLUser(
-                            id=resp.responded_by_user.id,
-                            name=resp.responded_by_user.name,
-                        ),
-                        timedout=False,
-                    )
-                )
+            event = await self._handle_response()
+            if event:
+                yield event
                 return
+
             await asyncio.sleep(self.poke_interval)

airflow/providers/standard/triggers/temporal.py

@@ -23,13 +23,9 @@ from typing import Any
 
 import pendulum
 
+from airflow.providers.common.compat.sdk import timezone
 from airflow.triggers.base import BaseTrigger, TaskSuccessEvent, TriggerEvent
 
-try:
-    from airflow.sdk import timezone
-except ImportError:
-    from airflow.utils import timezone  # type: ignore[attr-defined,no-redef]
-
 
 class DateTimeTrigger(BaseTrigger):
     """

airflow/providers/standard/utils/python_virtualenv.py

@@ -19,8 +19,11 @@
 
 from __future__ import annotations
 
+import logging
 import os
+import shlex
 import shutil
+import subprocess
 import warnings
 from pathlib import Path
 
@@ -28,7 +31,6 @@ import jinja2
 from jinja2 import select_autoescape
 
 from airflow.configuration import conf
-from airflow.utils.process_utils import execute_in_subprocess
 
 
 def _is_uv_installed() -> bool:
@@ -132,6 +134,37 @@ def _index_urls_to_uv_env_vars(index_urls: list[str] | None = None) -> dict[str,
     return uv_index_env_vars
 
 
+def _execute_in_subprocess(cmd: list[str], cwd: str | None = None, env: dict[str, str] | None = None) -> None:
+    """
+    Execute a process and stream output to logger.
+
+    :param cmd: command and arguments to run
+    :param cwd: Current working directory passed to the Popen constructor
+    :param env: Additional environment variables to set for the subprocess.
+    """
+    log = logging.getLogger(__name__)
+
+    log.info("Executing cmd: %s", " ".join(shlex.quote(c) for c in cmd))
+    with subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=0,
+        close_fds=True,
+        cwd=cwd,
+        env=env,
+    ) as proc:
+        log.info("Output:")
+        if proc.stdout:
+            with proc.stdout:
+                for line in iter(proc.stdout.readline, b""):
+                    log.info("%s", line.decode().rstrip())
+
+        exit_code = proc.wait()
+    if exit_code != 0:
+        raise subprocess.CalledProcessError(exit_code, cmd)
+
+
 def prepare_virtualenv(
     venv_directory: str,
     python_bin: str,
@@ -169,7 +202,7 @@ def prepare_virtualenv(
         venv_cmd = _generate_uv_cmd(venv_directory, python_bin, system_site_packages)
     else:
         venv_cmd = _generate_venv_cmd(venv_directory, python_bin, system_site_packages)
-    execute_in_subprocess(venv_cmd)
+    _execute_in_subprocess(venv_cmd)
 
     pip_cmd = None
     if requirements is not None and len(requirements) != 0:
@@ -188,7 +221,7 @@ def prepare_virtualenv(
             )
 
     if pip_cmd:
-        execute_in_subprocess(pip_cmd, env={**os.environ, **_index_urls_to_uv_env_vars(index_urls)})
+        _execute_in_subprocess(pip_cmd, env={**os.environ, **_index_urls_to_uv_env_vars(index_urls)})
 
     return f"{venv_directory}/bin/python"
 

airflow/providers/standard/utils/sensor_helper.py

@@ -16,6 +16,7 @@
 # under the License.
 from __future__ import annotations
 
+from collections.abc import Collection
 from typing import TYPE_CHECKING, Any, cast
 
 from sqlalchemy import func, select, tuple_
@@ -27,7 +28,7 @@ from airflow.utils.session import NEW_SESSION, provide_session
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
-    from sqlalchemy.sql import Executable
+    from sqlalchemy.sql import Select
 
 
 @provide_session
@@ -59,6 +60,7 @@ def _get_count(
             session.scalar(
                 _count_stmt(TI, states, dttm_filter, external_dag_id).where(TI.task_id.in_(external_task_ids))
             )
+            or 0
         ) / len(external_task_ids)
     elif external_task_group_id:
         external_task_group_task_ids = _get_external_task_group_task_ids(
@@ -68,20 +70,25 @@ def _get_count(
             count = 0
         else:
             count = (
-                session.scalar(
-                    _count_stmt(TI, states, dttm_filter, external_dag_id).where(
-                        tuple_(TI.task_id, TI.map_index).in_(external_task_group_task_ids)
+                (
+                    session.scalar(
+                        _count_stmt(TI, states, dttm_filter, external_dag_id).where(
+                            tuple_(TI.task_id, TI.map_index).in_(external_task_group_task_ids)
+                        )
                     )
+                    or 0
                 )
                 / len(external_task_group_task_ids)
                 * len(dttm_filter)
             )
     else:
-        count = session.scalar(_count_stmt(DR, states, dttm_filter, external_dag_id))
+        count = session.scalar(_count_stmt(DR, states, dttm_filter, external_dag_id)) or 0
     return cast("int", count)
 
 
-def _count_stmt(model, states, dttm_filter, external_dag_id) -> Executable:
+def _count_stmt(
+    model: type[DagRun] | type[TaskInstance], states: list[str], dttm_filter: list[Any], external_dag_id: str
+) -> Select[tuple[int]]:
     """
     Get the count of records against dttm filter and states.
 
@@ -97,7 +104,9 @@ def _count_stmt(model, states, dttm_filter, external_dag_id) -> Executable:
     )
 
 
-def _get_external_task_group_task_ids(dttm_filter, external_task_group_id, external_dag_id, session):
+def _get_external_task_group_task_ids(
+    dttm_filter: list[Any], external_task_group_id: str, external_dag_id: str, session: Session
+) -> list[tuple[str, int]]:
     """
     Get the count of records against dttm filter and states.
 
@@ -107,6 +116,8 @@ def _get_external_task_group_task_ids(dttm_filter, external_task_group_id, exter
     :param session: airflow session object
     """
     refreshed_dag_info = SerializedDagModel.get_dag(external_dag_id, session=session)
+    if not refreshed_dag_info:
+        return [(external_task_group_id, -1)]
     task_group = refreshed_dag_info.task_group_dict.get(external_task_group_id)
 
     if task_group:
@@ -129,7 +140,7 @@ def _get_external_task_group_task_ids(dttm_filter, external_task_group_id, exter
 
 def _get_count_by_matched_states(
     run_id_task_state_map: dict[str, dict[str, Any]],
-    states: list[str],
+    states: Collection[str],
 ):
     count = 0
     for _, task_states in run_id_task_state_map.items():

airflow/providers/standard/utils/skipmixin.py

@@ -22,7 +22,6 @@ from types import GeneratorType
 from typing import TYPE_CHECKING
 
 from airflow.exceptions import AirflowException
-from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
@@ -40,12 +39,7 @@ XCOM_SKIPMIXIN_FOLLOWED = "followed"
 
 
 def _ensure_tasks(nodes: Iterable[DAGNode]) -> Sequence[Operator]:
-    if AIRFLOW_V_3_0_PLUS:
-        from airflow.sdk import BaseOperator
-        from airflow.sdk.definitions.mappedoperator import MappedOperator
-    else:
-        from airflow.models.baseoperator import BaseOperator  # type: ignore[no-redef]
-        from airflow.models.mappedoperator import MappedOperator  # type: ignore[assignment,no-redef]
+    from airflow.providers.common.compat.sdk import BaseOperator, MappedOperator
 
     return [n for n in nodes if isinstance(n, (BaseOperator, MappedOperator))]
 

airflow/providers/standard/version_compat.py

@@ -34,36 +34,19 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
 
 AIRFLOW_V_3_0_PLUS: bool = get_base_airflow_version_tuple() >= (3, 0, 0)
 AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0)
+AIRFLOW_V_3_1_3_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 3)
 AIRFLOW_V_3_2_PLUS: bool = get_base_airflow_version_tuple() >= (3, 2, 0)
 
-# BaseOperator is not imported from SDK from 3.0 (and only done from 3.1) due to a bug with
-# DecoratedOperator -- where `DecoratedOperator._handle_output` needed `xcom_push` to exist on `BaseOperator`
-# even though it wasn't used.
+# BaseOperator: Use 3.1+ due to xcom_push method missing in SDK BaseOperator 3.0.x
+# This is needed for DecoratedOperator compatibility
 if AIRFLOW_V_3_1_PLUS:
-    from airflow.sdk import BaseHook, BaseOperator, timezone
-    from airflow.sdk.definitions.context import context_merge
+    from airflow.sdk import BaseOperator
 else:
-    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
     from airflow.models.baseoperator import BaseOperator  # type: ignore[no-redef]
-    from airflow.utils import timezone  # type: ignore[attr-defined,no-redef]
-    from airflow.utils.context import context_merge  # type: ignore[no-redef, attr-defined]
-
-if AIRFLOW_V_3_0_PLUS:
-    from airflow.sdk import BaseOperatorLink
-    from airflow.sdk.bases.sensor import BaseSensorOperator, PokeReturnValue
-else:
-    from airflow.models.baseoperatorlink import BaseOperatorLink  # type: ignore[no-redef]
-    from airflow.sensors.base import BaseSensorOperator, PokeReturnValue  # type: ignore[no-redef]
 
 __all__ = [
     "AIRFLOW_V_3_0_PLUS",
     "AIRFLOW_V_3_1_PLUS",
     "AIRFLOW_V_3_2_PLUS",
     "BaseOperator",
-    "BaseOperatorLink",
-    "BaseHook",
-    "BaseSensorOperator",
-    "PokeReturnValue",
-    "context_merge",
-    "timezone",
 ]

{apache_airflow_providers_standard-1.9.0.dist-info → apache_airflow_providers_standard-1.9.2rc1.dist-info}/METADATA

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-standard
-Version: 1.9.0
+Version: 1.9.2rc1
 Summary: Provider package apache-airflow-providers-standard for Apache Airflow
 Keywords: airflow-provider,standard,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
 Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
+License-Expression: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Environment :: Web Environment
@@ -14,16 +15,18 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.10.0
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: apache-airflow>=2.10.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.8.0rc1
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-standard/1.9.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-standard/1.9.0
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-standard/1.9.2/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-standard/1.9.2
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -54,7 +57,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-standard``
 
-Release: ``1.9.0``
+Release: ``1.9.2``
 
 
 Airflow Standard Provider
@@ -67,7 +70,7 @@ This is a provider package for ``standard`` provider. All classes for this provi
 are in ``airflow.providers.standard`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.9.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.9.2/>`_.
 
 Installation
 ------------
@@ -81,12 +84,32 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13
 Requirements
 ------------
 
-==================  ==================
-PIP package         Version required
-==================  ==================
-``apache-airflow``  ``>=2.10.0``
-==================  ==================
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.10.0``
+``apache-airflow-providers-common-compat``  ``>=1.8.0``
+==========================================  ==================
+
+Cross provider package dependencies
+-----------------------------------
+
+Those are dependencies that might be needed in order to use all the features of the package.
+You need to install the specified providers in order to use them.
+
+You can install such cross-provider dependencies when installing from PyPI. For example:
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-standard[common.compat]
+
+
+==================================================================================================================  =================
+Dependent package                                                                                                   Extra
+==================================================================================================================  =================
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
+==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.9.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.9.2/changelog.html>`_.