apache-airflow-providers-standard 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

airflow/providers/standard/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "0.0.2"
+__version__ = "0.0.3"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.8.0"
+    "2.9.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-standard:{__version__}` needs Apache Airflow 2.8.0+"
+        f"The package `apache-airflow-providers-standard:{__version__}` needs Apache Airflow 2.9.0+"
     )

airflow/providers/standard/get_provider_info.py

@@ -28,9 +28,9 @@ def get_provider_info():
         "name": "Standard",
         "description": "Airflow Standard Provider\n",
         "state": "ready",
-        "source-date-epoch": 1732434919,
-        "versions": ["0.0.2", "0.0.1"],
-        "dependencies": ["apache-airflow>=2.8.0", "apache-airflow-providers-common-sql>=1.20.0"],
+        "source-date-epoch": 1734536895,
+        "versions": ["0.0.3", "0.0.2", "0.0.1"],
+        "dependencies": ["apache-airflow>=2.9.0", "apache-airflow-providers-common-sql>=1.20.0"],
         "integrations": [
             {
                 "integration-name": "Standard",
@@ -53,6 +53,7 @@ def get_provider_info():
                     "airflow.providers.standard.operators.python",
                     "airflow.providers.standard.operators.generic_transfer",
                     "airflow.providers.standard.operators.trigger_dagrun",
+                    "airflow.providers.standard.operators.latest_only",
                 ],
             }
         ],
@@ -67,6 +68,7 @@ def get_provider_info():
                     "airflow.providers.standard.sensors.bash",
                     "airflow.providers.standard.sensors.python",
                     "airflow.providers.standard.sensors.filesystem",
+                    "airflow.providers.standard.sensors.external_task",
                 ],
             }
         ],
@@ -80,6 +82,16 @@ def get_provider_info():
                 ],
             }
         ],
+        "triggers": [
+            {
+                "integration-name": "Standard",
+                "python-modules": [
+                    "airflow.providers.standard.triggers.external_task",
+                    "airflow.providers.standard.triggers.file",
+                    "airflow.providers.standard.triggers.temporal",
+                ],
+            }
+        ],
         "config": {
             "standard": {
                 "description": "Options for the standard provider operators.",

airflow/providers/standard/operators/bash.py

@@ -20,7 +20,6 @@ from __future__ import annotations
 import os
 import shutil
 import tempfile
-import warnings
 from collections.abc import Container, Sequence
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Callable, cast
@@ -29,9 +28,12 @@ from airflow.exceptions import AirflowException, AirflowSkipException
 from airflow.models.baseoperator import BaseOperator
 from airflow.providers.standard.hooks.subprocess import SubprocessHook, SubprocessResult, working_directory
 from airflow.utils.operator_helpers import context_to_airflow_vars
+from airflow.utils.session import NEW_SESSION, provide_session
 from airflow.utils.types import ArgNotSet
 
 if TYPE_CHECKING:
+    from sqlalchemy.orm import Session as SASession
+
     from airflow.models.taskinstance import TaskInstance
     from airflow.utils.context import Context
 
@@ -158,7 +160,6 @@ class BashOperator(BaseOperator):
         env: dict[str, str] | None = None,
         append_env: bool = False,
         output_encoding: str = "utf-8",
-        skip_exit_code: int | None = None,
         skip_on_exit_code: int | Container[int] | None = 99,
         cwd: str | None = None,
         output_processor: Callable[[str], Any] = lambda result: result,
@@ -168,11 +169,6 @@ class BashOperator(BaseOperator):
         self.bash_command = bash_command
         self.env = env
         self.output_encoding = output_encoding
-        if skip_exit_code is not None:
-            warnings.warn(
-                "skip_exit_code is deprecated. Please use skip_on_exit_code", DeprecationWarning, stacklevel=2
-            )
-            skip_on_exit_code = skip_exit_code
         self.skip_on_exit_code = (
             skip_on_exit_code
             if isinstance(skip_on_exit_code, Container)
@@ -199,8 +195,10 @@ class BashOperator(BaseOperator):
         """Returns hook for running the bash command."""
         return SubprocessHook()
 
+    # TODO: This should be replaced with Task SDK API call
     @staticmethod
-    def refresh_bash_command(ti: TaskInstance) -> None:
+    @provide_session
+    def refresh_bash_command(ti: TaskInstance, session: SASession = NEW_SESSION) -> None:
         """
         Rewrite the underlying rendered bash_command value for a task instance in the metadatabase.
 
@@ -212,7 +210,16 @@ class BashOperator(BaseOperator):
         """
         from airflow.models.renderedtifields import RenderedTaskInstanceFields
 
-        RenderedTaskInstanceFields._update_runtime_evaluated_template_fields(ti)
+        """Update rendered task instance fields for cases where runtime evaluated, not templated."""
+        # Note: Need lazy import to break the partly loaded class loop
+        from airflow.models.taskinstance import TaskInstance
+
+        # If called via remote API the DAG needs to be re-loaded
+        TaskInstance.ensure_dag(ti, session=session)
+
+        rtif = RenderedTaskInstanceFields(ti)
+        RenderedTaskInstanceFields.write(rtif, session=session)
+        RenderedTaskInstanceFields.delete_old_records(ti.task_id, ti.dag_id, session=session)
 
     def get_env(self, context) -> dict:
         """Build the set of environment variables to be exposed for the bash command."""
@@ -249,7 +256,7 @@ class BashOperator(BaseOperator):
         # displays the executed command (otherwise it will display as an ArgNotSet type).
         if self._init_bash_command_not_set:
             is_inline_command = self._is_inline_command(bash_command=cast(str, self.bash_command))
-            ti = cast("TaskInstance", context["ti"])
+            ti = context["ti"]
             self.refresh_bash_command(ti)
         else:
             is_inline_command = self._is_inline_command(bash_command=cast(str, self._unrendered_bash_command))

airflow/providers/standard/operators/latest_only.py

@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Contains an operator to run downstream tasks only for the latest scheduled DagRun."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
+
+import pendulum
+
+from airflow.operators.branch import BaseBranchOperator
+
+if TYPE_CHECKING:
+    from airflow.models import DAG, DagRun
+    from airflow.utils.context import Context
+
+
+class LatestOnlyOperator(BaseBranchOperator):
+    """
+    Skip tasks that are not running during the most recent schedule interval.
+
+    If the task is run outside the latest schedule interval (i.e. external_trigger),
+    all directly downstream tasks will be skipped.
+
+    Note that downstream tasks are never skipped if the given DAG_Run is
+    marked as externally triggered.
+    """
+
+    ui_color = "#e9ffdb"  # nyanza
+
+    def choose_branch(self, context: Context) -> str | Iterable[str]:
+        # If the DAG Run is externally triggered, then return without
+        # skipping downstream tasks
+        dag_run: DagRun = context["dag_run"]  # type: ignore[assignment]
+        if dag_run.external_trigger:
+            self.log.info("Externally triggered DAG_Run: allowing execution to proceed.")
+            return list(context["task"].get_direct_relative_ids(upstream=False))
+
+        dag: DAG = context["dag"]
+        next_info = dag.next_dagrun_info(dag.get_run_data_interval(dag_run), restricted=False)
+        now = pendulum.now("UTC")
+
+        if next_info is None:
+            self.log.info("Last scheduled execution: allowing execution to proceed.")
+            return list(context["task"].get_direct_relative_ids(upstream=False))
+
+        left_window, right_window = next_info.data_interval
+        self.log.info(
+            "Checking latest only with left_window: %s right_window: %s now: %s",
+            left_window,
+            right_window,
+            now,
+        )
+
+        if not left_window < now <= right_window:
+            self.log.info("Not latest execution, skipping downstream.")
+            # we return an empty list, thus the parent BaseBranchOperator
+            # won't exclude any downstream tasks from skipping.
+            return []
+        else:
+            self.log.info("Latest, allowing execution to proceed.")
+            return list(context["task"].get_direct_relative_ids(upstream=False))

airflow/providers/standard/operators/python.py

@@ -26,7 +26,6 @@ import subprocess
 import sys
 import textwrap
 import types
-import warnings
 from abc import ABCMeta, abstractmethod
 from collections.abc import Collection, Container, Iterable, Mapping, Sequence
 from functools import cache
@@ -41,7 +40,6 @@ from airflow.exceptions import (
     AirflowException,
     AirflowSkipException,
     DeserializingResultError,
-    RemovedInAirflow3Warning,
 )
 from airflow.models.baseoperator import BaseOperator
 from airflow.models.skipmixin import SkipMixin
@@ -49,22 +47,22 @@ from airflow.models.taskinstance import _CURRENT_CONTEXT
 from airflow.models.variable import Variable
 from airflow.operators.branch import BranchMixIn
 from airflow.providers.standard.utils.python_virtualenv import prepare_virtualenv, write_python_script
-from airflow.providers.standard.utils.version_references import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS
-from airflow.settings import _ENABLE_AIP_44
+from airflow.providers.standard.version_compat import (
+    AIRFLOW_V_2_10_PLUS,
+    AIRFLOW_V_3_0_PLUS,
+)
 from airflow.typing_compat import Literal
 from airflow.utils import hashlib_wrapper
 from airflow.utils.context import context_copy_partial, context_merge
 from airflow.utils.file import get_unique_dag_module_name
 from airflow.utils.operator_helpers import KeywordParameters
 from airflow.utils.process_utils import execute_in_subprocess, execute_in_subprocess_with_kwargs
-from airflow.utils.session import create_session
 
 log = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
     from pendulum.datetime import DateTime
 
-    from airflow.serialization.enums import Encoding
     from airflow.utils.context import Context
 
 
@@ -417,7 +415,6 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
         skip_on_exit_code: int | Container[int] | None = None,
         env_vars: dict[str, str] | None = None,
         inherit_env: bool = True,
-        use_dill: bool = False,
         use_airflow_context: bool = False,
         **kwargs,
     ):
@@ -439,18 +436,6 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
         )
         self.string_args = string_args or []
 
-        if use_dill:
-            warnings.warn(
-                "`use_dill` is deprecated and will be removed in a future version. "
-                "Please provide serializer='dill' instead.",
-                RemovedInAirflow3Warning,
-                stacklevel=3,
-            )
-            if serializer:
-                raise AirflowException(
-                    "Both 'use_dill' and 'serializer' parameters are set. Please set only one of them"
-                )
-            serializer = "dill"
         serializer = serializer or "pickle"
         if serializer not in _SERIALIZERS:
             msg = (
@@ -524,10 +509,6 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
             self._write_args(input_path)
             self._write_string_args(string_args_path)
 
-            if self.use_airflow_context and not _ENABLE_AIP_44:
-                error_msg = "`get_current_context()` needs to be used with AIP-44 enabled."
-                raise AirflowException(error_msg)
-
             jinja_context = {
                 "op_args": self.op_args,
                 "op_kwargs": op_kwargs,
@@ -547,18 +528,19 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
                 render_template_as_native_obj=self.dag.render_template_as_native_obj,
             )
             if self.use_airflow_context:
-                from airflow.serialization.serialized_objects import BaseSerialization
-
-                context = get_current_context()
-                with create_session() as session:
-                    # FIXME: DetachedInstanceError
-                    dag_run, task_instance = context["dag_run"], context["task_instance"]
-                    session.add_all([dag_run, task_instance])
-                    serializable_context: dict[Encoding, Any] = BaseSerialization.serialize(
-                        context, use_pydantic_models=True
-                    )
-                with airflow_context_path.open("w+") as file:
-                    json.dump(serializable_context, file)
+                # TODO: replace with commented code when context serialization is implemented in AIP-72
+                raise AirflowException(
+                    "The `use_airflow_context=True` is not yet implemented. "
+                    "It will work in Airflow 3 after AIP-72 context "
+                    "serialization is ready."
+                )
+                # context = get_current_context()
+                # with create_session() as session:
+                #     dag_run, task_instance = context["dag_run"], context["task_instance"]
+                #     session.add_all([dag_run, task_instance])
+                #     serializable_context: dict[Encoding, Any] = # Get serializable context here
+                # with airflow_context_path.open("w+") as file:
+                #     json.dump(serializable_context, file)
 
             env_vars = dict(os.environ) if self.inherit_env else {}
             if self.env_vars:
@@ -669,10 +651,8 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
         environment. If set to ``True``, the virtual environment will inherit the environment variables
         of the parent process (``os.environ``). If set to ``False``, the virtual environment will be
         executed with a clean environment.
-    :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
-        the args and result (pickle is default). This allows more complex types
-        but requires you to include dill in your requirements.
     :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
+        NOT YET IMPLEMENTED - waits for AIP-72 context serialization.
     """
 
     template_fields: Sequence[str] = tuple(
@@ -700,7 +680,6 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
         venv_cache_path: None | os.PathLike[str] = None,
         env_vars: dict[str, str] | None = None,
         inherit_env: bool = True,
-        use_dill: bool = False,
         use_airflow_context: bool = False,
         **kwargs,
     ):
@@ -718,15 +697,18 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
             raise AirflowException(
                 "Passing non-string types (e.g. int or float) as python_version not supported"
             )
-        if use_airflow_context and not AIRFLOW_V_3_0_PLUS:
-            raise AirflowException(
-                "The `use_airflow_context=True` is only supported in Airflow 3.0.0 and later."
-            )
         if use_airflow_context and (not expect_airflow and not system_site_packages):
             raise AirflowException(
                 "The `use_airflow_context` parameter is set to True, but "
                 "expect_airflow and system_site_packages are set to False."
             )
+        # TODO: remove when context serialization is implemented in AIP-72
+        if use_airflow_context and not AIRFLOW_V_3_0_PLUS:
+            raise AirflowException(
+                "The `use_airflow_context=True` is not yet implemented. "
+                "It will work in Airflow 3 after AIP-72 context "
+                "serialization is ready."
+            )
         if not requirements:
             self.requirements: list[str] = []
         elif isinstance(requirements, str):
@@ -755,7 +737,6 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
             skip_on_exit_code=skip_on_exit_code,
             env_vars=env_vars,
             inherit_env=inherit_env,
-            use_dill=use_dill,
             use_airflow_context=use_airflow_context,
             **kwargs,
         )
@@ -972,10 +953,8 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
         environment. If set to ``True``, the virtual environment will inherit the environment variables
         of the parent process (``os.environ``). If set to ``False``, the virtual environment will be
         executed with a clean environment.
-    :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
-        the args and result (pickle is default). This allows more complex types
-        but requires you to include dill in your requirements.
     :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
+        NOT YET IMPLEMENTED - waits for AIP-72 context serialization.
     """
 
     template_fields: Sequence[str] = tuple({"python"}.union(PythonOperator.template_fields))
@@ -996,20 +975,22 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
         skip_on_exit_code: int | Container[int] | None = None,
         env_vars: dict[str, str] | None = None,
         inherit_env: bool = True,
-        use_dill: bool = False,
         use_airflow_context: bool = False,
         **kwargs,
     ):
         if not python:
             raise ValueError("Python Path must be defined in ExternalPythonOperator")
-        if use_airflow_context and not AIRFLOW_V_3_0_PLUS:
-            raise AirflowException(
-                "The `use_airflow_context=True` is only supported in Airflow 3.0.0 and later."
-            )
         if use_airflow_context and not expect_airflow:
             raise AirflowException(
                 "The `use_airflow_context` parameter is set to True, but expect_airflow is set to False."
             )
+        # TODO: remove when context serialization is implemented in AIP-72
+        if use_airflow_context:
+            raise AirflowException(
+                "The `use_airflow_context=True` is not yet implemented. "
+                "It will work in Airflow 3 after AIP-72 context "
+                "serialization is ready."
+            )
         self.python = python
         self.expect_pendulum = expect_pendulum
         super().__init__(
@@ -1024,7 +1005,6 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
             skip_on_exit_code=skip_on_exit_code,
             env_vars=env_vars,
             inherit_env=inherit_env,
-            use_dill=use_dill,
             use_airflow_context=use_airflow_context,
             **kwargs,
         )

airflow/providers/standard/operators/trigger_dagrun.py

@@ -27,7 +27,6 @@ from sqlalchemy import select
 from sqlalchemy.orm.exc import NoResultFound
 
 from airflow.api.common.trigger_dag import trigger_dag
-from airflow.api_internal.internal_api_call import InternalApiConfig
 from airflow.configuration import conf
 from airflow.exceptions import (
     AirflowException,
@@ -40,7 +39,7 @@ from airflow.models.dag import DagModel
 from airflow.models.dagbag import DagBag
 from airflow.models.dagrun import DagRun
 from airflow.models.xcom import XCom
-from airflow.triggers.external_task import DagStateTrigger
+from airflow.providers.standard.triggers.external_task import DagStateTrigger
 from airflow.utils import timezone
 from airflow.utils.helpers import build_airflow_url_with_query
 from airflow.utils.session import provide_session
@@ -182,14 +181,6 @@ class TriggerDagRunOperator(BaseOperator):
         self.logical_date = logical_date
 
     def execute(self, context: Context):
-        if InternalApiConfig.get_use_internal_api():
-            if self.reset_dag_run:
-                raise AirflowException("Parameter reset_dag_run=True is broken with Database Isolation Mode.")
-            if self.wait_for_completion:
-                raise AirflowException(
-                    "Parameter wait_for_completion=True is broken with Database Isolation Mode."
-                )
-
         if isinstance(self.logical_date, datetime.datetime):
             parsed_logical_date = self.logical_date
         elif isinstance(self.logical_date, str):

airflow/providers/standard/sensors/date_time.py

@@ -22,7 +22,8 @@ from collections.abc import Sequence
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, NoReturn
 
-from airflow.providers.standard.utils.version_references import AIRFLOW_V_3_0_PLUS
+from airflow.providers.standard.triggers.temporal import DateTimeTrigger
+from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.sensors.base import BaseSensorOperator
 
 try:
@@ -40,7 +41,6 @@ except ImportError:
         timeout: datetime.timedelta | None = None
 
 
-from airflow.triggers.temporal import DateTimeTrigger
 from airflow.utils import timezone
 
 if TYPE_CHECKING:
@@ -111,7 +111,7 @@ class DateTimeSensorAsync(DateTimeSensor):
     """
 
     start_trigger_args = StartTriggerArgs(
-        trigger_cls="airflow.triggers.temporal.DateTimeTrigger",
+        trigger_cls="airflow.providers.standard.triggers.temporal.DateTimeTrigger",
         trigger_kwargs={"moment": "", "end_from_trigger": False},
         next_method="execute_complete",
         next_kwargs=None,