apache-airflow-providers-standard 0.2.0rc1__tar.gz → 0.3.0rc1__tar.gz

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-standard
-Version: 0.2.0rc1
+Version: 0.3.0rc1
 Summary: Provider package apache-airflow-providers-standard for Apache Airflow
 Keywords: airflow-provider,standard,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow>=2.9.0rc0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -54,7 +54,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-standard``
 
-Release: ``0.2.0``
+Release: ``0.3.0``
 
 
 Airflow Standard Provider
@@ -67,7 +67,7 @@ This is a provider package for ``standard`` provider. All classes for this provi
 are in ``airflow.providers.standard`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0/>`_.
 
 Installation
 ------------
@@ -88,5 +88,5 @@ PIP package         Version required
 ==================  ==================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0/changelog.html>`_.
 

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-standard``
 
-Release: ``0.2.0``
+Release: ``0.3.0``
 
 
 Airflow Standard Provider
@@ -36,7 +36,7 @@ This is a provider package for ``standard`` provider. All classes for this provi
 are in ``airflow.providers.standard`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0/>`_.
 
 Installation
 ------------
@@ -57,4 +57,4 @@ PIP package         Version required
 ==================  ==================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0/changelog.html>`_.

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-standard"
-version = "0.2.0.rc1"
+version = "0.3.0.rc1"
 description = "Provider package apache-airflow-providers-standard for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -66,6 +66,7 @@ dev = [
     "apache-airflow-task-sdk",
     "apache-airflow-devel-common",
     # Additional devel dependencies (do not remove this line and add extra development dependencies)
+    "apache-airflow-providers-mysql",
 ]
 
 [tool.uv.sources]
@@ -79,8 +80,8 @@ apache-airflow-providers-fab = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-standard/0.2.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-standard/0.3.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/get_provider_info.py

@@ -27,8 +27,8 @@ def get_provider_info():
         "name": "Standard",
         "description": "Airflow Standard Provider\n",
         "state": "ready",
-        "source-date-epoch": 1742480519,
-        "versions": ["0.2.0", "0.1.1", "0.1.0", "0.0.3", "0.0.2", "0.0.1"],
+        "source-date-epoch": 1743477899,
+        "versions": ["0.3.0", "0.2.0", "0.1.1", "0.1.0", "0.0.3", "0.0.2", "0.0.1"],
         "integrations": [
             {
                 "integration-name": "Standard",

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/operators/bash.py

@@ -253,7 +253,7 @@ class BashOperator(BaseOperator):
         """
         with working_directory(cwd=self.cwd) as cwd:
             with tempfile.NamedTemporaryFile(mode="w", dir=cwd, suffix=".sh") as file:
-                file.write(cast(str, self.bash_command))
+                file.write(cast("str", self.bash_command))
                 file.flush()
 
                 bash_script = os.path.basename(file.name)

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/operators/datetime.py

@@ -77,9 +77,14 @@ class BranchDateTimeOperator(BaseBranchOperator):
 
     def choose_branch(self, context: Context) -> str | Iterable[str]:
         if self.use_task_logical_date:
-            now = context["logical_date"]
+            now = context.get("logical_date")
+            if not now:
+                dag_run = context.get("dag_run")
+                now = dag_run.run_after  # type: ignore[union-attr, assignment]
         else:
             now = timezone.coerce_datetime(timezone.utcnow())
+        if TYPE_CHECKING:
+            assert isinstance(now, datetime.datetime)
         lower, upper = target_times_as_dates(now, self.target_lower, self.target_upper)
         lower = timezone.coerce_datetime(lower, self.dag.timezone)
         upper = timezone.coerce_datetime(upper, self.dag.timezone)

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/operators/python.py

@@ -56,7 +56,7 @@ if AIRFLOW_V_3_0_PLUS:
     from airflow.providers.standard.utils.skipmixin import SkipMixin
 else:
     from airflow.models.skipmixin import SkipMixin
-    from airflow.operators.branch import BranchMixIn
+    from airflow.operators.branch import BranchMixIn  # type: ignore[no-redef]
 
 
 log = logging.getLogger(__name__)
@@ -460,8 +460,7 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
         serializer = serializer or "pickle"
         if serializer not in _SERIALIZERS:
             msg = (
-                f"Unsupported serializer {serializer!r}. "
-                f"Expected one of {', '.join(map(repr, _SERIALIZERS))}"
+                f"Unsupported serializer {serializer!r}. Expected one of {', '.join(map(repr, _SERIALIZERS))}"
             )
             raise AirflowException(msg)
 
@@ -1142,7 +1141,6 @@ def _get_current_context() -> Mapping[str, Any]:
 
     if not _CURRENT_CONTEXT:
         raise RuntimeError(
-            "Current context was requested but no context was found! "
-            "Are you running within an Airflow task?"
+            "Current context was requested but no context was found! Are you running within an Airflow task?"
         )
     return _CURRENT_CONTEXT[-1]

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/operators/weekday.py

@@ -116,10 +116,13 @@ class BranchDayOfWeekOperator(BaseBranchOperator):
 
     def choose_branch(self, context: Context) -> str | Iterable[str]:
         if self.use_task_logical_date:
-            now = context["logical_date"]
+            now = context.get("logical_date")
+            if not now:
+                dag_run = context.get("dag_run")
+                now = dag_run.run_after  # type: ignore[union-attr, assignment]
         else:
             now = timezone.make_naive(timezone.utcnow(), self.dag.timezone)
 
-        if now.isoweekday() in self._week_day_num:
+        if now.isoweekday() in self._week_day_num:  # type: ignore[union-attr]
             return self.follow_task_ids_if_true
         return self.follow_task_ids_if_false

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/sensors/external_task.py

@@ -31,21 +31,26 @@ from airflow.providers.standard.operators.empty import EmptyOperator
 from airflow.providers.standard.triggers.external_task import WorkflowTrigger
 from airflow.providers.standard.utils.sensor_helper import _get_count, _get_external_task_group_task_ids
 from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
-from airflow.sensors.base import BaseSensorOperator
 from airflow.utils.file import correct_maybe_zipped
 from airflow.utils.session import NEW_SESSION, provide_session
 from airflow.utils.state import State, TaskInstanceState
 
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.sdk.bases.sensor import BaseSensorOperator
+else:
+    from airflow.sensors.base import BaseSensorOperator
+
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
-    from airflow.models.baseoperator import BaseOperator
     from airflow.models.taskinstancekey import TaskInstanceKey
 
     try:
+        from airflow.sdk import BaseOperator
         from airflow.sdk.definitions.context import Context
     except ImportError:
         # TODO: Remove once provider drops support for Airflow 2
+        from airflow.models.baseoperator import BaseOperator
         from airflow.utils.context import Context
 
 
@@ -65,15 +70,16 @@ class ExternalDagLink(BaseOperatorLink):
     name = "External DAG"
 
     def get_link(self, operator: BaseOperator, *, ti_key: TaskInstanceKey) -> str:
-        from airflow.models.renderedtifields import RenderedTaskInstanceFields
-
         if TYPE_CHECKING:
             assert isinstance(operator, (ExternalTaskMarker, ExternalTaskSensor))
 
-        if template_fields := RenderedTaskInstanceFields.get_templated_fields(ti_key):
-            external_dag_id: str = template_fields.get("external_dag_id", operator.external_dag_id)
-        else:
-            external_dag_id = operator.external_dag_id
+        external_dag_id = operator.external_dag_id
+
+        if not AIRFLOW_V_3_0_PLUS:
+            from airflow.models.renderedtifields import RenderedTaskInstanceFields
+
+            if template_fields := RenderedTaskInstanceFields.get_templated_fields(ti_key):
+                external_dag_id: str = template_fields.get("external_dag_id", operator.external_dag_id)  # type: ignore[no-redef]
 
         if AIRFLOW_V_3_0_PLUS:
             from airflow.utils.helpers import build_airflow_dagrun_url
@@ -245,16 +251,22 @@ class ExternalTaskSensor(BaseSensorOperator):
         self.poll_interval = poll_interval
 
     def _get_dttm_filter(self, context):
+        logical_date = context.get("logical_date")
+        if logical_date is None:
+            dag_run = context.get("dag_run")
+            if TYPE_CHECKING:
+                assert dag_run
+
+            logical_date = dag_run.run_after
         if self.execution_delta:
-            dttm = context["logical_date"] - self.execution_delta
+            dttm = logical_date - self.execution_delta
         elif self.execution_date_fn:
             dttm = self._handle_execution_date_fn(context=context)
         else:
-            dttm = context["logical_date"]
+            dttm = logical_date
         return dttm if isinstance(dttm, list) else [dttm]
 
-    @provide_session
-    def poke(self, context: Context, session: Session = NEW_SESSION) -> bool:
+    def poke(self, context: Context) -> bool:
         # delay check to poke rather than __init__ in case it was supplied as XComArgs
         if self.external_task_ids and len(self.external_task_ids) > len(set(self.external_task_ids)):
             raise ValueError("Duplicate task_ids passed in external_task_ids parameter")
@@ -285,15 +297,62 @@ class ExternalTaskSensor(BaseSensorOperator):
                 serialized_dttm_filter,
             )
 
-        # In poke mode this will check dag existence only once
-        if self.check_existence and not self._has_checked_existence:
-            self._check_for_existence(session=session)
+        if AIRFLOW_V_3_0_PLUS:
+            return self._poke_af3(context, dttm_filter)
+        else:
+            return self._poke_af2(dttm_filter)
+
+    def _poke_af3(self, context: Context, dttm_filter: list[datetime.datetime]) -> bool:
+        self._has_checked_existence = True
+        ti = context["ti"]
+
+        def _get_count(states: list[str]) -> int:
+            if self.external_task_ids:
+                return ti.get_ti_count(
+                    dag_id=self.external_dag_id,
+                    task_ids=self.external_task_ids,  # type: ignore[arg-type]
+                    logical_dates=dttm_filter,
+                    states=states,
+                )
+            elif self.external_task_group_id:
+                return ti.get_ti_count(
+                    dag_id=self.external_dag_id,
+                    task_group_id=self.external_task_group_id,
+                    logical_dates=dttm_filter,
+                    states=states,
+                )
+            else:
+                return ti.get_dr_count(
+                    dag_id=self.external_dag_id,
+                    logical_dates=dttm_filter,
+                    states=states,
+                )
 
-        count_failed = -1
         if self.failed_states:
-            count_failed = self.get_count(dttm_filter, session, self.failed_states)
+            count = _get_count(self.failed_states)
+            count_failed = self._calculate_count(count, dttm_filter)
+            self._handle_failed_states(count_failed)
 
-        # Fail if anything in the list has failed.
+        if self.skipped_states:
+            count = _get_count(self.skipped_states)
+            count_skipped = self._calculate_count(count, dttm_filter)
+            self._handle_skipped_states(count_skipped)
+
+        count = _get_count(self.allowed_states)
+        count_allowed = self._calculate_count(count, dttm_filter)
+        return count_allowed == len(dttm_filter)
+
+    def _calculate_count(self, count: int, dttm_filter: list[datetime.datetime]) -> float | int:
+        """Calculate the normalized count based on the type of check."""
+        if self.external_task_ids:
+            return count / len(self.external_task_ids)
+        elif self.external_task_group_id:
+            return count / len(dttm_filter)
+        else:
+            return count
+
+    def _handle_failed_states(self, count_failed: float | int) -> None:
+        """Handle failed states and raise appropriate exceptions."""
         if count_failed > 0:
             if self.external_task_ids:
                 if self.soft_fail:
@@ -315,7 +374,6 @@ class ExternalTaskSensor(BaseSensorOperator):
                     f"The external task_group '{self.external_task_group_id}' "
                     f"in DAG '{self.external_dag_id}' failed."
                 )
-
             else:
                 if self.soft_fail:
                     raise AirflowSkipException(
@@ -323,12 +381,8 @@ class ExternalTaskSensor(BaseSensorOperator):
                     )
                 raise AirflowException(f"The external DAG {self.external_dag_id} failed.")
 
-        count_skipped = -1
-        if self.skipped_states:
-            count_skipped = self.get_count(dttm_filter, session, self.skipped_states)
-
-        # Skip if anything in the list has skipped. Note if we are checking multiple tasks and one skips
-        # before another errors, we'll skip first.
+    def _handle_skipped_states(self, count_skipped: float | int) -> None:
+        """Handle skipped states and raise appropriate exceptions."""
         if count_skipped > 0:
             if self.external_task_ids:
                 raise AirflowSkipException(
@@ -346,7 +400,19 @@ class ExternalTaskSensor(BaseSensorOperator):
                     "Skipping."
                 )
 
-        # only go green if every single task has reached an allowed state
+    @provide_session
+    def _poke_af2(self, dttm_filter: list[datetime.datetime], session: Session = NEW_SESSION) -> bool:
+        if self.check_existence and not self._has_checked_existence:
+            self._check_for_existence(session=session)
+
+        if self.failed_states:
+            count_failed = self.get_count(dttm_filter, session, self.failed_states)
+            self._handle_failed_states(count_failed)
+
+        if self.skipped_states:
+            count_skipped = self.get_count(dttm_filter, session, self.skipped_states)
+            self._handle_skipped_states(count_skipped)
+
         count_allowed = self.get_count(dttm_filter, session, self.allowed_states)
         return count_allowed == len(dttm_filter)
 
@@ -398,8 +464,7 @@ class ExternalTaskSensor(BaseSensorOperator):
             for external_task_id in self.external_task_ids:
                 if not refreshed_dag_info.has_task(external_task_id):
                     raise AirflowException(
-                        f"The external task {external_task_id} in "
-                        f"DAG {self.external_dag_id} does not exist."
+                        f"The external task {external_task_id} in DAG {self.external_dag_id} does not exist."
                     )
 
         if self.external_task_group_id:
@@ -482,6 +547,9 @@ class ExternalTaskMarker(EmptyOperator):
     """
 
     template_fields = ["external_dag_id", "external_task_id", "logical_date"]
+    if not AIRFLOW_V_3_0_PLUS:
+        template_fields.append("execution_date")
+
     ui_color = "#4db7db"
     operator_extra_links = [ExternalDagLink()]
 
@@ -509,6 +577,9 @@ class ExternalTaskMarker(EmptyOperator):
                 f"Expected str or datetime.datetime type for logical_date. Got {type(logical_date)}"
             )
 
+        if not AIRFLOW_V_3_0_PLUS:
+            self.execution_date = self.logical_date
+
         if recursion_depth <= 0:
             raise ValueError("recursion_depth should be a positive integer")
         self.recursion_depth = recursion_depth

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/triggers/external_task.py

@@ -208,4 +208,4 @@ class DagStateTrigger(BaseTrigger):
             )
             .scalar()
         )
-        return typing.cast(int, count)
+        return typing.cast("int", count)

{apache_airflow_providers_standard-0.2.0rc1 → apache_airflow_providers_standard-0.3.0rc1}/src/airflow/providers/standard/utils/sensor_helper.py

@@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, cast
 from sqlalchemy import func, select, tuple_
 
 from airflow.models import DagBag, DagRun, TaskInstance
+from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils.session import NEW_SESSION, provide_session
 
 if TYPE_CHECKING:
@@ -71,10 +72,12 @@ def _get_count(
                         tuple_(TI.task_id, TI.map_index).in_(external_task_group_task_ids)
                     )
                 )
-            ) / len(external_task_group_task_ids)
+                / len(external_task_group_task_ids)
+                * len(dttm_filter)
+            )
     else:
         count = session.scalar(_count_stmt(DR, states, dttm_filter, external_dag_id))
-    return cast(int, count)
+    return cast("int", count)
 
 
 def _count_stmt(model, states, dttm_filter, external_dag_id) -> Executable:
@@ -86,8 +89,10 @@ def _count_stmt(model, states, dttm_filter, external_dag_id) -> Executable:
     :param dttm_filter: date time filter for logical date
     :param external_dag_id: The ID of the external DAG.
     """
+    date_field = model.logical_date if AIRFLOW_V_3_0_PLUS else model.execution_date
+
     return select(func.count()).where(
-        model.dag_id == external_dag_id, model.state.in_(states), model.logical_date.in_(dttm_filter)
+        model.dag_id == external_dag_id, model.state.in_(states), date_field.in_(dttm_filter)
     )
 
 
@@ -104,11 +109,13 @@ def _get_external_task_group_task_ids(dttm_filter, external_task_group_id, exter
     task_group = refreshed_dag_info.task_group_dict.get(external_task_group_id)
 
     if task_group:
+        date_field = TaskInstance.logical_date if AIRFLOW_V_3_0_PLUS else TaskInstance.execution_date
+
         group_tasks = session.scalars(
             select(TaskInstance).filter(
                 TaskInstance.dag_id == external_dag_id,
                 TaskInstance.task_id.in_(task.task_id for task in task_group),
-                TaskInstance.logical_date.in_(dttm_filter),
+                date_field.in_(dttm_filter),
             )
         )