apache-airflow-providers-standard 0.0.1rc1__py3-none-any.whl → 0.0.2__py3-none-any.whl

airflow/providers/standard/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "0.0.1"
+__version__ = "0.0.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

airflow/providers/standard/get_provider_info.py

@@ -28,8 +28,8 @@ def get_provider_info():
         "name": "Standard",
         "description": "Airflow Standard Provider\n",
         "state": "ready",
-        "source-date-epoch": 1718603992,
-        "versions": ["0.0.1"],
+        "source-date-epoch": 1732434919,
+        "versions": ["0.0.2", "0.0.1"],
         "dependencies": ["apache-airflow>=2.8.0", "apache-airflow-providers-common-sql>=1.20.0"],
         "integrations": [
             {
@@ -52,6 +52,7 @@ def get_provider_info():
                     "airflow.providers.standard.operators.bash",
                     "airflow.providers.standard.operators.python",
                     "airflow.providers.standard.operators.generic_transfer",
+                    "airflow.providers.standard.operators.trigger_dagrun",
                 ],
             }
         ],
@@ -65,6 +66,7 @@ def get_provider_info():
                     "airflow.providers.standard.sensors.weekday",
                     "airflow.providers.standard.sensors.bash",
                     "airflow.providers.standard.sensors.python",
+                    "airflow.providers.standard.sensors.filesystem",
                 ],
             }
         ],

airflow/providers/standard/hooks/subprocess.py

@@ -20,9 +20,9 @@ import contextlib
 import os
 import signal
 from collections import namedtuple
+from collections.abc import Iterator
 from subprocess import PIPE, STDOUT, Popen
 from tempfile import TemporaryDirectory, gettempdir
-from typing import Iterator
 
 from airflow.hooks.base import BaseHook
 

airflow/providers/standard/operators/bash.py

@@ -21,8 +21,9 @@ import os
 import shutil
 import tempfile
 import warnings
+from collections.abc import Container, Sequence
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Callable, Container, Sequence, cast
+from typing import TYPE_CHECKING, Any, Callable, cast
 
 from airflow.exceptions import AirflowException, AirflowSkipException
 from airflow.models.baseoperator import BaseOperator
@@ -97,7 +98,7 @@ class BashOperator(BaseOperator):
 
         .. code-block:: python
 
-            bash_command = "set -e; python3 script.py '{{ next_execution_date }}'"
+            bash_command = "set -e; python3 script.py '{{ data_interval_end }}'"
 
     .. note::
 

airflow/providers/standard/operators/datetime.py

@@ -17,7 +17,8 @@
 from __future__ import annotations
 
 import datetime
-from typing import TYPE_CHECKING, Iterable
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
 
 from airflow.exceptions import AirflowException
 from airflow.operators.branch import BaseBranchOperator
@@ -44,7 +45,7 @@ class BranchDateTimeOperator(BaseBranchOperator):
     :param target_lower: target lower bound.
     :param target_upper: target upper bound.
     :param use_task_logical_date: If ``True``, uses task's logical date to compare with targets.
-        Execution date is useful for backfilling. If ``False``, uses system's date.
+        Logical date is useful for backfilling. If ``False``, uses system's date.
     """
 
     def __init__(

airflow/providers/standard/operators/generic_transfer.py

@@ -17,7 +17,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Sequence
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
 
 from airflow.hooks.base import BaseHook
 from airflow.models import BaseOperator

airflow/providers/standard/operators/python.py

@@ -28,11 +28,11 @@ import textwrap
 import types
 import warnings
 from abc import ABCMeta, abstractmethod
-from collections.abc import Container
+from collections.abc import Collection, Container, Iterable, Mapping, Sequence
 from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import TYPE_CHECKING, Any, Callable, Collection, Iterable, Mapping, NamedTuple, Sequence, cast
+from typing import TYPE_CHECKING, Any, Callable, NamedTuple, cast
 
 import lazy_object_proxy
 
@@ -114,13 +114,13 @@ class PythonOperator(BaseOperator):
     function. This set of kwargs correspond exactly to what you can use in your jinja templates.
     For this to work, you need to define ``**kwargs`` in your function header, or you can add directly the
     keyword arguments you would like to get - for example with the below code your callable will get
-    the values of ``ti`` and ``next_ds`` context variables.
+    the values of ``ti`` context variables.
 
     With explicit arguments:
 
     .. code-block:: python
 
-       def my_python_callable(ti, next_ds):
+       def my_python_callable(ti):
            pass
 
     With kwargs:
@@ -129,7 +129,6 @@ class PythonOperator(BaseOperator):
 
        def my_python_callable(**kwargs):
            ti = kwargs["ti"]
-           next_ds = kwargs["next_ds"]
 
 
     :param python_callable: A reference to an object that is callable
@@ -310,7 +309,7 @@ class ShortCircuitOperator(PythonOperator, SkipMixin):
             self.skip(
                 dag_run=dag_run,
                 tasks=to_skip,
-                execution_date=cast("DateTime", dag_run.execution_date),  # type: ignore[call-arg]
+                execution_date=cast("DateTime", dag_run.logical_date),  # type: ignore[call-arg, union-attr]
                 map_index=context["ti"].map_index,
             )
 
@@ -360,34 +359,36 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
         "ds_nodash",
         "expanded_ti_count",
         "inlets",
-        "next_ds",
-        "next_ds_nodash",
         "outlets",
-        "prev_ds",
-        "prev_ds_nodash",
         "run_id",
         "task_instance_key_str",
         "test_mode",
-        "tomorrow_ds",
-        "tomorrow_ds_nodash",
         "ts",
         "ts_nodash",
         "ts_nodash_with_tz",
+        # The following should be removed when Airflow 2 support is dropped.
+        "next_ds",
+        "next_ds_nodash",
+        "prev_ds",
+        "prev_ds_nodash",
+        "tomorrow_ds",
+        "tomorrow_ds_nodash",
         "yesterday_ds",
         "yesterday_ds_nodash",
     }
     PENDULUM_SERIALIZABLE_CONTEXT_KEYS = {
         "data_interval_end",
         "data_interval_start",
-        "execution_date",
         "logical_date",
-        "next_execution_date",
         "prev_data_interval_end_success",
         "prev_data_interval_start_success",
-        "prev_execution_date",
-        "prev_execution_date_success",
         "prev_start_date_success",
         "prev_end_date_success",
+        # The following should be removed when Airflow 2 support is dropped.
+        "execution_date",
+        "next_execution_date",
+        "prev_execution_date",
+        "prev_execution_date_success",
     }
 
     AIRFLOW_SERIALIZABLE_CONTEXT_KEYS = {
@@ -397,7 +398,9 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
         "dag_run",
         "task",
         "params",
-        "triggering_asset_events" if AIRFLOW_V_3_0_PLUS else "triggering_dataset_events",
+        "triggering_asset_events",
+        # The following should be removed when Airflow 2 support is dropped.
+        "triggering_dataset_events",
     }
 
     def __init__(

airflow/providers/standard/operators/trigger_dagrun.py

@@ -0,0 +1,305 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import datetime
+import json
+import time
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, cast
+
+from sqlalchemy import select
+from sqlalchemy.orm.exc import NoResultFound
+
+from airflow.api.common.trigger_dag import trigger_dag
+from airflow.api_internal.internal_api_call import InternalApiConfig
+from airflow.configuration import conf
+from airflow.exceptions import (
+    AirflowException,
+    AirflowSkipException,
+    DagNotFound,
+    DagRunAlreadyExists,
+)
+from airflow.models import BaseOperator, BaseOperatorLink
+from airflow.models.dag import DagModel
+from airflow.models.dagbag import DagBag
+from airflow.models.dagrun import DagRun
+from airflow.models.xcom import XCom
+from airflow.triggers.external_task import DagStateTrigger
+from airflow.utils import timezone
+from airflow.utils.helpers import build_airflow_url_with_query
+from airflow.utils.session import provide_session
+from airflow.utils.state import DagRunState
+from airflow.utils.types import DagRunTriggeredByType, DagRunType
+
+XCOM_LOGICAL_DATE_ISO = "trigger_logical_date_iso"
+XCOM_RUN_ID = "trigger_run_id"
+
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm.session import Session
+
+    from airflow.models.taskinstancekey import TaskInstanceKey
+    from airflow.utils.context import Context
+
+
+class TriggerDagRunLink(BaseOperatorLink):
+    """
+    Operator link for TriggerDagRunOperator.
+
+    It allows users to access DAG triggered by task using TriggerDagRunOperator.
+    """
+
+    name = "Triggered DAG"
+
+    def get_link(self, operator: BaseOperator, *, ti_key: TaskInstanceKey) -> str:
+        from airflow.models.renderedtifields import RenderedTaskInstanceFields
+        from airflow.models.taskinstance import TaskInstance
+
+        ti = TaskInstance.get_task_instance(
+            dag_id=ti_key.dag_id, run_id=ti_key.run_id, task_id=ti_key.task_id, map_index=ti_key.map_index
+        )
+        if TYPE_CHECKING:
+            assert ti is not None
+
+        template_fields = RenderedTaskInstanceFields.get_templated_fields(ti)
+        untemplated_trigger_dag_id = cast(TriggerDagRunOperator, operator).trigger_dag_id
+        if template_fields:
+            trigger_dag_id = template_fields.get("trigger_dag_id", untemplated_trigger_dag_id)
+        else:
+            trigger_dag_id = untemplated_trigger_dag_id
+
+        # Fetch the correct dag_run_id for the triggerED dag which is
+        # stored in xcom during execution of the triggerING task.
+        triggered_dag_run_id = XCom.get_value(ti_key=ti_key, key=XCOM_RUN_ID)
+
+        query = {
+            "dag_id": trigger_dag_id,
+            "dag_run_id": triggered_dag_run_id,
+        }
+        return build_airflow_url_with_query(query)
+
+
+class TriggerDagRunOperator(BaseOperator):
+    """
+    Triggers a DAG run for a specified DAG ID.
+
+    Note that if database isolation mode is enabled, not all features are supported.
+
+    :param trigger_dag_id: The ``dag_id`` of the DAG to trigger (templated).
+    :param trigger_run_id: The run ID to use for the triggered DAG run (templated).
+        If not provided, a run ID will be automatically generated.
+    :param conf: Configuration for the DAG run (templated).
+    :param logical_date: Logical date for the triggered DAG (templated).
+    :param reset_dag_run: Whether clear existing DAG run if already exists.
+        This is useful when backfill or rerun an existing DAG run.
+        This only resets (not recreates) the DAG run.
+        DAG run conf is immutable and will not be reset on rerun of an existing DAG run.
+        When reset_dag_run=False and dag run exists, DagRunAlreadyExists will be raised.
+        When reset_dag_run=True and dag run exists, existing DAG run will be cleared to rerun.
+    :param wait_for_completion: Whether or not wait for DAG run completion. (default: False)
+    :param poke_interval: Poke interval to check DAG run status when wait_for_completion=True.
+        (default: 60)
+    :param allowed_states: Optional list of allowed DAG run states of the triggered DAG. This is useful when
+        setting ``wait_for_completion`` to True. Must be a valid DagRunState.
+        Default is ``[DagRunState.SUCCESS]``.
+    :param failed_states: Optional list of failed or disallowed DAG run states of the triggered DAG. This is
+        useful when setting ``wait_for_completion`` to True. Must be a valid DagRunState.
+        Default is ``[DagRunState.FAILED]``.
+    :param skip_when_already_exists: Set to true to mark the task as SKIPPED if a DAG run of the triggered
+        DAG for the same logical date already exists.
+    :param deferrable: If waiting for completion, whether or not to defer the task until done,
+        default is ``False``.
+    """
+
+    template_fields: Sequence[str] = (
+        "trigger_dag_id",
+        "trigger_run_id",
+        "logical_date",
+        "conf",
+        "wait_for_completion",
+        "skip_when_already_exists",
+    )
+    template_fields_renderers = {"conf": "py"}
+    ui_color = "#ffefeb"
+    operator_extra_links = [TriggerDagRunLink()]
+
+    def __init__(
+        self,
+        *,
+        trigger_dag_id: str,
+        trigger_run_id: str | None = None,
+        conf: dict | None = None,
+        logical_date: str | datetime.datetime | None = None,
+        reset_dag_run: bool = False,
+        wait_for_completion: bool = False,
+        poke_interval: int = 60,
+        allowed_states: list[str | DagRunState] | None = None,
+        failed_states: list[str | DagRunState] | None = None,
+        skip_when_already_exists: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.trigger_dag_id = trigger_dag_id
+        self.trigger_run_id = trigger_run_id
+        self.conf = conf
+        self.reset_dag_run = reset_dag_run
+        self.wait_for_completion = wait_for_completion
+        self.poke_interval = poke_interval
+        if allowed_states:
+            self.allowed_states = [DagRunState(s) for s in allowed_states]
+        else:
+            self.allowed_states = [DagRunState.SUCCESS]
+        if failed_states or failed_states == []:
+            self.failed_states = [DagRunState(s) for s in failed_states]
+        else:
+            self.failed_states = [DagRunState.FAILED]
+        self.skip_when_already_exists = skip_when_already_exists
+        self._defer = deferrable
+
+        if logical_date is not None and not isinstance(logical_date, (str, datetime.datetime)):
+            type_name = type(logical_date).__name__
+            raise TypeError(
+                f"Expected str or datetime.datetime type for parameter 'logical_date'. Got {type_name}"
+            )
+
+        self.logical_date = logical_date
+
+    def execute(self, context: Context):
+        if InternalApiConfig.get_use_internal_api():
+            if self.reset_dag_run:
+                raise AirflowException("Parameter reset_dag_run=True is broken with Database Isolation Mode.")
+            if self.wait_for_completion:
+                raise AirflowException(
+                    "Parameter wait_for_completion=True is broken with Database Isolation Mode."
+                )
+
+        if isinstance(self.logical_date, datetime.datetime):
+            parsed_logical_date = self.logical_date
+        elif isinstance(self.logical_date, str):
+            parsed_logical_date = timezone.parse(self.logical_date)
+        else:
+            parsed_logical_date = timezone.utcnow()
+
+        try:
+            json.dumps(self.conf)
+        except TypeError:
+            raise AirflowException("conf parameter should be JSON Serializable")
+
+        if self.trigger_run_id:
+            run_id = str(self.trigger_run_id)
+        else:
+            run_id = DagRun.generate_run_id(DagRunType.MANUAL, parsed_logical_date)
+
+        try:
+            dag_run = trigger_dag(
+                dag_id=self.trigger_dag_id,
+                run_id=run_id,
+                conf=self.conf,
+                logical_date=parsed_logical_date,
+                replace_microseconds=False,
+                triggered_by=DagRunTriggeredByType.OPERATOR,
+            )
+
+        except DagRunAlreadyExists as e:
+            if self.reset_dag_run:
+                dag_run = e.dag_run
+                self.log.info("Clearing %s on %s", self.trigger_dag_id, dag_run.logical_date)
+
+                # Get target dag object and call clear()
+                dag_model = DagModel.get_current(self.trigger_dag_id)
+                if dag_model is None:
+                    raise DagNotFound(f"Dag id {self.trigger_dag_id} not found in DagModel")
+
+                # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72
+                dag_bag = DagBag(dag_folder=dag_model.fileloc, read_dags_from_db=True)
+                dag = dag_bag.get_dag(self.trigger_dag_id)
+                dag.clear(start_date=dag_run.logical_date, end_date=dag_run.logical_date)
+            else:
+                if self.skip_when_already_exists:
+                    raise AirflowSkipException(
+                        "Skipping due to skip_when_already_exists is set to True and DagRunAlreadyExists"
+                    )
+                raise e
+        if dag_run is None:
+            raise RuntimeError("The dag_run should be set here!")
+        # Store the run id from the dag run (either created or found above) to
+        # be used when creating the extra link on the webserver.
+        ti = context["task_instance"]
+        ti.xcom_push(key=XCOM_RUN_ID, value=dag_run.run_id)
+
+        if self.wait_for_completion:
+            # Kick off the deferral process
+            if self._defer:
+                self.defer(
+                    trigger=DagStateTrigger(
+                        dag_id=self.trigger_dag_id,
+                        states=self.allowed_states + self.failed_states,
+                        logical_dates=[dag_run.logical_date],
+                        poll_interval=self.poke_interval,
+                    ),
+                    method_name="execute_complete",
+                )
+            # wait for dag to complete
+            while True:
+                self.log.info(
+                    "Waiting for %s on %s to become allowed state %s ...",
+                    self.trigger_dag_id,
+                    dag_run.logical_date,
+                    self.allowed_states,
+                )
+                time.sleep(self.poke_interval)
+
+                # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72
+                dag_run.refresh_from_db()
+                state = dag_run.state
+                if state in self.failed_states:
+                    raise AirflowException(f"{self.trigger_dag_id} failed with failed states {state}")
+                if state in self.allowed_states:
+                    self.log.info("%s finished with allowed state %s", self.trigger_dag_id, state)
+                    return
+
+    @provide_session
+    def execute_complete(self, context: Context, session: Session, event: tuple[str, dict[str, Any]]):
+        # This logical_date is parsed from the return trigger event
+        provided_logical_date = event[1]["logical_dates"][0]
+        try:
+            # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72
+            dag_run = session.execute(
+                select(DagRun).where(
+                    DagRun.dag_id == self.trigger_dag_id, DagRun.logical_date == provided_logical_date
+                )
+            ).scalar_one()
+        except NoResultFound:
+            raise AirflowException(
+                f"No DAG run found for DAG {self.trigger_dag_id} and logical date {self.logical_date}"
+            )
+
+        state = dag_run.state
+
+        if state in self.failed_states:
+            raise AirflowException(f"{self.trigger_dag_id} failed with failed state {state}")
+        if state in self.allowed_states:
+            self.log.info("%s finished with allowed state %s", self.trigger_dag_id, state)
+            return
+
+        raise AirflowException(
+            f"{self.trigger_dag_id} return {state} which is not in {self.failed_states}"
+            f" or {self.allowed_states}"
+        )

airflow/providers/standard/operators/weekday.py

@@ -17,7 +17,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
 
 from airflow.operators.branch import BaseBranchOperator
 from airflow.utils import timezone

airflow/providers/standard/sensors/bash.py

@@ -18,9 +18,10 @@
 from __future__ import annotations
 
 import os
+from collections.abc import Sequence
 from subprocess import PIPE, STDOUT, Popen
 from tempfile import NamedTemporaryFile, TemporaryDirectory, gettempdir
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING
 
 from airflow.exceptions import AirflowFailException
 from airflow.sensors.base import BaseSensorOperator
@@ -70,9 +71,10 @@ class BashSensor(BaseSensorOperator):
         """Execute the bash command in a temporary directory."""
         bash_command = self.bash_command
         self.log.info("Tmp dir root location: %s", gettempdir())
-        with TemporaryDirectory(prefix="airflowtmp") as tmp_dir, NamedTemporaryFile(
-            dir=tmp_dir, prefix=self.task_id
-        ) as f:
+        with (
+            TemporaryDirectory(prefix="airflowtmp") as tmp_dir,
+            NamedTemporaryFile(dir=tmp_dir, prefix=self.task_id) as f,
+        ):
             f.write(bytes(bash_command, "utf_8"))
             f.flush()
             fname = f.name

airflow/providers/standard/sensors/date_time.py

@@ -18,8 +18,9 @@
 from __future__ import annotations
 
 import datetime
+from collections.abc import Sequence
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, NoReturn, Sequence
+from typing import TYPE_CHECKING, Any, NoReturn
 
 from airflow.providers.standard.utils.version_references import AIRFLOW_V_3_0_PLUS
 from airflow.sensors.base import BaseSensorOperator
@@ -54,7 +55,7 @@ class DateTimeSensor(BaseSensorOperator):
     It handles some cases for which ``TimeSensor`` and ``TimeDeltaSensor`` are not suited.
 
     **Example** 1 :
-        If a task needs to wait for 11am on each ``execution_date``. Using
+        If a task needs to wait for 11am on each ``logical_date``. Using
         ``TimeSensor`` or ``TimeDeltaSensor``, all backfill tasks started at
         1am have to wait for 10 hours. This is unnecessary, e.g. a backfill
         task with ``{{ ds }} = '1970-01-01'`` does not need to wait because
@@ -69,7 +70,7 @@ class DateTimeSensor(BaseSensorOperator):
 
             DateTimeSensor(
                 task_id="wait_for_0100",
-                target_time="{{ next_execution_date.tomorrow().replace(hour=1) }}",
+                target_time="{{ data_interval_end.tomorrow().replace(hour=1) }}",
             )
 
     :param target_time: datetime after which the job succeeds. (templated)

airflow/providers/standard/sensors/filesystem.py

@@ -0,0 +1,139 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import datetime
+import os
+from collections.abc import Sequence
+from functools import cached_property
+from glob import glob
+from typing import TYPE_CHECKING, Any
+
+from airflow.configuration import conf
+from airflow.exceptions import AirflowException
+from airflow.providers.standard.hooks.filesystem import FSHook
+from airflow.sensors.base import BaseSensorOperator
+from airflow.triggers.base import StartTriggerArgs
+from airflow.triggers.file import FileTrigger
+
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
+
+class FileSensor(BaseSensorOperator):
+    """
+    Waits for a file or folder to land in a filesystem.
+
+    If the path given is a directory then this sensor will only return true if
+    any files exist inside it (either directly, or within a subdirectory)
+
+    :param fs_conn_id: reference to the File (path)
+        connection id
+    :param filepath: File or folder name (relative to
+        the base path set within the connection), can be a glob.
+    :param recursive: when set to ``True``, enables recursive directory matching behavior of
+        ``**`` in glob filepath parameter. Defaults to ``False``.
+    :param deferrable: If waiting for completion, whether to defer the task until done,
+        default is ``False``.
+    :param start_from_trigger: Start the task directly from the triggerer without going into the worker.
+    :param trigger_kwargs: The keyword arguments passed to the trigger when start_from_trigger is set to True
+        during dynamic task mapping. This argument is not used in standard usage.
+
+    .. seealso::
+        For more information on how to use this sensor, take a look at the guide:
+        :ref:`howto/operator:FileSensor`
+
+
+    """
+
+    template_fields: Sequence[str] = ("filepath",)
+    ui_color = "#91818a"
+    start_trigger_args = StartTriggerArgs(
+        trigger_cls="airflow.triggers.file.FileTrigger",
+        trigger_kwargs={},
+        next_method="execute_complete",
+        next_kwargs=None,
+        timeout=None,
+    )
+    start_from_trigger = False
+
+    def __init__(
+        self,
+        *,
+        filepath,
+        fs_conn_id="fs_default",
+        recursive=False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+        start_from_trigger: bool = False,
+        trigger_kwargs: dict[str, Any] | None = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.filepath = filepath
+        self.fs_conn_id = fs_conn_id
+        self.recursive = recursive
+        self.deferrable = deferrable
+
+        self.start_from_trigger = start_from_trigger
+
+        if self.deferrable and self.start_from_trigger:
+            self.start_trigger_args.timeout = datetime.timedelta(seconds=self.timeout)
+            self.start_trigger_args.trigger_kwargs = dict(
+                filepath=self.path,
+                recursive=self.recursive,
+                poke_interval=self.poke_interval,
+            )
+
+    @cached_property
+    def path(self) -> str:
+        hook = FSHook(self.fs_conn_id)
+        basepath = hook.get_path()
+        full_path = os.path.join(basepath, self.filepath)
+        return full_path
+
+    def poke(self, context: Context) -> bool:
+        self.log.info("Poking for file %s", self.path)
+        for path in glob(self.path, recursive=self.recursive):
+            if os.path.isfile(path):
+                mod_time = datetime.datetime.fromtimestamp(os.path.getmtime(path)).strftime("%Y%m%d%H%M%S")
+                self.log.info("Found File %s last modified: %s", path, mod_time)
+                return True
+
+            for _, _, files in os.walk(path):
+                if files:
+                    return True
+        return False
+
+    def execute(self, context: Context) -> None:
+        if not self.deferrable:
+            super().execute(context=context)
+        if not self.poke(context=context):
+            self.defer(
+                timeout=datetime.timedelta(seconds=self.timeout),
+                trigger=FileTrigger(
+                    filepath=self.path,
+                    recursive=self.recursive,
+                    poke_interval=self.poke_interval,
+                ),
+                method_name="execute_complete",
+            )
+
+    def execute_complete(self, context: Context, event: bool | None = None) -> None:
+        if not event:
+            raise AirflowException("%s task failed as %s not found.", self.task_id, self.filepath)
+        self.log.info("%s completed successfully as %s found.", self.task_id, self.filepath)

airflow/providers/standard/sensors/python.py

@@ -17,7 +17,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Callable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Callable
 
 from airflow.sensors.base import BaseSensorOperator, PokeReturnValue
 from airflow.utils.context import context_merge

airflow/providers/standard/sensors/weekday.py

@@ -17,7 +17,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
 
 from airflow.sensors.base import BaseSensorOperator
 from airflow.utils import timezone

{apache_airflow_providers_standard-0.0.1rc1.dist-info → apache_airflow_providers_standard-0.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apache-airflow-providers-standard
-Version: 0.0.1rc1
+Version: 0.0.2
 Summary: Provider package apache-airflow-providers-standard for Apache Airflow
 Keywords: airflow-provider,standard,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
+Requires-Dist: apache-airflow>=2.8.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.2/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.2
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -74,7 +74,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-standard``
 
-Release: ``0.0.1.rc1``
+Release: ``0.0.2``
 
 
 Airflow Standard Provider
@@ -87,7 +87,7 @@ This is a provider package for ``standard`` provider. All classes for this provi
 are in ``airflow.providers.standard`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.2/>`_.
 
 Installation
 ------------
@@ -109,4 +109,4 @@ PIP package                              Version required
 =======================================  ==================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.2/changelog.html>`_.

apache_airflow_providers_standard-0.0.2.dist-info/RECORD

@@ -0,0 +1,30 @@
+airflow/providers/standard/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
+airflow/providers/standard/__init__.py,sha256=Y9NnjAMDPzmmwIz6hIjtW8mk2me2U25AprTqwzTpXNI,1495
+airflow/providers/standard/get_provider_info.py,sha256=ONygoc2VOl9a0kjZAVAXW9x3K34iJwzsU-OqPcpU5uM,4368
+airflow/providers/standard/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/standard/hooks/filesystem.py,sha256=fDZwW_EYD8z1QXnReqI7gIwSbDPZNTKtqQvgktiP02o,2870
+airflow/providers/standard/hooks/package_index.py,sha256=U7_s_02-wwz9kTkzKr3JAhVQj2spuntWd_GmjfpV-y4,3769
+airflow/providers/standard/hooks/subprocess.py,sha256=GAmdF69jwUcpc7DH5I42GnJRs6NMQvHwFhimWpIdTU4,4920
+airflow/providers/standard/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/standard/operators/bash.py,sha256=JND5TfySXqQcJUfzJLv8IZ2_PojNzsubjXjaMyTTdA4,13440
+airflow/providers/standard/operators/datetime.py,sha256=3ArdXBAiolyrjLZP9VX4zD4OCWUcdxAOWst7f4IcD0k,4587
+airflow/providers/standard/operators/generic_transfer.py,sha256=V9aJ9PtSETWpKXUsAHbqpJPlb-PJ4EN3sXKDrDQ8tvE,5085
+airflow/providers/standard/operators/python.py,sha256=a3sotdSdTv-Q-7TOxibRQZ5e473Y0Gj9wTdoOVCP7no,52088
+airflow/providers/standard/operators/trigger_dagrun.py,sha256=uXO43o70ewAO4Xko_FhJE3xyC-uTGiM6SGzOp3bZQ-c,13020
+airflow/providers/standard/operators/weekday.py,sha256=f_Qjqsb00YhtDSb329aPmjZi6bR4EZTmhvvluCexGDE,4641
+airflow/providers/standard/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/standard/sensors/bash.py,sha256=rjbO7Ws30RwDZiVdjx8YRyS6M1ahiZu3dB_bUSrUfTs,4863
+airflow/providers/standard/sensors/date_time.py,sha256=-lA_Zb2UkFe-jN8hT0_1pAuGqj5k5dQQ4NtPeWLhl6s,5887
+airflow/providers/standard/sensors/filesystem.py,sha256=iT8q3k9tcq7wQ_w1N13AvYcGKNies-vCOo-qk5JVnL8,5336
+airflow/providers/standard/sensors/python.py,sha256=iljexBkGJ6spV2Z4POSt-Xm1Fw2DdbAb12vYKTmwzVo,3243
+airflow/providers/standard/sensors/time.py,sha256=kknnhVx-Ff3lRljY7jBEDc9EKEkwXkCf3m7KRm8BU_8,4817
+airflow/providers/standard/sensors/time_delta.py,sha256=b9U15IpJ2yFoJMNKLJ6FcMvuijf_jMqmMPI733rfI_o,4957
+airflow/providers/standard/sensors/weekday.py,sha256=f1uwCSvA4tibz88JKsij1zRieHcbTF2N4OT5GQumbaU,3722
+airflow/providers/standard/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/standard/utils/python_virtualenv.py,sha256=FR3241l5Obuo2BBwwBs-s87pRpCLyJnh3sUtHxrgRuM,7759
+airflow/providers/standard/utils/python_virtualenv_script.jinja2,sha256=xipJ3A8yuJBQxdOZba3NLzM9KP07TeFg-2z1GBB1Bmc,3268
+airflow/providers/standard/utils/version_references.py,sha256=_AxsWDE9afM-lkY2LzTLHy6qNqrTv5q_tqgRb1zS1Mo,1117
+apache_airflow_providers_standard-0.0.2.dist-info/entry_points.txt,sha256=mW2YRh3mVdZdaP5-iGSNgmcCh3YYdALIn28BCLBZZ40,104
+apache_airflow_providers_standard-0.0.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+apache_airflow_providers_standard-0.0.2.dist-info/METADATA,sha256=dcypwsVb7ehfOsQEbFBFtyQ2x2QSlz3mEmyL7WpKRjg,4845
+apache_airflow_providers_standard-0.0.2.dist-info/RECORD,,

apache_airflow_providers_standard-0.0.1rc1.dist-info/RECORD

@@ -1,28 +0,0 @@
-airflow/providers/standard/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/standard/__init__.py,sha256=OEhrQnhJMzOX1WvAZQPlXAEAfVco-55lw8voEe-x47w,1495
-airflow/providers/standard/get_provider_info.py,sha256=8CFrRtQJetNh97vyGFqLlcIFRiDsbXmUAymiv7R-E8w,4215
-airflow/providers/standard/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/standard/hooks/filesystem.py,sha256=fDZwW_EYD8z1QXnReqI7gIwSbDPZNTKtqQvgktiP02o,2870
-airflow/providers/standard/hooks/package_index.py,sha256=U7_s_02-wwz9kTkzKr3JAhVQj2spuntWd_GmjfpV-y4,3769
-airflow/providers/standard/hooks/subprocess.py,sha256=43tVi3bhmdJ_zowxYYuooJzJdp4Vw2jrz6R6L6g4Q0s,4911
-airflow/providers/standard/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/standard/operators/bash.py,sha256=fK0S14O_cDpDUgMT4-X9uLl_voJKeWWmLYBQxyxq4KA,13415
-airflow/providers/standard/operators/datetime.py,sha256=nk0gwO_H8vIIu8ztA4zryWqZeoSokfINTu4o2vPUcPc,4562
-airflow/providers/standard/operators/generic_transfer.py,sha256=zdY9-8UDDG3YUwvEjfH4Nxe7KaAoXUtZtoWnnadD04s,5058
-airflow/providers/standard/operators/python.py,sha256=-1aLnyCJp_GoH7JVJxg62IRGt69aqPcAgmf-p7D-io8,51929
-airflow/providers/standard/operators/weekday.py,sha256=u1tkO_04IGeshL17NhC6G7lIYLhgDnZjNrIKyAfd4b4,4614
-airflow/providers/standard/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/standard/sensors/bash.py,sha256=Gyi8zMwSESPyanlG9jbN9u-an6Vj33s7lTlmjWmjgSA,4821
-airflow/providers/standard/sensors/date_time.py,sha256=t6EDZBFSsUuhNPxD9R7N9ZP6HOxw2aBTirtpUiilQl8,5864
-airflow/providers/standard/sensors/python.py,sha256=0JoExKvQOJKKQlTEOXztg17UC5bpZDyIjQUeATV13LE,3216
-airflow/providers/standard/sensors/time.py,sha256=kknnhVx-Ff3lRljY7jBEDc9EKEkwXkCf3m7KRm8BU_8,4817
-airflow/providers/standard/sensors/time_delta.py,sha256=b9U15IpJ2yFoJMNKLJ6FcMvuijf_jMqmMPI733rfI_o,4957
-airflow/providers/standard/sensors/weekday.py,sha256=PMg0eoGuD0xNLSJIBY9C1Y0aqSZn6PkQ_j_eYo7lnks,3695
-airflow/providers/standard/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/standard/utils/python_virtualenv.py,sha256=FR3241l5Obuo2BBwwBs-s87pRpCLyJnh3sUtHxrgRuM,7759
-airflow/providers/standard/utils/python_virtualenv_script.jinja2,sha256=xipJ3A8yuJBQxdOZba3NLzM9KP07TeFg-2z1GBB1Bmc,3268
-airflow/providers/standard/utils/version_references.py,sha256=_AxsWDE9afM-lkY2LzTLHy6qNqrTv5q_tqgRb1zS1Mo,1117
-apache_airflow_providers_standard-0.0.1rc1.dist-info/entry_points.txt,sha256=mW2YRh3mVdZdaP5-iGSNgmcCh3YYdALIn28BCLBZZ40,104
-apache_airflow_providers_standard-0.0.1rc1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-apache_airflow_providers_standard-0.0.1rc1.dist-info/METADATA,sha256=b_PTbScMBE49UpqTjnVXWOd49HxX1WUca1tRJWLIDaU,4858
-apache_airflow_providers_standard-0.0.1rc1.dist-info/RECORD,,