apache-airflow-providers-standard 1.2.0__py3-none-any.whl → 1.3.0rc1__py3-none-any.whl

airflow/providers/standard/example_dags/example_external_task_child_deferrable.py

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from datetime import datetime
+
+from airflow import DAG
+from airflow.providers.standard.operators.bash import BashOperator
+
+with DAG(
+    dag_id="child_dag",
+    start_date=datetime(2022, 1, 1),
+    schedule="@once",
+    catchup=False,
+    tags=["example", "async", "core"],
+) as dag:
+    BashOperator(
+        task_id="child_task",
+        bash_command="echo 1; sleep 1; echo 2; sleep 2; echo 3; sleep 3",
+    )

airflow/providers/standard/example_dags/example_external_task_marker_dag.py

@@ -0,0 +1,98 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example DAG demonstrating setting up inter-DAG dependencies using ExternalTaskSensor and
+ExternalTaskMarker.
+
+In this example, child_task1 in example_external_task_marker_child depends on parent_task in
+example_external_task_marker_parent. When parent_task is cleared with 'Recursive' selected,
+the presence of ExternalTaskMarker tells Airflow to clear child_task1 and its downstream tasks.
+
+ExternalTaskSensor will keep poking for the status of remote ExternalTaskMarker task at a regular
+interval till one of the following will happen:
+
+ExternalTaskMarker reaches the states mentioned in the allowed_states list.
+In this case, ExternalTaskSensor will exit with a success status code
+
+ExternalTaskMarker reaches the states mentioned in the failed_states list
+In this case, ExternalTaskSensor will raise an AirflowException and user need to handle this
+with multiple downstream tasks
+
+ExternalTaskSensor times out. In this case, ExternalTaskSensor will raise AirflowSkipException
+or AirflowSensorTimeout exception
+
+"""
+
+from __future__ import annotations
+
+import pendulum
+
+from airflow.providers.standard.operators.empty import EmptyOperator
+from airflow.providers.standard.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor
+from airflow.sdk import DAG
+
+start_date = pendulum.datetime(2021, 1, 1, tz="UTC")
+
+with DAG(
+    dag_id="example_external_task_marker_parent",
+    start_date=start_date,
+    catchup=False,
+    schedule=None,
+    tags=["example2"],
+) as parent_dag:
+    # [START howto_operator_external_task_marker]
+    parent_task = ExternalTaskMarker(
+        task_id="parent_task",
+        external_dag_id="example_external_task_marker_child",
+        external_task_id="child_task1",
+    )
+    # [END howto_operator_external_task_marker]
+
+with DAG(
+    dag_id="example_external_task_marker_child",
+    start_date=start_date,
+    schedule=None,
+    catchup=False,
+    tags=["example2"],
+) as child_dag:
+    # [START howto_operator_external_task_sensor]
+    child_task1 = ExternalTaskSensor(
+        task_id="child_task1",
+        external_dag_id=parent_dag.dag_id,
+        external_task_id=parent_task.task_id,
+        timeout=600,
+        allowed_states=["success"],
+        failed_states=["failed", "skipped"],
+        mode="reschedule",
+    )
+    # [END howto_operator_external_task_sensor]
+
+    # [START howto_operator_external_task_sensor_with_task_group]
+    child_task2 = ExternalTaskSensor(
+        task_id="child_task2",
+        external_dag_id=parent_dag.dag_id,
+        external_task_group_id="parent_dag_task_group_id",
+        timeout=600,
+        allowed_states=["success"],
+        failed_states=["failed", "skipped"],
+        mode="reschedule",
+    )
+    # [END howto_operator_external_task_sensor_with_task_group]
+
+    child_task3 = EmptyOperator(task_id="child_task3")
+    child_task1 >> child_task2 >> child_task3

airflow/providers/standard/example_dags/example_external_task_parent_deferrable.py

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow import DAG
+from airflow.providers.standard.operators.empty import EmptyOperator
+from airflow.providers.standard.operators.trigger_dagrun import TriggerDagRunOperator
+from airflow.providers.standard.sensors.external_task import ExternalTaskSensor
+from airflow.utils.timezone import datetime
+
+with DAG(
+    dag_id="example_external_task",
+    start_date=datetime(2022, 1, 1),
+    schedule="@once",
+    catchup=False,
+    tags=["example", "async", "core"],
+) as dag:
+    start = EmptyOperator(task_id="start")
+
+    # [START howto_external_task_async_sensor]
+    external_task_sensor = ExternalTaskSensor(
+        task_id="parent_task_sensor",
+        external_task_id="child_task",
+        external_dag_id="child_dag",
+        deferrable=True,
+    )
+    # [END howto_external_task_async_sensor]
+
+    trigger_child_task = TriggerDagRunOperator(
+        task_id="trigger_child_task",
+        trigger_dag_id="child_dag",
+        allowed_states=[
+            "success",
+            "failed",
+        ],
+        logical_date="{{ logical_date }}",
+        poke_interval=5,
+        reset_dag_run=True,
+        wait_for_completion=True,
+    )
+
+    end = EmptyOperator(task_id="end")
+
+    start >> [trigger_child_task, external_task_sensor] >> end
+
+    from tests_common.test_utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "teardown" task with trigger rule is part of the DAG
+    list(dag.tasks) >> watcher()

airflow/providers/standard/example_dags/example_latest_only.py

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Example of the LatestOnlyOperator"""
+
+from __future__ import annotations
+
+import datetime
+
+from airflow.providers.standard.operators.empty import EmptyOperator
+from airflow.providers.standard.operators.latest_only import LatestOnlyOperator
+from airflow.sdk import DAG
+
+with DAG(
+    dag_id="latest_only",
+    schedule=datetime.timedelta(hours=4),
+    start_date=datetime.datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example2", "example3"],
+) as dag:
+    # [START howto_operator_latest_only]
+    latest_only = LatestOnlyOperator(task_id="latest_only")
+    # [END howto_operator_latest_only]
+    task1 = EmptyOperator(task_id="task1")
+
+    latest_only >> task1

airflow/providers/standard/example_dags/example_python_decorator.py

@@ -0,0 +1,132 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example DAG demonstrating the usage of the TaskFlow API to execute Python functions natively and within a
+virtual environment.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from pprint import pprint
+
+import pendulum
+
+from airflow.sdk import dag, task
+
+log = logging.getLogger(__name__)
+
+PATH_TO_PYTHON_BINARY = sys.executable
+
+
+@dag(
+    schedule=None,
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    catchup=False,
+    tags=["example"],
+)
+def example_python_decorator():
+    # [START howto_operator_python]
+    @task(task_id="print_the_context")
+    def print_context(ds=None, **kwargs):
+        """Print the Airflow context and ds variable from the context."""
+        pprint(kwargs)
+        print(ds)
+        return "Whatever you return gets printed in the logs"
+
+    run_this = print_context()
+    # [END howto_operator_python]
+
+    # [START howto_operator_python_render_sql]
+    @task(task_id="log_sql_query", templates_dict={"query": "sql/sample.sql"}, templates_exts=[".sql"])
+    def log_sql(**kwargs):
+        log.info("Python task decorator query: %s", str(kwargs["templates_dict"]["query"]))
+
+    log_the_sql = log_sql()
+    # [END howto_operator_python_render_sql]
+
+    # [START howto_operator_python_kwargs]
+    # Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
+    @task
+    def my_sleeping_function(random_base):
+        """This is a function that will run within the DAG execution"""
+        time.sleep(random_base)
+
+    for i in range(5):
+        sleeping_task = my_sleeping_function.override(task_id=f"sleep_for_{i}")(random_base=i / 10)
+
+        run_this >> log_the_sql >> sleeping_task
+    # [END howto_operator_python_kwargs]
+
+    # [START howto_operator_python_venv]
+    @task.virtualenv(
+        task_id="virtualenv_python", requirements=["colorama==0.4.0"], system_site_packages=False
+    )
+    def callable_virtualenv():
+        """
+        Example function that will be performed in a virtual environment.
+
+        Importing at the module level ensures that it will not attempt to import the
+        library before it is installed.
+        """
+        from time import sleep
+
+        from colorama import Back, Fore, Style
+
+        print(Fore.RED + "some red text")
+        print(Back.GREEN + "and with a green background")
+        print(Style.DIM + "and in dim text")
+        print(Style.RESET_ALL)
+        for _ in range(4):
+            print(Style.DIM + "Please wait...", flush=True)
+            sleep(1)
+        print("Finished")
+
+    virtualenv_task = callable_virtualenv()
+    # [END howto_operator_python_venv]
+
+    sleeping_task >> virtualenv_task
+
+    # [START howto_operator_external_python]
+    @task.external_python(task_id="external_python", python=PATH_TO_PYTHON_BINARY)
+    def callable_external_python():
+        """
+        Example function that will be performed in a virtual environment.
+
+        Importing at the module level ensures that it will not attempt to import the
+        library before it is installed.
+        """
+        import sys
+        from time import sleep
+
+        print(f"Running task via {sys.executable}")
+        print("Sleeping")
+        for _ in range(4):
+            print("Please wait...", flush=True)
+            sleep(1)
+        print("Finished")
+
+    external_python_task = callable_external_python()
+    # [END howto_operator_external_python]
+
+    run_this >> external_python_task >> virtualenv_task
+
+
+example_dag = example_python_decorator()

airflow/providers/standard/example_dags/example_python_operator.py

@@ -0,0 +1,147 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example DAG demonstrating the usage of the classic Python operators to execute Python functions natively and
+within a virtual environment.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from pprint import pprint
+
+import pendulum
+
+from airflow.providers.standard.operators.python import (
+    ExternalPythonOperator,
+    PythonOperator,
+    PythonVirtualenvOperator,
+)
+from airflow.sdk import DAG
+
+log = logging.getLogger(__name__)
+
+PATH_TO_PYTHON_BINARY = sys.executable
+
+
+with DAG(
+    dag_id="example_python_operator",
+    schedule=None,
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    catchup=False,
+    tags=["example"],
+) as dag:
+    # [START howto_operator_python]
+    def print_context(ds=None, **kwargs):
+        """Print the Airflow context and ds variable from the context."""
+        print("::group::All kwargs")
+        pprint(kwargs)
+        print("::endgroup::")
+        print("::group::Context variable ds")
+        print(ds)
+        print("::endgroup::")
+        return "Whatever you return gets printed in the logs"
+
+    run_this = PythonOperator(task_id="print_the_context", python_callable=print_context)
+    # [END howto_operator_python]
+
+    # [START howto_operator_python_render_sql]
+    def log_sql(**kwargs):
+        log.info("Python task decorator query: %s", str(kwargs["templates_dict"]["query"]))
+
+    log_the_sql = PythonOperator(
+        task_id="log_sql_query",
+        python_callable=log_sql,
+        templates_dict={"query": "sql/sample.sql"},
+        templates_exts=[".sql"],
+    )
+    # [END howto_operator_python_render_sql]
+
+    # [START howto_operator_python_kwargs]
+    # Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
+    def my_sleeping_function(random_base):
+        """This is a function that will run within the DAG execution"""
+        time.sleep(random_base)
+
+    for i in range(5):
+        sleeping_task = PythonOperator(
+            task_id=f"sleep_for_{i}", python_callable=my_sleeping_function, op_kwargs={"random_base": i / 10}
+        )
+
+        run_this >> log_the_sql >> sleeping_task
+    # [END howto_operator_python_kwargs]
+
+    # [START howto_operator_python_venv]
+    def callable_virtualenv():
+        """
+        Example function that will be performed in a virtual environment.
+
+        Importing at the function level ensures that it will not attempt to import the
+        library before it is installed.
+        """
+        from time import sleep
+
+        from colorama import Back, Fore, Style
+
+        print(Fore.RED + "some red text")
+        print(Back.GREEN + "and with a green background")
+        print(Style.DIM + "and in dim text")
+        print(Style.RESET_ALL)
+        for _ in range(4):
+            print(Style.DIM + "Please wait...", flush=True)
+            sleep(1)
+        print("Finished")
+
+    virtualenv_task = PythonVirtualenvOperator(
+        task_id="virtualenv_python",
+        python_callable=callable_virtualenv,
+        requirements=["colorama==0.4.0"],
+        system_site_packages=False,
+    )
+    # [END howto_operator_python_venv]
+
+    sleeping_task >> virtualenv_task
+
+    # [START howto_operator_external_python]
+    def callable_external_python():
+        """
+        Example function that will be performed in a virtual environment.
+
+        Importing at the module level ensures that it will not attempt to import the
+        library before it is installed.
+        """
+        import sys
+        from time import sleep
+
+        print(f"Running task via {sys.executable}")
+        print("Sleeping")
+        for _ in range(4):
+            print("Please wait...", flush=True)
+            sleep(1)
+        print("Finished")
+
+    external_python_task = ExternalPythonOperator(
+        task_id="external_python",
+        python_callable=callable_external_python,
+        python=PATH_TO_PYTHON_BINARY,
+    )
+    # [END howto_operator_external_python]
+
+    run_this >> external_python_task >> virtualenv_task

airflow/providers/standard/example_dags/example_sensor_decorator.py

@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Example DAG demonstrating the usage of the sensor decorator."""
+
+from __future__ import annotations
+
+# [START tutorial]
+# [START import_module]
+import pendulum
+
+from airflow.sdk import PokeReturnValue, dag, task
+
+# [END import_module]
+
+
+# [START instantiate_dag]
+@dag(
+    schedule=None,
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    catchup=False,
+    tags=["example"],
+)
+def example_sensor_decorator():
+    # [END instantiate_dag]
+
+    # [START wait_function]
+    # Using a sensor operator to wait for the upstream data to be ready.
+    @task.sensor(poke_interval=60, timeout=3600, mode="reschedule")
+    def wait_for_upstream() -> PokeReturnValue:
+        return PokeReturnValue(is_done=True, xcom_value="xcom_value")
+
+    # [END wait_function]
+
+    # [START dummy_function]
+    @task
+    def dummy_operator() -> None:
+        pass
+
+    # [END dummy_function]
+
+    # [START main_flow]
+    wait_for_upstream() >> dummy_operator()
+    # [END main_flow]
+
+
+# [START dag_invocation]
+tutorial_etl_dag = example_sensor_decorator()
+# [END dag_invocation]
+
+# [END tutorial]