apache-airflow-providers-standard 1.0.0.dev0__py3-none-any.whl → 1.0.0.dev1__py3-none-any.whl

airflow/providers/standard/get_provider_info.py

@@ -45,6 +45,7 @@ def get_provider_info():
                 "python-modules": [
                     "airflow.providers.standard.operators.datetime",
                     "airflow.providers.standard.operators.weekday",
+                    "airflow.providers.standard.operators.bash",
                 ],
             }
         ],
@@ -56,6 +57,7 @@ def get_provider_info():
                     "airflow.providers.standard.sensors.time_delta",
                     "airflow.providers.standard.sensors.time",
                     "airflow.providers.standard.sensors.weekday",
+                    "airflow.providers.standard.sensors.bash",
                 ],
             }
         ],

airflow/providers/standard/operators/bash.py

@@ -0,0 +1,256 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+import shutil
+import warnings
+from functools import cached_property
+from typing import TYPE_CHECKING, Any, Callable, Container, Sequence, cast
+
+from airflow.exceptions import AirflowException, AirflowSkipException
+from airflow.hooks.subprocess import SubprocessHook
+from airflow.models.baseoperator import BaseOperator
+from airflow.utils.operator_helpers import context_to_airflow_vars
+from airflow.utils.types import ArgNotSet
+
+if TYPE_CHECKING:
+    from airflow.models.taskinstance import TaskInstance
+    from airflow.utils.context import Context
+
+
+class BashOperator(BaseOperator):
+    r"""
+    Execute a Bash script, command or set of commands.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BashOperator`
+
+    If BaseOperator.do_xcom_push is True, the last line written to stdout
+    will also be pushed to an XCom when the bash command completes
+
+    :param bash_command: The command, set of commands or reference to a
+        Bash script (must be '.sh' or '.bash') to be executed. (templated)
+    :param env: If env is not None, it must be a dict that defines the
+        environment variables for the new process; these are used instead
+        of inheriting the current process environment, which is the default
+        behavior. (templated)
+    :param append_env: If False(default) uses the environment variables passed in env params
+        and does not inherit the current process environment. If True, inherits the environment variables
+        from current passes and then environment variable passed by the user will either update the existing
+        inherited environment variables or the new variables gets appended to it
+    :param output_encoding: Output encoding of Bash command
+    :param skip_on_exit_code: If task exits with this exit code, leave the task
+        in ``skipped`` state (default: 99). If set to ``None``, any non-zero
+        exit code will be treated as a failure.
+    :param cwd: Working directory to execute the command in (templated).
+        If None (default), the command is run in a temporary directory.
+        To use current DAG folder as the working directory,
+        you might set template ``{{ dag_run.dag.folder }}``.
+    :param output_processor: Function to further process the output of the bash script
+        (default is lambda output: output).
+
+    Airflow will evaluate the exit code of the Bash command. In general, a non-zero exit code will result in
+    task failure and zero will result in task success.
+    Exit code ``99`` (or another set in ``skip_on_exit_code``)
+    will throw an :class:`airflow.exceptions.AirflowSkipException`, which will leave the task in ``skipped``
+    state. You can have all non-zero exit codes be treated as a failure by setting ``skip_on_exit_code=None``.
+
+    .. list-table::
+       :widths: 25 25
+       :header-rows: 1
+
+       * - Exit code
+         - Behavior
+       * - 0
+         - success
+       * - `skip_on_exit_code` (default: 99)
+         - raise :class:`airflow.exceptions.AirflowSkipException`
+       * - otherwise
+         - raise :class:`airflow.exceptions.AirflowException`
+
+    .. note::
+
+        Airflow will not recognize a non-zero exit code unless the whole shell exit with a non-zero exit
+        code.  This can be an issue if the non-zero exit arises from a sub-command.  The easiest way of
+        addressing this is to prefix the command with ``set -e;``
+
+        .. code-block:: python
+
+            bash_command = "set -e; python3 script.py '{{ next_execution_date }}'"
+
+    .. note::
+
+        Add a space after the script name when directly calling a ``.sh`` script with the
+        ``bash_command`` argument -- for example ``bash_command="my_script.sh "``.  This
+        is because Airflow tries to apply load this file and process it as a Jinja template to
+        it ends with ``.sh``, which will likely not be what most users want.
+
+    .. warning::
+
+        Care should be taken with "user" input or when using Jinja templates in the
+        ``bash_command``, as this bash operator does not perform any escaping or
+        sanitization of the command.
+
+        This applies mostly to using "dag_run" conf, as that can be submitted via
+        users in the Web UI. Most of the default template variables are not at
+        risk.
+
+    For example, do **not** do this:
+
+    .. code-block:: python
+
+        bash_task = BashOperator(
+            task_id="bash_task",
+            bash_command='echo "Here is the message: \'{{ dag_run.conf["message"] if dag_run else "" }}\'"',
+        )
+
+    Instead, you should pass this via the ``env`` kwarg and use double-quotes
+    inside the bash_command, as below:
+
+    .. code-block:: python
+
+        bash_task = BashOperator(
+            task_id="bash_task",
+            bash_command="echo \"here is the message: '$message'\"",
+            env={"message": '{{ dag_run.conf["message"] if dag_run else "" }}'},
+        )
+
+    .. versionadded:: 2.10.0
+       The `output_processor` parameter.
+
+    """
+
+    template_fields: Sequence[str] = ("bash_command", "env", "cwd")
+    template_fields_renderers = {"bash_command": "bash", "env": "json"}
+    template_ext: Sequence[str] = (".sh", ".bash")
+    ui_color = "#f0ede4"
+
+    def __init__(
+        self,
+        *,
+        bash_command: str | ArgNotSet,
+        env: dict[str, str] | None = None,
+        append_env: bool = False,
+        output_encoding: str = "utf-8",
+        skip_exit_code: int | None = None,
+        skip_on_exit_code: int | Container[int] | None = 99,
+        cwd: str | None = None,
+        output_processor: Callable[[str], Any] = lambda result: result,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.bash_command = bash_command
+        self.env = env
+        self.output_encoding = output_encoding
+        if skip_exit_code is not None:
+            warnings.warn(
+                "skip_exit_code is deprecated. Please use skip_on_exit_code", DeprecationWarning, stacklevel=2
+            )
+            skip_on_exit_code = skip_exit_code
+        self.skip_on_exit_code = (
+            skip_on_exit_code
+            if isinstance(skip_on_exit_code, Container)
+            else [skip_on_exit_code]
+            if skip_on_exit_code is not None
+            else []
+        )
+        self.cwd = cwd
+        self.append_env = append_env
+        self.output_processor = output_processor
+
+        # When using the @task.bash decorator, the Bash command is not known until the underlying Python
+        # callable is executed and therefore set to NOTSET initially. This flag is useful during execution to
+        # determine whether the bash_command value needs to re-rendered.
+        self._init_bash_command_not_set = isinstance(self.bash_command, ArgNotSet)
+
+    @cached_property
+    def subprocess_hook(self):
+        """Returns hook for running the bash command."""
+        return SubprocessHook()
+
+    @staticmethod
+    def refresh_bash_command(ti: TaskInstance) -> None:
+        """
+        Rewrite the underlying rendered bash_command value for a task instance in the metadatabase.
+
+        TaskInstance.get_rendered_template_fields() cannot be used because this will retrieve the
+        RenderedTaskInstanceFields from the metadatabase which doesn't have the runtime-evaluated bash_command
+        value.
+
+        :meta private:
+        """
+        from airflow.models.renderedtifields import RenderedTaskInstanceFields
+
+        RenderedTaskInstanceFields._update_runtime_evaluated_template_fields(ti)
+
+    def get_env(self, context):
+        """Build the set of environment variables to be exposed for the bash command."""
+        system_env = os.environ.copy()
+        env = self.env
+        if env is None:
+            env = system_env
+        else:
+            if self.append_env:
+                system_env.update(env)
+                env = system_env
+
+        airflow_context_vars = context_to_airflow_vars(context, in_env_var_format=True)
+        self.log.debug(
+            "Exporting env vars: %s",
+            " ".join(f"{k}={v!r}" for k, v in airflow_context_vars.items()),
+        )
+        env.update(airflow_context_vars)
+        return env
+
+    def execute(self, context: Context):
+        bash_path = shutil.which("bash") or "bash"
+        if self.cwd is not None:
+            if not os.path.exists(self.cwd):
+                raise AirflowException(f"Can not find the cwd: {self.cwd}")
+            if not os.path.isdir(self.cwd):
+                raise AirflowException(f"The cwd {self.cwd} must be a directory")
+        env = self.get_env(context)
+
+        # Because the bash_command value is evaluated at runtime using the @task.bash decorator, the
+        # RenderedTaskInstanceField data needs to be rewritten and the bash_command value re-rendered -- the
+        # latter because the returned command from the decorated callable could contain a Jinja expression.
+        # Both will ensure the correct Bash command is executed and that the Rendered Template view in the UI
+        # displays the executed command (otherwise it will display as an ArgNotSet type).
+        if self._init_bash_command_not_set:
+            ti = cast("TaskInstance", context["ti"])
+            self.refresh_bash_command(ti)
+
+        result = self.subprocess_hook.run_command(
+            command=[bash_path, "-c", self.bash_command],
+            env=env,
+            output_encoding=self.output_encoding,
+            cwd=self.cwd,
+        )
+        if result.exit_code in self.skip_on_exit_code:
+            raise AirflowSkipException(f"Bash command returned exit code {result.exit_code}. Skipping.")
+        elif result.exit_code != 0:
+            raise AirflowException(
+                f"Bash command failed. The command returned a non-zero exit code {result.exit_code}."
+            )
+
+        return self.output_processor(result.output)
+
+    def on_kill(self) -> None:
+        self.subprocess_hook.send_sigterm()

airflow/providers/standard/sensors/bash.py

@@ -0,0 +1,114 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+from subprocess import PIPE, STDOUT, Popen
+from tempfile import NamedTemporaryFile, TemporaryDirectory, gettempdir
+from typing import TYPE_CHECKING, Sequence
+
+from airflow.exceptions import AirflowFailException
+from airflow.sensors.base import BaseSensorOperator
+
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
+
+class BashSensor(BaseSensorOperator):
+    """
+    Executes a bash command/script.
+
+    Return True if and only if the return code is 0.
+
+    :param bash_command: The command, set of commands or reference to a
+        bash script (must be '.sh') to be executed.
+
+    :param env: If env is not None, it must be a mapping that defines the
+        environment variables for the new process; these are used instead
+        of inheriting the current process environment, which is the default
+        behavior. (templated)
+    :param output_encoding: output encoding of bash command.
+    :param retry_exit_code: If task exits with this code, treat the sensor
+        as not-yet-complete and retry the check later according to the
+        usual retry/timeout settings. Any other non-zero return code will
+        be treated as an error, and cause the sensor to fail. If set to
+        ``None`` (the default), any non-zero exit code will cause a retry
+        and the task will never raise an error except on time-out.
+
+    .. seealso::
+        For more information on how to use this sensor,take a look at the guide:
+        :ref:`howto/operator:BashSensor`
+    """
+
+    template_fields: Sequence[str] = ("bash_command", "env")
+
+    def __init__(
+        self, *, bash_command, env=None, output_encoding="utf-8", retry_exit_code: int | None = None, **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.bash_command = bash_command
+        self.env = env
+        self.output_encoding = output_encoding
+        self.retry_exit_code = retry_exit_code
+
+    def poke(self, context: Context):
+        """Execute the bash command in a temporary directory."""
+        bash_command = self.bash_command
+        self.log.info("Tmp dir root location: %s", gettempdir())
+        with TemporaryDirectory(prefix="airflowtmp") as tmp_dir, NamedTemporaryFile(
+            dir=tmp_dir, prefix=self.task_id
+        ) as f:
+            f.write(bytes(bash_command, "utf_8"))
+            f.flush()
+            fname = f.name
+            script_location = tmp_dir + "/" + fname
+            self.log.info("Temporary script location: %s", script_location)
+            self.log.info("Running command: %s", bash_command)
+
+            with Popen(
+                ["bash", fname],
+                stdout=PIPE,
+                stderr=STDOUT,
+                close_fds=True,
+                cwd=tmp_dir,
+                env=self.env,
+                preexec_fn=os.setsid,
+            ) as resp:
+                if resp.stdout:
+                    self.log.info("Output:")
+                    for line in iter(resp.stdout.readline, b""):
+                        self.log.info(line.decode(self.output_encoding).strip())
+                resp.wait()
+                self.log.info("Command exited with return code %s", resp.returncode)
+
+                # zero code means success, the sensor can go green
+                if resp.returncode == 0:
+                    return True
+
+                # we have a retry exit code, sensor retries if return code matches, otherwise error
+                elif self.retry_exit_code is not None:
+                    if resp.returncode == self.retry_exit_code:
+                        self.log.info("Return code matches retry code, will retry later")
+                        return False
+                    else:
+                        raise AirflowFailException(f"Command exited with return code {resp.returncode}")
+
+                # backwards compatibility: sensor retries no matter the error code
+                else:
+                    self.log.info("Non-zero return code and no retry code set, will retry later")
+                    return False

{apache_airflow_providers_standard-1.0.0.dev0.dist-info → apache_airflow_providers_standard-1.0.0.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-standard
-Version: 1.0.0.dev0
+Version: 1.0.0.dev1
 Summary: Provider package apache-airflow-providers-standard for Apache Airflow
 Keywords: airflow-provider,standard,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -74,7 +74,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-standard``
 
-Release: ``1.0.0.dev0``
+Release: ``1.0.0.dev1``
 
 
 Airflow Standard Provider

{apache_airflow_providers_standard-1.0.0.dev0.dist-info → apache_airflow_providers_standard-1.0.0.dev1.dist-info}/RECORD

@@ -1,15 +1,17 @@
 airflow/providers/standard/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
 airflow/providers/standard/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/standard/get_provider_info.py,sha256=YOJ5MXuFZYg9TBhXtStPLjwKRCXwBj5sT1N_QE64i6s,2431
+airflow/providers/standard/get_provider_info.py,sha256=nFqgVL44xY7Xwa-Y2mn8SLMD-NH7lW_aNqgT17f0gLo,2559
 airflow/providers/standard/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/standard/operators/bash.py,sha256=mxHM1Uc57Twh-OS1r3nCX7QKpatsBVb1PJVI3OEGJCk,10801
 airflow/providers/standard/operators/datetime.py,sha256=nk0gwO_H8vIIu8ztA4zryWqZeoSokfINTu4o2vPUcPc,4562
 airflow/providers/standard/operators/weekday.py,sha256=ws1FHHxFVEZhq8MDfBvNmI9pjH5QcHqueRkuanhayeQ,4474
 airflow/providers/standard/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/standard/sensors/bash.py,sha256=Gyi8zMwSESPyanlG9jbN9u-an6Vj33s7lTlmjWmjgSA,4821
 airflow/providers/standard/sensors/date_time.py,sha256=WKoMTLuSYwNbsvvkndRmiTGa6GN3jeolAGazbPDMeUk,5179
 airflow/providers/standard/sensors/time.py,sha256=nhKqn7eQKnx-F-MSG_yrlN7RM8ZM3iOSUwgqxc_F2SQ,4142
 airflow/providers/standard/sensors/time_delta.py,sha256=YMNELt0m89PvWsYmWWO-m6RW-ICPOZ8rcABrmt-4xxc,4660
 airflow/providers/standard/sensors/weekday.py,sha256=PMg0eoGuD0xNLSJIBY9C1Y0aqSZn6PkQ_j_eYo7lnks,3695
-apache_airflow_providers_standard-1.0.0.dev0.dist-info/entry_points.txt,sha256=mW2YRh3mVdZdaP5-iGSNgmcCh3YYdALIn28BCLBZZ40,104
-apache_airflow_providers_standard-1.0.0.dev0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_standard-1.0.0.dev0.dist-info/METADATA,sha256=unzipTVuHhQdPFzlb3VhMm9OXHFvX72zrRwrk9NUev0,4698
-apache_airflow_providers_standard-1.0.0.dev0.dist-info/RECORD,,
+apache_airflow_providers_standard-1.0.0.dev1.dist-info/entry_points.txt,sha256=mW2YRh3mVdZdaP5-iGSNgmcCh3YYdALIn28BCLBZZ40,104
+apache_airflow_providers_standard-1.0.0.dev1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_standard-1.0.0.dev1.dist-info/METADATA,sha256=Q_aHdns_3O3-ukEP8lq8HJvSd9QZ0ENFnK-MSDOO_1g,4698
+apache_airflow_providers_standard-1.0.0.dev1.dist-info/RECORD,,