apache-airflow-providers-standard 1.0.0.dev1__py3-none-any.whl → 1.1.0rc1__py3-none-any.whl

airflow/providers/standard/decorators/sensor.py

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Callable, ClassVar
+
+from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
+
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.sdk.bases.decorator import get_unique_task_id, task_decorator_factory
+else:
+    from airflow.decorators.base import get_unique_task_id, task_decorator_factory  # type: ignore[no-redef]
+
+
+from airflow.providers.standard.sensors.python import PythonSensor
+
+if TYPE_CHECKING:
+    from airflow.sdk.bases.decorator import TaskDecorator
+
+
+class DecoratedSensorOperator(PythonSensor):
+    """
+    Wraps a Python callable and captures args/kwargs when called for execution.
+
+    :param python_callable: A reference to an object that is callable
+    :param task_id: task Id
+    :param op_args: a list of positional arguments that will get unpacked when
+        calling your callable (templated)
+    :param op_kwargs: a dictionary of keyword arguments that will get unpacked
+        in your function (templated)
+    :param kwargs_to_upstream: For certain operators, we might need to upstream certain arguments
+        that would otherwise be absorbed by the DecoratedOperator (for example python_callable for the
+        PythonOperator). This gives a user the option to upstream kwargs as needed.
+    """
+
+    template_fields: Sequence[str] = ("op_args", "op_kwargs")
+    template_fields_renderers: ClassVar[dict[str, str]] = {"op_args": "py", "op_kwargs": "py"}
+
+    custom_operator_name = "@task.sensor"
+
+    # since we won't mutate the arguments, we should just do the shallow copy
+    # there are some cases we can't deepcopy the objects (e.g protobuf).
+    shallow_copy_attrs: Sequence[str] = ("python_callable",)
+
+    def __init__(
+        self,
+        *,
+        task_id: str,
+        **kwargs,
+    ) -> None:
+        kwargs["task_id"] = get_unique_task_id(task_id, kwargs.get("dag"), kwargs.get("task_group"))
+        super().__init__(**kwargs)
+
+
+def sensor_task(python_callable: Callable | None = None, **kwargs) -> TaskDecorator:
+    """
+    Wrap a function into an Airflow operator.
+
+    Accepts kwargs for operator kwarg. Can be reused in a single DAG.
+    :param python_callable: Function to decorate
+    """
+    return task_decorator_factory(
+        python_callable=python_callable,
+        multiple_outputs=False,
+        decorated_operator_class=DecoratedSensorOperator,
+        **kwargs,
+    )

airflow/providers/standard/decorators/short_circuit.py

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable
+
+from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
+
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.sdk.bases.decorator import task_decorator_factory
+else:
+    from airflow.decorators.base import task_decorator_factory  # type: ignore[no-redef]
+
+from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
+from airflow.providers.standard.operators.python import ShortCircuitOperator
+
+if TYPE_CHECKING:
+    from airflow.sdk.bases.decorator import TaskDecorator
+
+
+class _ShortCircuitDecoratedOperator(_PythonDecoratedOperator, ShortCircuitOperator):
+    """Wraps a Python callable and captures args/kwargs when called for execution."""
+
+    template_fields = ShortCircuitOperator.template_fields
+    custom_operator_name: str = "@task.short_circuit"
+
+
+def short_circuit_task(
+    python_callable: Callable | None = None,
+    multiple_outputs: bool | None = None,
+    **kwargs,
+) -> TaskDecorator:
+    """
+    Wrap a function into an ShortCircuitOperator.
+
+    Accepts kwargs for operator kwarg. Can be reused in a single DAG.
+
+    This function is only used only used during type checking or auto-completion.
+
+    :param python_callable: Function to decorate
+    :param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
+        multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys. Defaults to False.
+
+    :meta private:
+    """
+    return task_decorator_factory(
+        python_callable=python_callable,
+        multiple_outputs=multiple_outputs,
+        decorated_operator_class=_ShortCircuitDecoratedOperator,
+        **kwargs,
+    )

airflow/providers/standard/get_provider_info.py

@@ -15,8 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 #
 # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
 # `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -27,16 +26,23 @@ def get_provider_info():
         "package-name": "apache-airflow-providers-standard",
         "name": "Standard",
         "description": "Airflow Standard Provider\n",
-        "state": "not-ready",
-        "source-date-epoch": 1718603992,
-        "versions": ["1.0.0"],
-        "dependencies": ["apache-airflow>=2.10.0"],
         "integrations": [
             {
                 "integration-name": "Standard",
                 "external-doc-url": "https://airflow.apache.org/",
                 "tags": ["apache"],
-                "how-to-guide": ["/docs/apache-airflow-providers-standard/operators.rst"],
+                "how-to-guide": [
+                    "/docs/apache-airflow-providers-standard/operators/bash.rst",
+                    "/docs/apache-airflow-providers-standard/operators/python.rst",
+                    "/docs/apache-airflow-providers-standard/operators/datetime.rst",
+                    "/docs/apache-airflow-providers-standard/operators/trigger_dag_run.rst",
+                    "/docs/apache-airflow-providers-standard/operators/latest_only.rst",
+                    "/docs/apache-airflow-providers-standard/sensors/bash.rst",
+                    "/docs/apache-airflow-providers-standard/sensors/python.rst",
+                    "/docs/apache-airflow-providers-standard/sensors/datetime.rst",
+                    "/docs/apache-airflow-providers-standard/sensors/file.rst",
+                    "/docs/apache-airflow-providers-standard/sensors/external_task_sensor.rst",
+                ],
             }
         ],
         "operators": [
@@ -46,6 +52,12 @@ def get_provider_info():
                     "airflow.providers.standard.operators.datetime",
                     "airflow.providers.standard.operators.weekday",
                     "airflow.providers.standard.operators.bash",
+                    "airflow.providers.standard.operators.python",
+                    "airflow.providers.standard.operators.empty",
+                    "airflow.providers.standard.operators.trigger_dagrun",
+                    "airflow.providers.standard.operators.latest_only",
+                    "airflow.providers.standard.operators.smooth",
+                    "airflow.providers.standard.operators.branch",
                 ],
             }
         ],
@@ -58,7 +70,77 @@ def get_provider_info():
                     "airflow.providers.standard.sensors.time",
                     "airflow.providers.standard.sensors.weekday",
                     "airflow.providers.standard.sensors.bash",
+                    "airflow.providers.standard.sensors.python",
+                    "airflow.providers.standard.sensors.filesystem",
+                    "airflow.providers.standard.sensors.external_task",
+                ],
+            }
+        ],
+        "hooks": [
+            {
+                "integration-name": "Standard",
+                "python-modules": [
+                    "airflow.providers.standard.hooks.filesystem",
+                    "airflow.providers.standard.hooks.package_index",
+                    "airflow.providers.standard.hooks.subprocess",
+                ],
+            }
+        ],
+        "triggers": [
+            {
+                "integration-name": "Standard",
+                "python-modules": [
+                    "airflow.providers.standard.triggers.external_task",
+                    "airflow.providers.standard.triggers.file",
+                    "airflow.providers.standard.triggers.temporal",
                 ],
             }
         ],
+        "extra-links": [
+            "airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunLink",
+            "airflow.providers.standard.sensors.external_task.ExternalDagLink",
+        ],
+        "config": {
+            "standard": {
+                "description": "Options for the standard provider operators.",
+                "options": {
+                    "venv_install_method": {
+                        "description": "Which python tooling should be used to install the virtual environment.\n\nThe following options are available:\n- ``auto``: Automatically select, use ``uv`` if available, otherwise use ``pip``.\n- ``pip``: Use pip to install the virtual environment.\n- ``uv``: Use uv to install the virtual environment. Must be available in environment PATH.\n",
+                        "version_added": None,
+                        "type": "string",
+                        "example": "uv",
+                        "default": "auto",
+                    }
+                },
+            }
+        },
+        "task-decorators": [
+            {"class-name": "airflow.providers.standard.decorators.python.python_task", "name": "python"},
+            {"class-name": "airflow.providers.standard.decorators.bash.bash_task", "name": "bash"},
+            {
+                "class-name": "airflow.providers.standard.decorators.branch_external_python.branch_external_python_task",
+                "name": "branch_external_python",
+            },
+            {
+                "class-name": "airflow.providers.standard.decorators.branch_python.branch_task",
+                "name": "branch",
+            },
+            {
+                "class-name": "airflow.providers.standard.decorators.branch_virtualenv.branch_virtualenv_task",
+                "name": "branch_virtualenv",
+            },
+            {
+                "class-name": "airflow.providers.standard.decorators.external_python.external_python_task",
+                "name": "external_python",
+            },
+            {
+                "class-name": "airflow.providers.standard.decorators.python_virtualenv.virtualenv_task",
+                "name": "virtualenv",
+            },
+            {"class-name": "airflow.providers.standard.decorators.sensor.sensor_task", "name": "sensor"},
+            {
+                "class-name": "airflow.providers.standard.decorators.short_circuit.short_circuit_task",
+                "name": "short_circuit",
+            },
+        ],
     }

airflow/providers/standard/hooks/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/standard/hooks/filesystem.py

@@ -0,0 +1,89 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from airflow.hooks.base import BaseHook
+
+
+class FSHook(BaseHook):
+    """
+    Allows for interaction with an file server.
+
+    Connection should have a name and a path specified under extra:
+
+    example:
+    Connection Id: fs_test
+    Connection Type: File (path)
+    Host, Schema, Login, Password, Port: empty
+    Extra: {"path": "/tmp"}
+    """
+
+    conn_name_attr = "fs_conn_id"
+    default_conn_name = "fs_default"
+    conn_type = "fs"
+    hook_name = "File (path)"
+
+    @classmethod
+    def get_connection_form_widgets(cls) -> dict[str, Any]:
+        """Return connection widgets to add to connection form."""
+        from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
+        from flask_babel import lazy_gettext
+        from wtforms import StringField
+
+        return {"path": StringField(lazy_gettext("Path"), widget=BS3TextFieldWidget())}
+
+    @classmethod
+    def get_ui_field_behaviour(cls) -> dict[str, Any]:
+        """Return custom field behaviour."""
+        return {
+            "hidden_fields": ["host", "schema", "port", "login", "password", "extra"],
+            "relabeling": {},
+            "placeholders": {},
+        }
+
+    def __init__(self, fs_conn_id: str = default_conn_name, **kwargs):
+        super().__init__(**kwargs)
+        conn = self.get_connection(fs_conn_id)
+        self.basepath = conn.extra_dejson.get("path", "")
+        self.conn = conn
+
+    def get_conn(self) -> None:
+        pass
+
+    def get_path(self) -> str:
+        """
+        Get the path to the filesystem location.
+
+        :return: the path.
+        """
+        return self.basepath
+
+    def test_connection(self):
+        """Test File connection."""
+        try:
+            p = self.get_path()
+            if not p:
+                return False, "File Path is undefined."
+            if not Path(p).exists():
+                return False, f"Path {p} does not exist."
+            return True, f"Path {p} is existing."
+        except Exception as e:
+            return False, str(e)

airflow/providers/standard/hooks/package_index.py

@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Hook for additional Package Indexes (Python)."""
+
+from __future__ import annotations
+
+import subprocess
+from typing import Any
+from urllib.parse import quote, urlparse
+
+from airflow.hooks.base import BaseHook
+
+
+class PackageIndexHook(BaseHook):
+    """Specify package indexes/Python package sources using Airflow connections."""
+
+    conn_name_attr = "pi_conn_id"
+    default_conn_name = "package_index_default"
+    conn_type = "package_index"
+    hook_name = "Package Index (Python)"
+
+    def __init__(self, pi_conn_id: str = default_conn_name, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.pi_conn_id = pi_conn_id
+        self.conn = None
+
+    @staticmethod
+    def get_ui_field_behaviour() -> dict[str, Any]:
+        """Return custom field behaviour."""
+        return {
+            "hidden_fields": ["schema", "port", "extra"],
+            "relabeling": {"host": "Package Index URL"},
+            "placeholders": {
+                "host": "Example: https://my-package-mirror.net/pypi/repo-name/simple",
+                "login": "Username for package index",
+                "password": "Password for package index (will be masked)",
+            },
+        }
+
+    @staticmethod
+    def _get_basic_auth_conn_url(index_url: str, user: str | None, password: str | None) -> str:
+        """Return a connection URL with basic auth credentials based on connection config."""
+        url = urlparse(index_url)
+        host = url.netloc.split("@")[-1]
+        if user:
+            if password:
+                host = f"{quote(user)}:{quote(password)}@{host}"
+            else:
+                host = f"{quote(user)}@{host}"
+        return url._replace(netloc=host).geturl()
+
+    def get_conn(self) -> Any:
+        """Return connection for the hook."""
+        return self.get_connection_url()
+
+    def get_connection_url(self) -> Any:
+        """Return a connection URL with embedded credentials."""
+        conn = self.get_connection(self.pi_conn_id)
+        index_url = conn.host
+        if not index_url:
+            raise ValueError("Please provide an index URL.")
+        return self._get_basic_auth_conn_url(index_url, conn.login, conn.password)
+
+    def test_connection(self) -> tuple[bool, str]:
+        """Test connection to package index url."""
+        conn_url = self.get_connection_url()
+        proc = subprocess.run(
+            ["pip", "search", "not-existing-test-package", "--no-input", "--index", conn_url],
+            check=False,
+            capture_output=True,
+        )
+        conn = self.get_connection(self.pi_conn_id)
+        if proc.returncode not in [
+            0,  # executed successfully, found package
+            23,  # executed successfully, didn't find any packages
+            #      (but we do not expect it to find 'not-existing-test-package')
+        ]:
+            return False, f"Connection test to {conn.host} failed. Error: {str(proc.stderr)}"
+
+        return True, f"Connection to {conn.host} tested successfully!"

airflow/providers/standard/hooks/subprocess.py

@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import contextlib
+import os
+import signal
+from collections import namedtuple
+from collections.abc import Iterator
+from subprocess import PIPE, STDOUT, Popen
+from tempfile import TemporaryDirectory, gettempdir
+
+from airflow.hooks.base import BaseHook
+
+SubprocessResult = namedtuple("SubprocessResult", ["exit_code", "output"])
+
+
+@contextlib.contextmanager
+def working_directory(cwd: str | None = None) -> Iterator[str]:
+    """
+    Context manager for handling (temporary) working directory.
+
+    Use the given cwd as working directory, if provided.
+    Otherwise, create a temporary directory.
+    """
+    with contextlib.ExitStack() as stack:
+        if cwd is None:
+            cwd = stack.enter_context(TemporaryDirectory(prefix="airflowtmp"))
+        yield cwd
+
+
+class SubprocessHook(BaseHook):
+    """Hook for running processes with the ``subprocess`` module."""
+
+    def __init__(self, **kwargs) -> None:
+        self.sub_process: Popen[bytes] | None = None
+        super().__init__(**kwargs)
+
+    def run_command(
+        self,
+        command: list[str],
+        env: dict[str, str] | None = None,
+        output_encoding: str = "utf-8",
+        cwd: str | None = None,
+    ) -> SubprocessResult:
+        """
+        Execute the command.
+
+        If ``cwd`` is None, execute the command in a temporary directory which will be cleaned afterwards.
+        If ``env`` is not supplied, ``os.environ`` is passed
+
+        :param command: the command to run
+        :param env: Optional dict containing environment variables to be made available to the shell
+            environment in which ``command`` will be executed.  If omitted, ``os.environ`` will be used.
+            Note, that in case you have Sentry configured, original variables from the environment
+            will also be passed to the subprocess with ``SUBPROCESS_`` prefix. See:
+            https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/errors.html for details.
+        :param output_encoding: encoding to use for decoding stdout
+        :param cwd: Working directory to run the command in.
+            If None (default), the command is run in a temporary directory.
+        :return: :class:`namedtuple` containing ``exit_code`` and ``output``, the last line from stderr
+            or stdout
+        """
+        self.log.info("Tmp dir root location: %s", gettempdir())
+        with working_directory(cwd=cwd) as cwd:
+
+            def pre_exec():
+                # Restore default signal disposition and invoke setsid
+                for sig in ("SIGPIPE", "SIGXFZ", "SIGXFSZ"):
+                    if hasattr(signal, sig):
+                        signal.signal(getattr(signal, sig), signal.SIG_DFL)
+                os.setsid()
+
+            self.log.info("Running command: %s", command)
+
+            self.sub_process = Popen(
+                command,
+                stdout=PIPE,
+                stderr=STDOUT,
+                cwd=cwd,
+                env=env if env or env == {} else os.environ,
+                preexec_fn=pre_exec,
+            )
+
+            self.log.info("Output:")
+            line = ""
+            if self.sub_process is None:
+                raise RuntimeError("The subprocess should be created here and is None!")
+            if self.sub_process.stdout is not None:
+                for raw_line in iter(self.sub_process.stdout.readline, b""):
+                    line = raw_line.decode(output_encoding, errors="backslashreplace").rstrip()
+                    self.log.info("%s", line)
+
+            self.sub_process.wait()
+
+            self.log.info("Command exited with return code %s", self.sub_process.returncode)
+            return_code: int = self.sub_process.returncode
+
+        return SubprocessResult(exit_code=return_code, output=line)
+
+    def send_sigterm(self):
+        """Send SIGTERM signal to ``self.sub_process`` if one exists."""
+        self.log.info("Sending SIGTERM signal to process group")
+        if self.sub_process and hasattr(self.sub_process, "pid"):
+            os.killpg(os.getpgid(self.sub_process.pid), signal.SIGTERM)