apache-airflow-providers-sftp 5.5.1rc1__tar.gz → 5.7.0__tar.gz

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/NOTICE

@@ -1,5 +1,5 @@
 Apache Airflow
-Copyright 2016-2025 The Apache Software Foundation
+Copyright 2016-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-sftp
-Version: 5.5.1rc1
+Version: 5.7.0
 Summary: Provider package apache-airflow-providers-sftp for Apache Airflow
 Keywords: airflow-provider,sftp,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,22 +22,22 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
 License-File: LICENSE
 License-File: NOTICE
-Requires-Dist: apache-airflow>=2.11.0rc1
-Requires-Dist: apache-airflow-providers-ssh>=4.0.0rc1
-Requires-Dist: apache-airflow-providers-common-compat>=1.10.1rc1
+Requires-Dist: apache-airflow>=2.11.0
+Requires-Dist: apache-airflow-providers-ssh>=4.0.0
+Requires-Dist: apache-airflow-providers-common-compat>=1.12.0
 Requires-Dist: paramiko>=2.9.0,<4.0.0
 Requires-Dist: asyncssh>=2.12.0
-Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
+Requires-Dist: sshfs>=2023.1.0 ; extra == "sshfs"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-sftp/5.5.1/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-sftp/5.5.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
-Provides-Extra: common-compat
 Provides-Extra: openlineage
+Provides-Extra: sshfs
 
 
 .. Licensed to the Apache Software Foundation (ASF) under one
@@ -64,7 +64,7 @@ Provides-Extra: openlineage
 
 Package ``apache-airflow-providers-sftp``
 
-Release: ``5.5.1``
+Release: ``5.7.0``
 
 
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
@@ -77,7 +77,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
 are in ``airflow.providers.sftp`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.5.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0/>`_.
 
 Installation
 ------------
@@ -96,7 +96,7 @@ PIP package                                 Version required
 ==========================================  ==================
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-ssh``            ``>=4.0.0``
-``apache-airflow-providers-common-compat``  ``>=1.10.1``
+``apache-airflow-providers-common-compat``  ``>=1.12.0``
 ``paramiko``                                ``>=2.9.0,<4.0.0``
 ``asyncssh``                                ``>=2.12.0``
 ==========================================  ==================
@@ -125,13 +125,13 @@ Dependent package
 Optional dependencies
 ----------------------
 
-=================  ==========================================
-Extra              Dependencies
-=================  ==========================================
-``common.compat``  ``apache-airflow-providers-common-compat``
-``openlineage``    ``apache-airflow-providers-openlineage``
-=================  ==========================================
+===============  ========================================
+Extra            Dependencies
+===============  ========================================
+``openlineage``  ``apache-airflow-providers-openlineage``
+``sshfs``        ``sshfs>=2023.1.0``
+===============  ========================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.5.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0/changelog.html>`_.
 

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-sftp``
 
-Release: ``5.5.1``
+Release: ``5.7.0``
 
 
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
@@ -36,7 +36,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
 are in ``airflow.providers.sftp`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.5.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0/>`_.
 
 Installation
 ------------
@@ -55,7 +55,7 @@ PIP package                                 Version required
 ==========================================  ==================
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-ssh``            ``>=4.0.0``
-``apache-airflow-providers-common-compat``  ``>=1.10.1``
+``apache-airflow-providers-common-compat``  ``>=1.12.0``
 ``paramiko``                                ``>=2.9.0,<4.0.0``
 ``asyncssh``                                ``>=2.12.0``
 ==========================================  ==================
@@ -84,12 +84,12 @@ Dependent package
 Optional dependencies
 ----------------------
 
-=================  ==========================================
-Extra              Dependencies
-=================  ==========================================
-``common.compat``  ``apache-airflow-providers-common-compat``
-``openlineage``    ``apache-airflow-providers-openlineage``
-=================  ==========================================
+===============  ========================================
+Extra            Dependencies
+===============  ========================================
+``openlineage``  ``apache-airflow-providers-openlineage``
+``sshfs``        ``sshfs>=2023.1.0``
+===============  ========================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.5.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0/changelog.html>`_.

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/docs/changelog.rst

@@ -27,6 +27,40 @@
 Changelog
 ---------
 
+5.7.0
+.....
+
+Features
+~~~~~~~~
+
+* ``Add SSH/SFTP support for ObjectStoragePath (#60757)``
+
+Misc
+~~~~
+
+* ``Use common provider's get_async_connection in other providers (#56791)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+
+5.6.0
+.....
+
+Features
+~~~~~~~~
+
+* ``Enhance SFTPSensor documentation and functionality (#60313)``
+
+Misc
+~~~~
+
+* ``New year means updated Copyright notices (#60344)``
+* ``Migrate sftp provider to use airflow.sdk.configuration.conf (#59970)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``TaskInstance unused method cleanup (#59835)``
+
 5.5.1
 .....
 

apache_airflow_providers_sftp-5.7.0/docs/filesystems/index.rst

@@ -0,0 +1,26 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Filesystems
+===========
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Filesystem Providers
+    :glob:
+
+    *

apache_airflow_providers_sftp-5.7.0/docs/filesystems/sftp.rst

@@ -0,0 +1,63 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+SFTP Filesystem
+===============
+
+Use ``ObjectStoragePath`` with SFTP/SSH servers via the `sshfs <https://github.com/fsspec/sshfs>`__ library.
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-sftp[sshfs]
+
+URL format: ``sftp://connection_id@hostname/path/to/file`` (also supports ``ssh://``).
+
+Configuration
+-------------
+
+Uses the standard SFTP connection. The following extras are supported:
+
+* ``key_file`` - path to private key file
+* ``private_key`` - private key content (PEM format)
+* ``private_key_passphrase`` - passphrase for encrypted keys
+* ``no_host_key_check`` - set to ``true`` to skip host key verification
+
+See :doc:`/connections/sftp` for details.
+
+Example
+-------
+
+.. code-block:: python
+
+    from airflow.sdk import ObjectStoragePath
+
+    path = ObjectStoragePath("sftp://my_conn@myserver/data/file.csv")
+
+    # read
+    with path.open() as f:
+        data = f.read()
+
+    # write
+    with path.open("w") as f:
+        f.write("content")
+
+    # list
+    for p in path.parent.iterdir():
+        print(p.name)
+
+    # copy
+    path.copy(ObjectStoragePath("file:///tmp/local.csv"))

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/docs/index.rst

@@ -29,6 +29,13 @@
     Changelog <changelog>
     Security <security>
 
+.. toctree::
+    :hidden:
+    :maxdepth: 1
+    :caption: Guides
+
+    Filesystems <filesystems/index>
+
 .. toctree::
     :hidden:
     :maxdepth: 1
@@ -70,7 +77,7 @@ apache-airflow-providers-sftp package
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
 
 
-Release: 5.5.1
+Release: 5.7.0
 
 Provider package
 ----------------
@@ -95,7 +102,7 @@ PIP package                                 Version required
 ==========================================  ==================
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-ssh``            ``>=4.0.0``
-``apache-airflow-providers-common-compat``  ``>=1.10.1``
+``apache-airflow-providers-common-compat``  ``>=1.12.0``
 ``paramiko``                                ``>=2.9.0,<4.0.0``
 ``asyncssh``                                ``>=2.12.0``
 ==========================================  ==================
@@ -127,5 +134,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-sftp 5.5.1 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.5.1.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.5.1.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.5.1.tar.gz.sha512>`__)
-* `The apache-airflow-providers-sftp 5.5.1 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.5.1-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.5.1-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.5.1-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-sftp 5.7.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.7.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.7.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.7.0.tar.gz.sha512>`__)
+* `The apache-airflow-providers-sftp 5.7.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.7.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.7.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_sftp-5.7.0-py3-none-any.whl.sha512>`__)

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/docs/sensors/sftp_sensor.rst

@@ -30,14 +30,26 @@ To get more information about this sensor visit :class:`~airflow.providers.sftp.
 
 We can also use TaskFlow API. It takes the same arguments as the :class:`~airflow.providers.sftp.sensors.sftp.SFTPSensor` along with -
 
+python_callable (optional)
+    A callable that will be executed after files matching the sensor criteria are found.
+    This allows you to process the found files with custom logic. The callable receives:
+
+    - Positional arguments from ``op_args``
+    - Keyword arguments from ``op_kwargs``, with ``files_found`` automatically added
+      (if ``op_kwargs`` is provided and not empty) containing the list of files that matched
+      the sensor criteria
+
+    The return value of the callable is stored in XCom along with the ``files_found`` list,
+    accessible via ``{"files_found": [...], "decorator_return_value": <callable_return_value>}``.
+
 op_args (optional)
-    A list of positional arguments that will get unpacked when
-    calling your callable (templated)
-op_kwargs (optional)
-    A dictionary of keyword arguments that will get unpacked
-    in your function (templated)
+    A list of positional arguments that will get unpacked when calling your callable (templated).
+    Only used when ``python_callable`` is provided.
 
-Whatever returned by the python callable is put into XCom.
+op_kwargs (optional)
+    A dictionary of keyword arguments that will get unpacked in your function (templated).
+    If provided and not empty, the ``files_found`` list is automatically added to this dictionary
+    when the callable is invoked. Only used when ``python_callable`` is provided.
 
 .. exampleinclude:: /../../sftp/tests/system/sftp/example_sftp_sensor.py
     :language: python

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/provider.yaml

@@ -22,12 +22,14 @@ description: |
     `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
 
 state: ready
-source-date-epoch: 1765299399
+source-date-epoch: 1769461567
 # Note that those versions are maintained by release manager - do not update them manually
 # with the exception of case where other provider in sources has >= new provider version.
 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
 # to be done in the same PR
 versions:
+  - 5.7.0
+  - 5.6.0
   - 5.5.1
   - 5.5.0
   - 5.4.2
@@ -122,3 +124,6 @@ triggers:
   - integration-name: SSH File Transfer Protocol (SFTP)
     python-modules:
       - airflow.providers.sftp.triggers.sftp
+
+filesystems:
+  - airflow.providers.sftp.fs.sftp

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-sftp"
-version = "5.5.1rc1"
+version = "5.7.0"
 description = "Provider package apache-airflow-providers-sftp for Apache Airflow"
 readme = "README.rst"
 license = "Apache-2.0"
@@ -58,9 +58,9 @@ requires-python = ">=3.10"
 # Make sure to run ``prek update-providers-dependencies --all-files``
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
-    "apache-airflow>=2.11.0rc1",
-    "apache-airflow-providers-ssh>=4.0.0rc1",
-    "apache-airflow-providers-common-compat>=1.10.1rc1",
+    "apache-airflow>=2.11.0",
+    "apache-airflow-providers-ssh>=4.0.0",
+    "apache-airflow-providers-common-compat>=1.12.0",
     # TODO: Bump to >= 4.0.0 once https://github.com/apache/airflow/issues/54079 is handled
     "paramiko>=2.9.0,<4.0.0",
     "asyncssh>=2.12.0",
@@ -69,12 +69,12 @@ dependencies = [
 # The optional dependencies should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 [project.optional-dependencies]
-"common.compat" = [
-    "apache-airflow-providers-common-compat"
-]
 "openlineage" = [
     "apache-airflow-providers-openlineage"
 ]
+"sshfs" = [
+    "sshfs>=2023.1.0",
+]
 
 [dependency-groups]
 dev = [
@@ -113,8 +113,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-sftp/5.5.1"
-"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-sftp/5.5.1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.7.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/src/airflow/providers/sftp/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "5.5.1"
+__version__ = "5.7.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.11.0"

apache_airflow_providers_sftp-5.7.0/src/airflow/providers/sftp/fs/sftp.py

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from airflow.sdk.bases.hook import BaseHook
+
+if TYPE_CHECKING:
+    from fsspec import AbstractFileSystem
+
+schemes = ["sftp", "ssh"]
+
+
+def get_fs(conn_id: str | None, storage_options: dict[str, Any] | None = None) -> AbstractFileSystem:
+    try:
+        from sshfs import SSHFileSystem
+    except ImportError:
+        raise ImportError(
+            "Airflow FS SFTP/SSH protocol requires the sshfs library. "
+            "Install with: pip install apache-airflow-providers-sftp[sshfs]"
+        )
+
+    if conn_id is None:
+        return SSHFileSystem(**(storage_options or {}))
+
+    conn = BaseHook.get_connection(conn_id)
+    extras = conn.extra_dejson
+
+    options: dict[str, Any] = {
+        "host": conn.host,
+        "port": conn.port or 22,
+        "username": conn.login,
+    }
+
+    if conn.password:
+        options["password"] = conn.password
+
+    if key_file := extras.get("key_file"):
+        options["client_keys"] = [key_file]
+
+    if private_key := extras.get("private_key"):
+        options["client_keys"] = [private_key]
+        if passphrase := extras.get("private_key_passphrase"):
+            options["passphrase"] = passphrase
+
+    if str(extras.get("no_host_key_check", "")).lower() == "true":
+        options["known_hosts"] = None
+
+    options.update(storage_options or {})
+    return SSHFileSystem(**options)

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/src/airflow/providers/sftp/get_provider_info.py

@@ -71,4 +71,5 @@ def get_provider_info():
                 "python-modules": ["airflow.providers.sftp.triggers.sftp"],
             }
         ],
+        "filesystems": ["airflow.providers.sftp.fs.sftp"],
     }

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/src/airflow/providers/sftp/hooks/sftp.py

@@ -33,10 +33,10 @@ from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any, cast
 
 import asyncssh
-from asgiref.sync import sync_to_async
 from paramiko.config import SSH_PORT
 
 from airflow.exceptions import AirflowProviderDeprecationWarning
+from airflow.providers.common.compat.connection import get_async_connection
 from airflow.providers.common.compat.sdk import AirflowException, BaseHook, Connection
 from airflow.providers.sftp.exceptions import ConnectionNotOpenedException
 from airflow.providers.ssh.hooks.ssh import SSHHook
@@ -756,7 +756,7 @@ class SFTPHookAsync(BaseHook):
         - known_hosts
         - passphrase
         """
-        conn = await sync_to_async(self.get_connection)(self.sftp_conn_id)
+        conn = await get_async_connection(self.sftp_conn_id)
         if conn.extra is not None:
             self._parse_extras(conn)  # type: ignore[arg-type]
 

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/src/airflow/providers/sftp/sensors/sftp.py

@@ -26,8 +26,7 @@ from typing import TYPE_CHECKING, Any
 
 from paramiko.sftp import SFTP_NO_SUCH_FILE
 
-from airflow.configuration import conf
-from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator, PokeReturnValue
+from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator, PokeReturnValue, conf
 from airflow.providers.sftp.hooks.sftp import SFTPHook
 from airflow.providers.sftp.triggers.sftp import SFTPTrigger
 from airflow.utils.timezone import convert_to_utc, parse
@@ -44,6 +43,15 @@ class SFTPSensor(BaseSensorOperator):
     :param file_pattern: The pattern that will be used to match the file (fnmatch format)
     :param sftp_conn_id: The connection to run the sensor against
     :param newer_than: DateTime for which the file or file path should be newer than, comparison is inclusive
+    :param python_callable: Optional callable that will be called after files are found. The callable
+        will receive the found files list in ``op_kwargs['files_found']`` if ``op_kwargs`` is provided
+        and not empty. The return value of the callable will be stored in XCom along with the
+        files_found list.
+    :param op_args: A list of positional arguments that will get unpacked when calling your callable
+        (templated). Only used when ``python_callable`` is provided.
+    :param op_kwargs: A dictionary of keyword arguments that will get unpacked in your callable
+        (templated). If provided and not empty, the ``files_found`` list will be automatically added
+        to this dictionary. Only used when ``python_callable`` is provided.
     :param deferrable: If waiting for completion, whether to defer the task until done, default is ``False``.
     """
 

apache_airflow_providers_sftp-5.7.0/tests/unit/sftp/fs/test_sftp.py

@@ -0,0 +1,222 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+
+pytest.importorskip("sshfs")
+
+TEST_CONN_ID = "sftp_test_conn"
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _setup_connections():
+    with pytest.MonkeyPatch.context() as mp_ctx:
+        mp_ctx.setenv(
+            f"AIRFLOW_CONN_{TEST_CONN_ID}".upper(),
+            "sftp://testuser:testpass@testhost:2222",
+        )
+        yield
+
+
+class TestSftpFilesystem:
+    def test_schemes(self):
+        from airflow.providers.sftp.fs.sftp import schemes
+
+        assert "sftp" in schemes
+        assert "ssh" in schemes
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_with_connection(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        get_fs(conn_id=TEST_CONN_ID)
+
+        mock_sshfs.assert_called_once()
+        call_kwargs = mock_sshfs.call_args.kwargs
+        assert call_kwargs["host"] == "testhost"
+        assert call_kwargs["port"] == 2222
+        assert call_kwargs["username"] == "testuser"
+        assert call_kwargs["password"] == "testpass"
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_without_connection(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        # When conn_id is None, storage_options are passed directly to SSHFileSystem
+        storage_options = {"host": "manual-host", "username": "manual-user"}
+        get_fs(conn_id=None, storage_options=storage_options)
+
+        mock_sshfs.assert_called_once()
+        call_kwargs = mock_sshfs.call_args.kwargs
+        assert call_kwargs["host"] == "manual-host"
+        assert call_kwargs["username"] == "manual-user"
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_storage_options_merge(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        storage_options = {"custom_option": "custom_value"}
+        get_fs(conn_id=TEST_CONN_ID, storage_options=storage_options)
+
+        mock_sshfs.assert_called_once()
+        call_kwargs = mock_sshfs.call_args.kwargs
+        assert call_kwargs["custom_option"] == "custom_value"
+        assert call_kwargs["host"] == "testhost"
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_storage_options_override(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        storage_options = {"port": 3333}
+        get_fs(conn_id=TEST_CONN_ID, storage_options=storage_options)
+
+        mock_sshfs.assert_called_once()
+        call_kwargs = mock_sshfs.call_args.kwargs
+        assert call_kwargs["port"] == 3333
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_with_key_file(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_KEY_FILE",
+                "sftp://testuser@testhost?key_file=%2Fpath%2Fto%2Fkey",
+            )
+
+            get_fs(conn_id="sftp_key_file")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert call_kwargs["client_keys"] == ["/path/to/key"]
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_with_private_key(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_PRIVATE_KEY",
+                "sftp://testuser@testhost?private_key=PRIVATE_KEY_CONTENT&private_key_passphrase=secret",
+            )
+
+            get_fs(conn_id="sftp_private_key")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert call_kwargs["client_keys"] == ["PRIVATE_KEY_CONTENT"]
+            assert call_kwargs["passphrase"] == "secret"
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_with_private_key_no_passphrase(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_PRIVATE_KEY_NO_PASS",
+                "sftp://testuser@testhost?private_key=PRIVATE_KEY_CONTENT",
+            )
+
+            get_fs(conn_id="sftp_private_key_no_pass")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert call_kwargs["client_keys"] == ["PRIVATE_KEY_CONTENT"]
+            assert "passphrase" not in call_kwargs
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_with_no_host_key_check(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_NO_HOST_CHECK",
+                "sftp://testuser@testhost?no_host_key_check=true",
+            )
+
+            get_fs(conn_id="sftp_no_host_check")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert call_kwargs["known_hosts"] is None
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_default_port(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_DEFAULT_PORT",
+                "sftp://testuser@testhost",
+            )
+
+            get_fs(conn_id="sftp_default_port")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert call_kwargs["port"] == 22
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_without_password(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_NO_PASSWORD",
+                "sftp://testuser@testhost",
+            )
+
+            get_fs(conn_id="sftp_no_password")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert "password" not in call_kwargs
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_host_key_check_enabled_by_default(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_HOST_CHECK_DEFAULT",
+                "sftp://testuser@testhost",
+            )
+
+            get_fs(conn_id="sftp_host_check_default")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert "known_hosts" not in call_kwargs
+
+    @patch("sshfs.SSHFileSystem", autospec=True)
+    def test_get_fs_host_key_check_explicit_false(self, mock_sshfs):
+        from airflow.providers.sftp.fs.sftp import get_fs
+
+        with pytest.MonkeyPatch.context() as mp_ctx:
+            mp_ctx.setenv(
+                "AIRFLOW_CONN_SFTP_HOST_CHECK_FALSE",
+                "sftp://testuser@testhost?no_host_key_check=false",
+            )
+
+            get_fs(conn_id="sftp_host_check_false")
+
+            mock_sshfs.assert_called_once()
+            call_kwargs = mock_sshfs.call_args.kwargs
+            assert "known_hosts" not in call_kwargs

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/tests/unit/sftp/hooks/test_sftp.py

@@ -734,7 +734,7 @@ class MockAirflowConnectionWithPrivate:
 
 class TestSFTPHookAsync:
     @patch("asyncssh.connect", new_callable=AsyncMock)
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     @pytest.mark.asyncio
     async def test_extra_dejson_fields_for_connection_building_known_hosts_none(
         self, mock_get_connection, mock_connect, caplog
@@ -775,7 +775,7 @@ class TestSFTPHookAsync:
     )
     @patch("asyncssh.connect", new_callable=AsyncMock)
     @patch("asyncssh.import_private_key")
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     @pytest.mark.asyncio
     async def test_extra_dejson_fields_for_connection_with_host_key(
         self,
@@ -799,7 +799,7 @@ class TestSFTPHookAsync:
         assert hook.known_hosts == f"localhost {mock_host_key}".encode()
 
     @patch("asyncssh.connect", new_callable=AsyncMock)
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     @pytest.mark.asyncio
     async def test_extra_dejson_fields_for_connection_raises_valuerror(
         self, mock_get_connection, mock_connect
@@ -820,7 +820,7 @@ class TestSFTPHookAsync:
     @patch("paramiko.SSHClient.connect")
     @patch("asyncssh.import_private_key")
     @patch("asyncssh.connect", new_callable=AsyncMock)
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     @pytest.mark.asyncio
     async def test_no_host_key_check_set_logs_warning(
         self, mock_get_connection, mock_connect, mock_import_pkey, mock_ssh_connect, caplog
@@ -833,7 +833,7 @@ class TestSFTPHookAsync:
         assert "No Host Key Verification. This won't protect against Man-In-The-Middle attacks" in caplog.text
 
     @patch("asyncssh.connect", new_callable=AsyncMock)
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     @pytest.mark.asyncio
     async def test_extra_dejson_fields_for_connection_building(self, mock_get_connection, mock_connect):
         """
@@ -861,7 +861,7 @@ class TestSFTPHookAsync:
     @pytest.mark.asyncio
     @patch("asyncssh.connect", new_callable=AsyncMock)
     @patch("asyncssh.import_private_key")
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     async def test_connection_private(self, mock_get_connection, mock_import_private_key, mock_connect):
         """
         Assert that connection details with private key passed through the extra field in the Airflow connection
@@ -888,7 +888,7 @@ class TestSFTPHookAsync:
 
     @pytest.mark.asyncio
     @patch("asyncssh.connect", new_callable=AsyncMock)
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     async def test_connection_port_default_to_22(self, mock_get_connection, mock_connect):
         from unittest.mock import Mock, call
 
@@ -917,7 +917,7 @@ class TestSFTPHookAsync:
 
     @pytest.mark.asyncio
     @patch("asyncssh.connect", new_callable=AsyncMock)
-    @patch("airflow.providers.sftp.hooks.sftp.SFTPHookAsync.get_connection")
+    @patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
     async def test_init_argument_not_ignored(self, mock_get_connection, mock_connect):
         from unittest.mock import Mock, call
 

{apache_airflow_providers_sftp-5.5.1rc1 → apache_airflow_providers_sftp-5.7.0}/tests/unit/sftp/operators/test_sftp.py

@@ -138,16 +138,16 @@ class TestSFTPOperator:
                 do_xcom_push=True,
             )
 
-        tis = {ti.task_id: ti for ti in dag_maker.create_dagrun().task_instances}
-        tis["put_test_task"].run()
-        tis["check_file_task"].run()
+        dr = dag_maker.create_dagrun()
+        dag_maker.run_ti("put_test_task", dr)
+        check_file_task_ti = dag_maker.run_ti("check_file_task", dr)
 
-        pulled = tis["check_file_task"].xcom_pull(task_ids="check_file_task", key="return_value")
+        pulled = check_file_task_ti.xcom_pull(task_ids="check_file_task", key="return_value")
         assert pulled.strip() == test_local_file_content
 
     @pytest.mark.skipif(AIRFLOW_V_3_0_PLUS, reason="Pickle support is removed in Airflow 3")
     @conf_vars({("core", "enable_xcom_pickling"): "True"})
-    def test_file_transfer_no_intermediate_dir_error_put(self, create_task_instance_of_operator):
+    def test_file_transfer_no_intermediate_dir_error_put(self, dag_maker):
         test_local_file_content = (
             b"This is local file content \n which is multiline "
             b"continuing....with other character\nanother line here \n this is last line"
@@ -158,20 +158,17 @@ class TestSFTPOperator:
 
         # Try to put test file to remote. This should raise an error with
         # "No such file" as the directory does not exist.
-        ti2 = create_task_instance_of_operator(
-            SFTPOperator,
-            dag_id="unit_tests_sftp_op_file_transfer_no_intermediate_dir_error_put",
-            task_id="test_sftp",
-            sftp_hook=self.sftp_hook,
-            local_filepath=self.test_local_filepath,
-            remote_filepath=self.test_remote_filepath_int_dir,
-            operation=SFTPOperation.PUT,
-            create_intermediate_dirs=False,
-        )
-        with (
-            pytest.raises(AirflowException) as ctx,
-        ):
-            ti2.run()
+        with dag_maker(dag_id="unit_tests_sftp_op_file_transfer_no_intermediate_dir_error_put"):
+            SFTPOperator(
+                task_id="test_sftp",
+                sftp_hook=self.sftp_hook,
+                local_filepath=self.test_local_filepath,
+                remote_filepath=self.test_remote_filepath_int_dir,
+                operation=SFTPOperation.PUT,
+                create_intermediate_dirs=False,
+            )
+        with pytest.raises(AirflowException) as ctx:
+            dag_maker.run_ti("test_sftp")
         assert "No such file" in str(ctx.value)
 
     @pytest.mark.skipif(AIRFLOW_V_3_0_PLUS, reason="Pickle support is removed in Airflow 3")
@@ -201,10 +198,9 @@ class TestSFTPOperator:
                 do_xcom_push=True,
             )
         dagrun = dag_maker.create_dagrun(logical_date=timezone.utcnow())
-        tis = {ti.task_id: ti for ti in dagrun.task_instances}
-        tis["test_sftp"].run()
-        tis["test_check_file"].run()
-        pulled = tis["test_check_file"].xcom_pull(task_ids="test_check_file", key="return_value")
+        dag_maker.run_ti("test_sftp", dagrun)
+        test_check_file_ti = dag_maker.run_ti("test_check_file", dagrun)
+        pulled = test_check_file_ti.xcom_pull(task_ids="test_check_file", key="return_value")
         assert pulled.strip() == test_local_file_content
 
     @conf_vars({("core", "enable_xcom_pickling"): "False"})
@@ -232,12 +228,10 @@ class TestSFTPOperator:
                 do_xcom_push=True,
             )
         dagrun = dag_maker.create_dagrun(logical_date=timezone.utcnow())
-        tis = {ti.task_id: ti for ti in dagrun.task_instances}
+        dag_maker.run_ti("put_test_task", dagrun)
+        check_file_task_ti = dag_maker.run_ti("check_file_task", dagrun)
 
-        tis["put_test_task"].run()
-        tis["check_file_task"].run()
-
-        pulled = tis["check_file_task"].xcom_pull(task_ids="check_file_task", key="return_value")
+        pulled = check_file_task_ti.xcom_pull(task_ids="check_file_task", key="return_value")
         assert pulled.strip() == b64encode(test_local_file_content).decode("utf-8")
 
     @pytest.fixture
@@ -258,8 +252,7 @@ class TestSFTPOperator:
                 remote_filepath=self.test_remote_filepath,
                 operation=SFTPOperation.GET,
             )
-        for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances:
-            ti.run()
+        dag_maker.run_ti("test_sftp")
 
         # Test the received content.
         with open(self.test_local_filepath, "rb") as file:
@@ -276,8 +269,7 @@ class TestSFTPOperator:
                 remote_filepath=self.test_remote_filepath,
                 operation=SFTPOperation.GET,
             )
-        for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances:
-            ti.run()
+        dag_maker.run_ti("test_sftp")
 
         # Test the received content.
         content_received = None
@@ -297,14 +289,11 @@ class TestSFTPOperator:
                 operation=SFTPOperation.GET,
             )
 
-        for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances:
-            # This should raise an error with "No such file" as the directory
-            # does not exist.
-            with (
-                pytest.raises(AirflowException) as ctx,
-            ):
-                ti.run()
-            assert "No such file" in str(ctx.value)
+        # This should raise an error with "No such file" as the directory
+        # does not exist.
+        with pytest.raises(AirflowException) as ctx:
+            dag_maker.run_ti("test_sftp")
+        assert "No such file" in str(ctx.value)
 
     @pytest.mark.skipif(AIRFLOW_V_3_0_PLUS, reason="Pickle support is removed in Airflow 3")
     @conf_vars({("core", "enable_xcom_pickling"): "True"})
@@ -318,9 +307,7 @@ class TestSFTPOperator:
                 operation=SFTPOperation.GET,
                 create_intermediate_dirs=True,
             )
-
-        for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances:
-            ti.run()
+        dag_maker.run_ti("test_sftp")
 
         # Test the received content.
         content_received = None

apache_airflow_providers_sftp-5.7.0/tests/unit/sftp/sensors/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

apache_airflow_providers_sftp-5.7.0/tests/unit/sftp/triggers/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.