PyPI - apache-airflow-providers-sftp - Versions diffs - 5.0.0rc1__tar.gz → 5.1.0__tar.gz - Mend

apache-airflow-providers-sftp 5.0.0rc1tar.gz → 5.1.0tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apache-airflow-providers-sftp
-Version: 5.0.0rc1
+Version: 5.1.0
 Summary: Provider package apache-airflow-providers-sftp for Apache Airflow
 Keywords: airflow-provider,sftp,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,15 +20,15 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-ssh>=2.1.0rc0
-Requires-Dist: apache-airflow>=2.9.0rc0
-Requires-Dist: asyncssh>=2.12.0
+Requires-Dist: apache-airflow>=2.9.0
+Requires-Dist: apache-airflow-providers-ssh>=2.1.0
 Requires-Dist: paramiko>=2.9.0
+Requires-Dist: asyncssh>=2.12.0
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://x.com/ApacheAirflow
@@ -37,23 +37,6 @@ Provides-Extra: common-compat
 Provides-Extra: openlineage
-.. Licensed to the Apache Software Foundation (ASF) under one
-   or more contributor license agreements.  See the NOTICE file
-   distributed with this work for additional information
-   regarding copyright ownership.  The ASF licenses this file
-   to you under the Apache License, Version 2.0 (the
-   "License"); you may not use this file except in compliance
-   with the License.  You may obtain a copy of the License at
-..   http://www.apache.org/licenses/LICENSE-2.0
-.. Unless required by applicable law or agreed to in writing,
-   software distributed under the License is distributed on an
-   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-   KIND, either express or implied.  See the License for the
-   specific language governing permissions and limitations
-   under the License.
  .. Licensed to the Apache Software Foundation (ASF) under one
     or more contributor license agreements.  See the NOTICE file
     distributed with this work for additional information
@@ -71,8 +54,7 @@ Provides-Extra: openlineage
     specific language governing permissions and limitations
     under the License.
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-    OVERWRITTEN WHEN PREPARING PACKAGES.
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
  .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
     `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -80,7 +62,7 @@ Provides-Extra: openlineage
 Package ``apache-airflow-providers-sftp``
-Release: ``5.0.0.rc1``
+Release: ``5.1.0``
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
@@ -93,7 +75,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
 are in ``airflow.providers.sftp`` python package.
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/>`_.
 Installation
 ------------
@@ -138,4 +120,5 @@ Dependent package
 ==================================================================================================================  =================
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/changelog.html>`_.

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0}/README.rst RENAMED Viewed

@@ -1,21 +1,4 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
-   or more contributor license agreements.  See the NOTICE file
-   distributed with this work for additional information
-   regarding copyright ownership.  The ASF licenses this file
-   to you under the Apache License, Version 2.0 (the
-   "License"); you may not use this file except in compliance
-   with the License.  You may obtain a copy of the License at
-..   http://www.apache.org/licenses/LICENSE-2.0
-.. Unless required by applicable law or agreed to in writing,
-   software distributed under the License is distributed on an
-   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-   KIND, either express or implied.  See the License for the
-   specific language governing permissions and limitations
-   under the License.
  .. Licensed to the Apache Software Foundation (ASF) under one
     or more contributor license agreements.  See the NOTICE file
     distributed with this work for additional information
@@ -33,8 +16,7 @@
     specific language governing permissions and limitations
     under the License.
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-    OVERWRITTEN WHEN PREPARING PACKAGES.
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
  .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
     `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -42,7 +24,7 @@
 Package ``apache-airflow-providers-sftp``
-Release: ``5.0.0.rc1``
+Release: ``5.1.0``
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
@@ -55,7 +37,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
 are in ``airflow.providers.sftp`` python package.
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/>`_.
 Installation
 ------------
@@ -100,4 +82,4 @@ Dependent package
 ==================================================================================================================  =================
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/changelog.html>`_.

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0}/pyproject.toml RENAMED Viewed

@@ -1,4 +1,3 @@
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,10 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
-# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
 # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
 [build-system]
 requires = ["flit_core==3.10.1"]
@@ -27,7 +25,7 @@ build-backend = "flit_core.buildapi"
 [project]
 name = "apache-airflow-providers-sftp"
-version = "5.0.0.rc1"
+version = "5.1.0"
 description = "Provider package apache-airflow-providers-sftp for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -53,16 +51,29 @@ classifiers = [
     "Topic :: System :: Monitoring",
 ]
 requires-python = "~=3.9"
+# The dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
 dependencies = [
-    "apache-airflow-providers-ssh>=2.1.0rc0",
-    "apache-airflow>=2.9.0rc0",
-    "asyncssh>=2.12.0",
+    "apache-airflow>=2.9.0",
+    "apache-airflow-providers-ssh>=2.1.0",
     "paramiko>=2.9.0",
+    "asyncssh>=2.12.0",
+]
+# The optional dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+[project.optional-dependencies]
+"common.compat" = [
+    "apache-airflow-providers-common-compat"
+]
+"openlineage" = [
+    "apache-airflow-providers-openlineage"
 ]
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.0.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -71,13 +82,9 @@ dependencies = [
 [project.entry-points."apache_airflow_provider"]
 provider_info = "airflow.providers.sftp.get_provider_info:get_provider_info"
-[project.optional-dependencies]
-"common.compat" = [
-    "apache-airflow-providers-common-compat",
-]
-"openlineage" = [
-    "apache-airflow-providers-openlineage",
-]
 [tool.flit.module]
 name = "airflow.providers.sftp"
+[tool.pytest.ini_options]
+ignore = "tests/system/"

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0/src}/airflow/providers/sftp/LICENSE RENAMED Viewed

@@ -199,55 +199,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-============================================================================
-   APACHE AIRFLOW SUBCOMPONENTS:
-   The Apache Airflow project contains subcomponents with separate copyright
-   notices and license terms. Your use of the source code for the these
-   subcomponents is subject to the terms and conditions of the following
-   licenses.
-========================================================================
-Third party Apache 2.0 licenses
-========================================================================
-The following components are provided under the Apache 2.0 License.
-See project link for details. The text of each license is also included
-at 3rd-party-licenses/LICENSE-[project].txt.
-    (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
-    (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
-    (ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
-    (ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
-========================================================================
-MIT licenses
-========================================================================
-The following components are provided under the MIT License. See project link for details.
-The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
-    (MIT License) jquery v3.5.1 (https://jquery.org/license/)
-    (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
-    (MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
-    (MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
-    (MIT License) dataTables v1.10.25 (https://datatables.net)
-    (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
-    (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
-    (MIT License) MomentJS v2.24.0 (http://momentjs.com/)
-    (MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
-========================================================================
-BSD 3-Clause licenses
-========================================================================
-The following components are provided under the BSD 3-Clause license. See project links for details.
-The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
-    (BSD 3 License) d3 v5.16.0 (https://d3js.org)
-    (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
-    (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
-========================================================================
-See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0/src}/airflow/providers/sftp/__init__.py RENAMED Viewed

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 __all__ = ["__version__"]
-__version__ = "5.0.0"
+__version__ = "5.1.0"
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0/src}/airflow/providers/sftp/get_provider_info.py RENAMED Viewed

@@ -15,8 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 #
 # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
 # `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -28,8 +27,9 @@ def get_provider_info():
         "name": "SFTP",
         "description": "`SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1734536539,
+        "source-date-epoch": 1739964267,
         "versions": [
+            "5.1.0",
             "5.0.0",
             "4.11.1",
             "4.11.0",
@@ -72,17 +72,11 @@ def get_provider_info():
             "1.1.0",
             "1.0.0",
         ],
-        "dependencies": [
-            "apache-airflow>=2.9.0",
-            "apache-airflow-providers-ssh>=2.1.0",
-            "paramiko>=2.9.0",
-            "asyncssh>=2.12.0",
-        ],
         "integrations": [
             {
                 "integration-name": "SSH File Transfer Protocol (SFTP)",
                 "external-doc-url": "https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/",
-                "logo": "/integration-logos/sftp/SFTP.png",
+                "logo": "/docs/integration-logos/SFTP.png",
                 "tags": ["protocol"],
             }
         ],
@@ -122,4 +116,14 @@ def get_provider_info():
                 "python-modules": ["airflow.providers.sftp.triggers.sftp"],
             }
         ],
+        "dependencies": [
+            "apache-airflow>=2.9.0",
+            "apache-airflow-providers-ssh>=2.1.0",
+            "paramiko>=2.9.0",
+            "asyncssh>=2.12.0",
+        ],
+        "optional-dependencies": {
+            "common.compat": ["apache-airflow-providers-common-compat"],
+            "openlineage": ["apache-airflow-providers-openlineage"],
+        },
     }

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0/src}/airflow/providers/sftp/hooks/sftp.py RENAMED Viewed

@@ -22,19 +22,24 @@ from __future__ import annotations
 import datetime
 import os
 import stat
-from collections.abc import Sequence
+import warnings
+from collections.abc import Generator, Sequence
+from contextlib import closing, contextmanager
 from fnmatch import fnmatch
+from io import BytesIO
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable
 import asyncssh
 from asgiref.sync import sync_to_async
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.hooks.base import BaseHook
 from airflow.providers.ssh.hooks.ssh import SSHHook
 if TYPE_CHECKING:
-    import paramiko
+    from paramiko.sftp_attr import SFTPAttributes
+    from paramiko.sftp_client import SFTPClient
     from airflow.models.connection import Connection
@@ -51,8 +56,6 @@ class SFTPHook(SSHHook):
         - In contrast with FTPHook describe_directory only returns size, type and
           modify. It doesn't return unix.owner, unix.mode, perm, unix.group and
           unique.
-        - retrieve_file and store_file only take a local full path and not a
-           buffer.
         - If no mode is passed to create_directory it will be created with 777
           permissions.
@@ -84,7 +87,22 @@ class SFTPHook(SSHHook):
         *args,
         **kwargs,
     ) -> None:
-        self.conn: paramiko.SFTPClient | None = None
+        # TODO: remove support for ssh_hook when it is removed from SFTPOperator
+        if kwargs.get("ssh_hook") is not None:
+            warnings.warn(
+                "Parameter `ssh_hook` is deprecated and will be ignored.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
+        ftp_conn_id = kwargs.pop("ftp_conn_id", None)
+        if ftp_conn_id:
+            warnings.warn(
+                "Parameter `ftp_conn_id` is deprecated. Please use `ssh_conn_id` instead.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
+            ssh_conn_id = ftp_conn_id
         kwargs["ssh_conn_id"] = ssh_conn_id
         kwargs["host_proxy_cmd"] = host_proxy_cmd
@@ -92,17 +110,12 @@ class SFTPHook(SSHHook):
         super().__init__(*args, **kwargs)
-    def get_conn(self) -> paramiko.SFTPClient:  # type: ignore[override]
-        """Open an SFTP connection to the remote host."""
-        if self.conn is None:
-            self.conn = super().get_conn().open_sftp()
-        return self.conn
-    def close_conn(self) -> None:
-        """Close the SFTP connection."""
-        if self.conn is not None:
-            self.conn.close()
-            self.conn = None
+    @contextmanager
+    def get_conn(self) -> Generator[SFTPClient, None, None]:
+        """Context manager that closes the connection after use."""
+        with closing(super().get_conn()) as conn:
+            with closing(conn.open_sftp()) as sftp:
+                yield sftp
     def describe_directory(self, path: str) -> dict[str, dict[str, str | int | None]]:
         """
@@ -113,17 +126,17 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote directory
         """
-        conn = self.get_conn()
-        flist = sorted(conn.listdir_attr(path), key=lambda x: x.filename)
-        files = {}
-        for f in flist:
-            modify = datetime.datetime.fromtimestamp(f.st_mtime).strftime("%Y%m%d%H%M%S")  # type: ignore
-            files[f.filename] = {
-                "size": f.st_size,
-                "type": "dir" if stat.S_ISDIR(f.st_mode) else "file",  # type: ignore
-                "modify": modify,
-            }
-        return files
+        with self.get_conn() as conn:  # type: SFTPClient
+            flist = sorted(conn.listdir_attr(path), key=lambda x: x.filename)
+            files = {}
+            for f in flist:
+                modify = datetime.datetime.fromtimestamp(f.st_mtime).strftime("%Y%m%d%H%M%S")  # type: ignore
+                files[f.filename] = {
+                    "size": f.st_size,
+                    "type": "dir" if stat.S_ISDIR(f.st_mode) else "file",  # type: ignore
+                    "modify": modify,
+                }
+            return files
     def list_directory(self, path: str) -> list[str]:
         """
@@ -131,18 +144,17 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote directory to list
         """
-        conn = self.get_conn()
-        files = sorted(conn.listdir(path))
-        return files
+        with self.get_conn() as conn:
+            return sorted(conn.listdir(path))
-    def list_directory_with_attr(self, path: str) -> list[paramiko.SFTPAttributes]:
+    def list_directory_with_attr(self, path: str) -> list[SFTPAttributes]:
         """
         List files in a directory on the remote system including their SFTPAttributes.
         :param path: full path to the remote directory to list
         """
-        conn = self.get_conn()
-        return [file for file in conn.listdir_attr(path)]
+        with self.get_conn() as conn:
+            return [file for file in conn.listdir_attr(path)]
     def mkdir(self, path: str, mode: int = 0o777) -> None:
         """
@@ -154,8 +166,8 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote directory to create
         :param mode: int permissions of octal mode for directory
         """
-        conn = self.get_conn()
-        conn.mkdir(path, mode=mode)
+        with self.get_conn() as conn:
+            conn.mkdir(path, mode=mode)
     def isdir(self, path: str) -> bool:
         """
@@ -163,12 +175,11 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote directory to check
         """
-        conn = self.get_conn()
-        try:
-            result = stat.S_ISDIR(conn.stat(path).st_mode)  # type: ignore
-        except OSError:
-            result = False
-        return result
+        with self.get_conn() as conn:
+            try:
+                return stat.S_ISDIR(conn.stat(path).st_mode)  # type: ignore
+            except OSError:
+                return False
     def isfile(self, path: str) -> bool:
         """
@@ -176,12 +187,11 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote file to check
         """
-        conn = self.get_conn()
-        try:
-            result = stat.S_ISREG(conn.stat(path).st_mode)  # type: ignore
-        except OSError:
-            result = False
-        return result
+        with self.get_conn() as conn:
+            try:
+                return stat.S_ISREG(conn.stat(path).st_mode)  # type: ignore
+            except OSError:
+                return False
     def create_directory(self, path: str, mode: int = 0o777) -> None:
         """
@@ -195,7 +205,6 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote directory to create
         :param mode: int permissions of octal mode for directory
         """
-        conn = self.get_conn()
         if self.isdir(path):
             self.log.info("%s already exists", path)
             return
@@ -207,16 +216,28 @@ class SFTPHook(SSHHook):
                 self.create_directory(dirname, mode)
             if basename:
                 self.log.info("Creating %s", path)
-                conn.mkdir(path, mode=mode)
+                with self.get_conn() as conn:
+                    conn.mkdir(path, mode=mode)
-    def delete_directory(self, path: str) -> None:
+    def delete_directory(self, path: str, include_files: bool = False) -> None:
         """
         Delete a directory on the remote system.
         :param path: full path to the remote directory to delete
         """
-        conn = self.get_conn()
-        conn.rmdir(path)
+        files: list[str] = []
+        dirs: list[str] = []
+        if include_files is True:
+            files, dirs, _ = self.get_tree_map(path)
+            dirs = dirs[::-1]  # reverse the order for deleting deepest directories first
+        with self.get_conn() as conn:
+            for file_path in files:
+                conn.remove(file_path)
+            for dir_path in dirs:
+                conn.rmdir(dir_path)
+            conn.rmdir(path)
     def retrieve_file(self, remote_full_path: str, local_full_path: str, prefetch: bool = True) -> None:
         """
@@ -226,11 +247,14 @@ class SFTPHook(SSHHook):
         at that location.
         :param remote_full_path: full path to the remote file
-        :param local_full_path: full path to the local file
+        :param local_full_path: full path to the local file or a file-like buffer
         :param prefetch: controls whether prefetch is performed (default: True)
         """
-        conn = self.get_conn()
-        conn.get(remote_full_path, local_full_path, prefetch=prefetch)
+        with self.get_conn() as conn:
+            if isinstance(local_full_path, BytesIO):
+                conn.getfo(remote_full_path, local_full_path, prefetch=prefetch)
+            else:
+                conn.get(remote_full_path, local_full_path, prefetch=prefetch)
     def store_file(self, remote_full_path: str, local_full_path: str, confirm: bool = True) -> None:
         """
@@ -240,10 +264,13 @@ class SFTPHook(SSHHook):
         from that location.
         :param remote_full_path: full path to the remote file
-        :param local_full_path: full path to the local file
+        :param local_full_path: full path to the local file or a file-like buffer
         """
-        conn = self.get_conn()
-        conn.put(local_full_path, remote_full_path, confirm=confirm)
+        with self.get_conn() as conn:
+            if isinstance(local_full_path, BytesIO):
+                conn.putfo(local_full_path, remote_full_path, confirm=confirm)
+            else:
+                conn.put(local_full_path, remote_full_path, confirm=confirm)
     def delete_file(self, path: str) -> None:
         """
@@ -251,8 +278,53 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote file
         """
-        conn = self.get_conn()
-        conn.remove(path)
+        with self.get_conn() as conn:
+            conn.remove(path)
+    def retrieve_directory(self, remote_full_path: str, local_full_path: str, prefetch: bool = True) -> None:
+        """
+        Transfer the remote directory to a local location.
+        If local_full_path is a string path, the directory will be put
+        at that location.
+        :param remote_full_path: full path to the remote directory
+        :param local_full_path: full path to the local directory
+        :param prefetch: controls whether prefetch is performed (default: True)
+        """
+        if Path(local_full_path).exists():
+            raise AirflowException(f"{local_full_path} already exists")
+        Path(local_full_path).mkdir(parents=True)
+        files, dirs, _ = self.get_tree_map(remote_full_path)
+        for dir_path in dirs:
+            new_local_path = os.path.join(local_full_path, os.path.relpath(dir_path, remote_full_path))
+            Path(new_local_path).mkdir(parents=True, exist_ok=True)
+        for file_path in files:
+            new_local_path = os.path.join(local_full_path, os.path.relpath(file_path, remote_full_path))
+            self.retrieve_file(file_path, new_local_path, prefetch)
+    def store_directory(self, remote_full_path: str, local_full_path: str, confirm: bool = True) -> None:
+        """
+        Transfer a local directory to the remote location.
+        If local_full_path is a string path, the directory will be read
+        from that location.
+        :param remote_full_path: full path to the remote directory
+        :param local_full_path: full path to the local directory
+        """
+        if self.path_exists(remote_full_path):
+            raise AirflowException(f"{remote_full_path} already exists")
+        self.create_directory(remote_full_path)
+        for root, dirs, files in os.walk(local_full_path):
+            for dir_name in dirs:
+                dir_path = os.path.join(root, dir_name)
+                new_remote_path = os.path.join(remote_full_path, os.path.relpath(dir_path, local_full_path))
+                self.create_directory(new_remote_path)
+            for file_name in files:
+                file_path = os.path.join(root, file_name)
+                new_remote_path = os.path.join(remote_full_path, os.path.relpath(file_path, local_full_path))
+                self.store_file(new_remote_path, file_path, confirm)
     def get_mod_time(self, path: str) -> str:
         """
@@ -260,9 +332,9 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote file
         """
-        conn = self.get_conn()
-        ftp_mdtm = conn.stat(path).st_mtime
-        return datetime.datetime.fromtimestamp(ftp_mdtm).strftime("%Y%m%d%H%M%S")  # type: ignore
+        with self.get_conn() as conn:
+            ftp_mdtm = conn.stat(path).st_mtime
+            return datetime.datetime.fromtimestamp(ftp_mdtm).strftime("%Y%m%d%H%M%S")  # type: ignore
     def path_exists(self, path: str) -> bool:
         """
@@ -270,12 +342,12 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote file or directory
         """
-        conn = self.get_conn()
-        try:
-            conn.stat(path)
-        except OSError:
-            return False
-        return True
+        with self.get_conn() as conn:
+            try:
+                conn.stat(path)
+            except OSError:
+                return False
+            return True
     @staticmethod
     def _is_path_match(path: str, prefix: str | None = None, delimiter: str | None = None) -> bool:
@@ -369,9 +441,9 @@ class SFTPHook(SSHHook):
     def test_connection(self) -> tuple[bool, str]:
         """Test the SFTP connection by calling path with directory."""
         try:
-            conn = self.get_conn()
-            conn.normalize(".")
-            return True, "Connection successfully tested"
+            with self.get_conn() as conn:
+                conn.normalize(".")
+                return True, "Connection successfully tested"
         except Exception as e:
             return False, str(e)
@@ -386,7 +458,6 @@ class SFTPHook(SSHHook):
         for file in self.list_directory(path):
             if fnmatch(file, fnmatch_pattern):
                 return file
         return ""
     def get_files_by_pattern(self, path, fnmatch_pattern) -> list[str]:
@@ -554,17 +625,13 @@ class SFTPHookAsync(BaseHook):
         :param path: full path to the remote file
         """
-        ssh_conn = None
-        try:
-            ssh_conn = await self._get_conn()
-            sftp_client = await ssh_conn.start_sftp_client()
-            ftp_mdtm = await sftp_client.stat(path)
-            modified_time = ftp_mdtm.mtime
-            mod_time = datetime.datetime.fromtimestamp(modified_time).strftime("%Y%m%d%H%M%S")  # type: ignore[arg-type]
-            self.log.info("Found File %s last modified: %s", str(path), str(mod_time))
-            return mod_time
-        except asyncssh.SFTPNoSuchFile:
-            raise AirflowException("No files matching")
-        finally:
-            if ssh_conn:
-                ssh_conn.close()
+        async with await self._get_conn() as ssh_conn:
+            try:
+                sftp_client = await ssh_conn.start_sftp_client()
+                ftp_mdtm = await sftp_client.stat(path)
+                modified_time = ftp_mdtm.mtime
+                mod_time = datetime.datetime.fromtimestamp(modified_time).strftime("%Y%m%d%H%M%S")  # type: ignore[arg-type]
+                self.log.info("Found File %s last modified: %s", str(path), str(mod_time))
+                return mod_time
+            except asyncssh.SFTPNoSuchFile:
+                raise AirflowException("No files matching")

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0/src}/airflow/providers/sftp/operators/sftp.py RENAMED Viewed

@@ -37,6 +37,7 @@ class SFTPOperation:
     PUT = "put"
     GET = "get"
+    DELETE = "delete"
 class SFTPOperator(BaseOperator):
@@ -53,8 +54,8 @@ class SFTPOperator(BaseOperator):
         Nullable. If provided, it will replace the `remote_host` which was
         defined in `sftp_hook` or predefined in the connection of `ssh_conn_id`.
     :param local_filepath: local file path or list of local file paths to get or put. (templated)
-    :param remote_filepath: remote file path or list of remote file paths to get or put. (templated)
-    :param operation: specify operation 'get' or 'put', defaults to put
+    :param remote_filepath: remote file path or list of remote file paths to get, put, or delete. (templated)
+    :param operation: specify operation 'get', 'put', or 'delete', defaults to put
     :param confirm: specify if the SFTP operation should be confirmed, defaults to True
     :param create_intermediate_dirs: create missing intermediate directories when
         copying from remote to local and vice-versa. Default is False.
@@ -84,7 +85,7 @@ class SFTPOperator(BaseOperator):
         sftp_hook: SFTPHook | None = None,
         ssh_conn_id: str | None = None,
         remote_host: str | None = None,
-        local_filepath: str | list[str],
+        local_filepath: str | list[str] | None = None,
         remote_filepath: str | list[str],
         operation: str = SFTPOperation.PUT,
         confirm: bool = True,
@@ -102,7 +103,9 @@ class SFTPOperator(BaseOperator):
         self.remote_filepath = remote_filepath
     def execute(self, context: Any) -> str | list[str] | None:
-        if isinstance(self.local_filepath, str):
+        if self.local_filepath is None:
+            local_filepath_array = []
+        elif isinstance(self.local_filepath, str):
             local_filepath_array = [self.local_filepath]
         else:
             local_filepath_array = self.local_filepath
@@ -112,16 +115,21 @@ class SFTPOperator(BaseOperator):
         else:
             remote_filepath_array = self.remote_filepath
-        if len(local_filepath_array) != len(remote_filepath_array):
+        if self.operation.lower() in (SFTPOperation.GET, SFTPOperation.PUT) and len(
+            local_filepath_array
+        ) != len(remote_filepath_array):
             raise ValueError(
                 f"{len(local_filepath_array)} paths in local_filepath "
                 f"!= {len(remote_filepath_array)} paths in remote_filepath"
             )
-        if self.operation.lower() not in (SFTPOperation.GET, SFTPOperation.PUT):
+        if self.operation.lower() == SFTPOperation.DELETE and local_filepath_array:
+            raise ValueError("local_filepath should not be provided for delete operation")
+        if self.operation.lower() not in (SFTPOperation.GET, SFTPOperation.PUT, SFTPOperation.DELETE):
             raise TypeError(
                 f"Unsupported operation value {self.operation}, "
-                f"expected {SFTPOperation.GET} or {SFTPOperation.PUT}."
+                f"expected {SFTPOperation.GET} or {SFTPOperation.PUT} or {SFTPOperation.DELETE}."
             )
         file_msg = None
@@ -144,24 +152,43 @@ class SFTPOperator(BaseOperator):
                 )
                 self.sftp_hook.remote_host = self.remote_host
-            for _local_filepath, _remote_filepath in zip(local_filepath_array, remote_filepath_array):
-                if self.operation.lower() == SFTPOperation.GET:
-                    local_folder = os.path.dirname(_local_filepath)
-                    if self.create_intermediate_dirs:
-                        Path(local_folder).mkdir(parents=True, exist_ok=True)
-                    file_msg = f"from {_remote_filepath} to {_local_filepath}"
-                    self.log.info("Starting to transfer %s", file_msg)
-                    self.sftp_hook.retrieve_file(_remote_filepath, _local_filepath)
-                else:
-                    remote_folder = os.path.dirname(_remote_filepath)
-                    if self.create_intermediate_dirs:
-                        self.sftp_hook.create_directory(remote_folder)
-                    file_msg = f"from {_local_filepath} to {_remote_filepath}"
-                    self.log.info("Starting to transfer file %s", file_msg)
-                    self.sftp_hook.store_file(_remote_filepath, _local_filepath, confirm=self.confirm)
+            if self.operation.lower() in (SFTPOperation.GET, SFTPOperation.PUT):
+                for _local_filepath, _remote_filepath in zip(local_filepath_array, remote_filepath_array):
+                    if self.operation.lower() == SFTPOperation.GET:
+                        local_folder = os.path.dirname(_local_filepath)
+                        if self.create_intermediate_dirs:
+                            Path(local_folder).mkdir(parents=True, exist_ok=True)
+                        file_msg = f"from {_remote_filepath} to {_local_filepath}"
+                        self.log.info("Starting to transfer %s", file_msg)
+                        if self.sftp_hook.isdir(_remote_filepath):
+                            self.sftp_hook.retrieve_directory(_remote_filepath, _local_filepath)
+                        else:
+                            self.sftp_hook.retrieve_file(_remote_filepath, _local_filepath)
+                    elif self.operation.lower() == SFTPOperation.PUT:
+                        remote_folder = os.path.dirname(_remote_filepath)
+                        if self.create_intermediate_dirs:
+                            self.sftp_hook.create_directory(remote_folder)
+                        file_msg = f"from {_local_filepath} to {_remote_filepath}"
+                        self.log.info("Starting to transfer file %s", file_msg)
+                        if os.path.isdir(_local_filepath):
+                            self.sftp_hook.store_directory(
+                                _remote_filepath, _local_filepath, confirm=self.confirm
+                            )
+                        else:
+                            self.sftp_hook.store_file(_remote_filepath, _local_filepath, confirm=self.confirm)
+            elif self.operation.lower() == SFTPOperation.DELETE:
+                for _remote_filepath in remote_filepath_array:
+                    file_msg = f"{_remote_filepath}"
+                    self.log.info("Starting to delete %s", file_msg)
+                    if self.sftp_hook.isdir(_remote_filepath):
+                        self.sftp_hook.delete_directory(_remote_filepath, include_files=True)
+                    else:
+                        self.sftp_hook.delete_file(_remote_filepath)
         except Exception as e:
-            raise AirflowException(f"Error while transferring {file_msg}, error: {e}")
+            raise AirflowException(
+                f"Error while processing {self.operation.upper()} operation {file_msg}, error: {e}"
+            )
         return self.local_filepath

{apache_airflow_providers_sftp-5.0.0rc1 → apache_airflow_providers_sftp-5.1.0/src}/airflow/providers/sftp/sensors/sftp.py RENAMED Viewed

@@ -34,7 +34,11 @@ from airflow.sensors.base import BaseSensorOperator, PokeReturnValue
 from airflow.utils.timezone import convert_to_utc, parse
 if TYPE_CHECKING:
-    from airflow.utils.context import Context
+    try:
+        from airflow.sdk.definitions.context import Context
+    except ImportError:
+        # TODO: Remove once provider drops support for Airflow 2
+        from airflow.utils.context import Context
 class SFTPSensor(BaseSensorOperator):