apache-airflow-providers-sftp 5.1.0rc1__py3-none-any.whl → 5.1.1rc1__py3-none-any.whl

airflow/providers/sftp/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "5.1.0"
+__version__ = "5.1.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/sftp/get_provider_info.py

@@ -27,8 +27,9 @@ def get_provider_info():
         "name": "SFTP",
         "description": "`SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1739964267,
+        "source-date-epoch": 1741509612,
         "versions": [
+            "5.1.1",
             "5.1.0",
             "5.0.0",
             "4.11.1",
@@ -126,4 +127,5 @@ def get_provider_info():
             "common.compat": ["apache-airflow-providers-common-compat"],
             "openlineage": ["apache-airflow-providers-openlineage"],
         },
+        "devel-dependencies": [],
     }

airflow/providers/sftp/hooks/sftp.py

@@ -24,7 +24,7 @@ import os
 import stat
 import warnings
 from collections.abc import Generator, Sequence
-from contextlib import closing, contextmanager
+from contextlib import contextmanager
 from fnmatch import fnmatch
 from io import BytesIO
 from pathlib import Path
@@ -38,6 +38,7 @@ from airflow.hooks.base import BaseHook
 from airflow.providers.ssh.hooks.ssh import SSHHook
 
 if TYPE_CHECKING:
+    from paramiko import SSHClient
     from paramiko.sftp_attr import SFTPAttributes
     from paramiko.sftp_client import SFTPClient
 
@@ -87,6 +88,8 @@ class SFTPHook(SSHHook):
         *args,
         **kwargs,
     ) -> None:
+        self.conn: SFTPClient | None = None
+
         # TODO: remove support for ssh_hook when it is removed from SFTPOperator
         if kwargs.get("ssh_hook") is not None:
             warnings.warn(
@@ -108,14 +111,46 @@ class SFTPHook(SSHHook):
         kwargs["host_proxy_cmd"] = host_proxy_cmd
         self.ssh_conn_id = ssh_conn_id
 
+        self._ssh_conn: SSHClient | None = None
+        self._sftp_conn: SFTPClient | None = None
+        self._conn_count = 0
+
         super().__init__(*args, **kwargs)
 
+    def get_conn(self) -> SFTPClient:  # type: ignore[override]
+        """Open an SFTP connection to the remote host."""
+        if self.conn is None:
+            self.conn = super().get_conn().open_sftp()
+        return self.conn
+
+    def close_conn(self) -> None:
+        """Close the SFTP connection."""
+        if self.conn is not None:
+            self.conn.close()
+            self.conn = None
+
     @contextmanager
-    def get_conn(self) -> Generator[SFTPClient, None, None]:
+    def get_managed_conn(self) -> Generator[SFTPClient, None, None]:
         """Context manager that closes the connection after use."""
-        with closing(super().get_conn()) as conn:
-            with closing(conn.open_sftp()) as sftp:
-                yield sftp
+        if self._sftp_conn is None:
+            ssh_conn: SSHClient = super().get_conn()
+            self._ssh_conn = ssh_conn
+            self._sftp_conn = ssh_conn.open_sftp()
+        self._conn_count += 1
+
+        try:
+            yield self._sftp_conn
+        finally:
+            self._conn_count -= 1
+            if self._conn_count == 0 and self._ssh_conn is not None and self._sftp_conn is not None:
+                self._sftp_conn.close()
+                self._sftp_conn = None
+                self._ssh_conn.close()
+                self._ssh_conn = None
+
+    def get_conn_count(self) -> int:
+        """Get the number of open connections."""
+        return self._conn_count
 
     def describe_directory(self, path: str) -> dict[str, dict[str, str | int | None]]:
         """
@@ -126,7 +161,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote directory
         """
-        with self.get_conn() as conn:  # type: SFTPClient
+        with self.get_managed_conn() as conn:  # type: SFTPClient
             flist = sorted(conn.listdir_attr(path), key=lambda x: x.filename)
             files = {}
             for f in flist:
@@ -144,7 +179,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote directory to list
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             return sorted(conn.listdir(path))
 
     def list_directory_with_attr(self, path: str) -> list[SFTPAttributes]:
@@ -153,7 +188,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote directory to list
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             return [file for file in conn.listdir_attr(path)]
 
     def mkdir(self, path: str, mode: int = 0o777) -> None:
@@ -166,7 +201,7 @@ class SFTPHook(SSHHook):
         :param path: full path to the remote directory to create
         :param mode: int permissions of octal mode for directory
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             conn.mkdir(path, mode=mode)
 
     def isdir(self, path: str) -> bool:
@@ -175,7 +210,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote directory to check
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             try:
                 return stat.S_ISDIR(conn.stat(path).st_mode)  # type: ignore
             except OSError:
@@ -187,7 +222,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote file to check
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             try:
                 return stat.S_ISREG(conn.stat(path).st_mode)  # type: ignore
             except OSError:
@@ -216,7 +251,7 @@ class SFTPHook(SSHHook):
                 self.create_directory(dirname, mode)
             if basename:
                 self.log.info("Creating %s", path)
-                with self.get_conn() as conn:
+                with self.get_managed_conn() as conn:
                     conn.mkdir(path, mode=mode)
 
     def delete_directory(self, path: str, include_files: bool = False) -> None:
@@ -232,7 +267,7 @@ class SFTPHook(SSHHook):
             files, dirs, _ = self.get_tree_map(path)
             dirs = dirs[::-1]  # reverse the order for deleting deepest directories first
 
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             for file_path in files:
                 conn.remove(file_path)
             for dir_path in dirs:
@@ -250,7 +285,7 @@ class SFTPHook(SSHHook):
         :param local_full_path: full path to the local file or a file-like buffer
         :param prefetch: controls whether prefetch is performed (default: True)
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             if isinstance(local_full_path, BytesIO):
                 conn.getfo(remote_full_path, local_full_path, prefetch=prefetch)
             else:
@@ -266,7 +301,7 @@ class SFTPHook(SSHHook):
         :param remote_full_path: full path to the remote file
         :param local_full_path: full path to the local file or a file-like buffer
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             if isinstance(local_full_path, BytesIO):
                 conn.putfo(local_full_path, remote_full_path, confirm=confirm)
             else:
@@ -278,7 +313,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote file
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             conn.remove(path)
 
     def retrieve_directory(self, remote_full_path: str, local_full_path: str, prefetch: bool = True) -> None:
@@ -295,13 +330,14 @@ class SFTPHook(SSHHook):
         if Path(local_full_path).exists():
             raise AirflowException(f"{local_full_path} already exists")
         Path(local_full_path).mkdir(parents=True)
-        files, dirs, _ = self.get_tree_map(remote_full_path)
-        for dir_path in dirs:
-            new_local_path = os.path.join(local_full_path, os.path.relpath(dir_path, remote_full_path))
-            Path(new_local_path).mkdir(parents=True, exist_ok=True)
-        for file_path in files:
-            new_local_path = os.path.join(local_full_path, os.path.relpath(file_path, remote_full_path))
-            self.retrieve_file(file_path, new_local_path, prefetch)
+        with self.get_managed_conn():
+            files, dirs, _ = self.get_tree_map(remote_full_path)
+            for dir_path in dirs:
+                new_local_path = os.path.join(local_full_path, os.path.relpath(dir_path, remote_full_path))
+                Path(new_local_path).mkdir(parents=True, exist_ok=True)
+            for file_path in files:
+                new_local_path = os.path.join(local_full_path, os.path.relpath(file_path, remote_full_path))
+                self.retrieve_file(file_path, new_local_path, prefetch)
 
     def store_directory(self, remote_full_path: str, local_full_path: str, confirm: bool = True) -> None:
         """
@@ -315,16 +351,21 @@ class SFTPHook(SSHHook):
         """
         if self.path_exists(remote_full_path):
             raise AirflowException(f"{remote_full_path} already exists")
-        self.create_directory(remote_full_path)
-        for root, dirs, files in os.walk(local_full_path):
-            for dir_name in dirs:
-                dir_path = os.path.join(root, dir_name)
-                new_remote_path = os.path.join(remote_full_path, os.path.relpath(dir_path, local_full_path))
-                self.create_directory(new_remote_path)
-            for file_name in files:
-                file_path = os.path.join(root, file_name)
-                new_remote_path = os.path.join(remote_full_path, os.path.relpath(file_path, local_full_path))
-                self.store_file(new_remote_path, file_path, confirm)
+        with self.get_managed_conn():
+            self.create_directory(remote_full_path)
+            for root, dirs, files in os.walk(local_full_path):
+                for dir_name in dirs:
+                    dir_path = os.path.join(root, dir_name)
+                    new_remote_path = os.path.join(
+                        remote_full_path, os.path.relpath(dir_path, local_full_path)
+                    )
+                    self.create_directory(new_remote_path)
+                for file_name in files:
+                    file_path = os.path.join(root, file_name)
+                    new_remote_path = os.path.join(
+                        remote_full_path, os.path.relpath(file_path, local_full_path)
+                    )
+                    self.store_file(new_remote_path, file_path, confirm)
 
     def get_mod_time(self, path: str) -> str:
         """
@@ -332,7 +373,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote file
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             ftp_mdtm = conn.stat(path).st_mtime
             return datetime.datetime.fromtimestamp(ftp_mdtm).strftime("%Y%m%d%H%M%S")  # type: ignore
 
@@ -342,7 +383,7 @@ class SFTPHook(SSHHook):
 
         :param path: full path to the remote file or directory
         """
-        with self.get_conn() as conn:
+        with self.get_managed_conn() as conn:
             try:
                 conn.stat(path)
             except OSError:
@@ -441,7 +482,7 @@ class SFTPHook(SSHHook):
     def test_connection(self) -> tuple[bool, str]:
         """Test the SFTP connection by calling path with directory."""
         try:
-            with self.get_conn() as conn:
+            with self.get_managed_conn() as conn:
                 conn.normalize(".")
                 return True, "Connection successfully tested"
         except Exception as e:

airflow/providers/sftp/sensors/sftp.py

@@ -129,7 +129,6 @@ class SFTPSensor(BaseSensorOperator):
             else:
                 files_found.append(actual_file_to_check)
 
-        self.hook.close_conn()
         if not len(files_found):
             return False
 

{apache_airflow_providers_sftp-5.1.0rc1.dist-info → apache_airflow_providers_sftp-5.1.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: apache-airflow-providers-sftp
-Version: 5.1.0rc1
+Version: 5.1.1rc1
 Summary: Provider package apache-airflow-providers-sftp for Apache Airflow
 Keywords: airflow-provider,sftp,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -27,8 +27,8 @@ Requires-Dist: asyncssh>=2.12.0
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://x.com/ApacheAirflow
@@ -37,32 +37,31 @@ Provides-Extra: common-compat
 Provides-Extra: openlineage
 
 
- .. Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
+.. Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
 
- ..   http://www.apache.org/licenses/LICENSE-2.0
+..   http://www.apache.org/licenses/LICENSE-2.0
 
- .. Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
+.. Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
 
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
-
- .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
-    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 
+.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+   ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY
 
 Package ``apache-airflow-providers-sftp``
 
-Release: ``5.1.0``
+Release: ``5.1.1``
 
 
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
@@ -75,7 +74,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
 are in ``airflow.providers.sftp`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/>`_.
 
 Installation
 ------------
@@ -120,5 +119,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/changelog.html>`_.
 

{apache_airflow_providers_sftp-5.1.0rc1.dist-info → apache_airflow_providers_sftp-5.1.1rc1.dist-info}/RECORD

@@ -1,18 +1,18 @@
 airflow/providers/sftp/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/sftp/__init__.py,sha256=cozZuS7gnm3FJTNUVcuHiETa115Qw8xdtX6-YMm8Kl8,1491
-airflow/providers/sftp/get_provider_info.py,sha256=r-DSixnfFdgYqggP4wv6SGfxh5aVzrvSfGPfTRHo3AQ,4269
+airflow/providers/sftp/__init__.py,sha256=N2tDz6dqlCE4CDTlsZJXq39AESjhHCOXWX6QyKt_ZCo,1491
+airflow/providers/sftp/get_provider_info.py,sha256=YKtWfzGpCEAE0RAeyWbZ5r3W-DAU5rxk7zjGUcH46r0,4324
 airflow/providers/sftp/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/sftp/decorators/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/sftp/decorators/sensors/sftp.py,sha256=deps7xldfdLcO5mwxuZ9SJ7wFNrf2DZWMSt_KYMmGG4,2888
 airflow/providers/sftp/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/hooks/sftp.py,sha256=aDdBE7yNkh-H0SFc3Uqkx0LP4lWtu3rDqqeYwqxrvN0,24945
+airflow/providers/sftp/hooks/sftp.py,sha256=srBUSfeqxI37T7ytSfAQo8zX-GDWoDdWKFXHD1xwHCs,26457
 airflow/providers/sftp/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/sftp/operators/sftp.py,sha256=skNYBFzdZUnkU_SpDf_m6HxnUuTVd5KDwJDRJnly7NE,11670
 airflow/providers/sftp/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/sensors/sftp.py,sha256=2McTofMV0IPM_NmxzH9EwWmmeLWuQSrfF8hq6aMtmGE,8077
+airflow/providers/sftp/sensors/sftp.py,sha256=GF8GGnkuSKrg2kqNMGWLm99SE9rtrtDqWxoOPE1QdY8,8046
 airflow/providers/sftp/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/sftp/triggers/sftp.py,sha256=fSi-I5FocNQblHt4GYfGispFgOOl8XQ9Vk9ZFLcv_Sw,6182
-apache_airflow_providers_sftp-5.1.0rc1.dist-info/entry_points.txt,sha256=Fa1IkUHV6qnIuwLd0U7tKoklbLXXVrbB2hhG6N7Q-zo,100
-apache_airflow_providers_sftp-5.1.0rc1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-apache_airflow_providers_sftp-5.1.0rc1.dist-info/METADATA,sha256=r3I0JViv1sqlwuHbQok8IAoFfRMsbz7kxOnK0Hwx0II,5704
-apache_airflow_providers_sftp-5.1.0rc1.dist-info/RECORD,,
+apache_airflow_providers_sftp-5.1.1rc1.dist-info/entry_points.txt,sha256=Fa1IkUHV6qnIuwLd0U7tKoklbLXXVrbB2hhG6N7Q-zo,100
+apache_airflow_providers_sftp-5.1.1rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+apache_airflow_providers_sftp-5.1.1rc1.dist-info/METADATA,sha256=VV9CC3toyzU5oIqBP4ijawbV5MS1I-lDp-0N6AfJrTo,5690
+apache_airflow_providers_sftp-5.1.1rc1.dist-info/RECORD,,

{apache_airflow_providers_sftp-5.1.0rc1.dist-info → apache_airflow_providers_sftp-5.1.1rc1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.11.0
 Root-Is-Purelib: true
 Tag: py3-none-any