apache-airflow-providers-sftp 5.1.2__py3-none-any.whl → 5.2.0rc1__py3-none-any.whl

airflow/providers/sftp/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "5.1.2"
+__version__ = "5.2.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/sftp/decorators/sensors/sftp.py

@@ -20,7 +20,16 @@ from __future__ import annotations
 from collections.abc import Sequence
 from typing import Callable
 
-from airflow.decorators.base import TaskDecorator, get_unique_task_id, task_decorator_factory
+from airflow.providers.sftp.version_compat import AIRFLOW_V_3_0_PLUS
+
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.sdk.bases.decorator import TaskDecorator, get_unique_task_id, task_decorator_factory
+else:
+    from airflow.decorators.base import (  # type: ignore[no-redef]
+        TaskDecorator,
+        get_unique_task_id,
+        task_decorator_factory,
+    )
 from airflow.providers.sftp.sensors.sftp import SFTPSensor
 
 

airflow/providers/sftp/get_provider_info.py

@@ -26,54 +26,6 @@ def get_provider_info():
         "package-name": "apache-airflow-providers-sftp",
         "name": "SFTP",
         "description": "`SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__\n",
-        "state": "ready",
-        "source-date-epoch": 1743836705,
-        "versions": [
-            "5.1.2",
-            "5.1.1",
-            "5.1.0",
-            "5.0.0",
-            "4.11.1",
-            "4.11.0",
-            "4.10.3",
-            "4.10.2",
-            "4.10.1",
-            "4.10.0",
-            "4.9.1",
-            "4.9.0",
-            "4.8.1",
-            "4.8.0",
-            "4.7.0",
-            "4.6.1",
-            "4.6.0",
-            "4.5.0",
-            "4.4.0",
-            "4.3.1",
-            "4.3.0",
-            "4.2.4",
-            "4.2.3",
-            "4.2.2",
-            "4.2.1",
-            "4.2.0",
-            "4.1.0",
-            "4.0.0",
-            "3.0.0",
-            "2.6.0",
-            "2.5.2",
-            "2.5.1",
-            "2.5.0",
-            "2.4.1",
-            "2.4.0",
-            "2.3.0",
-            "2.2.0",
-            "2.1.1",
-            "2.1.0",
-            "2.0.0",
-            "1.2.0",
-            "1.1.1",
-            "1.1.0",
-            "1.0.0",
-        ],
         "integrations": [
             {
                 "integration-name": "SSH File Transfer Protocol (SFTP)",
@@ -118,15 +70,4 @@ def get_provider_info():
                 "python-modules": ["airflow.providers.sftp.triggers.sftp"],
             }
         ],
-        "dependencies": [
-            "apache-airflow>=2.9.0",
-            "apache-airflow-providers-ssh>=2.1.0",
-            "paramiko>=2.9.0",
-            "asyncssh>=2.12.0",
-        ],
-        "optional-dependencies": {
-            "common.compat": ["apache-airflow-providers-common-compat"],
-            "openlineage": ["apache-airflow-providers-openlineage"],
-        },
-        "devel-dependencies": [],
     }

airflow/providers/sftp/hooks/sftp.py

@@ -19,6 +19,7 @@
 
 from __future__ import annotations
 
+import concurrent.futures
 import datetime
 import os
 import stat
@@ -339,6 +340,63 @@ class SFTPHook(SSHHook):
                 new_local_path = os.path.join(local_full_path, os.path.relpath(file_path, remote_full_path))
                 self.retrieve_file(file_path, new_local_path, prefetch)
 
+    def retrieve_directory_concurrently(
+        self, remote_full_path: str, local_full_path: str, workers: int = os.cpu_count() or 2
+    ) -> None:
+        """
+        Transfer the remote directory to a local location concurrently.
+
+        If local_full_path is a string path, the directory will be put
+        at that location.
+
+        :param remote_full_path: full path to the remote directory
+        :param local_full_path: full path to the local directory
+        :param prefetch: controls whether prefetch is performed (default: True)
+        :param workers: number of workers to use for concurrent transfer (default: number of CPUs or 2 if undetermined)
+        """
+
+        def retrieve_file_chunk(
+            conn: SFTPClient, local_file_chunk: list[str], remote_file_chunk: list[str], prefetch: bool = True
+        ):
+            for local_file, remote_file in zip(local_file_chunk, remote_file_chunk):
+                conn.get(remote_file, local_file, prefetch=prefetch)
+
+        with self.get_managed_conn():
+            if Path(local_full_path).exists():
+                raise AirflowException(f"{local_full_path} already exists")
+            Path(local_full_path).mkdir(parents=True)
+            new_local_file_paths, remote_file_paths = [], []
+            files, dirs, _ = self.get_tree_map(remote_full_path)
+            for dir_path in dirs:
+                new_local_path = os.path.join(local_full_path, os.path.relpath(dir_path, remote_full_path))
+                Path(new_local_path).mkdir(parents=True, exist_ok=True)
+            for file in files:
+                remote_file_paths.append(file)
+                new_local_file_paths.append(
+                    os.path.join(local_full_path, os.path.relpath(file, remote_full_path))
+                )
+        remote_file_chunks = [remote_file_paths[i::workers] for i in range(workers)]
+        local_file_chunks = [new_local_file_paths[i::workers] for i in range(workers)]
+        self.log.info("Opening %s new SFTP connections", workers)
+        conns = [SFTPHook(ssh_conn_id=self.ssh_conn_id).get_conn() for _ in range(workers)]
+        try:
+            self.log.info("Retrieving files concurrently with %s threads", workers)
+            with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+                futures = [
+                    executor.submit(
+                        retrieve_file_chunk,
+                        conns[i],
+                        local_file_chunks[i],
+                        remote_file_chunks[i],
+                    )
+                    for i in range(workers)
+                ]
+                for future in concurrent.futures.as_completed(futures):
+                    future.result()
+        finally:
+            for conn in conns:
+                conn.close()
+
     def store_directory(self, remote_full_path: str, local_full_path: str, confirm: bool = True) -> None:
         """
         Transfer a local directory to the remote location.
@@ -367,6 +425,71 @@ class SFTPHook(SSHHook):
                     )
                     self.store_file(new_remote_path, file_path, confirm)
 
+    def store_directory_concurrently(
+        self,
+        remote_full_path: str,
+        local_full_path: str,
+        confirm: bool = True,
+        workers: int = os.cpu_count() or 2,
+    ) -> None:
+        """
+        Transfer a local directory to the remote location concurrently.
+
+        If local_full_path is a string path, the directory will be read
+        from that location.
+
+        :param remote_full_path: full path to the remote directory
+        :param local_full_path: full path to the local directory
+        :param confirm: whether to confirm the file size after transfer (default: True)
+        :param workers: number of workers to use for concurrent transfer (default: number of CPUs or 2 if undetermined)
+        """
+
+        def store_file_chunk(
+            conn: SFTPClient, local_file_chunk: list[str], remote_file_chunk: list[str], confirm: bool
+        ):
+            for local_file, remote_file in zip(local_file_chunk, remote_file_chunk):
+                conn.put(local_file, remote_file, confirm=confirm)
+
+        with self.get_managed_conn():
+            if self.path_exists(remote_full_path):
+                raise AirflowException(f"{remote_full_path} already exists")
+            self.create_directory(remote_full_path)
+
+            local_file_paths, new_remote_file_paths = [], []
+            for root, dirs, files in os.walk(local_full_path):
+                for dir_name in dirs:
+                    dir_path = os.path.join(root, dir_name)
+                    new_remote_path = os.path.join(
+                        remote_full_path, os.path.relpath(dir_path, local_full_path)
+                    )
+                    self.create_directory(new_remote_path)
+                for file_name in files:
+                    file_path = os.path.join(root, file_name)
+                    new_remote_path = os.path.join(
+                        remote_full_path, os.path.relpath(file_path, local_full_path)
+                    )
+                    local_file_paths.append(file_path)
+                    new_remote_file_paths.append(new_remote_path)
+
+        remote_file_chunks = [new_remote_file_paths[i::workers] for i in range(workers)]
+        local_file_chunks = [local_file_paths[i::workers] for i in range(workers)]
+        self.log.info("Opening %s new SFTP connections", workers)
+        conns = [SFTPHook(ssh_conn_id=self.ssh_conn_id).get_conn() for _ in range(workers)]
+        try:
+            self.log.info("Storing files concurrently with %s threads", workers)
+            with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+                futures = [
+                    executor.submit(
+                        store_file_chunk, conns[i], local_file_chunks[i], remote_file_chunks[i], confirm
+                    )
+                    for i in range(workers)
+                ]
+                for future in concurrent.futures.as_completed(futures):
+                    future.result()
+        finally:
+            for conn in conns:
+                conn.close()
+
     def get_mod_time(self, path: str) -> str:
         """
         Get an entry's modification time.

airflow/providers/sftp/operators/sftp.py

@@ -74,6 +74,8 @@ class SFTPOperator(BaseOperator):
                 create_intermediate_dirs=True,
                 dag=dag,
             )
+    :param concurrency: Number of threads when transferring directories. Each thread opens a new SFTP connection.
+        This parameter is used only when transferring directories, not individual files. (Default is 1)
 
     """
 
@@ -90,6 +92,7 @@ class SFTPOperator(BaseOperator):
         operation: str = SFTPOperation.PUT,
         confirm: bool = True,
         create_intermediate_dirs: bool = False,
+        concurrency: int = 1,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -101,6 +104,7 @@ class SFTPOperator(BaseOperator):
         self.create_intermediate_dirs = create_intermediate_dirs
         self.local_filepath = local_filepath
         self.remote_filepath = remote_filepath
+        self.concurrency = concurrency
 
     def execute(self, context: Any) -> str | list[str] | None:
         if self.local_filepath is None:
@@ -132,6 +136,9 @@ class SFTPOperator(BaseOperator):
                 f"expected {SFTPOperation.GET} or {SFTPOperation.PUT} or {SFTPOperation.DELETE}."
             )
 
+        if self.concurrency < 1:
+            raise ValueError(f"concurrency should be greater than 0, got {self.concurrency}")
+
         file_msg = None
         try:
             if self.ssh_conn_id:
@@ -161,7 +168,14 @@ class SFTPOperator(BaseOperator):
                         file_msg = f"from {_remote_filepath} to {_local_filepath}"
                         self.log.info("Starting to transfer %s", file_msg)
                         if self.sftp_hook.isdir(_remote_filepath):
-                            self.sftp_hook.retrieve_directory(_remote_filepath, _local_filepath)
+                            if self.concurrency > 1:
+                                self.sftp_hook.retrieve_directory_concurrently(
+                                    _remote_filepath,
+                                    _local_filepath,
+                                    workers=self.concurrency,
+                                )
+                            elif self.concurrency == 1:
+                                self.sftp_hook.retrieve_directory(_remote_filepath, _local_filepath)
                         else:
                             self.sftp_hook.retrieve_file(_remote_filepath, _local_filepath)
                     elif self.operation.lower() == SFTPOperation.PUT:
@@ -171,9 +185,17 @@ class SFTPOperator(BaseOperator):
                         file_msg = f"from {_local_filepath} to {_remote_filepath}"
                         self.log.info("Starting to transfer file %s", file_msg)
                         if os.path.isdir(_local_filepath):
-                            self.sftp_hook.store_directory(
-                                _remote_filepath, _local_filepath, confirm=self.confirm
-                            )
+                            if self.concurrency > 1:
+                                self.sftp_hook.store_directory_concurrently(
+                                    _remote_filepath,
+                                    _local_filepath,
+                                    confirm=self.confirm,
+                                    workers=self.concurrency,
+                                )
+                            elif self.concurrency == 1:
+                                self.sftp_hook.store_directory(
+                                    _remote_filepath, _local_filepath, confirm=self.confirm
+                                )
                         else:
                             self.sftp_hook.store_file(_remote_filepath, _local_filepath, confirm=self.confirm)
             elif self.operation.lower() == SFTPOperation.DELETE:

airflow/providers/sftp/version_compat.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
+# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
+# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
+# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
+#
+from __future__ import annotations
+
+
+def get_base_airflow_version_tuple() -> tuple[int, int, int]:
+    from packaging.version import Version
+
+    from airflow import __version__
+
+    airflow_version = Version(__version__)
+    return airflow_version.major, airflow_version.minor, airflow_version.micro
+
+
+AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
+AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)

{apache_airflow_providers_sftp-5.1.2.dist-info → apache_airflow_providers_sftp-5.2.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-sftp
-Version: 5.1.2
+Version: 5.2.0rc1
 Summary: Provider package apache-airflow-providers-sftp for Apache Airflow
 Keywords: airflow-provider,sftp,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,15 +20,15 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0
-Requires-Dist: apache-airflow-providers-ssh>=2.1.0
+Requires-Dist: apache-airflow>=2.9.0rc0
+Requires-Dist: apache-airflow-providers-ssh>=2.1.0rc0
 Requires-Dist: paramiko>=2.9.0
 Requires-Dist: asyncssh>=2.12.0
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.2
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.2.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -61,7 +61,7 @@ Provides-Extra: openlineage
 
 Package ``apache-airflow-providers-sftp``
 
-Release: ``5.1.2``
+Release: ``5.2.0``
 
 
 `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
@@ -74,7 +74,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
 are in ``airflow.providers.sftp`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.2.0/>`_.
 
 Installation
 ------------
@@ -119,5 +119,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.2.0/changelog.html>`_.
 

apache_airflow_providers_sftp-5.2.0rc1.dist-info/RECORD

@@ -0,0 +1,19 @@
+airflow/providers/sftp/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+airflow/providers/sftp/__init__.py,sha256=MgTmkrkMblXGjX5-oMRC3zs1BqvT-AiuyFUkdo0vxHQ,1491
+airflow/providers/sftp/get_provider_info.py,sha256=_IqUGQ-rKpZsSAsXdTsGYzfzJ3X57duhn-2b0-rFOz0,2905
+airflow/providers/sftp/version_compat.py,sha256=aHg90_DtgoSnQvILFICexMyNlHlALBdaeWqkX3dFDug,1605
+airflow/providers/sftp/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/sftp/decorators/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/sftp/decorators/sensors/sftp.py,sha256=htmylsF0h93Loova_rq7_ebJQAjBgJlbHgC9rSVpwww,3152
+airflow/providers/sftp/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/sftp/hooks/sftp.py,sha256=JlTt-JFP-2L8pGFARYDJXOJ-wEQuI0VZjK6ZQi9C3us,32315
+airflow/providers/sftp/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/sftp/operators/sftp.py,sha256=WSHPIGLUZD62knz_uBweVfMIi4tqMCDmUHBl9umKrbY,12940
+airflow/providers/sftp/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/sftp/sensors/sftp.py,sha256=WpotlHzsWRgtJnu44zEKU3hagKT1J634M-jQYvQ9Z3M,8301
+airflow/providers/sftp/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/sftp/triggers/sftp.py,sha256=fSi-I5FocNQblHt4GYfGispFgOOl8XQ9Vk9ZFLcv_Sw,6182
+apache_airflow_providers_sftp-5.2.0rc1.dist-info/entry_points.txt,sha256=Fa1IkUHV6qnIuwLd0U7tKoklbLXXVrbB2hhG6N7Q-zo,100
+apache_airflow_providers_sftp-5.2.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_sftp-5.2.0rc1.dist-info/METADATA,sha256=HLOKnf-O-20vDvA4v6_NK_3i6Fgos-v86fuhkirFCWM,5686
+apache_airflow_providers_sftp-5.2.0rc1.dist-info/RECORD,,

{apache_airflow_providers_sftp-5.1.2.dist-info → apache_airflow_providers_sftp-5.2.0rc1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.11.0
+Generator: flit 3.12.0
 Root-Is-Purelib: true
 Tag: py3-none-any

apache_airflow_providers_sftp-5.1.2.dist-info/RECORD

@@ -1,18 +0,0 @@
-airflow/providers/sftp/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/sftp/__init__.py,sha256=WdG_S0JGjH55B8nVQJCyjdE-82LLOGB5teA3VeIPUC8,1491
-airflow/providers/sftp/get_provider_info.py,sha256=RSPeKARVeBql8BB3Wt4PRAHxTvydeRz5Y46QsDzZlcY,4345
-airflow/providers/sftp/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/decorators/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/decorators/sensors/sftp.py,sha256=deps7xldfdLcO5mwxuZ9SJ7wFNrf2DZWMSt_KYMmGG4,2888
-airflow/providers/sftp/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/hooks/sftp.py,sha256=srBUSfeqxI37T7ytSfAQo8zX-GDWoDdWKFXHD1xwHCs,26457
-airflow/providers/sftp/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/operators/sftp.py,sha256=skNYBFzdZUnkU_SpDf_m6HxnUuTVd5KDwJDRJnly7NE,11670
-airflow/providers/sftp/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/sensors/sftp.py,sha256=WpotlHzsWRgtJnu44zEKU3hagKT1J634M-jQYvQ9Z3M,8301
-airflow/providers/sftp/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/sftp/triggers/sftp.py,sha256=fSi-I5FocNQblHt4GYfGispFgOOl8XQ9Vk9ZFLcv_Sw,6182
-apache_airflow_providers_sftp-5.1.2.dist-info/entry_points.txt,sha256=Fa1IkUHV6qnIuwLd0U7tKoklbLXXVrbB2hhG6N7Q-zo,100
-apache_airflow_providers_sftp-5.1.2.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
-apache_airflow_providers_sftp-5.1.2.dist-info/METADATA,sha256=5Ls4zFXTOZP0560ym6NzVQXRpWhPz1Sb8K9nJW0aRJQ,5677
-apache_airflow_providers_sftp-5.1.2.dist-info/RECORD,,