apache-airflow-providers-sftp 5.1.0rc1__tar.gz → 5.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/PKG-INFO +26 -27
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/README.rst +20 -21
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/pyproject.toml +30 -10
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/__init__.py +1 -1
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/get_provider_info.py +3 -1
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/hooks/sftp.py +77 -36
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/sensors/sftp.py +0 -1
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/LICENSE +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/decorators/__init__.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/decorators/sensors/__init__.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/decorators/sensors/sftp.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/hooks/__init__.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/operators/__init__.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/operators/sftp.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/sensors/__init__.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/triggers/__init__.py +0 -0
- {apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/src/airflow/providers/sftp/triggers/sftp.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: apache-airflow-providers-sftp
|
3
|
-
Version: 5.1.
|
3
|
+
Version: 5.1.1
|
4
4
|
Summary: Provider package apache-airflow-providers-sftp for Apache Airflow
|
5
5
|
Keywords: airflow-provider,sftp,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
@@ -20,15 +20,15 @@ Classifier: Programming Language :: Python :: 3.10
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
23
|
-
Requires-Dist: apache-airflow>=2.9.
|
24
|
-
Requires-Dist: apache-airflow-providers-ssh>=2.1.
|
23
|
+
Requires-Dist: apache-airflow>=2.9.0
|
24
|
+
Requires-Dist: apache-airflow-providers-ssh>=2.1.0
|
25
25
|
Requires-Dist: paramiko>=2.9.0
|
26
26
|
Requires-Dist: asyncssh>=2.12.0
|
27
27
|
Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
|
28
28
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
29
29
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
30
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
31
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
30
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/changelog.html
|
31
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1
|
32
32
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
33
33
|
Project-URL: Source Code, https://github.com/apache/airflow
|
34
34
|
Project-URL: Twitter, https://x.com/ApacheAirflow
|
@@ -37,32 +37,31 @@ Provides-Extra: common-compat
|
|
37
37
|
Provides-Extra: openlineage
|
38
38
|
|
39
39
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
40
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
41
|
+
or more contributor license agreements. See the NOTICE file
|
42
|
+
distributed with this work for additional information
|
43
|
+
regarding copyright ownership. The ASF licenses this file
|
44
|
+
to you under the Apache License, Version 2.0 (the
|
45
|
+
"License"); you may not use this file except in compliance
|
46
|
+
with the License. You may obtain a copy of the License at
|
47
47
|
|
48
|
-
|
48
|
+
.. http://www.apache.org/licenses/LICENSE-2.0
|
49
49
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
50
|
+
.. Unless required by applicable law or agreed to in writing,
|
51
|
+
software distributed under the License is distributed on an
|
52
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
53
|
+
KIND, either express or implied. See the License for the
|
54
|
+
specific language governing permissions and limitations
|
55
|
+
under the License.
|
56
56
|
|
57
|
-
|
58
|
-
|
59
|
-
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
60
|
-
`PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
57
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
|
61
58
|
|
59
|
+
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
60
|
+
``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY
|
62
61
|
|
63
62
|
Package ``apache-airflow-providers-sftp``
|
64
63
|
|
65
|
-
Release: ``5.1.
|
64
|
+
Release: ``5.1.1``
|
66
65
|
|
67
66
|
|
68
67
|
`SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
|
@@ -75,7 +74,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
|
|
75
74
|
are in ``airflow.providers.sftp`` python package.
|
76
75
|
|
77
76
|
You can find package information and changelog for the provider
|
78
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
77
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/>`_.
|
79
78
|
|
80
79
|
Installation
|
81
80
|
------------
|
@@ -120,5 +119,5 @@ Dependent package
|
|
120
119
|
================================================================================================================== =================
|
121
120
|
|
122
121
|
The changelog for the provider package can be found in the
|
123
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
122
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/changelog.html>`_.
|
124
123
|
|
@@ -1,30 +1,29 @@
|
|
1
1
|
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
2
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
or more contributor license agreements. See the NOTICE file
|
4
|
+
distributed with this work for additional information
|
5
|
+
regarding copyright ownership. The ASF licenses this file
|
6
|
+
to you under the Apache License, Version 2.0 (the
|
7
|
+
"License"); you may not use this file except in compliance
|
8
|
+
with the License. You may obtain a copy of the License at
|
9
9
|
|
10
|
-
|
10
|
+
.. http://www.apache.org/licenses/LICENSE-2.0
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
12
|
+
.. Unless required by applicable law or agreed to in writing,
|
13
|
+
software distributed under the License is distributed on an
|
14
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
KIND, either express or implied. See the License for the
|
16
|
+
specific language governing permissions and limitations
|
17
|
+
under the License.
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
22
|
-
`PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
19
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
|
23
20
|
|
21
|
+
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
22
|
+
``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY
|
24
23
|
|
25
24
|
Package ``apache-airflow-providers-sftp``
|
26
25
|
|
27
|
-
Release: ``5.1.
|
26
|
+
Release: ``5.1.1``
|
28
27
|
|
29
28
|
|
30
29
|
`SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
|
@@ -37,7 +36,7 @@ This is a provider package for ``sftp`` provider. All classes for this provider
|
|
37
36
|
are in ``airflow.providers.sftp`` python package.
|
38
37
|
|
39
38
|
You can find package information and changelog for the provider
|
40
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/>`_.
|
41
40
|
|
42
41
|
Installation
|
43
42
|
------------
|
@@ -82,4 +81,4 @@ Dependent package
|
|
82
81
|
================================================================================================================== =================
|
83
82
|
|
84
83
|
The changelog for the provider package can be found in the
|
85
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
84
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/changelog.html>`_.
|
{apache_airflow_providers_sftp-5.1.0rc1 → apache_airflow_providers_sftp-5.1.1}/pyproject.toml
RENAMED
@@ -20,12 +20,12 @@
|
|
20
20
|
# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
|
21
21
|
# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
22
22
|
[build-system]
|
23
|
-
requires = ["flit_core==3.
|
23
|
+
requires = ["flit_core==3.11.0"]
|
24
24
|
build-backend = "flit_core.buildapi"
|
25
25
|
|
26
26
|
[project]
|
27
27
|
name = "apache-airflow-providers-sftp"
|
28
|
-
version = "5.1.
|
28
|
+
version = "5.1.1"
|
29
29
|
description = "Provider package apache-airflow-providers-sftp for Apache Airflow"
|
30
30
|
readme = "README.rst"
|
31
31
|
authors = [
|
@@ -52,11 +52,13 @@ classifiers = [
|
|
52
52
|
]
|
53
53
|
requires-python = "~=3.9"
|
54
54
|
|
55
|
-
# The dependencies should be modified in place in the generated file
|
55
|
+
# The dependencies should be modified in place in the generated file.
|
56
56
|
# Any change in the dependencies is preserved when the file is regenerated
|
57
|
+
# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
|
58
|
+
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
57
59
|
dependencies = [
|
58
|
-
"apache-airflow>=2.9.
|
59
|
-
"apache-airflow-providers-ssh>=2.1.
|
60
|
+
"apache-airflow>=2.9.0",
|
61
|
+
"apache-airflow-providers-ssh>=2.1.0",
|
60
62
|
"paramiko>=2.9.0",
|
61
63
|
"asyncssh>=2.12.0",
|
62
64
|
]
|
@@ -71,9 +73,30 @@ dependencies = [
|
|
71
73
|
"apache-airflow-providers-openlineage"
|
72
74
|
]
|
73
75
|
|
76
|
+
[dependency-groups]
|
77
|
+
dev = [
|
78
|
+
"apache-airflow",
|
79
|
+
"apache-airflow-task-sdk",
|
80
|
+
"apache-airflow-devel-common",
|
81
|
+
"apache-airflow-providers-common-compat",
|
82
|
+
"apache-airflow-providers-openlineage",
|
83
|
+
"apache-airflow-providers-ssh",
|
84
|
+
# Additional devel dependencies (do not remove this line and add extra development dependencies)
|
85
|
+
]
|
86
|
+
|
87
|
+
[tool.uv.sources]
|
88
|
+
# These names must match the names as defined in the pyproject.toml of the workspace items,
|
89
|
+
# *not* the workspace folder paths
|
90
|
+
apache-airflow = {workspace = true}
|
91
|
+
apache-airflow-devel-common = {workspace = true}
|
92
|
+
apache-airflow-task-sdk = {workspace = true}
|
93
|
+
apache-airflow-providers-common-sql = {workspace = true}
|
94
|
+
apache-airflow-providers-fab = {workspace = true}
|
95
|
+
apache-airflow-providers-standard = {workspace = true}
|
96
|
+
|
74
97
|
[project.urls]
|
75
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
76
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.
|
98
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1"
|
99
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-sftp/5.1.1/changelog.html"
|
77
100
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
78
101
|
"Source Code" = "https://github.com/apache/airflow"
|
79
102
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
@@ -85,6 +108,3 @@ provider_info = "airflow.providers.sftp.get_provider_info:get_provider_info"
|
|
85
108
|
|
86
109
|
[tool.flit.module]
|
87
110
|
name = "airflow.providers.sftp"
|
88
|
-
|
89
|
-
[tool.pytest.ini_options]
|
90
|
-
ignore = "tests/system/"
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "5.1.
|
32
|
+
__version__ = "5.1.1"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
35
|
"2.9.0"
|
@@ -27,8 +27,9 @@ def get_provider_info():
|
|
27
27
|
"name": "SFTP",
|
28
28
|
"description": "`SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__\n",
|
29
29
|
"state": "ready",
|
30
|
-
"source-date-epoch":
|
30
|
+
"source-date-epoch": 1741509612,
|
31
31
|
"versions": [
|
32
|
+
"5.1.1",
|
32
33
|
"5.1.0",
|
33
34
|
"5.0.0",
|
34
35
|
"4.11.1",
|
@@ -126,4 +127,5 @@ def get_provider_info():
|
|
126
127
|
"common.compat": ["apache-airflow-providers-common-compat"],
|
127
128
|
"openlineage": ["apache-airflow-providers-openlineage"],
|
128
129
|
},
|
130
|
+
"devel-dependencies": [],
|
129
131
|
}
|
@@ -24,7 +24,7 @@ import os
|
|
24
24
|
import stat
|
25
25
|
import warnings
|
26
26
|
from collections.abc import Generator, Sequence
|
27
|
-
from contextlib import
|
27
|
+
from contextlib import contextmanager
|
28
28
|
from fnmatch import fnmatch
|
29
29
|
from io import BytesIO
|
30
30
|
from pathlib import Path
|
@@ -38,6 +38,7 @@ from airflow.hooks.base import BaseHook
|
|
38
38
|
from airflow.providers.ssh.hooks.ssh import SSHHook
|
39
39
|
|
40
40
|
if TYPE_CHECKING:
|
41
|
+
from paramiko import SSHClient
|
41
42
|
from paramiko.sftp_attr import SFTPAttributes
|
42
43
|
from paramiko.sftp_client import SFTPClient
|
43
44
|
|
@@ -87,6 +88,8 @@ class SFTPHook(SSHHook):
|
|
87
88
|
*args,
|
88
89
|
**kwargs,
|
89
90
|
) -> None:
|
91
|
+
self.conn: SFTPClient | None = None
|
92
|
+
|
90
93
|
# TODO: remove support for ssh_hook when it is removed from SFTPOperator
|
91
94
|
if kwargs.get("ssh_hook") is not None:
|
92
95
|
warnings.warn(
|
@@ -108,14 +111,46 @@ class SFTPHook(SSHHook):
|
|
108
111
|
kwargs["host_proxy_cmd"] = host_proxy_cmd
|
109
112
|
self.ssh_conn_id = ssh_conn_id
|
110
113
|
|
114
|
+
self._ssh_conn: SSHClient | None = None
|
115
|
+
self._sftp_conn: SFTPClient | None = None
|
116
|
+
self._conn_count = 0
|
117
|
+
|
111
118
|
super().__init__(*args, **kwargs)
|
112
119
|
|
120
|
+
def get_conn(self) -> SFTPClient: # type: ignore[override]
|
121
|
+
"""Open an SFTP connection to the remote host."""
|
122
|
+
if self.conn is None:
|
123
|
+
self.conn = super().get_conn().open_sftp()
|
124
|
+
return self.conn
|
125
|
+
|
126
|
+
def close_conn(self) -> None:
|
127
|
+
"""Close the SFTP connection."""
|
128
|
+
if self.conn is not None:
|
129
|
+
self.conn.close()
|
130
|
+
self.conn = None
|
131
|
+
|
113
132
|
@contextmanager
|
114
|
-
def
|
133
|
+
def get_managed_conn(self) -> Generator[SFTPClient, None, None]:
|
115
134
|
"""Context manager that closes the connection after use."""
|
116
|
-
|
117
|
-
|
118
|
-
|
135
|
+
if self._sftp_conn is None:
|
136
|
+
ssh_conn: SSHClient = super().get_conn()
|
137
|
+
self._ssh_conn = ssh_conn
|
138
|
+
self._sftp_conn = ssh_conn.open_sftp()
|
139
|
+
self._conn_count += 1
|
140
|
+
|
141
|
+
try:
|
142
|
+
yield self._sftp_conn
|
143
|
+
finally:
|
144
|
+
self._conn_count -= 1
|
145
|
+
if self._conn_count == 0 and self._ssh_conn is not None and self._sftp_conn is not None:
|
146
|
+
self._sftp_conn.close()
|
147
|
+
self._sftp_conn = None
|
148
|
+
self._ssh_conn.close()
|
149
|
+
self._ssh_conn = None
|
150
|
+
|
151
|
+
def get_conn_count(self) -> int:
|
152
|
+
"""Get the number of open connections."""
|
153
|
+
return self._conn_count
|
119
154
|
|
120
155
|
def describe_directory(self, path: str) -> dict[str, dict[str, str | int | None]]:
|
121
156
|
"""
|
@@ -126,7 +161,7 @@ class SFTPHook(SSHHook):
|
|
126
161
|
|
127
162
|
:param path: full path to the remote directory
|
128
163
|
"""
|
129
|
-
with self.
|
164
|
+
with self.get_managed_conn() as conn: # type: SFTPClient
|
130
165
|
flist = sorted(conn.listdir_attr(path), key=lambda x: x.filename)
|
131
166
|
files = {}
|
132
167
|
for f in flist:
|
@@ -144,7 +179,7 @@ class SFTPHook(SSHHook):
|
|
144
179
|
|
145
180
|
:param path: full path to the remote directory to list
|
146
181
|
"""
|
147
|
-
with self.
|
182
|
+
with self.get_managed_conn() as conn:
|
148
183
|
return sorted(conn.listdir(path))
|
149
184
|
|
150
185
|
def list_directory_with_attr(self, path: str) -> list[SFTPAttributes]:
|
@@ -153,7 +188,7 @@ class SFTPHook(SSHHook):
|
|
153
188
|
|
154
189
|
:param path: full path to the remote directory to list
|
155
190
|
"""
|
156
|
-
with self.
|
191
|
+
with self.get_managed_conn() as conn:
|
157
192
|
return [file for file in conn.listdir_attr(path)]
|
158
193
|
|
159
194
|
def mkdir(self, path: str, mode: int = 0o777) -> None:
|
@@ -166,7 +201,7 @@ class SFTPHook(SSHHook):
|
|
166
201
|
:param path: full path to the remote directory to create
|
167
202
|
:param mode: int permissions of octal mode for directory
|
168
203
|
"""
|
169
|
-
with self.
|
204
|
+
with self.get_managed_conn() as conn:
|
170
205
|
conn.mkdir(path, mode=mode)
|
171
206
|
|
172
207
|
def isdir(self, path: str) -> bool:
|
@@ -175,7 +210,7 @@ class SFTPHook(SSHHook):
|
|
175
210
|
|
176
211
|
:param path: full path to the remote directory to check
|
177
212
|
"""
|
178
|
-
with self.
|
213
|
+
with self.get_managed_conn() as conn:
|
179
214
|
try:
|
180
215
|
return stat.S_ISDIR(conn.stat(path).st_mode) # type: ignore
|
181
216
|
except OSError:
|
@@ -187,7 +222,7 @@ class SFTPHook(SSHHook):
|
|
187
222
|
|
188
223
|
:param path: full path to the remote file to check
|
189
224
|
"""
|
190
|
-
with self.
|
225
|
+
with self.get_managed_conn() as conn:
|
191
226
|
try:
|
192
227
|
return stat.S_ISREG(conn.stat(path).st_mode) # type: ignore
|
193
228
|
except OSError:
|
@@ -216,7 +251,7 @@ class SFTPHook(SSHHook):
|
|
216
251
|
self.create_directory(dirname, mode)
|
217
252
|
if basename:
|
218
253
|
self.log.info("Creating %s", path)
|
219
|
-
with self.
|
254
|
+
with self.get_managed_conn() as conn:
|
220
255
|
conn.mkdir(path, mode=mode)
|
221
256
|
|
222
257
|
def delete_directory(self, path: str, include_files: bool = False) -> None:
|
@@ -232,7 +267,7 @@ class SFTPHook(SSHHook):
|
|
232
267
|
files, dirs, _ = self.get_tree_map(path)
|
233
268
|
dirs = dirs[::-1] # reverse the order for deleting deepest directories first
|
234
269
|
|
235
|
-
with self.
|
270
|
+
with self.get_managed_conn() as conn:
|
236
271
|
for file_path in files:
|
237
272
|
conn.remove(file_path)
|
238
273
|
for dir_path in dirs:
|
@@ -250,7 +285,7 @@ class SFTPHook(SSHHook):
|
|
250
285
|
:param local_full_path: full path to the local file or a file-like buffer
|
251
286
|
:param prefetch: controls whether prefetch is performed (default: True)
|
252
287
|
"""
|
253
|
-
with self.
|
288
|
+
with self.get_managed_conn() as conn:
|
254
289
|
if isinstance(local_full_path, BytesIO):
|
255
290
|
conn.getfo(remote_full_path, local_full_path, prefetch=prefetch)
|
256
291
|
else:
|
@@ -266,7 +301,7 @@ class SFTPHook(SSHHook):
|
|
266
301
|
:param remote_full_path: full path to the remote file
|
267
302
|
:param local_full_path: full path to the local file or a file-like buffer
|
268
303
|
"""
|
269
|
-
with self.
|
304
|
+
with self.get_managed_conn() as conn:
|
270
305
|
if isinstance(local_full_path, BytesIO):
|
271
306
|
conn.putfo(local_full_path, remote_full_path, confirm=confirm)
|
272
307
|
else:
|
@@ -278,7 +313,7 @@ class SFTPHook(SSHHook):
|
|
278
313
|
|
279
314
|
:param path: full path to the remote file
|
280
315
|
"""
|
281
|
-
with self.
|
316
|
+
with self.get_managed_conn() as conn:
|
282
317
|
conn.remove(path)
|
283
318
|
|
284
319
|
def retrieve_directory(self, remote_full_path: str, local_full_path: str, prefetch: bool = True) -> None:
|
@@ -295,13 +330,14 @@ class SFTPHook(SSHHook):
|
|
295
330
|
if Path(local_full_path).exists():
|
296
331
|
raise AirflowException(f"{local_full_path} already exists")
|
297
332
|
Path(local_full_path).mkdir(parents=True)
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
333
|
+
with self.get_managed_conn():
|
334
|
+
files, dirs, _ = self.get_tree_map(remote_full_path)
|
335
|
+
for dir_path in dirs:
|
336
|
+
new_local_path = os.path.join(local_full_path, os.path.relpath(dir_path, remote_full_path))
|
337
|
+
Path(new_local_path).mkdir(parents=True, exist_ok=True)
|
338
|
+
for file_path in files:
|
339
|
+
new_local_path = os.path.join(local_full_path, os.path.relpath(file_path, remote_full_path))
|
340
|
+
self.retrieve_file(file_path, new_local_path, prefetch)
|
305
341
|
|
306
342
|
def store_directory(self, remote_full_path: str, local_full_path: str, confirm: bool = True) -> None:
|
307
343
|
"""
|
@@ -315,16 +351,21 @@ class SFTPHook(SSHHook):
|
|
315
351
|
"""
|
316
352
|
if self.path_exists(remote_full_path):
|
317
353
|
raise AirflowException(f"{remote_full_path} already exists")
|
318
|
-
self.
|
319
|
-
|
320
|
-
for
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
354
|
+
with self.get_managed_conn():
|
355
|
+
self.create_directory(remote_full_path)
|
356
|
+
for root, dirs, files in os.walk(local_full_path):
|
357
|
+
for dir_name in dirs:
|
358
|
+
dir_path = os.path.join(root, dir_name)
|
359
|
+
new_remote_path = os.path.join(
|
360
|
+
remote_full_path, os.path.relpath(dir_path, local_full_path)
|
361
|
+
)
|
362
|
+
self.create_directory(new_remote_path)
|
363
|
+
for file_name in files:
|
364
|
+
file_path = os.path.join(root, file_name)
|
365
|
+
new_remote_path = os.path.join(
|
366
|
+
remote_full_path, os.path.relpath(file_path, local_full_path)
|
367
|
+
)
|
368
|
+
self.store_file(new_remote_path, file_path, confirm)
|
328
369
|
|
329
370
|
def get_mod_time(self, path: str) -> str:
|
330
371
|
"""
|
@@ -332,7 +373,7 @@ class SFTPHook(SSHHook):
|
|
332
373
|
|
333
374
|
:param path: full path to the remote file
|
334
375
|
"""
|
335
|
-
with self.
|
376
|
+
with self.get_managed_conn() as conn:
|
336
377
|
ftp_mdtm = conn.stat(path).st_mtime
|
337
378
|
return datetime.datetime.fromtimestamp(ftp_mdtm).strftime("%Y%m%d%H%M%S") # type: ignore
|
338
379
|
|
@@ -342,7 +383,7 @@ class SFTPHook(SSHHook):
|
|
342
383
|
|
343
384
|
:param path: full path to the remote file or directory
|
344
385
|
"""
|
345
|
-
with self.
|
386
|
+
with self.get_managed_conn() as conn:
|
346
387
|
try:
|
347
388
|
conn.stat(path)
|
348
389
|
except OSError:
|
@@ -441,7 +482,7 @@ class SFTPHook(SSHHook):
|
|
441
482
|
def test_connection(self) -> tuple[bool, str]:
|
442
483
|
"""Test the SFTP connection by calling path with directory."""
|
443
484
|
try:
|
444
|
-
with self.
|
485
|
+
with self.get_managed_conn() as conn:
|
445
486
|
conn.normalize(".")
|
446
487
|
return True, "Connection successfully tested"
|
447
488
|
except Exception as e:
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|