sftppath 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sftppath/AGENTS.md +411 -0
- sftppath/README.md +143 -0
- sftppath/__init__.py +70 -0
- sftppath/_asyncssh/__init__.py +7 -0
- sftppath/_asyncssh/_attrs.py +99 -0
- sftppath/_asyncssh/_backend.py +212 -0
- sftppath/_asyncssh/_cache.py +130 -0
- sftppath/_asyncssh/_client.py +187 -0
- sftppath/_asyncssh/_connect.py +75 -0
- sftppath/_asyncssh/_file.py +113 -0
- sftppath/_asyncssh/_loop.py +164 -0
- sftppath/_asyncssh/_translate.py +98 -0
- sftppath/_asyncssh/_walk.py +442 -0
- sftppath/_backend.py +133 -0
- sftppath/_checkfile.py +216 -0
- sftppath/_errors.py +104 -0
- sftppath/_exit.py +57 -0
- sftppath/_namespace.py +35 -0
- sftppath/_paramiko.py +564 -0
- sftppath/_paramiko_config.py +65 -0
- sftppath/_path.py +732 -0
- sftppath/_rename.py +33 -0
- sftppath/_select.py +128 -0
- sftppath/_sshconfig.py +258 -0
- sftppath/_version.py +11 -0
- sftppath/exceptions.py +28 -0
- sftppath/py.typed +0 -0
- sftppath/uri.py +478 -0
- sftppath-0.1.0.dist-info/METADATA +190 -0
- sftppath-0.1.0.dist-info/RECORD +33 -0
- sftppath-0.1.0.dist-info/WHEEL +4 -0
- sftppath-0.1.0.dist-info/entry_points.txt +2 -0
- sftppath-0.1.0.dist-info/licenses/LICENSE +21 -0
sftppath/AGENTS.md
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
# `sftppath` — public API header
|
|
2
|
+
|
|
3
|
+
Header-file-style reference for the `sftppath` package: every public export with
|
|
4
|
+
its signature, arguments, contract and gotchas, so the package can be used
|
|
5
|
+
without reading its source. It ships inside the package and is
|
|
6
|
+
self-contained. Development documentation lives with the source at
|
|
7
|
+
<https://github.com/jose-pr/sftppath>.
|
|
8
|
+
|
|
9
|
+
Names are imported from `sftppath` (the path and the backends), from
|
|
10
|
+
`sftppath.exceptions` (the errors, also exported from the root) and from
|
|
11
|
+
`sftppath.uri` (the `sftp:` scheme of `uripath`); a module whose name starts
|
|
12
|
+
with an underscore is private. `import sftppath` imports neither SSH library
|
|
13
|
+
nor `uripath`: `SFTPBackend` and `AsyncSSHBackend` are loaded the first time
|
|
14
|
+
they are named, and naming one whose library is not installed raises an
|
|
15
|
+
`ImportError` naming its extra.
|
|
16
|
+
|
|
17
|
+
`pip install "sftppath[paramiko]"` or `"sftppath[asyncssh]"` installs an SSH
|
|
18
|
+
library; with neither, a path can still be built, named, joined and compared
|
|
19
|
+
(its endpoint keywords are not checked), and its first request raises an
|
|
20
|
+
`ImportError` naming both extras. `[uri]` adds `uripath` and with it the `sftp:`
|
|
21
|
+
scheme, `[all]` is the three together. The one required dependency is
|
|
22
|
+
`pathapi>=0.1.0,<0.2`. The root's `__version__` is the version of the
|
|
23
|
+
installed distribution.
|
|
24
|
+
|
|
25
|
+
## SFTP paths (`sftppath`)
|
|
26
|
+
|
|
27
|
+
### `SFTPPath`
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
class SFTPPath(Path):
|
|
31
|
+
backend
|
|
32
|
+
def __init__(
|
|
33
|
+
self, *segments, backend=None, host=None, port=None, user=None,
|
|
34
|
+
password=None, ssh_config=..., timeout=..., connect_opts=None, **options,
|
|
35
|
+
) -> None: ...
|
|
36
|
+
def as_uri(self) -> str: ...
|
|
37
|
+
def is_local(self) -> bool: ...
|
|
38
|
+
def readlink(self) -> SFTPPath: ...
|
|
39
|
+
def replace(self, target) -> SFTPPath: ...
|
|
40
|
+
def rename(self, target) -> SFTPPath: ...
|
|
41
|
+
def hardlink_to(self, target) -> None: ...
|
|
42
|
+
def chmod(self, mode, *, follow_symlinks=True) -> None: ...
|
|
43
|
+
def checksum(self, algorithm="md5") -> str: ...
|
|
44
|
+
def supported_checksums(self) -> FrozenSet[str]: ...
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
A `pathapi.Path` for a directory tree on an SFTP server, over paramiko or
|
|
48
|
+
asyncssh. It needs no URI. Nothing connects when a path is built; the first
|
|
49
|
+
request does.
|
|
50
|
+
|
|
51
|
+
- **Construction.** `SFTPPath("/srv/x", host="h", user="u", password="p")` builds
|
|
52
|
+
the path's own backend from the endpoint keywords, of the class chosen by
|
|
53
|
+
"Backend selection" below, and passes it `ssh_config=`, `timeout=`,
|
|
54
|
+
`connect_opts=` (when given) and `**options`, the keywords of that class
|
|
55
|
+
(`known_hosts=`, `hostkeypolicy=` for paramiko; `max_concurrency=`,
|
|
56
|
+
`sftp_version=` for asyncssh): a keyword the class does not take is a
|
|
57
|
+
`TypeError`. `backend=` instead shares one connection between paths built
|
|
58
|
+
separately, or plugs in a `BaseSFTPBackend` of your own. Giving `backend=`
|
|
59
|
+
together with any other keyword is a `TypeError`, and so is giving neither
|
|
60
|
+
(unless a segment is an `SFTPPath`, whose backend is kept) and giving the
|
|
61
|
+
other keywords without `host=`. Segments are `str`, `SFTPPath` or a pure
|
|
62
|
+
path; an absolute segment restarts the join, empty and `.` segments
|
|
63
|
+
collapse. Every path derived from one (`/`, `joinpath`, `parent`,
|
|
64
|
+
`with_name`, `iterdir`, `relative_to`) keeps its backend.
|
|
65
|
+
- **`port`** is `None` unless the caller names one: the `Port` of the host in
|
|
66
|
+
the ssh_config, else 22, is used then. A port given outranks the config.
|
|
67
|
+
`ssh_config` is `~/.ssh/config` unless a path, an iterable of paths or
|
|
68
|
+
`None` (no config) is given; `DEFAULT_SSH_CONFIG` names the default. A host
|
|
69
|
+
that is not a plain host name or address -- an IP address (an IPv6 one may
|
|
70
|
+
carry a `%zone`), or ASCII letters, digits, `.`, `_`, `-` (not first) and the
|
|
71
|
+
letters and digits of other scripts -- raises `ValueError` naming it where a
|
|
72
|
+
connection or an ssh_config lookup would start, on both backends; building,
|
|
73
|
+
joining and comparing such a path still work.
|
|
74
|
+
- **Backend selection**, highest first: `backend=` → the class attribute
|
|
75
|
+
`_default_backend_cls` of a subclass → the environment variable
|
|
76
|
+
`SFTPPATH_BACKEND` (`auto`|`asyncssh`|`paramiko`; the first request of a path
|
|
77
|
+
whose named backend is not installed raises `ImportError`) → auto (asyncssh if importable, else
|
|
78
|
+
paramiko). The choice is made once per process.
|
|
79
|
+
- **Names.** The tree has no working directory: `srv/x` and `/srv/x` name one
|
|
80
|
+
file, read from the server root, and `/` is sent for the root (never an empty
|
|
81
|
+
argument, which a server reads as its start directory). `..` is sent as
|
|
82
|
+
written. A name the server sends that is not UTF-8 is the string of its bytes
|
|
83
|
+
as lone surrogates (pathlib's spelling on POSIX), on both backends: it lists
|
|
84
|
+
beside its siblings, and `stat()`, `open()`, `unlink()`, `rm()` and `copy()`
|
|
85
|
+
send the same bytes back.
|
|
86
|
+
- **Text forms.** `str()` is the path alone. `repr()` names the endpoint
|
|
87
|
+
(`SFTPPath('/srv/x', host='h', port=22, user='u')`, with `port=` and `user=`
|
|
88
|
+
only when set) and never a password. `as_uri()` is
|
|
89
|
+
`sftp://[user@]host[:port]/path` without a password; `NotImplementedError` on a
|
|
90
|
+
backend that has no `host`.
|
|
91
|
+
- **`is_local()`** is true when the host is a loopback or unspecified address or
|
|
92
|
+
`localhost`, or the backend has no host; a host name is not resolved.
|
|
93
|
+
`PathSyncer` skips its `quick_check` when both sides are local.
|
|
94
|
+
- **Identity.** `==` compares the names only (as `MemPath` does); two paths are
|
|
95
|
+
one tree (`copy()`, `move()` and `PathSyncer` ask) when they share a backend
|
|
96
|
+
object or their backends have equal `endpoint`s. `_node_key()` follows the
|
|
97
|
+
same rule and applies `..` to the text. A pickle holds the text and the
|
|
98
|
+
backend, which carries no connection and a password or `connect_opts` only
|
|
99
|
+
if it was given them; `copy.copy` and `copy.deepcopy` share the backend.
|
|
100
|
+
- **Listing and stat.** A directory is listed with one request that carries every
|
|
101
|
+
entry's attributes (a link is not followed): `walk()`, `glob()` and
|
|
102
|
+
`rm(recursive=True)` test each child from that listing, and an entry that is
|
|
103
|
+
not a link is not stat-ed again. `iterdir()` yields children without
|
|
104
|
+
attributes, so `is_dir()` on each costs one request; the children an
|
|
105
|
+
`SFTPURIPath` yields carry them. A listed name that is not one path
|
|
106
|
+
component (`..`, anything with `/`) is skipped. `stat()` returns the
|
|
107
|
+
client's `os.stat_result`-shaped attributes.
|
|
108
|
+
- **Errors** are the same on both backends and none is chained to the SSH
|
|
109
|
+
library's exception. A server status is the pathlib exception with `errno`
|
|
110
|
+
and `filename` (the remote path; `filename2` for the second path of
|
|
111
|
+
`rename()` and `symlink_to()`); an "operation unsupported" status is
|
|
112
|
+
`NotImplementedError`. An error `SFTPPath` builds itself (`mkdir()` of an
|
|
113
|
+
existing path, `rmdir()` or `iterdir()` of the wrong kind of entry) carries
|
|
114
|
+
the path as `filename`. A connection that goes away, mid-request or
|
|
115
|
+
mid-transfer, is `ConnectionResetError`; a refused login is
|
|
116
|
+
`SFTPAuthenticationError` (a `PermissionError`, `EACCES`); a host key that
|
|
117
|
+
is unknown, changed or refused is `SFTPHostKeyError` (a `ConnectionError`,
|
|
118
|
+
`ECONNABORTED`) and no credential was sent; any other failure of the
|
|
119
|
+
handshake or the session is `ConnectionAbortedError`; a refused TCP
|
|
120
|
+
connection is `ConnectionRefusedError`; a timeout is `TimeoutError` (a
|
|
121
|
+
banner that never arrives is reported as `ConnectionAbortedError` by
|
|
122
|
+
paramiko). The SSH library's own message for a `ConnectionAbortedError` (and
|
|
123
|
+
for the rare request failure reported as `OSError(EIO)`) is logged at DEBUG on
|
|
124
|
+
the logger `sftppath`, as `SFTP connection failed: <ExceptionClass>:
|
|
125
|
+
<message>`; the exception carries only the class name. A DEBUG record can
|
|
126
|
+
hold what the SSH library put in its message, so enable that logger for
|
|
127
|
+
troubleshooting and keep it out of shared logs.
|
|
128
|
+
- **Writes** go through as the stream is used and `open("w")` truncates the
|
|
129
|
+
file at once: a `with` block that raised leaves what it had written, as a
|
|
130
|
+
local file does (the stream has no `discard()`). `write()` returns the number
|
|
131
|
+
of bytes written and `truncate()` works on both backends. A file opened for
|
|
132
|
+
writing is flushed and closed when the interpreter exits if the program left
|
|
133
|
+
it open (a text wrapper's pending text included), waiting at most five
|
|
134
|
+
seconds.
|
|
135
|
+
- **Links and attributes.** `readlink()` returns the target as a path on the
|
|
136
|
+
same backend (dot segments collapse, as `pathlib` does) and `symlink_to()`
|
|
137
|
+
stores its target verbatim, on both backends; `hardlink_to(target)` and
|
|
138
|
+
`chmod(follow_symlinks=False)` on asyncssh only (paramiko →
|
|
139
|
+
`NotImplementedError`); `chown()` takes numeric ids only. A partial `chown()`
|
|
140
|
+
(`chown(None, gid)`) needs the current id of the field it leaves alone: a
|
|
141
|
+
server that names owners instead of numbering them (SFTP version 4) makes it
|
|
142
|
+
`NotImplementedError` naming the field, and nothing is sent (`st_uid` and
|
|
143
|
+
`st_gid` are 0 for an owner name that is not a number). `_utime()` sets the
|
|
144
|
+
times with one attributes request on both backends, in whole seconds; SFTPv3
|
|
145
|
+
carries the access and modification time as a pair, so the one left alone is
|
|
146
|
+
read first (one more request).
|
|
147
|
+
- **`rename(target)`** is onto a path of the same server (a `str` target is a
|
|
148
|
+
sibling name; a path of another server is `NotImplementedError`, so `move()`
|
|
149
|
+
copies). An existing target is replaced via `posix-rename@openssh.com` where
|
|
150
|
+
supported, else `FileExistsError`. `replace(target)` onto a path of the same
|
|
151
|
+
server is that one request when the server has the extension: atomic, the
|
|
152
|
+
target is not removed first, and a refusal that names its cause
|
|
153
|
+
(`PermissionError`, `FileNotFoundError`, ...) leaves both entries as they
|
|
154
|
+
were. A server without the extension, one that refuses it with a bare
|
|
155
|
+
failure, and a directory at `target` (judged by `Path.replace()`, which
|
|
156
|
+
checks before it removes: `OSError(ENOTEMPTY)`, `IsADirectoryError`) get the
|
|
157
|
+
generic `replace()`: the file at `target` is removed, then this entry is
|
|
158
|
+
renamed. `move(overwrite=True)` onto an existing file of the same server
|
|
159
|
+
uses it, so it is one `posix-rename` and no removal.
|
|
160
|
+
- **`checksum(algorithm)` and `supported_checksums()`** (`NativeChecksum`): both
|
|
161
|
+
backends send the `check-file-handle` extension (OpenSSH lacks it); an
|
|
162
|
+
"operation unsupported" answer is remembered per connection and algorithm
|
|
163
|
+
(one request per algorithm, not per file) and a failure for one file refuses
|
|
164
|
+
nothing; an algorithm other than `md5`, `sha1`, `sha224`, `sha256`,
|
|
165
|
+
`sha384`, `sha512` or `crc32`, or a digest of the wrong size, is
|
|
166
|
+
`NotImplementedError` (no request for the first); every failure is
|
|
167
|
+
`NotImplementedError`. `supported_checksums()` is empty until a digest has
|
|
168
|
+
been produced.
|
|
169
|
+
- **Recursive `copy()` and `rm()`.** When the backend has its own walks
|
|
170
|
+
(`supports_tree`: asyncssh), a directory copied to a path of the same server
|
|
171
|
+
and tree, and `rm(recursive=True)` with the default `follow_symlinks=False`,
|
|
172
|
+
run them as concurrent native requests bounded by `max_concurrency`;
|
|
173
|
+
otherwise (and for every backend that does not set it) the generic walks of
|
|
174
|
+
`pathapi` run, and no SSH library is imported by `copy()` or `rm()`.
|
|
175
|
+
`progress` is not called during the native fan-out, and the native `rm()`
|
|
176
|
+
looks only at the root before it lists.
|
|
177
|
+
|
|
178
|
+
### `SFTPBackend`, `AsyncSSHBackend`, `BaseSFTPBackend`
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
DEFAULT_PORT = 22
|
|
182
|
+
|
|
183
|
+
class SFTPEndpoint(NamedTuple):
|
|
184
|
+
host: str
|
|
185
|
+
port: int | None
|
|
186
|
+
user: str | None
|
|
187
|
+
|
|
188
|
+
class BaseSFTPBackend:
|
|
189
|
+
endpoint
|
|
190
|
+
supports_lchmod = False
|
|
191
|
+
supports_hardlink = False
|
|
192
|
+
supports_tree = False
|
|
193
|
+
def client(self): ...
|
|
194
|
+
def close(self) -> None: ...
|
|
195
|
+
def __enter__(self) -> Self: ...
|
|
196
|
+
def __exit__(self, *exc_info) -> None: ...
|
|
197
|
+
def tree_copy(
|
|
198
|
+
self, path, target, *, overwrite, follow_symlinks, preserve_metadata,
|
|
199
|
+
ignore_error, on_error=None,
|
|
200
|
+
) -> None: ...
|
|
201
|
+
def tree_rm(self, path, *, missing_ok, on_error) -> None: ...
|
|
202
|
+
def checksum(self, path, algorithm) -> str: ...
|
|
203
|
+
def supported_checksums(self, path) -> FrozenSet[str]: ...
|
|
204
|
+
|
|
205
|
+
class CheckFileSFTPBackend(BaseSFTPBackend): ...
|
|
206
|
+
|
|
207
|
+
class SFTPBackend(CheckFileSFTPBackend):
|
|
208
|
+
host
|
|
209
|
+
port
|
|
210
|
+
user
|
|
211
|
+
password
|
|
212
|
+
connect_opts
|
|
213
|
+
hostkeypolicy
|
|
214
|
+
ssh_config
|
|
215
|
+
known_hosts
|
|
216
|
+
timeout
|
|
217
|
+
DEFAULT_TIMEOUT = 30.0
|
|
218
|
+
def __init__(
|
|
219
|
+
self, host, port=None, *, user=None, password=None, connect_opts=None,
|
|
220
|
+
hostkeypolicy=None, ssh_config=..., known_hosts=..., timeout=30.0,
|
|
221
|
+
) -> None: ...
|
|
222
|
+
def opts(self): ...
|
|
223
|
+
def transport(self) -> Transport: ...
|
|
224
|
+
|
|
225
|
+
class AsyncSSHBackend(CheckFileSFTPBackend):
|
|
226
|
+
host
|
|
227
|
+
port
|
|
228
|
+
user
|
|
229
|
+
password
|
|
230
|
+
connect_opts
|
|
231
|
+
max_concurrency
|
|
232
|
+
sftp_version
|
|
233
|
+
timeout
|
|
234
|
+
DEFAULT_MAX_CONCURRENCY = 16
|
|
235
|
+
def __init__(
|
|
236
|
+
self, host, port=None, *, user=None, password=None, connect_opts=None,
|
|
237
|
+
max_concurrency=None, sftp_version=4, ssh_config=..., timeout=60.0,
|
|
238
|
+
) -> None: ...
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
A backend is the connections to one SFTP server and account; its `endpoint`
|
|
242
|
+
(an `SFTPEndpoint`, with the host name in lower case and no password) is what
|
|
243
|
+
two backends must share for their paths to name the same files. Nothing
|
|
244
|
+
connects when one is built (`ValueError` for an empty `host`). `close()`
|
|
245
|
+
closes every connection the backend holds, whichever thread opened it, and
|
|
246
|
+
returns when they are closed; it is harmless twice and before any use, and a
|
|
247
|
+
later use connects afresh. Leaving a `with` block calls it. `host` may be a
|
|
248
|
+
name from the user's ssh_config.
|
|
249
|
+
|
|
250
|
+
- `BaseSFTPBackend.client()` is the override point: a paramiko-shaped
|
|
251
|
+
`SFTPClient` (`stat`, `lstat`, `listdir_attr`, `open`, `mkdir`, `chmod`,
|
|
252
|
+
`chown`, `utime`, `remove`, `rmdir`, `rename`, `posix_rename`, `symlink`,
|
|
253
|
+
`readlink`, `link`), cached or newly opened, with no argument.
|
|
254
|
+
`supports_lchmod`, `supports_hardlink`, `checksum()` and
|
|
255
|
+
`supported_checksums()` are the other switches. `supports_tree` (False;
|
|
256
|
+
asyncssh True) says the backend has its own concurrent recursive `copy()` and
|
|
257
|
+
`rm()` (`tree_copy(path, target, *, overwrite, follow_symlinks,
|
|
258
|
+
preserve_metadata, ignore_error, on_error=None)`, `tree_rm(path, *,
|
|
259
|
+
missing_ok, on_error)`); `on_error(error, path)` decides whether the failure
|
|
260
|
+
of one entry is skipped. `CheckFileSFTPBackend` adds the `check-file-handle`
|
|
261
|
+
`checksum()` and `supported_checksums()` to a backend that supplies the
|
|
262
|
+
extended request.
|
|
263
|
+
- **`SFTPBackend`** (paramiko; `pip install "sftppath[paramiko]"`) connects with
|
|
264
|
+
`paramiko.SSHClient` using `connect_opts` merged with `opts()`, the keyword
|
|
265
|
+
arguments `SSHClient.connect()` is called with for the host (a `ProxyCommand`
|
|
266
|
+
starts its process there); `transport()` connects and returns the
|
|
267
|
+
`paramiko.Transport`. Host keys are verified: `known_hosts` default is
|
|
268
|
+
`~/.ssh/known_hosts` plus the ssh_config `UserKnownHostsFile` (`None` loads
|
|
269
|
+
none; a path or list loads exactly those), `hostkeypolicy` default
|
|
270
|
+
`paramiko.RejectPolicy()`; a changed key always fails with `SFTPHostKeyError`.
|
|
271
|
+
Opt-out: `SFTPBackend(host, hostkeypolicy=paramiko.AutoAddPolicy(),
|
|
272
|
+
known_hosts=None)` accepts any server key, and a network
|
|
273
|
+
man-in-the-middle then receives the password. `timeout` fills paramiko's
|
|
274
|
+
connect/banner/auth/channel timeouts (`connect_opts` values win; `None`
|
|
275
|
+
leaves them unset); requests on an open connection are unbounded.
|
|
276
|
+
ssh_config: `HostName`, `Port`, `User`, `IdentityFile`, `ProxyCommand`,
|
|
277
|
+
`Include`; `ProxyJump` → `NotImplementedError` unless `connect_opts["sock"]` is
|
|
278
|
+
given. A `ProxyCommand` expands `%h`, `%n`, `%p`, `%r` and `%%` with the host,
|
|
279
|
+
port and user this connection uses (a user with white space, a quote, a shell
|
|
280
|
+
metacharacter or a leading `-` is refused when `%r` is used, on asyncssh too:
|
|
281
|
+
a `ValueError` before asyncssh is given the user, when the config's
|
|
282
|
+
`ProxyCommand` would pass it on); an `Include` inside a `Host` or `Match`
|
|
283
|
+
block applies the included file's blocks only where the enclosing one does.
|
|
284
|
+
Connections are cached per (backend, thread) and replaced when dropped.
|
|
285
|
+
- **`AsyncSSHBackend`** (`pip install "sftppath[asyncssh]"`) connects with
|
|
286
|
+
asyncssh, which verifies host keys against `known_hosts` and the ssh_config;
|
|
287
|
+
the opt-out is `connect_opts={"known_hosts": None}`. `ssh_config` is passed to
|
|
288
|
+
asyncssh as its `config`. The port is the one given, else the ssh_config
|
|
289
|
+
`Port` of the host. `timeout` bounds single requests (a timed-out request is
|
|
290
|
+
cancelled and raises `TimeoutError`); recursive `copy()`/`rm()`, streamed
|
|
291
|
+
reads/writes and a native `checksum()` are unbounded (use asyncssh's
|
|
292
|
+
`connect_timeout`/`keepalive_interval`). One connection per backend, served
|
|
293
|
+
by one shared background event loop thread; not fork-safe (rebuilt after
|
|
294
|
+
`fork()`). A sync `Path` call made on that loop thread (inside a callback
|
|
295
|
+
running there) raises `RuntimeError`. `max_concurrency` (`None` → the class
|
|
296
|
+
attribute `DEFAULT_MAX_CONCURRENCY`, 16) bounds requests in flight and files
|
|
297
|
+
open during recursive `copy()` (target on the same server and the same tree:
|
|
298
|
+
two backends with different endpoints copy through the generic walk) and
|
|
299
|
+
`rm()`. Such a call that raises, times out or is interrupted
|
|
300
|
+
(`KeyboardInterrupt`) has stopped when the exception reaches the caller: no
|
|
301
|
+
further request is sent, and a destination file whose copy did not complete
|
|
302
|
+
is removed. Both walks leave behind what the generic `copy()`/`rm()` leave: a
|
|
303
|
+
source is opened before the destination it replaces is touched, `copy()`
|
|
304
|
+
applies permission bits as `Path.copy()` does, `rm()`'s `missing_ok` covers
|
|
305
|
+
the path it was called on only, and `ignore_error` is offered each error
|
|
306
|
+
once.
|
|
307
|
+
|
|
308
|
+
## The `sftp:` scheme (`sftppath.uri`, `uri` extra)
|
|
309
|
+
|
|
310
|
+
```python
|
|
311
|
+
class SFTPURIPath(DelegatingURIPath):
|
|
312
|
+
def __init__(self, *args, ssh_config=..., **kwargs): ...
|
|
313
|
+
def readlink(self) -> SFTPURIPath: ...
|
|
314
|
+
def replace(self, target) -> SFTPURIPath: ...
|
|
315
|
+
def hardlink_to(self, target): ...
|
|
316
|
+
def checksum(self, algorithm="md5") -> str: ...
|
|
317
|
+
def supported_checksums(self) -> FrozenSet[str]: ...
|
|
318
|
+
|
|
319
|
+
class SFTPURIBackend:
|
|
320
|
+
backend_class
|
|
321
|
+
options
|
|
322
|
+
def __init__(self, backend_class=None, **options) -> None: ...
|
|
323
|
+
def backend_for(self, source) -> BaseSFTPBackend: ...
|
|
324
|
+
def close(self) -> None: ...
|
|
325
|
+
def __enter__(self) -> Self: ...
|
|
326
|
+
def __exit__(self, *exc_info) -> None: ...
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
Importing `sftppath.uri` needs `uripath` (`pip install "sftppath[uri]"`); without
|
|
330
|
+
it the import raises an `ImportError` naming that extra. `uripath` finds the
|
|
331
|
+
class itself, through the entry point `sftp` of the group `uripath.schemes`,
|
|
332
|
+
the first time a path of the scheme is used, so `URIPath("sftp://u@h/srv/x")`
|
|
333
|
+
needs no import of this package. A path of the scheme is an `SFTPPath` behind
|
|
334
|
+
a URI: the URI layer (parsing, joining, how paths of one endpoint share a
|
|
335
|
+
backend, `str()` without the password, pickling) is `uripath`'s, and every
|
|
336
|
+
request is made by the `SFTPPath` that the URI names, so the errors, the
|
|
337
|
+
caches and the native walks are the ones above. An error that names the
|
|
338
|
+
remote path names the URI. A URI without a path (`sftp://h`) is the root.
|
|
339
|
+
`os.fspath()` and `host_fspath()` return `.path`, the path on the server.
|
|
340
|
+
|
|
341
|
+
- `ssh_config` is a keyword of the constructor with the meaning above. It is
|
|
342
|
+
carried to every path derived from the path on the same endpoint (`/`,
|
|
343
|
+
`parent`, `with_name()`, `with_source()`, a join, a destination string), and
|
|
344
|
+
dropped, with the backend, when the derived path is on another host. Paths of
|
|
345
|
+
one endpoint share the backend they build for themselves unless their
|
|
346
|
+
`ssh_config` differs: one built with another `ssh_config` than the path it
|
|
347
|
+
derives from has a backend of its own.
|
|
348
|
+
- `backend=` is an `SFTPURIBackend`, or a `BaseSFTPBackend` used as it is for
|
|
349
|
+
every path whatever host the URI names. Without one, a path builds an
|
|
350
|
+
`SFTPURIBackend` of the class that "Backend selection" chooses; the user, the
|
|
351
|
+
password and the port of the URI go to the backend of each server it names.
|
|
352
|
+
A supplied backend is not part of a pickle or a copy of the path (`copy.copy`
|
|
353
|
+
shares it).
|
|
354
|
+
- `SFTPURIBackend(backend_class=None, **options)` is how the connections of
|
|
355
|
+
the `sftp:` paths are made: `backend_class` is a `BaseSFTPBackend` subclass
|
|
356
|
+
(`SFTPBackend`, `AsyncSSHBackend`, or your own; `None` → the selection
|
|
357
|
+
above) and `options` its constructor keywords (`connect_opts`, `ssh_config`,
|
|
358
|
+
`timeout`, `known_hosts`, ...). `backend_for(source)` is the backend for the
|
|
359
|
+
server and account of a `uripath.source.Source`, the same object for as long
|
|
360
|
+
as a path or a connection holds it, so paths to one server share a
|
|
361
|
+
connection per backend. `close()` closes every connection those backends
|
|
362
|
+
hold and is harmless twice; leaving a `with` block calls it.
|
|
363
|
+
- Two paths are on different filesystems when both carry a supplied backend
|
|
364
|
+
and those differ; `rename()`, `replace()` and `hardlink_to()` onto another
|
|
365
|
+
host are `NotImplementedError` (so `move()` copies), and a directory copied
|
|
366
|
+
to another host, to another supplied backend or to a local path takes the
|
|
367
|
+
generic copy.
|
|
368
|
+
|
|
369
|
+
## Exceptions
|
|
370
|
+
|
|
371
|
+
### `sftppath.exceptions`
|
|
372
|
+
|
|
373
|
+
```python
|
|
374
|
+
class SFTPPathError(PathError): ...
|
|
375
|
+
class SFTPAuthenticationError(SFTPPathError, PermissionError): ...
|
|
376
|
+
class SFTPHostKeyError(SFTPPathError, ConnectionError): ...
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
`SFTPPathError` is the base of what this package raises on its own, itself a
|
|
380
|
+
`pathapi.PathError`; the root exports the three. Every failure of the
|
|
381
|
+
server is the `OSError` type pathlib raises for it, and no exception of paramiko
|
|
382
|
+
or asyncssh crosses the API. A bad argument is a plain `ValueError` or
|
|
383
|
+
`TypeError`.
|
|
384
|
+
|
|
385
|
+
- `SFTPAuthenticationError` — the server refused the credentials offered.
|
|
386
|
+
- `SFTPHostKeyError` — the host key was unknown, changed or not accepted; no
|
|
387
|
+
credential was sent.
|
|
388
|
+
|
|
389
|
+
## Environment variables
|
|
390
|
+
|
|
391
|
+
### `SFTPPATH_BACKEND`
|
|
392
|
+
|
|
393
|
+
`auto` (the default), `asyncssh` or `paramiko`: the backend class a path builds
|
|
394
|
+
when it is given endpoint keywords and no `backend=`. Any other value, an empty
|
|
395
|
+
one and a differently cased one included, is a `ValueError` naming the
|
|
396
|
+
variable; for a named backend whose library is not installed, the first
|
|
397
|
+
request of a path raises an `ImportError` naming its extra. It is read the first time a backend is needed and kept for
|
|
398
|
+
the life of the process.
|
|
399
|
+
|
|
400
|
+
## Gotchas
|
|
401
|
+
|
|
402
|
+
- A relative path is read from the server root: `SFTPPath("data/x", host=h)` is
|
|
403
|
+
`/data/x`, not a file in the home directory of the account.
|
|
404
|
+
- `SFTPPath.readlink()` returns a path, so its dot segments collapse; the
|
|
405
|
+
`SFTPURIPath` keeps the text of the target verbatim in `.path`.
|
|
406
|
+
- Both libraries connect lazily and cache the connection; a program that opens
|
|
407
|
+
many backends should `close()` them (or use `with`).
|
|
408
|
+
- `AsyncSSHBackend` runs on one shared loop thread: do not call a path method
|
|
409
|
+
from a callback that runs on it.
|
|
410
|
+
- `known_hosts=None` / `AutoAddPolicy` turn host-key verification off; the
|
|
411
|
+
password is then sent to whoever answers.
|
sftppath/README.md
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# sftppath
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/sftppath/)
|
|
4
|
+
[](https://pypi.org/project/sftppath/)
|
|
5
|
+
[](https://github.com/jose-pr/sftppath/blob/main/LICENSE)
|
|
6
|
+
[](https://jose-pr.github.io/sftppath/)
|
|
7
|
+
[](https://github.com/jose-pr/sftppath/actions/workflows/test.yml)
|
|
8
|
+
|
|
9
|
+
**`pathlib` for SFTP servers.** `SFTPPath("/srv/data/report.txt", host="files.example.org")`
|
|
10
|
+
has the method names, signatures and exception types of `pathlib.Path`, over
|
|
11
|
+
paramiko or asyncssh, and needs no URI. It builds on
|
|
12
|
+
[`pathapi`](https://github.com/jose-pr/pathapi) alone; with
|
|
13
|
+
[`uripath`](https://github.com/jose-pr/uripath) installed it also provides the
|
|
14
|
+
`sftp:` scheme. It is documented at <https://jose-pr.github.io/sftppath/>.
|
|
15
|
+
|
|
16
|
+
## Features
|
|
17
|
+
|
|
18
|
+
- **A `pathlib` path on a server**: `open`, `read_text`, `iterdir`, `glob`,
|
|
19
|
+
`stat`, `mkdir`, `rename`, `replace`, `chmod`, `symlink_to`, `readlink` and
|
|
20
|
+
the rest, plus `walk`, `copy`, `move`, `rm` and `PathSyncer` from
|
|
21
|
+
`pathapi`. Nothing connects until the first request.
|
|
22
|
+
- **Two backends, one behaviour**: `SFTPBackend` (paramiko) and `AsyncSSHBackend`
|
|
23
|
+
(asyncssh) raise the same exceptions for the same server answers; the choice is
|
|
24
|
+
a keyword, a subclass attribute or `SFTPPATH_BACKEND`.
|
|
25
|
+
- **Native tree operations**: on asyncssh a recursive `copy()` and `rm()` run as
|
|
26
|
+
concurrent requests on one connection, bounded by `max_concurrency`.
|
|
27
|
+
- **ssh_config**: `HostName`, `Port`, `User`, `IdentityFile`, `ProxyCommand` and
|
|
28
|
+
`Include` apply to the host you name; host keys are verified.
|
|
29
|
+
- **Server-side checksums**: `checksum()` uses the `check-file-handle`
|
|
30
|
+
extension where the server has it, and `PathSyncer` falls back to streaming
|
|
31
|
+
where it does not.
|
|
32
|
+
- **No SSH library at import**: `import sftppath` loads neither library nor
|
|
33
|
+
`uripath`. Errors are `OSError` subclasses, `SFTPAuthenticationError` and
|
|
34
|
+
`SFTPHostKeyError`; no library exception crosses the API.
|
|
35
|
+
- **The `sftp:` scheme** for `uripath`: `URIPath("sftp://user@host/dir")` is an
|
|
36
|
+
`SFTPURIPath` that delegates to an `SFTPPath`.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "sftppath[paramiko]"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Requires `pathapi`, which pip installs with it, and Python 3.9 or newer.
|
|
45
|
+
|
|
46
|
+
| Install | Adds |
|
|
47
|
+
| --- | --- |
|
|
48
|
+
| `pip install sftppath` | the path and the backend contract; no SSH library |
|
|
49
|
+
| `pip install "sftppath[paramiko]"` | the paramiko backend (`paramiko>=5.0,<6`) |
|
|
50
|
+
| `pip install "sftppath[asyncssh]"` | the asyncssh backend and the native tree operations (`asyncssh>=2.20`) |
|
|
51
|
+
| `pip install "sftppath[uri]"` | `uripath` and with it the `sftp:` scheme |
|
|
52
|
+
| `pip install "sftppath[all]"` | the three above |
|
|
53
|
+
|
|
54
|
+
## Quick start
|
|
55
|
+
|
|
56
|
+
**A path on a server**: the endpoint goes in as keywords; the user, password and
|
|
57
|
+
port may come from your `~/.ssh/config` instead:
|
|
58
|
+
|
|
59
|
+
<!-- example: skip: connects to a server -->
|
|
60
|
+
```python
|
|
61
|
+
from sftppath import SFTPPath
|
|
62
|
+
|
|
63
|
+
report = SFTPPath("/srv/data/report.txt", host="files.example.org", user="deploy")
|
|
64
|
+
print(report.read_text())
|
|
65
|
+
for child in report.parent.iterdir():
|
|
66
|
+
print(child.name, child.stat().st_size)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
**Share a connection**: a backend is the connections to one server and
|
|
70
|
+
account, and a context manager:
|
|
71
|
+
|
|
72
|
+
<!-- example: skip: connects to a server -->
|
|
73
|
+
```python
|
|
74
|
+
from sftppath import SFTPBackend, SFTPPath
|
|
75
|
+
|
|
76
|
+
with SFTPBackend("files.example.org", user="deploy", timeout=10) as backend:
|
|
77
|
+
root = SFTPPath("/srv/data", backend=backend)
|
|
78
|
+
(root / "out").mkdir(exist_ok=True)
|
|
79
|
+
(root / "out" / "n.txt").write_text("1")
|
|
80
|
+
root.copy(SFTPPath("/srv/backup", backend=backend), recursive=True)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**Mirror a tree** with `PathSyncer`, onto a local directory:
|
|
84
|
+
|
|
85
|
+
<!-- example: skip: connects to a server -->
|
|
86
|
+
```python
|
|
87
|
+
from pathapi import LocalPath
|
|
88
|
+
from pathapi.sync import PathSyncer
|
|
89
|
+
from sftppath import SFTPPath
|
|
90
|
+
|
|
91
|
+
remote = SFTPPath("/var/log", host="files.example.org", user="deploy")
|
|
92
|
+
PathSyncer(remove_missing=True).sync(remote, LocalPath("mirror"))
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**The scheme**, with `uripath` installed:
|
|
96
|
+
|
|
97
|
+
<!-- example: skip: connects to a server -->
|
|
98
|
+
```python
|
|
99
|
+
from uripath import URIPath
|
|
100
|
+
|
|
101
|
+
print(URIPath("sftp://deploy@files.example.org/srv/data/report.txt").read_text())
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## API overview
|
|
105
|
+
|
|
106
|
+
| Module | Purpose |
|
|
107
|
+
| --- | --- |
|
|
108
|
+
| `sftppath` | `SFTPPath`, the backends `SFTPBackend`, `AsyncSSHBackend`, `BaseSFTPBackend` and `CheckFileSFTPBackend`, `SFTPEndpoint`, `DEFAULT_PORT`, `DEFAULT_SSH_CONFIG`, the errors and `__version__` |
|
|
109
|
+
| `sftppath.exceptions` | `SFTPPathError`, `SFTPAuthenticationError` and `SFTPHostKeyError` |
|
|
110
|
+
| `sftppath.uri` | `SFTPURIPath` and `SFTPURIBackend`, the `sftp:` scheme of `uripath` (needs the `uri` extra) |
|
|
111
|
+
|
|
112
|
+
The full reference is on the [documentation site](https://jose-pr.github.io/sftppath/);
|
|
113
|
+
an agent-oriented API summary ships inside the package as `sftppath/AGENTS.md`.
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install -e ".[dev,all]"
|
|
119
|
+
python -m pytest -q -rs
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The tests start SSH servers of their own on the loopback interface with
|
|
123
|
+
throwaway keys; they never reach another host. A test for a library or an extra
|
|
124
|
+
that is not installed is skipped, so check the skip list (`-rs`). Contributor
|
|
125
|
+
notes (virtual environments, formatting, CI, releases) are in the repository's
|
|
126
|
+
[`AGENTS.md`](https://github.com/jose-pr/sftppath/blob/main/AGENTS.md). To
|
|
127
|
+
preview the documentation: `pip install -e ".[all,docs]"` then `mkdocs serve`.
|
|
128
|
+
|
|
129
|
+
Changes are recorded in the
|
|
130
|
+
[changelog](https://github.com/jose-pr/sftppath/blob/main/CHANGELOG.md).
|
|
131
|
+
|
|
132
|
+
### Releasing
|
|
133
|
+
|
|
134
|
+
This project follows [Semantic Versioning](https://semver.org/) and keeps a
|
|
135
|
+
[`CHANGELOG.md`](https://github.com/jose-pr/sftppath/blob/main/CHANGELOG.md). Pushing a tag matching `v*` triggers the release
|
|
136
|
+
workflow: test gate → build (checking the tag names the version built) → a strict
|
|
137
|
+
docs build as a gate → GitHub release → publish. The release workflow never deploys
|
|
138
|
+
the docs site itself: for a final release its last job dispatches the docs workflow
|
|
139
|
+
at the tag, which owns every Pages deploy.
|
|
140
|
+
|
|
141
|
+
## License
|
|
142
|
+
|
|
143
|
+
MIT — see [LICENSE](https://github.com/jose-pr/sftppath/blob/main/LICENSE).
|
sftppath/__init__.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""pathlib-style paths for SFTP servers, on paramiko or asyncssh.
|
|
2
|
+
|
|
3
|
+
`SFTPPath` needs no URI and imports neither SSH library until a backend is
|
|
4
|
+
built: `SFTPBackend` (paramiko) and `AsyncSSHBackend` (asyncssh) are resolved
|
|
5
|
+
on first use (PEP 562), so `import sftppath` needs neither extra and never
|
|
6
|
+
imports `uripath`; naming one whose library is missing raises an `ImportError`
|
|
7
|
+
that names the extra. The `sftp:` scheme of `uripath` is `sftppath.uri`.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import importlib as _importlib
|
|
13
|
+
import typing as _ty
|
|
14
|
+
|
|
15
|
+
from ._backend import DEFAULT_PORT, BaseSFTPBackend, SFTPEndpoint
|
|
16
|
+
from ._checkfile import CheckFileSFTPBackend
|
|
17
|
+
from ._path import SFTPPath
|
|
18
|
+
from ._sshconfig import DEFAULT_SSH_CONFIG
|
|
19
|
+
from ._version import __version__
|
|
20
|
+
from .exceptions import SFTPAuthenticationError, SFTPHostKeyError, SFTPPathError
|
|
21
|
+
|
|
22
|
+
if _ty.TYPE_CHECKING:
|
|
23
|
+
from ._asyncssh import AsyncSSHBackend as AsyncSSHBackend
|
|
24
|
+
from ._paramiko import SFTPBackend as SFTPBackend
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"AsyncSSHBackend",
|
|
28
|
+
"BaseSFTPBackend",
|
|
29
|
+
"CheckFileSFTPBackend",
|
|
30
|
+
"DEFAULT_PORT",
|
|
31
|
+
"DEFAULT_SSH_CONFIG",
|
|
32
|
+
"SFTPAuthenticationError",
|
|
33
|
+
"SFTPBackend",
|
|
34
|
+
"SFTPEndpoint",
|
|
35
|
+
"SFTPHostKeyError",
|
|
36
|
+
"SFTPPath",
|
|
37
|
+
"SFTPPathError",
|
|
38
|
+
"__version__",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
#: Public name -> (the private module that defines it, imported on first use;
|
|
42
|
+
#: the library and extra it needs).
|
|
43
|
+
_LAZY = {
|
|
44
|
+
"AsyncSSHBackend": ("._asyncssh", "asyncssh"),
|
|
45
|
+
"SFTPBackend": ("._paramiko", "paramiko"),
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def __getattr__(name: str):
|
|
50
|
+
entry = _LAZY.get(name)
|
|
51
|
+
if entry is None:
|
|
52
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
53
|
+
module, library = entry
|
|
54
|
+
try:
|
|
55
|
+
loaded = _importlib.import_module(module, __name__)
|
|
56
|
+
except ModuleNotFoundError as error:
|
|
57
|
+
if (error.name or "").partition(".")[0] != library:
|
|
58
|
+
raise
|
|
59
|
+
raise ImportError(
|
|
60
|
+
f"{name} needs the {library!r} extra: "
|
|
61
|
+
f'pip install "sftppath[{library}]"',
|
|
62
|
+
name=library,
|
|
63
|
+
) from error
|
|
64
|
+
value = getattr(loaded, name)
|
|
65
|
+
globals()[name] = value
|
|
66
|
+
return value
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def __dir__():
|
|
70
|
+
return sorted(set(globals()) | set(_LAZY))
|