pathlib-next 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/CHANGELOG.md +29 -1
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/PKG-INFO +1 -1
- pathlib_next-0.6.0/docs/guides/schemes.md +62 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/pyproject.toml +1 -1
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/schemes/__init__.py +4 -0
- pathlib_next-0.6.0/src/pathlib_next/uri/schemes/archive.py +266 -0
- pathlib_next-0.6.0/src/pathlib_next/uri/schemes/data.py +58 -0
- pathlib_next-0.6.0/src/pathlib_next/uri/schemes/ftp.py +205 -0
- pathlib_next-0.6.0/tests/test_archive_uri.py +198 -0
- pathlib_next-0.6.0/tests/test_data_uri.py +76 -0
- pathlib_next-0.6.0/tests/test_ftp.py +313 -0
- pathlib_next-0.5.0/docs/guides/schemes.md +0 -39
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/.gitignore +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/LICENSE +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/README.md +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/docs/api/reference.md +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/docs/changelog.md +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/docs/divergences.md +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/docs/guides/extending.md +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/docs/index.md +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/examples/http_listing.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/examples/local_and_mem.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/examples/sftp_sync.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/mkdocs.yml +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/__init__.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/fspath.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/mempath.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/path.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/protocols/__init__.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/protocols/fs.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/protocols/io.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/py.typed +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/testing.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/__init__.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/query.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/schemes/file.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/schemes/http.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/schemes/sftp.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/uri/source.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/utils/__init__.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/utils/glob.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/utils/stat.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/src/pathlib_next/utils/sync.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/conftest.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_contract.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_glob.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_http.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_local.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_mempath.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_parity_io.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_parity_pure.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_pathname.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_query.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_sftp.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_smoke.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_source.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_sync.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_uri_parse.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_uri_path.py +0 -0
- {pathlib_next-0.5.0 → pathlib_next-0.6.0}/tests/test_utils.py +0 -0
|
@@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.6.0] - 2026-07-11
|
|
11
|
+
|
|
12
|
+
### Added (Phase 7a new schemes, stdlib-only, no new deps)
|
|
13
|
+
- `data:` scheme (RFC 2397, `pathlib_next.uri.schemes.data.DataUri`):
|
|
14
|
+
read-only, no backend/connection -- the entire file content is embedded
|
|
15
|
+
in the URI (`data:[<mediatype>][;base64],<data>`). `stat().st_size` is
|
|
16
|
+
the decoded payload length; `iterdir()` raises `NotADirectoryError`
|
|
17
|
+
(it's always a single file); write operations raise `NotImplementedError`.
|
|
18
|
+
- `ftp:`/`ftps:` scheme (`pathlib_next.uri.schemes.ftp.FtpPath`): full
|
|
19
|
+
read/write/list access via stdlib `ftplib`, with a thread-keyed LRU
|
|
20
|
+
connection cache mirroring `sftp.py`. Listing/stat prefer MLSD (RFC
|
|
21
|
+
3659); servers without it fall back to NLST (listing) and SIZE
|
|
22
|
+
(file-only stat). Writes buffer in memory and upload via STOR/APPE on
|
|
23
|
+
`close()`. `chmod()` uses the common but non-standard `SITE CHMOD`
|
|
24
|
+
extension (may not be supported by every server).
|
|
25
|
+
- `zip:`/`tar:` archive paths (`pathlib_next.uri.schemes.archive`):
|
|
26
|
+
`<scheme>:<archive-uri>!/<inner-path>` (Java-style `!/` separator,
|
|
27
|
+
URI form proposed to and confirmed by the user before implementation).
|
|
28
|
+
The archive half is itself any absolute URI with an explicit scheme, so
|
|
29
|
+
archives are readable straight off any other backend (`file:`, `http:`,
|
|
30
|
+
`sftp:`, `ftp:`, `data:`, ...). Read is supported for both schemes.
|
|
31
|
+
Write is `zip:`-only, and only for brand-new entries in a local (`file:`)
|
|
32
|
+
outer archive (overwriting/deleting/renaming an existing entry would
|
|
33
|
+
need a full-archive rewrite -- not implemented, raises
|
|
34
|
+
`NotImplementedError`). `tar:` auto-detects `.tar.gz`/`.tar.bz2`/`.tar.xz`
|
|
35
|
+
compression and is always read-only.
|
|
36
|
+
|
|
10
37
|
## [0.5.0] - 2026-07-11
|
|
11
38
|
|
|
12
39
|
### Fixed (critical -- found while writing Phase 6 examples)
|
|
@@ -208,7 +235,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
|
208
235
|
- Sync error handling.
|
|
209
236
|
- Generic Path Protocol based pathlib implementation for URI paths with file access support for sftp, http, file schemes.
|
|
210
237
|
|
|
211
|
-
[Unreleased]: https://github.com/jose-pr/pathlib_next/compare/v0.
|
|
238
|
+
[Unreleased]: https://github.com/jose-pr/pathlib_next/compare/v0.6.0...HEAD
|
|
239
|
+
[0.6.0]: https://github.com/jose-pr/pathlib_next/compare/v0.5.0...v0.6.0
|
|
212
240
|
[0.5.0]: https://github.com/jose-pr/pathlib_next/compare/v0.4.1...v0.5.0
|
|
213
241
|
[0.4.1]: https://github.com/jose-pr/pathlib_next/compare/v0.4.0...v0.4.1
|
|
214
242
|
[0.4.0]: https://github.com/jose-pr/pathlib_next/releases/tag/v0.4.0
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Schemes
|
|
2
|
+
|
|
3
|
+
Capability matrix for every built-in `Path`/`UriPath` implementation. A
|
|
4
|
+
"No" means the method raises `NotImplementedError` (or, for `http:`, isn't
|
|
5
|
+
meaningful for a read-only scheme) -- everything else (name/suffix parsing,
|
|
6
|
+
`glob()`, `walk()`, `copy()`, ...) is derived from these primitives and
|
|
7
|
+
works identically across all of them.
|
|
8
|
+
|
|
9
|
+
| Capability | `LocalPath` | `file:` (`FileUri`) | `mem:` (`MemPath`) | `http(s):` (`HttpPath`) | `sftp:` (`SftpPath`) | `data:` (`DataUri`) | `ftp(s):` (`FtpPath`) | `zip:` (`ZipUri`) | `tar:` (`TarUri`) |
|
|
10
|
+
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
11
|
+
| Read (`read_text`/`read_bytes`/`open`) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
|
|
12
|
+
| Write (`write_text`/`write_bytes`/`open("w")`) | Yes | Yes | Yes | No | Yes | No | Yes | New entries only, local outer archive only | No |
|
|
13
|
+
| List (`iterdir`) | Yes | Yes | Yes | Yes (scrapes an HTML index) | Yes | No (`NotADirectoryError`) | Yes (MLSD, falls back to NLST) | Yes | Yes |
|
|
14
|
+
| Stat (`stat`, `exists`, `is_dir`, `is_file`, ...) | Yes | Yes | Yes | Yes (via `HEAD`, falls back to `GET`) | Yes | Yes (`st_size` from decoded payload) | Yes (MLSD, falls back to SIZE for files) | Yes | Yes |
|
|
15
|
+
| `mkdir` | Yes | Yes | Yes | No | Yes | No | Yes | Local outer archive only (zero-length `name/` entry) | No |
|
|
16
|
+
| Delete (`unlink`/`rmdir`/`rm`) | Yes | Yes | Yes | No | Yes | No | Yes | No | No |
|
|
17
|
+
| `rename` | Yes | Yes | No (`move()` falls back to copy+unlink) | No | Yes | No | Yes | No | No |
|
|
18
|
+
| `chmod` | Yes | Yes | No | No | Yes (no `follow_symlinks=False`; paramiko has no `lchmod`) | No | Yes (`SITE CHMOD`, server-dependent) | No | No |
|
|
19
|
+
| Extra required | none | none | none | `http` | `sftp` | none (stdlib) | none (stdlib `ftplib`) | none (stdlib `zipfile`) | none (stdlib `tarfile`) |
|
|
20
|
+
|
|
21
|
+
Notes:
|
|
22
|
+
|
|
23
|
+
- **`file:`** is a thin `UriPath` wrapper around `LocalPath` -- it has
|
|
24
|
+
identical capabilities, just addressed by URI instead of a native path.
|
|
25
|
+
- **`mem:` (`MemPath`)** isn't a `UriPath` at all -- it's a plain `Path`
|
|
26
|
+
subclass (Track A of [Extending](extending.md)), backed by nested dicts
|
|
27
|
+
(`MemPathBackend`). No `as_uri()` scheme is registered for it; construct
|
|
28
|
+
it directly via `MemPath(...)`.
|
|
29
|
+
- **`http(s):`** is inherently read-only: there's no portable "create/
|
|
30
|
+
delete/rename a resource" over plain HTTP. Directory listing depends on
|
|
31
|
+
the server producing an Apache/nginx-style HTML index that
|
|
32
|
+
`htmllistparse` can parse (confirmed against Python's own
|
|
33
|
+
`http.server.SimpleHTTPRequestHandler`).
|
|
34
|
+
- **`sftp:`** has the fullest capability set of the URI schemes (it's a
|
|
35
|
+
real remote filesystem protocol). Connections are cached per
|
|
36
|
+
`(backend, source, thread)` (see `pathlib_next.uri.schemes.sftp`).
|
|
37
|
+
- **`data:`** (RFC 2397) has no server or connection at all -- the entire
|
|
38
|
+
"file" content lives in the URI string itself
|
|
39
|
+
(`data:[<mediatype>][;base64],<data>`). It's always a single file, never a
|
|
40
|
+
directory.
|
|
41
|
+
- **`ftp(s):`** uses the same connection-cache pattern as `sftp:` (stdlib
|
|
42
|
+
`ftplib`, no extra required). Prefers MLSD (RFC 3659) for listing/stat --
|
|
43
|
+
gives type/size/modify in one round trip -- and falls back to NLST/SIZE on
|
|
44
|
+
servers that don't support it (that fallback path can't distinguish "file
|
|
45
|
+
doesn't exist" from "is a directory" for `stat()`, since SIZE only works on
|
|
46
|
+
files).
|
|
47
|
+
- **`zip:`/`tar:`** address an entry *inside* an archive:
|
|
48
|
+
`zip:<archive-uri>!/<inner-path>` (Java-style `!/` separator, as in JAR
|
|
49
|
+
URLs / NIO `ZipFileSystem`). The `<archive-uri>` half is itself any
|
|
50
|
+
absolute URI with an explicit scheme -- `file:`, `http:`, `sftp:`,
|
|
51
|
+
`ftp:`, even `data:` -- so an archive is readable straight off any other
|
|
52
|
+
backend; `zip:file:///backups/site.zip!/index.html` and
|
|
53
|
+
`zip:sftp://host/nightly.zip!/index.html` both work the same way.
|
|
54
|
+
`segments`/`name`/`parent`/`glob()`/... all operate on the *inner* path.
|
|
55
|
+
Writing a brand-new entry (not overwriting/deleting/renaming an existing
|
|
56
|
+
one -- that would need a full-archive rewrite, not implemented) only
|
|
57
|
+
works when the outer archive is itself a local `file:` URI; every other
|
|
58
|
+
outer scheme is read-only. `tar:` (with transparent `.tar.gz`/`.tar.bz2`/
|
|
59
|
+
`.tar.xz` decompression) is read-only regardless of the outer scheme.
|
|
60
|
+
- See [Divergences from pathlib](../divergences.md) for the "explicitly out
|
|
61
|
+
of scope" list (`resolve`, `symlink_to`, `owner`, `expanduser`, ...) that
|
|
62
|
+
applies uniformly across every non-`LocalPath` implementation.
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io as _io
|
|
4
|
+
import re as _re
|
|
5
|
+
import tarfile as _tarfile
|
|
6
|
+
import time as _time
|
|
7
|
+
import zipfile as _zipfile
|
|
8
|
+
|
|
9
|
+
from ...utils.stat import FileStat
|
|
10
|
+
from .. import UriPath
|
|
11
|
+
from .file import FileUri
|
|
12
|
+
|
|
13
|
+
_SEP = "!/"
|
|
14
|
+
_SCHEME_RE = _re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*:")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _split_archive_path(path: str) -> "tuple[str, str]":
|
|
18
|
+
"""Split `<archive-uri>!/<inner-path>` (Java-style separator, as used
|
|
19
|
+
for JAR URLs / NIO ZipFileSystem) into its two halves. No separator (or
|
|
20
|
+
a bare trailing "!") means the archive root."""
|
|
21
|
+
if _SEP in path:
|
|
22
|
+
archive, inner = path.split(_SEP, 1)
|
|
23
|
+
elif path.endswith("!"):
|
|
24
|
+
archive, inner = path[:-1], ""
|
|
25
|
+
else:
|
|
26
|
+
archive, inner = path, ""
|
|
27
|
+
return archive, inner
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _open_outer(archive_uri: str) -> "UriPath":
|
|
31
|
+
if not _SCHEME_RE.match(archive_uri):
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f"Archive URI {archive_uri!r} has no scheme -- prefix it "
|
|
34
|
+
"explicitly, e.g. 'file:///path/to/archive.zip'."
|
|
35
|
+
)
|
|
36
|
+
return UriPath(archive_uri)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _ArchiveBackend:
|
|
40
|
+
"""Lazily opens+caches the archive handle for one outer archive URI.
|
|
41
|
+
Shared by every `ArchiveUri` instance derived from the same one (backend
|
|
42
|
+
propagation is the existing `UriPath` machinery -- see `ArchiveUri._init`).
|
|
43
|
+
NOT deduplicated across independently-constructed top-level `UriPath(...)`
|
|
44
|
+
strings pointing at the same archive -- derive sibling members via `/`/
|
|
45
|
+
`joinpath` from a shared instance rather than reparsing the same archive
|
|
46
|
+
URI twice, especially for writable zips (two separate handles writing
|
|
47
|
+
the same underlying file can corrupt it)."""
|
|
48
|
+
|
|
49
|
+
__slots__ = ("outer", "_handle")
|
|
50
|
+
|
|
51
|
+
def __init__(self, outer: "UriPath"):
|
|
52
|
+
self.outer = outer
|
|
53
|
+
self._handle = None
|
|
54
|
+
|
|
55
|
+
def _open(self):
|
|
56
|
+
raise NotImplementedError
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def handle(self):
|
|
60
|
+
if self._handle is None:
|
|
61
|
+
self._handle = self._open()
|
|
62
|
+
return self._handle
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def writable(self) -> bool:
|
|
66
|
+
return False
|
|
67
|
+
|
|
68
|
+
def names(self) -> "list[str]":
|
|
69
|
+
raise NotImplementedError
|
|
70
|
+
|
|
71
|
+
def read_member(self, path: str):
|
|
72
|
+
raise NotImplementedError
|
|
73
|
+
|
|
74
|
+
def member_stat(self, path: str) -> FileStat:
|
|
75
|
+
raise NotImplementedError
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _ZipBackend(_ArchiveBackend):
|
|
79
|
+
__slots__ = ()
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def writable(self) -> bool:
|
|
83
|
+
# Writing individual entries in place (via ZipFile.open(name, "w"))
|
|
84
|
+
# only works when we hold the archive open in "a" mode against a
|
|
85
|
+
# real seekable local file -- not for remote/embedded outer URIs.
|
|
86
|
+
return isinstance(self.outer, FileUri)
|
|
87
|
+
|
|
88
|
+
def _open(self):
|
|
89
|
+
if self.writable:
|
|
90
|
+
return _zipfile.ZipFile(str(self.outer.filepath), mode="a")
|
|
91
|
+
return _zipfile.ZipFile(_io.BytesIO(self.outer.read_bytes()), mode="r")
|
|
92
|
+
|
|
93
|
+
def names(self):
|
|
94
|
+
return self.handle.namelist()
|
|
95
|
+
|
|
96
|
+
def read_member(self, path):
|
|
97
|
+
return self.handle.open(path, "r") # raises KeyError if missing
|
|
98
|
+
|
|
99
|
+
def write_member(self, path: str, data: bytes):
|
|
100
|
+
# zipfile only persists the central directory to disk when the
|
|
101
|
+
# *archive* (not just the entry) is closed -- close and drop the
|
|
102
|
+
# cached handle so both this backend's next operation and any
|
|
103
|
+
# independently-opened ZipUri see the write.
|
|
104
|
+
self.handle.writestr(path, data)
|
|
105
|
+
self.handle.close()
|
|
106
|
+
self._handle = None
|
|
107
|
+
|
|
108
|
+
def member_stat(self, path):
|
|
109
|
+
info = self.handle.getinfo(path)
|
|
110
|
+
mtime = int(_time.mktime((*info.date_time, 0, 0, -1))) if info.date_time else 0
|
|
111
|
+
return FileStat(st_size=info.file_size, st_mtime=mtime, is_dir=path.endswith("/"))
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class _TarBackend(_ArchiveBackend):
|
|
115
|
+
__slots__ = ()
|
|
116
|
+
|
|
117
|
+
def _open(self):
|
|
118
|
+
return _tarfile.open(fileobj=_io.BytesIO(self.outer.read_bytes()), mode="r:*")
|
|
119
|
+
|
|
120
|
+
def names(self):
|
|
121
|
+
return self.handle.getnames()
|
|
122
|
+
|
|
123
|
+
def read_member(self, path):
|
|
124
|
+
try:
|
|
125
|
+
member = self.handle.getmember(path)
|
|
126
|
+
except KeyError:
|
|
127
|
+
raise
|
|
128
|
+
f = self.handle.extractfile(member)
|
|
129
|
+
if f is None:
|
|
130
|
+
raise IsADirectoryError(path)
|
|
131
|
+
return f
|
|
132
|
+
|
|
133
|
+
def member_stat(self, path):
|
|
134
|
+
info = self.handle.getmember(path)
|
|
135
|
+
return FileStat(st_size=info.size, st_mtime=int(info.mtime), is_dir=info.isdir())
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class ArchiveUri(UriPath):
|
|
139
|
+
"""Common base for `zip:`/`tar:` archive paths:
|
|
140
|
+
`<scheme>:<archive-uri>!/<inner-path>` (Java-style separator; the
|
|
141
|
+
`<archive-uri>` half is itself any absolute URI with an explicit scheme
|
|
142
|
+
-- `file:`, `http:`, `sftp:`, `ftp:`, ... -- so archives are readable
|
|
143
|
+
straight off any existing backend). `segments`/`name`/`parent`/`glob`/
|
|
144
|
+
... all operate on the *inner* path; the outer archive handle is the
|
|
145
|
+
`backend` (`_ZipBackend`/`_TarBackend`), propagated through the normal
|
|
146
|
+
backend machinery to every path derived from this one."""
|
|
147
|
+
|
|
148
|
+
__slots__ = ()
|
|
149
|
+
_backend_cls: type = None
|
|
150
|
+
|
|
151
|
+
def _init(self, source, path, query, fragment, /, **kwargs):
|
|
152
|
+
backend = kwargs.get("backend", None) or self._backend
|
|
153
|
+
if backend is None:
|
|
154
|
+
# Fresh top-level construction (e.g. UriPath("zip:...!/...")) --
|
|
155
|
+
# `path` is still the raw "<archive-uri>!/<inner>" string.
|
|
156
|
+
archive_str, inner = _split_archive_path(path)
|
|
157
|
+
backend = self._backend_cls(_open_outer(archive_str))
|
|
158
|
+
else:
|
|
159
|
+
# Derived instance (with_segments/joinpath/_make_child_relpath)
|
|
160
|
+
# -- backend already known, `path` is already just the inner
|
|
161
|
+
# path (no "archive!/" prefix to strip).
|
|
162
|
+
inner = path
|
|
163
|
+
kwargs["backend"] = backend
|
|
164
|
+
super()._init(source, inner, query, fragment, **kwargs)
|
|
165
|
+
|
|
166
|
+
def as_uri(self, /, sanitize=False):
|
|
167
|
+
outer_uri = self.backend.outer.as_uri(sanitize=sanitize)
|
|
168
|
+
return f"{self.source.scheme}:{outer_uri}!/{self.path}"
|
|
169
|
+
|
|
170
|
+
def _names(self):
|
|
171
|
+
return self.backend.names()
|
|
172
|
+
|
|
173
|
+
def _listdir(self):
|
|
174
|
+
prefix = f"{self.path}/" if self.path else ""
|
|
175
|
+
seen = set()
|
|
176
|
+
for name in self._names():
|
|
177
|
+
if not name.startswith(prefix):
|
|
178
|
+
continue
|
|
179
|
+
rest = name[len(prefix) :]
|
|
180
|
+
if not rest:
|
|
181
|
+
continue
|
|
182
|
+
child = rest.split("/", 1)[0]
|
|
183
|
+
if child and child not in seen:
|
|
184
|
+
seen.add(child)
|
|
185
|
+
yield child
|
|
186
|
+
|
|
187
|
+
def stat(self, *, follow_symlinks=True):
|
|
188
|
+
path = self.path
|
|
189
|
+
if path == "":
|
|
190
|
+
return FileStat(is_dir=True)
|
|
191
|
+
names = self._names()
|
|
192
|
+
if path in names:
|
|
193
|
+
return self.backend.member_stat(path)
|
|
194
|
+
dirmarker = f"{path}/"
|
|
195
|
+
if dirmarker in names or any(n.startswith(dirmarker) for n in names):
|
|
196
|
+
return FileStat(is_dir=True)
|
|
197
|
+
raise FileNotFoundError(self)
|
|
198
|
+
|
|
199
|
+
def _open(self, mode="r", buffering=-1):
|
|
200
|
+
if "r" in mode:
|
|
201
|
+
try:
|
|
202
|
+
return self.backend.read_member(self.path)
|
|
203
|
+
except KeyError as error:
|
|
204
|
+
raise FileNotFoundError(self) from error
|
|
205
|
+
raise NotImplementedError(f"open(mode={mode!r}) on {self.source.scheme}:")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class _ZipWriteStream(_io.BytesIO):
|
|
209
|
+
"""Buffers a new entry's content in memory; on close(), writes it via
|
|
210
|
+
`writestr()` and flushes+reopens the archive (see
|
|
211
|
+
`_ZipBackend.write_member`)."""
|
|
212
|
+
|
|
213
|
+
def __init__(self, backend: "_ZipBackend", path: str):
|
|
214
|
+
super().__init__()
|
|
215
|
+
self._backend = backend
|
|
216
|
+
self._path = path
|
|
217
|
+
|
|
218
|
+
def close(self):
|
|
219
|
+
if not self.closed:
|
|
220
|
+
self._backend.write_member(self._path, self.getvalue())
|
|
221
|
+
super().close()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class ZipUri(ArchiveUri):
|
|
225
|
+
"""`zip:` scheme. Read/write: writing a new entry works when the outer
|
|
226
|
+
archive is a local `file:` URI (opened in `zipfile`'s "a" -- append --
|
|
227
|
+
mode, so existing entries are untouched); every other outer scheme is
|
|
228
|
+
read-only (fetched fully into memory first). Overwriting/deleting/
|
|
229
|
+
renaming an existing entry is not implemented (would require rewriting
|
|
230
|
+
the whole archive) -- only adding new entries and (for directories)
|
|
231
|
+
zero-length `"name/"` marker entries."""
|
|
232
|
+
|
|
233
|
+
__SCHEMES = ("zip",)
|
|
234
|
+
__slots__ = ()
|
|
235
|
+
_backend_cls = _ZipBackend
|
|
236
|
+
|
|
237
|
+
def _open(self, mode="r", buffering=-1):
|
|
238
|
+
if "r" in mode:
|
|
239
|
+
return super()._open(mode, buffering)
|
|
240
|
+
if not self.backend.writable:
|
|
241
|
+
raise NotImplementedError(
|
|
242
|
+
"zip: write support requires a local (file:) outer archive"
|
|
243
|
+
)
|
|
244
|
+
if mode not in ("w", "x"):
|
|
245
|
+
raise NotImplementedError(f"open(mode={mode!r})")
|
|
246
|
+
if mode == "x" and self.exists():
|
|
247
|
+
raise FileExistsError(self)
|
|
248
|
+
return _ZipWriteStream(self.backend, self.path)
|
|
249
|
+
|
|
250
|
+
def _mkdir(self, mode):
|
|
251
|
+
if not self.backend.writable:
|
|
252
|
+
raise NotImplementedError(
|
|
253
|
+
"zip: write support requires a local (file:) outer archive"
|
|
254
|
+
)
|
|
255
|
+
if self.exists():
|
|
256
|
+
raise FileExistsError(self)
|
|
257
|
+
self.backend.write_member(f"{self.path}/", b"")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class TarUri(ArchiveUri):
|
|
261
|
+
"""`tar:` scheme (also handles `.tar.gz`/`.tar.bz2`/`.tar.xz` via
|
|
262
|
+
`tarfile`'s auto-detected "r:*" mode). Read-only."""
|
|
263
|
+
|
|
264
|
+
__SCHEMES = ("tar",)
|
|
265
|
+
__slots__ = ()
|
|
266
|
+
_backend_cls = _TarBackend
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io as _io
|
|
4
|
+
import urllib.parse as _urlparse
|
|
5
|
+
from base64 import b64decode as _b64decode
|
|
6
|
+
|
|
7
|
+
from ...utils.stat import FileStat
|
|
8
|
+
from .. import UriPath
|
|
9
|
+
|
|
10
|
+
_DEFAULT_MEDIATYPE = "text/plain;charset=US-ASCII"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DataUri(UriPath):
|
|
14
|
+
"""`data:` scheme (RFC 2397): a read-only "file" whose entire content is
|
|
15
|
+
embedded in the URI itself (`data:[<mediatype>][;base64],<data>`) -- no
|
|
16
|
+
filesystem, no directories, no backend/connection. Percent-encoded
|
|
17
|
+
reserved characters (`?`/`#`) in the payload are only decoded correctly
|
|
18
|
+
if the URI was built with them escaped, per RFC 2397/3986."""
|
|
19
|
+
|
|
20
|
+
__SCHEMES = ("data",)
|
|
21
|
+
__slots__ = ()
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def _header(self) -> str:
|
|
25
|
+
header, _, _ = self.path.partition(",")
|
|
26
|
+
return header
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def _is_base64(self) -> bool:
|
|
30
|
+
return self._header.rsplit(";", 1)[-1].strip().lower() == "base64"
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def mediatype(self) -> str:
|
|
34
|
+
header = self._header
|
|
35
|
+
if not header:
|
|
36
|
+
return _DEFAULT_MEDIATYPE
|
|
37
|
+
if self._is_base64:
|
|
38
|
+
header = header.rsplit(";", 1)[0]
|
|
39
|
+
return header or _DEFAULT_MEDIATYPE
|
|
40
|
+
|
|
41
|
+
def _content(self) -> bytes:
|
|
42
|
+
header, sep, data = self.path.partition(",")
|
|
43
|
+
if not sep:
|
|
44
|
+
raise FileNotFoundError(self)
|
|
45
|
+
if self._is_base64:
|
|
46
|
+
return _b64decode(_urlparse.unquote_to_bytes(data))
|
|
47
|
+
return _urlparse.unquote_to_bytes(data)
|
|
48
|
+
|
|
49
|
+
def stat(self, *, follow_symlinks=True):
|
|
50
|
+
return FileStat(st_size=len(self._content()), is_dir=False)
|
|
51
|
+
|
|
52
|
+
def _open(self, mode="r", buffering=-1):
|
|
53
|
+
if "r" not in mode:
|
|
54
|
+
raise NotImplementedError("data: URIs are read-only")
|
|
55
|
+
return _io.BytesIO(self._content())
|
|
56
|
+
|
|
57
|
+
def _listdir(self):
|
|
58
|
+
raise NotADirectoryError(self)
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import datetime as _dt
|
|
4
|
+
import ftplib as _ftplib
|
|
5
|
+
import io as _io
|
|
6
|
+
import threading as _thread
|
|
7
|
+
import typing as _ty
|
|
8
|
+
|
|
9
|
+
from ... import utils as _utils
|
|
10
|
+
from ...utils.stat import FileStat
|
|
11
|
+
from .. import Source, Uri, UriPath
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BaseFtpBackend(object):
|
|
15
|
+
"""Protocol for obtaining a connected+logged-in `ftplib.FTP` (or
|
|
16
|
+
`FTP_TLS`) for a `Source`. Subclass this to plug in custom connection
|
|
17
|
+
handling (e.g. tests mock it directly, no real server);
|
|
18
|
+
`FtpBackend` is the real implementation."""
|
|
19
|
+
|
|
20
|
+
__slots__ = ()
|
|
21
|
+
|
|
22
|
+
@_utils.notimplemented
|
|
23
|
+
def client(self, source: Source, tls: bool) -> "_ftplib.FTP": ...
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class FtpBackend(BaseFtpBackend):
|
|
27
|
+
"""Connects via stdlib `ftplib.FTP` (`ftp:`) or `ftplib.FTP_TLS`
|
|
28
|
+
(`ftps:`, with `PROT P` for an encrypted data channel too)."""
|
|
29
|
+
|
|
30
|
+
__slots__ = ("timeout",)
|
|
31
|
+
|
|
32
|
+
def __init__(self, timeout: float = None) -> None:
|
|
33
|
+
self.timeout = timeout
|
|
34
|
+
|
|
35
|
+
def client(self, source: Source, tls: bool):
|
|
36
|
+
cls = _ftplib.FTP_TLS if tls else _ftplib.FTP
|
|
37
|
+
client = cls(timeout=self.timeout)
|
|
38
|
+
client.connect(str(source.host), source.port or 21)
|
|
39
|
+
user, password = source.parsed_userinfo()
|
|
40
|
+
client.login(user or "anonymous", password or "")
|
|
41
|
+
if tls:
|
|
42
|
+
client.prot_p()
|
|
43
|
+
client.set_pasv(True)
|
|
44
|
+
return client
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _create_ftpclient(backend: BaseFtpBackend, source: Source, tls: bool, thread_id: int):
|
|
48
|
+
return backend.client(source, tls)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
_CACHED_CLIENTS = _utils.LRU(_create_ftpclient, maxsize=128)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse_mlsd_time(value: str) -> int:
|
|
55
|
+
# MLSD "modify" fact: YYYYMMDDHHMMSS[.sss], always UTC (RFC 3659).
|
|
56
|
+
try:
|
|
57
|
+
return int(
|
|
58
|
+
_dt.datetime.strptime(value[:14], "%Y%m%d%H%M%S").timestamp()
|
|
59
|
+
)
|
|
60
|
+
except ValueError:
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class _FtpWriteStream(_io.BytesIO):
|
|
65
|
+
"""Buffers the whole write in memory, uploads on close() via
|
|
66
|
+
STOR/APPE. Simple and works with any ftplib client, at the cost of
|
|
67
|
+
holding the full file content in memory for the duration of the write."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, client: "_ftplib.FTP", path: str, append: bool = False):
|
|
70
|
+
super().__init__()
|
|
71
|
+
self._client = client
|
|
72
|
+
self._path = path
|
|
73
|
+
self._cmd = "APPE" if append else "STOR"
|
|
74
|
+
|
|
75
|
+
def close(self):
|
|
76
|
+
if not self.closed:
|
|
77
|
+
self.seek(0)
|
|
78
|
+
self._client.storbinary(f"{self._cmd} {self._path}", self)
|
|
79
|
+
super().close()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class FtpPath(UriPath):
|
|
83
|
+
"""`ftp:`/`ftps:` scheme: full read/write access via stdlib `ftplib`,
|
|
84
|
+
with a thread-keyed LRU connection cache (`_CACHED_CLIENTS`, mirroring
|
|
85
|
+
`sftp.py`). Directory listing and stat prefer MLSD (RFC 3659 -- gives
|
|
86
|
+
type/size/modify facts in one round trip); servers that don't support it
|
|
87
|
+
fall back to NLST for listing (names only) and SIZE for file stat
|
|
88
|
+
(no portable "not found vs. is a directory" distinction in that path)."""
|
|
89
|
+
|
|
90
|
+
__SCHEMES = ("ftp", "ftps")
|
|
91
|
+
__slots__ = ()
|
|
92
|
+
|
|
93
|
+
if _ty.TYPE_CHECKING:
|
|
94
|
+
backend: BaseFtpBackend
|
|
95
|
+
|
|
96
|
+
def _initbackend(self):
|
|
97
|
+
return FtpBackend()
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def _tls(self):
|
|
101
|
+
return self.source.scheme == "ftps"
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def _ftpclient(self) -> "_ftplib.FTP":
|
|
105
|
+
thread_id = _thread.get_ident()
|
|
106
|
+
client = _CACHED_CLIENTS(self.backend, self.source, self._tls, thread_id)
|
|
107
|
+
try:
|
|
108
|
+
client.voidcmd("NOOP")
|
|
109
|
+
except (OSError, EOFError, _ftplib.error_temp, _ftplib.error_proto):
|
|
110
|
+
client = _CACHED_CLIENTS.invalidate(
|
|
111
|
+
self.backend, self.source, self._tls, thread_id
|
|
112
|
+
)
|
|
113
|
+
return client
|
|
114
|
+
|
|
115
|
+
def _mlsd_entry(self):
|
|
116
|
+
"""This entry's MLSD facts from its parent's listing, or None if
|
|
117
|
+
not found or the server doesn't support MLSD."""
|
|
118
|
+
parent = self.path.rsplit("/", 1)[0] or "/"
|
|
119
|
+
try:
|
|
120
|
+
for name, facts in self._ftpclient.mlsd(parent):
|
|
121
|
+
if name == self.name:
|
|
122
|
+
return facts
|
|
123
|
+
except _ftplib.error_perm:
|
|
124
|
+
return None
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
def _listdir(self):
|
|
128
|
+
try:
|
|
129
|
+
for name, _facts in self._ftpclient.mlsd(self.path):
|
|
130
|
+
if name not in (".", ".."):
|
|
131
|
+
yield name
|
|
132
|
+
except _ftplib.error_perm:
|
|
133
|
+
for name in self._ftpclient.nlst(self.path):
|
|
134
|
+
base = name.rsplit("/", 1)[-1]
|
|
135
|
+
if base not in (".", ".."):
|
|
136
|
+
yield base
|
|
137
|
+
|
|
138
|
+
def stat(self, *, follow_symlinks=True):
|
|
139
|
+
facts = self._mlsd_entry()
|
|
140
|
+
if facts is not None:
|
|
141
|
+
kind = facts.get("type", "file")
|
|
142
|
+
size = int(facts.get("size", 0) or 0)
|
|
143
|
+
modify = facts.get("modify")
|
|
144
|
+
mtime = _parse_mlsd_time(modify) if modify else 0
|
|
145
|
+
return FileStat(
|
|
146
|
+
st_size=size, st_mtime=mtime, is_dir=kind in ("dir", "cdir", "pdir")
|
|
147
|
+
)
|
|
148
|
+
# MLSD unsupported (or entry not found via it) -- SIZE only works
|
|
149
|
+
# for files, so this can't tell "missing" apart from "is a
|
|
150
|
+
# directory"; either way there's no file to report a size for.
|
|
151
|
+
try:
|
|
152
|
+
size = self._ftpclient.size(self.path)
|
|
153
|
+
except _ftplib.error_perm as error:
|
|
154
|
+
raise FileNotFoundError(self) from error
|
|
155
|
+
if size is None:
|
|
156
|
+
raise FileNotFoundError(self)
|
|
157
|
+
return FileStat(st_size=size, is_dir=False)
|
|
158
|
+
|
|
159
|
+
def _open(self, mode="r", buffering=-1):
|
|
160
|
+
if "r" in mode:
|
|
161
|
+
buf = _io.BytesIO()
|
|
162
|
+
try:
|
|
163
|
+
self._ftpclient.retrbinary(f"RETR {self.path}", buf.write)
|
|
164
|
+
except _ftplib.error_perm as error:
|
|
165
|
+
raise FileNotFoundError(self) from error
|
|
166
|
+
buf.seek(0)
|
|
167
|
+
return buf
|
|
168
|
+
if mode not in ("w", "x", "a"):
|
|
169
|
+
raise NotImplementedError(f"open(mode={mode!r})")
|
|
170
|
+
if mode == "x" and self.exists():
|
|
171
|
+
raise FileExistsError(self)
|
|
172
|
+
return _FtpWriteStream(self._ftpclient, self.path, append=(mode == "a"))
|
|
173
|
+
|
|
174
|
+
def _mkdir(self, mode):
|
|
175
|
+
try:
|
|
176
|
+
self._ftpclient.mkd(self.path)
|
|
177
|
+
except _ftplib.error_perm as error:
|
|
178
|
+
if self.exists():
|
|
179
|
+
raise FileExistsError(self) from error
|
|
180
|
+
raise FileNotFoundError(self) from error
|
|
181
|
+
|
|
182
|
+
def unlink(self, missing_ok=False):
|
|
183
|
+
try:
|
|
184
|
+
self._ftpclient.delete(self.path)
|
|
185
|
+
except _ftplib.error_perm as error:
|
|
186
|
+
if missing_ok and not self.exists():
|
|
187
|
+
return
|
|
188
|
+
raise FileNotFoundError(self) from error
|
|
189
|
+
|
|
190
|
+
def rmdir(self):
|
|
191
|
+
self._ftpclient.rmd(self.path)
|
|
192
|
+
|
|
193
|
+
def rename(self, target: "FtpPath | Uri | str"):
|
|
194
|
+
# A plain str target is a sibling rename (relative to self's
|
|
195
|
+
# parent), matching sftp.py's rename() semantics.
|
|
196
|
+
if not isinstance(target, Uri):
|
|
197
|
+
target = Uri(self.parent, target)
|
|
198
|
+
self._ftpclient.rename(self.path, target.path)
|
|
199
|
+
|
|
200
|
+
def chmod(self, mode, *, follow_symlinks=True):
|
|
201
|
+
# SITE CHMOD is a common but non-standard FTP extension; not every
|
|
202
|
+
# server supports it (ftplib.error_perm propagates uncaught if so).
|
|
203
|
+
if not follow_symlinks:
|
|
204
|
+
raise NotImplementedError("chmod(follow_symlinks=False)")
|
|
205
|
+
self._ftpclient.voidcmd(f"SITE CHMOD {mode:o} {self.path}")
|