pathlib-next 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/CHANGELOG.md +21 -1
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/PKG-INFO +4 -1
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/docs/divergences.md +2 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/docs/guides/schemes.md +27 -11
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/pyproject.toml +2 -1
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/__init__.py +8 -0
- pathlib_next-0.7.0/src/pathlib_next/uri/schemes/s3.py +179 -0
- pathlib_next-0.7.0/src/pathlib_next/uri/schemes/webdav.py +180 -0
- pathlib_next-0.7.0/tests/test_s3.py +210 -0
- pathlib_next-0.7.0/tests/test_webdav.py +250 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/.gitignore +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/LICENSE +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/README.md +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/docs/api/reference.md +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/docs/changelog.md +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/docs/guides/extending.md +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/docs/index.md +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/examples/http_listing.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/examples/local_and_mem.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/examples/sftp_sync.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/mkdocs.yml +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/__init__.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/fspath.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/mempath.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/path.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/protocols/__init__.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/protocols/fs.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/protocols/io.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/py.typed +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/testing.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/__init__.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/query.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/archive.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/data.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/file.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/ftp.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/http.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/schemes/sftp.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/uri/source.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/utils/__init__.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/utils/glob.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/utils/stat.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/src/pathlib_next/utils/sync.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/conftest.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_archive_uri.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_contract.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_data_uri.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_ftp.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_glob.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_http.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_local.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_mempath.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_parity_io.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_parity_pure.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_pathname.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_query.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_sftp.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_smoke.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_source.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_sync.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_uri_parse.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_uri_path.py +0 -0
- {pathlib_next-0.6.0 → pathlib_next-0.7.0}/tests/test_utils.py +0 -0
|
@@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.7.0] - 2026-07-11
|
|
11
|
+
|
|
12
|
+
### Added (Phase 7b new schemes, optional extras)
|
|
13
|
+
- `dav:`/`davs:` scheme (`pathlib_next.uri.schemes.webdav.DavPath`):
|
|
14
|
+
extends `HttpPath` with WebDAV (RFC 4918) PROPFIND for real stat/listdir
|
|
15
|
+
metadata (replacing HTML-index scraping) and PUT/DELETE/MKCOL/MOVE for
|
|
16
|
+
full read/write access. Requests go to the equivalent `http:`/`https:`
|
|
17
|
+
URL; `as_uri()` still reports `dav:`/`davs:`. Reuses the `http` extra,
|
|
18
|
+
no new dependency. `rmdir()` is recursive by WebDAV spec, unlike
|
|
19
|
+
`pathlib.Path.rmdir()`'s "must be empty" contract -- documented, not
|
|
20
|
+
silent.
|
|
21
|
+
- `s3:` scheme (`pathlib_next.uri.schemes.s3.S3Path`,
|
|
22
|
+
`s3://bucket/key/path`): read/write/list via `boto3`. New `s3` extra.
|
|
23
|
+
S3 has no real directories -- `is_dir()` is prefix emulation (any
|
|
24
|
+
object key under `"<path>/"`), `mkdir()` creates a zero-byte `"<path>/"`
|
|
25
|
+
marker object, `rmdir()` requires no other keys under that prefix.
|
|
26
|
+
`rename()` uses server-side `copy_object`+`delete_object` (same-bucket
|
|
27
|
+
only) instead of the generic download+upload+delete `move()` fallback.
|
|
28
|
+
|
|
10
29
|
## [0.6.0] - 2026-07-11
|
|
11
30
|
|
|
12
31
|
### Added (Phase 7a new schemes, stdlib-only, no new deps)
|
|
@@ -235,7 +254,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
|
235
254
|
- Sync error handling.
|
|
236
255
|
- Generic Path Protocol based pathlib implementation for URI paths with file access support for sftp, http, file schemes.
|
|
237
256
|
|
|
238
|
-
[Unreleased]: https://github.com/jose-pr/pathlib_next/compare/v0.
|
|
257
|
+
[Unreleased]: https://github.com/jose-pr/pathlib_next/compare/v0.7.0...HEAD
|
|
258
|
+
[0.7.0]: https://github.com/jose-pr/pathlib_next/compare/v0.6.0...v0.7.0
|
|
239
259
|
[0.6.0]: https://github.com/jose-pr/pathlib_next/compare/v0.5.0...v0.6.0
|
|
240
260
|
[0.5.0]: https://github.com/jose-pr/pathlib_next/compare/v0.4.1...v0.5.0
|
|
241
261
|
[0.4.1]: https://github.com/jose-pr/pathlib_next/compare/v0.4.0...v0.4.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pathlib_next
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Generic Path Protocol based pathlib
|
|
5
5
|
Project-URL: Homepage, https://github.com/jose-pr/pathlib_next/
|
|
6
6
|
Project-URL: Documentation, https://jose-pr.github.io/pathlib_next/
|
|
@@ -26,6 +26,9 @@ Requires-Dist: bs4; extra == 'http'
|
|
|
26
26
|
Requires-Dist: htmllistparse; extra == 'http'
|
|
27
27
|
Requires-Dist: requests; extra == 'http'
|
|
28
28
|
Requires-Dist: uritools; extra == 'http'
|
|
29
|
+
Provides-Extra: s3
|
|
30
|
+
Requires-Dist: boto3; extra == 's3'
|
|
31
|
+
Requires-Dist: uritools; extra == 's3'
|
|
29
32
|
Provides-Extra: sftp
|
|
30
33
|
Requires-Dist: paramiko; extra == 'sftp'
|
|
31
34
|
Requires-Dist: uritools; extra == 'sftp'
|
|
@@ -20,6 +20,8 @@ The divergences below apply to `Uri`/`UriPath` and `MemPath`.
|
|
|
20
20
|
| `Path.move(target, ...)` | Not in `pathlib` at all | Our own extension: tries `rename()`, falls back to copy+unlink | N/A -- pure extension, no pathlib method to diverge from. |
|
|
21
21
|
| `Path.rm(recursive=, missing_ok=, ignore_error=)` | Not in `pathlib` (closest: `shutil.rmtree`) | Our own extension | N/A -- pure extension. |
|
|
22
22
|
| `PathSyncer` / `Query` / `Source` | N/A | Our own extensions | N/A -- pure extensions, no pathlib equivalent. |
|
|
23
|
+
| `DavPath.rmdir()` | `pathlib.Path.rmdir()` requires the directory to be empty, raises `OSError` otherwise | WebDAV `DELETE` on a collection is recursive by spec (RFC 4918) -- deletes non-empty collections too | Not our choice -- inherent to the WebDAV protocol's `DELETE` semantics; no separate "delete if empty" verb exists to build an empty-check on top of. **Caution**, not a feature. |
|
|
24
|
+
| `S3Path` directories | N/A (pathlib directories are real filesystem entries) | `is_dir()` is prefix emulation (any object key under `"<path>/"`); `mkdir()` creates a zero-byte `"<path>/"` marker object; `rmdir()` requires no other keys under that prefix (pathlib-parity "must be empty", unlike `DavPath.rmdir()` above) | S3 has no native directory concept -- this is the same prefix convention the AWS console itself uses for an empty "folder". |
|
|
23
25
|
|
|
24
26
|
## Explicitly out of scope (not implemented on `Pathname`/`Path`)
|
|
25
27
|
|
|
@@ -6,17 +6,17 @@ meaningful for a read-only scheme) -- everything else (name/suffix parsing,
|
|
|
6
6
|
`glob()`, `walk()`, `copy()`, ...) is derived from these primitives and
|
|
7
7
|
works identically across all of them.
|
|
8
8
|
|
|
9
|
-
| Capability | `LocalPath` | `file:` (`FileUri`) | `mem:` (`MemPath`) | `http(s):` (`HttpPath`) | `sftp:` (`SftpPath`) | `data:` (`DataUri`) | `ftp(s):` (`FtpPath`) | `zip:` (`ZipUri`) | `tar:` (`TarUri`) |
|
|
10
|
-
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
11
|
-
| Read (`read_text`/`read_bytes`/`open`) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
|
|
12
|
-
| Write (`write_text`/`write_bytes`/`open("w")`) | Yes | Yes | Yes | No | Yes | No | Yes | New entries only, local outer archive only | No |
|
|
13
|
-
| List (`iterdir`) | Yes | Yes | Yes | Yes (scrapes an HTML index) | Yes | No (`NotADirectoryError`) | Yes (MLSD, falls back to NLST) | Yes | Yes |
|
|
14
|
-
| Stat (`stat`, `exists`, `is_dir`, `is_file`, ...) | Yes | Yes | Yes | Yes (via `HEAD`, falls back to `GET`) | Yes | Yes (`st_size` from decoded payload) | Yes (MLSD, falls back to SIZE for files) | Yes | Yes |
|
|
15
|
-
| `mkdir` | Yes | Yes | Yes | No | Yes | No | Yes | Local outer archive only (zero-length `name/` entry) | No |
|
|
16
|
-
| Delete (`unlink`/`rmdir`/`rm`) | Yes | Yes | Yes | No | Yes | No | Yes | No | No |
|
|
17
|
-
| `rename` | Yes | Yes | No (`move()` falls back to copy+unlink) | No | Yes | No | Yes | No | No |
|
|
18
|
-
| `chmod` | Yes | Yes | No | No | Yes (no `follow_symlinks=False`; paramiko has no `lchmod`) | No | Yes (`SITE CHMOD`, server-dependent) | No | No |
|
|
19
|
-
| Extra required | none | none | none | `http` | `sftp` | none (stdlib) | none (stdlib `ftplib`) | none (stdlib `zipfile`) | none (stdlib `tarfile`) |
|
|
9
|
+
| Capability | `LocalPath` | `file:` (`FileUri`) | `mem:` (`MemPath`) | `http(s):` (`HttpPath`) | `sftp:` (`SftpPath`) | `data:` (`DataUri`) | `ftp(s):` (`FtpPath`) | `zip:` (`ZipUri`) | `tar:` (`TarUri`) | `dav(s):` (`DavPath`) | `s3:` (`S3Path`) |
|
|
10
|
+
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
11
|
+
| Read (`read_text`/`read_bytes`/`open`) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
|
|
12
|
+
| Write (`write_text`/`write_bytes`/`open("w")`) | Yes | Yes | Yes | No | Yes | No | Yes | New entries only, local outer archive only | No | Yes | Yes |
|
|
13
|
+
| List (`iterdir`) | Yes | Yes | Yes | Yes (scrapes an HTML index) | Yes | No (`NotADirectoryError`) | Yes (MLSD, falls back to NLST) | Yes | Yes | Yes (PROPFIND) | Yes (prefix + delimiter emulation) |
|
|
14
|
+
| Stat (`stat`, `exists`, `is_dir`, `is_file`, ...) | Yes | Yes | Yes | Yes (via `HEAD`, falls back to `GET`) | Yes | Yes (`st_size` from decoded payload) | Yes (MLSD, falls back to SIZE for files) | Yes | Yes | Yes (PROPFIND) | Yes (`is_dir` is prefix emulation) |
|
|
15
|
+
| `mkdir` | Yes | Yes | Yes | No | Yes | No | Yes | Local outer archive only (zero-length `name/` entry) | No | Yes (MKCOL) | Yes (zero-byte `key/` marker object) |
|
|
16
|
+
| Delete (`unlink`/`rmdir`/`rm`) | Yes | Yes | Yes | No | Yes | No | Yes | No | No | Yes (`rmdir` is recursive, see notes) | Yes (`rmdir` requires empty prefix) |
|
|
17
|
+
| `rename` | Yes | Yes | No (`move()` falls back to copy+unlink) | No | Yes | No | Yes | No | No | Yes (MOVE) | Yes (server-side `copy_object`+delete, same bucket) |
|
|
18
|
+
| `chmod` | Yes | Yes | No | No | Yes (no `follow_symlinks=False`; paramiko has no `lchmod`) | No | Yes (`SITE CHMOD`, server-dependent) | No | No | No | No |
|
|
19
|
+
| Extra required | none | none | none | `http` | `sftp` | none (stdlib) | none (stdlib `ftplib`) | none (stdlib `zipfile`) | none (stdlib `tarfile`) | `http` (reused) | `s3` |
|
|
20
20
|
|
|
21
21
|
Notes:
|
|
22
22
|
|
|
@@ -57,6 +57,22 @@ Notes:
|
|
|
57
57
|
works when the outer archive is itself a local `file:` URI; every other
|
|
58
58
|
outer scheme is read-only. `tar:` (with transparent `.tar.gz`/`.tar.bz2`/
|
|
59
59
|
`.tar.xz` decompression) is read-only regardless of the outer scheme.
|
|
60
|
+
- **`dav(s):`** is WebDAV (RFC 4918) layered on `HttpPath` -- PROPFIND
|
|
61
|
+
replaces HTML-index scraping for real stat/listdir metadata, and PUT/
|
|
62
|
+
DELETE/MKCOL/MOVE give it full write support (unlike plain `http(s):`).
|
|
63
|
+
Requests go out over the equivalent `http:`/`https:` URL; `as_uri()`
|
|
64
|
+
still reports `dav:`/`davs:`. Reuses the `http` extra, no new dependency.
|
|
65
|
+
**Caution:** `rmdir()` maps to WebDAV `DELETE`, which is recursive by
|
|
66
|
+
spec -- unlike `pathlib.Path.rmdir()`, it does not require the
|
|
67
|
+
collection to be empty first.
|
|
68
|
+
- **`s3:`** (`s3://bucket/key/path`) has no real directories: `is_dir()`
|
|
69
|
+
is prefix emulation (any object key under `"<path>/"`), and `mkdir()`
|
|
70
|
+
creates a zero-byte `"<path>/"` marker object (the same convention the
|
|
71
|
+
AWS console itself uses for an empty "folder") -- `rmdir()` requires no
|
|
72
|
+
other keys under that prefix (pathlib's "must be empty" semantics, unlike
|
|
73
|
+
`dav:`'s recursive `DELETE`). A single `boto3` client is cached per
|
|
74
|
+
backend (documented thread-safe, unlike `sftp:`/`ftp:`'s per-thread
|
|
75
|
+
connection pools).
|
|
60
76
|
- See [Divergences from pathlib](../divergences.md) for the "explicitly out
|
|
61
77
|
of scope" list (`resolve`, `symlink_to`, `owner`, `expanduser`, ...) that
|
|
62
78
|
applies uniformly across every non-`LocalPath` implementation.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pathlib_next"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
authors = [{ name = "Jose A" }]
|
|
9
9
|
description = "Generic Path Protocol based pathlib"
|
|
10
10
|
readme = "README.md"
|
|
@@ -19,6 +19,7 @@ dependencies = []
|
|
|
19
19
|
uri = ["uritools"]
|
|
20
20
|
http = ["requests", "pathlib_next[uri]", 'htmllistparse', 'bs4']
|
|
21
21
|
sftp = ["paramiko", "pathlib_next[uri]"]
|
|
22
|
+
s3 = ["boto3", "pathlib_next[uri]"]
|
|
22
23
|
dev = ['build', 'twine', 'hatchling', 'pytest', 'pytest-cov']
|
|
23
24
|
docs = ['mkdocs', 'mkdocs-material', 'mkdocstrings[python]']
|
|
24
25
|
|
|
@@ -8,7 +8,15 @@ try:
|
|
|
8
8
|
from .http import HttpPath as HttpPath
|
|
9
9
|
except ImportError:
|
|
10
10
|
pass
|
|
11
|
+
try:
|
|
12
|
+
from .webdav import DavPath as DavPath
|
|
13
|
+
except ImportError:
|
|
14
|
+
pass
|
|
11
15
|
try:
|
|
12
16
|
from .sftp import SftpPath as SftpPath
|
|
13
17
|
except ImportError:
|
|
14
18
|
pass
|
|
19
|
+
try:
|
|
20
|
+
from .s3 import S3Path as S3Path
|
|
21
|
+
except ImportError:
|
|
22
|
+
pass
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io as _io
|
|
4
|
+
import typing as _ty
|
|
5
|
+
|
|
6
|
+
import botocore.exceptions as _botoexc
|
|
7
|
+
|
|
8
|
+
from ... import utils as _utils
|
|
9
|
+
from ...utils.stat import FileStat
|
|
10
|
+
from .. import Source, Uri, UriPath
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaseS3Backend(object):
|
|
14
|
+
"""Protocol for obtaining a `boto3` S3 client. Subclass this to plug in
|
|
15
|
+
custom credential/session handling (e.g. tests mock it directly, no
|
|
16
|
+
real AWS account needed); `S3Backend` is the real implementation."""
|
|
17
|
+
|
|
18
|
+
__slots__ = ()
|
|
19
|
+
|
|
20
|
+
@_utils.notimplemented
|
|
21
|
+
def client(self): ...
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class S3Backend(BaseS3Backend):
|
|
25
|
+
"""Lazily creates+caches a `boto3` S3 client. Unlike `sftp.py`'s/
|
|
26
|
+
`ftp.py`'s per-thread connection pools, a single `boto3` client is
|
|
27
|
+
reused across threads -- it's documented as thread-safe."""
|
|
28
|
+
|
|
29
|
+
__slots__ = ("client_kwargs", "_client")
|
|
30
|
+
|
|
31
|
+
def __init__(self, **client_kwargs):
|
|
32
|
+
self.client_kwargs = client_kwargs
|
|
33
|
+
self._client = None
|
|
34
|
+
|
|
35
|
+
def client(self):
|
|
36
|
+
if self._client is None:
|
|
37
|
+
import boto3
|
|
38
|
+
|
|
39
|
+
self._client = boto3.client("s3", **self.client_kwargs)
|
|
40
|
+
return self._client
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _is_not_found(error: _botoexc.ClientError) -> bool:
|
|
44
|
+
code = error.response.get("Error", {}).get("Code")
|
|
45
|
+
return code in ("404", "NoSuchKey", "NotFound")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _S3WriteStream(_io.BytesIO):
|
|
49
|
+
def __init__(self, path: "S3Path"):
|
|
50
|
+
super().__init__()
|
|
51
|
+
self._path = path
|
|
52
|
+
|
|
53
|
+
def close(self):
|
|
54
|
+
if not self.closed:
|
|
55
|
+
self._path._client.put_object(
|
|
56
|
+
Bucket=self._path.bucket, Key=self._path.key, Body=self.getvalue()
|
|
57
|
+
)
|
|
58
|
+
super().close()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class S3Path(UriPath):
|
|
62
|
+
"""`s3:` scheme (`s3://bucket/key/path`): read/write/list via `boto3`.
|
|
63
|
+
Requires the `s3` extra. S3 has no real directories -- `is_dir()` is
|
|
64
|
+
prefix emulation (any object key under `"<path>/"`), and `mkdir()`
|
|
65
|
+
creates a zero-byte `"<path>/"` marker object (the same convention the
|
|
66
|
+
AWS console itself uses for an empty "folder"); see
|
|
67
|
+
`docs/divergences.md`. `rename()` uses server-side `copy_object` +
|
|
68
|
+
`delete_object` (same-bucket only) instead of the generic
|
|
69
|
+
download+upload+delete `move()` fallback."""
|
|
70
|
+
|
|
71
|
+
__SCHEMES = ("s3",)
|
|
72
|
+
__slots__ = ()
|
|
73
|
+
|
|
74
|
+
if _ty.TYPE_CHECKING:
|
|
75
|
+
backend: BaseS3Backend
|
|
76
|
+
|
|
77
|
+
def _initbackend(self):
|
|
78
|
+
return S3Backend()
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def bucket(self) -> str:
|
|
82
|
+
return self.source.host
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def key(self) -> str:
|
|
86
|
+
return self.path.lstrip("/")
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def _client(self):
|
|
90
|
+
return self.backend.client()
|
|
91
|
+
|
|
92
|
+
def stat(self, *, follow_symlinks=True):
|
|
93
|
+
key = self.key
|
|
94
|
+
if key == "":
|
|
95
|
+
try:
|
|
96
|
+
self._client.head_bucket(Bucket=self.bucket)
|
|
97
|
+
except _botoexc.ClientError as error:
|
|
98
|
+
raise FileNotFoundError(self) from error
|
|
99
|
+
return FileStat(is_dir=True)
|
|
100
|
+
try:
|
|
101
|
+
head = self._client.head_object(Bucket=self.bucket, Key=key)
|
|
102
|
+
return FileStat(
|
|
103
|
+
st_size=head["ContentLength"],
|
|
104
|
+
st_mtime=int(head["LastModified"].timestamp()),
|
|
105
|
+
is_dir=False,
|
|
106
|
+
)
|
|
107
|
+
except _botoexc.ClientError as error:
|
|
108
|
+
if not _is_not_found(error):
|
|
109
|
+
raise
|
|
110
|
+
# Not an object at this exact key -- emulate a directory: any
|
|
111
|
+
# object under the "<key>/" prefix means this is a "directory".
|
|
112
|
+
resp = self._client.list_objects_v2(
|
|
113
|
+
Bucket=self.bucket, Prefix=f"{key}/", MaxKeys=1
|
|
114
|
+
)
|
|
115
|
+
if resp.get("KeyCount", 0) > 0:
|
|
116
|
+
return FileStat(is_dir=True)
|
|
117
|
+
raise FileNotFoundError(self)
|
|
118
|
+
|
|
119
|
+
def _listdir(self):
|
|
120
|
+
prefix = f"{self.key}/" if self.key else ""
|
|
121
|
+
seen = set()
|
|
122
|
+
paginator = self._client.get_paginator("list_objects_v2")
|
|
123
|
+
for page in paginator.paginate(
|
|
124
|
+
Bucket=self.bucket, Prefix=prefix, Delimiter="/"
|
|
125
|
+
):
|
|
126
|
+
for common in page.get("CommonPrefixes", []):
|
|
127
|
+
name = common["Prefix"][len(prefix) :].rstrip("/")
|
|
128
|
+
if name and name not in seen:
|
|
129
|
+
seen.add(name)
|
|
130
|
+
yield name
|
|
131
|
+
for obj in page.get("Contents", []):
|
|
132
|
+
name = obj["Key"][len(prefix) :]
|
|
133
|
+
if name and name not in seen:
|
|
134
|
+
seen.add(name)
|
|
135
|
+
yield name
|
|
136
|
+
|
|
137
|
+
def _open(self, mode="r", buffering=-1):
|
|
138
|
+
if "r" in mode:
|
|
139
|
+
try:
|
|
140
|
+
resp = self._client.get_object(Bucket=self.bucket, Key=self.key)
|
|
141
|
+
except _botoexc.ClientError as error:
|
|
142
|
+
raise FileNotFoundError(self) from error
|
|
143
|
+
return _io.BytesIO(resp["Body"].read())
|
|
144
|
+
if mode not in ("w", "x"):
|
|
145
|
+
raise NotImplementedError(f"open(mode={mode!r})")
|
|
146
|
+
if mode == "x" and self.exists():
|
|
147
|
+
raise FileExistsError(self)
|
|
148
|
+
return _S3WriteStream(self)
|
|
149
|
+
|
|
150
|
+
def _mkdir(self, mode):
|
|
151
|
+
if self.exists():
|
|
152
|
+
raise FileExistsError(self)
|
|
153
|
+
self._client.put_object(Bucket=self.bucket, Key=f"{self.key}/", Body=b"")
|
|
154
|
+
|
|
155
|
+
def unlink(self, missing_ok=False):
|
|
156
|
+
if not missing_ok and not self.exists():
|
|
157
|
+
raise FileNotFoundError(self)
|
|
158
|
+
# delete_object() is idempotent (no error for a missing key) --
|
|
159
|
+
# the exists() check above is what actually enforces missing_ok.
|
|
160
|
+
self._client.delete_object(Bucket=self.bucket, Key=self.key)
|
|
161
|
+
|
|
162
|
+
def rmdir(self):
|
|
163
|
+
marker = f"{self.key}/"
|
|
164
|
+
resp = self._client.list_objects_v2(Bucket=self.bucket, Prefix=marker, MaxKeys=2)
|
|
165
|
+
contents = resp.get("Contents", [])
|
|
166
|
+
if any(obj["Key"] != marker for obj in contents):
|
|
167
|
+
raise OSError(f"Directory not empty: {self}")
|
|
168
|
+
self._client.delete_object(Bucket=self.bucket, Key=marker)
|
|
169
|
+
|
|
170
|
+
def rename(self, target: "S3Path | Uri | str"):
|
|
171
|
+
if not isinstance(target, Uri):
|
|
172
|
+
target = Uri(self.parent, target)
|
|
173
|
+
dest_key = target.path.lstrip("/")
|
|
174
|
+
self._client.copy_object(
|
|
175
|
+
Bucket=self.bucket,
|
|
176
|
+
Key=dest_key,
|
|
177
|
+
CopySource={"Bucket": self.bucket, "Key": self.key},
|
|
178
|
+
)
|
|
179
|
+
self._client.delete_object(Bucket=self.bucket, Key=self.key)
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io as _io
|
|
4
|
+
import stat as _stat
|
|
5
|
+
import urllib.parse as _urlparse
|
|
6
|
+
import xml.etree.ElementTree as _ET
|
|
7
|
+
|
|
8
|
+
import uritools as _uritools
|
|
9
|
+
|
|
10
|
+
from ... import utils as _utils
|
|
11
|
+
from ...utils.stat import FileStat
|
|
12
|
+
from .. import Uri
|
|
13
|
+
from .http import HttpPath
|
|
14
|
+
|
|
15
|
+
_NS = {"D": "DAV:"}
|
|
16
|
+
|
|
17
|
+
_PROPFIND_BODY = b"""<?xml version="1.0" encoding="utf-8"?>
|
|
18
|
+
<D:propfind xmlns:D="DAV:">
|
|
19
|
+
<D:prop>
|
|
20
|
+
<D:resourcetype/>
|
|
21
|
+
<D:getcontentlength/>
|
|
22
|
+
<D:getlastmodified/>
|
|
23
|
+
</D:prop>
|
|
24
|
+
</D:propfind>"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _parse_response(elem) -> "tuple[str, bool, int, str]":
|
|
28
|
+
href = elem.findtext("D:href", namespaces=_NS) or ""
|
|
29
|
+
prop = elem.find("D:propstat/D:prop", _NS)
|
|
30
|
+
resourcetype = prop.find("D:resourcetype", _NS) if prop is not None else None
|
|
31
|
+
is_dir = (
|
|
32
|
+
resourcetype is not None
|
|
33
|
+
and resourcetype.find("D:collection", _NS) is not None
|
|
34
|
+
)
|
|
35
|
+
size_text = prop.findtext("D:getcontentlength", namespaces=_NS) if prop is not None else None
|
|
36
|
+
size = int(size_text) if size_text else 0
|
|
37
|
+
lm = prop.findtext("D:getlastmodified", namespaces=_NS) if prop is not None else None
|
|
38
|
+
return _urlparse.unquote(href), is_dir, size, lm
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class _DavWriteStream(_io.BytesIO):
|
|
42
|
+
def __init__(self, path: "DavPath"):
|
|
43
|
+
super().__init__()
|
|
44
|
+
self._path = path
|
|
45
|
+
|
|
46
|
+
def close(self):
|
|
47
|
+
if not self.closed:
|
|
48
|
+
resp = self._path.backend.request(
|
|
49
|
+
"PUT", self._path._wire_uri(), data=self.getvalue()
|
|
50
|
+
)
|
|
51
|
+
resp.raise_for_status()
|
|
52
|
+
super().close()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DavPath(HttpPath):
|
|
56
|
+
"""`dav:`/`davs:` scheme: WebDAV (RFC 4918) over HTTP(S). Extends
|
|
57
|
+
`HttpPath` with PROPFIND (stat/listdir -- real directory metadata,
|
|
58
|
+
replacing `HttpPath`'s HTML-index scraping) and PUT/DELETE/MKCOL/MOVE
|
|
59
|
+
(full write support). Requests go to the equivalent `http:`/`https:`
|
|
60
|
+
URL (`_wire_uri()`); `as_uri()` still reports `dav:`/`davs:`. Reuses
|
|
61
|
+
`HttpPath`'s `HttpBackend` (session + requests_args) and the `http`
|
|
62
|
+
extra -- no new dependency.
|
|
63
|
+
|
|
64
|
+
Caution: `rmdir()` maps to WebDAV `DELETE`, which is recursive by
|
|
65
|
+
spec (RFC 4918) -- unlike `pathlib.Path.rmdir()`, it does NOT require
|
|
66
|
+
the collection to be empty first.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
__SCHEMES = ("dav", "davs")
|
|
70
|
+
__slots__ = ()
|
|
71
|
+
|
|
72
|
+
def _wire_uri(self) -> str:
|
|
73
|
+
scheme = "https" if self.source.scheme == "davs" else "http"
|
|
74
|
+
return _uritools.uricompose(
|
|
75
|
+
scheme=scheme,
|
|
76
|
+
userinfo=self.source.userinfo,
|
|
77
|
+
host=self.source.host,
|
|
78
|
+
port=self.source.port,
|
|
79
|
+
path=self.path,
|
|
80
|
+
query=self.query or None,
|
|
81
|
+
fragment=self.fragment or None,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def _propfind(self, depth="0"):
|
|
85
|
+
resp = self.backend.request(
|
|
86
|
+
"PROPFIND",
|
|
87
|
+
self._wire_uri(),
|
|
88
|
+
headers={"Depth": depth, "Content-Type": "application/xml"},
|
|
89
|
+
data=_PROPFIND_BODY,
|
|
90
|
+
)
|
|
91
|
+
if resp.status_code == 404:
|
|
92
|
+
raise FileNotFoundError(self)
|
|
93
|
+
if resp.status_code == 403:
|
|
94
|
+
raise PermissionError(self)
|
|
95
|
+
resp.raise_for_status()
|
|
96
|
+
return _ET.fromstring(resp.content)
|
|
97
|
+
|
|
98
|
+
def stat(self, *, follow_symlinks=True):
|
|
99
|
+
root = self._propfind(depth="0")
|
|
100
|
+
responses = root.findall("D:response", _NS)
|
|
101
|
+
if not responses:
|
|
102
|
+
raise FileNotFoundError(self)
|
|
103
|
+
_href, is_dir, size, lm = _parse_response(responses[0])
|
|
104
|
+
return FileStat(st_size=size, st_mtime=_utils.parsedate(lm), is_dir=is_dir)
|
|
105
|
+
|
|
106
|
+
def is_dir(self):
|
|
107
|
+
# Shadow HttpPath.is_dir()'s _isdir-slot cache (never populated by
|
|
108
|
+
# our stat() override) -- derive from stat() like any other scheme.
|
|
109
|
+
return _stat.S_ISDIR(self._st_mode() or 0)
|
|
110
|
+
|
|
111
|
+
def is_file(self):
|
|
112
|
+
return _stat.S_ISREG(self._st_mode() or 0)
|
|
113
|
+
|
|
114
|
+
def _listdir(self):
|
|
115
|
+
root = self._propfind(depth="1")
|
|
116
|
+
self_path = _urlparse.urlsplit(self._wire_uri()).path.rstrip("/")
|
|
117
|
+
for elem in root.findall("D:response", _NS):
|
|
118
|
+
href, _is_dir, _size, _lm = _parse_response(elem)
|
|
119
|
+
href_path = _urlparse.urlsplit(href).path.rstrip("/")
|
|
120
|
+
if not href_path or href_path == self_path:
|
|
121
|
+
continue # the "." entry describing self, per RFC 4918
|
|
122
|
+
name = href_path.rsplit("/", 1)[-1]
|
|
123
|
+
if name:
|
|
124
|
+
yield name
|
|
125
|
+
|
|
126
|
+
def iterdir(self):
|
|
127
|
+
# Reinstate the plain name+_make_child_relpath contract (HttpPath
|
|
128
|
+
# overrides this to expect FileEntry-like objects from _listdir(),
|
|
129
|
+
# which ours doesn't return).
|
|
130
|
+
for name in self._listdir():
|
|
131
|
+
yield self._make_child_relpath(name)
|
|
132
|
+
|
|
133
|
+
def _open(self, mode="r", buffering=-1):
|
|
134
|
+
if "r" in mode:
|
|
135
|
+
buffer_size = _io.DEFAULT_BUFFER_SIZE if buffering < 0 else buffering
|
|
136
|
+
req = self.backend.request("GET", self._wire_uri(), stream=True)
|
|
137
|
+
resp = req.raw
|
|
138
|
+
resp.auto_close = False
|
|
139
|
+
return (
|
|
140
|
+
resp
|
|
141
|
+
if buffer_size == 0
|
|
142
|
+
else _io.BufferedReader(resp, buffer_size=buffer_size)
|
|
143
|
+
)
|
|
144
|
+
if mode not in ("w", "x"):
|
|
145
|
+
raise NotImplementedError(f"open(mode={mode!r})")
|
|
146
|
+
if mode == "x" and self.exists():
|
|
147
|
+
raise FileExistsError(self)
|
|
148
|
+
return _DavWriteStream(self)
|
|
149
|
+
|
|
150
|
+
def _mkdir(self, mode):
|
|
151
|
+
resp = self.backend.request("MKCOL", self._wire_uri())
|
|
152
|
+
if resp.status_code == 409:
|
|
153
|
+
raise FileNotFoundError(self)
|
|
154
|
+
if resp.status_code in (405, 501):
|
|
155
|
+
raise FileExistsError(self)
|
|
156
|
+
resp.raise_for_status()
|
|
157
|
+
|
|
158
|
+
def unlink(self, missing_ok=False):
|
|
159
|
+
resp = self.backend.request("DELETE", self._wire_uri())
|
|
160
|
+
if resp.status_code == 404:
|
|
161
|
+
if missing_ok:
|
|
162
|
+
return
|
|
163
|
+
raise FileNotFoundError(self)
|
|
164
|
+
resp.raise_for_status()
|
|
165
|
+
|
|
166
|
+
def rmdir(self):
|
|
167
|
+
self.unlink()
|
|
168
|
+
|
|
169
|
+
def rename(self, target: "DavPath | Uri | str"):
|
|
170
|
+
if not isinstance(target, Uri):
|
|
171
|
+
target = Uri(self.parent, target)
|
|
172
|
+
dest = self.with_path(target.path)._wire_uri()
|
|
173
|
+
resp = self.backend.request(
|
|
174
|
+
"MOVE", self._wire_uri(), headers={"Destination": dest, "Overwrite": "F"}
|
|
175
|
+
)
|
|
176
|
+
if resp.status_code == 404:
|
|
177
|
+
raise FileNotFoundError(self)
|
|
178
|
+
if resp.status_code == 412:
|
|
179
|
+
raise FileExistsError(target)
|
|
180
|
+
resp.raise_for_status()
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Unit-only S3 tests: a fake boto3-shaped client via BaseS3Backend, no
|
|
2
|
+
real AWS account. Covers prefix-emulated directories, list_objects_v2
|
|
3
|
+
paging shape, and copy_object+delete_object rename.
|
|
4
|
+
"""
|
|
5
|
+
import datetime
|
|
6
|
+
import io
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
botocore = pytest.importorskip("botocore")
|
|
11
|
+
import botocore.exceptions as _botoexc
|
|
12
|
+
|
|
13
|
+
from pathlib_next.uri.schemes.s3 import BaseS3Backend, S3Path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _client_error(code):
|
|
17
|
+
return _botoexc.ClientError({"Error": {"Code": code, "Message": code}}, "Operation")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _Paginator:
|
|
21
|
+
def __init__(self, client):
|
|
22
|
+
self._client = client
|
|
23
|
+
|
|
24
|
+
def paginate(self, **kwargs):
|
|
25
|
+
yield self._client.list_objects_v2(**kwargs)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _FakeS3Client:
|
|
29
|
+
def __init__(self):
|
|
30
|
+
self.objects = {} # key -> bytes
|
|
31
|
+
|
|
32
|
+
def head_bucket(self, Bucket):
|
|
33
|
+
return {}
|
|
34
|
+
|
|
35
|
+
def head_object(self, Bucket, Key):
|
|
36
|
+
if Key not in self.objects:
|
|
37
|
+
raise _client_error("404")
|
|
38
|
+
data = self.objects[Key]
|
|
39
|
+
return {
|
|
40
|
+
"ContentLength": len(data),
|
|
41
|
+
"LastModified": datetime.datetime(2026, 1, 1, 12, 0, 0),
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
def list_objects_v2(self, Bucket, Prefix="", Delimiter=None, MaxKeys=None):
|
|
45
|
+
contents = []
|
|
46
|
+
common = set()
|
|
47
|
+
for key in sorted(self.objects):
|
|
48
|
+
if not key.startswith(Prefix):
|
|
49
|
+
continue
|
|
50
|
+
rest = key[len(Prefix) :]
|
|
51
|
+
if Delimiter and Delimiter in rest:
|
|
52
|
+
common.add(Prefix + rest.split(Delimiter, 1)[0] + Delimiter)
|
|
53
|
+
else:
|
|
54
|
+
contents.append({"Key": key})
|
|
55
|
+
if MaxKeys:
|
|
56
|
+
contents = contents[:MaxKeys]
|
|
57
|
+
result = {"KeyCount": len(contents) + len(common)}
|
|
58
|
+
if contents:
|
|
59
|
+
result["Contents"] = contents
|
|
60
|
+
if common:
|
|
61
|
+
result["CommonPrefixes"] = [{"Prefix": p} for p in sorted(common)]
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
def get_paginator(self, name):
|
|
65
|
+
assert name == "list_objects_v2"
|
|
66
|
+
return _Paginator(self)
|
|
67
|
+
|
|
68
|
+
def get_object(self, Bucket, Key):
|
|
69
|
+
if Key not in self.objects:
|
|
70
|
+
raise _client_error("NoSuchKey")
|
|
71
|
+
return {"Body": io.BytesIO(self.objects[Key])}
|
|
72
|
+
|
|
73
|
+
def put_object(self, Bucket, Key, Body):
|
|
74
|
+
self.objects[Key] = Body if isinstance(Body, bytes) else bytes(Body)
|
|
75
|
+
|
|
76
|
+
def delete_object(self, Bucket, Key):
|
|
77
|
+
self.objects.pop(Key, None)
|
|
78
|
+
|
|
79
|
+
def copy_object(self, Bucket, Key, CopySource):
|
|
80
|
+
self.objects[Key] = self.objects[CopySource["Key"]]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class _FakeBackend(BaseS3Backend):
|
|
84
|
+
def __init__(self):
|
|
85
|
+
self._client = _FakeS3Client()
|
|
86
|
+
|
|
87
|
+
def client(self):
|
|
88
|
+
return self._client
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _s3(uri, backend=None):
|
|
92
|
+
return S3Path(uri, backend=backend or _FakeBackend())
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_scheme_dispatch():
|
|
96
|
+
assert isinstance(_s3("s3://bucket/a"), S3Path)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_bucket_and_key():
|
|
100
|
+
p = _s3("s3://bucket/docs/readme.txt")
|
|
101
|
+
assert p.bucket == "bucket"
|
|
102
|
+
assert p.key == "docs/readme.txt"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_stat_file():
|
|
106
|
+
backend = _FakeBackend()
|
|
107
|
+
backend._client.objects["docs/readme.txt"] = b"hello world"
|
|
108
|
+
st = _s3("s3://bucket/docs/readme.txt", backend).stat()
|
|
109
|
+
assert st.st_size == 11
|
|
110
|
+
assert not st.is_dir()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_stat_dir_via_prefix():
|
|
114
|
+
backend = _FakeBackend()
|
|
115
|
+
backend._client.objects["docs/readme.txt"] = b"hello"
|
|
116
|
+
assert _s3("s3://bucket/docs", backend).stat().is_dir()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_stat_root_is_dir():
|
|
120
|
+
assert _s3("s3://bucket/", _FakeBackend()).stat().is_dir()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_stat_missing_raises_file_not_found():
|
|
124
|
+
with pytest.raises(FileNotFoundError):
|
|
125
|
+
_s3("s3://bucket/missing.txt", _FakeBackend()).stat()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_listdir_root_and_nested():
|
|
129
|
+
backend = _FakeBackend()
|
|
130
|
+
backend._client.objects["docs/readme.txt"] = b"x"
|
|
131
|
+
backend._client.objects["docs/sub/file.txt"] = b"y"
|
|
132
|
+
backend._client.objects["top.txt"] = b"z"
|
|
133
|
+
root = _s3("s3://bucket/", backend)
|
|
134
|
+
assert sorted(root._listdir()) == ["docs", "top.txt"]
|
|
135
|
+
docs = _s3("s3://bucket/docs", backend)
|
|
136
|
+
assert sorted(docs._listdir()) == ["readme.txt", "sub"]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def test_read_bytes():
|
|
140
|
+
backend = _FakeBackend()
|
|
141
|
+
backend._client.objects["a.txt"] = b"hello"
|
|
142
|
+
assert _s3("s3://bucket/a.txt", backend).read_bytes() == b"hello"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def test_read_missing_raises_file_not_found():
|
|
146
|
+
with pytest.raises(FileNotFoundError):
|
|
147
|
+
_s3("s3://bucket/missing.txt", _FakeBackend()).read_bytes()
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_write_bytes_uploads_via_put_object_on_close():
|
|
151
|
+
backend = _FakeBackend()
|
|
152
|
+
_s3("s3://bucket/new.txt", backend).write_bytes(b"new content")
|
|
153
|
+
assert backend._client.objects["new.txt"] == b"new content"
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_mkdir_creates_marker_object():
|
|
157
|
+
backend = _FakeBackend()
|
|
158
|
+
_s3("s3://bucket/newdir", backend).mkdir()
|
|
159
|
+
assert "newdir/" in backend._client.objects
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_mkdir_existing_raises_file_exists():
|
|
163
|
+
backend = _FakeBackend()
|
|
164
|
+
backend._client.objects["newdir/"] = b""
|
|
165
|
+
with pytest.raises(FileExistsError):
|
|
166
|
+
_s3("s3://bucket/newdir", backend).mkdir()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def test_unlink_deletes_object():
|
|
170
|
+
backend = _FakeBackend()
|
|
171
|
+
backend._client.objects["a.txt"] = b"x"
|
|
172
|
+
_s3("s3://bucket/a.txt", backend).unlink()
|
|
173
|
+
assert "a.txt" not in backend._client.objects
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_unlink_missing_without_missing_ok_raises():
|
|
177
|
+
with pytest.raises(FileNotFoundError):
|
|
178
|
+
_s3("s3://bucket/missing.txt", _FakeBackend()).unlink()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_unlink_missing_ok():
|
|
182
|
+
_s3("s3://bucket/missing.txt", _FakeBackend()).unlink(missing_ok=True)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def test_rmdir_empty_deletes_marker():
|
|
186
|
+
backend = _FakeBackend()
|
|
187
|
+
backend._client.objects["dir/"] = b""
|
|
188
|
+
_s3("s3://bucket/dir", backend).rmdir()
|
|
189
|
+
assert "dir/" not in backend._client.objects
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def test_rmdir_non_empty_raises_oserror():
|
|
193
|
+
backend = _FakeBackend()
|
|
194
|
+
backend._client.objects["dir/"] = b""
|
|
195
|
+
backend._client.objects["dir/file.txt"] = b"x"
|
|
196
|
+
with pytest.raises(OSError):
|
|
197
|
+
_s3("s3://bucket/dir", backend).rmdir()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_rename_uses_copy_then_delete():
|
|
201
|
+
backend = _FakeBackend()
|
|
202
|
+
backend._client.objects["a.txt"] = b"content"
|
|
203
|
+
_s3("s3://bucket/a.txt", backend).rename("b.txt")
|
|
204
|
+
assert backend._client.objects.get("b.txt") == b"content"
|
|
205
|
+
assert "a.txt" not in backend._client.objects
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def test_chmod_not_implemented():
|
|
209
|
+
with pytest.raises(NotImplementedError):
|
|
210
|
+
_s3("s3://bucket/a.txt").chmod(0o644)
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Unit-only WebDAV tests: a fake `requests`-like session plugged into the
|
|
2
|
+
real `HttpBackend`, no real server. Covers PROPFIND-based stat/listdir and
|
|
3
|
+
PUT/DELETE/MKCOL/MOVE.
|
|
4
|
+
"""
|
|
5
|
+
import io
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
requests = pytest.importorskip("requests")
|
|
10
|
+
|
|
11
|
+
from pathlib_next.uri.schemes.http import HttpBackend
|
|
12
|
+
from pathlib_next.uri.schemes.webdav import DavPath
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _FakeResponse:
|
|
16
|
+
def __init__(self, status_code=200, content=b"", headers=None):
|
|
17
|
+
self.status_code = status_code
|
|
18
|
+
self.content = content
|
|
19
|
+
self.headers = headers or {}
|
|
20
|
+
self.raw = io.BytesIO(content)
|
|
21
|
+
self.is_redirect = False
|
|
22
|
+
|
|
23
|
+
def raise_for_status(self):
|
|
24
|
+
if self.status_code >= 400:
|
|
25
|
+
raise requests.HTTPError(str(self.status_code))
|
|
26
|
+
|
|
27
|
+
def close(self):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _FakeSession:
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self.calls = []
|
|
34
|
+
self.responses = {}
|
|
35
|
+
|
|
36
|
+
def request(self, method, url, **kwargs):
|
|
37
|
+
self.calls.append((method, url, kwargs))
|
|
38
|
+
if method == "PROPFIND":
|
|
39
|
+
key = (method, url, kwargs.get("headers", {}).get("Depth"))
|
|
40
|
+
else:
|
|
41
|
+
key = (method, url)
|
|
42
|
+
resp = self.responses.get(key)
|
|
43
|
+
if resp is None:
|
|
44
|
+
return _FakeResponse(404)
|
|
45
|
+
return resp() if callable(resp) else resp
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _dav(uri, session=None):
|
|
49
|
+
return DavPath(uri, backend=HttpBackend(session or _FakeSession(), {}))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
_MULTISTATUS_FILE = b"""<?xml version="1.0"?>
|
|
53
|
+
<D:multistatus xmlns:D="DAV:">
|
|
54
|
+
<D:response>
|
|
55
|
+
<D:href>/docs/readme.txt</D:href>
|
|
56
|
+
<D:propstat>
|
|
57
|
+
<D:prop>
|
|
58
|
+
<D:resourcetype/>
|
|
59
|
+
<D:getcontentlength>11</D:getcontentlength>
|
|
60
|
+
<D:getlastmodified>Mon, 01 Jan 2026 12:00:00 GMT</D:getlastmodified>
|
|
61
|
+
</D:prop>
|
|
62
|
+
<D:status>HTTP/1.1 200 OK</D:status>
|
|
63
|
+
</D:propstat>
|
|
64
|
+
</D:response>
|
|
65
|
+
</D:multistatus>"""
|
|
66
|
+
|
|
67
|
+
_MULTISTATUS_DIR = b"""<?xml version="1.0"?>
|
|
68
|
+
<D:multistatus xmlns:D="DAV:">
|
|
69
|
+
<D:response>
|
|
70
|
+
<D:href>/docs/</D:href>
|
|
71
|
+
<D:propstat>
|
|
72
|
+
<D:prop>
|
|
73
|
+
<D:resourcetype><D:collection/></D:resourcetype>
|
|
74
|
+
<D:getlastmodified>Mon, 01 Jan 2026 12:00:00 GMT</D:getlastmodified>
|
|
75
|
+
</D:prop>
|
|
76
|
+
<D:status>HTTP/1.1 200 OK</D:status>
|
|
77
|
+
</D:propstat>
|
|
78
|
+
</D:response>
|
|
79
|
+
</D:multistatus>"""
|
|
80
|
+
|
|
81
|
+
_MULTISTATUS_LISTING = b"""<?xml version="1.0"?>
|
|
82
|
+
<D:multistatus xmlns:D="DAV:">
|
|
83
|
+
<D:response>
|
|
84
|
+
<D:href>/docs/</D:href>
|
|
85
|
+
<D:propstat>
|
|
86
|
+
<D:prop>
|
|
87
|
+
<D:resourcetype><D:collection/></D:resourcetype>
|
|
88
|
+
</D:prop>
|
|
89
|
+
<D:status>HTTP/1.1 200 OK</D:status>
|
|
90
|
+
</D:propstat>
|
|
91
|
+
</D:response>
|
|
92
|
+
<D:response>
|
|
93
|
+
<D:href>/docs/readme.txt</D:href>
|
|
94
|
+
<D:propstat>
|
|
95
|
+
<D:prop>
|
|
96
|
+
<D:resourcetype/>
|
|
97
|
+
<D:getcontentlength>11</D:getcontentlength>
|
|
98
|
+
</D:prop>
|
|
99
|
+
<D:status>HTTP/1.1 200 OK</D:status>
|
|
100
|
+
</D:propstat>
|
|
101
|
+
</D:response>
|
|
102
|
+
<D:response>
|
|
103
|
+
<D:href>/docs/sub/</D:href>
|
|
104
|
+
<D:propstat>
|
|
105
|
+
<D:prop>
|
|
106
|
+
<D:resourcetype><D:collection/></D:resourcetype>
|
|
107
|
+
</D:prop>
|
|
108
|
+
<D:status>HTTP/1.1 200 OK</D:status>
|
|
109
|
+
</D:propstat>
|
|
110
|
+
</D:response>
|
|
111
|
+
</D:multistatus>"""
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_scheme_dispatch():
|
|
115
|
+
assert isinstance(_dav("dav://host/docs/readme.txt"), DavPath)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_wire_uri_translates_scheme():
|
|
119
|
+
assert _dav("dav://host/a")._wire_uri() == "http://host/a"
|
|
120
|
+
assert _dav("davs://host/a")._wire_uri() == "https://host/a"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_stat_file():
|
|
124
|
+
session = _FakeSession()
|
|
125
|
+
session.responses[("PROPFIND", "http://host/docs/readme.txt", "0")] = _FakeResponse(
|
|
126
|
+
207, _MULTISTATUS_FILE
|
|
127
|
+
)
|
|
128
|
+
st = _dav("dav://host/docs/readme.txt", session).stat()
|
|
129
|
+
assert st.st_size == 11
|
|
130
|
+
assert not st.is_dir()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_stat_dir():
|
|
134
|
+
session = _FakeSession()
|
|
135
|
+
session.responses[("PROPFIND", "http://host/docs/", "0")] = _FakeResponse(
|
|
136
|
+
207, _MULTISTATUS_DIR
|
|
137
|
+
)
|
|
138
|
+
assert _dav("dav://host/docs/", session).stat().is_dir()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_stat_missing_raises_file_not_found():
|
|
142
|
+
with pytest.raises(FileNotFoundError):
|
|
143
|
+
_dav("dav://host/missing.txt").stat()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_is_dir_is_file():
|
|
147
|
+
session = _FakeSession()
|
|
148
|
+
session.responses[("PROPFIND", "http://host/docs/", "0")] = _FakeResponse(
|
|
149
|
+
207, _MULTISTATUS_DIR
|
|
150
|
+
)
|
|
151
|
+
session.responses[("PROPFIND", "http://host/docs/readme.txt", "0")] = _FakeResponse(
|
|
152
|
+
207, _MULTISTATUS_FILE
|
|
153
|
+
)
|
|
154
|
+
root = _dav("dav://host/docs/", session)
|
|
155
|
+
assert root.is_dir()
|
|
156
|
+
assert not root.is_file()
|
|
157
|
+
f = _dav("dav://host/docs/readme.txt", session)
|
|
158
|
+
assert f.is_file()
|
|
159
|
+
assert not f.is_dir()
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_listdir():
|
|
163
|
+
session = _FakeSession()
|
|
164
|
+
session.responses[("PROPFIND", "http://host/docs/", "1")] = _FakeResponse(
|
|
165
|
+
207, _MULTISTATUS_LISTING
|
|
166
|
+
)
|
|
167
|
+
p = _dav("dav://host/docs/", session)
|
|
168
|
+
assert sorted(p._listdir()) == ["readme.txt", "sub"]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_iterdir_yields_children():
|
|
172
|
+
session = _FakeSession()
|
|
173
|
+
session.responses[("PROPFIND", "http://host/docs/", "1")] = _FakeResponse(
|
|
174
|
+
207, _MULTISTATUS_LISTING
|
|
175
|
+
)
|
|
176
|
+
p = _dav("dav://host/docs/", session)
|
|
177
|
+
assert sorted(c.name for c in p.iterdir()) == ["readme.txt", "sub"]
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_open_read_downloads_via_get():
|
|
181
|
+
session = _FakeSession()
|
|
182
|
+
session.responses[("GET", "http://host/docs/readme.txt")] = _FakeResponse(
|
|
183
|
+
200, b"hello world"
|
|
184
|
+
)
|
|
185
|
+
p = _dav("dav://host/docs/readme.txt", session)
|
|
186
|
+
assert p.read_bytes() == b"hello world"
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def test_open_write_uploads_via_put_on_close():
|
|
190
|
+
session = _FakeSession()
|
|
191
|
+
session.responses[("PUT", "http://host/docs/new.txt")] = _FakeResponse(201)
|
|
192
|
+
p = _dav("dav://host/docs/new.txt", session)
|
|
193
|
+
p.write_bytes(b"new content")
|
|
194
|
+
method, url, kwargs = session.calls[-1]
|
|
195
|
+
assert method == "PUT"
|
|
196
|
+
assert kwargs["data"] == b"new content"
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def test_mkdir_sends_mkcol():
|
|
200
|
+
session = _FakeSession()
|
|
201
|
+
session.responses[("MKCOL", "http://host/docs/newdir")] = _FakeResponse(201)
|
|
202
|
+
p = _dav("dav://host/docs/newdir", session)
|
|
203
|
+
p._mkdir(0o777)
|
|
204
|
+
assert session.calls[-1][:2] == ("MKCOL", "http://host/docs/newdir")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_mkdir_existing_raises_file_exists():
|
|
208
|
+
session = _FakeSession()
|
|
209
|
+
session.responses[("MKCOL", "http://host/docs/newdir")] = _FakeResponse(405)
|
|
210
|
+
p = _dav("dav://host/docs/newdir", session)
|
|
211
|
+
with pytest.raises(FileExistsError):
|
|
212
|
+
p._mkdir(0o777)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def test_unlink_sends_delete():
|
|
216
|
+
session = _FakeSession()
|
|
217
|
+
session.responses[("DELETE", "http://host/docs/readme.txt")] = _FakeResponse(204)
|
|
218
|
+
p = _dav("dav://host/docs/readme.txt", session)
|
|
219
|
+
p.unlink()
|
|
220
|
+
assert session.calls[-1][:2] == ("DELETE", "http://host/docs/readme.txt")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def test_unlink_missing_ok():
|
|
224
|
+
session = _FakeSession()
|
|
225
|
+
session.responses[("DELETE", "http://host/missing.txt")] = _FakeResponse(404)
|
|
226
|
+
_dav("dav://host/missing.txt", session).unlink(missing_ok=True)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def test_unlink_missing_without_missing_ok_raises():
|
|
230
|
+
session = _FakeSession()
|
|
231
|
+
session.responses[("DELETE", "http://host/missing.txt")] = _FakeResponse(404)
|
|
232
|
+
p = _dav("dav://host/missing.txt", session)
|
|
233
|
+
with pytest.raises(FileNotFoundError):
|
|
234
|
+
p.unlink()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def test_rename_sends_move_with_destination_header():
|
|
238
|
+
session = _FakeSession()
|
|
239
|
+
session.responses[("MOVE", "http://host/docs/a.txt")] = _FakeResponse(201)
|
|
240
|
+
p = _dav("dav://host/docs/a.txt", session)
|
|
241
|
+
p.rename("b.txt")
|
|
242
|
+
method, url, kwargs = session.calls[-1]
|
|
243
|
+
assert method == "MOVE"
|
|
244
|
+
assert kwargs["headers"]["Destination"] == "http://host/docs/b.txt"
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def test_chmod_not_implemented():
|
|
248
|
+
p = _dav("dav://host/docs/a.txt")
|
|
249
|
+
with pytest.raises(NotImplementedError):
|
|
250
|
+
p.chmod(0o644)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|