objpath 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- objpath/AGENTS.md +311 -0
- objpath/README.md +153 -0
- objpath/__init__.py +68 -0
- objpath/_az.py +547 -0
- objpath/_az_backend.py +130 -0
- objpath/_az_errors.py +110 -0
- objpath/_clients.py +108 -0
- objpath/_digests.py +10 -0
- objpath/_failures.py +84 -0
- objpath/_gs.py +447 -0
- objpath/_gs_backend.py +95 -0
- objpath/_gs_errors.py +112 -0
- objpath/_keys.py +119 -0
- objpath/_optional.py +60 -0
- objpath/_places.py +26 -0
- objpath/_reader.py +83 -0
- objpath/_s3.py +489 -0
- objpath/_s3_backend.py +68 -0
- objpath/_s3_errors.py +106 -0
- objpath/_storepath.py +579 -0
- objpath/_uri_store.py +162 -0
- objpath/_version.py +11 -0
- objpath/_writer.py +113 -0
- objpath/exceptions.py +31 -0
- objpath/py.typed +0 -0
- objpath/uri.py +213 -0
- objpath-0.1.0.dist-info/METADATA +204 -0
- objpath-0.1.0.dist-info/RECORD +31 -0
- objpath-0.1.0.dist-info/WHEEL +4 -0
- objpath-0.1.0.dist-info/entry_points.txt +4 -0
- objpath-0.1.0.dist-info/licenses/LICENSE +21 -0
objpath/AGENTS.md
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
# `objpath` — public API header
|
|
2
|
+
|
|
3
|
+
Header-file-style reference for the `objpath` package: every public export with
|
|
4
|
+
its signature, arguments, contract and gotchas, so the package can be used
|
|
5
|
+
without reading its source. It ships inside the package and is
|
|
6
|
+
self-contained. Development documentation lives with the source at
|
|
7
|
+
<https://github.com/jose-pr/objpath>.
|
|
8
|
+
|
|
9
|
+
Names are imported from `objpath` (the paths, the backends, the exceptions and
|
|
10
|
+
`__version__`), from `objpath.exceptions` (the exceptions) and from
|
|
11
|
+
`objpath.uri` (the `s3:`, `gs:` and `az:` schemes of `uripath`). Every module
|
|
12
|
+
whose name starts with `_` is private. `import objpath` imports no store SDK and
|
|
13
|
+
never `uripath`: `S3Path`, `GSPath`, `AZPath` and the backends are loaded the
|
|
14
|
+
first time they are named, and a store's SDK is imported when its client is
|
|
15
|
+
built.
|
|
16
|
+
|
|
17
|
+
Install with `pip install objpath`, which needs only `pathapi`. One extra
|
|
18
|
+
per store brings its SDK, and a store used without its extra raises an
|
|
19
|
+
`ImportError` that names it:
|
|
20
|
+
|
|
21
|
+
| Extra | Adds | For |
|
|
22
|
+
| --- | --- | --- |
|
|
23
|
+
| `s3` | `boto3` | `S3Path`, `S3Backend` (and S3-compatible stores) |
|
|
24
|
+
| `gs` | `google-cloud-storage` | `GSPath`, `GSBackend` |
|
|
25
|
+
| `az` | `azure-storage-blob`, `azure-identity` | `AZPath`, `AZBackend` |
|
|
26
|
+
| `uri` | `uripath` | `objpath.uri`: the `s3:`, `gs:` and `az:` schemes |
|
|
27
|
+
| `all` | all of the above | |
|
|
28
|
+
|
|
29
|
+
`objpath.__version__` is the installed version, read from the distribution's
|
|
30
|
+
metadata.
|
|
31
|
+
|
|
32
|
+
## Paths (`objpath`)
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
class S3Path(Path):
|
|
36
|
+
def __init__(
|
|
37
|
+
self, *segments, bucket=None, backend=None, **client_kwargs,
|
|
38
|
+
): ...
|
|
39
|
+
class GSPath(Path):
|
|
40
|
+
def __init__(
|
|
41
|
+
self, *segments, bucket=None, backend=None, timeout=..., retry=...,
|
|
42
|
+
**client_kwargs,
|
|
43
|
+
): ...
|
|
44
|
+
class AZPath(Path):
|
|
45
|
+
def __init__(
|
|
46
|
+
self, *segments, container=None, account=None, backend=None,
|
|
47
|
+
**client_kwargs,
|
|
48
|
+
): ...
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`S3Path`, `GSPath` and `AZPath` are `pathapi.Path` classes (so
|
|
52
|
+
`pathlib.Path`'s methods, `copy()`, `move()`, `rm()`, `walk()`, `glob()` and
|
|
53
|
+
`PathSyncer` work on them) over an Amazon S3 bucket, a Google Cloud Storage
|
|
54
|
+
bucket and an Azure Blob Storage container. They need no URI. `segments` are the
|
|
55
|
+
object key, joined and parsed like a POSIX path; the bucket or container is a
|
|
56
|
+
keyword.
|
|
57
|
+
|
|
58
|
+
- **Place.** `bucket=` (`container=` for `AZPath`) names the bucket. It is
|
|
59
|
+
required, unless a segment is a path of the same class and supplies it
|
|
60
|
+
(`path / "x"`, `path.parent`, `S3Path(path, "x")` keep it); `bucket=` then
|
|
61
|
+
moves the path to another bucket of the same backend. A path of another bucket joined as
|
|
62
|
+
a later segment contributes its names only. Without a bucket: `TypeError`.
|
|
63
|
+
A name that is not a `str` is a `TypeError`; an empty one or one with a `/`
|
|
64
|
+
is a `ValueError`.
|
|
65
|
+
- **Backend.** Client options (`region_name=`, `endpoint_url=`,
|
|
66
|
+
`aws_access_key_id=`, `config=`, ... for S3; `project=`, `credentials=`,
|
|
67
|
+
`client_options=`, `timeout=`, `retry=`, ... for GCS; `credential=`,
|
|
68
|
+
`account_url=`, `connection_string=`, ... for Azure) build the path's own
|
|
69
|
+
backend (`S3Backend(**options)`), which every path derived from it keeps.
|
|
70
|
+
`backend=` shares one client between paths built separately, or plugs in a
|
|
71
|
+
backend of your own. Giving `backend=` together with a client option (for
|
|
72
|
+
`AZPath`, also `account=`) is a `TypeError`. Without either, a path derived from
|
|
73
|
+
another keeps its backend and a new path uses the default backend: one per process
|
|
74
|
+
for `S3Path` and `GSPath`, one per account for `AZPath(account=...)`, shared by
|
|
75
|
+
every path built that way, so they share one client and one connection pool, and
|
|
76
|
+
the credentials, region and endpoint the environment gave the first request are
|
|
77
|
+
theirs. `AZPath` with none of `account=`, `backend=`, a client option or a
|
|
78
|
+
`AZPath` segment is a `TypeError`.
|
|
79
|
+
- **Nothing connects when a path is built.** A backend builds its SDK client on
|
|
80
|
+
the first request.
|
|
81
|
+
- **Names.** `x` and `/x` are the same key (they compare equal and hash alike;
|
|
82
|
+
`is_absolute()` tells them apart). Exactly one trailing `/` goes (`dir/` is
|
|
83
|
+
`dir`; `dir//` is the key `dir/`), `.` segments are dropped, `..` is a name
|
|
84
|
+
like any other, and an interior empty segment is part of the key (`a//b` is
|
|
85
|
+
the key `a//b`, and is never listed). Leading `/`s are not part of a key.
|
|
86
|
+
- **Identity.** Two paths are equal when they are of one class, name one bucket
|
|
87
|
+
(`AZPath`: one account and container) and spell one key; the backend is not part of
|
|
88
|
+
it. They are `_same_filesystem()` when they also share a backend object.
|
|
89
|
+
- **Text.** `str(path)` is the URL: `s3://bucket/key`, `gs://bucket/key`,
|
|
90
|
+
`az://account/container/key` (`az:///container/key` when the backend names no
|
|
91
|
+
account). `as_uri()` is the same, percent-encoded. `repr()` is
|
|
92
|
+
`S3Path('dir/key', bucket='bkt')`, naming `backend=` when it is not the default;
|
|
93
|
+
it never shows a client option.
|
|
94
|
+
- **Attributes.** `path.bucket` (`S3Path`, `GSPath`), `path.container` and
|
|
95
|
+
`path.account` (`AZPath`; `account` is the backend's, `None` when it names none),
|
|
96
|
+
`path.key` (the object key; `""` for the root), `path.backend`.
|
|
97
|
+
- **Pickling and copying.** A path pickles its place and its names, and for
|
|
98
|
+
`AZPath` its account, and not its backend: the options of a client carry
|
|
99
|
+
credentials, so the receiving process builds the default backend. A backend
|
|
100
|
+
whose class sets `picklable = True` travels with the path. `copy.copy()` and
|
|
101
|
+
`copy.deepcopy()` keep the backend.
|
|
102
|
+
|
|
103
|
+
Directories do not exist in an object store, and the classes emulate them:
|
|
104
|
+
|
|
105
|
+
- `is_dir()` is prefix emulation (any key under `"<path>/"`), and `mkdir()` writes a
|
|
106
|
+
zero-byte `key/` marker object; `rmdir()` needs an empty prefix; a key that is
|
|
107
|
+
both an object and a prefix is the object, in `stat()` and in listings. There is
|
|
108
|
+
no hierarchy: writes below a missing "directory" succeed, but a write (`"w"`) or
|
|
109
|
+
`rename()` onto a prefix directory raises `IsADirectoryError` (one list request;
|
|
110
|
+
credentials that may write but not list are not stopped). `mkdir()` treats only
|
|
111
|
+
`FileNotFoundError` from `stat()` as absent.
|
|
112
|
+
- A listing is one level, with a delimiter. `_scandir()`/`iterdir()` yield each
|
|
113
|
+
name with the `FileStat` the listing gave (size and `st_mtime`; no mode), so
|
|
114
|
+
`is_dir()` and `stat()` of a listed child ask nothing more.
|
|
115
|
+
- A prefix that holds keys a directory walk does not reach (a key with an empty, `.`
|
|
116
|
+
or `..` segment; the subtree under a key that is also an object; a `name/` key
|
|
117
|
+
that holds data) is refused by `copy(recursive=True)`, `move()` and
|
|
118
|
+
`rm(recursive=True)`: `UnreachableKeysError` naming the keys, nothing created or
|
|
119
|
+
removed (`rm(..., ignore_error=...)` is offered the error and then removes only the
|
|
120
|
+
keys a walk reaches). Remove the named keys, or `unlink()` the object that hides a
|
|
121
|
+
subtree, first. `iterdir()` and `walk()` skip such keys.
|
|
122
|
+
|
|
123
|
+
Reading and writing:
|
|
124
|
+
|
|
125
|
+
- `open("rb")` streams the object. A read is one version of the object: the S3
|
|
126
|
+
reader makes one request per open, and the GCS and Azure readers fetch ranged
|
|
127
|
+
requests (4 MiB for GCS, `AZBackend.read_chunk` = 512 KiB for Azure) that each
|
|
128
|
+
require the generation or ETag the first one reported, so the stream stays
|
|
129
|
+
seekable and holds one chunk. An object replaced while it is being read raises
|
|
130
|
+
`ObjectChangedError`, never a mix of two versions; one deleted meanwhile is
|
|
131
|
+
`FileNotFoundError`. S3's stream is not seekable (`seek()` and `tell()` raise
|
|
132
|
+
`io.UnsupportedOperation`).
|
|
133
|
+
- `open("wb")`, `"xb"` and `"r+b"` spool the writes (in memory up to 4 MiB, 8 MiB for
|
|
134
|
+
S3, then a temporary file) and upload on `close()`: nothing is uploaded when the
|
|
135
|
+
`with` block over the file raised, and nothing is created. `"x"` is a conditional
|
|
136
|
+
create (`IfNoneMatch`, `if_generation_match=0`, `If-None-Match: *`; on S3 an upload
|
|
137
|
+
above 5 GiB falls back to a check-then-put, and a store without conditional writes
|
|
138
|
+
to the same check). Uploads above the spool size are multipart or chunked
|
|
139
|
+
(S3 managed transfer; GCS resumable upload in 2 MiB requests; Azure blocks). `"a"`
|
|
140
|
+
is unsupported (`NotImplementedError`). `st_mtime` is `LastModified`, `updated`
|
|
141
|
+
or `last_modified`.
|
|
142
|
+
- `rename(target)`: a `str` target is a sibling. Server-side copy + delete within one
|
|
143
|
+
bucket (container) reached by one backend; a target in another bucket, another
|
|
144
|
+
backend or another class raises `NotImplementedError` (so `move()` copies), and so
|
|
145
|
+
does a prefix directory. A missing path onto its own name is `FileNotFoundError`.
|
|
146
|
+
Azure waits for the service copy at most `AZBackend.copy_timeout` (300 s),
|
|
147
|
+
then aborts it and raises `TimeoutError` with the source untouched.
|
|
148
|
+
- `copy(target)` of a file onto a key of the same store reached by the same backend object --
|
|
149
|
+
within a bucket or between buckets (Azure: within one container) -- is one request of the
|
|
150
|
+
store (`CopyObject`, `copy_blob`, the service copy) and no byte passes through this process. It makes the checks of `Path.copy()` first and in
|
|
151
|
+
its order (the same file, a target that is a directory or holds keys, an existing
|
|
152
|
+
target without `overwrite=True`); an existing target is replaced by the copy itself,
|
|
153
|
+
never deleted first. `progress` is called once with the total. A store that answers
|
|
154
|
+
"not implemented" gets the streamed copy, and a copy between different backends or
|
|
155
|
+
classes always streams. `copy(recursive=True)` and `move()` of a prefix copy each file
|
|
156
|
+
this way; a file named as the root of a `recursive=True` copy is read and written.
|
|
157
|
+
- `rm(recursive=True)` batch-deletes (1000 keys per request on S3, 100 on GCS, 256 on
|
|
158
|
+
Azure, each key on its own where the batch is refused); at the bucket root it is
|
|
159
|
+
`PermissionError`. A key a batch refuses is offered to `ignore_error(error, path)`
|
|
160
|
+
with that key's path, one error per key. `rm(recursive=True)` of a missing bucket is
|
|
161
|
+
a missing path (`missing_ok`, `ignore_error` decide). There are no symlinks or
|
|
162
|
+
bindings: `follow_symlinks=` and `follow_binds=` are checked and decide nothing.
|
|
163
|
+
- `checksum(algorithm="md5")` (`NativeChecksum`) is the digest the store keeps, read with
|
|
164
|
+
one request and no byte of the content. S3: the plain MD5 that the ETag is for an object
|
|
165
|
+
one request wrote and KMS or a customer key does not encrypt, or the stored full-object
|
|
166
|
+
`crc32`, `crc32c`, `crc64nvme`, `sha1` or `sha256`. GCS: `md5` or `crc32c`. Azure: the
|
|
167
|
+
`Content-MD5` property, which a client can set. A multipart ETag, a composite checksum,
|
|
168
|
+
an algorithm the object has none of, a content-encoded GCS object: `NotImplementedError`,
|
|
169
|
+
never a guess. `FileNotFoundError` for a missing path, `IsADirectoryError` for a prefix
|
|
170
|
+
directory. `supported_checksums()` is `{"md5"}` (`{"md5", "crc32c"}` for GCS), asks
|
|
171
|
+
nothing and is advisory. `PathSyncer` compares two object-store paths by these digests.
|
|
172
|
+
- There is no `chmod()`, `symlink_to()` or `readlink()`.
|
|
173
|
+
- `is_local()` is `False`: a bucket is not a host name, nothing is resolved, and
|
|
174
|
+
`PathSyncer(quick_check=True)` treats a store as non-local.
|
|
175
|
+
- Timeouts and retries are the SDK's own: pass `config=botocore.config.Config(...)` (S3),
|
|
176
|
+
`timeout=` / `retry=` (GCS, on `GSBackend`) or `retry_*`, `connection_timeout=`,
|
|
177
|
+
`read_timeout=` (Azure) to change them. With the SDK defaults a connection and a
|
|
178
|
+
read may each wait a minute or more.
|
|
179
|
+
|
|
180
|
+
## Backends (`objpath`)
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
class BaseS3Backend:
|
|
184
|
+
def client(self): ...
|
|
185
|
+
def close(self) -> None: ...
|
|
186
|
+
picklable: bool
|
|
187
|
+
class S3Backend(BaseS3Backend):
|
|
188
|
+
client_kwargs: dict
|
|
189
|
+
def __init__(self, **client_kwargs): ...
|
|
190
|
+
|
|
191
|
+
class BaseGSBackend:
|
|
192
|
+
def client(self): ...
|
|
193
|
+
def call_options(self) -> dict: ...
|
|
194
|
+
def close(self) -> None: ...
|
|
195
|
+
class GSBackend(BaseGSBackend):
|
|
196
|
+
client_kwargs: dict
|
|
197
|
+
def __init__(self, *, timeout=..., retry=..., **client_kwargs): ...
|
|
198
|
+
|
|
199
|
+
class BaseAZBackend:
|
|
200
|
+
account: str | None
|
|
201
|
+
read_chunk: int
|
|
202
|
+
copy_timeout: float
|
|
203
|
+
def client(self): ...
|
|
204
|
+
def close(self) -> None: ...
|
|
205
|
+
class AZBackend(BaseAZBackend):
|
|
206
|
+
client_kwargs: dict
|
|
207
|
+
def __init__(self, account=None, **client_kwargs): ...
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
A backend gives the paths that hold it their SDK client. Subclass the `Base...Backend`
|
|
211
|
+
of a store and define `client()` to plug in your own credential or session handling (or a
|
|
212
|
+
fake in a test); the real classes build the client on first use, once however many threads
|
|
213
|
+
ask first, and keep it. A client is shared between threads (the SDKs document that).
|
|
214
|
+
|
|
215
|
+
- `S3Backend(**client_kwargs)` is `boto3.client("s3", **client_kwargs)`; boto3's own
|
|
216
|
+
credential and endpoint configuration by default. `BaseS3Backend.client()` returns
|
|
217
|
+
the boto3 (or botocore) S3 client.
|
|
218
|
+
- `GSBackend(*, timeout=, retry=, **client_kwargs)` is `google.cloud.storage.Client(
|
|
219
|
+
**client_kwargs)` (an emulator: `client_options={"api_endpoint": url},
|
|
220
|
+
use_auth_w_custom_endpoint=False`, or set `STORAGE_EMULATOR_HOST`). `timeout` (seconds
|
|
221
|
+
or `(connect, read)`) and `retry` (`google.api_core.retry.Retry`, or `None` for none)
|
|
222
|
+
are passed to every SDK call a path makes (`call_options()` is the hook); left out, the
|
|
223
|
+
SDK's defaults apply, under which an unreachable endpoint fails after about two minutes.
|
|
224
|
+
- `AZBackend(account=None, **client_kwargs)`: `connection_string=` goes to
|
|
225
|
+
`BlobServiceClient.from_connection_string(...)`; otherwise the options go to
|
|
226
|
+
`BlobServiceClient`, and `account` alone derives
|
|
227
|
+
`account_url="https://<account>.blob.core.windows.net"` authenticated by
|
|
228
|
+
azure-identity's `DefaultAzureCredential` unless `credential=` is passed (`ImportError`
|
|
229
|
+
naming `objpath[az]` if azure-identity is missing). `backend.account` is `account`, else the
|
|
230
|
+
account `connection_string=` (`AccountName`) or `account_url=` names, else `None`.
|
|
231
|
+
- `read_chunk` (bytes of a blob one ranged read request asks for, default 524288; the SDK holds
|
|
232
|
+
about nine times a response while it handles it, so a larger number trades memory for requests) and
|
|
233
|
+
`copy_timeout` (seconds, default 300.0) are class attributes of `BaseAZBackend`: assign them on
|
|
234
|
+
`AZBackend` or on your own subclass.
|
|
235
|
+
- Every backend is a context manager. `close()` closes the SDK client the backend built
|
|
236
|
+
(when the SDK's client has a `close()`) and drops it: the next `client()` builds
|
|
237
|
+
another. It is complete when it returns and harmless when called again.
|
|
238
|
+
- A backend pickles and copies (`copy.copy`, `deepcopy`) without the client it built and
|
|
239
|
+
keeps its options, credentials included: the copy builds its own client on first use.
|
|
240
|
+
A path never pickles its backend unless the backend's `picklable` is true (it is
|
|
241
|
+
`False`).
|
|
242
|
+
|
|
243
|
+
## The `s3:`, `gs:` and `az:` schemes (`objpath.uri`, `uri` extra)
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
class S3URIPath(DelegatingURIPath): bucket: str; key: str
|
|
247
|
+
class GSURIPath(DelegatingURIPath): bucket: str; key: str
|
|
248
|
+
class AZURIPath(DelegatingURIPath): account: str; container: str; key: str
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Importing `objpath.uri` needs `uripath` (`ImportError` naming `pip install
|
|
252
|
+
"objpath[uri]"` without it) and imports no SDK. `uripath` loads the module by itself,
|
|
253
|
+
through the entry points of the group `uripath.schemes`, the first time a path of one of the
|
|
254
|
+
schemes is used: `URIPath("s3://bucket/key")` is an `S3URIPath`. Each class is a
|
|
255
|
+
`uripath` URI path (see its header for everything about URIs) whose requests are done by
|
|
256
|
+
the matching path of this package, on `backend`: a `BaseS3Backend`, `BaseGSBackend` or
|
|
257
|
+
`BaseAZBackend` used as it is by the path it is given to and by the paths derived from it
|
|
258
|
+
on the same bucket; without one, the
|
|
259
|
+
default backend of the process (`s3:`, `gs:`) or of the account in the authority (`az:`).
|
|
260
|
+
|
|
261
|
+
- `s3://bucket/key`, `gs://bucket/key`: `bucket` is the host. `az://account/container/key`:
|
|
262
|
+
`account` is the host and `container` the first segment; `az://account` is a directory
|
|
263
|
+
whose children are the containers, and reading or writing it is `IsADirectoryError`.
|
|
264
|
+
- The key is the path without its leading `/`s and without one trailing `/`; an interior
|
|
265
|
+
empty segment is part of it (`s3://b/a//b` is the key `a//b`). A key that begins with
|
|
266
|
+
`/` cannot be addressed.
|
|
267
|
+
- A path with no bucket (`s3:///key`) or no account (`az:x`) does not exist:
|
|
268
|
+
`FileNotFoundError`, nothing is sent and no credential is built.
|
|
269
|
+
- `rename()` and server-side `copy()` between two URIs need the same authority (and, for
|
|
270
|
+
`az:`, container) and the same supplied backend (or each its default); otherwise
|
|
271
|
+
`rename()` raises `NotImplementedError` and `move()` copies. Callbacks of `rm()` and
|
|
272
|
+
`copy()` (`ignore_error`, `on_error`, `progress`) are given URI paths.
|
|
273
|
+
- A URI path pickles as its URI text (no password: the URI holds none) and not its
|
|
274
|
+
backend, as a path does.
|
|
275
|
+
|
|
276
|
+
## Exceptions (`objpath.exceptions`)
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
class ObjPathError(pathapi.PathError): ...
|
|
280
|
+
class ObjectChangedError(ObjPathError, OSError): ...
|
|
281
|
+
class UnreachableKeysError(ObjPathError, OSError): ...
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
A failure of a store is the `OSError` type `pathlib` raises for the same condition, with
|
|
285
|
+
the path as `filename`: `FileNotFoundError` (a key or a bucket), `PermissionError`,
|
|
286
|
+
`OSError(EIO)`; a request that runs out of time is `TimeoutError`, an endpoint that cannot
|
|
287
|
+
be reached `ConnectionError`, a connection that breaks while the answer or a body arrives
|
|
288
|
+
`ConnectionResetError`. None is chained to the SDK's exception, and the message carries the
|
|
289
|
+
SDK's exception type but not its text (it holds the request URL and so any query-string
|
|
290
|
+
credential). They reach `exists()` and `walk(on_error=)` as `OSError`. An exception that is
|
|
291
|
+
not a reply or a connection failure (a missing SDK, bad credentials, a bug) propagates as
|
|
292
|
+
itself.
|
|
293
|
+
|
|
294
|
+
- `ObjectChangedError` (`errno.ESTALE`, `EIO` where the platform has none): the object a
|
|
295
|
+
stream was reading was replaced.
|
|
296
|
+
- `UnreachableKeysError` (`errno.EINVAL`): a copy or removal of a prefix was refused because
|
|
297
|
+
it holds keys a listing does not show; nothing changed.
|
|
298
|
+
- A bad argument is a plain `ValueError` or `TypeError`.
|
|
299
|
+
|
|
300
|
+
## Gotchas
|
|
301
|
+
|
|
302
|
+
- A key that begins with `/` is not addressable (`S3Path("//lead.txt")` is the key `lead.txt`)
|
|
303
|
+
and a key with an empty segment (`a//b`) is addressable but never listed.
|
|
304
|
+
- `import objpath` and `objpath.uri` load no SDK; naming `objpath.S3Path` imports
|
|
305
|
+
`botocore.exceptions` and `boto3.exceptions` when they are installed, to know which errors to
|
|
306
|
+
translate.
|
|
307
|
+
- A test double that stands for a client subclasses the `Base...Backend` of the store and
|
|
308
|
+
defines `client()`; a `Mock` client needs the methods a path calls (`head_object`,
|
|
309
|
+
`list_objects_v2`, ...), whose names are the SDK's.
|
|
310
|
+
- Listing a very large prefix is one request per page; `rm(recursive=True)` and a recursive copy
|
|
311
|
+
list the whole prefix first.
|
objpath/README.md
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# objpath
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/objpath/)
|
|
4
|
+
[](https://pypi.org/project/objpath/)
|
|
5
|
+
[](https://github.com/jose-pr/objpath/blob/main/LICENSE)
|
|
6
|
+
[](https://jose-pr.github.io/objpath/)
|
|
7
|
+
[](https://github.com/jose-pr/objpath/actions/workflows/test.yml)
|
|
8
|
+
|
|
9
|
+
**`pathlib` for S3, Google Cloud Storage and Azure Blob buckets.**
|
|
10
|
+
`S3Path("reports/2026.csv", bucket="data")` has the method names, signatures and
|
|
11
|
+
exception types of `pathlib.Path`, and `GSPath` and `AZPath` do the same for the
|
|
12
|
+
other two stores; none needs a URI. They build on
|
|
13
|
+
[`pathapi`](https://github.com/jose-pr/pathapi) alone; with
|
|
14
|
+
[`uripath`](https://github.com/jose-pr/uripath) installed the same package also
|
|
15
|
+
provides the `s3:`, `gs:` and `az:` schemes. It is documented at
|
|
16
|
+
<https://jose-pr.github.io/objpath/>.
|
|
17
|
+
|
|
18
|
+
## Features
|
|
19
|
+
|
|
20
|
+
| Store | Class | Extra | Directories | Server-side copy | Checksum | Append |
|
|
21
|
+
| --- | --- | --- | --- | --- | --- | --- |
|
|
22
|
+
| Amazon S3 and S3-compatible | `S3Path` | `s3` | key prefixes, `mkdir()` writes a marker object | `CopyObject`, same client | ETag MD5, `crc32`, `crc32c`, `crc64nvme`, `sha1`, `sha256` | no |
|
|
23
|
+
| Google Cloud Storage | `GSPath` | `gs` | key prefixes, `mkdir()` writes a marker object | `copy_blob`, same client | `md5`, `crc32c` | no |
|
|
24
|
+
| Azure Blob Storage | `AZPath` | `az` | key prefixes, `mkdir()` writes a marker blob | service copy, same container and client | `Content-MD5` | no |
|
|
25
|
+
|
|
26
|
+
- **A `pathlib` path on a bucket**: `open`, `read_text`, `iterdir`, `glob`, `stat`,
|
|
27
|
+
`mkdir`, `rename` and the rest, plus `walk`, `copy`, `move`, `rm` and `PathSyncer` from
|
|
28
|
+
`pathapi`. Nothing connects until the first request.
|
|
29
|
+
- **Directories without directories**: `is_dir()` is a prefix test, listings use a
|
|
30
|
+
delimiter and carry the size and time of each object, and the keys a walk cannot reach
|
|
31
|
+
(an empty segment, a name that is both an object and a prefix) are named and refused
|
|
32
|
+
instead of copied or removed half way.
|
|
33
|
+
- **Batch deletes and server-side copies**: `rm(recursive=True)` deletes a thousand keys a
|
|
34
|
+
request, and a copy within a store moves no byte through your process.
|
|
35
|
+
- **Safe writes**: uploads are spooled and sent on `close()`; a `with` block that raised
|
|
36
|
+
uploads nothing. A read is one version of the object, and an object replaced meanwhile
|
|
37
|
+
is an error, not a mix.
|
|
38
|
+
- **The SDKs stay out of the way**: `import objpath` loads no SDK and not `uripath`; a
|
|
39
|
+
store used without its extra raises an `ImportError` naming it. No SDK exception crosses
|
|
40
|
+
the API: errors are the `OSError` types `pathlib` raises, `TimeoutError` and
|
|
41
|
+
`ConnectionError`.
|
|
42
|
+
- **The `s3:`, `gs:` and `az:` schemes** for `uripath`: `URIPath("s3://data/reports")` is
|
|
43
|
+
an `S3URIPath` that delegates to an `S3Path`.
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install "objpath[s3]"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Requires `pathapi`, which pip installs with it, and Python 3.9 or newer.
|
|
52
|
+
|
|
53
|
+
| Install | Adds |
|
|
54
|
+
| --- | --- |
|
|
55
|
+
| `pip install objpath` | the paths and the backend contracts; no SDK |
|
|
56
|
+
| `pip install "objpath[s3]"` | `boto3>=1.36.0,<2` for `S3Path` |
|
|
57
|
+
| `pip install "objpath[gs]"` | `google-cloud-storage>=3.0,<4` for `GSPath` |
|
|
58
|
+
| `pip install "objpath[az]"` | `azure-storage-blob` and `azure-identity` for `AZPath` |
|
|
59
|
+
| `pip install "objpath[uri]"` | `uripath` and with it the `s3:`, `gs:` and `az:` schemes |
|
|
60
|
+
| `pip install "objpath[all]"` | all of the above |
|
|
61
|
+
|
|
62
|
+
## Quick start
|
|
63
|
+
|
|
64
|
+
**A path in a bucket**: the bucket is a keyword, the segments are the key, and the
|
|
65
|
+
credentials are the ones boto3 finds:
|
|
66
|
+
|
|
67
|
+
<!-- example: skip: connects to S3 -->
|
|
68
|
+
```python
|
|
69
|
+
from objpath import S3Path
|
|
70
|
+
|
|
71
|
+
report = S3Path("reports/2026.csv", bucket="data")
|
|
72
|
+
print(report.read_text())
|
|
73
|
+
for child in report.parent.iterdir():
|
|
74
|
+
print(child.name, child.stat().st_size)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
**Another store, or another endpoint**: client options build the path's own backend, and a
|
|
78
|
+
backend shared between paths is one client and one connection pool:
|
|
79
|
+
|
|
80
|
+
<!-- example: skip: connects to a store -->
|
|
81
|
+
```python
|
|
82
|
+
from objpath import AZPath, GSPath, S3Backend, S3Path
|
|
83
|
+
|
|
84
|
+
blob = AZPath("logs/app.log", container="ops", account="mycompany")
|
|
85
|
+
gcs = GSPath("models/latest", bucket="ml", project="my-project")
|
|
86
|
+
|
|
87
|
+
with S3Backend(endpoint_url="https://minio.example.org", region_name="us-east-1") as backend:
|
|
88
|
+
root = S3Path(bucket="backups", backend=backend)
|
|
89
|
+
(root / "2026").mkdir(exist_ok=True)
|
|
90
|
+
(root / "2026" / "n.txt").write_text("1")
|
|
91
|
+
root.copy(S3Path("mirror", bucket="backups", backend=backend), recursive=True)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**Mirror a tree** with `PathSyncer`, onto a local directory:
|
|
95
|
+
|
|
96
|
+
<!-- example: skip: connects to S3 -->
|
|
97
|
+
```python
|
|
98
|
+
from pathapi import LocalPath
|
|
99
|
+
from pathapi.sync import PathSyncer
|
|
100
|
+
from objpath import S3Path
|
|
101
|
+
|
|
102
|
+
PathSyncer(remove_missing=True).sync(S3Path("logs", bucket="data"), LocalPath("mirror"))
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**The schemes**, with `uripath` installed:
|
|
106
|
+
|
|
107
|
+
<!-- example: skip: connects to S3 -->
|
|
108
|
+
```python
|
|
109
|
+
from uripath import URIPath
|
|
110
|
+
|
|
111
|
+
print(URIPath("s3://data/reports/2026.csv").read_text())
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## API overview
|
|
115
|
+
|
|
116
|
+
| Module | Purpose |
|
|
117
|
+
| --- | --- |
|
|
118
|
+
| `objpath` | `S3Path`, `GSPath`, `AZPath`, the backends `S3Backend`, `GSBackend`, `AZBackend` and the contracts `BaseS3Backend`, `BaseGSBackend`, `BaseAZBackend`, the exceptions and `__version__` |
|
|
119
|
+
| `objpath.exceptions` | `ObjPathError`, `ObjectChangedError` and `UnreachableKeysError` |
|
|
120
|
+
| `objpath.uri` | `S3URIPath`, `GSURIPath` and `AZURIPath`, the schemes of `uripath` (needs the `uri` extra) |
|
|
121
|
+
|
|
122
|
+
The full reference is on the [documentation site](https://jose-pr.github.io/objpath/);
|
|
123
|
+
an agent-oriented API summary ships inside the package as `objpath/AGENTS.md`.
|
|
124
|
+
|
|
125
|
+
## Development
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
pip install -e ".[dev,all]"
|
|
129
|
+
python -m pytest -q -rs
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The S3 tests run against `moto` in the test process, and the Google Cloud Storage and
|
|
133
|
+
Azure tests against fake servers on the loopback interface; none reaches another host. A
|
|
134
|
+
test for an extra that is not installed is skipped, so check the skip list (`-rs`).
|
|
135
|
+
Contributor notes (virtual environments, formatting, CI, releases) are in the repository's
|
|
136
|
+
[`AGENTS.md`](https://github.com/jose-pr/objpath/blob/main/AGENTS.md). To preview the
|
|
137
|
+
documentation: `pip install -e ".[all,docs]"` then `mkdocs serve`.
|
|
138
|
+
|
|
139
|
+
Changes are recorded in the
|
|
140
|
+
[changelog](https://github.com/jose-pr/objpath/blob/main/CHANGELOG.md).
|
|
141
|
+
|
|
142
|
+
### Releasing
|
|
143
|
+
|
|
144
|
+
This project follows [Semantic Versioning](https://semver.org/) and keeps a
|
|
145
|
+
[`CHANGELOG.md`](https://github.com/jose-pr/objpath/blob/main/CHANGELOG.md). Pushing a tag matching `v*` triggers the release
|
|
146
|
+
workflow: test gate → build (checking the tag names the version built) → a strict
|
|
147
|
+
docs build as a gate → GitHub release → publish. The release workflow never deploys
|
|
148
|
+
the docs site itself: for a final release its last job dispatches the docs workflow
|
|
149
|
+
at the tag, which owns every Pages deploy.
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT — see [LICENSE](https://github.com/jose-pr/objpath/blob/main/LICENSE).
|
objpath/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""pathlib-style paths for S3, Google Cloud Storage and Azure Blob Storage.
|
|
2
|
+
|
|
3
|
+
`S3Path`, `GSPath` and `AZPath` need no URI. None of them imports its SDK
|
|
4
|
+
until a client is built, and none is imported until it is first named
|
|
5
|
+
(PEP 562), so `import objpath` needs no extra and never imports `uripath`. The
|
|
6
|
+
`s3:`, `gs:` and `az:` schemes of `uripath` are `objpath.uri`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import importlib as _importlib
|
|
12
|
+
import typing as _ty
|
|
13
|
+
|
|
14
|
+
from ._version import __version__
|
|
15
|
+
from .exceptions import ObjectChangedError, ObjPathError, UnreachableKeysError
|
|
16
|
+
|
|
17
|
+
if _ty.TYPE_CHECKING:
|
|
18
|
+
from ._az import AZPath as AZPath
|
|
19
|
+
from ._az_backend import AZBackend as AZBackend
|
|
20
|
+
from ._az_backend import BaseAZBackend as BaseAZBackend
|
|
21
|
+
from ._gs import GSPath as GSPath
|
|
22
|
+
from ._gs_backend import BaseGSBackend as BaseGSBackend
|
|
23
|
+
from ._gs_backend import GSBackend as GSBackend
|
|
24
|
+
from ._s3 import S3Path as S3Path
|
|
25
|
+
from ._s3_backend import BaseS3Backend as BaseS3Backend
|
|
26
|
+
from ._s3_backend import S3Backend as S3Backend
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"AZBackend",
|
|
30
|
+
"AZPath",
|
|
31
|
+
"BaseAZBackend",
|
|
32
|
+
"BaseGSBackend",
|
|
33
|
+
"BaseS3Backend",
|
|
34
|
+
"GSBackend",
|
|
35
|
+
"GSPath",
|
|
36
|
+
"ObjPathError",
|
|
37
|
+
"ObjectChangedError",
|
|
38
|
+
"S3Backend",
|
|
39
|
+
"S3Path",
|
|
40
|
+
"UnreachableKeysError",
|
|
41
|
+
"__version__",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
#: Public name -> the private module that defines it, imported on first use.
|
|
45
|
+
_LAZY = {
|
|
46
|
+
"AZBackend": "._az_backend",
|
|
47
|
+
"AZPath": "._az",
|
|
48
|
+
"BaseAZBackend": "._az_backend",
|
|
49
|
+
"BaseGSBackend": "._gs_backend",
|
|
50
|
+
"BaseS3Backend": "._s3_backend",
|
|
51
|
+
"GSBackend": "._gs_backend",
|
|
52
|
+
"GSPath": "._gs",
|
|
53
|
+
"S3Backend": "._s3_backend",
|
|
54
|
+
"S3Path": "._s3",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def __getattr__(name: str):
|
|
59
|
+
module = _LAZY.get(name)
|
|
60
|
+
if module is None:
|
|
61
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
62
|
+
value = getattr(_importlib.import_module(module, __name__), name)
|
|
63
|
+
globals()[name] = value
|
|
64
|
+
return value
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def __dir__():
|
|
68
|
+
return sorted(set(globals()) | set(_LAZY))
|