asanypath 2.1.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/workflows/publish.yml +2 -2
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/workflows/unittests.yml +3 -6
- {asanypath-2.1.0 → asanypath-2.2.0}/.gitignore +1 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.pre-commit-config.yaml +7 -1
- {asanypath-2.1.0 → asanypath-2.2.0}/PKG-INFO +132 -7
- {asanypath-2.1.0 → asanypath-2.2.0}/README.md +125 -4
- {asanypath-2.1.0 → asanypath-2.2.0}/hatch_build.py +1 -1
- {asanypath-2.1.0 → asanypath-2.2.0}/pyproject.toml +27 -7
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/__init__.py +2 -0
- asanypath-2.2.0/src/asanypath/_resume.py +43 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/_transfer.py +13 -5
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/_version.py +2 -2
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/artifactory.py +54 -2
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/asanypath.py +1 -1
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/asanypath.pyi +1 -1
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/azure.py +54 -3
- asanypath-2.2.0/src/asanypath/cache.py +175 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/cli.py +2 -2
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/cloud.py +227 -43
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/ftp.py +44 -8
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/gcs.py +43 -3
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/http.py +27 -17
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/local.py +22 -16
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/options.py +30 -0
- asanypath-2.2.0/src/asanypath/py.typed +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/s3.py +69 -10
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/ssh.py +136 -39
- asanypath-2.2.0/src/asanypath/unsupported.py +64 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_artifactory.py +37 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_azure.py +42 -0
- asanypath-2.2.0/tests/test_cache.py +242 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_cli.py +2 -2
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_cloudpathmixin.py +23 -2
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_ftp.py +19 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_gcs.py +69 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_local.py +6 -0
- asanypath-2.2.0/tests/test_options.py +32 -0
- asanypath-2.2.0/tests/test_py_typed.py +14 -0
- asanypath-2.2.0/tests/test_resume.py +198 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_s3.py +166 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_ssh.py +39 -0
- asanypath-2.1.0/src/asanypath/unsupported.py +0 -202
- asanypath-2.1.0/src/asanypath_native.pyi +0 -123
- asanypath-2.1.0/uv.lock +0 -2798
- {asanypath-2.1.0 → asanypath-2.2.0}/.copilot/skills/async-python-lib/SKILL.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.copilot/skills/gitguro/SKILL.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/CODEOWNERS +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/.github/dependabot.yml +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/CODE_OF_CONDUCT.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/CONTRIBUTING.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/LICENSE +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/SECURITY.md +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/asanypath.png +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/compose.yml +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/scripts/bench.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/__init__.pyi +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/_credstore.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/batcher.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/common.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/exceptions.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/src/asanypath/sync.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/__init__.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/conftest.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_asanypath.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_batcher.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_commonpathmixin.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_credstore.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_http.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_http_auth.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_sync.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_syncpath.py +0 -0
- {asanypath-2.1.0 → asanypath-2.2.0}/tests/test_unsupported_path.py +0 -0
|
@@ -102,7 +102,7 @@ jobs:
|
|
|
102
102
|
rm rust/Cargo.toml.bak
|
|
103
103
|
|
|
104
104
|
- name: Set up uv
|
|
105
|
-
uses: astral-sh/setup-uv@
|
|
105
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
106
106
|
with:
|
|
107
107
|
enable-cache: false
|
|
108
108
|
|
|
@@ -128,7 +128,7 @@ jobs:
|
|
|
128
128
|
persist-credentials: false
|
|
129
129
|
|
|
130
130
|
- name: Set up uv
|
|
131
|
-
uses: astral-sh/setup-uv@
|
|
131
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
132
132
|
with:
|
|
133
133
|
enable-cache: false
|
|
134
134
|
|
|
@@ -44,16 +44,13 @@ jobs:
|
|
|
44
44
|
python-version: ${{ matrix.python-version }}
|
|
45
45
|
|
|
46
46
|
- name: Set up uv
|
|
47
|
-
uses: astral-sh/setup-uv@
|
|
47
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
48
48
|
|
|
49
49
|
- name: Ruff checks
|
|
50
50
|
run: uvx ruff check src tests && uvx ruff format --check src tests
|
|
51
51
|
|
|
52
|
-
- name:
|
|
53
|
-
run: uv
|
|
54
|
-
|
|
55
|
-
- name: Install project and native extension
|
|
56
|
-
run: uv pip install -e ".[dev]"
|
|
52
|
+
- name: Install project and native extension from uv.lock
|
|
53
|
+
run: uv sync --locked --extra dev --python "${{ matrix.python-version }}"
|
|
57
54
|
|
|
58
55
|
- name: Test with pytest and coverage
|
|
59
56
|
env:
|
|
@@ -22,4 +22,10 @@ repos:
|
|
|
22
22
|
entry: cargo check --manifest-path rust/Cargo.toml --locked
|
|
23
23
|
language: system
|
|
24
24
|
files: ^rust/.*\.(rs|toml|lock)$
|
|
25
|
-
pass_filenames: false
|
|
25
|
+
pass_filenames: false
|
|
26
|
+
# uv.lock must only reference the public index (a mirror URL leaks internal hosts).
|
|
27
|
+
- id: uv-lock-public-index
|
|
28
|
+
name: uv.lock uses only public index URLs
|
|
29
|
+
language: pygrep
|
|
30
|
+
entry: '(registry|url) = "https?://(?!pypi\.org/|files\.pythonhosted\.org/)'
|
|
31
|
+
files: ^uv\.lock$
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: asanypath
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Async pathlib for local and cloud storage
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -19,11 +19,12 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
19
19
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
20
|
Requires-Python: >=3.10
|
|
21
21
|
Requires-Dist: anyio>=4
|
|
22
|
-
Requires-Dist: asanypath-native~=2.
|
|
23
|
-
Requires-Dist: msgspec>=0.
|
|
22
|
+
Requires-Dist: asanypath-native~=2.2.0
|
|
23
|
+
Requires-Dist: msgspec>=0.22.0
|
|
24
24
|
Requires-Dist: yarl>=1.2
|
|
25
25
|
Provides-Extra: all
|
|
26
26
|
Requires-Dist: aioftp>=0.21; extra == 'all'
|
|
27
|
+
Requires-Dist: diskcache>=5.6; extra == 'all'
|
|
27
28
|
Requires-Dist: keyring>=24; extra == 'all'
|
|
28
29
|
Provides-Extra: bench
|
|
29
30
|
Requires-Dist: adlfs; extra == 'bench'
|
|
@@ -37,12 +38,15 @@ Requires-Dist: gcloud-aio-storage; extra == 'bench'
|
|
|
37
38
|
Requires-Dist: gcsfs; extra == 'bench'
|
|
38
39
|
Requires-Dist: requests; extra == 'bench'
|
|
39
40
|
Requires-Dist: s3fs; extra == 'bench'
|
|
41
|
+
Provides-Extra: cache
|
|
42
|
+
Requires-Dist: diskcache>=5.6; extra == 'cache'
|
|
40
43
|
Provides-Extra: cli
|
|
41
44
|
Requires-Dist: rich-click>=1.9.7; extra == 'cli'
|
|
42
45
|
Provides-Extra: credstore
|
|
43
46
|
Requires-Dist: keyring>=24; extra == 'credstore'
|
|
44
47
|
Provides-Extra: dev
|
|
45
48
|
Requires-Dist: aioftp>=0.21; extra == 'dev'
|
|
49
|
+
Requires-Dist: diskcache>=5.6; extra == 'dev'
|
|
46
50
|
Requires-Dist: keyring>=24; extra == 'dev'
|
|
47
51
|
Requires-Dist: pre-commit>=4; extra == 'dev'
|
|
48
52
|
Requires-Dist: pytest-asyncio>=1.3; extra == 'dev'
|
|
@@ -161,11 +165,131 @@ for child in p.parent.iterdir(): # yields synchronously
|
|
|
161
165
|
Local paths (`file://` or bare `/path`) bypass the proxy entirely and use
|
|
162
166
|
`pathlib.Path` directly — zero threading overhead.
|
|
163
167
|
|
|
168
|
+
### Large uploads stream automatically
|
|
169
|
+
|
|
170
|
+
Writing or copying a large object never buffers the whole thing in memory — it
|
|
171
|
+
streams in bounded-size parts using each backend's native mechanism (S3
|
|
172
|
+
multipart, Azure block blobs, GCS resumable, SSH chunked writes, FTP data
|
|
173
|
+
connection, Artifactory streamed `PUT`). This is the **default** behaviour — no
|
|
174
|
+
flag, no API change:
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
# open("wb") spills to disk past 16 MiB and uploads in parts on close
|
|
178
|
+
async with AsAnyPath("s3://bucket/huge.bin").open("wb") as f:
|
|
179
|
+
f.write(data)
|
|
180
|
+
|
|
181
|
+
# local -> cloud copy streams straight from the source file (async and sync)
|
|
182
|
+
await AsAnyPath("/data/movie.mp4").copy("s3://bucket/movie.mp4")
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Peak memory per upload is one part — **8 MiB by default**. Objects smaller than
|
|
186
|
+
one part upload in a single request. (`write_bytes(data)` / `write_text(...)`
|
|
187
|
+
take the whole payload in memory by definition — use `open("wb")` or `copy()`
|
|
188
|
+
for large data.)
|
|
189
|
+
|
|
190
|
+
**Tuning the memory footprint.** The part size *is* the max RAM held per
|
|
191
|
+
in-flight chunk (parts upload sequentially, so there is no concurrency
|
|
192
|
+
multiplier). Set it globally via env, or per call:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
export ASANYPATH_UPLOAD_CHUNK_SIZE=33554432 # 32 MiB parts (bytes)
|
|
196
|
+
export ASANYPATH_SPOOL_MAX_SIZE=67108864 # 64 MiB before open("wb") spills to disk
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
# per call: copy(chunk_size=...) or, for writes, open(..., buffering=...)
|
|
201
|
+
await src.copy("s3://bucket/huge.bin", chunk_size=32 * 1024 * 1024)
|
|
202
|
+
async with dst.open("wb", buffering=32 * 1024 * 1024) as f:
|
|
203
|
+
f.write(data)
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Bigger parts mean fewer requests and a larger single-object ceiling (S3 allows
|
|
207
|
+
10,000 parts, so 8 MiB parts cap an object at ~80 GB). Backend minimums are
|
|
208
|
+
applied automatically: **S3** clamps to ≥ 5 MiB parts, **GCS** rounds down to a
|
|
209
|
+
256 KiB multiple.
|
|
210
|
+
|
|
164
211
|
Unknown URI schemes now construct as `UnsupportedProtocolPath` placeholders.
|
|
165
212
|
Pure path operations (e.g. `.name`, `.parent`, joins) still work, while backend
|
|
166
213
|
operations (e.g. `.exists()`, `.read_bytes()`, `.open()`) raise
|
|
167
214
|
`UnsupportedProtocolError` on use.
|
|
168
215
|
|
|
216
|
+
### Download caching (opt-in)
|
|
217
|
+
|
|
218
|
+
Reading the same object or URL twice always re-downloads it — there is **no
|
|
219
|
+
cache by default**, and the read path is unchanged unless you opt in, so
|
|
220
|
+
non-caching callers pay zero overhead.
|
|
221
|
+
|
|
222
|
+
Enable it with the optional `cache` extra (`pip install asanypath[cache]`,
|
|
223
|
+
backed by [`diskcache`](https://pypi.org/project/diskcache/) — a pure-Python,
|
|
224
|
+
zero-dependency store). Opt in per read with `use_cache=True`:
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
from asanypath import AsAnyPath
|
|
228
|
+
|
|
229
|
+
p = AsAnyPath("s3://bucket/model.bin")
|
|
230
|
+
data = await p.read_bytes(use_cache=True) # first call fetches + stores
|
|
231
|
+
data = await p.read_bytes(use_cache=True) # subsequent calls served from disk
|
|
232
|
+
text = await p.read_text(use_cache=True)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Configure the location and limits once (location defaults to
|
|
236
|
+
`~/.cache/asanypath`, honoring `$XDG_CACHE_HOME` and `$ASANYPATH_CACHE_DIR`):
|
|
237
|
+
|
|
238
|
+
```python
|
|
239
|
+
from asanypath import configure_cache
|
|
240
|
+
|
|
241
|
+
configure_cache(
|
|
242
|
+
directory="/var/cache/asanypath", # default: ~/.cache/asanypath
|
|
243
|
+
size_limit=2 * 1024**3, # total on-disk cap in bytes (default 1 GiB, LRU)
|
|
244
|
+
ttl=3600, # per-entry expiry in seconds (default: none)
|
|
245
|
+
enabled=True, # make use_cache default to True process-wide
|
|
246
|
+
)
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Or via environment: `ASANYPATH_CACHE_DIR`, `ASANYPATH_CACHE_SIZE_LIMIT` (bytes),
|
|
250
|
+
`ASANYPATH_CACHE_TTL` (seconds).
|
|
251
|
+
|
|
252
|
+
Behaviour and trade-offs:
|
|
253
|
+
|
|
254
|
+
- **Off by default, opt-in per call.** With `configure_cache(enabled=True)` the
|
|
255
|
+
default flips process-wide; an explicit `use_cache=False` always bypasses.
|
|
256
|
+
- **Works for every remote backend** (S3, GCS, Azure, Artifactory, HTTP, SSH,
|
|
257
|
+
FTP). Local paths are never cached.
|
|
258
|
+
- **Total-size bounded, LRU-evicted;** disk I/O runs in a worker thread so the
|
|
259
|
+
event loop stays responsive. Concurrent reads of the same key collapse to a
|
|
260
|
+
single fetch.
|
|
261
|
+
- **Staleness is on you.** Entries are keyed by URL with no revalidation — a
|
|
262
|
+
cached object that changes upstream is served stale until it expires (`ttl`)
|
|
263
|
+
or is evicted. Prefer caching immutable/versioned objects, or set a `ttl`.
|
|
264
|
+
|
|
265
|
+
### Resumable downloads
|
|
266
|
+
|
|
267
|
+
`download(dst)` streams an object to a local file and **resumes an interrupted
|
|
268
|
+
transfer** instead of starting over:
|
|
269
|
+
|
|
270
|
+
```python
|
|
271
|
+
await AsAnyPath("s3://bucket/huge.bin").download("/data/huge.bin")
|
|
272
|
+
# process killed mid-download... run the same call again — it continues
|
|
273
|
+
await AsAnyPath("s3://bucket/huge.bin").download("/data/huge.bin")
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
Bytes stream into `<dst>.part` with a `<dst>.part.meta` sidecar recording the
|
|
277
|
+
object's ETag (or `Last-Modified`) and total size. On resume the identity is
|
|
278
|
+
re-checked: if the upstream object is unchanged the download continues from the
|
|
279
|
+
saved offset; if it changed, the partial is discarded and it restarts cleanly.
|
|
280
|
+
On completion `<dst>.part` is atomically renamed to `dst`.
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
# tune the range-request size; disable resume to always start fresh
|
|
284
|
+
await path.download("/data/huge.bin", chunk_size=32 * 1024 * 1024, resume=False)
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
Resume needs range reads **and** a HEAD probe, so it's available on the object
|
|
288
|
+
stores (S3/GCS/Azure) and Artifactory/HTTP where the server supports it.
|
|
289
|
+
Backends without those (e.g. FTP) download in a single pass automatically. This
|
|
290
|
+
needs no extra dependency — resume state is plain sidecar files, independent of
|
|
291
|
+
the optional download cache.
|
|
292
|
+
|
|
169
293
|
## Installation
|
|
170
294
|
|
|
171
295
|
Most cloud backends (S3, GCS, Azure, Artifactory, HTTP) are included in the base
|
|
@@ -438,10 +562,11 @@ Factory that returns the appropriate path instance based on the URI scheme.
|
|
|
438
562
|
|
|
439
563
|
All path implementations support:
|
|
440
564
|
|
|
441
|
-
- `open(mode, buffering)` — file-like streaming with lazy range reads
|
|
442
|
-
- `copy(dst, recursive)` — copy file or tree (cross-backend)
|
|
443
|
-
- `
|
|
444
|
-
- `
|
|
565
|
+
- `open(mode, buffering)` — file-like streaming with lazy range reads; for writes, `buffering` sets the upload part size
|
|
566
|
+
- `copy(dst, recursive, chunk_size)` — copy file or tree (cross-backend); `chunk_size` tunes the upload part size
|
|
567
|
+
- `download(dst, resume, chunk_size)` — stream an object to a local file, resuming an interrupted transfer
|
|
568
|
+
- `read_bytes()` / `write_bytes(data)` — `read_bytes(use_cache=True)` serves from / stores in the opt-in [download cache](#download-caching-opt-in)
|
|
569
|
+
- `read_text(encoding)` / `write_text(data, encoding)` — `read_text` also accepts `use_cache`
|
|
445
570
|
- `exists()` / `is_file()` / `is_dir()`
|
|
446
571
|
- `stat()` / `checksums()` (cloud backends)
|
|
447
572
|
- `mkdir(parents, exist_ok)` / `touch(exist_ok)`
|
|
@@ -106,11 +106,131 @@ for child in p.parent.iterdir(): # yields synchronously
|
|
|
106
106
|
Local paths (`file://` or bare `/path`) bypass the proxy entirely and use
|
|
107
107
|
`pathlib.Path` directly — zero threading overhead.
|
|
108
108
|
|
|
109
|
+
### Large uploads stream automatically
|
|
110
|
+
|
|
111
|
+
Writing or copying a large object never buffers the whole thing in memory — it
|
|
112
|
+
streams in bounded-size parts using each backend's native mechanism (S3
|
|
113
|
+
multipart, Azure block blobs, GCS resumable, SSH chunked writes, FTP data
|
|
114
|
+
connection, Artifactory streamed `PUT`). This is the **default** behaviour — no
|
|
115
|
+
flag, no API change:
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
# open("wb") spills to disk past 16 MiB and uploads in parts on close
|
|
119
|
+
async with AsAnyPath("s3://bucket/huge.bin").open("wb") as f:
|
|
120
|
+
f.write(data)
|
|
121
|
+
|
|
122
|
+
# local -> cloud copy streams straight from the source file (async and sync)
|
|
123
|
+
await AsAnyPath("/data/movie.mp4").copy("s3://bucket/movie.mp4")
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Peak memory per upload is one part — **8 MiB by default**. Objects smaller than
|
|
127
|
+
one part upload in a single request. (`write_bytes(data)` / `write_text(...)`
|
|
128
|
+
take the whole payload in memory by definition — use `open("wb")` or `copy()`
|
|
129
|
+
for large data.)
|
|
130
|
+
|
|
131
|
+
**Tuning the memory footprint.** The part size *is* the max RAM held per
|
|
132
|
+
in-flight chunk (parts upload sequentially, so there is no concurrency
|
|
133
|
+
multiplier). Set it globally via env, or per call:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
export ASANYPATH_UPLOAD_CHUNK_SIZE=33554432 # 32 MiB parts (bytes)
|
|
137
|
+
export ASANYPATH_SPOOL_MAX_SIZE=67108864 # 64 MiB before open("wb") spills to disk
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
# per call: copy(chunk_size=...) or, for writes, open(..., buffering=...)
|
|
142
|
+
await src.copy("s3://bucket/huge.bin", chunk_size=32 * 1024 * 1024)
|
|
143
|
+
async with dst.open("wb", buffering=32 * 1024 * 1024) as f:
|
|
144
|
+
f.write(data)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Bigger parts mean fewer requests and a larger single-object ceiling (S3 allows
|
|
148
|
+
10,000 parts, so 8 MiB parts cap an object at ~80 GB). Backend minimums are
|
|
149
|
+
applied automatically: **S3** clamps to ≥ 5 MiB parts, **GCS** rounds down to a
|
|
150
|
+
256 KiB multiple.
|
|
151
|
+
|
|
109
152
|
Unknown URI schemes now construct as `UnsupportedProtocolPath` placeholders.
|
|
110
153
|
Pure path operations (e.g. `.name`, `.parent`, joins) still work, while backend
|
|
111
154
|
operations (e.g. `.exists()`, `.read_bytes()`, `.open()`) raise
|
|
112
155
|
`UnsupportedProtocolError` on use.
|
|
113
156
|
|
|
157
|
+
### Download caching (opt-in)
|
|
158
|
+
|
|
159
|
+
Reading the same object or URL twice always re-downloads it — there is **no
|
|
160
|
+
cache by default**, and the read path is unchanged unless you opt in, so
|
|
161
|
+
non-caching callers pay zero overhead.
|
|
162
|
+
|
|
163
|
+
Enable it with the optional `cache` extra (`pip install asanypath[cache]`,
|
|
164
|
+
backed by [`diskcache`](https://pypi.org/project/diskcache/) — a pure-Python,
|
|
165
|
+
zero-dependency store). Opt in per read with `use_cache=True`:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from asanypath import AsAnyPath
|
|
169
|
+
|
|
170
|
+
p = AsAnyPath("s3://bucket/model.bin")
|
|
171
|
+
data = await p.read_bytes(use_cache=True) # first call fetches + stores
|
|
172
|
+
data = await p.read_bytes(use_cache=True) # subsequent calls served from disk
|
|
173
|
+
text = await p.read_text(use_cache=True)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Configure the location and limits once (location defaults to
|
|
177
|
+
`~/.cache/asanypath`, honoring `$XDG_CACHE_HOME` and `$ASANYPATH_CACHE_DIR`):
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from asanypath import configure_cache
|
|
181
|
+
|
|
182
|
+
configure_cache(
|
|
183
|
+
directory="/var/cache/asanypath", # default: ~/.cache/asanypath
|
|
184
|
+
size_limit=2 * 1024**3, # total on-disk cap in bytes (default 1 GiB, LRU)
|
|
185
|
+
ttl=3600, # per-entry expiry in seconds (default: none)
|
|
186
|
+
enabled=True, # make use_cache default to True process-wide
|
|
187
|
+
)
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Or via environment: `ASANYPATH_CACHE_DIR`, `ASANYPATH_CACHE_SIZE_LIMIT` (bytes),
|
|
191
|
+
`ASANYPATH_CACHE_TTL` (seconds).
|
|
192
|
+
|
|
193
|
+
Behaviour and trade-offs:
|
|
194
|
+
|
|
195
|
+
- **Off by default, opt-in per call.** With `configure_cache(enabled=True)` the
|
|
196
|
+
default flips process-wide; an explicit `use_cache=False` always bypasses.
|
|
197
|
+
- **Works for every remote backend** (S3, GCS, Azure, Artifactory, HTTP, SSH,
|
|
198
|
+
FTP). Local paths are never cached.
|
|
199
|
+
- **Total-size bounded, LRU-evicted;** disk I/O runs in a worker thread so the
|
|
200
|
+
event loop stays responsive. Concurrent reads of the same key collapse to a
|
|
201
|
+
single fetch.
|
|
202
|
+
- **Staleness is on you.** Entries are keyed by URL with no revalidation — a
|
|
203
|
+
cached object that changes upstream is served stale until it expires (`ttl`)
|
|
204
|
+
or is evicted. Prefer caching immutable/versioned objects, or set a `ttl`.
|
|
205
|
+
|
|
206
|
+
### Resumable downloads
|
|
207
|
+
|
|
208
|
+
`download(dst)` streams an object to a local file and **resumes an interrupted
|
|
209
|
+
transfer** instead of starting over:
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
await AsAnyPath("s3://bucket/huge.bin").download("/data/huge.bin")
|
|
213
|
+
# process killed mid-download... run the same call again — it continues
|
|
214
|
+
await AsAnyPath("s3://bucket/huge.bin").download("/data/huge.bin")
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
Bytes stream into `<dst>.part` with a `<dst>.part.meta` sidecar recording the
|
|
218
|
+
object's ETag (or `Last-Modified`) and total size. On resume the identity is
|
|
219
|
+
re-checked: if the upstream object is unchanged the download continues from the
|
|
220
|
+
saved offset; if it changed, the partial is discarded and it restarts cleanly.
|
|
221
|
+
On completion `<dst>.part` is atomically renamed to `dst`.
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
# tune the range-request size; disable resume to always start fresh
|
|
225
|
+
await path.download("/data/huge.bin", chunk_size=32 * 1024 * 1024, resume=False)
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Resume needs range reads **and** a HEAD probe, so it's available on the object
|
|
229
|
+
stores (S3/GCS/Azure) and Artifactory/HTTP where the server supports it.
|
|
230
|
+
Backends without those (e.g. FTP) download in a single pass automatically. This
|
|
231
|
+
needs no extra dependency — resume state is plain sidecar files, independent of
|
|
232
|
+
the optional download cache.
|
|
233
|
+
|
|
114
234
|
## Installation
|
|
115
235
|
|
|
116
236
|
Most cloud backends (S3, GCS, Azure, Artifactory, HTTP) are included in the base
|
|
@@ -383,10 +503,11 @@ Factory that returns the appropriate path instance based on the URI scheme.
|
|
|
383
503
|
|
|
384
504
|
All path implementations support:
|
|
385
505
|
|
|
386
|
-
- `open(mode, buffering)` — file-like streaming with lazy range reads
|
|
387
|
-
- `copy(dst, recursive)` — copy file or tree (cross-backend)
|
|
388
|
-
- `
|
|
389
|
-
- `
|
|
506
|
+
- `open(mode, buffering)` — file-like streaming with lazy range reads; for writes, `buffering` sets the upload part size
|
|
507
|
+
- `copy(dst, recursive, chunk_size)` — copy file or tree (cross-backend); `chunk_size` tunes the upload part size
|
|
508
|
+
- `download(dst, resume, chunk_size)` — stream an object to a local file, resuming an interrupted transfer
|
|
509
|
+
- `read_bytes()` / `write_bytes(data)` — `read_bytes(use_cache=True)` serves from / stores in the opt-in [download cache](#download-caching-opt-in)
|
|
510
|
+
- `read_text(encoding)` / `write_text(data, encoding)` — `read_text` also accepts `use_cache`
|
|
390
511
|
- `exists()` / `is_file()` / `is_dir()`
|
|
391
512
|
- `stat()` / `checksums()` (cloud backends)
|
|
392
513
|
- `mkdir(parents, exist_ok)` / `touch(exist_ok)`
|
|
@@ -20,7 +20,7 @@ asanypath-native = { path = "rust" }
|
|
|
20
20
|
root = "src"
|
|
21
21
|
|
|
22
22
|
[tool.hatch.build.targets.sdist]
|
|
23
|
-
exclude = ["rust"]
|
|
23
|
+
exclude = ["rust", "uv.lock"]
|
|
24
24
|
|
|
25
25
|
[tool.hatch.metadata.hooks.custom]
|
|
26
26
|
|
|
@@ -58,6 +58,7 @@ dev = [
|
|
|
58
58
|
cli = ["rich-click>=1.9.7"]
|
|
59
59
|
ftp = ["aioftp>=0.21"]
|
|
60
60
|
credstore = ["keyring>=24"]
|
|
61
|
+
cache = ["diskcache>=5.6"]
|
|
61
62
|
bench = [
|
|
62
63
|
"asyncssh>=2.14",
|
|
63
64
|
"fsspec",
|
|
@@ -71,7 +72,7 @@ bench = [
|
|
|
71
72
|
"aiofiles",
|
|
72
73
|
"requests",
|
|
73
74
|
]
|
|
74
|
-
all = ["asanypath[credstore,ftp]"]
|
|
75
|
+
all = ["asanypath[credstore,ftp,cache]"]
|
|
75
76
|
[project.scripts]
|
|
76
77
|
asanypath = "asanypath.cli:main"
|
|
77
78
|
|
|
@@ -88,6 +89,9 @@ python_version = "3.10"
|
|
|
88
89
|
warn_return_any = true
|
|
89
90
|
warn_unused_configs = true
|
|
90
91
|
disallow_untyped_defs = true
|
|
92
|
+
# asanypath_native.pyi lives in rust/ (its canonical home, shipped in the native
|
|
93
|
+
# wheel by maturin); keep it resolvable for dev type-checking.
|
|
94
|
+
mypy_path = ["src", "rust"]
|
|
91
95
|
|
|
92
96
|
[[tool.mypy.overrides]]
|
|
93
97
|
module = "asanypath.cli"
|
|
@@ -99,7 +103,7 @@ disable_error_code = ["attr-defined", "arg-type", "assignment"]
|
|
|
99
103
|
module = "asanypath.local"
|
|
100
104
|
disallow_untyped_defs = false
|
|
101
105
|
warn_return_any = false
|
|
102
|
-
disable_error_code = ["type-var", "misc", "override", "return-value"]
|
|
106
|
+
disable_error_code = ["type-var", "misc", "override", "return-value", "assignment", "no-redef"]
|
|
103
107
|
|
|
104
108
|
[[tool.mypy.overrides]]
|
|
105
109
|
module = "asanypath.common"
|
|
@@ -111,30 +115,46 @@ disable_error_code = ["arg-type", "assignment", "misc", "attr-defined"]
|
|
|
111
115
|
module = "asanypath.http"
|
|
112
116
|
disallow_untyped_defs = false
|
|
113
117
|
warn_return_any = false
|
|
114
|
-
disable_error_code = ["override", "type-arg", "return-value", "assignment"]
|
|
118
|
+
disable_error_code = ["override", "type-arg", "return-value", "assignment", "attr-defined"]
|
|
115
119
|
|
|
116
120
|
[[tool.mypy.overrides]]
|
|
117
121
|
module = ["asanypath.s3", "asanypath.gcs", "asanypath.azure", "asanypath.artifactory"]
|
|
118
122
|
disallow_untyped_defs = false
|
|
119
123
|
warn_return_any = false
|
|
120
|
-
disable_error_code = ["no-any-return", "attr-defined", "assignment", "override"]
|
|
124
|
+
disable_error_code = ["no-any-return", "attr-defined", "assignment", "override", "arg-type", "misc"]
|
|
125
|
+
|
|
126
|
+
[[tool.mypy.overrides]]
|
|
127
|
+
module = ["asanypath.ssh", "asanypath.ftp"]
|
|
128
|
+
disallow_untyped_defs = false
|
|
129
|
+
warn_return_any = false
|
|
130
|
+
disable_error_code = ["attr-defined", "override", "arg-type", "misc", "no-any-return", "assignment"]
|
|
131
|
+
|
|
132
|
+
[[tool.mypy.overrides]]
|
|
133
|
+
module = "asanypath.sync"
|
|
134
|
+
disallow_untyped_defs = false
|
|
135
|
+
warn_return_any = false
|
|
136
|
+
disable_error_code = ["no-any-return", "misc", "attr-defined", "arg-type"]
|
|
121
137
|
|
|
122
138
|
[[tool.mypy.overrides]]
|
|
123
139
|
module = "asanypath.cloud"
|
|
124
140
|
disallow_untyped_defs = false
|
|
125
141
|
warn_return_any = false
|
|
126
|
-
disable_error_code = ["no-any-return", "arg-type"]
|
|
142
|
+
disable_error_code = ["no-any-return", "arg-type", "attr-defined", "misc", "assignment"]
|
|
127
143
|
|
|
128
144
|
[[tool.mypy.overrides]]
|
|
129
145
|
module = "asanypath.asanypath"
|
|
130
146
|
disallow_untyped_defs = false
|
|
131
147
|
warn_return_any = false
|
|
132
|
-
disable_error_code = ["no-any-return", "misc", "union-attr", "type-var"]
|
|
148
|
+
disable_error_code = ["no-any-return", "misc", "union-attr", "type-var", "attr-defined"]
|
|
133
149
|
|
|
134
150
|
[[tool.mypy.overrides]]
|
|
135
151
|
module = "asanypath.batcher"
|
|
136
152
|
disallow_untyped_defs = false
|
|
137
153
|
|
|
154
|
+
[[tool.mypy.overrides]]
|
|
155
|
+
module = "diskcache.*"
|
|
156
|
+
ignore_missing_imports = true
|
|
157
|
+
|
|
138
158
|
[tool.pytest.ini_options]
|
|
139
159
|
asyncio_mode = "auto"
|
|
140
160
|
testpaths = ["tests"]
|
|
@@ -13,6 +13,7 @@ from asanypath.asanypath import (
|
|
|
13
13
|
register_protocol,
|
|
14
14
|
)
|
|
15
15
|
from asanypath.azure import AzurePath
|
|
16
|
+
from asanypath.cache import configure_cache
|
|
16
17
|
from asanypath.cloud import CloudPathMixin
|
|
17
18
|
from asanypath.common import CommonPurePathMixin
|
|
18
19
|
from asanypath.exceptions import (
|
|
@@ -62,6 +63,7 @@ __all__ = [
|
|
|
62
63
|
"GCSPath",
|
|
63
64
|
"AzurePath",
|
|
64
65
|
"ArtifactoryPath",
|
|
66
|
+
"configure_cache",
|
|
65
67
|
"register_protocol",
|
|
66
68
|
"list_protocols",
|
|
67
69
|
"AsAnyPathError",
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Copyright (C) 2026 Axis Communications AB, Lund, Sweden
|
|
2
|
+
#
|
|
3
|
+
# Use of this source code is governed by an MIT-style
|
|
4
|
+
# license that can be found in the LICENSE file or at
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
"""Sidecar state for resumable downloads.
|
|
8
|
+
|
|
9
|
+
A partial download lives in ``<dst>.part`` with a ``<dst>.part.meta`` sidecar
|
|
10
|
+
recording the remote object's identity (ETag/Last-Modified) and total size. On
|
|
11
|
+
resume the identity is re-checked so a changed upstream object restarts cleanly
|
|
12
|
+
rather than stitching mismatched bytes. State is plain files — no dependency on
|
|
13
|
+
the optional download cache.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import msgspec
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ResumeMeta(msgspec.Struct, frozen=True):
|
|
24
|
+
"""Identity and size of the object a ``.part`` file is downloading."""
|
|
25
|
+
|
|
26
|
+
identity: str | None
|
|
27
|
+
size: int
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
_encoder = msgspec.json.Encoder()
|
|
31
|
+
_decoder = msgspec.json.Decoder(ResumeMeta)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def read_meta(path: Path) -> ResumeMeta | None:
|
|
35
|
+
"""Return the sidecar meta at *path*, or ``None`` if missing/corrupt."""
|
|
36
|
+
try:
|
|
37
|
+
return _decoder.decode(path.read_bytes())
|
|
38
|
+
except (OSError, msgspec.DecodeError):
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def write_meta(path: Path, meta: ResumeMeta) -> None:
|
|
43
|
+
path.write_bytes(_encoder.encode(meta))
|
|
@@ -15,6 +15,8 @@ from hashlib import sha256
|
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
from typing import Any
|
|
17
17
|
|
|
18
|
+
from asanypath.exceptions import UnsupportedProtocolError
|
|
19
|
+
|
|
18
20
|
_CHECKSUM_PREFERENCE = ("sha256", "sha1", "md5")
|
|
19
21
|
|
|
20
22
|
# Default streaming chunk size for chunk_size=None ("auto").
|
|
@@ -79,19 +81,25 @@ def _normalized_checksums(checksums: dict[str, str]) -> dict[str, str]:
|
|
|
79
81
|
def is_remote_destination(dst: Any) -> bool:
|
|
80
82
|
protocol = getattr(dst, "protocol", None)
|
|
81
83
|
if protocol is not None:
|
|
82
|
-
return protocol != "file"
|
|
84
|
+
return bool(protocol != "file")
|
|
83
85
|
value = str(dst)
|
|
84
86
|
return "://" in value and not value.startswith("file:")
|
|
85
87
|
|
|
86
88
|
|
|
87
89
|
def has_async_transfer_api(dst: Any) -> bool:
|
|
88
|
-
|
|
89
|
-
getattr(dst, "exists", None)
|
|
90
|
-
|
|
90
|
+
try:
|
|
91
|
+
exists = getattr(dst, "exists", None)
|
|
92
|
+
write_bytes = getattr(dst, "write_bytes", None)
|
|
93
|
+
except UnsupportedProtocolError:
|
|
94
|
+
return False
|
|
95
|
+
return inspect.iscoroutinefunction(exists) and inspect.iscoroutinefunction(write_bytes)
|
|
91
96
|
|
|
92
97
|
|
|
93
98
|
def has_write_api(dst: Any) -> bool:
|
|
94
|
-
|
|
99
|
+
try:
|
|
100
|
+
return callable(getattr(dst, "write_bytes", None))
|
|
101
|
+
except UnsupportedProtocolError:
|
|
102
|
+
return False
|
|
95
103
|
|
|
96
104
|
|
|
97
105
|
def run_sync_maybe(value: Any) -> Any:
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.
|
|
22
|
-
__version_tuple__ = version_tuple = (2,
|
|
21
|
+
__version__ = version = '2.2.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 2, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|