asanypath 2.2.0__tar.gz → 2.3.0__tar.gz

{asanypath-2.2.0 → asanypath-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: asanypath
-Version: 2.2.0
+Version: 2.3.0
 Summary: Async pathlib for local and cloud storage
 License-Expression: MIT
 License-File: LICENSE
@@ -19,12 +19,11 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: anyio>=4
-Requires-Dist: asanypath-native~=2.2.0
+Requires-Dist: asanypath-native~=2.3.0
 Requires-Dist: msgspec>=0.22.0
 Requires-Dist: yarl>=1.2
 Provides-Extra: all
 Requires-Dist: aioftp>=0.21; extra == 'all'
-Requires-Dist: diskcache>=5.6; extra == 'all'
 Requires-Dist: keyring>=24; extra == 'all'
 Provides-Extra: bench
 Requires-Dist: adlfs; extra == 'bench'
@@ -38,15 +37,12 @@ Requires-Dist: gcloud-aio-storage; extra == 'bench'
 Requires-Dist: gcsfs; extra == 'bench'
 Requires-Dist: requests; extra == 'bench'
 Requires-Dist: s3fs; extra == 'bench'
-Provides-Extra: cache
-Requires-Dist: diskcache>=5.6; extra == 'cache'
 Provides-Extra: cli
 Requires-Dist: rich-click>=1.9.7; extra == 'cli'
 Provides-Extra: credstore
 Requires-Dist: keyring>=24; extra == 'credstore'
 Provides-Extra: dev
 Requires-Dist: aioftp>=0.21; extra == 'dev'
-Requires-Dist: diskcache>=5.6; extra == 'dev'
 Requires-Dist: keyring>=24; extra == 'dev'
 Requires-Dist: pre-commit>=4; extra == 'dev'
 Requires-Dist: pytest-asyncio>=1.3; extra == 'dev'
@@ -149,6 +145,46 @@ await src.copy(dst, recursive=True)
 await path.rmdir(recursive=True)
 ```
 
+Copies automatically use spare transfer concurrency for large ranged reads.
+The default target is 8 streams, with an 8 MiB minimum range size and at most
+8 range streams per file. Files smaller than 16 MiB stay single-stream. When
+enough files are active to meet the target, automatic copies use one request
+per file instead of adding ranges. The target is best-effort, not a minimum
+number of requests that must always be launched.
+
+S3, GCS, Azure, Artifactory, SSH, and range-capable HTTP/HTTPS sources use the
+shared reader for local downloads and source reads in cross-backend copies.
+FTP/FTPS, sources without size metadata, and HTTP servers without advertised
+range support retain single-stream downloads. Local-only copies and native
+server-side copy/rename operations retain their existing implementations.
+Uploads remain single writes; this does not introduce multipart uploads.
+
+Concurrent library copies share a default budget within the event loop. To
+configure a batch explicitly:
+
+```python
+import asyncio
+from asanypath import TransferBudget
+
+with TransferBudget(floor=8, ceiling=32, min_range_size=8 * 1024 * 1024, per_file_cap=8):
+    await asyncio.gather(*(src.copy(dst) for src, dst in pairs))
+```
+
+The ceiling bounds concurrent reads in this budget, not metadata requests or
+other processes. Parallel range buffers are bounded by the active streams
+times the range size; cross-backend destinations still buffer the complete
+object for their existing write API. `floor=1` disables automatic splitting.
+Explicit `chunk_size` requests retain sequential streaming when splitting
+would be inappropriate, and `chunk_size=0` disables streaming to local paths.
+
+S3/Azure ETags, GCS generations, and HTTP conditional requests pin the source
+version when available. Other sources are checked against fresh metadata
+after the download, which cannot guarantee a snapshot if the backend does
+not expose a reliable change token. Use `atomic=True` to keep an existing
+local destination intact on failure. Moves remove the source only after a
+successful copy. The updated Python package requires the matching native
+extension for conditional cloud reads.
+
 For synchronous code (no `await` needed):
 
 ```python
@@ -219,9 +255,8 @@ Reading the same object or URL twice always re-downloads it — there is **no
 cache by default**, and the read path is unchanged unless you opt in, so
 non-caching callers pay zero overhead.
 
-Enable it with the optional `cache` extra (`pip install asanypath[cache]`,
-backed by [`diskcache`](https://pypi.org/project/diskcache/) — a pure-Python,
-zero-dependency store). Opt in per read with `use_cache=True`:
+Caching is included in the native extension with no extra dependencies or
+installation extras. Opt in per read with `use_cache=True`:
 
 ```python
 from asanypath import AsAnyPath
@@ -240,7 +275,7 @@ from asanypath import configure_cache
 
 configure_cache(
     directory="/var/cache/asanypath",  # default: ~/.cache/asanypath
-    size_limit=2 * 1024**3,            # total on-disk cap in bytes (default 1 GiB, LRU)
+    size_limit=2 * 1024**3,            # stored key + payload budget (default 1 GiB, LRU)
     ttl=3600,                          # per-entry expiry in seconds (default: none)
     enabled=True,                      # make use_cache default to True process-wide
 )
@@ -255,9 +290,15 @@ Behaviour and trade-offs:
   default flips process-wide; an explicit `use_cache=False` always bypasses.
 - **Works for every remote backend** (S3, GCS, Azure, Artifactory, HTTP, SSH,
   FTP). Local paths are never cached.
-- **Total-size bounded, LRU-evicted;** disk I/O runs in a worker thread so the
+- **Size-bounded, LRU-evicted;** disk I/O runs in a worker thread so the
   event loop stays responsive. Concurrent reads of the same key collapse to a
-  single fetch.
+  single fetch within a cache instance. The byte budget counts UTF-8 keys and
+  payloads; SQLite metadata and transaction journals add filesystem overhead.
+  Objects larger than the budget are returned without being cached.
+- **Byte-only native storage:** SQLite stores raw bytes, never Python pickles.
+  The database lives under `native-v1/` in the configured cache directory.
+  Legacy DiskCache files are ignored, not migrated or deleted. Multiple
+  processes can share the new store; use the same size limit across them.
 - **Staleness is on you.** Entries are keyed by URL with no revalidation — a
   cached object that changes upstream is served stale until it expires (`ttl`)
   or is evicted. Prefer caching immutable/versioned objects, or set a `ttl`.
@@ -356,6 +397,12 @@ asanypath cp /local/file.txt s3://bucket/file.txt
 # Copy cloud → local
 asanypath cp s3://bucket/file.txt /local/file.txt
 
+# Allow 32 concurrent files; fill spare capacity toward 8 streams for large files
+asanypath cp -r s3://bucket/data/ ./data/ -j 32 --concurrency-floor 8
+
+# Disable automatic range splitting (also supported by mv and sync)
+asanypath cp s3://bucket/file.txt ./file.txt --concurrency-floor 1
+
 # Delete a file
 asanypath rm s3://bucket/file.txt --force
 

{asanypath-2.2.0 → asanypath-2.3.0}/README.md

@@ -90,6 +90,46 @@ await src.copy(dst, recursive=True)
 await path.rmdir(recursive=True)
 ```
 
+Copies automatically use spare transfer concurrency for large ranged reads.
+The default target is 8 streams, with an 8 MiB minimum range size and at most
+8 range streams per file. Files smaller than 16 MiB stay single-stream. When
+enough files are active to meet the target, automatic copies use one request
+per file instead of adding ranges. The target is best-effort, not a minimum
+number of requests that must always be launched.
+
+S3, GCS, Azure, Artifactory, SSH, and range-capable HTTP/HTTPS sources use the
+shared reader for local downloads and source reads in cross-backend copies.
+FTP/FTPS, sources without size metadata, and HTTP servers without advertised
+range support retain single-stream downloads. Local-only copies and native
+server-side copy/rename operations retain their existing implementations.
+Uploads remain single writes; this does not introduce multipart uploads.
+
+Concurrent library copies share a default budget within the event loop. To
+configure a batch explicitly:
+
+```python
+import asyncio
+from asanypath import TransferBudget
+
+with TransferBudget(floor=8, ceiling=32, min_range_size=8 * 1024 * 1024, per_file_cap=8):
+    await asyncio.gather(*(src.copy(dst) for src, dst in pairs))
+```
+
+The ceiling bounds concurrent reads in this budget, not metadata requests or
+other processes. Parallel range buffers are bounded by the active streams
+times the range size; cross-backend destinations still buffer the complete
+object for their existing write API. `floor=1` disables automatic splitting.
+Explicit `chunk_size` requests retain sequential streaming when splitting
+would be inappropriate, and `chunk_size=0` disables streaming to local paths.
+
+S3/Azure ETags, GCS generations, and HTTP conditional requests pin the source
+version when available. Other sources are checked against fresh metadata
+after the download, which cannot guarantee a snapshot if the backend does
+not expose a reliable change token. Use `atomic=True` to keep an existing
+local destination intact on failure. Moves remove the source only after a
+successful copy. The updated Python package requires the matching native
+extension for conditional cloud reads.
+
 For synchronous code (no `await` needed):
 
 ```python
@@ -160,9 +200,8 @@ Reading the same object or URL twice always re-downloads it — there is **no
 cache by default**, and the read path is unchanged unless you opt in, so
 non-caching callers pay zero overhead.
 
-Enable it with the optional `cache` extra (`pip install asanypath[cache]`,
-backed by [`diskcache`](https://pypi.org/project/diskcache/) — a pure-Python,
-zero-dependency store). Opt in per read with `use_cache=True`:
+Caching is included in the native extension with no extra dependencies or
+installation extras. Opt in per read with `use_cache=True`:
 
 ```python
 from asanypath import AsAnyPath
@@ -181,7 +220,7 @@ from asanypath import configure_cache
 
 configure_cache(
     directory="/var/cache/asanypath",  # default: ~/.cache/asanypath
-    size_limit=2 * 1024**3,            # total on-disk cap in bytes (default 1 GiB, LRU)
+    size_limit=2 * 1024**3,            # stored key + payload budget (default 1 GiB, LRU)
     ttl=3600,                          # per-entry expiry in seconds (default: none)
     enabled=True,                      # make use_cache default to True process-wide
 )
@@ -196,9 +235,15 @@ Behaviour and trade-offs:
   default flips process-wide; an explicit `use_cache=False` always bypasses.
 - **Works for every remote backend** (S3, GCS, Azure, Artifactory, HTTP, SSH,
   FTP). Local paths are never cached.
-- **Total-size bounded, LRU-evicted;** disk I/O runs in a worker thread so the
+- **Size-bounded, LRU-evicted;** disk I/O runs in a worker thread so the
   event loop stays responsive. Concurrent reads of the same key collapse to a
-  single fetch.
+  single fetch within a cache instance. The byte budget counts UTF-8 keys and
+  payloads; SQLite metadata and transaction journals add filesystem overhead.
+  Objects larger than the budget are returned without being cached.
+- **Byte-only native storage:** SQLite stores raw bytes, never Python pickles.
+  The database lives under `native-v1/` in the configured cache directory.
+  Legacy DiskCache files are ignored, not migrated or deleted. Multiple
+  processes can share the new store; use the same size limit across them.
 - **Staleness is on you.** Entries are keyed by URL with no revalidation — a
   cached object that changes upstream is served stale until it expires (`ttl`)
   or is evicted. Prefer caching immutable/versioned objects, or set a `ttl`.
@@ -297,6 +342,12 @@ asanypath cp /local/file.txt s3://bucket/file.txt
 # Copy cloud → local
 asanypath cp s3://bucket/file.txt /local/file.txt
 
+# Allow 32 concurrent files; fill spare capacity toward 8 streams for large files
+asanypath cp -r s3://bucket/data/ ./data/ -j 32 --concurrency-floor 8
+
+# Disable automatic range splitting (also supported by mv and sync)
+asanypath cp s3://bucket/file.txt ./file.txt --concurrency-floor 1
+
 # Delete a file
 asanypath rm s3://bucket/file.txt --force
 

{asanypath-2.2.0 → asanypath-2.3.0}/pyproject.toml

@@ -58,7 +58,6 @@ dev = [
 cli = ["rich-click>=1.9.7"]
 ftp = ["aioftp>=0.21"]
 credstore = ["keyring>=24"]
-cache = ["diskcache>=5.6"]
 bench = [
     "asyncssh>=2.14",
     "fsspec",
@@ -72,7 +71,7 @@ bench = [
     "aiofiles",
     "requests",
 ]
-all = ["asanypath[credstore,ftp,cache]"]
+all = ["asanypath[credstore,ftp]"]
 [project.scripts]
 asanypath = "asanypath.cli:main"
 
@@ -151,10 +150,6 @@ disable_error_code = ["no-any-return", "misc", "union-attr", "type-var", "attr-d
 module = "asanypath.batcher"
 disallow_untyped_defs = false
 
-[[tool.mypy.overrides]]
-module = "diskcache.*"
-ignore_missing_imports = true
-
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 testpaths = ["tests"]

{asanypath-2.2.0 → asanypath-2.3.0}/src/asanypath/__init__.py

@@ -4,6 +4,7 @@
 # license that can be found in the LICENSE file or at
 # https://opensource.org/licenses/MIT.
 
+from asanypath._transfer import TransferBudget
 from asanypath.artifactory import ArtifactoryPath
 from asanypath.asanypath import (
     PROTOCOL_MAP,
@@ -53,6 +54,7 @@ __all__ = [
     "AsAnyPath",
     "AsyncPath",
     "SyncPath",
+    "TransferBudget",
     "BackendOptions",
     "AccessGrant",
     "AccessPolicy",

{asanypath-2.2.0 → asanypath-2.3.0}/src/asanypath/__init__.pyi

@@ -6,6 +6,7 @@
 
 from __future__ import annotations
 
+from ._transfer import TransferBudget as TransferBudget
 from ._version import __version__ as __version__
 from .artifactory import ArtifactoryPath as ArtifactoryPath
 from .asanypath import (

asanypath-2.3.0/src/asanypath/_backend_opts.py

@@ -0,0 +1,61 @@
+"""CLI backend options: ``scheme.key=value`` mapped onto path constructor kwargs."""
+
+from __future__ import annotations
+
+import inspect
+import re
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from asanypath.asanypath import PROTOCOL_MAP, _match_base_url_flavor
+
+Options = dict[str, dict[str, Any]]
+
+
+def option_keys(cls: type) -> dict[str, Callable[[str], Any]]:
+    """Return ``{key: coercer}`` for the keyword arguments a path class accepts."""
+    keys: dict[str, Callable[[str], Any]] = {}
+    for name, param in inspect.signature(cls.__init__).parameters.items():
+        if name == "self" or param.kind in (
+            param.VAR_POSITIONAL,
+            param.VAR_KEYWORD,
+            param.POSITIONAL_ONLY,
+        ):
+            continue
+        ann = str(param.annotation)
+        keys[name] = int if re.search(r"\bint\b", ann) and "str" not in ann else str
+    return keys
+
+
+def parse_options(specs: Iterable[str]) -> Options:
+    """Parse ``scheme.key=value`` specs; raise ValueError on anything unsupported."""
+    parsed: Options = {}
+    for spec in specs:
+        lhs, eq, value = spec.partition("=")
+        scheme, dot, key = lhs.partition(".")
+        if not eq or not dot or not key:
+            raise ValueError(f"invalid option {spec!r}; expected SCHEME.KEY=VALUE")
+        scheme = scheme.lower()
+        if scheme not in PROTOCOL_MAP:
+            raise ValueError(f"unknown protocol {scheme!r} in option {spec!r}")
+        keys = option_keys(PROTOCOL_MAP[scheme])
+        if key not in keys:
+            valid = ", ".join(sorted(keys)) or "none"
+            raise ValueError(f"{scheme} does not support option {key!r} (valid: {valid})")
+        try:
+            parsed.setdefault(scheme, {})[key] = keys[key](value)
+        except ValueError:
+            raise ValueError(f"invalid value {value!r} for {scheme}.{key}") from None
+    return parsed
+
+
+def kwargs_for(path: str, options: Options) -> dict[str, Any]:
+    """Return the options that apply to the backend serving ``path``."""
+    if not options:
+        return {}
+    flavor = _match_base_url_flavor(path)
+    merged: dict[str, Any] = {}
+    for scheme, values in options.items():
+        if PROTOCOL_MAP[scheme] is flavor:
+            merged.update(values)
+    return merged