asanypath 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- asanypath/__init__.py +81 -0
- asanypath/__init__.pyi +62 -0
- asanypath/_credstore.py +102 -0
- asanypath/_transfer.py +216 -0
- asanypath/_version.py +24 -0
- asanypath/artifactory.py +366 -0
- asanypath/asanypath.py +556 -0
- asanypath/asanypath.pyi +225 -0
- asanypath/azure.py +274 -0
- asanypath/batcher.py +206 -0
- asanypath/cli.py +2566 -0
- asanypath/cloud.py +1042 -0
- asanypath/common.py +355 -0
- asanypath/exceptions.py +43 -0
- asanypath/ftp.py +543 -0
- asanypath/gcs.py +253 -0
- asanypath/http.py +516 -0
- asanypath/local.py +817 -0
- asanypath/options.py +57 -0
- asanypath/s3.py +277 -0
- asanypath/ssh.py +776 -0
- asanypath/sync.py +220 -0
- asanypath/unsupported.py +202 -0
- asanypath-2.0.0.dist-info/METADATA +591 -0
- asanypath-2.0.0.dist-info/RECORD +28 -0
- asanypath-2.0.0.dist-info/WHEEL +4 -0
- asanypath-2.0.0.dist-info/entry_points.txt +2 -0
- asanypath-2.0.0.dist-info/licenses/LICENSE +21 -0
asanypath/asanypath.pyi
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Copyright (C) 2026 Axis Communications AB, Lund, Sweden
|
|
2
|
+
#
|
|
3
|
+
# Use of this source code is governed by an MIT-style
|
|
4
|
+
# license that can be found in the LICENSE file or at
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
10
|
+
from os import PathLike, stat_result
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Self, TypeAlias
|
|
13
|
+
|
|
14
|
+
from anyio import AsyncFile
|
|
15
|
+
from anyio import Path as AioPath
|
|
16
|
+
|
|
17
|
+
class BackendOptions:
|
|
18
|
+
headers: dict[str, str]
|
|
19
|
+
query: dict[str, str]
|
|
20
|
+
provider: dict[str, Any]
|
|
21
|
+
|
|
22
|
+
class AccessGrant:
|
|
23
|
+
principal: str
|
|
24
|
+
actions: frozenset[str]
|
|
25
|
+
def __init__(self, principal: str, actions: frozenset[str]) -> None: ...
|
|
26
|
+
|
|
27
|
+
class AccessPolicy:
|
|
28
|
+
owner: str | None
|
|
29
|
+
group: str | None
|
|
30
|
+
grants: tuple[AccessGrant, ...]
|
|
31
|
+
provider: dict[str, Any]
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
*,
|
|
35
|
+
owner: str | None = None,
|
|
36
|
+
group: str | None = None,
|
|
37
|
+
grants: tuple[AccessGrant, ...] = (),
|
|
38
|
+
provider: dict[str, Any] = {},
|
|
39
|
+
) -> None: ...
|
|
40
|
+
|
|
41
|
+
class AccessPolicyPatch:
|
|
42
|
+
owner: str | None
|
|
43
|
+
group: str | None
|
|
44
|
+
grants: tuple[AccessGrant, ...]
|
|
45
|
+
provider: dict[str, Any]
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
*,
|
|
49
|
+
owner: str | None = None,
|
|
50
|
+
group: str | None = None,
|
|
51
|
+
grants: tuple[AccessGrant, ...] = (),
|
|
52
|
+
provider: dict[str, Any] = {},
|
|
53
|
+
) -> None: ...
|
|
54
|
+
|
|
55
|
+
PathPart: TypeAlias = str | Path | AioPath | PathLike[str] | PathLike[bytes] | AsAnyPurePath
|
|
56
|
+
|
|
57
|
+
PROTOCOL_MAP: dict[str, type[AsAnyPurePath]]
|
|
58
|
+
|
|
59
|
+
class AsAnyPurePath:
|
|
60
|
+
def __new__(cls, arg: PathPart = ".", *args: PathPart, **kwargs: Any) -> Self: ...
|
|
61
|
+
def __rtruediv__(self, other: str | PathLike[str]) -> Self: ...
|
|
62
|
+
def __truediv__(self, other: str | PathLike[str]) -> Self: ...
|
|
63
|
+
@property
|
|
64
|
+
def anchor(self) -> str: ...
|
|
65
|
+
def as_posix(self) -> str: ...
|
|
66
|
+
def as_uri(self) -> str: ...
|
|
67
|
+
@property
|
|
68
|
+
def drive(self) -> str: ...
|
|
69
|
+
def full_match(self, pattern: str, *, case_sensitive: bool | None = None) -> bool: ...
|
|
70
|
+
def is_absolute(self) -> bool: ...
|
|
71
|
+
def is_relative_to(self, other: str | PathLike[str]) -> bool: ...
|
|
72
|
+
def joinpath(self, *args: str | PathLike[str]) -> Self: ...
|
|
73
|
+
def match(self, pattern: str, *, case_sensitive: bool | None = None) -> bool: ...
|
|
74
|
+
@property
|
|
75
|
+
def name(self) -> str: ...
|
|
76
|
+
@property
|
|
77
|
+
def parent(self) -> Self: ...
|
|
78
|
+
@property
|
|
79
|
+
def parents(self) -> tuple[Self, ...]: ...
|
|
80
|
+
@property
|
|
81
|
+
def parts(self) -> tuple[str, ...]: ...
|
|
82
|
+
def relative_to(self, *other: str | PathLike[str], walk_up: bool = False) -> Self: ...
|
|
83
|
+
@property
|
|
84
|
+
def root(self) -> str: ...
|
|
85
|
+
@property
|
|
86
|
+
def stem(self) -> str: ...
|
|
87
|
+
@property
|
|
88
|
+
def suffix(self) -> str: ...
|
|
89
|
+
@property
|
|
90
|
+
def suffixes(self) -> list[str]: ...
|
|
91
|
+
def with_name(self, name: str) -> Self: ...
|
|
92
|
+
def with_parent(self, parent: str | PathLike[str] | Self) -> Self: ...
|
|
93
|
+
def with_segments(self, *pathsegments: str | PathLike[str] | Self) -> Self: ...
|
|
94
|
+
def with_stem(self, stem: str) -> Self: ...
|
|
95
|
+
def with_suffix(self, suffix: str) -> Self: ...
|
|
96
|
+
|
|
97
|
+
class AsAnyPath(AsAnyPurePath):
|
|
98
|
+
def __new__(cls, arg: PathPart = ".", *args: PathPart, **kwargs: Any) -> Self: ...
|
|
99
|
+
def __bytes__(self) -> bytes: ...
|
|
100
|
+
def __fspath__(self) -> str: ...
|
|
101
|
+
def absolute(self) -> Self: ...
|
|
102
|
+
def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: ...
|
|
103
|
+
@classmethod
|
|
104
|
+
def cwd(cls) -> Self: ...
|
|
105
|
+
async def exists(self) -> bool: ...
|
|
106
|
+
def expanduser(self) -> Self: ...
|
|
107
|
+
async def get_access_policy(
|
|
108
|
+
self, *, backend_options: BackendOptions | None = None
|
|
109
|
+
) -> AccessPolicy: ...
|
|
110
|
+
async def glob(
|
|
111
|
+
self, pattern: str, *, case_sensitive: bool | None = None
|
|
112
|
+
) -> AsyncIterator[Self]: ...
|
|
113
|
+
def group(self) -> str: ...
|
|
114
|
+
def hardlink_to(self, target: str | bytes | PathLike[str] | PathLike[bytes]) -> None: ...
|
|
115
|
+
@classmethod
|
|
116
|
+
def home(cls) -> Self: ...
|
|
117
|
+
def is_block_device(self) -> bool: ...
|
|
118
|
+
def is_char_device(self) -> bool: ...
|
|
119
|
+
async def is_dir(self) -> bool: ...
|
|
120
|
+
def is_fifo(self) -> bool: ...
|
|
121
|
+
async def is_file(self) -> bool: ...
|
|
122
|
+
def is_junction(self) -> bool: ...
|
|
123
|
+
def is_mount(self) -> bool: ...
|
|
124
|
+
def is_socket(self) -> bool: ...
|
|
125
|
+
def is_symlink(self) -> bool: ...
|
|
126
|
+
async def iterdir(self) -> AsyncIterator[Self]: ...
|
|
127
|
+
def lchmod(self, mode: int) -> None: ...
|
|
128
|
+
def lstat(self) -> stat_result: ...
|
|
129
|
+
async def mkdir(
|
|
130
|
+
self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False
|
|
131
|
+
) -> None: ...
|
|
132
|
+
async def copy(
|
|
133
|
+
self,
|
|
134
|
+
dst: PathPart,
|
|
135
|
+
*,
|
|
136
|
+
recursive: bool = False,
|
|
137
|
+
remove_src: bool = False,
|
|
138
|
+
force: bool = False,
|
|
139
|
+
atomic: bool = False,
|
|
140
|
+
chunk_size: int | None = None,
|
|
141
|
+
destination_backend_options: BackendOptions | None = None,
|
|
142
|
+
on_progress: Callable[[int, int, Self, Self], None | Awaitable[None]] | None = None,
|
|
143
|
+
on_done: Callable[[Self, Self, int, bool, BaseException | None], None | Awaitable[None]]
|
|
144
|
+
| None = None,
|
|
145
|
+
) -> Self: ...
|
|
146
|
+
async def move(
|
|
147
|
+
self,
|
|
148
|
+
target: PathPart,
|
|
149
|
+
*,
|
|
150
|
+
force: bool = False,
|
|
151
|
+
atomic: bool = False,
|
|
152
|
+
chunk_size: int | None = None,
|
|
153
|
+
destination_backend_options: BackendOptions | None = None,
|
|
154
|
+
) -> Self: ...
|
|
155
|
+
def open(
|
|
156
|
+
self,
|
|
157
|
+
mode: str = "r",
|
|
158
|
+
buffering: int = -1,
|
|
159
|
+
encoding: str | None = None,
|
|
160
|
+
errors: str | None = None,
|
|
161
|
+
newline: str | None = None,
|
|
162
|
+
*,
|
|
163
|
+
backend_options: BackendOptions | None = None,
|
|
164
|
+
) -> AsyncFile[Any]: ...
|
|
165
|
+
def owner(self) -> str: ...
|
|
166
|
+
async def read_bytes(self) -> bytes: ...
|
|
167
|
+
def iter_bytes(self, chunk_size: int | None = None) -> AsyncIterator[bytes]: ...
|
|
168
|
+
async def read_text(
|
|
169
|
+
self, encoding: str | None = None, errors: str | None = None, newline: str | None = None
|
|
170
|
+
) -> str: ...
|
|
171
|
+
def readlink(self) -> Self: ...
|
|
172
|
+
async def rename(
|
|
173
|
+
self,
|
|
174
|
+
target: PathPart,
|
|
175
|
+
*,
|
|
176
|
+
force: bool = False,
|
|
177
|
+
atomic: bool = False,
|
|
178
|
+
chunk_size: int | None = None,
|
|
179
|
+
destination_backend_options: BackendOptions | None = None,
|
|
180
|
+
) -> Self: ...
|
|
181
|
+
async def replace(
|
|
182
|
+
self, target: PathPart, *, destination_backend_options: BackendOptions | None = None
|
|
183
|
+
) -> Self: ...
|
|
184
|
+
def resolve(self, strict: bool = False) -> Self: ...
|
|
185
|
+
async def rglob(
|
|
186
|
+
self, pattern: str, *, case_sensitive: bool | None = None
|
|
187
|
+
) -> AsyncIterator[Self]: ...
|
|
188
|
+
async def rmdir(self) -> None: ...
|
|
189
|
+
def samefile(self, other_path: str | PathLike[str]) -> bool: ...
|
|
190
|
+
async def stat(self, *, follow_symlinks: bool = True) -> stat_result: ...
|
|
191
|
+
def symlink_to(
|
|
192
|
+
self,
|
|
193
|
+
target: str | bytes | PathLike[str] | PathLike[bytes] | Self,
|
|
194
|
+
target_is_directory: bool = False,
|
|
195
|
+
) -> None: ...
|
|
196
|
+
async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: ...
|
|
197
|
+
async def unlink(self, missing_ok: bool = False) -> None: ...
|
|
198
|
+
async def update_access_policy(
|
|
199
|
+
self,
|
|
200
|
+
policy_patch: AccessPolicyPatch,
|
|
201
|
+
*,
|
|
202
|
+
backend_options: BackendOptions | None = None,
|
|
203
|
+
) -> None: ...
|
|
204
|
+
async def walk(
|
|
205
|
+
self,
|
|
206
|
+
top_down: bool = True,
|
|
207
|
+
on_error: Callable[[OSError], object] | None = None,
|
|
208
|
+
follow_symlinks: bool = False,
|
|
209
|
+
) -> AsyncIterator[tuple[Self, list[str], list[str]]]: ...
|
|
210
|
+
async def write_bytes(
|
|
211
|
+
self, data: bytes, *, backend_options: BackendOptions | None = None
|
|
212
|
+
) -> int: ...
|
|
213
|
+
async def write_text(
|
|
214
|
+
self,
|
|
215
|
+
data: str,
|
|
216
|
+
encoding: str | None = None,
|
|
217
|
+
errors: str | None = None,
|
|
218
|
+
newline: str | None = None,
|
|
219
|
+
*,
|
|
220
|
+
backend_options: BackendOptions | None = None,
|
|
221
|
+
) -> int: ...
|
|
222
|
+
async def checksums(self) -> dict[str, str]: ...
|
|
223
|
+
|
|
224
|
+
def register_protocol(protocol: str, path_class: type[AsAnyPurePath]) -> None: ...
|
|
225
|
+
def list_protocols() -> list[str]: ...
|
asanypath/azure.py
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
# Copyright (C) 2026 Axis Communications AB, Lund, Sweden
|
|
2
|
+
#
|
|
3
|
+
# Use of this source code is governed by an MIT-style
|
|
4
|
+
# license that can be found in the LICENSE file or at
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
"""Azure Blob Storage path implementation (HTTP-style adapter).
|
|
8
|
+
|
|
9
|
+
Uses the Azure Blob Storage REST API:
|
|
10
|
+
https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
from os import getenv
|
|
18
|
+
from types import SimpleNamespace
|
|
19
|
+
|
|
20
|
+
import msgspec
|
|
21
|
+
|
|
22
|
+
from asanypath.cloud import CloudPathMixin
|
|
23
|
+
from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
|
|
24
|
+
from asanypath_native import (
|
|
25
|
+
az_copy_batch,
|
|
26
|
+
az_delete_batch,
|
|
27
|
+
az_exists_batch,
|
|
28
|
+
az_get_batch,
|
|
29
|
+
az_get_container_acl,
|
|
30
|
+
az_head_batch,
|
|
31
|
+
az_is_dir,
|
|
32
|
+
az_is_dir_batch,
|
|
33
|
+
az_list_batch,
|
|
34
|
+
az_list_containers,
|
|
35
|
+
az_presign,
|
|
36
|
+
az_put_batch,
|
|
37
|
+
az_put_container_acl,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
AZURE_API_VERSION = "2023-11-03"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _parse_connection_string(cs: str | None) -> dict[str, str]:
|
|
44
|
+
"""Parse an Azure Storage connection string into asanypath config keys."""
|
|
45
|
+
if not cs:
|
|
46
|
+
return {}
|
|
47
|
+
parts: dict[str, str] = {}
|
|
48
|
+
for seg in cs.split(";"):
|
|
49
|
+
key, sep, val = seg.partition("=")
|
|
50
|
+
if sep and key.strip():
|
|
51
|
+
parts[key.strip().lower()] = val.strip()
|
|
52
|
+
out: dict[str, str] = {}
|
|
53
|
+
if name := parts.get("accountname"):
|
|
54
|
+
out["account_name"] = name
|
|
55
|
+
if key := parts.get("accountkey"):
|
|
56
|
+
out["account_key"] = key
|
|
57
|
+
if sas := parts.get("sharedaccesssignature"):
|
|
58
|
+
out["sas_token"] = sas
|
|
59
|
+
if endpoint := parts.get("blobendpoint"):
|
|
60
|
+
out["endpoint_url"] = endpoint
|
|
61
|
+
elif name := parts.get("accountname"):
|
|
62
|
+
proto = parts.get("defaultendpointsprotocol", "https")
|
|
63
|
+
suffix = parts.get("endpointsuffix", "core.windows.net")
|
|
64
|
+
out["endpoint_url"] = f"{proto}://{name}.blob.{suffix}"
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class AzurePath(CloudPathMixin):
|
|
69
|
+
"""Async path implementation for Azure Blob Storage.
|
|
70
|
+
|
|
71
|
+
Uses the Azure Blob Storage REST API via the shared-key or SAS authentication.
|
|
72
|
+
Supports: exists, is_file, is_dir, iterdir, read/write, unlink, touch, rename, stat.
|
|
73
|
+
|
|
74
|
+
Example:
|
|
75
|
+
```python
|
|
76
|
+
path = AzurePath("az://container/blob/path.txt",
|
|
77
|
+
account_name="myaccount",
|
|
78
|
+
account_key="base64-encoded-key")
|
|
79
|
+
content = await path.read_bytes()
|
|
80
|
+
```
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
protocol: str = "az"
|
|
84
|
+
_supports_range_read: bool = True
|
|
85
|
+
_is_dir_batch_fn = staticmethod(az_is_dir_batch)
|
|
86
|
+
_copy_batch_fn = staticmethod(az_copy_batch)
|
|
87
|
+
|
|
88
|
+
@staticmethod
|
|
89
|
+
def _batcher_key_fn(
|
|
90
|
+
endpoint: str,
|
|
91
|
+
container: str,
|
|
92
|
+
account_name: str = "",
|
|
93
|
+
account_key: str | None = None,
|
|
94
|
+
sas_token: str | None = None,
|
|
95
|
+
) -> str:
|
|
96
|
+
return f"{endpoint}|{container}|{account_name}"
|
|
97
|
+
|
|
98
|
+
_batcher_ops = {
|
|
99
|
+
"get": {"batch_fn": az_get_batch, "items_key": "blob_paths", "wrap": bytes},
|
|
100
|
+
"exists": {"batch_fn": az_exists_batch, "items_key": "blob_paths"},
|
|
101
|
+
"head": {"batch_fn": az_head_batch, "items_key": "blob_paths"},
|
|
102
|
+
"delete": {"batch_fn": az_delete_batch, "items_key": "blob_paths"},
|
|
103
|
+
"put": {"batch_fn": az_put_batch},
|
|
104
|
+
"list": {"batch_fn": az_list_batch, "items_key": "prefixes"},
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
def __init__(
|
|
108
|
+
self,
|
|
109
|
+
*args,
|
|
110
|
+
endpoint_url: str | None = None,
|
|
111
|
+
account_name: str | None = None,
|
|
112
|
+
account_key: str | None = None,
|
|
113
|
+
sas_token: str | None = None,
|
|
114
|
+
**kwargs,
|
|
115
|
+
):
|
|
116
|
+
super().__init__(*args, **kwargs)
|
|
117
|
+
cfg = self.env_config
|
|
118
|
+
self._account_name = account_name or cfg.account_name
|
|
119
|
+
self._account_key = account_key or cfg.account_key
|
|
120
|
+
self._sas_token = sas_token or cfg.sas_token
|
|
121
|
+
self._endpoint_url = (
|
|
122
|
+
endpoint_url
|
|
123
|
+
or cfg.endpoint_url
|
|
124
|
+
or (f"https://{self._account_name}.blob.core.windows.net")
|
|
125
|
+
)
|
|
126
|
+
# Promote explicit params into class cache so derived instances
|
|
127
|
+
# (parent, /, with_name, etc.) inherit them.
|
|
128
|
+
if account_name:
|
|
129
|
+
cfg.account_name = account_name
|
|
130
|
+
if account_key:
|
|
131
|
+
cfg.account_key = account_key
|
|
132
|
+
if sas_token:
|
|
133
|
+
cfg.sas_token = sas_token
|
|
134
|
+
if endpoint_url:
|
|
135
|
+
cfg.endpoint_url = endpoint_url
|
|
136
|
+
|
|
137
|
+
# URL format: az://container/blob/path
|
|
138
|
+
self._container = self._path.host or ""
|
|
139
|
+
self._blob_path = self._path.path.lstrip("/")
|
|
140
|
+
|
|
141
|
+
@classmethod
|
|
142
|
+
def _create_env_config(cls) -> SimpleNamespace:
|
|
143
|
+
"""Build env config from Azure environment variables / connection string."""
|
|
144
|
+
conn = _parse_connection_string(getenv("AZURE_STORAGE_CONNECTION_STRING"))
|
|
145
|
+
account_name = getenv("AZURE_STORAGE_ACCOUNT") or conn.get("account_name")
|
|
146
|
+
return SimpleNamespace(
|
|
147
|
+
account_name=account_name,
|
|
148
|
+
account_key=getenv("AZURE_STORAGE_KEY") or conn.get("account_key"),
|
|
149
|
+
sas_token=getenv("AZURE_STORAGE_SAS_TOKEN") or conn.get("sas_token"),
|
|
150
|
+
endpoint_url=(
|
|
151
|
+
getenv("AZURE_STORAGE_ENDPOINT")
|
|
152
|
+
or conn.get("endpoint_url")
|
|
153
|
+
or (f"https://{account_name}.blob.core.windows.net" if account_name else None)
|
|
154
|
+
),
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
# ------------------------------------------------------------------
|
|
158
|
+
# CloudPathMixin hooks
|
|
159
|
+
# ------------------------------------------------------------------
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def _item_path(self) -> str:
|
|
163
|
+
return self._blob_path
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def _native_kwargs(self) -> dict:
|
|
167
|
+
return {
|
|
168
|
+
"endpoint": self._endpoint_url,
|
|
169
|
+
"container": self._container,
|
|
170
|
+
"account_name": self._account_name or "",
|
|
171
|
+
"account_key": self._account_key,
|
|
172
|
+
"sas_token": self._sas_token,
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
# ------------------------------------------------------------------
|
|
176
|
+
# Backend-specific operations
|
|
177
|
+
# ------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
async def is_dir(self) -> bool:
|
|
180
|
+
"""Check if path is a directory prefix (has blobs with this prefix)."""
|
|
181
|
+
return await az_is_dir(prefix=self._blob_path, **self._native_kwargs)
|
|
182
|
+
|
|
183
|
+
async def _list_containers(self) -> list[str] | None:
|
|
184
|
+
"""List containers when at the account root (``az://``); else None."""
|
|
185
|
+
if self._container:
|
|
186
|
+
return None
|
|
187
|
+
names = await az_list_containers(
|
|
188
|
+
endpoint=self._endpoint_url,
|
|
189
|
+
account_name=self._account_name or "",
|
|
190
|
+
account_key=self._account_key,
|
|
191
|
+
sas_token=self._sas_token,
|
|
192
|
+
use_h2=False,
|
|
193
|
+
)
|
|
194
|
+
return [f"az://{name}" for name in names]
|
|
195
|
+
|
|
196
|
+
async def stat(self, *, follow_symlinks: bool = True) -> object:
|
|
197
|
+
"""Get blob metadata from HEAD response."""
|
|
198
|
+
from types import SimpleNamespace
|
|
199
|
+
|
|
200
|
+
headers = await self._get_batcher().head(
|
|
201
|
+
item=self._blob_path,
|
|
202
|
+
**self._native_kwargs,
|
|
203
|
+
)
|
|
204
|
+
size = int(headers.get("Content-Length", headers.get("content-length", 0)))
|
|
205
|
+
last_modified = headers.get("Last-Modified", headers.get("last-modified"))
|
|
206
|
+
mtime = None
|
|
207
|
+
if last_modified:
|
|
208
|
+
mtime = (
|
|
209
|
+
datetime.strptime(last_modified, "%a, %d %b %Y %H:%M:%S GMT")
|
|
210
|
+
.replace(tzinfo=timezone.utc)
|
|
211
|
+
.timestamp()
|
|
212
|
+
)
|
|
213
|
+
return SimpleNamespace(st_size=size, st_ctime=mtime, st_mtime=mtime)
|
|
214
|
+
|
|
215
|
+
async def checksums(self) -> dict[str, str]:
|
|
216
|
+
"""Return Content-MD5 from blob properties."""
|
|
217
|
+
headers = await self._get_batcher().head(
|
|
218
|
+
item=self._blob_path,
|
|
219
|
+
**self._native_kwargs,
|
|
220
|
+
)
|
|
221
|
+
md5 = headers.get("Content-MD5", headers.get("content-md5", ""))
|
|
222
|
+
return {"md5": md5}
|
|
223
|
+
|
|
224
|
+
async def get_access_policy(
|
|
225
|
+
self, *, backend_options: BackendOptions | None = None
|
|
226
|
+
) -> AccessPolicy:
|
|
227
|
+
"""Return the containing container's public access and stored policies."""
|
|
228
|
+
data = await az_get_container_acl(**self._native_kwargs)
|
|
229
|
+
if isinstance(data, str):
|
|
230
|
+
data = msgspec.json.decode(data)
|
|
231
|
+
if not isinstance(data, Mapping):
|
|
232
|
+
raise TypeError("native Azure ACL response must be a mapping")
|
|
233
|
+
public_access = data.get("public_access")
|
|
234
|
+
grants = (
|
|
235
|
+
(AccessGrant("everyone", frozenset({"read"})),)
|
|
236
|
+
if public_access in {"blob", "container", "true"}
|
|
237
|
+
else ()
|
|
238
|
+
)
|
|
239
|
+
return AccessPolicy(grants=grants, provider={"azure_container_acl": data})
|
|
240
|
+
|
|
241
|
+
async def update_access_policy(
|
|
242
|
+
self, policy_patch: AccessPolicyPatch, *, backend_options: BackendOptions | None = None
|
|
243
|
+
) -> None:
|
|
244
|
+
"""Replace the container ACL using ``provider["azure_container_acl"]``."""
|
|
245
|
+
acl = policy_patch.provider.get("azure_container_acl")
|
|
246
|
+
if not isinstance(acl, Mapping):
|
|
247
|
+
raise ValueError("Azure ACL replacement requires provider['azure_container_acl']")
|
|
248
|
+
await az_put_container_acl(
|
|
249
|
+
acl_json=msgspec.json.encode(acl).decode(), **self._native_kwargs
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
async def presign(self, *, expires: int = 3600, method: str = "GET") -> str:
|
|
253
|
+
"""Generate a presigned URL for this Azure blob using a Service SAS token.
|
|
254
|
+
|
|
255
|
+
Requires account_key to be set. If a SAS token is already configured,
|
|
256
|
+
returns the blob URL with the existing SAS token appended.
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
expires: URL validity in seconds (default 3600).
|
|
260
|
+
method: HTTP method (used to determine SAS permission: r, w, d, c).
|
|
261
|
+
|
|
262
|
+
Returns:
|
|
263
|
+
A presigned URL string.
|
|
264
|
+
"""
|
|
265
|
+
return az_presign(
|
|
266
|
+
endpoint=self._endpoint_url,
|
|
267
|
+
container=self._container,
|
|
268
|
+
blob_path=self._blob_path,
|
|
269
|
+
account_name=self._account_name or "",
|
|
270
|
+
account_key=self._account_key,
|
|
271
|
+
sas_token=self._sas_token,
|
|
272
|
+
method=method,
|
|
273
|
+
expires=expires,
|
|
274
|
+
)
|
asanypath/batcher.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# Copyright (C) 2026 Axis Communications AB, Lund, Sweden
|
|
2
|
+
#
|
|
3
|
+
# Use of this source code is governed by an MIT-style
|
|
4
|
+
# license that can be found in the LICENSE file or at
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
"""Generic micro-batcher for native Rust batch operations.
|
|
8
|
+
|
|
9
|
+
Transparently collects concurrent calls arriving in the same event-loop
|
|
10
|
+
tick and dispatches them as a single batch, amortizing the PyO3 bridge
|
|
11
|
+
overhead across all concurrent operations.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
from collections.abc import Callable, Coroutine
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
# Batch size threshold: use HTTP/2 multiplexing when batch is large enough
|
|
21
|
+
# to benefit from stream multiplexing over fewer connections; use HTTP/1.1
|
|
22
|
+
# (parallel TCP connections) for smaller batches where framing overhead hurts.
|
|
23
|
+
H2_BATCH_THRESHOLD = 20
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MicroBatcher:
|
|
27
|
+
"""Collects calls arriving in the same event-loop tick into one batch.
|
|
28
|
+
|
|
29
|
+
Each operation is registered with :meth:`register`, which returns a
|
|
30
|
+
submit coroutine callers use to enqueue work. At the end of the
|
|
31
|
+
current tick, all enqueued items are flushed together via the
|
|
32
|
+
backend-provided *flush_fn*.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
key_fn:
|
|
37
|
+
Callable that produces a hashable grouping key from the keyword
|
|
38
|
+
arguments passed to the submit function. Calls with the same
|
|
39
|
+
key are batched together.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
self._pending: dict[str, list] = {}
|
|
44
|
+
self._scheduled: set[str] = set()
|
|
45
|
+
self._ops: dict[str, _OpSpec] = {}
|
|
46
|
+
|
|
47
|
+
def __getattr__(self, name: str) -> Callable[..., Coroutine]:
|
|
48
|
+
raise AttributeError(name)
|
|
49
|
+
|
|
50
|
+
# -----------------------------------------------------------------
|
|
51
|
+
# Registration
|
|
52
|
+
# -----------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
def register(
|
|
55
|
+
self,
|
|
56
|
+
op: str,
|
|
57
|
+
*,
|
|
58
|
+
flush_fn: Callable[..., Coroutine],
|
|
59
|
+
key_fn: Callable[..., str],
|
|
60
|
+
item_key: str = "item",
|
|
61
|
+
unpack_put: bool = False,
|
|
62
|
+
) -> Callable[..., Coroutine]:
|
|
63
|
+
"""Register a batch operation and return a *submit* coroutine.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
op:
|
|
68
|
+
Unique name for this operation (e.g. ``"get"``, ``"put"``).
|
|
69
|
+
flush_fn:
|
|
70
|
+
``async def flush_fn(items, futures, use_h2, **kwargs)``
|
|
71
|
+
Called with the collected items list and their futures.
|
|
72
|
+
key_fn:
|
|
73
|
+
``def key_fn(**kwargs) -> str`` — returns a grouping key.
|
|
74
|
+
item_key:
|
|
75
|
+
Name of the keyword argument that carries the per-call item
|
|
76
|
+
(default ``"item"``).
|
|
77
|
+
unpack_put:
|
|
78
|
+
If *True*, the item is expected to be a tuple and will be
|
|
79
|
+
appended as-is (for put-style ops with ``(path, data)``).
|
|
80
|
+
"""
|
|
81
|
+
spec = _OpSpec(
|
|
82
|
+
op=op, flush_fn=flush_fn, key_fn=key_fn, item_key=item_key, unpack_put=unpack_put
|
|
83
|
+
)
|
|
84
|
+
self._ops[op] = spec
|
|
85
|
+
|
|
86
|
+
async def submit(**kwargs: Any) -> Any:
|
|
87
|
+
return await self._submit(spec, kwargs)
|
|
88
|
+
|
|
89
|
+
# Also make it accessible via attribute: batcher.get(item=..., ...)
|
|
90
|
+
setattr(self, op, submit)
|
|
91
|
+
return submit
|
|
92
|
+
|
|
93
|
+
# -----------------------------------------------------------------
|
|
94
|
+
# Internal machinery
|
|
95
|
+
# -----------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
async def _submit(self, spec: _OpSpec, kwargs: dict) -> Any:
|
|
98
|
+
item = kwargs.pop(spec.item_key)
|
|
99
|
+
key = f"{spec.op}|{spec.key_fn(**kwargs)}"
|
|
100
|
+
fut: asyncio.Future = asyncio.get_running_loop().create_future()
|
|
101
|
+
pending = self._pending.setdefault(key, [])
|
|
102
|
+
pending.append((item, fut))
|
|
103
|
+
if key not in self._scheduled:
|
|
104
|
+
self._scheduled.add(key)
|
|
105
|
+
asyncio.get_running_loop().call_soon(self._flush, key, spec, kwargs)
|
|
106
|
+
return await fut
|
|
107
|
+
|
|
108
|
+
def _flush(self, key: str, spec: _OpSpec, kwargs: dict) -> None:
|
|
109
|
+
items = self._pending.pop(key, [])
|
|
110
|
+
self._scheduled.discard(key)
|
|
111
|
+
if not items:
|
|
112
|
+
return
|
|
113
|
+
collected = [i for i, _ in items]
|
|
114
|
+
futures = [f for _, f in items]
|
|
115
|
+
use_h2 = len(collected) >= H2_BATCH_THRESHOLD
|
|
116
|
+
asyncio.ensure_future(
|
|
117
|
+
spec.flush_fn(items=collected, futures=futures, use_h2=use_h2, **kwargs)
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class _OpSpec:
|
|
122
|
+
"""Internal descriptor for a registered operation."""
|
|
123
|
+
|
|
124
|
+
__slots__ = ("op", "flush_fn", "key_fn", "item_key", "unpack_put")
|
|
125
|
+
|
|
126
|
+
def __init__(self, *, op: str, flush_fn, key_fn, item_key: str, unpack_put: bool):
|
|
127
|
+
self.op = op
|
|
128
|
+
self.flush_fn = flush_fn
|
|
129
|
+
self.key_fn = key_fn
|
|
130
|
+
self.item_key = item_key
|
|
131
|
+
self.unpack_put = unpack_put
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# Generic flush factory + batcher builder
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _make_flush(
|
|
140
|
+
batch_fn: Callable[..., Coroutine],
|
|
141
|
+
*,
|
|
142
|
+
items_key: str = "items",
|
|
143
|
+
wrap: Callable | None = None,
|
|
144
|
+
) -> Callable[..., Coroutine]:
|
|
145
|
+
"""Create a flush function for a batch operation.
|
|
146
|
+
|
|
147
|
+
Parameters
|
|
148
|
+
----------
|
|
149
|
+
batch_fn:
|
|
150
|
+
The native Rust batch function to call.
|
|
151
|
+
items_key:
|
|
152
|
+
Keyword argument name to pass the items list to *batch_fn*.
|
|
153
|
+
wrap:
|
|
154
|
+
Optional callable applied to each result value (e.g. ``bytes``).
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
async def _flush(items: list, futures: list, use_h2: bool, **kw: Any) -> None:
|
|
158
|
+
try:
|
|
159
|
+
kw[items_key] = items
|
|
160
|
+
results = await batch_fn(use_h2=use_h2, **kw) or [None] * len(futures)
|
|
161
|
+
for fut, val in zip(futures, results):
|
|
162
|
+
if not fut.done():
|
|
163
|
+
fut.set_result(wrap(val) if wrap else val)
|
|
164
|
+
except Exception as exc:
|
|
165
|
+
for fut in futures:
|
|
166
|
+
if not fut.done():
|
|
167
|
+
fut.set_exception(exc)
|
|
168
|
+
|
|
169
|
+
return _flush
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def make_batcher(
|
|
173
|
+
key_fn: Callable[..., str],
|
|
174
|
+
ops: dict[str, dict[str, Any]],
|
|
175
|
+
) -> MicroBatcher:
|
|
176
|
+
"""Build a :class:`MicroBatcher` from a declarative ops table.
|
|
177
|
+
|
|
178
|
+
Parameters
|
|
179
|
+
----------
|
|
180
|
+
key_fn:
|
|
181
|
+
Callable producing a grouping key from operation kwargs.
|
|
182
|
+
ops:
|
|
183
|
+
Mapping of operation name to a dict with keys:
|
|
184
|
+
|
|
185
|
+
- ``batch_fn`` (required): the native Rust batch function.
|
|
186
|
+
- ``items_key`` (default ``"items"``): kwarg name for the items list.
|
|
187
|
+
- ``returns_results`` (default ``True``): whether the batch fn
|
|
188
|
+
returns per-item results.
|
|
189
|
+
- ``wrap`` (default ``None``): optional per-result transform.
|
|
190
|
+
- ``unpack_put`` (default ``False``): passed to
|
|
191
|
+
:meth:`MicroBatcher.register`.
|
|
192
|
+
"""
|
|
193
|
+
b = MicroBatcher()
|
|
194
|
+
for name, spec in ops.items():
|
|
195
|
+
flush = _make_flush(
|
|
196
|
+
spec["batch_fn"],
|
|
197
|
+
items_key=spec.get("items_key", "items"),
|
|
198
|
+
wrap=spec.get("wrap"),
|
|
199
|
+
)
|
|
200
|
+
b.register(
|
|
201
|
+
name,
|
|
202
|
+
flush_fn=flush,
|
|
203
|
+
key_fn=key_fn,
|
|
204
|
+
unpack_put=spec.get("unpack_put", False),
|
|
205
|
+
)
|
|
206
|
+
return b
|