arraylake 1.0.0__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-1.0.0 → arraylake-1.0.2}/.gitignore +3 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/PKG-INFO +5 -3
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/_version.py +2 -2
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/api_utils.py +16 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/auth.py +1 -8
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/client.py +224 -46
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/metastore/http_metastore.py +21 -10
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/types.py +162 -1
- {arraylake-1.0.0 → arraylake-1.0.2}/pyproject.toml +20 -14
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_api_utils.py +19 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_client.py +123 -22
- {arraylake-1.0.0 → arraylake-1.0.2}/uv.lock +598 -427
- {arraylake-1.0.0 → arraylake-1.0.2}/README.md +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/__main__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/asyn.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/compute.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/config.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/main.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/repo.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/cli/utils.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/compute/doctor.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/compute/http_client.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/compute/services.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/compute/types.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/config.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/config.yaml +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/credentials.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/diagnostics.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/display/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/display/repo.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/display/repolist.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/exceptions.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/log_util.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/metastore/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/metastore/abc.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/py.typed +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/repos/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/repos/icechunk/storage.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/repos/icechunk/types.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/repos/icechunk/virtual.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/token.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/arraylake/tuning.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/compute/test_client.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/compute/test_doctor.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/config.yaml +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/conftest.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/icechunk/__init__.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/icechunk/conftest.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/icechunk/test_storage.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/icechunk/test_virtual.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_asyn.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_cli.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_config.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_credentials.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_diagnostics.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_performance.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_subscriptions.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_token_handler.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_types.py +0 -0
- {arraylake-1.0.0 → arraylake-1.0.2}/tests/test_virtual_chunks.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arraylake
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: Python client for ArrayLake
|
|
5
5
|
Project-URL: Documentation, https://docs.earthmover.io/
|
|
6
6
|
Project-URL: Changelog, https://docs.earthmover.io/changelog
|
|
@@ -16,10 +16,10 @@ Requires-Dist: donfig<1.0,>=0.7
|
|
|
16
16
|
Requires-Dist: httpx<0.29,>=0.23
|
|
17
17
|
Requires-Dist: icechunk<3.0.0,>=2.0.1
|
|
18
18
|
Requires-Dist: numpy<3.0,>=1.23
|
|
19
|
-
Requires-Dist: packaging<
|
|
19
|
+
Requires-Dist: packaging<27.0,>=23.0
|
|
20
20
|
Requires-Dist: pydantic[email]<3.0,>=2.9
|
|
21
21
|
Requires-Dist: python-dateutil<3.0,>=2.8
|
|
22
|
-
Requires-Dist: rich<
|
|
22
|
+
Requires-Dist: rich<16.0,>=12.6
|
|
23
23
|
Requires-Dist: ruamel-yaml<1.0,>=0.17
|
|
24
24
|
Requires-Dist: structlog<26.0,>=24.1
|
|
25
25
|
Requires-Dist: typer<1.0,>=0.12.0
|
|
@@ -27,6 +27,8 @@ Requires-Dist: zarr<4.0,>=3.1.0
|
|
|
27
27
|
Provides-Extra: maximal
|
|
28
28
|
Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
|
|
29
29
|
Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
|
|
30
|
+
Provides-Extra: obstore
|
|
31
|
+
Requires-Dist: obstore>=0.9.2; extra == 'obstore'
|
|
30
32
|
Provides-Extra: xarray
|
|
31
33
|
Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
|
|
32
34
|
Requires-Dist: xarray>=2024.10.0; extra == 'xarray'
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (1, 0,
|
|
21
|
+
__version__ = version = '1.0.2'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 2)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import functools
|
|
4
5
|
import json
|
|
5
6
|
import os
|
|
6
7
|
import random
|
|
8
|
+
import uuid
|
|
7
9
|
import warnings
|
|
8
10
|
import weakref
|
|
9
11
|
from collections.abc import AsyncGenerator, Callable, Generator, Mapping
|
|
@@ -82,6 +84,19 @@ def _get_proxy_from_env() -> str | None:
|
|
|
82
84
|
return None
|
|
83
85
|
|
|
84
86
|
|
|
87
|
+
@functools.cache
|
|
88
|
+
def _get_session_id() -> str:
|
|
89
|
+
"""Session ID for this Python process.
|
|
90
|
+
|
|
91
|
+
Propagated as the ``arraylake-session-id`` header on every outbound request and
|
|
92
|
+
baked into ``ArraylakeHttpClient._default_headers`` at construction time, so it
|
|
93
|
+
rides along to dask workers via the existing pickle path on the client.
|
|
94
|
+
"""
|
|
95
|
+
session_id = str(uuid.uuid4())
|
|
96
|
+
logger.info("arraylake session id: %s", session_id)
|
|
97
|
+
return session_id
|
|
98
|
+
|
|
99
|
+
|
|
85
100
|
# this is a hashable key to use for the global client cache
|
|
86
101
|
# we only need one client for each key
|
|
87
102
|
@dataclass(eq=True, frozen=True)
|
|
@@ -302,6 +317,7 @@ class ArraylakeHttpClient:
|
|
|
302
317
|
# technically we don't enforce this, for now it's for debugging purposes
|
|
303
318
|
"client-name": "arraylake-python-client",
|
|
304
319
|
"client-version": arraylake.__version__,
|
|
320
|
+
"arraylake-session-id": _get_session_id(),
|
|
305
321
|
}
|
|
306
322
|
|
|
307
323
|
self._async_lock = asyncio.Lock()
|
|
@@ -101,14 +101,7 @@ def token():
|
|
|
101
101
|
raise typer.Exit(code=1)
|
|
102
102
|
try:
|
|
103
103
|
token_value = handler.tokens.id_token.get_secret_value()
|
|
104
|
-
|
|
105
|
-
Panel(
|
|
106
|
-
f"[dim]{token_value}[/dim]",
|
|
107
|
-
title="id_token",
|
|
108
|
-
subtitle=f"[link=https://jwt.io/#id_token={token_value}][blue]:link: Inspect with JWT.io[/blue][/link]",
|
|
109
|
-
subtitle_align="right",
|
|
110
|
-
)
|
|
111
|
-
)
|
|
104
|
+
typer.echo(token_value)
|
|
112
105
|
except (AuthException, AttributeError):
|
|
113
106
|
rich_console.print(Align(f"[yellow]{NOT_LOGGED_IN_MESSAGE}[/yellow]", align="center"))
|
|
114
107
|
raise typer.Exit(code=1)
|
|
@@ -18,7 +18,7 @@ import warnings
|
|
|
18
18
|
from collections.abc import AsyncGenerator, Callable, Generator, Mapping
|
|
19
19
|
from datetime import UTC
|
|
20
20
|
from functools import partial
|
|
21
|
-
from typing import Literal
|
|
21
|
+
from typing import TYPE_CHECKING, Literal
|
|
22
22
|
from urllib.parse import urlparse
|
|
23
23
|
from uuid import UUID
|
|
24
24
|
|
|
@@ -83,6 +83,9 @@ from arraylake.types import (
|
|
|
83
83
|
)
|
|
84
84
|
from arraylake.types import Repo as RepoModel
|
|
85
85
|
|
|
86
|
+
if TYPE_CHECKING:
|
|
87
|
+
import obstore as obs
|
|
88
|
+
|
|
86
89
|
logger = get_logger(__name__)
|
|
87
90
|
|
|
88
91
|
_VALID_NAME = r"(\w[\w\.\-_]+)"
|
|
@@ -243,52 +246,67 @@ class AsyncClient:
|
|
|
243
246
|
azure_creds = await mstore.get_azure_container_credentials_from_repo(repo_name)
|
|
244
247
|
return azure_creds
|
|
245
248
|
|
|
246
|
-
async def _get_s3_delegated_credentials_from_bucket(
|
|
247
|
-
|
|
248
|
-
|
|
249
|
+
async def _get_s3_delegated_credentials_from_bucket(
|
|
250
|
+
self, org: OrgName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
|
|
251
|
+
) -> S3Credentials:
|
|
252
|
+
"""Get delegated credentials for an S3 bucket.
|
|
253
|
+
|
|
254
|
+
Defaults to read-only credentials. Pass ``access="write"`` to request read+write
|
|
255
|
+
credentials; the caller must have ``CAN_MANAGE_USERS`` on the org.
|
|
249
256
|
|
|
250
257
|
Args:
|
|
251
258
|
org: Name of the organization that the bucket belongs to.
|
|
252
259
|
nickname: Nickname of the bucket.
|
|
260
|
+
access: ``"read"`` (default) or ``"write"``.
|
|
253
261
|
|
|
254
262
|
Returns:
|
|
255
263
|
S3Credentials: Temporary credentials for the S3 bucket.
|
|
256
264
|
"""
|
|
257
265
|
mstore = self._metastore_for_org(org)
|
|
258
266
|
bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
|
|
259
|
-
s3_creds = await mstore.get_s3_bucket_credentials_from_bucket(bucket_id)
|
|
267
|
+
s3_creds = await mstore.get_s3_bucket_credentials_from_bucket(bucket_id, access=access)
|
|
260
268
|
return s3_creds
|
|
261
269
|
|
|
262
|
-
async def _get_gcs_delegated_credentials_from_bucket(
|
|
263
|
-
|
|
264
|
-
|
|
270
|
+
async def _get_gcs_delegated_credentials_from_bucket(
|
|
271
|
+
self, org: RepoName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
|
|
272
|
+
) -> GSCredentials:
|
|
273
|
+
"""Get delegated credentials for a GCS bucket.
|
|
274
|
+
|
|
275
|
+
Defaults to read-only credentials. Pass ``access="write"`` to request read+write
|
|
276
|
+
credentials; the caller must have ``CAN_MANAGE_USERS`` on the org.
|
|
265
277
|
|
|
266
278
|
Args:
|
|
267
279
|
org: Name of the organization that the bucket belongs to.
|
|
268
280
|
nickname: Nickname of the bucket.
|
|
281
|
+
access: ``"read"`` (default) or ``"write"``.
|
|
269
282
|
|
|
270
283
|
Returns:
|
|
271
284
|
GSCredentials: Temporary credentials for the GCS bucket.
|
|
272
285
|
"""
|
|
273
286
|
mstore = self._metastore_for_org(org)
|
|
274
287
|
bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
|
|
275
|
-
gcs_creds = await mstore.get_gs_bucket_credentials_from_bucket(bucket_id)
|
|
288
|
+
gcs_creds = await mstore.get_gs_bucket_credentials_from_bucket(bucket_id, access=access)
|
|
276
289
|
return gcs_creds
|
|
277
290
|
|
|
278
|
-
async def _get_azure_delegated_credentials_from_bucket(
|
|
279
|
-
|
|
280
|
-
|
|
291
|
+
async def _get_azure_delegated_credentials_from_bucket(
|
|
292
|
+
self, org: RepoName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
|
|
293
|
+
) -> AzureCredentials:
|
|
294
|
+
"""Get delegated credentials for an Azure Blob Storage container.
|
|
295
|
+
|
|
296
|
+
Defaults to read-only credentials. Pass ``access="write"`` to request read+write
|
|
297
|
+
credentials; the caller must have ``CAN_MANAGE_USERS`` on the org.
|
|
281
298
|
|
|
282
299
|
Args:
|
|
283
300
|
org: Name of the organization that the bucket belongs to.
|
|
284
301
|
nickname: Nickname of the bucket.
|
|
302
|
+
access: ``"read"`` (default) or ``"write"``.
|
|
285
303
|
|
|
286
304
|
Returns:
|
|
287
305
|
AzureCredentials: Temporary credentials for the Azure Blob Storage container.
|
|
288
306
|
"""
|
|
289
307
|
mstore = self._metastore_for_org(org)
|
|
290
308
|
bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
|
|
291
|
-
azure_creds = await mstore.get_azure_container_credentials_from_bucket(bucket_id)
|
|
309
|
+
azure_creds = await mstore.get_azure_container_credentials_from_bucket(bucket_id, access=access)
|
|
292
310
|
return azure_creds
|
|
293
311
|
|
|
294
312
|
def _get_icechunk_s3_credentials_refresh_function_for_repo(self, org: OrgName, repo_name: RepoName) -> icechunk.S3StaticCredentials:
|
|
@@ -394,28 +412,32 @@ class AsyncClient:
|
|
|
394
412
|
bucket: BucketResponse,
|
|
395
413
|
org: OrgName,
|
|
396
414
|
repo_name: RepoName | None,
|
|
415
|
+
bucket_access: Literal["read", "write"] = "read",
|
|
397
416
|
) -> TempCredentials | None:
|
|
398
417
|
"""Checks if the bucket is configured for delegated or HMAC credentials and gets the
|
|
399
418
|
credentials if it is configured.
|
|
400
419
|
|
|
401
420
|
Returns None if delegated or HMAC credentials are not configured for the bucket.
|
|
421
|
+
|
|
422
|
+
``bucket_access`` only applies when ``repo_name`` is None (the bucket-scoped path);
|
|
423
|
+
repo-scoped credential fetches always inherit the caller's permissions on the repo.
|
|
402
424
|
"""
|
|
403
425
|
if _use_delegated_credentials(bucket):
|
|
404
426
|
if bucket.platform == "s3" or _is_r2_bucket(bucket):
|
|
405
427
|
if repo_name:
|
|
406
428
|
return await self._get_s3_delegated_credentials_from_repo(org, repo_name)
|
|
407
429
|
else:
|
|
408
|
-
return await self._get_s3_delegated_credentials_from_bucket(org, bucket.nickname)
|
|
430
|
+
return await self._get_s3_delegated_credentials_from_bucket(org, bucket.nickname, access=bucket_access)
|
|
409
431
|
elif bucket.platform == "gs":
|
|
410
432
|
if repo_name:
|
|
411
433
|
return await self._get_gcs_delegated_credentials_from_repo(org, repo_name)
|
|
412
434
|
else:
|
|
413
|
-
return await self._get_gcs_delegated_credentials_from_bucket(org, bucket.nickname)
|
|
435
|
+
return await self._get_gcs_delegated_credentials_from_bucket(org, bucket.nickname, access=bucket_access)
|
|
414
436
|
elif bucket.platform == "azure":
|
|
415
437
|
if repo_name:
|
|
416
438
|
return await self._get_azure_delegated_credentials_from_repo(org, repo_name)
|
|
417
439
|
else:
|
|
418
|
-
return await self._get_azure_delegated_credentials_from_bucket(org, bucket.nickname)
|
|
440
|
+
return await self._get_azure_delegated_credentials_from_bucket(org, bucket.nickname, access=bucket_access)
|
|
419
441
|
else:
|
|
420
442
|
raise ValueError(f"Unsupported platform for delegated credentials: {bucket.platform}")
|
|
421
443
|
elif _use_hmac_credentials(bucket):
|
|
@@ -558,16 +580,17 @@ class AsyncClient:
|
|
|
558
580
|
if response.virtual_chunk_credentials:
|
|
559
581
|
credentials_map: dict[str, icechunk.AnyS3Credential | icechunk.AnyGcsCredential | icechunk.AnyAzureCredential | None] = {}
|
|
560
582
|
for vcc_prefix, vcc_creds in response.virtual_chunk_credentials.items():
|
|
561
|
-
#
|
|
583
|
+
# Create VCC objects for all server-provided VCCs, not just runtime ones,
|
|
584
|
+
# because icechunk's config merge replaces virtual_chunk_containers
|
|
585
|
+
# rather than extending them.
|
|
586
|
+
store_config = vcc_creds.to_vcc_store_config(user_id=UUID(response.principal_id))
|
|
587
|
+
container = icechunk.VirtualChunkContainer(
|
|
588
|
+
url_prefix=vcc_prefix,
|
|
589
|
+
store=store_config,
|
|
590
|
+
)
|
|
562
591
|
if vcc_creds.runtime_vcc_name is not None:
|
|
563
|
-
bucket_config = await self.get_bucket_config(org=vcc_creds.org, nickname=vcc_creds.bucket_nickname)
|
|
564
|
-
store_config = bucket_config.to_vcc_store_config(user_id=UUID(response.principal_id))
|
|
565
|
-
container = icechunk.VirtualChunkContainer(
|
|
566
|
-
url_prefix=vcc_prefix,
|
|
567
|
-
store=store_config,
|
|
568
|
-
)
|
|
569
592
|
container.name = vcc_creds.runtime_vcc_name
|
|
570
|
-
|
|
593
|
+
config.set_virtual_chunk_container(container)
|
|
571
594
|
|
|
572
595
|
# TODO: build refresh closures here using vcc_creds.org, vcc_creds.bucket_nickname,
|
|
573
596
|
# and vcc_creds.platform so that credentials can be refreshed when they expire. (See https://github.com/earth-mover/arraylake/issues/5122)
|
|
@@ -1264,13 +1287,42 @@ class AsyncClient:
|
|
|
1264
1287
|
raise BucketNotFoundError(nickname)
|
|
1265
1288
|
return bucket_id
|
|
1266
1289
|
|
|
1267
|
-
def _make_bucket_config(
|
|
1290
|
+
def _make_bucket_config(
|
|
1291
|
+
self,
|
|
1292
|
+
*,
|
|
1293
|
+
nickname: BucketNickname,
|
|
1294
|
+
uri: str,
|
|
1295
|
+
region_name: str | None = None,
|
|
1296
|
+
endpoint_url: str | None = None,
|
|
1297
|
+
use_ssl: bool | None = None,
|
|
1298
|
+
force_path_style: bool | None = None,
|
|
1299
|
+
extra_config: dict | None = None,
|
|
1300
|
+
auth_config: dict | None = None,
|
|
1301
|
+
) -> dict:
|
|
1268
1302
|
if not nickname:
|
|
1269
1303
|
raise ValueError("nickname must be specified if uri is provided.")
|
|
1270
1304
|
|
|
1305
|
+
# Build extra_config from explicit parameters, with extra_config dict as fallback
|
|
1306
|
+
# for backwards compatibility. Explicit parameters take precedence.
|
|
1307
|
+
merged_extra_config: dict[str, str | bool] = {}
|
|
1308
|
+
if extra_config is not None:
|
|
1309
|
+
warnings.warn(
|
|
1310
|
+
"Passing 'extra_config' as a dict is deprecated. "
|
|
1311
|
+
"Use the explicit keyword arguments instead: region_name, endpoint_url, use_ssl, force_path_style.",
|
|
1312
|
+
DeprecationWarning,
|
|
1313
|
+
stacklevel=3,
|
|
1314
|
+
)
|
|
1315
|
+
merged_extra_config.update(extra_config)
|
|
1316
|
+
if region_name is not None:
|
|
1317
|
+
merged_extra_config["region_name"] = region_name
|
|
1318
|
+
if endpoint_url is not None:
|
|
1319
|
+
merged_extra_config["endpoint_url"] = endpoint_url
|
|
1320
|
+
if use_ssl is not None:
|
|
1321
|
+
merged_extra_config["use_ssl"] = use_ssl
|
|
1322
|
+
if force_path_style is not None:
|
|
1323
|
+
merged_extra_config["force_path_style"] = force_path_style
|
|
1324
|
+
|
|
1271
1325
|
# unpack optionals
|
|
1272
|
-
if extra_config is None:
|
|
1273
|
-
extra_config = {}
|
|
1274
1326
|
if auth_config is None:
|
|
1275
1327
|
auth_config = {"method": "anonymous"}
|
|
1276
1328
|
|
|
@@ -1279,7 +1331,7 @@ class AsyncClient:
|
|
|
1279
1331
|
platform: Literal["s3", "gs", "s3-compatible", "azure"] | None = (
|
|
1280
1332
|
"s3" if res.scheme == "s3" else "gs" if res.scheme == "gs" else "azure" if res.scheme == "az" else None
|
|
1281
1333
|
)
|
|
1282
|
-
if platform == "s3" and
|
|
1334
|
+
if platform == "s3" and merged_extra_config.get("endpoint_url"):
|
|
1283
1335
|
platform = "s3-compatible"
|
|
1284
1336
|
if platform not in ["s3", "gs", "s3-compatible", "azure"]:
|
|
1285
1337
|
raise ValueError(f"Invalid platform {platform} for uri {uri}")
|
|
@@ -1313,26 +1365,62 @@ class AsyncClient:
|
|
|
1313
1365
|
name=name,
|
|
1314
1366
|
prefix=prefix,
|
|
1315
1367
|
nickname=nickname,
|
|
1316
|
-
extra_config=
|
|
1368
|
+
extra_config=merged_extra_config,
|
|
1317
1369
|
auth_config=auth_config,
|
|
1318
1370
|
)
|
|
1319
1371
|
|
|
1320
1372
|
async def create_bucket_config(
|
|
1321
|
-
self,
|
|
1373
|
+
self,
|
|
1374
|
+
*,
|
|
1375
|
+
org: OrgName,
|
|
1376
|
+
nickname: BucketNickname,
|
|
1377
|
+
uri: URI,
|
|
1378
|
+
region_name: str | None = None,
|
|
1379
|
+
endpoint_url: str | None = None,
|
|
1380
|
+
use_ssl: bool | None = None,
|
|
1381
|
+
force_path_style: bool | None = None,
|
|
1382
|
+
extra_config: dict | None = None,
|
|
1383
|
+
auth_config: dict | None = None,
|
|
1322
1384
|
) -> BucketResponse:
|
|
1323
|
-
"""Create a new bucket config entry
|
|
1385
|
+
"""Create a new bucket config entry.
|
|
1324
1386
|
|
|
1325
1387
|
NOTE: This does not create any actual buckets in the object store.
|
|
1326
1388
|
|
|
1327
1389
|
Args:
|
|
1328
|
-
org: Name of the org
|
|
1329
|
-
nickname:
|
|
1390
|
+
org: Name of the org.
|
|
1391
|
+
nickname: Bucket nickname (e.g. ``"my-s3-bucket"``).
|
|
1330
1392
|
uri: The URI of the object store, of the form
|
|
1331
|
-
platform://bucket_name[/prefix].
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1393
|
+
``platform://bucket_name[/prefix]`` (e.g. ``"s3://my-bucket/prefix"``).
|
|
1394
|
+
region_name: Cloud region for the bucket (e.g. ``"us-west-2"``).
|
|
1395
|
+
Required for S3 buckets.
|
|
1396
|
+
endpoint_url: Custom endpoint URL for S3-compatible storage
|
|
1397
|
+
(e.g. ``"https://account-id.r2.cloudflarestorage.com"``).
|
|
1398
|
+
Setting this automatically marks the bucket as S3-compatible.
|
|
1399
|
+
use_ssl: Whether to use HTTPS for storage connections.
|
|
1400
|
+
Defaults to ``True``. Set to ``False`` for local or insecure endpoints.
|
|
1401
|
+
Only applies to S3-type buckets.
|
|
1402
|
+
force_path_style: Use path-style URLs (``endpoint/bucket``) instead of
|
|
1403
|
+
virtual-hosted style (``bucket.endpoint``). Defaults to ``False`` when
|
|
1404
|
+
``use_ssl=True``, ``True`` when ``use_ssl=False``. Often required for
|
|
1405
|
+
S3-compatible storage. Only applies to S3-type buckets.
|
|
1406
|
+
extra_config: Dictionary of additional config options. Prefer using the
|
|
1407
|
+
explicit parameters above when possible. Keys set here are overridden
|
|
1408
|
+
by any explicit parameters that are also provided.
|
|
1409
|
+
auth_config: Dictionary of auth parameters. Must include a ``"method"`` key.
|
|
1410
|
+
Defaults to ``{"method": "anonymous"}``.
|
|
1411
|
+
"""
|
|
1412
|
+
validated = NewBucket(
|
|
1413
|
+
**self._make_bucket_config(
|
|
1414
|
+
nickname=nickname,
|
|
1415
|
+
uri=uri,
|
|
1416
|
+
region_name=region_name,
|
|
1417
|
+
endpoint_url=endpoint_url,
|
|
1418
|
+
use_ssl=use_ssl,
|
|
1419
|
+
force_path_style=force_path_style,
|
|
1420
|
+
extra_config=extra_config,
|
|
1421
|
+
auth_config=auth_config,
|
|
1422
|
+
)
|
|
1423
|
+
)
|
|
1336
1424
|
mstore = self._metastore_for_org(org)
|
|
1337
1425
|
bucket = await mstore.create_bucket_config(validated)
|
|
1338
1426
|
return bucket
|
|
@@ -1359,6 +1447,42 @@ class AsyncClient:
|
|
|
1359
1447
|
bucket = await mstore.get_bucket_config(bucket_id)
|
|
1360
1448
|
return bucket
|
|
1361
1449
|
|
|
1450
|
+
async def get_obstore_for_bucket(
|
|
1451
|
+
self, *, org: OrgName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
|
|
1452
|
+
) -> obs.store.ObjectStore:
|
|
1453
|
+
"""Get an obstore ObjectStore for the named bucket.
|
|
1454
|
+
|
|
1455
|
+
Resolves the bucket nickname, fetches scoped credentials, and returns an
|
|
1456
|
+
``obstore.store.S3Store``, ``GCSStore``, or ``AzureStore`` ready for use.
|
|
1457
|
+
|
|
1458
|
+
Defaults to read-only credentials. Pass ``access="write"`` to request a
|
|
1459
|
+
read+write store; that variant requires ``CAN_MANAGE_USERS`` on the org.
|
|
1460
|
+
|
|
1461
|
+
Requires the optional ``obstore`` extra (``pip install arraylake[obstore]``).
|
|
1462
|
+
|
|
1463
|
+
Args:
|
|
1464
|
+
org: Name of the org that owns the bucket.
|
|
1465
|
+
nickname: Nickname of the bucket config.
|
|
1466
|
+
access: ``"read"`` (default) or ``"write"``.
|
|
1467
|
+
|
|
1468
|
+
Returns:
|
|
1469
|
+
An obstore ObjectStore scoped to the bucket and its configured prefix.
|
|
1470
|
+
|
|
1471
|
+
Raises:
|
|
1472
|
+
ImportError: If the optional ``obstore`` extra is not installed.
|
|
1473
|
+
"""
|
|
1474
|
+
# Fail fast before making any API calls if obstore isn't available.
|
|
1475
|
+
try:
|
|
1476
|
+
import obstore # noqa: F401
|
|
1477
|
+
except ImportError as e:
|
|
1478
|
+
raise ImportError(
|
|
1479
|
+
"The 'obstore' package is required for this functionality but is not installed. "
|
|
1480
|
+
"Install it with: pip install 'arraylake[obstore]'"
|
|
1481
|
+
) from e
|
|
1482
|
+
bucket = await self.get_bucket_config(org=org, nickname=nickname)
|
|
1483
|
+
creds = await self._maybe_get_credentials_for_icechunk(bucket, org, repo_name=None, bucket_access=access)
|
|
1484
|
+
return bucket.as_obstore_store(creds)
|
|
1485
|
+
|
|
1362
1486
|
async def list_bucket_configs(self, org: OrgName) -> list[BucketResponse]:
|
|
1363
1487
|
"""List all bucket config entries
|
|
1364
1488
|
|
|
@@ -1906,22 +2030,56 @@ class Client:
|
|
|
1906
2030
|
return sync(self.aclient.delete_repo, name, imsure=imsure, imreallysure=imreallysure)
|
|
1907
2031
|
|
|
1908
2032
|
def create_bucket_config(
|
|
1909
|
-
self,
|
|
2033
|
+
self,
|
|
2034
|
+
*,
|
|
2035
|
+
org: OrgName,
|
|
2036
|
+
nickname: BucketNickname,
|
|
2037
|
+
uri: URI,
|
|
2038
|
+
region_name: str | None = None,
|
|
2039
|
+
endpoint_url: str | None = None,
|
|
2040
|
+
use_ssl: bool | None = None,
|
|
2041
|
+
force_path_style: bool | None = None,
|
|
2042
|
+
extra_config: dict | None = None,
|
|
2043
|
+
auth_config: dict | None = None,
|
|
1910
2044
|
) -> BucketResponse:
|
|
1911
|
-
"""Create a new bucket config entry
|
|
2045
|
+
"""Create a new bucket config entry.
|
|
1912
2046
|
|
|
1913
2047
|
NOTE: This does not create any actual buckets in the object store.
|
|
1914
2048
|
|
|
1915
2049
|
Args:
|
|
1916
|
-
org: Name of the org
|
|
1917
|
-
nickname:
|
|
2050
|
+
org: Name of the org.
|
|
2051
|
+
nickname: Bucket nickname (e.g. ``"my-s3-bucket"``).
|
|
1918
2052
|
uri: The URI of the object store, of the form
|
|
1919
|
-
platform://bucket_name[/prefix].
|
|
1920
|
-
|
|
1921
|
-
|
|
2053
|
+
``platform://bucket_name[/prefix]`` (e.g. ``"s3://my-bucket/prefix"``).
|
|
2054
|
+
region_name: Cloud region for the bucket (e.g. ``"us-west-2"``).
|
|
2055
|
+
Required for S3 buckets.
|
|
2056
|
+
endpoint_url: Custom endpoint URL for S3-compatible storage
|
|
2057
|
+
(e.g. ``"https://account-id.r2.cloudflarestorage.com"``).
|
|
2058
|
+
Setting this automatically marks the bucket as S3-compatible.
|
|
2059
|
+
use_ssl: Whether to use HTTPS for storage connections.
|
|
2060
|
+
Defaults to ``True``. Set to ``False`` for local or insecure endpoints.
|
|
2061
|
+
Only applies to S3-type buckets.
|
|
2062
|
+
force_path_style: Use path-style URLs (``endpoint/bucket``) instead of
|
|
2063
|
+
virtual-hosted style (``bucket.endpoint``). Defaults to ``False`` when
|
|
2064
|
+
``use_ssl=True``, ``True`` when ``use_ssl=False``. Often required for
|
|
2065
|
+
S3-compatible storage. Only applies to S3-type buckets.
|
|
2066
|
+
extra_config: Dictionary of additional config options. Prefer using the
|
|
2067
|
+
explicit parameters above when possible. Keys set here are overridden
|
|
2068
|
+
by any explicit parameters that are also provided.
|
|
2069
|
+
auth_config: Dictionary of auth parameters. Must include a ``"method"`` key.
|
|
2070
|
+
Defaults to ``{"method": "anonymous"}``.
|
|
1922
2071
|
"""
|
|
1923
2072
|
return sync(
|
|
1924
|
-
self.aclient.create_bucket_config,
|
|
2073
|
+
self.aclient.create_bucket_config,
|
|
2074
|
+
org=org,
|
|
2075
|
+
nickname=nickname,
|
|
2076
|
+
uri=uri,
|
|
2077
|
+
region_name=region_name,
|
|
2078
|
+
endpoint_url=endpoint_url,
|
|
2079
|
+
use_ssl=use_ssl,
|
|
2080
|
+
force_path_style=force_path_style,
|
|
2081
|
+
extra_config=extra_config,
|
|
2082
|
+
auth_config=auth_config,
|
|
1925
2083
|
)
|
|
1926
2084
|
|
|
1927
2085
|
def set_default_bucket_config(self, *, org: OrgName, nickname: BucketNickname) -> None:
|
|
@@ -1942,6 +2100,26 @@ class Client:
|
|
|
1942
2100
|
"""
|
|
1943
2101
|
return sync(self.aclient.get_bucket_config, org=org, nickname=nickname)
|
|
1944
2102
|
|
|
2103
|
+
def get_obstore_for_bucket(
|
|
2104
|
+
self, *, org: OrgName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
|
|
2105
|
+
) -> obs.store.ObjectStore:
|
|
2106
|
+
"""Get an obstore ObjectStore for the named bucket.
|
|
2107
|
+
|
|
2108
|
+
Defaults to read-only credentials. Pass ``access="write"`` to request a
|
|
2109
|
+
read+write store; that variant requires ``CAN_MANAGE_USERS`` on the org.
|
|
2110
|
+
|
|
2111
|
+
Requires the optional ``obstore`` extra (``pip install arraylake[obstore]``).
|
|
2112
|
+
|
|
2113
|
+
Args:
|
|
2114
|
+
org: Name of the org that owns the bucket.
|
|
2115
|
+
nickname: Nickname of the bucket config.
|
|
2116
|
+
access: ``"read"`` (default) or ``"write"``.
|
|
2117
|
+
|
|
2118
|
+
Returns:
|
|
2119
|
+
An obstore ObjectStore scoped to the bucket and its configured prefix.
|
|
2120
|
+
"""
|
|
2121
|
+
return sync(self.aclient.get_obstore_for_bucket, org=org, nickname=nickname, access=access)
|
|
2122
|
+
|
|
1945
2123
|
def list_bucket_configs(self, org: OrgName) -> list[BucketResponse]:
|
|
1946
2124
|
"""List all buckets for the specified org
|
|
1947
2125
|
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from dataclasses import dataclass, field
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any, Literal
|
|
6
6
|
from uuid import UUID
|
|
7
7
|
|
|
8
8
|
from pydantic import TypeAdapter
|
|
@@ -300,21 +300,32 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
|
|
|
300
300
|
handle_response(response)
|
|
301
301
|
return AzureCredentials.model_validate_json(response.content)
|
|
302
302
|
|
|
303
|
-
async def get_s3_bucket_credentials_from_bucket(self, bucket_id: UUID) -> S3Credentials:
|
|
304
|
-
"""Gets the S3 credentials for a bucket.
|
|
305
|
-
|
|
303
|
+
async def get_s3_bucket_credentials_from_bucket(self, bucket_id: UUID, access: Literal["read", "write"] = "read") -> S3Credentials:
|
|
304
|
+
"""Gets the S3 credentials for a bucket.
|
|
305
|
+
|
|
306
|
+
Defaults to read-only credentials; pass ``access="write"`` for read+write.
|
|
307
|
+
"""
|
|
308
|
+
response = await self._request("GET", f"/orgs/{self._config.org}/buckets/{bucket_id}/credentials", params={"access": access})
|
|
306
309
|
handle_response(response)
|
|
307
310
|
return S3Credentials.model_validate_json(response.content)
|
|
308
311
|
|
|
309
|
-
async def get_gs_bucket_credentials_from_bucket(self, bucket_id: UUID) -> GSCredentials:
|
|
310
|
-
"""Gets the GCS credentials for a bucket.
|
|
311
|
-
|
|
312
|
+
async def get_gs_bucket_credentials_from_bucket(self, bucket_id: UUID, access: Literal["read", "write"] = "read") -> GSCredentials:
|
|
313
|
+
"""Gets the GCS credentials for a bucket.
|
|
314
|
+
|
|
315
|
+
Defaults to read-only credentials; pass ``access="write"`` for read+write.
|
|
316
|
+
"""
|
|
317
|
+
response = await self._request("GET", f"/orgs/{self._config.org}/buckets/{bucket_id}/credentials", params={"access": access})
|
|
312
318
|
handle_response(response)
|
|
313
319
|
return GSCredentials.model_validate_json(response.content)
|
|
314
320
|
|
|
315
|
-
async def get_azure_container_credentials_from_bucket(
|
|
316
|
-
|
|
317
|
-
|
|
321
|
+
async def get_azure_container_credentials_from_bucket(
|
|
322
|
+
self, bucket_id: UUID, access: Literal["read", "write"] = "read"
|
|
323
|
+
) -> AzureCredentials:
|
|
324
|
+
"""Gets the Azure credentials for a bucket.
|
|
325
|
+
|
|
326
|
+
Defaults to read-only credentials; pass ``access="write"`` for read+write.
|
|
327
|
+
"""
|
|
328
|
+
response = await self._request("GET", f"/orgs/{self._config.org}/buckets/{bucket_id}/credentials", params={"access": access})
|
|
318
329
|
handle_response(response)
|
|
319
330
|
return AzureCredentials.model_validate_json(response.content)
|
|
320
331
|
|