arraylake 0.31.0__tar.gz → 0.32.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-0.31.0 → arraylake-0.32.0}/PKG-INFO +1 -1
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/_version.py +2 -2
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/client.py +57 -43
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/metastore/http_metastore.py +20 -20
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/virtual.py +1 -12
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/types.py +6 -4
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_cli.py +9 -5
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_client.py +37 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_performance.py +24 -12
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_subscriptions.py +32 -0
- arraylake-0.32.0/tests/test_virtual_chunks.py +1201 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/uv.lock +100 -102
- arraylake-0.31.0/tests/test_virtual_chunks.py +0 -803
- {arraylake-0.31.0 → arraylake-0.32.0}/.gitignore +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/README.md +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/__main__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/api_utils.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/asyn.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/auth.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/compute.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/config.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/main.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/repo.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/utils.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/doctor.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/http_client.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/services.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/types.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/config.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/config.yaml +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/credentials.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/diagnostics.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/display/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/display/repo.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/display/repolist.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/exceptions.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/log_util.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/metastore/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/metastore/abc.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/py.typed +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/storage.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/types.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/token.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/tuning.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/pyproject.toml +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/compute/test_client.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/compute/test_doctor.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/config.yaml +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/conftest.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/__init__.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/conftest.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/test_storage.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/test_virtual.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_api_utils.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_asyn.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_config.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_credentials.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_diagnostics.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_token_handler.py +0 -0
- {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_types.py +0 -0
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.32.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 32, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -62,7 +62,7 @@ from arraylake.types import (
|
|
|
62
62
|
BucketNickname,
|
|
63
63
|
BucketPrefix,
|
|
64
64
|
BucketResponse,
|
|
65
|
-
|
|
65
|
+
ExplicitVirtualChunkAccessPolicyResponse,
|
|
66
66
|
GSCredentials,
|
|
67
67
|
NewBucket,
|
|
68
68
|
OptimizationConfig,
|
|
@@ -506,6 +506,13 @@ class AsyncClient:
|
|
|
506
506
|
# Single API call to get everything needed to open the repo
|
|
507
507
|
response = await mstore.open_repo(repo_name)
|
|
508
508
|
|
|
509
|
+
if response.status.mode == RepoOperationMode.OFFLINE:
|
|
510
|
+
raise ValueError(f"Repository is offline, check back after {response.status.estimated_end_time}")
|
|
511
|
+
elif response.status.mode == RepoOperationMode.MAINTENANCE:
|
|
512
|
+
warnings.warn(
|
|
513
|
+
f"Repository is in maintenance mode until {response.status.estimated_end_time}, do not attempt to write to the store"
|
|
514
|
+
)
|
|
515
|
+
|
|
509
516
|
# Build icechunk storage from the response
|
|
510
517
|
credential_refresh_func = self._maybe_get_credential_refresh_func_for_icechunk(
|
|
511
518
|
bucket=response.repo_bucket, org=org, repo_name=repo_name
|
|
@@ -546,18 +553,23 @@ class AsyncClient:
|
|
|
546
553
|
forbid_configs_unsafe_for_virtual_chunks=True,
|
|
547
554
|
)
|
|
548
555
|
else:
|
|
549
|
-
# Use server-provided VCC credentials
|
|
556
|
+
# Use server-provided VCC credentials
|
|
550
557
|
vcc_credentials = {}
|
|
551
558
|
if response.virtual_chunk_credentials:
|
|
552
559
|
credentials_map: dict[str, icechunk.AnyS3Credential | icechunk.AnyGcsCredential | icechunk.AnyAzureCredential | None] = {}
|
|
553
560
|
for vcc_prefix, vcc_creds in response.virtual_chunk_credentials.items():
|
|
554
|
-
# TODO:
|
|
555
|
-
#
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
+
# TODO: build refresh closures here using vcc_creds.org, vcc_creds.bucket_nickname,
|
|
562
|
+
# and vcc_creds.platform so that credentials can be refreshed when they expire. (See https://github.com/earth-mover/arraylake/issues/5122)
|
|
563
|
+
# TODO: support Azure virtual chunk credentials
|
|
564
|
+
if isinstance(vcc_creds.credentials, AzureCredentials):
|
|
565
|
+
raise NotImplementedError(f"Azure virtual chunk credentials are not yet supported. (prefix: {vcc_prefix}).")
|
|
566
|
+
# TODO: fix return type of get_icechunk_container_credentials (returns GcsBearerCredential
|
|
567
|
+
# which is not in icechunk's AnyGcsCredential union — icechunk typing gap)
|
|
568
|
+
credentials_map[vcc_prefix] = get_icechunk_container_credentials( # type: ignore[assignment]
|
|
569
|
+
bucket_platform=vcc_creds.platform,
|
|
570
|
+
credentials=vcc_creds.credentials,
|
|
571
|
+
credential_refresh_func=None,
|
|
572
|
+
)
|
|
561
573
|
vcc_credentials = icechunk.containers_credentials(credentials_map)
|
|
562
574
|
|
|
563
575
|
ic_repo: IcechunkRepository = await IcechunkRepository.open_async(
|
|
@@ -1369,42 +1381,42 @@ class AsyncClient:
|
|
|
1369
1381
|
bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
|
|
1370
1382
|
await mstore.delete_bucket_config(bucket_id)
|
|
1371
1383
|
|
|
1372
|
-
async def
|
|
1384
|
+
async def set_virtual_chunk_access_policy(
|
|
1373
1385
|
self,
|
|
1374
1386
|
org: OrgName,
|
|
1375
1387
|
*,
|
|
1376
1388
|
bucket_nickname: BucketNickname,
|
|
1377
1389
|
subprefix: str = "",
|
|
1378
1390
|
public: bool = False,
|
|
1379
|
-
) ->
|
|
1380
|
-
"""Set an explicit virtual chunk
|
|
1391
|
+
) -> ExplicitVirtualChunkAccessPolicyResponse:
|
|
1392
|
+
"""Set an explicit virtual chunk access policy on this org.
|
|
1381
1393
|
|
|
1382
1394
|
Args:
|
|
1383
1395
|
org: Name of the organization.
|
|
1384
|
-
bucket_nickname: Nickname of the bucket config to associate with this
|
|
1385
|
-
subprefix: Subprefix within the bucket for this
|
|
1386
|
-
public: Whether this
|
|
1396
|
+
bucket_nickname: Nickname of the bucket config to associate with this VCAP.
|
|
1397
|
+
subprefix: Subprefix within the bucket for this VCAP.
|
|
1398
|
+
public: Whether this VCAP allows public access.
|
|
1387
1399
|
"""
|
|
1388
1400
|
mstore = self._metastore_for_org(org)
|
|
1389
1401
|
bucket_id = await self._bucket_id_for_nickname(mstore, bucket_nickname)
|
|
1390
|
-
|
|
1402
|
+
vcap = await mstore.set_virtual_chunk_access_policy(
|
|
1391
1403
|
bucket_id=str(bucket_id),
|
|
1392
1404
|
subprefix=subprefix.strip("/"),
|
|
1393
1405
|
public=public,
|
|
1394
1406
|
)
|
|
1395
|
-
return
|
|
1407
|
+
return ExplicitVirtualChunkAccessPolicyResponse.model_validate(vcap.model_dump())
|
|
1396
1408
|
|
|
1397
|
-
async def
|
|
1398
|
-
"""List explicit virtual chunk
|
|
1409
|
+
async def list_virtual_chunk_access_policies(self, org: OrgName) -> list[ExplicitVirtualChunkAccessPolicyResponse]:
|
|
1410
|
+
"""List explicit virtual chunk access policies on this org.
|
|
1399
1411
|
|
|
1400
1412
|
Args:
|
|
1401
1413
|
org: Name of the organization.
|
|
1402
1414
|
"""
|
|
1403
1415
|
mstore = self._metastore_for_org(org)
|
|
1404
|
-
|
|
1405
|
-
return [
|
|
1416
|
+
vcaps = await mstore.list_virtual_chunk_access_policies()
|
|
1417
|
+
return [ExplicitVirtualChunkAccessPolicyResponse.model_validate(v.model_dump()) for v in vcaps if v.use_case == "explicit"]
|
|
1406
1418
|
|
|
1407
|
-
async def
|
|
1419
|
+
async def delete_virtual_chunk_access_policy(
|
|
1408
1420
|
self,
|
|
1409
1421
|
org: OrgName,
|
|
1410
1422
|
*,
|
|
@@ -1412,17 +1424,17 @@ class AsyncClient:
|
|
|
1412
1424
|
subprefix: str,
|
|
1413
1425
|
public: bool,
|
|
1414
1426
|
) -> None:
|
|
1415
|
-
"""Delete a virtual chunk
|
|
1427
|
+
"""Delete a virtual chunk access policy from this org (idempotent).
|
|
1416
1428
|
|
|
1417
1429
|
Args:
|
|
1418
1430
|
org: Name of the organization.
|
|
1419
|
-
bucket_nickname: Nickname of the bucket config for the
|
|
1420
|
-
subprefix: Subprefix of the
|
|
1421
|
-
public: Whether this
|
|
1431
|
+
bucket_nickname: Nickname of the bucket config for the VCAP.
|
|
1432
|
+
subprefix: Subprefix of the VCAP to delete.
|
|
1433
|
+
public: Whether this VCAP allows public access.
|
|
1422
1434
|
"""
|
|
1423
1435
|
mstore = self._metastore_for_org(org)
|
|
1424
1436
|
bucket_id = await self._bucket_id_for_nickname(mstore, bucket_nickname)
|
|
1425
|
-
await mstore.
|
|
1437
|
+
await mstore.delete_virtual_chunk_access_policy(
|
|
1426
1438
|
bucket_id=str(bucket_id),
|
|
1427
1439
|
subprefix=subprefix.strip("/"),
|
|
1428
1440
|
public=public,
|
|
@@ -1940,33 +1952,33 @@ class Client:
|
|
|
1940
1952
|
"""
|
|
1941
1953
|
return sync(self.aclient.delete_bucket_config, org=org, nickname=nickname, imsure=imsure, imreallysure=imreallysure)
|
|
1942
1954
|
|
|
1943
|
-
def
|
|
1955
|
+
def set_virtual_chunk_access_policy(
|
|
1944
1956
|
self,
|
|
1945
1957
|
org: OrgName,
|
|
1946
1958
|
*,
|
|
1947
1959
|
bucket_nickname: BucketNickname,
|
|
1948
1960
|
subprefix: str,
|
|
1949
1961
|
public: bool,
|
|
1950
|
-
) ->
|
|
1951
|
-
"""Set an explicit virtual chunk
|
|
1962
|
+
) -> ExplicitVirtualChunkAccessPolicyResponse:
|
|
1963
|
+
"""Set an explicit virtual chunk access policy on this org.
|
|
1952
1964
|
|
|
1953
1965
|
Args:
|
|
1954
1966
|
org: Name of the organization.
|
|
1955
|
-
bucket_nickname: Nickname of the bucket config to associate with this
|
|
1956
|
-
subprefix: Subprefix within the bucket for this
|
|
1957
|
-
public: Whether this
|
|
1967
|
+
bucket_nickname: Nickname of the bucket config to associate with this VCAP.
|
|
1968
|
+
subprefix: Subprefix within the bucket for this VCAP.
|
|
1969
|
+
public: Whether this VCAP allows public access.
|
|
1958
1970
|
"""
|
|
1959
|
-
return sync(self.aclient.
|
|
1971
|
+
return sync(self.aclient.set_virtual_chunk_access_policy, org, bucket_nickname=bucket_nickname, subprefix=subprefix, public=public)
|
|
1960
1972
|
|
|
1961
|
-
def
|
|
1962
|
-
"""List explicit virtual chunk
|
|
1973
|
+
def list_virtual_chunk_access_policies(self, org: OrgName) -> list[ExplicitVirtualChunkAccessPolicyResponse]:
|
|
1974
|
+
"""List explicit virtual chunk access policies on this org.
|
|
1963
1975
|
|
|
1964
1976
|
Args:
|
|
1965
1977
|
org: Name of the organization.
|
|
1966
1978
|
"""
|
|
1967
|
-
return sync(self.aclient.
|
|
1979
|
+
return sync(self.aclient.list_virtual_chunk_access_policies, org)
|
|
1968
1980
|
|
|
1969
|
-
def
|
|
1981
|
+
def delete_virtual_chunk_access_policy(
|
|
1970
1982
|
self,
|
|
1971
1983
|
org: OrgName,
|
|
1972
1984
|
*,
|
|
@@ -1974,15 +1986,17 @@ class Client:
|
|
|
1974
1986
|
subprefix: str,
|
|
1975
1987
|
public: bool,
|
|
1976
1988
|
) -> None:
|
|
1977
|
-
"""Delete a virtual chunk
|
|
1989
|
+
"""Delete a virtual chunk access policy from this org.
|
|
1978
1990
|
|
|
1979
1991
|
Args:
|
|
1980
1992
|
org: Name of the organization.
|
|
1981
|
-
bucket_nickname: Nickname of the bucket config for the
|
|
1982
|
-
subprefix: Subprefix of the
|
|
1983
|
-
public: Whether this
|
|
1993
|
+
bucket_nickname: Nickname of the bucket config for the VCAP.
|
|
1994
|
+
subprefix: Subprefix of the VCAP to delete.
|
|
1995
|
+
public: Whether this VCAP allows public access.
|
|
1984
1996
|
"""
|
|
1985
|
-
return sync(
|
|
1997
|
+
return sync(
|
|
1998
|
+
self.aclient.delete_virtual_chunk_access_policy, org, bucket_nickname=bucket_nickname, subprefix=subprefix, public=public
|
|
1999
|
+
)
|
|
1986
2000
|
|
|
1987
2001
|
def login(self, *, browser: bool = False) -> None:
|
|
1988
2002
|
"""Login to ArrayLake.
|
|
@@ -33,7 +33,7 @@ from arraylake.types import (
|
|
|
33
33
|
RepoOperationStatusResponse,
|
|
34
34
|
S3Credentials,
|
|
35
35
|
TokenAuthenticateBody,
|
|
36
|
-
|
|
36
|
+
VirtualChunkAccessPolicyResponse,
|
|
37
37
|
)
|
|
38
38
|
|
|
39
39
|
# type adapters
|
|
@@ -41,7 +41,7 @@ LIST_DATABASES_ADAPTER = TypeAdapter(list[Repo])
|
|
|
41
41
|
LIST_DATABASES_PAGINATED_ADAPTER = TypeAdapter(PaginatedResponse[Repo])
|
|
42
42
|
LIST_BUCKETS_ADAPTER = TypeAdapter(list[BucketResponse])
|
|
43
43
|
LIST_REPOS_FOR_BUCKET_ADAPTER = TypeAdapter(list[Repo])
|
|
44
|
-
|
|
44
|
+
LIST_VCAPS_PAGINATED_ADAPTER = TypeAdapter(PaginatedResponse[VirtualChunkAccessPolicyResponse])
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
@dataclass
|
|
@@ -328,19 +328,19 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
|
|
|
328
328
|
)
|
|
329
329
|
handle_response(response)
|
|
330
330
|
|
|
331
|
-
async def
|
|
331
|
+
async def set_virtual_chunk_access_policy(
|
|
332
332
|
self,
|
|
333
333
|
*,
|
|
334
334
|
bucket_id: str,
|
|
335
335
|
subprefix: str,
|
|
336
336
|
public: bool,
|
|
337
|
-
) ->
|
|
338
|
-
"""Set an explicit virtual chunk
|
|
337
|
+
) -> VirtualChunkAccessPolicyResponse:
|
|
338
|
+
"""Set an explicit virtual chunk access policy on this org (idempotent).
|
|
339
339
|
|
|
340
340
|
Args:
|
|
341
|
-
bucket_id: Bucket ID for this
|
|
342
|
-
subprefix: Subprefix within the bucket for this
|
|
343
|
-
public: Whether this
|
|
341
|
+
bucket_id: Bucket ID for this VCAP.
|
|
342
|
+
subprefix: Subprefix within the bucket for this VCAP.
|
|
343
|
+
public: Whether this VCAP allows public access.
|
|
344
344
|
"""
|
|
345
345
|
payload: dict[str, str | bool] = {
|
|
346
346
|
"use_case": "explicit",
|
|
@@ -350,35 +350,35 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
|
|
|
350
350
|
}
|
|
351
351
|
response = await self._request(
|
|
352
352
|
"PUT",
|
|
353
|
-
f"/orgs/{self._config.org}/settings/virtual-chunk-
|
|
353
|
+
f"/orgs/{self._config.org}/settings/virtual-chunk-access-policies",
|
|
354
354
|
content=json.dumps(payload),
|
|
355
355
|
)
|
|
356
356
|
handle_response(response)
|
|
357
|
-
return
|
|
357
|
+
return VirtualChunkAccessPolicyResponse.model_validate_json(response.content)
|
|
358
358
|
|
|
359
|
-
async def
|
|
360
|
-
"""List all virtual chunk
|
|
359
|
+
async def list_virtual_chunk_access_policies(self) -> list[VirtualChunkAccessPolicyResponse]:
|
|
360
|
+
"""List all virtual chunk access policies on this org."""
|
|
361
361
|
response = await self._request(
|
|
362
362
|
"GET",
|
|
363
|
-
f"/orgs/{self._config.org}/settings/virtual-chunk-
|
|
363
|
+
f"/orgs/{self._config.org}/settings/virtual-chunk-access-policies",
|
|
364
364
|
params={"size": 100},
|
|
365
365
|
)
|
|
366
366
|
handle_response(response)
|
|
367
|
-
return
|
|
367
|
+
return LIST_VCAPS_PAGINATED_ADAPTER.validate_json(response.content).items
|
|
368
368
|
|
|
369
|
-
async def
|
|
369
|
+
async def delete_virtual_chunk_access_policy(
|
|
370
370
|
self,
|
|
371
371
|
*,
|
|
372
372
|
bucket_id: str,
|
|
373
373
|
subprefix: str,
|
|
374
374
|
public: bool,
|
|
375
375
|
) -> None:
|
|
376
|
-
"""Delete an explicit virtual chunk
|
|
376
|
+
"""Delete an explicit virtual chunk access policy from this org (idempotent).
|
|
377
377
|
|
|
378
378
|
Args:
|
|
379
|
-
bucket_id: Bucket ID of the
|
|
380
|
-
subprefix: Subprefix of the
|
|
381
|
-
public: Whether this
|
|
379
|
+
bucket_id: Bucket ID of the VCAP to delete.
|
|
380
|
+
subprefix: Subprefix of the VCAP to delete.
|
|
381
|
+
public: Whether this VCAP allows public access.
|
|
382
382
|
"""
|
|
383
383
|
payload: dict[str, str | bool] = {
|
|
384
384
|
"use_case": "explicit",
|
|
@@ -388,7 +388,7 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
|
|
|
388
388
|
}
|
|
389
389
|
response = await self._request(
|
|
390
390
|
"DELETE",
|
|
391
|
-
f"/orgs/{self._config.org}/settings/virtual-chunk-
|
|
391
|
+
f"/orgs/{self._config.org}/settings/virtual-chunk-access-policies",
|
|
392
392
|
content=json.dumps(payload),
|
|
393
393
|
)
|
|
394
394
|
handle_response(response)
|
|
@@ -8,7 +8,6 @@ from arraylake.types import (
|
|
|
8
8
|
ICECHUNK_ANY_CREDENTIAL,
|
|
9
9
|
URI,
|
|
10
10
|
UUID,
|
|
11
|
-
AnonymousAuth,
|
|
12
11
|
BucketNickname,
|
|
13
12
|
BucketPrefix,
|
|
14
13
|
BucketResponse,
|
|
@@ -62,7 +61,7 @@ def get_icechunk_container_credentials(
|
|
|
62
61
|
# https://github.com/earth-mover/icechunk/pull/776
|
|
63
62
|
if credentials:
|
|
64
63
|
assert isinstance(credentials, GSCredentials)
|
|
65
|
-
return icechunk.
|
|
64
|
+
return icechunk.GcsCredentials.Static(icechunk.GcsStaticCredentials.BearerToken(credentials.access_token))
|
|
66
65
|
else:
|
|
67
66
|
return icechunk.GcsCredentials.Anonymous()
|
|
68
67
|
else:
|
|
@@ -164,13 +163,6 @@ def forbid_unsafe_virtual_bucket_configs(bucket: BucketResponse, bucket_nickname
|
|
|
164
163
|
f"But bucket {bucket_nickname} uses HMAC credentials."
|
|
165
164
|
)
|
|
166
165
|
|
|
167
|
-
# TODO: in future we could relax this to allow buckets with delegated credentials
|
|
168
|
-
elif not isinstance(auth_method, AnonymousAuth):
|
|
169
|
-
raise ValueError(
|
|
170
|
-
"Cannot use virtual chunk references that refer to a bucket which does not support anonymous public access, "
|
|
171
|
-
f"But bucket {bucket_nickname} uses {auth_method.method if auth_method else 'unknown'} credentials."
|
|
172
|
-
)
|
|
173
|
-
|
|
174
166
|
|
|
175
167
|
def forbid_unsafe_virtual_chunk_containers(
|
|
176
168
|
container_prefixes: Iterable[BucketPrefix],
|
|
@@ -178,9 +170,6 @@ def forbid_unsafe_virtual_chunk_containers(
|
|
|
178
170
|
"""
|
|
179
171
|
Raises if any virtual chunk container prefix is somehow unsafe.
|
|
180
172
|
"""
|
|
181
|
-
# TODO Once we allow auto-authorization of existing VCCs (needed for Flux), this fn will need to be changed.
|
|
182
|
-
# TODO It will need to actually fetch the existing VCCs, check they are all safe, and only then open the repo with authorization.
|
|
183
|
-
|
|
184
173
|
for url_prefix in container_prefixes:
|
|
185
174
|
if any(url_prefix.startswith(forbidden_prefix) for forbidden_prefix in FORBIDDEN_VIRTUAL_CHUNK_PREFIXES):
|
|
186
175
|
raise ValueError(
|
|
@@ -615,8 +615,8 @@ class BucketResponse(Bucket):
|
|
|
615
615
|
auth_config: AuthConfig | None = None
|
|
616
616
|
|
|
617
617
|
|
|
618
|
-
class
|
|
619
|
-
"""Response body for a virtual chunk
|
|
618
|
+
class VirtualChunkAccessPolicyResponse(BaseModel):
|
|
619
|
+
"""Response body for a virtual chunk access policy."""
|
|
620
620
|
|
|
621
621
|
use_case: str
|
|
622
622
|
bucket_id: str
|
|
@@ -628,8 +628,8 @@ class VirtualChunkLocationResponse(BaseModel):
|
|
|
628
628
|
created_by: str
|
|
629
629
|
|
|
630
630
|
|
|
631
|
-
class
|
|
632
|
-
"""Response body for an explicit virtual chunk
|
|
631
|
+
class ExplicitVirtualChunkAccessPolicyResponse(BaseModel):
|
|
632
|
+
"""Response body for an explicit virtual chunk access policy."""
|
|
633
633
|
|
|
634
634
|
bucket_id: str
|
|
635
635
|
subprefix: str
|
|
@@ -1162,3 +1162,5 @@ class OpenRepoResponse(BaseModel):
|
|
|
1162
1162
|
author: Author
|
|
1163
1163
|
# Principal (for user-agent string in icechunk storage)
|
|
1164
1164
|
principal_id: str
|
|
1165
|
+
# Repo operation status
|
|
1166
|
+
status: RepoOperationStatusResponse
|
|
@@ -119,15 +119,19 @@ def test_repo(invoke, helpers, sync_isolated_org_with_bucket) -> None:
|
|
|
119
119
|
data = json.loads(result.stdout)
|
|
120
120
|
assert data["mode"] == "online"
|
|
121
121
|
|
|
122
|
+
# set-status sets the mode via PUT (succeeds) then calls get_repo_object to
|
|
123
|
+
# display the result. For HMAC repos, get_repo_object is now blocked when
|
|
124
|
+
# the repo is not online (409), so the CLI command exits with an error even
|
|
125
|
+
# though the mode was successfully changed.
|
|
122
126
|
result = invoke(["repo", "set-status", repo_name, "offline"])
|
|
123
|
-
assert result.exit_code
|
|
124
|
-
data = result.stdout
|
|
125
|
-
assert data.startswith("offline")
|
|
127
|
+
assert result.exit_code != 0
|
|
126
128
|
|
|
127
129
|
result = invoke(["repo", "set-status", repo_name, "maintenance", "--output", "json"])
|
|
130
|
+
assert result.exit_code != 0
|
|
131
|
+
|
|
132
|
+
# Restore to online before deleting
|
|
133
|
+
result = invoke(["repo", "set-status", repo_name, "online"])
|
|
128
134
|
assert result.exit_code == 0
|
|
129
|
-
data = json.loads(result.stdout)
|
|
130
|
-
assert data["mode"] == "maintenance"
|
|
131
135
|
|
|
132
136
|
result = invoke(["repo", "delete", repo_name, "--confirm"])
|
|
133
137
|
assert result.exit_code == 0
|
|
@@ -1017,6 +1017,42 @@ async def test_get_icechunk_container_credentials_from_bucket_delegated_sync(iso
|
|
|
1017
1017
|
assert isinstance(cont_creds, icechunk.S3Credentials.Refreshable)
|
|
1018
1018
|
|
|
1019
1019
|
|
|
1020
|
+
@pytest.mark.asyncio
|
|
1021
|
+
async def test_credentials_offline_mode_raises(isolated_org, default_bucket, token, helpers):
|
|
1022
|
+
"""Test that repos in OFFLINE mode raise an error when requesting credentials."""
|
|
1023
|
+
async with isolated_org(default_bucket()) as (org_name, buckets):
|
|
1024
|
+
aclient = AsyncClient(token=token)
|
|
1025
|
+
repo_name = f"{org_name}/{helpers.random_repo_id()}"
|
|
1026
|
+
|
|
1027
|
+
await aclient.create_repo(repo_name)
|
|
1028
|
+
|
|
1029
|
+
# Set repo to OFFLINE mode
|
|
1030
|
+
_on, _rn = repo_name.split("/")
|
|
1031
|
+
await helpers.set_repo_system_status(token, _on, _rn, RepoOperationMode.OFFLINE, "Testing offline mode", True)
|
|
1032
|
+
|
|
1033
|
+
# Attempting to get repo should raise due to credential fetch failure
|
|
1034
|
+
with pytest.raises(ValueError, match="offline"):
|
|
1035
|
+
await aclient.get_repo(repo_name)
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
@pytest.mark.asyncio
|
|
1039
|
+
async def test_credentials_maintenance_mode_hmac_raises(isolated_org, default_bucket, token, helpers):
|
|
1040
|
+
"""Test that HMAC repos in MAINTENANCE mode raise an error (HMAC creds can't be scoped to read-only)."""
|
|
1041
|
+
async with isolated_org(default_bucket()) as (org_name, buckets):
|
|
1042
|
+
aclient = AsyncClient(token=token)
|
|
1043
|
+
repo_name = f"{org_name}/{helpers.random_repo_id()}"
|
|
1044
|
+
|
|
1045
|
+
await aclient.create_repo(repo_name)
|
|
1046
|
+
|
|
1047
|
+
# Set repo to MAINTENANCE mode
|
|
1048
|
+
_on, _rn = repo_name.split("/")
|
|
1049
|
+
await helpers.set_repo_system_status(token, _on, _rn, RepoOperationMode.MAINTENANCE, "Testing maintenance mode", True)
|
|
1050
|
+
|
|
1051
|
+
# HMAC repos cannot provide read-only credentials, so the server returns 409
|
|
1052
|
+
with pytest.raises(ArraylakeClientError, match="HMAC"):
|
|
1053
|
+
await aclient.get_repo(repo_name)
|
|
1054
|
+
|
|
1055
|
+
|
|
1020
1056
|
@pytest.mark.asyncio
|
|
1021
1057
|
async def test_containers_credentials_for_buckets_async(isolated_org, default_bucket, token):
|
|
1022
1058
|
bucket1 = default_bucket(nickname="mybucket1", name="bucket1")
|
|
@@ -1189,6 +1225,7 @@ def test_get_database(respx_mock, test_user) -> None:
|
|
|
1189
1225
|
"virtual_chunk_credentials": {},
|
|
1190
1226
|
"author": {"name": "Test User", "email": "test@example.com"},
|
|
1191
1227
|
"principal_id": principal_id,
|
|
1228
|
+
"status": {"mode": "online", "initiated_by": {}, "estimated_end_time": None},
|
|
1192
1229
|
},
|
|
1193
1230
|
)
|
|
1194
1231
|
)
|
|
@@ -4,10 +4,19 @@ Performance regression tests for client operations.
|
|
|
4
4
|
These tests track API call counts and patterns to detect performance regressions.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
+
from contextlib import asynccontextmanager
|
|
8
|
+
|
|
7
9
|
import pytest
|
|
8
10
|
|
|
9
11
|
from arraylake import AsyncClient
|
|
10
12
|
|
|
13
|
+
from tests.test_virtual_chunks import vcap_for_bucket
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@asynccontextmanager
|
|
17
|
+
async def _noop_cm():
|
|
18
|
+
yield
|
|
19
|
+
|
|
11
20
|
|
|
12
21
|
@pytest.mark.asyncio
|
|
13
22
|
class TestGetRepoCallCount:
|
|
@@ -47,19 +56,22 @@ class TestGetRepoCallCount:
|
|
|
47
56
|
async with isolated_org(repo_bucket, virtual_bucket) as (org_name, buckets):
|
|
48
57
|
aclient = AsyncClient(token=token)
|
|
49
58
|
|
|
50
|
-
#
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
async with api_call_counter() as counter:
|
|
60
|
-
await aclient.get_repo(
|
|
61
|
-
name, authorize_virtual_chunk_access=authorize_virtual_chunk_access if authorize_vccs_explicitly else None
|
|
59
|
+
# VCAP required for VCC validation when virtual chunks are used
|
|
60
|
+
vcap_cm = vcap_for_bucket(aclient, org_name, virtual_bucket) if use_vccs else _noop_cm()
|
|
61
|
+
async with vcap_cm:
|
|
62
|
+
# Setup: create repo (not counted)
|
|
63
|
+
name = f"{org_name}/foo"
|
|
64
|
+
await aclient.create_repo(
|
|
65
|
+
name,
|
|
66
|
+
bucket_config_nickname=repo_bucket.nickname,
|
|
67
|
+
authorize_virtual_chunk_access=authorize_virtual_chunk_access if use_vccs else {},
|
|
62
68
|
)
|
|
63
69
|
|
|
70
|
+
# Count only get_repo calls
|
|
71
|
+
async with api_call_counter() as counter:
|
|
72
|
+
await aclient.get_repo(
|
|
73
|
+
name, authorize_virtual_chunk_access=authorize_virtual_chunk_access if authorize_vccs_explicitly else None
|
|
74
|
+
)
|
|
75
|
+
|
|
64
76
|
assert len(counter.tracked_calls) <= expected_num_calls, counter.call_log()
|
|
65
77
|
assert counter.count_sequential_calls() <= expected_num_sequential_calls, counter.call_log()
|
|
@@ -41,6 +41,38 @@ async def create_subscription_repo(token: str, org_name: str, repo_name: str, ma
|
|
|
41
41
|
return resp.json()
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
async def create_filtered_subscription_repo(
|
|
45
|
+
token: str,
|
|
46
|
+
org_name: str,
|
|
47
|
+
repo_name: str,
|
|
48
|
+
marketplace_listing_id: str,
|
|
49
|
+
bucket_nickname: str,
|
|
50
|
+
target_filter: dict,
|
|
51
|
+
):
|
|
52
|
+
"""Create a filtered subscription repo in the specified org.
|
|
53
|
+
|
|
54
|
+
Filtered subscriptions have their own storage bucket and apply a target_filter
|
|
55
|
+
to select a subset of the provider's data.
|
|
56
|
+
|
|
57
|
+
Client does not yet support filtered subscription repo creation, so we use the lower-level HTTP client here.
|
|
58
|
+
"""
|
|
59
|
+
client = ArraylakeHttpClient("http://localhost:8000", token=token)
|
|
60
|
+
body = {
|
|
61
|
+
"name": repo_name,
|
|
62
|
+
"description": "Filtered subscriber repo for subscription tests",
|
|
63
|
+
"create_mode": "subscribe",
|
|
64
|
+
"bucket_nickname": bucket_nickname,
|
|
65
|
+
"subscription": {
|
|
66
|
+
"marketplace_listing_id": str(marketplace_listing_id),
|
|
67
|
+
"subscription_type": "filtered",
|
|
68
|
+
"target_filter": target_filter,
|
|
69
|
+
},
|
|
70
|
+
}
|
|
71
|
+
resp = await client._request("POST", f"/orgs/{org_name}/repos", content=json.dumps(body))
|
|
72
|
+
assert resp.is_success, f"Failed to create filtered subscription repo in org {org_name}: {resp.status_code} {resp.content}"
|
|
73
|
+
return resp.json()
|
|
74
|
+
|
|
75
|
+
|
|
44
76
|
async def run_subscription_workflow_test(token: str, parent_org: str, subscriber_org: str):
|
|
45
77
|
"""Helper function to test subscription workflow between a parent and subscriber repo.
|
|
46
78
|
|