arraylake 0.31.0__tar.gz → 0.32.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. {arraylake-0.31.0 → arraylake-0.32.0}/PKG-INFO +1 -1
  2. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/_version.py +2 -2
  3. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/client.py +57 -43
  4. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/metastore/http_metastore.py +20 -20
  5. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/virtual.py +1 -12
  6. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/types.py +6 -4
  7. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_cli.py +9 -5
  8. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_client.py +37 -0
  9. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_performance.py +24 -12
  10. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_subscriptions.py +32 -0
  11. arraylake-0.32.0/tests/test_virtual_chunks.py +1201 -0
  12. {arraylake-0.31.0 → arraylake-0.32.0}/uv.lock +100 -102
  13. arraylake-0.31.0/tests/test_virtual_chunks.py +0 -803
  14. {arraylake-0.31.0 → arraylake-0.32.0}/.gitignore +0 -0
  15. {arraylake-0.31.0 → arraylake-0.32.0}/README.md +0 -0
  16. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/__init__.py +0 -0
  17. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/__main__.py +0 -0
  18. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/api_utils.py +0 -0
  19. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/asyn.py +0 -0
  20. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/__init__.py +0 -0
  21. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/auth.py +0 -0
  22. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/compute.py +0 -0
  23. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/config.py +0 -0
  24. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/main.py +0 -0
  25. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/repo.py +0 -0
  26. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/cli/utils.py +0 -0
  27. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/doctor.py +0 -0
  28. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/http_client.py +0 -0
  29. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/services.py +0 -0
  30. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/compute/types.py +0 -0
  31. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/config.py +0 -0
  32. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/config.yaml +0 -0
  33. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/credentials.py +0 -0
  34. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/diagnostics.py +0 -0
  35. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/display/__init__.py +0 -0
  36. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/display/repo.py +0 -0
  37. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/display/repolist.py +0 -0
  38. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/exceptions.py +0 -0
  39. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/log_util.py +0 -0
  40. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/metastore/__init__.py +0 -0
  41. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/metastore/abc.py +0 -0
  42. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/py.typed +0 -0
  43. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/__init__.py +0 -0
  44. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/__init__.py +0 -0
  45. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/storage.py +0 -0
  46. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/repos/icechunk/types.py +0 -0
  47. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/token.py +0 -0
  48. {arraylake-0.31.0 → arraylake-0.32.0}/arraylake/tuning.py +0 -0
  49. {arraylake-0.31.0 → arraylake-0.32.0}/pyproject.toml +0 -0
  50. {arraylake-0.31.0 → arraylake-0.32.0}/tests/__init__.py +0 -0
  51. {arraylake-0.31.0 → arraylake-0.32.0}/tests/compute/test_client.py +0 -0
  52. {arraylake-0.31.0 → arraylake-0.32.0}/tests/compute/test_doctor.py +0 -0
  53. {arraylake-0.31.0 → arraylake-0.32.0}/tests/config.yaml +0 -0
  54. {arraylake-0.31.0 → arraylake-0.32.0}/tests/conftest.py +0 -0
  55. {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/__init__.py +0 -0
  56. {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/conftest.py +0 -0
  57. {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/test_storage.py +0 -0
  58. {arraylake-0.31.0 → arraylake-0.32.0}/tests/icechunk/test_virtual.py +0 -0
  59. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_api_utils.py +0 -0
  60. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_asyn.py +0 -0
  61. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_config.py +0 -0
  62. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_credentials.py +0 -0
  63. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_diagnostics.py +0 -0
  64. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_token_handler.py +0 -0
  65. {arraylake-0.31.0 → arraylake-0.32.0}/tests/test_types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arraylake
3
- Version: 0.31.0
3
+ Version: 0.32.0
4
4
  Summary: Python client for ArrayLake
5
5
  Project-URL: Documentation, https://docs.earthmover.io/
6
6
  Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.31.0'
32
- __version_tuple__ = version_tuple = (0, 31, 0)
31
+ __version__ = version = '0.32.0'
32
+ __version_tuple__ = version_tuple = (0, 32, 0)
33
33
 
34
34
  __commit_id__ = commit_id = None
@@ -62,7 +62,7 @@ from arraylake.types import (
62
62
  BucketNickname,
63
63
  BucketPrefix,
64
64
  BucketResponse,
65
- ExplicitVirtualChunkLocationResponse,
65
+ ExplicitVirtualChunkAccessPolicyResponse,
66
66
  GSCredentials,
67
67
  NewBucket,
68
68
  OptimizationConfig,
@@ -506,6 +506,13 @@ class AsyncClient:
506
506
  # Single API call to get everything needed to open the repo
507
507
  response = await mstore.open_repo(repo_name)
508
508
 
509
+ if response.status.mode == RepoOperationMode.OFFLINE:
510
+ raise ValueError(f"Repository is offline, check back after {response.status.estimated_end_time}")
511
+ elif response.status.mode == RepoOperationMode.MAINTENANCE:
512
+ warnings.warn(
513
+ f"Repository is in maintenance mode until {response.status.estimated_end_time}, do not attempt to write to the store"
514
+ )
515
+
509
516
  # Build icechunk storage from the response
510
517
  credential_refresh_func = self._maybe_get_credential_refresh_func_for_icechunk(
511
518
  bucket=response.repo_bucket, org=org, repo_name=repo_name
@@ -546,18 +553,23 @@ class AsyncClient:
546
553
  forbid_configs_unsafe_for_virtual_chunks=True,
547
554
  )
548
555
  else:
549
- # Use server-provided VCC credentials (currently only anonymous-access buckets)
556
+ # Use server-provided VCC credentials
550
557
  vcc_credentials = {}
551
558
  if response.virtual_chunk_credentials:
552
559
  credentials_map: dict[str, icechunk.AnyS3Credential | icechunk.AnyGcsCredential | icechunk.AnyAzureCredential | None] = {}
553
560
  for vcc_prefix, vcc_creds in response.virtual_chunk_credentials.items():
554
- # TODO: once non-anonymous VCC buckets are supported, build refresh closures here
555
- # using vcc_creds.org, vcc_creds.bucket_nickname, and vcc_creds.platform
556
- if vcc_creds.credentials is not None:
557
- raise NotImplementedError(
558
- f"Non-anonymous credentials for buckets are not yet supported for virtual chunks. (prefix: {vcc_prefix})."
559
- )
560
- credentials_map[vcc_prefix] = None
561
+ # TODO: build refresh closures here using vcc_creds.org, vcc_creds.bucket_nickname,
562
+ # and vcc_creds.platform so that credentials can be refreshed when they expire. (See https://github.com/earth-mover/arraylake/issues/5122)
563
+ # TODO: support Azure virtual chunk credentials
564
+ if isinstance(vcc_creds.credentials, AzureCredentials):
565
+ raise NotImplementedError(f"Azure virtual chunk credentials are not yet supported. (prefix: {vcc_prefix}).")
566
+ # TODO: fix return type of get_icechunk_container_credentials (returns GcsBearerCredential
567
+ # which is not in icechunk's AnyGcsCredential union — icechunk typing gap)
568
+ credentials_map[vcc_prefix] = get_icechunk_container_credentials( # type: ignore[assignment]
569
+ bucket_platform=vcc_creds.platform,
570
+ credentials=vcc_creds.credentials,
571
+ credential_refresh_func=None,
572
+ )
561
573
  vcc_credentials = icechunk.containers_credentials(credentials_map)
562
574
 
563
575
  ic_repo: IcechunkRepository = await IcechunkRepository.open_async(
@@ -1369,42 +1381,42 @@ class AsyncClient:
1369
1381
  bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
1370
1382
  await mstore.delete_bucket_config(bucket_id)
1371
1383
 
1372
- async def set_virtual_chunk_location(
1384
+ async def set_virtual_chunk_access_policy(
1373
1385
  self,
1374
1386
  org: OrgName,
1375
1387
  *,
1376
1388
  bucket_nickname: BucketNickname,
1377
1389
  subprefix: str = "",
1378
1390
  public: bool = False,
1379
- ) -> ExplicitVirtualChunkLocationResponse:
1380
- """Set an explicit virtual chunk location on this org.
1391
+ ) -> ExplicitVirtualChunkAccessPolicyResponse:
1392
+ """Set an explicit virtual chunk access policy on this org.
1381
1393
 
1382
1394
  Args:
1383
1395
  org: Name of the organization.
1384
- bucket_nickname: Nickname of the bucket config to associate with this VCL.
1385
- subprefix: Subprefix within the bucket for this VCL.
1386
- public: Whether this VCL allows public access.
1396
+ bucket_nickname: Nickname of the bucket config to associate with this VCAP.
1397
+ subprefix: Subprefix within the bucket for this VCAP.
1398
+ public: Whether this VCAP allows public access.
1387
1399
  """
1388
1400
  mstore = self._metastore_for_org(org)
1389
1401
  bucket_id = await self._bucket_id_for_nickname(mstore, bucket_nickname)
1390
- vcl = await mstore.set_virtual_chunk_location(
1402
+ vcap = await mstore.set_virtual_chunk_access_policy(
1391
1403
  bucket_id=str(bucket_id),
1392
1404
  subprefix=subprefix.strip("/"),
1393
1405
  public=public,
1394
1406
  )
1395
- return ExplicitVirtualChunkLocationResponse.model_validate(vcl.model_dump())
1407
+ return ExplicitVirtualChunkAccessPolicyResponse.model_validate(vcap.model_dump())
1396
1408
 
1397
- async def list_virtual_chunk_locations(self, org: OrgName) -> list[ExplicitVirtualChunkLocationResponse]:
1398
- """List explicit virtual chunk locations on this org.
1409
+ async def list_virtual_chunk_access_policies(self, org: OrgName) -> list[ExplicitVirtualChunkAccessPolicyResponse]:
1410
+ """List explicit virtual chunk access policies on this org.
1399
1411
 
1400
1412
  Args:
1401
1413
  org: Name of the organization.
1402
1414
  """
1403
1415
  mstore = self._metastore_for_org(org)
1404
- vcls = await mstore.list_virtual_chunk_locations()
1405
- return [ExplicitVirtualChunkLocationResponse.model_validate(v.model_dump()) for v in vcls if v.use_case == "explicit"]
1416
+ vcaps = await mstore.list_virtual_chunk_access_policies()
1417
+ return [ExplicitVirtualChunkAccessPolicyResponse.model_validate(v.model_dump()) for v in vcaps if v.use_case == "explicit"]
1406
1418
 
1407
- async def delete_virtual_chunk_location(
1419
+ async def delete_virtual_chunk_access_policy(
1408
1420
  self,
1409
1421
  org: OrgName,
1410
1422
  *,
@@ -1412,17 +1424,17 @@ class AsyncClient:
1412
1424
  subprefix: str,
1413
1425
  public: bool,
1414
1426
  ) -> None:
1415
- """Delete a virtual chunk location from this org (idempotent).
1427
+ """Delete a virtual chunk access policy from this org (idempotent).
1416
1428
 
1417
1429
  Args:
1418
1430
  org: Name of the organization.
1419
- bucket_nickname: Nickname of the bucket config for the VCL.
1420
- subprefix: Subprefix of the VCL to delete.
1421
- public: Whether this VCL allows public access.
1431
+ bucket_nickname: Nickname of the bucket config for the VCAP.
1432
+ subprefix: Subprefix of the VCAP to delete.
1433
+ public: Whether this VCAP allows public access.
1422
1434
  """
1423
1435
  mstore = self._metastore_for_org(org)
1424
1436
  bucket_id = await self._bucket_id_for_nickname(mstore, bucket_nickname)
1425
- await mstore.delete_virtual_chunk_location(
1437
+ await mstore.delete_virtual_chunk_access_policy(
1426
1438
  bucket_id=str(bucket_id),
1427
1439
  subprefix=subprefix.strip("/"),
1428
1440
  public=public,
@@ -1940,33 +1952,33 @@ class Client:
1940
1952
  """
1941
1953
  return sync(self.aclient.delete_bucket_config, org=org, nickname=nickname, imsure=imsure, imreallysure=imreallysure)
1942
1954
 
1943
- def set_virtual_chunk_location(
1955
+ def set_virtual_chunk_access_policy(
1944
1956
  self,
1945
1957
  org: OrgName,
1946
1958
  *,
1947
1959
  bucket_nickname: BucketNickname,
1948
1960
  subprefix: str,
1949
1961
  public: bool,
1950
- ) -> ExplicitVirtualChunkLocationResponse:
1951
- """Set an explicit virtual chunk location on this org.
1962
+ ) -> ExplicitVirtualChunkAccessPolicyResponse:
1963
+ """Set an explicit virtual chunk access policy on this org.
1952
1964
 
1953
1965
  Args:
1954
1966
  org: Name of the organization.
1955
- bucket_nickname: Nickname of the bucket config to associate with this VCL.
1956
- subprefix: Subprefix within the bucket for this VCL.
1957
- public: Whether this VCL allows public access.
1967
+ bucket_nickname: Nickname of the bucket config to associate with this VCAP.
1968
+ subprefix: Subprefix within the bucket for this VCAP.
1969
+ public: Whether this VCAP allows public access.
1958
1970
  """
1959
- return sync(self.aclient.set_virtual_chunk_location, org, bucket_nickname=bucket_nickname, subprefix=subprefix, public=public)
1971
+ return sync(self.aclient.set_virtual_chunk_access_policy, org, bucket_nickname=bucket_nickname, subprefix=subprefix, public=public)
1960
1972
 
1961
- def list_virtual_chunk_locations(self, org: OrgName) -> list[ExplicitVirtualChunkLocationResponse]:
1962
- """List explicit virtual chunk locations on this org.
1973
+ def list_virtual_chunk_access_policies(self, org: OrgName) -> list[ExplicitVirtualChunkAccessPolicyResponse]:
1974
+ """List explicit virtual chunk access policies on this org.
1963
1975
 
1964
1976
  Args:
1965
1977
  org: Name of the organization.
1966
1978
  """
1967
- return sync(self.aclient.list_virtual_chunk_locations, org)
1979
+ return sync(self.aclient.list_virtual_chunk_access_policies, org)
1968
1980
 
1969
- def delete_virtual_chunk_location(
1981
+ def delete_virtual_chunk_access_policy(
1970
1982
  self,
1971
1983
  org: OrgName,
1972
1984
  *,
@@ -1974,15 +1986,17 @@ class Client:
1974
1986
  subprefix: str,
1975
1987
  public: bool,
1976
1988
  ) -> None:
1977
- """Delete a virtual chunk location from this org.
1989
+ """Delete a virtual chunk access policy from this org.
1978
1990
 
1979
1991
  Args:
1980
1992
  org: Name of the organization.
1981
- bucket_nickname: Nickname of the bucket config for the VCL.
1982
- subprefix: Subprefix of the VCL to delete.
1983
- public: Whether this VCL allows public access.
1993
+ bucket_nickname: Nickname of the bucket config for the VCAP.
1994
+ subprefix: Subprefix of the VCAP to delete.
1995
+ public: Whether this VCAP allows public access.
1984
1996
  """
1985
- return sync(self.aclient.delete_virtual_chunk_location, org, bucket_nickname=bucket_nickname, subprefix=subprefix, public=public)
1997
+ return sync(
1998
+ self.aclient.delete_virtual_chunk_access_policy, org, bucket_nickname=bucket_nickname, subprefix=subprefix, public=public
1999
+ )
1986
2000
 
1987
2001
  def login(self, *, browser: bool = False) -> None:
1988
2002
  """Login to ArrayLake.
@@ -33,7 +33,7 @@ from arraylake.types import (
33
33
  RepoOperationStatusResponse,
34
34
  S3Credentials,
35
35
  TokenAuthenticateBody,
36
- VirtualChunkLocationResponse,
36
+ VirtualChunkAccessPolicyResponse,
37
37
  )
38
38
 
39
39
  # type adapters
@@ -41,7 +41,7 @@ LIST_DATABASES_ADAPTER = TypeAdapter(list[Repo])
41
41
  LIST_DATABASES_PAGINATED_ADAPTER = TypeAdapter(PaginatedResponse[Repo])
42
42
  LIST_BUCKETS_ADAPTER = TypeAdapter(list[BucketResponse])
43
43
  LIST_REPOS_FOR_BUCKET_ADAPTER = TypeAdapter(list[Repo])
44
- LIST_VCLS_PAGINATED_ADAPTER = TypeAdapter(PaginatedResponse[VirtualChunkLocationResponse])
44
+ LIST_VCAPS_PAGINATED_ADAPTER = TypeAdapter(PaginatedResponse[VirtualChunkAccessPolicyResponse])
45
45
 
46
46
 
47
47
  @dataclass
@@ -328,19 +328,19 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
328
328
  )
329
329
  handle_response(response)
330
330
 
331
- async def set_virtual_chunk_location(
331
+ async def set_virtual_chunk_access_policy(
332
332
  self,
333
333
  *,
334
334
  bucket_id: str,
335
335
  subprefix: str,
336
336
  public: bool,
337
- ) -> VirtualChunkLocationResponse:
338
- """Set an explicit virtual chunk location on this org (idempotent).
337
+ ) -> VirtualChunkAccessPolicyResponse:
338
+ """Set an explicit virtual chunk access policy on this org (idempotent).
339
339
 
340
340
  Args:
341
- bucket_id: Bucket ID for this VCL.
342
- subprefix: Subprefix within the bucket for this VCL.
343
- public: Whether this VCL allows public access.
341
+ bucket_id: Bucket ID for this VCAP.
342
+ subprefix: Subprefix within the bucket for this VCAP.
343
+ public: Whether this VCAP allows public access.
344
344
  """
345
345
  payload: dict[str, str | bool] = {
346
346
  "use_case": "explicit",
@@ -350,35 +350,35 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
350
350
  }
351
351
  response = await self._request(
352
352
  "PUT",
353
- f"/orgs/{self._config.org}/settings/virtual-chunk-locations",
353
+ f"/orgs/{self._config.org}/settings/virtual-chunk-access-policies",
354
354
  content=json.dumps(payload),
355
355
  )
356
356
  handle_response(response)
357
- return VirtualChunkLocationResponse.model_validate_json(response.content)
357
+ return VirtualChunkAccessPolicyResponse.model_validate_json(response.content)
358
358
 
359
- async def list_virtual_chunk_locations(self) -> list[VirtualChunkLocationResponse]:
360
- """List all virtual chunk locations on this org."""
359
+ async def list_virtual_chunk_access_policies(self) -> list[VirtualChunkAccessPolicyResponse]:
360
+ """List all virtual chunk access policies on this org."""
361
361
  response = await self._request(
362
362
  "GET",
363
- f"/orgs/{self._config.org}/settings/virtual-chunk-locations",
363
+ f"/orgs/{self._config.org}/settings/virtual-chunk-access-policies",
364
364
  params={"size": 100},
365
365
  )
366
366
  handle_response(response)
367
- return LIST_VCLS_PAGINATED_ADAPTER.validate_json(response.content).items
367
+ return LIST_VCAPS_PAGINATED_ADAPTER.validate_json(response.content).items
368
368
 
369
- async def delete_virtual_chunk_location(
369
+ async def delete_virtual_chunk_access_policy(
370
370
  self,
371
371
  *,
372
372
  bucket_id: str,
373
373
  subprefix: str,
374
374
  public: bool,
375
375
  ) -> None:
376
- """Delete an explicit virtual chunk location from this org (idempotent).
376
+ """Delete an explicit virtual chunk access policy from this org (idempotent).
377
377
 
378
378
  Args:
379
- bucket_id: Bucket ID of the VCL to delete.
380
- subprefix: Subprefix of the VCL to delete.
381
- public: Whether this VCL allows public access.
379
+ bucket_id: Bucket ID of the VCAP to delete.
380
+ subprefix: Subprefix of the VCAP to delete.
381
+ public: Whether this VCAP allows public access.
382
382
  """
383
383
  payload: dict[str, str | bool] = {
384
384
  "use_case": "explicit",
@@ -388,7 +388,7 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
388
388
  }
389
389
  response = await self._request(
390
390
  "DELETE",
391
- f"/orgs/{self._config.org}/settings/virtual-chunk-locations",
391
+ f"/orgs/{self._config.org}/settings/virtual-chunk-access-policies",
392
392
  content=json.dumps(payload),
393
393
  )
394
394
  handle_response(response)
@@ -8,7 +8,6 @@ from arraylake.types import (
8
8
  ICECHUNK_ANY_CREDENTIAL,
9
9
  URI,
10
10
  UUID,
11
- AnonymousAuth,
12
11
  BucketNickname,
13
12
  BucketPrefix,
14
13
  BucketResponse,
@@ -62,7 +61,7 @@ def get_icechunk_container_credentials(
62
61
  # https://github.com/earth-mover/icechunk/pull/776
63
62
  if credentials:
64
63
  assert isinstance(credentials, GSCredentials)
65
- return icechunk.GcsBearerCredential(bearer=credentials.access_token, expires_after=credentials.expiration)
64
+ return icechunk.GcsCredentials.Static(icechunk.GcsStaticCredentials.BearerToken(credentials.access_token))
66
65
  else:
67
66
  return icechunk.GcsCredentials.Anonymous()
68
67
  else:
@@ -164,13 +163,6 @@ def forbid_unsafe_virtual_bucket_configs(bucket: BucketResponse, bucket_nickname
164
163
  f"But bucket {bucket_nickname} uses HMAC credentials."
165
164
  )
166
165
 
167
- # TODO: in future we could relax this to allow buckets with delegated credentials
168
- elif not isinstance(auth_method, AnonymousAuth):
169
- raise ValueError(
170
- "Cannot use virtual chunk references that refer to a bucket which does not support anonymous public access, "
171
- f"But bucket {bucket_nickname} uses {auth_method.method if auth_method else 'unknown'} credentials."
172
- )
173
-
174
166
 
175
167
  def forbid_unsafe_virtual_chunk_containers(
176
168
  container_prefixes: Iterable[BucketPrefix],
@@ -178,9 +170,6 @@ def forbid_unsafe_virtual_chunk_containers(
178
170
  """
179
171
  Raises if any virtual chunk container prefix is somehow unsafe.
180
172
  """
181
- # TODO Once we allow auto-authorization of existing VCCs (needed for Flux), this fn will need to be changed.
182
- # TODO It will need to actually fetch the existing VCCs, check they are all safe, and only then open the repo with authorization.
183
-
184
173
  for url_prefix in container_prefixes:
185
174
  if any(url_prefix.startswith(forbidden_prefix) for forbidden_prefix in FORBIDDEN_VIRTUAL_CHUNK_PREFIXES):
186
175
  raise ValueError(
@@ -615,8 +615,8 @@ class BucketResponse(Bucket):
615
615
  auth_config: AuthConfig | None = None
616
616
 
617
617
 
618
- class VirtualChunkLocationResponse(BaseModel):
619
- """Response body for a virtual chunk location."""
618
+ class VirtualChunkAccessPolicyResponse(BaseModel):
619
+ """Response body for a virtual chunk access policy."""
620
620
 
621
621
  use_case: str
622
622
  bucket_id: str
@@ -628,8 +628,8 @@ class VirtualChunkLocationResponse(BaseModel):
628
628
  created_by: str
629
629
 
630
630
 
631
- class ExplicitVirtualChunkLocationResponse(BaseModel):
632
- """Response body for an explicit virtual chunk location."""
631
+ class ExplicitVirtualChunkAccessPolicyResponse(BaseModel):
632
+ """Response body for an explicit virtual chunk access policy."""
633
633
 
634
634
  bucket_id: str
635
635
  subprefix: str
@@ -1162,3 +1162,5 @@ class OpenRepoResponse(BaseModel):
1162
1162
  author: Author
1163
1163
  # Principal (for user-agent string in icechunk storage)
1164
1164
  principal_id: str
1165
+ # Repo operation status
1166
+ status: RepoOperationStatusResponse
@@ -119,15 +119,19 @@ def test_repo(invoke, helpers, sync_isolated_org_with_bucket) -> None:
119
119
  data = json.loads(result.stdout)
120
120
  assert data["mode"] == "online"
121
121
 
122
+ # set-status sets the mode via PUT (succeeds) then calls get_repo_object to
123
+ # display the result. For HMAC repos, get_repo_object is now blocked when
124
+ # the repo is not online (409), so the CLI command exits with an error even
125
+ # though the mode was successfully changed.
122
126
  result = invoke(["repo", "set-status", repo_name, "offline"])
123
- assert result.exit_code == 0
124
- data = result.stdout
125
- assert data.startswith("offline")
127
+ assert result.exit_code != 0
126
128
 
127
129
  result = invoke(["repo", "set-status", repo_name, "maintenance", "--output", "json"])
130
+ assert result.exit_code != 0
131
+
132
+ # Restore to online before deleting
133
+ result = invoke(["repo", "set-status", repo_name, "online"])
128
134
  assert result.exit_code == 0
129
- data = json.loads(result.stdout)
130
- assert data["mode"] == "maintenance"
131
135
 
132
136
  result = invoke(["repo", "delete", repo_name, "--confirm"])
133
137
  assert result.exit_code == 0
@@ -1017,6 +1017,42 @@ async def test_get_icechunk_container_credentials_from_bucket_delegated_sync(iso
1017
1017
  assert isinstance(cont_creds, icechunk.S3Credentials.Refreshable)
1018
1018
 
1019
1019
 
1020
+ @pytest.mark.asyncio
1021
+ async def test_credentials_offline_mode_raises(isolated_org, default_bucket, token, helpers):
1022
+ """Test that repos in OFFLINE mode raise an error when requesting credentials."""
1023
+ async with isolated_org(default_bucket()) as (org_name, buckets):
1024
+ aclient = AsyncClient(token=token)
1025
+ repo_name = f"{org_name}/{helpers.random_repo_id()}"
1026
+
1027
+ await aclient.create_repo(repo_name)
1028
+
1029
+ # Set repo to OFFLINE mode
1030
+ _on, _rn = repo_name.split("/")
1031
+ await helpers.set_repo_system_status(token, _on, _rn, RepoOperationMode.OFFLINE, "Testing offline mode", True)
1032
+
1033
+ # Attempting to get repo should raise due to credential fetch failure
1034
+ with pytest.raises(ValueError, match="offline"):
1035
+ await aclient.get_repo(repo_name)
1036
+
1037
+
1038
+ @pytest.mark.asyncio
1039
+ async def test_credentials_maintenance_mode_hmac_raises(isolated_org, default_bucket, token, helpers):
1040
+ """Test that HMAC repos in MAINTENANCE mode raise an error (HMAC creds can't be scoped to read-only)."""
1041
+ async with isolated_org(default_bucket()) as (org_name, buckets):
1042
+ aclient = AsyncClient(token=token)
1043
+ repo_name = f"{org_name}/{helpers.random_repo_id()}"
1044
+
1045
+ await aclient.create_repo(repo_name)
1046
+
1047
+ # Set repo to MAINTENANCE mode
1048
+ _on, _rn = repo_name.split("/")
1049
+ await helpers.set_repo_system_status(token, _on, _rn, RepoOperationMode.MAINTENANCE, "Testing maintenance mode", True)
1050
+
1051
+ # HMAC repos cannot provide read-only credentials, so the server returns 409
1052
+ with pytest.raises(ArraylakeClientError, match="HMAC"):
1053
+ await aclient.get_repo(repo_name)
1054
+
1055
+
1020
1056
  @pytest.mark.asyncio
1021
1057
  async def test_containers_credentials_for_buckets_async(isolated_org, default_bucket, token):
1022
1058
  bucket1 = default_bucket(nickname="mybucket1", name="bucket1")
@@ -1189,6 +1225,7 @@ def test_get_database(respx_mock, test_user) -> None:
1189
1225
  "virtual_chunk_credentials": {},
1190
1226
  "author": {"name": "Test User", "email": "test@example.com"},
1191
1227
  "principal_id": principal_id,
1228
+ "status": {"mode": "online", "initiated_by": {}, "estimated_end_time": None},
1192
1229
  },
1193
1230
  )
1194
1231
  )
@@ -4,10 +4,19 @@ Performance regression tests for client operations.
4
4
  These tests track API call counts and patterns to detect performance regressions.
5
5
  """
6
6
 
7
+ from contextlib import asynccontextmanager
8
+
7
9
  import pytest
8
10
 
9
11
  from arraylake import AsyncClient
10
12
 
13
+ from tests.test_virtual_chunks import vcap_for_bucket
14
+
15
+
16
+ @asynccontextmanager
17
+ async def _noop_cm():
18
+ yield
19
+
11
20
 
12
21
  @pytest.mark.asyncio
13
22
  class TestGetRepoCallCount:
@@ -47,19 +56,22 @@ class TestGetRepoCallCount:
47
56
  async with isolated_org(repo_bucket, virtual_bucket) as (org_name, buckets):
48
57
  aclient = AsyncClient(token=token)
49
58
 
50
- # Setup: create repo (not counted)
51
- name = f"{org_name}/foo"
52
- await aclient.create_repo(
53
- name,
54
- bucket_config_nickname=repo_bucket.nickname,
55
- authorize_virtual_chunk_access=authorize_virtual_chunk_access if use_vccs else {},
56
- )
57
-
58
- # Count only get_repo calls
59
- async with api_call_counter() as counter:
60
- await aclient.get_repo(
61
- name, authorize_virtual_chunk_access=authorize_virtual_chunk_access if authorize_vccs_explicitly else None
59
+ # VCAP required for VCC validation when virtual chunks are used
60
+ vcap_cm = vcap_for_bucket(aclient, org_name, virtual_bucket) if use_vccs else _noop_cm()
61
+ async with vcap_cm:
62
+ # Setup: create repo (not counted)
63
+ name = f"{org_name}/foo"
64
+ await aclient.create_repo(
65
+ name,
66
+ bucket_config_nickname=repo_bucket.nickname,
67
+ authorize_virtual_chunk_access=authorize_virtual_chunk_access if use_vccs else {},
62
68
  )
63
69
 
70
+ # Count only get_repo calls
71
+ async with api_call_counter() as counter:
72
+ await aclient.get_repo(
73
+ name, authorize_virtual_chunk_access=authorize_virtual_chunk_access if authorize_vccs_explicitly else None
74
+ )
75
+
64
76
  assert len(counter.tracked_calls) <= expected_num_calls, counter.call_log()
65
77
  assert counter.count_sequential_calls() <= expected_num_sequential_calls, counter.call_log()
@@ -41,6 +41,38 @@ async def create_subscription_repo(token: str, org_name: str, repo_name: str, ma
41
41
  return resp.json()
42
42
 
43
43
 
44
+ async def create_filtered_subscription_repo(
45
+ token: str,
46
+ org_name: str,
47
+ repo_name: str,
48
+ marketplace_listing_id: str,
49
+ bucket_nickname: str,
50
+ target_filter: dict,
51
+ ):
52
+ """Create a filtered subscription repo in the specified org.
53
+
54
+ Filtered subscriptions have their own storage bucket and apply a target_filter
55
+ to select a subset of the provider's data.
56
+
57
+ Client does not yet support filtered subscription repo creation, so we use the lower-level HTTP client here.
58
+ """
59
+ client = ArraylakeHttpClient("http://localhost:8000", token=token)
60
+ body = {
61
+ "name": repo_name,
62
+ "description": "Filtered subscriber repo for subscription tests",
63
+ "create_mode": "subscribe",
64
+ "bucket_nickname": bucket_nickname,
65
+ "subscription": {
66
+ "marketplace_listing_id": str(marketplace_listing_id),
67
+ "subscription_type": "filtered",
68
+ "target_filter": target_filter,
69
+ },
70
+ }
71
+ resp = await client._request("POST", f"/orgs/{org_name}/repos", content=json.dumps(body))
72
+ assert resp.is_success, f"Failed to create filtered subscription repo in org {org_name}: {resp.status_code} {resp.content}"
73
+ return resp.json()
74
+
75
+
44
76
  async def run_subscription_workflow_test(token: str, parent_org: str, subscriber_org: str):
45
77
  """Helper function to test subscription workflow between a parent and subscriber repo.
46
78