arraylake 0.33.0__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {arraylake-0.33.0 → arraylake-1.0.0}/PKG-INFO +5 -7
  2. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/_version.py +2 -2
  3. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/api_utils.py +8 -0
  4. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/client.py +14 -4
  5. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/virtual.py +0 -22
  6. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/types.py +75 -0
  7. {arraylake-0.33.0 → arraylake-1.0.0}/pyproject.toml +6 -8
  8. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_api_utils.py +30 -3
  9. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_client.py +41 -0
  10. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_virtual_chunks.py +21 -6
  11. {arraylake-0.33.0 → arraylake-1.0.0}/uv.lock +437 -431
  12. {arraylake-0.33.0 → arraylake-1.0.0}/.gitignore +0 -0
  13. {arraylake-0.33.0 → arraylake-1.0.0}/README.md +0 -0
  14. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/__init__.py +0 -0
  15. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/__main__.py +0 -0
  16. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/asyn.py +0 -0
  17. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/__init__.py +0 -0
  18. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/auth.py +0 -0
  19. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/compute.py +0 -0
  20. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/config.py +0 -0
  21. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/main.py +0 -0
  22. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/repo.py +0 -0
  23. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/utils.py +0 -0
  24. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/doctor.py +0 -0
  25. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/http_client.py +0 -0
  26. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/services.py +0 -0
  27. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/types.py +0 -0
  28. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/config.py +0 -0
  29. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/config.yaml +0 -0
  30. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/credentials.py +0 -0
  31. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/diagnostics.py +0 -0
  32. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/display/__init__.py +0 -0
  33. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/display/repo.py +0 -0
  34. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/display/repolist.py +0 -0
  35. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/exceptions.py +0 -0
  36. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/log_util.py +0 -0
  37. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/metastore/__init__.py +0 -0
  38. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/metastore/abc.py +0 -0
  39. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/metastore/http_metastore.py +0 -0
  40. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/py.typed +0 -0
  41. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/__init__.py +0 -0
  42. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/__init__.py +0 -0
  43. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/storage.py +0 -0
  44. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/types.py +0 -0
  45. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/token.py +0 -0
  46. {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/tuning.py +0 -0
  47. {arraylake-0.33.0 → arraylake-1.0.0}/tests/__init__.py +0 -0
  48. {arraylake-0.33.0 → arraylake-1.0.0}/tests/compute/test_client.py +0 -0
  49. {arraylake-0.33.0 → arraylake-1.0.0}/tests/compute/test_doctor.py +0 -0
  50. {arraylake-0.33.0 → arraylake-1.0.0}/tests/config.yaml +0 -0
  51. {arraylake-0.33.0 → arraylake-1.0.0}/tests/conftest.py +0 -0
  52. {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/__init__.py +0 -0
  53. {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/conftest.py +0 -0
  54. {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/test_storage.py +0 -0
  55. {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/test_virtual.py +0 -0
  56. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_asyn.py +0 -0
  57. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_cli.py +0 -0
  58. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_config.py +0 -0
  59. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_credentials.py +0 -0
  60. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_diagnostics.py +0 -0
  61. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_performance.py +0 -0
  62. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_subscriptions.py +0 -0
  63. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_token_handler.py +0 -0
  64. {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_types.py +0 -0
@@ -1,20 +1,20 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arraylake
3
- Version: 0.33.0
3
+ Version: 1.0.0
4
4
  Summary: Python client for ArrayLake
5
5
  Project-URL: Documentation, https://docs.earthmover.io/
6
6
  Project-URL: Changelog, https://docs.earthmover.io/changelog
7
7
  Author-email: Joseph Hamman <joe@earthmover.io>
8
8
  License-Expression: MIT
9
9
  Classifier: Programming Language :: Python
10
- Classifier: Programming Language :: Python :: 3.11
11
10
  Classifier: Programming Language :: Python :: 3.12
12
11
  Classifier: Programming Language :: Python :: 3.13
13
- Requires-Python: <3.14,>=3.11
12
+ Classifier: Programming Language :: Python :: 3.14
13
+ Requires-Python: <3.15,>=3.12
14
14
  Requires-Dist: click<9.0,>=8.1.3
15
15
  Requires-Dist: donfig<1.0,>=0.7
16
- Requires-Dist: httpx<0.28,>=0.23
17
- Requires-Dist: icechunk<2.0.0,>=1.1.0
16
+ Requires-Dist: httpx<0.29,>=0.23
17
+ Requires-Dist: icechunk<3.0.0,>=2.0.1
18
18
  Requires-Dist: numpy<3.0,>=1.23
19
19
  Requires-Dist: packaging<26.0,>=23.0
20
20
  Requires-Dist: pydantic[email]<3.0,>=2.9
@@ -26,8 +26,6 @@ Requires-Dist: typer<1.0,>=0.12.0
26
26
  Requires-Dist: zarr<4.0,>=3.1.0
27
27
  Provides-Extra: maximal
28
28
  Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
29
- Requires-Dist: numpy<2.3; extra == 'maximal'
30
- Requires-Dist: pyarrow<22; extra == 'maximal'
31
29
  Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
32
30
  Provides-Extra: xarray
33
31
  Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.33.0'
22
- __version_tuple__ = version_tuple = (0, 33, 0)
21
+ __version__ = version = '1.0.0'
22
+ __version_tuple__ = version_tuple = (1, 0, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -4,6 +4,7 @@ import asyncio
4
4
  import json
5
5
  import os
6
6
  import random
7
+ import warnings
7
8
  import weakref
8
9
  from collections.abc import AsyncGenerator, Callable, Generator, Mapping
9
10
  from contextlib import _AsyncGeneratorContextManager
@@ -430,8 +431,15 @@ def _exception_log_debug(request: httpx.Request, response: httpx.Response):
430
431
  )
431
432
 
432
433
 
434
+ _emitted_deprecation_warnings: set[str] = set()
435
+
436
+
433
437
  def handle_response(response: httpx.Response):
434
438
  """Convenience function to handle response status codes"""
439
+ deprecation_msg = response.headers.get("X-Deprecation-Warning")
440
+ if deprecation_msg and deprecation_msg not in _emitted_deprecation_warnings:
441
+ _emitted_deprecation_warnings.add(deprecation_msg)
442
+ warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2)
435
443
  try:
436
444
  response.raise_for_status()
437
445
  except httpx.RequestError as exc:
@@ -46,7 +46,6 @@ from arraylake.repos.icechunk.storage import (
46
46
  _get_icechunk_storage_obj,
47
47
  )
48
48
  from arraylake.repos.icechunk.virtual import (
49
- create_vcc_store_config_for_platform,
50
49
  create_virtual_chunk_container,
51
50
  forbid_unsafe_virtual_bucket_configs,
52
51
  forbid_unsafe_virtual_chunk_containers,
@@ -561,7 +560,8 @@ class AsyncClient:
561
560
  for vcc_prefix, vcc_creds in response.virtual_chunk_credentials.items():
562
561
  # Handle runtime VCCs (e.g., filtered subscriptions) - create VCC on the fly
563
562
  if vcc_creds.runtime_vcc_name is not None:
564
- store_config = create_vcc_store_config_for_platform(vcc_creds.platform)
563
+ bucket_config = await self.get_bucket_config(org=vcc_creds.org, nickname=vcc_creds.bucket_nickname)
564
+ store_config = bucket_config.to_vcc_store_config(user_id=UUID(response.principal_id))
565
565
  container = icechunk.VirtualChunkContainer(
566
566
  url_prefix=vcc_prefix,
567
567
  store=store_config,
@@ -734,6 +734,7 @@ class AsyncClient:
734
734
  config: icechunk.RepositoryConfig | None = None,
735
735
  authorize_virtual_chunk_access: Mapping[BucketPrefix, BucketNickname] | None = None,
736
736
  storage_options: StorageOptions | None = None,
737
+ spec_version: int = 2,
737
738
  ) -> IcechunkRepository:
738
739
  """Create a new repo.
739
740
 
@@ -751,6 +752,7 @@ class AsyncClient:
751
752
  and used for the containers. Must be supplied as complete urls.
752
753
  storage_options: Optional storage options for the underlying Icechunk storage.
753
754
  Currently supports `network_stream_timeout_seconds` for S3, Tigris, and R2 storage.
755
+ spec_version: The Icechunk spec version to use for the repo. Defaults to 2.
754
756
 
755
757
  Returns:
756
758
  icechunk.Repository object for the repo.
@@ -808,10 +810,15 @@ class AsyncClient:
808
810
  # TODO: make icechunk accept an empty dict instead of None
809
811
  prefixes_credential_mapping = None
810
812
 
813
+ create_kwargs: dict[str, object] = {
814
+ "config": config,
815
+ "authorize_virtual_chunk_access": prefixes_credential_mapping,
816
+ "spec_version": spec_version,
817
+ }
818
+
811
819
  ic_repo = IcechunkRepository.create(
812
820
  icechunk_storage,
813
- config=config,
814
- authorize_virtual_chunk_access=prefixes_credential_mapping,
821
+ **create_kwargs, # type: ignore[arg-type]
815
822
  )
816
823
 
817
824
  ic_repo.set_default_commit_metadata({"author_name": author.name, "author_email": author.email})
@@ -1667,6 +1674,7 @@ class Client:
1667
1674
  config: icechunk.RepositoryConfig | None = None,
1668
1675
  authorize_virtual_chunk_access: Mapping[BucketPrefix, BucketNickname] | None = None,
1669
1676
  storage_options: StorageOptions | None = None,
1677
+ spec_version: int = 2,
1670
1678
  ) -> IcechunkRepository:
1671
1679
  """Create a new repo
1672
1680
 
@@ -1685,6 +1693,7 @@ class Client:
1685
1693
  and used for the containers. Used for Icechunk repos only.
1686
1694
  storage_options: Optional storage options for the underlying Icechunk storage.
1687
1695
  Currently supports `network_stream_timeout_seconds` for S3, Tigris, and R2 storage.
1696
+ spec_version: The Icechunk spec version to use for the repo. Defaults to 2.
1688
1697
 
1689
1698
  Returns:
1690
1699
  icechunk.Repository: The created Icechunk repository.
@@ -1699,6 +1708,7 @@ class Client:
1699
1708
  config=config,
1700
1709
  authorize_virtual_chunk_access=authorize_virtual_chunk_access,
1701
1710
  storage_options=storage_options,
1711
+ spec_version=spec_version,
1702
1712
  )
1703
1713
 
1704
1714
  def import_repo(
@@ -19,28 +19,6 @@ from arraylake.types import (
19
19
  FORBIDDEN_VIRTUAL_CHUNK_PREFIXES = {"file://", "memory://", "http://"}
20
20
 
21
21
 
22
- def create_vcc_store_config_for_platform(platform: str) -> icechunk.AnyObjectStoreConfig:
23
- """Create an Icechunk ObjectStoreConfig for a virtual chunk container based on platform.
24
-
25
- This is used for runtime VCCs (e.g., filtered subscriptions) where we need to create
26
- a VCC dynamically based on the server-provided platform type.
27
-
28
- Args:
29
- platform: The storage platform (s3, s3-compatible, minio, gs, azure)
30
-
31
- Returns:
32
- An icechunk ObjectStoreConfig for the given platform
33
- """
34
- if platform in ("s3", "s3-compatible", "minio"):
35
- return icechunk.ObjectStoreConfig.S3(icechunk.S3Options())
36
- elif platform == "gs":
37
- return icechunk.ObjectStoreConfig.Gcs({})
38
- elif platform == "azure":
39
- return icechunk.ObjectStoreConfig.Azure({})
40
- else:
41
- raise ValueError(f"Unsupported platform for VCC store config: {platform}")
42
-
43
-
44
22
  def get_icechunk_container_credentials(
45
23
  bucket_platform: str,
46
24
  credentials: S3Credentials | GSCredentials | None,
@@ -289,11 +289,31 @@ class RepoSubscribeBody(RepoCreateBodyBase):
289
289
  None, description="Subscriber-defined filter applied on top of the provider's source filter. None means no additional filtering."
290
290
  )
291
291
  subscription_type: SubscriptionType = Field(default=SubscriptionType.DIRECT, description="The type/tier of subscription.")
292
+ prefix: BucketPrefix | None = Field(
293
+ None, description="Custom storage prefix for filtered/private subscriptions. If not provided, one is auto-generated."
294
+ )
295
+
296
+ @field_validator("prefix")
297
+ @classmethod
298
+ def validate_prefix_length(cls, v: str | None) -> str | None:
299
+ if v is not None and len(v) > MAX_BUCKET_PREFIX_LENGTH:
300
+ raise ValueError(f"Prefix must be at most {MAX_BUCKET_PREFIX_LENGTH} characters long. Got {len(v)}.")
301
+ return v
292
302
 
293
303
  @model_validator(mode="after")
294
304
  def validate_subscription_type_with_filter(self) -> RepoSubscribeBody:
295
305
  if self.target_filter is not None and self.subscription_type == SubscriptionType.DIRECT:
296
306
  raise ValueError("subscription_type cannot be 'direct' when target_filter is specified. Use 'filtered' or 'private' instead.")
307
+ # Filtered/private subscriptions require explicit target_filter with explicit allowlist
308
+ if self.subscription_type in (SubscriptionType.FILTERED, SubscriptionType.PRIVATE):
309
+ if self.target_filter is None:
310
+ raise ValueError(f"target_filter is required for {self.subscription_type.value} subscriptions.")
311
+ if self.target_filter.nodes is None:
312
+ raise ValueError(f"target_filter.nodes cannot be None for {self.subscription_type.value} subscriptions.")
313
+ if not self.target_filter.nodes.include_paths:
314
+ raise ValueError(f"target_filter must have explicit include_paths for {self.subscription_type.value} subscriptions.")
315
+ if self.target_filter.nodes.include_paths == ["/"]:
316
+ raise ValueError(f"target_filter cannot be the root group for {self.subscription_type.value} subscriptions.")
297
317
  return self
298
318
 
299
319
 
@@ -304,6 +324,16 @@ class RepoClaimBody(RepoCreateBodyBase):
304
324
  """
305
325
 
306
326
  claim_code: str = Field(..., description="The claim code to redeem")
327
+ prefix: BucketPrefix | None = Field(
328
+ None, description="Custom storage prefix for filtered/private subscriptions. If not provided, one is auto-generated."
329
+ )
330
+
331
+ @field_validator("prefix")
332
+ @classmethod
333
+ def validate_prefix_length(cls, v: str | None) -> str | None:
334
+ if v is not None and len(v) > MAX_BUCKET_PREFIX_LENGTH:
335
+ raise ValueError(f"Prefix must be at most {MAX_BUCKET_PREFIX_LENGTH} characters long. Got {len(v)}.")
336
+ return v
307
337
 
308
338
 
309
339
  class RepoModifyRequest(BaseModel):
@@ -471,6 +501,51 @@ class BucketModel(BaseModel):
471
501
  region = self.extra_config.get("region_name")
472
502
  return str(region) if region else None
473
503
 
504
+ def to_vcc_store_config(self, user_id: UUID | None = None) -> icechunk.AnyObjectStoreConfig:
505
+ """Create an Icechunk ObjectStoreConfig for use as a virtual chunk container.
506
+
507
+ This extracts the necessary configuration from the bucket's extra_config
508
+ to create a properly configured store config for accessing virtual chunks.
509
+
510
+ Args:
511
+ user_id: Optional user ID for logging purposes (used in GCS user_agent).
512
+
513
+ Returns:
514
+ An icechunk ObjectStoreConfig for the bucket's platform.
515
+ """
516
+ if self.platform in ("s3", "s3-compatible", "minio"):
517
+ endpoint_url = self.extra_config.get("endpoint_url")
518
+ if endpoint_url is not None:
519
+ endpoint_url = str(endpoint_url)
520
+
521
+ region = self.extra_config.get("region_name")
522
+ if region is not None:
523
+ region = str(region)
524
+
525
+ use_ssl = self.extra_config.get("use_ssl", True)
526
+ allow_http = not use_ssl
527
+
528
+ force_path_style_raw = self.extra_config.get("force_path_style", False if use_ssl else True)
529
+ force_path_style = bool(force_path_style_raw)
530
+
531
+ options = icechunk.S3Options(
532
+ region=region,
533
+ endpoint_url=endpoint_url,
534
+ allow_http=allow_http,
535
+ force_path_style=force_path_style,
536
+ )
537
+ return icechunk.ObjectStoreConfig.S3(options)
538
+ elif self.platform == "gs":
539
+ from arraylake import __version__ as arraylake_version
540
+
541
+ # Used for logging and debugging purposes in GCS access logs to help identify which client/version and optionally which user is accessing virtual chunks
542
+ config = {"user_agent": f"arraylake/{arraylake_version} (uuid={str(user_id)})" if user_id else f"arraylake/{arraylake_version}"}
543
+ return icechunk.ObjectStoreConfig.Gcs(config)
544
+ elif self.platform == "azure":
545
+ return icechunk.ObjectStoreConfig.Azure({})
546
+ else:
547
+ raise ValueError(f"Unsupported platform for VCC store config: {self.platform}")
548
+
474
549
 
475
550
  class Bucket(BucketModel):
476
551
  """A bucket configuration which actually exists in arraylake."""
@@ -21,21 +21,21 @@ readme = "README.md"
21
21
  authors = [
22
22
  { name = "Joseph Hamman", email = "joe@earthmover.io" },
23
23
  ]
24
- requires-python = ">=3.11, <3.14"
24
+ requires-python = ">=3.12, <3.15"
25
25
  dynamic = ["version"]
26
26
  license = "MIT"
27
27
  keywords = []
28
28
  classifiers = [
29
29
  "Programming Language :: Python",
30
- "Programming Language :: Python :: 3.11",
31
30
  "Programming Language :: Python :: 3.12",
32
31
  "Programming Language :: Python :: 3.13",
32
+ "Programming Language :: Python :: 3.14",
33
33
  ]
34
34
  dependencies = [
35
35
  "click >= 8.1.3, < 9.0",
36
36
  "donfig >= 0.7, < 1.0",
37
- "httpx >= 0.23, < 0.28",
38
- "icechunk >= 1.1.0, < 2.0.0",
37
+ "httpx >= 0.23, < 0.29",
38
+ "icechunk >= 2.0.1, < 3.0.0",
39
39
  "numpy >= 1.23, < 3.0",
40
40
  "packaging >= 23.0, < 26.0",
41
41
  "pydantic[email] >= 2.9, < 3.0",
@@ -55,8 +55,6 @@ xarray = [
55
55
  maximal = [
56
56
  "xarray[accel,etc,io,parallel] >= 2024.10.0",
57
57
  "cf_xarray >= 0.10.4",
58
- "numpy<2.3", # until dask[complete] updates their constraints
59
- "pyarrow<22", # restrict for better resolution. comes via dask[complete]
60
58
  ]
61
59
 
62
60
  [dependency-groups]
@@ -64,7 +62,7 @@ dev = [
64
62
  "cloudpickle~=3.0.0",
65
63
  "hatch>=1.14.2",
66
64
  "hypothesis>=6.88.1",
67
- "icechunk >= 1.1.9, < 2.0.0", # tests require higher min version than core package - see https://github.com/earth-mover/arraylake/pull/3404#discussion_r2490979373
65
+ "icechunk >= 2.0.1, < 3.0.0",
68
66
  "mypy~=1.11",
69
67
  "pytest-asyncio~=0.21.1",
70
68
  "pytest-click~=1.1.0",
@@ -77,7 +75,7 @@ dev = [
77
75
  "pytest-xdist~=3.5.0",
78
76
  "pytest~=7.3",
79
77
  "pyyaml~=6.0.1",
80
- "respx~=0.20.1",
78
+ "respx~=0.22.0",
81
79
  "types-cachetools~=5.3",
82
80
  "types-python-dateutil~=2.8.19",
83
81
  "types-pyyaml~=6.0.12",
@@ -16,6 +16,7 @@ from arraylake.api_utils import (
16
16
  AsyncRetryTransport,
17
17
  TokenAuth,
18
18
  UserAuth,
19
+ _emitted_deprecation_warnings,
19
20
  _get_proxy_from_env,
20
21
  calc_backoff,
21
22
  handle_response,
@@ -167,7 +168,7 @@ def test_token_auth_does_not_mutate_request(test_token, method, headers, json) -
167
168
  if k != "authorization": # keys are lower case
168
169
  assert header == request.headers.get(k)
169
170
  if json:
170
- assert json_lib.dumps(json).encode("utf-8") in request.content
171
+ assert json_lib.loads(request.content) == json
171
172
 
172
173
 
173
174
  @pytest.mark.asyncio
@@ -226,7 +227,7 @@ async def test_user_auth_does_not_mutate_request(test_token_file, method, header
226
227
  if k != "authorization": # keys are lower case
227
228
  assert header == request.headers.get(k)
228
229
  if json:
229
- assert json_lib.dumps(json).encode("utf-8") in request.content
230
+ assert json_lib.loads(request.content) == json
230
231
 
231
232
 
232
233
  @pytest.mark.asyncio
@@ -396,7 +397,7 @@ def test_handle_response_error(log_output: LogCapture) -> None:
396
397
  entries = log_output.entries[0]
397
398
  assert entries["log_level"] == "debug"
398
399
  assert entries["url"] == "https://earthmover.io/content/foo?a=b"
399
- assert entries["request_content"] == b'{"foo": "bar"}'
400
+ assert json_lib.loads(entries["request_content"]) == {"foo": "bar"}
400
401
  assert entries["request_headers"]["authorization"] == "[omitted]"
401
402
  assert entries["response_headers"]["content-type"] == "application/json"
402
403
  assert entries["response_headers"]["x-request-id"] == "7eea73c041524f268f804af813f7618f"
@@ -1011,6 +1012,32 @@ def test_arraylake_http_client_explicit_overrides_config() -> None:
1011
1012
  config.clear()
1012
1013
 
1013
1014
 
1015
+ def _ok_response(headers: dict[str, str] | None = None) -> httpx.Response:
1016
+ return httpx.Response(200, request=httpx.Request("GET", "https://example.com/api"), headers=headers or {})
1017
+
1018
+
1019
+ def test_deprecation_warnings() -> None:
1020
+ import warnings
1021
+
1022
+ _emitted_deprecation_warnings.clear()
1023
+
1024
+ with warnings.catch_warnings(record=True) as caught:
1025
+ warnings.simplefilter("always")
1026
+ # no header -> no warning
1027
+ handle_response(_ok_response())
1028
+ # header -> warning
1029
+ handle_response(_ok_response({"X-Deprecation-Warning": "A is deprecated"}))
1030
+ # same message again -> deduplicated
1031
+ handle_response(_ok_response({"X-Deprecation-Warning": "A is deprecated"}))
1032
+ # different message -> new warning
1033
+ handle_response(_ok_response({"X-Deprecation-Warning": "B is deprecated"}))
1034
+
1035
+ deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
1036
+ assert len(deprecation_warnings) == 2
1037
+ assert "A is deprecated" in str(deprecation_warnings[0].message)
1038
+ assert "B is deprecated" in str(deprecation_warnings[1].message)
1039
+
1040
+
1014
1041
  @pytest.mark.asyncio
1015
1042
  async def test_config_proxy_end_to_end_integration(respx_mock, monkeypatch) -> None:
1016
1043
  """Test end-to-end proxy configuration via arraylake config"""
@@ -201,6 +201,47 @@ async def test_create_repo_in_anon_bucket(isolated_org, minio_anon_bucket, token
201
201
  await aclient.create_repo(repo_name)
202
202
 
203
203
 
204
+ @pytest.mark.asyncio
205
+ async def test_create_repo_default_creates_ic2_repo(isolated_org, default_bucket, token):
206
+ """create_repo with default spec_version creates IC2 repos (spec_version=2)."""
207
+ async with isolated_org(default_bucket()) as (org_name, buckets):
208
+ client = AsyncClient(token=token)
209
+ repo = await client.create_repo(
210
+ f"{org_name}/test-default-repo",
211
+ bucket_config_nickname=buckets[0].nickname,
212
+ )
213
+
214
+ assert repo.spec_version == 2, f"Default create_repo should create IC2 repo (spec_version=2), got {repo.spec_version}"
215
+
216
+
217
+ @pytest.mark.asyncio
218
+ async def test_create_repo_spec_version_1_creates_ic1_repo(isolated_org, default_bucket, token):
219
+ """create_repo with spec_version=1 creates IC1 repos."""
220
+ async with isolated_org(default_bucket()) as (org_name, buckets):
221
+ client = AsyncClient(token=token)
222
+ repo = await client.create_repo(
223
+ f"{org_name}/test-ic1-explicit",
224
+ bucket_config_nickname=buckets[0].nickname,
225
+ spec_version=1,
226
+ )
227
+
228
+ assert repo.spec_version == 1, f"spec_version=1 should create IC1 repo, got {repo.spec_version}"
229
+
230
+
231
+ @pytest.mark.asyncio
232
+ async def test_create_repo_spec_version_2_creates_ic2_repo(isolated_org, default_bucket, token):
233
+ """create_repo with spec_version=2 creates IC2 repos."""
234
+ async with isolated_org(default_bucket()) as (org_name, buckets):
235
+ client = AsyncClient(token=token)
236
+ repo = await client.create_repo(
237
+ f"{org_name}/test-ic2-explicit",
238
+ bucket_config_nickname=buckets[0].nickname,
239
+ spec_version=2,
240
+ )
241
+
242
+ assert repo.spec_version == 2, f"spec_version=2 should create IC2 repo, got {repo.spec_version}"
243
+
244
+
204
245
  @pytest.mark.asyncio
205
246
  async def test_modify_repo(isolated_org, default_bucket, token) -> None:
206
247
  async with isolated_org(default_bucket()) as (org_name, buckets):
@@ -5,8 +5,13 @@ from uuid import uuid4
5
5
 
6
6
  import icechunk
7
7
  import pytest
8
+ from packaging.version import Version
8
9
 
9
10
  from arraylake import AsyncClient, Client
11
+
12
+ # Check if icechunk 2 is available
13
+ ICECHUNK_VERSION = Version(icechunk.__version__)
14
+ IS_IC2 = ICECHUNK_VERSION.major >= 2
10
15
  from arraylake.api_utils import ArraylakeHttpClient
11
16
  from arraylake.exceptions import ArraylakeClientError
12
17
  from arraylake.types import (
@@ -919,6 +924,7 @@ def assert_subscription_vcap_resolution(client: Client, sub_repo_name: str, vcc_
919
924
  client.get_repo(sub_repo_name)
920
925
 
921
926
 
927
+ @pytest.mark.skipif(not IS_IC2, reason="Paid marketplace listings require icechunk 2.x")
922
928
  async def test_filtered_subscription_implementation(two_isolated_orgs, default_bucket, token):
923
929
  """
924
930
  Filtered subscription: subscriber has its own bucket, VCC points to provider's chunks.
@@ -958,6 +964,7 @@ async def test_filtered_subscription_implementation(two_isolated_orgs, default_b
958
964
  repo_prefix = parent_repo_obj.prefix.strip("/")
959
965
 
960
966
  # Create the icechunk repo manually (anonymous bucket config, but MinIO allows writes)
967
+ # Use spec_version=2 since paid marketplace listings require IC2 repos
961
968
  ic_storage = icechunk.s3_storage(
962
969
  bucket=provider_repo_bucket.name,
963
970
  prefix=parent_repo_obj.prefix,
@@ -968,7 +975,7 @@ async def test_filtered_subscription_implementation(two_isolated_orgs, default_b
968
975
  secret_access_key="minio123",
969
976
  force_path_style=True,
970
977
  )
971
- icechunk.Repository.create(storage=ic_storage)
978
+ icechunk.Repository.create(storage=ic_storage, spec_version=2)
972
979
 
973
980
  # Construct VCC URL pointing to the provider repo's chunks directory
974
981
  vcc_url_prefix = f"s3://{provider_repo_bucket.name}/{repo_prefix}/chunks/"
@@ -1004,7 +1011,7 @@ async def test_filtered_subscription_implementation(two_isolated_orgs, default_b
1004
1011
 
1005
1012
 
1006
1013
  @asynccontextmanager
1007
- async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public):
1014
+ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public, pricing_model="paid"):
1008
1015
  """Set up a provider org with a repo containing virtual chunks, and a subscriber org.
1009
1016
 
1010
1017
  Creates:
@@ -1012,6 +1019,9 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
1012
1019
  VCAP covering the virtual chunk bucket (public or private based on `public` param)
1013
1020
  - Org B (subscriber): writable bucket on "testbucket"
1014
1021
 
1022
+ Args:
1023
+ pricing_model: "free" or "paid". Use "free" for direct subscriptions, "paid" for filtered.
1024
+
1015
1025
  Yields (client, async_client, org_a, org_b, parent_repo_obj,
1016
1026
  provider_repo_bucket, subscriber_bucket, vcc_url_prefix, listing).
1017
1027
  """
@@ -1052,6 +1062,7 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
1052
1062
  parent_repo_obj = await async_client.get_repo_object(parent_repo_name)
1053
1063
 
1054
1064
  # Create the icechunk repo manually (anonymous bucket config, but MinIO allows writes)
1065
+ # Use spec_version=2 for paid listings (which require IC2 repos)
1055
1066
  ic_storage = icechunk.s3_storage(
1056
1067
  bucket=provider_repo_bucket.name,
1057
1068
  prefix=parent_repo_obj.prefix,
@@ -1062,7 +1073,10 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
1062
1073
  secret_access_key="minio123",
1063
1074
  force_path_style=True,
1064
1075
  )
1065
- provider_ic_repo = icechunk.Repository.create(storage=ic_storage)
1076
+ create_kwargs = {"storage": ic_storage}
1077
+ if pricing_model == "paid":
1078
+ create_kwargs["spec_version"] = 2
1079
+ provider_ic_repo = icechunk.Repository.create(**create_kwargs)
1066
1080
 
1067
1081
  # Set VCC for virtual chunks on the provider repo
1068
1082
  config = provider_ic_repo.config
@@ -1076,9 +1090,8 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
1076
1090
  provider_ic_repo.save_config()
1077
1091
 
1078
1092
  # The subscription VCAP is auto-created when the marketplace listing is created
1079
- # Use paid pricing model to support filtered subscriptions
1080
1093
  async with vcap_for_bucket(async_client, org_a, virtual_chunk_bucket, public=public):
1081
- listing = await create_marketplace_listing(token, org_a, parent_repo_obj.id, pricing_model="paid")
1094
+ listing = await create_marketplace_listing(token, org_a, parent_repo_obj.id, pricing_model=pricing_model)
1082
1095
 
1083
1096
  yield (
1084
1097
  client,
@@ -1109,7 +1122,8 @@ class TestSubscriptionToRepoContainingVirtualChunks:
1109
1122
 
1110
1123
  async def test_direct_subscription(self, two_isolated_orgs, default_bucket, token, public):
1111
1124
  """Direct subscription: subscriber shares the provider's storage and VCCs."""
1112
- async with provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public) as (
1125
+ # Direct subscriptions are only allowed on free listings
1126
+ async with provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public, pricing_model="free") as (
1113
1127
  client,
1114
1128
  _,
1115
1129
  _,
@@ -1123,6 +1137,7 @@ class TestSubscriptionToRepoContainingVirtualChunks:
1123
1137
  await create_subscription_repo(token, org_b, "sub-repo", listing["id"])
1124
1138
  assert_subscription_vcap_resolution(client, f"{org_b}/sub-repo", {vcc_url_prefix}, public)
1125
1139
 
1140
+ @pytest.mark.skipif(not IS_IC2, reason="Paid marketplace listings require icechunk 2.x")
1126
1141
  async def test_filtered_subscription_provider_repo_contains_virtual_chunks(self, two_isolated_orgs, default_bucket, token, public):
1127
1142
  """Filtered subscription where the provider repo also contains virtual chunks.
1128
1143