arraylake 0.33.0__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-0.33.0 → arraylake-1.0.0}/PKG-INFO +5 -7
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/_version.py +2 -2
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/api_utils.py +8 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/client.py +14 -4
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/virtual.py +0 -22
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/types.py +75 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/pyproject.toml +6 -8
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_api_utils.py +30 -3
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_client.py +41 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_virtual_chunks.py +21 -6
- {arraylake-0.33.0 → arraylake-1.0.0}/uv.lock +437 -431
- {arraylake-0.33.0 → arraylake-1.0.0}/.gitignore +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/README.md +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/__main__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/asyn.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/auth.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/compute.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/config.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/main.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/repo.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/cli/utils.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/doctor.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/http_client.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/services.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/compute/types.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/config.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/config.yaml +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/credentials.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/diagnostics.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/display/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/display/repo.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/display/repolist.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/exceptions.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/log_util.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/metastore/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/metastore/abc.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/metastore/http_metastore.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/py.typed +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/storage.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/repos/icechunk/types.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/token.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/arraylake/tuning.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/compute/test_client.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/compute/test_doctor.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/config.yaml +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/conftest.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/__init__.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/conftest.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/test_storage.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/icechunk/test_virtual.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_asyn.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_cli.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_config.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_credentials.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_diagnostics.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_performance.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_subscriptions.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_token_handler.py +0 -0
- {arraylake-0.33.0 → arraylake-1.0.0}/tests/test_types.py +0 -0
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arraylake
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Summary: Python client for ArrayLake
|
|
5
5
|
Project-URL: Documentation, https://docs.earthmover.io/
|
|
6
6
|
Project-URL: Changelog, https://docs.earthmover.io/changelog
|
|
7
7
|
Author-email: Joseph Hamman <joe@earthmover.io>
|
|
8
8
|
License-Expression: MIT
|
|
9
9
|
Classifier: Programming Language :: Python
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
-
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Requires-Python: <3.15,>=3.12
|
|
14
14
|
Requires-Dist: click<9.0,>=8.1.3
|
|
15
15
|
Requires-Dist: donfig<1.0,>=0.7
|
|
16
|
-
Requires-Dist: httpx<0.
|
|
17
|
-
Requires-Dist: icechunk<
|
|
16
|
+
Requires-Dist: httpx<0.29,>=0.23
|
|
17
|
+
Requires-Dist: icechunk<3.0.0,>=2.0.1
|
|
18
18
|
Requires-Dist: numpy<3.0,>=1.23
|
|
19
19
|
Requires-Dist: packaging<26.0,>=23.0
|
|
20
20
|
Requires-Dist: pydantic[email]<3.0,>=2.9
|
|
@@ -26,8 +26,6 @@ Requires-Dist: typer<1.0,>=0.12.0
|
|
|
26
26
|
Requires-Dist: zarr<4.0,>=3.1.0
|
|
27
27
|
Provides-Extra: maximal
|
|
28
28
|
Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
|
|
29
|
-
Requires-Dist: numpy<2.3; extra == 'maximal'
|
|
30
|
-
Requires-Dist: pyarrow<22; extra == 'maximal'
|
|
31
29
|
Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
|
|
32
30
|
Provides-Extra: xarray
|
|
33
31
|
Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (
|
|
21
|
+
__version__ = version = '1.0.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -4,6 +4,7 @@ import asyncio
|
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
6
|
import random
|
|
7
|
+
import warnings
|
|
7
8
|
import weakref
|
|
8
9
|
from collections.abc import AsyncGenerator, Callable, Generator, Mapping
|
|
9
10
|
from contextlib import _AsyncGeneratorContextManager
|
|
@@ -430,8 +431,15 @@ def _exception_log_debug(request: httpx.Request, response: httpx.Response):
|
|
|
430
431
|
)
|
|
431
432
|
|
|
432
433
|
|
|
434
|
+
_emitted_deprecation_warnings: set[str] = set()
|
|
435
|
+
|
|
436
|
+
|
|
433
437
|
def handle_response(response: httpx.Response):
|
|
434
438
|
"""Convenience function to handle response status codes"""
|
|
439
|
+
deprecation_msg = response.headers.get("X-Deprecation-Warning")
|
|
440
|
+
if deprecation_msg and deprecation_msg not in _emitted_deprecation_warnings:
|
|
441
|
+
_emitted_deprecation_warnings.add(deprecation_msg)
|
|
442
|
+
warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2)
|
|
435
443
|
try:
|
|
436
444
|
response.raise_for_status()
|
|
437
445
|
except httpx.RequestError as exc:
|
|
@@ -46,7 +46,6 @@ from arraylake.repos.icechunk.storage import (
|
|
|
46
46
|
_get_icechunk_storage_obj,
|
|
47
47
|
)
|
|
48
48
|
from arraylake.repos.icechunk.virtual import (
|
|
49
|
-
create_vcc_store_config_for_platform,
|
|
50
49
|
create_virtual_chunk_container,
|
|
51
50
|
forbid_unsafe_virtual_bucket_configs,
|
|
52
51
|
forbid_unsafe_virtual_chunk_containers,
|
|
@@ -561,7 +560,8 @@ class AsyncClient:
|
|
|
561
560
|
for vcc_prefix, vcc_creds in response.virtual_chunk_credentials.items():
|
|
562
561
|
# Handle runtime VCCs (e.g., filtered subscriptions) - create VCC on the fly
|
|
563
562
|
if vcc_creds.runtime_vcc_name is not None:
|
|
564
|
-
|
|
563
|
+
bucket_config = await self.get_bucket_config(org=vcc_creds.org, nickname=vcc_creds.bucket_nickname)
|
|
564
|
+
store_config = bucket_config.to_vcc_store_config(user_id=UUID(response.principal_id))
|
|
565
565
|
container = icechunk.VirtualChunkContainer(
|
|
566
566
|
url_prefix=vcc_prefix,
|
|
567
567
|
store=store_config,
|
|
@@ -734,6 +734,7 @@ class AsyncClient:
|
|
|
734
734
|
config: icechunk.RepositoryConfig | None = None,
|
|
735
735
|
authorize_virtual_chunk_access: Mapping[BucketPrefix, BucketNickname] | None = None,
|
|
736
736
|
storage_options: StorageOptions | None = None,
|
|
737
|
+
spec_version: int = 2,
|
|
737
738
|
) -> IcechunkRepository:
|
|
738
739
|
"""Create a new repo.
|
|
739
740
|
|
|
@@ -751,6 +752,7 @@ class AsyncClient:
|
|
|
751
752
|
and used for the containers. Must be supplied as complete urls.
|
|
752
753
|
storage_options: Optional storage options for the underlying Icechunk storage.
|
|
753
754
|
Currently supports `network_stream_timeout_seconds` for S3, Tigris, and R2 storage.
|
|
755
|
+
spec_version: The Icechunk spec version to use for the repo. Defaults to 2.
|
|
754
756
|
|
|
755
757
|
Returns:
|
|
756
758
|
icechunk.Repository object for the repo.
|
|
@@ -808,10 +810,15 @@ class AsyncClient:
|
|
|
808
810
|
# TODO: make icechunk accept an empty dict instead of None
|
|
809
811
|
prefixes_credential_mapping = None
|
|
810
812
|
|
|
813
|
+
create_kwargs: dict[str, object] = {
|
|
814
|
+
"config": config,
|
|
815
|
+
"authorize_virtual_chunk_access": prefixes_credential_mapping,
|
|
816
|
+
"spec_version": spec_version,
|
|
817
|
+
}
|
|
818
|
+
|
|
811
819
|
ic_repo = IcechunkRepository.create(
|
|
812
820
|
icechunk_storage,
|
|
813
|
-
|
|
814
|
-
authorize_virtual_chunk_access=prefixes_credential_mapping,
|
|
821
|
+
**create_kwargs, # type: ignore[arg-type]
|
|
815
822
|
)
|
|
816
823
|
|
|
817
824
|
ic_repo.set_default_commit_metadata({"author_name": author.name, "author_email": author.email})
|
|
@@ -1667,6 +1674,7 @@ class Client:
|
|
|
1667
1674
|
config: icechunk.RepositoryConfig | None = None,
|
|
1668
1675
|
authorize_virtual_chunk_access: Mapping[BucketPrefix, BucketNickname] | None = None,
|
|
1669
1676
|
storage_options: StorageOptions | None = None,
|
|
1677
|
+
spec_version: int = 2,
|
|
1670
1678
|
) -> IcechunkRepository:
|
|
1671
1679
|
"""Create a new repo
|
|
1672
1680
|
|
|
@@ -1685,6 +1693,7 @@ class Client:
|
|
|
1685
1693
|
and used for the containers. Used for Icechunk repos only.
|
|
1686
1694
|
storage_options: Optional storage options for the underlying Icechunk storage.
|
|
1687
1695
|
Currently supports `network_stream_timeout_seconds` for S3, Tigris, and R2 storage.
|
|
1696
|
+
spec_version: The Icechunk spec version to use for the repo. Defaults to 2.
|
|
1688
1697
|
|
|
1689
1698
|
Returns:
|
|
1690
1699
|
icechunk.Repository: The created Icechunk repository.
|
|
@@ -1699,6 +1708,7 @@ class Client:
|
|
|
1699
1708
|
config=config,
|
|
1700
1709
|
authorize_virtual_chunk_access=authorize_virtual_chunk_access,
|
|
1701
1710
|
storage_options=storage_options,
|
|
1711
|
+
spec_version=spec_version,
|
|
1702
1712
|
)
|
|
1703
1713
|
|
|
1704
1714
|
def import_repo(
|
|
@@ -19,28 +19,6 @@ from arraylake.types import (
|
|
|
19
19
|
FORBIDDEN_VIRTUAL_CHUNK_PREFIXES = {"file://", "memory://", "http://"}
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
def create_vcc_store_config_for_platform(platform: str) -> icechunk.AnyObjectStoreConfig:
|
|
23
|
-
"""Create an Icechunk ObjectStoreConfig for a virtual chunk container based on platform.
|
|
24
|
-
|
|
25
|
-
This is used for runtime VCCs (e.g., filtered subscriptions) where we need to create
|
|
26
|
-
a VCC dynamically based on the server-provided platform type.
|
|
27
|
-
|
|
28
|
-
Args:
|
|
29
|
-
platform: The storage platform (s3, s3-compatible, minio, gs, azure)
|
|
30
|
-
|
|
31
|
-
Returns:
|
|
32
|
-
An icechunk ObjectStoreConfig for the given platform
|
|
33
|
-
"""
|
|
34
|
-
if platform in ("s3", "s3-compatible", "minio"):
|
|
35
|
-
return icechunk.ObjectStoreConfig.S3(icechunk.S3Options())
|
|
36
|
-
elif platform == "gs":
|
|
37
|
-
return icechunk.ObjectStoreConfig.Gcs({})
|
|
38
|
-
elif platform == "azure":
|
|
39
|
-
return icechunk.ObjectStoreConfig.Azure({})
|
|
40
|
-
else:
|
|
41
|
-
raise ValueError(f"Unsupported platform for VCC store config: {platform}")
|
|
42
|
-
|
|
43
|
-
|
|
44
22
|
def get_icechunk_container_credentials(
|
|
45
23
|
bucket_platform: str,
|
|
46
24
|
credentials: S3Credentials | GSCredentials | None,
|
|
@@ -289,11 +289,31 @@ class RepoSubscribeBody(RepoCreateBodyBase):
|
|
|
289
289
|
None, description="Subscriber-defined filter applied on top of the provider's source filter. None means no additional filtering."
|
|
290
290
|
)
|
|
291
291
|
subscription_type: SubscriptionType = Field(default=SubscriptionType.DIRECT, description="The type/tier of subscription.")
|
|
292
|
+
prefix: BucketPrefix | None = Field(
|
|
293
|
+
None, description="Custom storage prefix for filtered/private subscriptions. If not provided, one is auto-generated."
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
@field_validator("prefix")
|
|
297
|
+
@classmethod
|
|
298
|
+
def validate_prefix_length(cls, v: str | None) -> str | None:
|
|
299
|
+
if v is not None and len(v) > MAX_BUCKET_PREFIX_LENGTH:
|
|
300
|
+
raise ValueError(f"Prefix must be at most {MAX_BUCKET_PREFIX_LENGTH} characters long. Got {len(v)}.")
|
|
301
|
+
return v
|
|
292
302
|
|
|
293
303
|
@model_validator(mode="after")
|
|
294
304
|
def validate_subscription_type_with_filter(self) -> RepoSubscribeBody:
|
|
295
305
|
if self.target_filter is not None and self.subscription_type == SubscriptionType.DIRECT:
|
|
296
306
|
raise ValueError("subscription_type cannot be 'direct' when target_filter is specified. Use 'filtered' or 'private' instead.")
|
|
307
|
+
# Filtered/private subscriptions require explicit target_filter with explicit allowlist
|
|
308
|
+
if self.subscription_type in (SubscriptionType.FILTERED, SubscriptionType.PRIVATE):
|
|
309
|
+
if self.target_filter is None:
|
|
310
|
+
raise ValueError(f"target_filter is required for {self.subscription_type.value} subscriptions.")
|
|
311
|
+
if self.target_filter.nodes is None:
|
|
312
|
+
raise ValueError(f"target_filter.nodes cannot be None for {self.subscription_type.value} subscriptions.")
|
|
313
|
+
if not self.target_filter.nodes.include_paths:
|
|
314
|
+
raise ValueError(f"target_filter must have explicit include_paths for {self.subscription_type.value} subscriptions.")
|
|
315
|
+
if self.target_filter.nodes.include_paths == ["/"]:
|
|
316
|
+
raise ValueError(f"target_filter cannot be the root group for {self.subscription_type.value} subscriptions.")
|
|
297
317
|
return self
|
|
298
318
|
|
|
299
319
|
|
|
@@ -304,6 +324,16 @@ class RepoClaimBody(RepoCreateBodyBase):
|
|
|
304
324
|
"""
|
|
305
325
|
|
|
306
326
|
claim_code: str = Field(..., description="The claim code to redeem")
|
|
327
|
+
prefix: BucketPrefix | None = Field(
|
|
328
|
+
None, description="Custom storage prefix for filtered/private subscriptions. If not provided, one is auto-generated."
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
@field_validator("prefix")
|
|
332
|
+
@classmethod
|
|
333
|
+
def validate_prefix_length(cls, v: str | None) -> str | None:
|
|
334
|
+
if v is not None and len(v) > MAX_BUCKET_PREFIX_LENGTH:
|
|
335
|
+
raise ValueError(f"Prefix must be at most {MAX_BUCKET_PREFIX_LENGTH} characters long. Got {len(v)}.")
|
|
336
|
+
return v
|
|
307
337
|
|
|
308
338
|
|
|
309
339
|
class RepoModifyRequest(BaseModel):
|
|
@@ -471,6 +501,51 @@ class BucketModel(BaseModel):
|
|
|
471
501
|
region = self.extra_config.get("region_name")
|
|
472
502
|
return str(region) if region else None
|
|
473
503
|
|
|
504
|
+
def to_vcc_store_config(self, user_id: UUID | None = None) -> icechunk.AnyObjectStoreConfig:
|
|
505
|
+
"""Create an Icechunk ObjectStoreConfig for use as a virtual chunk container.
|
|
506
|
+
|
|
507
|
+
This extracts the necessary configuration from the bucket's extra_config
|
|
508
|
+
to create a properly configured store config for accessing virtual chunks.
|
|
509
|
+
|
|
510
|
+
Args:
|
|
511
|
+
user_id: Optional user ID for logging purposes (used in GCS user_agent).
|
|
512
|
+
|
|
513
|
+
Returns:
|
|
514
|
+
An icechunk ObjectStoreConfig for the bucket's platform.
|
|
515
|
+
"""
|
|
516
|
+
if self.platform in ("s3", "s3-compatible", "minio"):
|
|
517
|
+
endpoint_url = self.extra_config.get("endpoint_url")
|
|
518
|
+
if endpoint_url is not None:
|
|
519
|
+
endpoint_url = str(endpoint_url)
|
|
520
|
+
|
|
521
|
+
region = self.extra_config.get("region_name")
|
|
522
|
+
if region is not None:
|
|
523
|
+
region = str(region)
|
|
524
|
+
|
|
525
|
+
use_ssl = self.extra_config.get("use_ssl", True)
|
|
526
|
+
allow_http = not use_ssl
|
|
527
|
+
|
|
528
|
+
force_path_style_raw = self.extra_config.get("force_path_style", False if use_ssl else True)
|
|
529
|
+
force_path_style = bool(force_path_style_raw)
|
|
530
|
+
|
|
531
|
+
options = icechunk.S3Options(
|
|
532
|
+
region=region,
|
|
533
|
+
endpoint_url=endpoint_url,
|
|
534
|
+
allow_http=allow_http,
|
|
535
|
+
force_path_style=force_path_style,
|
|
536
|
+
)
|
|
537
|
+
return icechunk.ObjectStoreConfig.S3(options)
|
|
538
|
+
elif self.platform == "gs":
|
|
539
|
+
from arraylake import __version__ as arraylake_version
|
|
540
|
+
|
|
541
|
+
# Used for logging and debugging purposes in GCS access logs to help identify which client/version and optionally which user is accessing virtual chunks
|
|
542
|
+
config = {"user_agent": f"arraylake/{arraylake_version} (uuid={str(user_id)})" if user_id else f"arraylake/{arraylake_version}"}
|
|
543
|
+
return icechunk.ObjectStoreConfig.Gcs(config)
|
|
544
|
+
elif self.platform == "azure":
|
|
545
|
+
return icechunk.ObjectStoreConfig.Azure({})
|
|
546
|
+
else:
|
|
547
|
+
raise ValueError(f"Unsupported platform for VCC store config: {self.platform}")
|
|
548
|
+
|
|
474
549
|
|
|
475
550
|
class Bucket(BucketModel):
|
|
476
551
|
"""A bucket configuration which actually exists in arraylake."""
|
|
@@ -21,21 +21,21 @@ readme = "README.md"
|
|
|
21
21
|
authors = [
|
|
22
22
|
{ name = "Joseph Hamman", email = "joe@earthmover.io" },
|
|
23
23
|
]
|
|
24
|
-
requires-python = ">=3.
|
|
24
|
+
requires-python = ">=3.12, <3.15"
|
|
25
25
|
dynamic = ["version"]
|
|
26
26
|
license = "MIT"
|
|
27
27
|
keywords = []
|
|
28
28
|
classifiers = [
|
|
29
29
|
"Programming Language :: Python",
|
|
30
|
-
"Programming Language :: Python :: 3.11",
|
|
31
30
|
"Programming Language :: Python :: 3.12",
|
|
32
31
|
"Programming Language :: Python :: 3.13",
|
|
32
|
+
"Programming Language :: Python :: 3.14",
|
|
33
33
|
]
|
|
34
34
|
dependencies = [
|
|
35
35
|
"click >= 8.1.3, < 9.0",
|
|
36
36
|
"donfig >= 0.7, < 1.0",
|
|
37
|
-
"httpx >= 0.23, < 0.
|
|
38
|
-
"icechunk >=
|
|
37
|
+
"httpx >= 0.23, < 0.29",
|
|
38
|
+
"icechunk >= 2.0.1, < 3.0.0",
|
|
39
39
|
"numpy >= 1.23, < 3.0",
|
|
40
40
|
"packaging >= 23.0, < 26.0",
|
|
41
41
|
"pydantic[email] >= 2.9, < 3.0",
|
|
@@ -55,8 +55,6 @@ xarray = [
|
|
|
55
55
|
maximal = [
|
|
56
56
|
"xarray[accel,etc,io,parallel] >= 2024.10.0",
|
|
57
57
|
"cf_xarray >= 0.10.4",
|
|
58
|
-
"numpy<2.3", # until dask[complete] updates their constraints
|
|
59
|
-
"pyarrow<22", # restrict for better resolution. comes via dask[complete]
|
|
60
58
|
]
|
|
61
59
|
|
|
62
60
|
[dependency-groups]
|
|
@@ -64,7 +62,7 @@ dev = [
|
|
|
64
62
|
"cloudpickle~=3.0.0",
|
|
65
63
|
"hatch>=1.14.2",
|
|
66
64
|
"hypothesis>=6.88.1",
|
|
67
|
-
"icechunk >=
|
|
65
|
+
"icechunk >= 2.0.1, < 3.0.0",
|
|
68
66
|
"mypy~=1.11",
|
|
69
67
|
"pytest-asyncio~=0.21.1",
|
|
70
68
|
"pytest-click~=1.1.0",
|
|
@@ -77,7 +75,7 @@ dev = [
|
|
|
77
75
|
"pytest-xdist~=3.5.0",
|
|
78
76
|
"pytest~=7.3",
|
|
79
77
|
"pyyaml~=6.0.1",
|
|
80
|
-
"respx~=0.
|
|
78
|
+
"respx~=0.22.0",
|
|
81
79
|
"types-cachetools~=5.3",
|
|
82
80
|
"types-python-dateutil~=2.8.19",
|
|
83
81
|
"types-pyyaml~=6.0.12",
|
|
@@ -16,6 +16,7 @@ from arraylake.api_utils import (
|
|
|
16
16
|
AsyncRetryTransport,
|
|
17
17
|
TokenAuth,
|
|
18
18
|
UserAuth,
|
|
19
|
+
_emitted_deprecation_warnings,
|
|
19
20
|
_get_proxy_from_env,
|
|
20
21
|
calc_backoff,
|
|
21
22
|
handle_response,
|
|
@@ -167,7 +168,7 @@ def test_token_auth_does_not_mutate_request(test_token, method, headers, json) -
|
|
|
167
168
|
if k != "authorization": # keys are lower case
|
|
168
169
|
assert header == request.headers.get(k)
|
|
169
170
|
if json:
|
|
170
|
-
assert json_lib.
|
|
171
|
+
assert json_lib.loads(request.content) == json
|
|
171
172
|
|
|
172
173
|
|
|
173
174
|
@pytest.mark.asyncio
|
|
@@ -226,7 +227,7 @@ async def test_user_auth_does_not_mutate_request(test_token_file, method, header
|
|
|
226
227
|
if k != "authorization": # keys are lower case
|
|
227
228
|
assert header == request.headers.get(k)
|
|
228
229
|
if json:
|
|
229
|
-
assert json_lib.
|
|
230
|
+
assert json_lib.loads(request.content) == json
|
|
230
231
|
|
|
231
232
|
|
|
232
233
|
@pytest.mark.asyncio
|
|
@@ -396,7 +397,7 @@ def test_handle_response_error(log_output: LogCapture) -> None:
|
|
|
396
397
|
entries = log_output.entries[0]
|
|
397
398
|
assert entries["log_level"] == "debug"
|
|
398
399
|
assert entries["url"] == "https://earthmover.io/content/foo?a=b"
|
|
399
|
-
assert entries["request_content"] ==
|
|
400
|
+
assert json_lib.loads(entries["request_content"]) == {"foo": "bar"}
|
|
400
401
|
assert entries["request_headers"]["authorization"] == "[omitted]"
|
|
401
402
|
assert entries["response_headers"]["content-type"] == "application/json"
|
|
402
403
|
assert entries["response_headers"]["x-request-id"] == "7eea73c041524f268f804af813f7618f"
|
|
@@ -1011,6 +1012,32 @@ def test_arraylake_http_client_explicit_overrides_config() -> None:
|
|
|
1011
1012
|
config.clear()
|
|
1012
1013
|
|
|
1013
1014
|
|
|
1015
|
+
def _ok_response(headers: dict[str, str] | None = None) -> httpx.Response:
|
|
1016
|
+
return httpx.Response(200, request=httpx.Request("GET", "https://example.com/api"), headers=headers or {})
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def test_deprecation_warnings() -> None:
|
|
1020
|
+
import warnings
|
|
1021
|
+
|
|
1022
|
+
_emitted_deprecation_warnings.clear()
|
|
1023
|
+
|
|
1024
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
1025
|
+
warnings.simplefilter("always")
|
|
1026
|
+
# no header -> no warning
|
|
1027
|
+
handle_response(_ok_response())
|
|
1028
|
+
# header -> warning
|
|
1029
|
+
handle_response(_ok_response({"X-Deprecation-Warning": "A is deprecated"}))
|
|
1030
|
+
# same message again -> deduplicated
|
|
1031
|
+
handle_response(_ok_response({"X-Deprecation-Warning": "A is deprecated"}))
|
|
1032
|
+
# different message -> new warning
|
|
1033
|
+
handle_response(_ok_response({"X-Deprecation-Warning": "B is deprecated"}))
|
|
1034
|
+
|
|
1035
|
+
deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
|
|
1036
|
+
assert len(deprecation_warnings) == 2
|
|
1037
|
+
assert "A is deprecated" in str(deprecation_warnings[0].message)
|
|
1038
|
+
assert "B is deprecated" in str(deprecation_warnings[1].message)
|
|
1039
|
+
|
|
1040
|
+
|
|
1014
1041
|
@pytest.mark.asyncio
|
|
1015
1042
|
async def test_config_proxy_end_to_end_integration(respx_mock, monkeypatch) -> None:
|
|
1016
1043
|
"""Test end-to-end proxy configuration via arraylake config"""
|
|
@@ -201,6 +201,47 @@ async def test_create_repo_in_anon_bucket(isolated_org, minio_anon_bucket, token
|
|
|
201
201
|
await aclient.create_repo(repo_name)
|
|
202
202
|
|
|
203
203
|
|
|
204
|
+
@pytest.mark.asyncio
|
|
205
|
+
async def test_create_repo_default_creates_ic2_repo(isolated_org, default_bucket, token):
|
|
206
|
+
"""create_repo with default spec_version creates IC2 repos (spec_version=2)."""
|
|
207
|
+
async with isolated_org(default_bucket()) as (org_name, buckets):
|
|
208
|
+
client = AsyncClient(token=token)
|
|
209
|
+
repo = await client.create_repo(
|
|
210
|
+
f"{org_name}/test-default-repo",
|
|
211
|
+
bucket_config_nickname=buckets[0].nickname,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
assert repo.spec_version == 2, f"Default create_repo should create IC2 repo (spec_version=2), got {repo.spec_version}"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@pytest.mark.asyncio
|
|
218
|
+
async def test_create_repo_spec_version_1_creates_ic1_repo(isolated_org, default_bucket, token):
|
|
219
|
+
"""create_repo with spec_version=1 creates IC1 repos."""
|
|
220
|
+
async with isolated_org(default_bucket()) as (org_name, buckets):
|
|
221
|
+
client = AsyncClient(token=token)
|
|
222
|
+
repo = await client.create_repo(
|
|
223
|
+
f"{org_name}/test-ic1-explicit",
|
|
224
|
+
bucket_config_nickname=buckets[0].nickname,
|
|
225
|
+
spec_version=1,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
assert repo.spec_version == 1, f"spec_version=1 should create IC1 repo, got {repo.spec_version}"
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@pytest.mark.asyncio
|
|
232
|
+
async def test_create_repo_spec_version_2_creates_ic2_repo(isolated_org, default_bucket, token):
|
|
233
|
+
"""create_repo with spec_version=2 creates IC2 repos."""
|
|
234
|
+
async with isolated_org(default_bucket()) as (org_name, buckets):
|
|
235
|
+
client = AsyncClient(token=token)
|
|
236
|
+
repo = await client.create_repo(
|
|
237
|
+
f"{org_name}/test-ic2-explicit",
|
|
238
|
+
bucket_config_nickname=buckets[0].nickname,
|
|
239
|
+
spec_version=2,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
assert repo.spec_version == 2, f"spec_version=2 should create IC2 repo, got {repo.spec_version}"
|
|
243
|
+
|
|
244
|
+
|
|
204
245
|
@pytest.mark.asyncio
|
|
205
246
|
async def test_modify_repo(isolated_org, default_bucket, token) -> None:
|
|
206
247
|
async with isolated_org(default_bucket()) as (org_name, buckets):
|
|
@@ -5,8 +5,13 @@ from uuid import uuid4
|
|
|
5
5
|
|
|
6
6
|
import icechunk
|
|
7
7
|
import pytest
|
|
8
|
+
from packaging.version import Version
|
|
8
9
|
|
|
9
10
|
from arraylake import AsyncClient, Client
|
|
11
|
+
|
|
12
|
+
# Check if icechunk 2 is available
|
|
13
|
+
ICECHUNK_VERSION = Version(icechunk.__version__)
|
|
14
|
+
IS_IC2 = ICECHUNK_VERSION.major >= 2
|
|
10
15
|
from arraylake.api_utils import ArraylakeHttpClient
|
|
11
16
|
from arraylake.exceptions import ArraylakeClientError
|
|
12
17
|
from arraylake.types import (
|
|
@@ -919,6 +924,7 @@ def assert_subscription_vcap_resolution(client: Client, sub_repo_name: str, vcc_
|
|
|
919
924
|
client.get_repo(sub_repo_name)
|
|
920
925
|
|
|
921
926
|
|
|
927
|
+
@pytest.mark.skipif(not IS_IC2, reason="Paid marketplace listings require icechunk 2.x")
|
|
922
928
|
async def test_filtered_subscription_implementation(two_isolated_orgs, default_bucket, token):
|
|
923
929
|
"""
|
|
924
930
|
Filtered subscription: subscriber has its own bucket, VCC points to provider's chunks.
|
|
@@ -958,6 +964,7 @@ async def test_filtered_subscription_implementation(two_isolated_orgs, default_b
|
|
|
958
964
|
repo_prefix = parent_repo_obj.prefix.strip("/")
|
|
959
965
|
|
|
960
966
|
# Create the icechunk repo manually (anonymous bucket config, but MinIO allows writes)
|
|
967
|
+
# Use spec_version=2 since paid marketplace listings require IC2 repos
|
|
961
968
|
ic_storage = icechunk.s3_storage(
|
|
962
969
|
bucket=provider_repo_bucket.name,
|
|
963
970
|
prefix=parent_repo_obj.prefix,
|
|
@@ -968,7 +975,7 @@ async def test_filtered_subscription_implementation(two_isolated_orgs, default_b
|
|
|
968
975
|
secret_access_key="minio123",
|
|
969
976
|
force_path_style=True,
|
|
970
977
|
)
|
|
971
|
-
icechunk.Repository.create(storage=ic_storage)
|
|
978
|
+
icechunk.Repository.create(storage=ic_storage, spec_version=2)
|
|
972
979
|
|
|
973
980
|
# Construct VCC URL pointing to the provider repo's chunks directory
|
|
974
981
|
vcc_url_prefix = f"s3://{provider_repo_bucket.name}/{repo_prefix}/chunks/"
|
|
@@ -1004,7 +1011,7 @@ async def test_filtered_subscription_implementation(two_isolated_orgs, default_b
|
|
|
1004
1011
|
|
|
1005
1012
|
|
|
1006
1013
|
@asynccontextmanager
|
|
1007
|
-
async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public):
|
|
1014
|
+
async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public, pricing_model="paid"):
|
|
1008
1015
|
"""Set up a provider org with a repo containing virtual chunks, and a subscriber org.
|
|
1009
1016
|
|
|
1010
1017
|
Creates:
|
|
@@ -1012,6 +1019,9 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
|
|
|
1012
1019
|
VCAP covering the virtual chunk bucket (public or private based on `public` param)
|
|
1013
1020
|
- Org B (subscriber): writable bucket on "testbucket"
|
|
1014
1021
|
|
|
1022
|
+
Args:
|
|
1023
|
+
pricing_model: "free" or "paid". Use "free" for direct subscriptions, "paid" for filtered.
|
|
1024
|
+
|
|
1015
1025
|
Yields (client, async_client, org_a, org_b, parent_repo_obj,
|
|
1016
1026
|
provider_repo_bucket, subscriber_bucket, vcc_url_prefix, listing).
|
|
1017
1027
|
"""
|
|
@@ -1052,6 +1062,7 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
|
|
|
1052
1062
|
parent_repo_obj = await async_client.get_repo_object(parent_repo_name)
|
|
1053
1063
|
|
|
1054
1064
|
# Create the icechunk repo manually (anonymous bucket config, but MinIO allows writes)
|
|
1065
|
+
# Use spec_version=2 for paid listings (which require IC2 repos)
|
|
1055
1066
|
ic_storage = icechunk.s3_storage(
|
|
1056
1067
|
bucket=provider_repo_bucket.name,
|
|
1057
1068
|
prefix=parent_repo_obj.prefix,
|
|
@@ -1062,7 +1073,10 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
|
|
|
1062
1073
|
secret_access_key="minio123",
|
|
1063
1074
|
force_path_style=True,
|
|
1064
1075
|
)
|
|
1065
|
-
|
|
1076
|
+
create_kwargs = {"storage": ic_storage}
|
|
1077
|
+
if pricing_model == "paid":
|
|
1078
|
+
create_kwargs["spec_version"] = 2
|
|
1079
|
+
provider_ic_repo = icechunk.Repository.create(**create_kwargs)
|
|
1066
1080
|
|
|
1067
1081
|
# Set VCC for virtual chunks on the provider repo
|
|
1068
1082
|
config = provider_ic_repo.config
|
|
@@ -1076,9 +1090,8 @@ async def provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token,
|
|
|
1076
1090
|
provider_ic_repo.save_config()
|
|
1077
1091
|
|
|
1078
1092
|
# The subscription VCAP is auto-created when the marketplace listing is created
|
|
1079
|
-
# Use paid pricing model to support filtered subscriptions
|
|
1080
1093
|
async with vcap_for_bucket(async_client, org_a, virtual_chunk_bucket, public=public):
|
|
1081
|
-
listing = await create_marketplace_listing(token, org_a, parent_repo_obj.id, pricing_model=
|
|
1094
|
+
listing = await create_marketplace_listing(token, org_a, parent_repo_obj.id, pricing_model=pricing_model)
|
|
1082
1095
|
|
|
1083
1096
|
yield (
|
|
1084
1097
|
client,
|
|
@@ -1109,7 +1122,8 @@ class TestSubscriptionToRepoContainingVirtualChunks:
|
|
|
1109
1122
|
|
|
1110
1123
|
async def test_direct_subscription(self, two_isolated_orgs, default_bucket, token, public):
|
|
1111
1124
|
"""Direct subscription: subscriber shares the provider's storage and VCCs."""
|
|
1112
|
-
|
|
1125
|
+
# Direct subscriptions are only allowed on free listings
|
|
1126
|
+
async with provider_with_virtual_chunks(two_isolated_orgs, default_bucket, token, public, pricing_model="free") as (
|
|
1113
1127
|
client,
|
|
1114
1128
|
_,
|
|
1115
1129
|
_,
|
|
@@ -1123,6 +1137,7 @@ class TestSubscriptionToRepoContainingVirtualChunks:
|
|
|
1123
1137
|
await create_subscription_repo(token, org_b, "sub-repo", listing["id"])
|
|
1124
1138
|
assert_subscription_vcap_resolution(client, f"{org_b}/sub-repo", {vcc_url_prefix}, public)
|
|
1125
1139
|
|
|
1140
|
+
@pytest.mark.skipif(not IS_IC2, reason="Paid marketplace listings require icechunk 2.x")
|
|
1126
1141
|
async def test_filtered_subscription_provider_repo_contains_virtual_chunks(self, two_isolated_orgs, default_bucket, token, public):
|
|
1127
1142
|
"""Filtered subscription where the provider repo also contains virtual chunks.
|
|
1128
1143
|
|