arraylake 0.18.0__tar.gz → 0.19.0__tar.gz

{arraylake-0.18.0 → arraylake-0.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arraylake
-Version: 0.18.0
+Version: 0.19.0
 Summary: Python client for ArrayLake
 Project-URL: Documentation, https://docs.earthmover.io/
 Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -39,7 +39,7 @@ Requires-Dist: cfgrib~=0.9; extra == 'grib'
 Requires-Dist: eccodes>=2.37; extra == 'grib'
 Requires-Dist: kerchunk<0.3,>=0.2.8; extra == 'grib'
 Provides-Extra: icechunk
-Requires-Dist: icechunk<0.3.0,>=0.2.11; extra == 'icechunk'
+Requires-Dist: icechunk<0.3.0,>=0.2.15; extra == 'icechunk'
 Provides-Extra: maximal
 Requires-Dist: dask>=2022.10.2; extra == 'maximal'
 Requires-Dist: distributed>=2022.10.2; extra == 'maximal'
@@ -73,7 +73,7 @@ Requires-Dist: tenacity==8.4.1; extra == 'min-reqs'
 Requires-Dist: typer==0.12.0; extra == 'min-reqs'
 Requires-Dist: zarr==2.18.0; extra == 'min-reqs'
 Provides-Extra: minicechunk
-Requires-Dist: icechunk==0.2.11; extra == 'minicechunk'
+Requires-Dist: icechunk==0.2.15; extra == 'minicechunk'
 Requires-Dist: numpy==1.25; extra == 'minicechunk'
 Requires-Dist: zarr==3.0.0; extra == 'minicechunk'
 Provides-Extra: virtual

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.18.0'
-__version_tuple__ = version_tuple = (0, 18, 0)
+__version__ = version = '0.19.0'
+__version_tuple__ = version_tuple = (0, 19, 0)

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/cli/repo.py

@@ -84,6 +84,20 @@ def _repos_table(repos, org):
     return table
 
 
+def _make_json_safe(obj):
+    """Convert an object to a JSON-safe format."""
+    if isinstance(obj, dict):
+        return {k: _make_json_safe(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [_make_json_safe(i) for i in obj]
+    elif isinstance(obj, set):
+        return _make_json_safe(list(obj))
+    elif isinstance(obj, Enum):
+        return obj.value
+    else:
+        return obj
+
+
 @app.command(name="list")
 @coro  # type: ignore
 async def list_repos(
@@ -111,7 +125,7 @@ async def list_repos(
         repos = await AsyncClient().list_repos(org, filter_metadata=metadata_filter_dict)
 
     if output == "json":
-        repos = [r._asdict() for r in repos]
+        repos = [_make_json_safe(r._asdict()) for r in repos]
         print_json(data=repos)
     elif repos:
         rich_console.print(_repos_table(repos, org))

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/client.py

@@ -54,12 +54,10 @@ from arraylake.types import (
     ApiClientResponse,
     Author,
     BucketResponse,
-    ExpirationConfig,
-    GCConfig,
     GSCredentials,
     HmacAuth,
-    MaintenanceWindow,
     NewBucket,
+    OptimizationConfig,
     OrgActions,
 )
 from arraylake.types import Repo as RepoModel
@@ -702,12 +700,16 @@ class AsyncClient:
         else:
             credentials = None
 
+        # If config is not set, set scatter_initial_credentials to True by default
+        scatter_initial_credentials = config.get("icechunk.scatter_initial_credentials", True)
+
         return _get_icechunk_storage_obj(
             bucket_config=repo_model.bucket,
             prefix=repo_model.prefix,
             credential_type=_get_credential_type(credentials, credential_refresh_func),
             credentials=credentials,
             credential_refresh_func=credential_refresh_func,
+            scatter_initial_credentials=scatter_initial_credentials,
         )
 
     async def get_icechunk_storage(self, name: str, credentials_override=None):  # icechunk.AnyCredential | None  # -> icechunk.Storage
@@ -784,9 +786,7 @@ class AsyncClient:
         add_metadata: Optional[RepoMetadataT] = None,
         remove_metadata: Optional[list[str]] = None,
         update_metadata: Optional[RepoMetadataT] = None,
-        maintenance_window: Optional[MaintenanceWindow] = None,
-        gc_config: Optional[GCConfig] = None,
-        expiration_config: Optional[ExpirationConfig] = None,
+        optimization_config: Optional[OptimizationConfig] = None,
     ) -> None:
         """Modify a repo's metadata or description.
 
@@ -799,9 +799,7 @@ class AsyncClient:
             remove_metadata: List of metadata keys to remove from the repo.
             update_metadata: Optional dictionary of metadata to update on the repo.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
-            maintenance_window: Optional maintenance window to set for the repo.
-            gc_config: Optional garbage collection configuration for the repo.
-            expiration_config: Optional expiration configuration for the repo.
+            optimization_config: Optional optimization configuration for the repo.
         """
         org, repo_name = _parse_org_and_repo(name)
         mstore = self._metastore_for_org(org)
@@ -811,9 +809,7 @@ class AsyncClient:
             add_metadata=add_metadata,
             remove_metadata=remove_metadata,
             update_metadata=update_metadata,
-            maintenance_window=maintenance_window,
-            gc_config=gc_config,
-            expiration_config=expiration_config,
+            optimization_config=optimization_config,
         )
 
     async def delete_repo(self, name: str, *, imsure: bool = False, imreallysure: bool = False) -> None:
@@ -1276,9 +1272,7 @@ class Client:
         add_metadata: Optional[RepoMetadataT] = None,
         remove_metadata: Optional[list[str]] = None,
         update_metadata: Optional[RepoMetadataT] = None,
-        maintenance_window: Optional[MaintenanceWindow] = None,
-        gc_config: Optional[GCConfig] = None,
-        expiration_config: Optional[ExpirationConfig] = None,
+        optimization_config: Optional[OptimizationConfig] = None,
     ) -> None:
         """Modify a repo's metadata or description.
 
@@ -1291,9 +1285,7 @@ class Client:
             remove_metadata: List of metadata keys to remove from the repo.
             update_metadata: Optional dictionary of metadata to update on the repo.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
-            maintenance_window: Optional maintenance window to set for the repo.
-            gc_config: Optional garbage collection configuration for the repo.
-            expiration_config: Optional expiration configuration for the repo.
+            optimization_config: Optional optimization configurations for the repo.
         """
         return sync(
             self.aclient.modify_repo,
@@ -1302,9 +1294,7 @@ class Client:
             add_metadata=add_metadata,
             remove_metadata=remove_metadata,
             update_metadata=update_metadata,
-            maintenance_window=maintenance_window,
-            gc_config=gc_config,
-            expiration_config=expiration_config,
+            optimization_config=optimization_config,
         )
 
     def delete_repo(self, name: str, *, imsure: bool = False, imreallysure: bool = False) -> None:

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/compute/doctor.py

@@ -1,8 +1,8 @@
-from dataclasses import dataclass
 from typing import TYPE_CHECKING, Literal, TypeAlias
 
 import numpy as np
 import zarr
+from pydantic import BaseModel
 
 from arraylake.log_util import get_logger
 
@@ -24,8 +24,7 @@ WMS_REQUIRED_CF_KEYS: list[CFAttributeKey] = ["latitude", "longitude"]
 REQUIRED_CF_KEYS: list[CFAttributeKey] = EDR_REQUIRED_CF_KEYS + WMS_REQUIRED_CF_KEYS
 
 
-@dataclass
-class CFDiagnosisMissing:
+class CFDiagnosisMissing(BaseModel):
     name: CFAttributeKey
     attr_name: CFAttributeName
     required: bool
@@ -42,8 +41,7 @@ class CFDiagnosisMissing:
         return False
 
 
-@dataclass
-class CFDiagnosisInvalid:
+class CFDiagnosisInvalid(BaseModel):
     name: CFAttributeKey
     attr_name: CFAttributeName
     issue: str
@@ -52,8 +50,7 @@ class CFDiagnosisInvalid:
         return hash((self.name, self.issue))
 
 
-@dataclass
-class CFDiagnosisSummary:
+class CFDiagnosisSummary(BaseModel):
     found: set[CFAttributeKey]  # All of the correct CF keys found
     missing: set[CFDiagnosisMissing]
     invalid: set[CFDiagnosisInvalid]
@@ -93,35 +90,53 @@ class CFDiagnosisSummary:
         return incompatible
 
 
-def _guess_cf_axes_and_coords(ds: "Dataset") -> "Dataset":
+def _guess_cf_axes_and_coords(ds: "Dataset", interactive: bool = True) -> "Dataset":
     import cf_xarray  # noqa
 
     new_ds = ds.cf.guess_coord_axis()
     cf_rep = new_ds.cf
     coords = cf_rep.coordinates
     axes = cf_rep.axes
+
+    # In non-interactive mode (server context), make reasonable assumptions
     if "longitude" not in coords and "X" in axes:
         if len(axes["X"]) == 1 and any(key not in new_ds[axes["X"][0]].attrs for key in ["standard_name", "units"]):
-            message = f"Does the X axis ({axes['X'][0]!r}) represent longitude? (y/n): "
-            modify_x = input(message).lower() == "y"
+            if interactive:
+                message = f"Does the X axis ({axes['X'][0]!r}) represent longitude? (y/n): "
+                modify_x = input(message).lower() == "y"
+            else:
+                # In server mode, assume X axis represents longitude if it's missing standard_name
+                modify_x = "standard_name" not in new_ds[axes["X"][0]].attrs
             if modify_x:
                 new_ds[axes["X"][0]].attrs["standard_name"] = "longitude"
     if "latitude" not in coords and "Y" in axes:
         if len(axes["Y"]) == 1 and any(key not in new_ds[axes["Y"][0]].attrs for key in ["standard_name", "units"]):
-            message = f"Does the Y axis ({axes['Y'][0]!r}) represent latitude? (y/n): "
-            modify_y = input(message).lower() == "y"
+            if interactive:
+                message = f"Does the Y axis ({axes['Y'][0]!r}) represent latitude? (y/n): "
+                modify_y = input(message).lower() == "y"
+            else:
+                # In server mode, assume Y axis represents latitude if it's missing standard_name
+                modify_y = "standard_name" not in new_ds[axes["Y"][0]].attrs
             if modify_y:
                 new_ds[axes["Y"][0]].attrs["standard_name"] = "latitude"
     if "vertical" not in coords and "Z" in axes:
         if len(axes["Z"]) == 1 and any(key not in new_ds[axes["Z"][0]].attrs for key in ["standard_name", "units"]):
-            message = f"Does the Z axis ({axes['Z'][0]!r}) represent vertical? (y/n): "
-            modify_z = input(message).lower() == "y"
+            if interactive:
+                message = f"Does the Z axis ({axes['Z'][0]!r}) represent vertical? (y/n): "
+                modify_z = input(message).lower() == "y"
+            else:
+                # In server mode, assume Z axis represents vertical if it's missing standard_name
+                modify_z = "standard_name" not in new_ds[axes["Z"][0]].attrs
             if modify_z:
                 new_ds[axes["Z"][0]].attrs["standard_name"] = "vertical"
     if "time" not in coords and "T" in axes:
         if len(axes["T"]) == 1 and any(key not in new_ds[axes["T"][0]].attrs for key in ["standard_name", "units"]):
-            message = f"Does the T axis ({axes['T'][0]!r}) represent time? (y/n): "
-            modify_t = input(message).lower() == "y"
+            if interactive:
+                message = f"Does the T axis ({axes['T'][0]!r}) represent time? (y/n): "
+                modify_t = input(message).lower() == "y"
+            else:
+                # In server mode, assume T axis represents time if it's missing standard_name
+                modify_t = "standard_name" not in new_ds[axes["T"][0]].attrs
             if modify_t:
                 new_ds[axes["T"][0]].attrs["standard_name"] = "time"
 
@@ -136,11 +151,12 @@ def _is_wms_compatible(keys: set[CFAttributeKey]) -> bool:
     return all(k in keys for k in WMS_REQUIRED_CF_KEYS)
 
 
-def check_cf_attribute_completeness(ds: "Dataset") -> CFDiagnosisSummary:
+def check_cf_attribute_completeness(ds: "Dataset", interactive: bool = True) -> CFDiagnosisSummary:
     """Returns a dictionary of the CF compliance errors in the provided dataset.
 
     Args:
         ds: The xarray dataset to check.
+        interactive: Whether to prompt user for input when guessing CF attributes.
 
     Returns:
         dict: A diagnostic dictionary containing the found, missing, and invalid CF compliance errors.
@@ -151,7 +167,7 @@ def check_cf_attribute_completeness(ds: "Dataset") -> CFDiagnosisSummary:
     found, missing, invalid = set(), set(), set()
 
     # Guess the axes, to use for fixing missing attributes
-    proposed_fix_ds = _guess_cf_axes_and_coords(ds)
+    proposed_fix_ds = _guess_cf_axes_and_coords(ds, interactive=interactive)
 
     for k in SPATIAL_CF_KEYS:
         if k in cf_keys:

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/config.yaml

@@ -4,6 +4,8 @@ service:
   ssl:
     verify: True
     cafile:
+icechunk:
+  scatter_initial_credentials: True
 chunkstore:
   uri:
   hash_method: hashlib.sha256

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/repos/icechunk/storage.py

@@ -48,6 +48,7 @@ def _get_icechunk_storage_obj(
     credential_type: CredentialType,
     credentials: TempCredentials | None,
     credential_refresh_func: Callable | None,
+    scatter_initial_credentials: bool,
 ):  # Removed output type so icechunk import is not required
     """Gets the Icechunk storage object.
 
@@ -64,6 +65,10 @@ def _get_icechunk_storage_obj(
             Optional function to refresh credentials. This function must
             be synchronous, cannot take in any args, and return a
             icechunk.S3StaticCredentials or icechunk.GcsBearerCredential object.
+        scatter_initial_credentials:
+            Whether to immediately call and store the value returned by get_credentials.
+            Ensures that the same set of credentials are available for all workers in a
+            distributed setup.
 
     Returns:
         Icechunk Storage object
@@ -112,6 +117,7 @@ def _get_icechunk_storage_obj(
                 anonymous=credential_type == CredentialType.ANONYMOUS,
                 from_env=credential_type == CredentialType.ENV,
                 get_credentials=credential_refresh_func,
+                scatter_initial_credentials=scatter_initial_credentials,
             )
         # Use s3_storage to create the storage object for s3 or s3-compatible buckets
         return icechunk.s3_storage(
@@ -128,6 +134,7 @@ def _get_icechunk_storage_obj(
             from_env=credential_type == CredentialType.ENV,
             get_credentials=credential_refresh_func,
             force_path_style=bool(force_path_style),  # mypy thinks force_path_style could be a str
+            scatter_initial_credentials=scatter_initial_credentials,
         )
     # Otherwise, check if the bucket is a GCS bucket
     elif bucket_config.platform in ("gs"):
@@ -145,6 +152,7 @@ def _get_icechunk_storage_obj(
             from_env=credential_type == CredentialType.ENV,
             config=None,
             get_credentials=credential_refresh_func,
+            scatter_initial_credentials=scatter_initial_credentials,
         )
     else:
         raise ValueError(f"Unsupported bucket platform: {bucket_config.platform}")

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/repos/v1/metastore/http_metastore.py

@@ -39,12 +39,10 @@ from arraylake.types import (
     ApiClientResponse,
     BucketModifyRequest,
     BucketResponse,
-    ExpirationConfig,
-    GCConfig,
     GSCredentials,
-    MaintenanceWindow,
     NewBucket,
     NewRepoOperationStatus,
+    OptimizationConfig,
     OrgActions,
     PermissionBody,
     PermissionCheckResponse,
@@ -204,18 +202,14 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
         add_metadata: RepoMetadataT | None = None,
         remove_metadata: list[str] | None = None,
         update_metadata: RepoMetadataT | None = None,
-        maintenance_window: MaintenanceWindow | None = None,
-        gc_config: GCConfig | None = None,
-        expiration_config: ExpirationConfig | None = None,
+        optimization_config: OptimizationConfig | None = None,
     ) -> None:
         repo_modify_request = RepoModifyRequest(
             description=description,
             add_metadata=add_metadata,
             remove_metadata=remove_metadata,
             update_metadata=update_metadata,
-            maintenance_window=maintenance_window,
-            gc_config=gc_config,
-            expiration_config=expiration_config,
+            optimization_config=optimization_config,
         )
         response = await self._request("PATCH", f"/orgs/{self._config.org}/{name}", content=repo_modify_request.model_dump_json())
         handle_response(response)

{arraylake-0.18.0 → arraylake-0.19.0}/arraylake/types.py

@@ -100,17 +100,13 @@ class RepoModifyRequest(BaseModel):
     add_metadata: RepoMetadataT | None = None
     remove_metadata: list[str] | None = None
     update_metadata: RepoMetadataT | None = None
-    gc_config: GCConfig | None = None
-    expiration_config: ExpirationConfig | None = None
-    maintenance_window: MaintenanceWindow | None = None
+    optimization_config: OptimizationConfig | None = None
 
 
 class RepoModifyBody(BaseModel):
     description: str | None = None
     metadata: RepoMetadataT | None = None
-    gc_config: GCConfig | None = None
-    expiration_config: ExpirationConfig | None = None
-    maintenance_window: MaintenanceWindow | None = None
+    optimization_config: Optional[OptimizationConfig] = None
 
 
 # Used for `/orgs/{org}/{repo}/visibility` endpoint
@@ -392,13 +388,27 @@ class GCConfig(BaseModel):
 
     @field_serializer("dangling_chunks", "dangling_manifests", "dangling_attributes", "dangling_transaction_logs", "dangling_snapshots")
     @classmethod
-    def serialize_duration(self, value: GCAction) -> float | None:
+    def serialize_dangling(self, value: GCAction) -> float | None:
         if isinstance(value, GCKeep):
             return None
         else:
             assert isinstance(value, GCDeleteOlderThan)
             return value.date.total_seconds()
 
+    @field_validator("gc_every", mode="before")
+    @classmethod
+    def parse_gc_every(cls, v: Any) -> timedelta | None:
+        if v is not None:
+            return timedelta_from_seconds(v)
+        return None
+
+    @field_serializer("gc_every")
+    @classmethod
+    def serialize_gc_every(self, value: timedelta | None) -> float | None:
+        if value is not None:
+            return value.total_seconds()
+        return None
+
 
 class ExpirationConfig(BaseModel):
     """Configuration for expiration"""
@@ -416,14 +426,14 @@ class ExpirationConfig(BaseModel):
 
     @field_serializer("expire_every", "expire_versions_older_than")
     @classmethod
-    def serialize_duration(self, value: timedelta | None) -> float | None:
+    def serialize_expire_every(self, value: timedelta | None) -> float | None:
         if value is not None:
             return value.total_seconds()
         return None
 
 
-class MaintenanceWindow(BaseModel):
-    """The preferred time to run maintenance tasks on a repo"""
+class OptimizationWindow(BaseModel):
+    """The preferred time to run optimization tasks on a repo"""
 
     duration: timedelta
     start_time: time  # Time of day in UTC
@@ -440,6 +450,14 @@ class MaintenanceWindow(BaseModel):
         return duration.total_seconds()
 
 
+class OptimizationConfig(BaseModel):
+    """High-level config for repo optimizations"""
+
+    window: Optional[OptimizationWindow] = None  # If None, uses the org default
+    gc_config: Optional[GCConfig] = None
+    expiration_config: Optional[ExpirationConfig] = None
+
+
 class Repo(ModelWithID):
     org: str
     name: str
@@ -453,9 +471,13 @@ class Repo(ModelWithID):
     status: RepoOperationStatusResponse
     kind: RepoKind = RepoKind.V1
     prefix: Optional[str] = None
-    maintenance_window: Optional[MaintenanceWindow] = None  # If None, uses the org default
-    gc_config: Optional[GCConfig] = None
-    expiration_config: Optional[ExpirationConfig] = None
+    optimization_config: OptimizationConfig = Field(
+        default_factory=lambda: OptimizationConfig(
+            window=None,
+            gc_config=None,
+            expiration_config=None,
+        )
+    )
     permissions: list[Permission[RepoActions]] = Field(default_factory=list)
 
     def _asdict(self):

{arraylake-0.18.0 → arraylake-0.19.0}/pyproject.toml

@@ -87,11 +87,11 @@ maximal = [
   "xarray >= 2024.10.0",
 ]
 icechunk = [
-  "icechunk >= 0.2.11, < 0.3.0"
+  "icechunk >= 0.2.15, < 0.3.0"
 ]
 # TODO: move to dependency group once pip supports it
 minicechunk = [
-  "icechunk==0.2.11",
+  "icechunk==0.2.15",
   "numpy==1.25",  # min zarr 3 numpy
   "zarr==3.0.0",
 ]

{arraylake-0.18.0 → arraylake-0.19.0}/tests/config.yaml

@@ -5,3 +5,5 @@ service:
   uri: http://localhost:8000
 s3:
   endpoint_url: http://localhost:9000
+icechunk:
+  scatter_initial_credentials: False

{arraylake-0.18.0 → arraylake-0.19.0}/tests/icechunk/test_client.py

@@ -7,6 +7,7 @@ import zarr
 from icechunk import IcechunkError, Repository, S3Credentials, Storage
 
 from arraylake import AsyncClient, Client
+from arraylake.config import config
 from arraylake.types import DBID
 from arraylake.types import Repo as RepoModel
 from arraylake.types import RepoKind, RepoOperationMode, RepoOperationStatusResponse
@@ -197,7 +198,8 @@ async def test_get_icechunk_storage_from_repo_model(isolated_org_with_bucket, to
         status=RepoOperationStatusResponse(mode=RepoOperationMode.ONLINE, initiated_by={}),
         bucket=bucket_config,
     )
-    storage = await aclient._get_icechunk_storage_from_repo_model(repo_model=repo_model)
+    with config.set({"icechunk.scatter_initial_credentials": False}):
+        storage = await aclient._get_icechunk_storage_from_repo_model(repo_model=repo_model)
     assert isinstance(storage, Storage)
 
 

{arraylake-0.18.0 → arraylake-0.19.0}/tests/icechunk/test_storage.py

@@ -31,6 +31,7 @@ def test_get_icechunk_storage_obj_s3_credentials(
         credentials=s3_credentials,
         credential_type=CredentialType.PROVIDED,
         credential_refresh_func=None,
+        scatter_initial_credentials=False,
     )
     assert isinstance(storage, icechunk.Storage)
 
@@ -45,6 +46,7 @@ def test_get_icechunk_storage_obj_gcs_credentials(
         credentials=gs_credentials,
         credential_type=CredentialType.PROVIDED,
         credential_refresh_func=None,
+        scatter_initial_credentials=False,
     )
     assert isinstance(storage, icechunk.Storage)
 
@@ -54,7 +56,12 @@ def test_get_icechunk_storage_obj_from_env(s3_bucket_config: BucketResponse, mon
         monkeypatch.setenv(var, "test")
 
     storage = _get_icechunk_storage_obj(
-        bucket_config=s3_bucket_config, prefix=None, credentials=None, credential_type=CredentialType.ENV, credential_refresh_func=None
+        bucket_config=s3_bucket_config,
+        prefix=None,
+        credentials=None,
+        credential_type=CredentialType.ENV,
+        credential_refresh_func=None,
+        scatter_initial_credentials=False,
     )
     assert isinstance(storage, icechunk.Storage)
 
@@ -70,6 +77,7 @@ def test_get_icechunk_storage_obj_s3_no_credentials(s3_bucket_config: BucketResp
         credentials=None,
         credential_type=CredentialType.ANONYMOUS,
         credential_refresh_func=None,
+        scatter_initial_credentials=False,
     )
     assert isinstance(storage, icechunk.Storage)
 
@@ -81,6 +89,7 @@ def test_get_icechunk_storage_obj_tigris_no_credentials(tigris_bucket_config: Bu
         credentials=None,
         credential_type=CredentialType.ANONYMOUS,
         credential_refresh_func=None,
+        scatter_initial_credentials=False,
     )
     assert isinstance(storage, icechunk.Storage)
 
@@ -95,6 +104,7 @@ def test_get_icechunk_storage_obj_tigris_no_region(tigris_bucket_config: BucketR
             credentials=None,
             credential_type=CredentialType.ENV,
             credential_refresh_func=None,
+            scatter_initial_credentials=False,
         )
     assert "Region cannot be None for Tigris buckets" in str(e.value)
 
@@ -109,13 +119,19 @@ def test_get_icechunk_storage_obj_tigris_auto_region(tigris_bucket_config: Bucke
             credentials=None,
             credential_type=CredentialType.ENV,
             credential_refresh_func=None,
+            scatter_initial_credentials=False,
         )
     assert "Region cannot be auto for Tigris buckets" in str(e.value)
 
 
 def test_get_icechunk_storage_obj_gcs_no_credentials(gcs_bucket_config: BucketResponse):
     storage = _get_icechunk_storage_obj(
-        bucket_config=gcs_bucket_config, prefix=None, credentials=None, credential_type=CredentialType.ENV, credential_refresh_func=None
+        bucket_config=gcs_bucket_config,
+        prefix=None,
+        credentials=None,
+        credential_type=CredentialType.ENV,
+        credential_refresh_func=None,
+        scatter_initial_credentials=False,
     )
     assert isinstance(storage, icechunk.Storage)
 
@@ -130,5 +146,6 @@ def test_get_icechunk_storage_obj_invalid_credential_input(
             credentials=s3_credentials,
             credential_type=CredentialType.PROVIDED,
             credential_refresh_func=credential_refresh_func,
+            scatter_initial_credentials=True,
         )
     assert "Cannot provide both static credentials and a credential refresh function" in str(e.value)

{arraylake-0.18.0 → arraylake-0.19.0}/tests/test_client.py

@@ -25,7 +25,8 @@ from arraylake.types import (
     GCConfig,
     GCDeleteOlderThan,
     GCKeep,
-    MaintenanceWindow,
+    OptimizationConfig,
+    OptimizationWindow,
     RepoKind,
     UserInfo,
 )
@@ -187,7 +188,7 @@ async def test_modify_repo(token, isolated_org_with_bucket, repo_kind_to_test: R
 
     # Modify metadata
     new_description = "This is a modified test repo"
-    maintenance_window = MaintenanceWindow(
+    optimization_window = OptimizationWindow(
         duration=3600,  # 1 hour
         start_time=dt_time(2, 0),  # 2:00 AM UTC
         day_of_week=1,  # Monday
@@ -207,6 +208,11 @@ async def test_modify_repo(token, isolated_org_with_bucket, repo_kind_to_test: R
         expire_every=None,
         enabled=True,
     )
+    optimization_config = OptimizationConfig(
+        expiration_config=expiration_config,
+        gc_config=gc_config,
+        window=optimization_window,
+    )
     add_metadata = {"legume": "soybean", "fats": "butter"}
     remove_metadata = ["dairy"]
     update_metadata = {"fruits": ["pear", "kiwi"], "vegetable": ["broccoli", "carrot"], "healthy": False}
@@ -225,9 +231,7 @@ async def test_modify_repo(token, isolated_org_with_bucket, repo_kind_to_test: R
         add_metadata=add_metadata,
         remove_metadata=remove_metadata,
         update_metadata=update_metadata,
-        maintenance_window=maintenance_window,
-        gc_config=gc_config,
-        expiration_config=expiration_config,
+        optimization_config=optimization_config,
     )
     updated_repo_obj = await aclient.get_repo_object(repo_name)
     assert updated_repo_obj.metadata.keys() == expected_metadata.keys()
@@ -238,9 +242,9 @@ async def test_modify_repo(token, isolated_org_with_bucket, repo_kind_to_test: R
             assert updated_repo_obj.metadata[key] == value
     assert updated_repo_obj.description == new_description
     assert updated_repo_obj.updated > current_repo_obj.updated
-    assert updated_repo_obj.maintenance_window == maintenance_window
-    assert updated_repo_obj.gc_config == gc_config
-    assert updated_repo_obj.expiration_config == expiration_config
+    assert updated_repo_obj.optimization_config.window == optimization_window
+    assert updated_repo_obj.optimization_config.gc_config == gc_config
+    assert updated_repo_obj.optimization_config.expiration_config == expiration_config
 
 
 def test_modify_repo_raises(token, isolated_org_with_bucket, repo_kind_to_test: RepoKind) -> None:
@@ -284,9 +288,9 @@ async def test_modify_repo_no_change(token, isolated_org_with_bucket, repo_kind_
     assert updated_repo_obj.metadata == metadata
     assert updated_repo_obj.description is None
     assert updated_repo_obj.updated == current_repo_obj.updated
-    assert updated_repo_obj.maintenance_window is None
-    assert updated_repo_obj.gc_config is None
-    assert updated_repo_obj.expiration_config is None
+    assert updated_repo_obj.optimization_config.window is None
+    assert updated_repo_obj.optimization_config.gc_config is None
+    assert updated_repo_obj.optimization_config.expiration_config is None
 
 
 def test_list_repos_filter_metadata(token, isolated_org_with_bucket, repo_kind_to_test: RepoKind):

{arraylake-0.18.0 → arraylake-0.19.0}/tests/test_config.py

@@ -19,3 +19,4 @@ def test_config_defaults_set():
     assert "verify" in arraylake.config.defaults[0]["service"]["ssl"]
     assert "hash_method" in arraylake.config.defaults[0]["chunkstore"]
     assert "inline_threshold_bytes" in arraylake.config.defaults[0]["chunkstore"]
+    assert "scatter_initial_credentials" in arraylake.config.defaults[0]["icechunk"]