arraylake 0.32.0__tar.gz → 0.33.0__tar.gz

{arraylake-0.32.0 → arraylake-0.33.0}/.gitignore

@@ -104,6 +104,7 @@ celerybeat.pid
 
 # Environments
 .env
+!jobs/subsync/.env
 .venv
 env/
 venv/

{arraylake-0.32.0 → arraylake-0.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arraylake
-Version: 0.32.0
+Version: 0.33.0
 Summary: Python client for ArrayLake
 Project-URL: Documentation, https://docs.earthmover.io/
 Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -28,7 +28,7 @@ Provides-Extra: maximal
 Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
 Requires-Dist: numpy<2.3; extra == 'maximal'
 Requires-Dist: pyarrow<22; extra == 'maximal'
-Requires-Dist: xarray[complete]>=2024.10.0; extra == 'maximal'
+Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
 Provides-Extra: xarray
 Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
 Requires-Dist: xarray>=2024.10.0; extra == 'xarray'

arraylake-0.33.0/arraylake/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.33.0'
+__version_tuple__ = version_tuple = (0, 33, 0)
+
+__commit_id__ = commit_id = None

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/cli/compute.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from enum import Enum
+from enum import StrEnum
 
 import typer
 from rich import print_json
@@ -14,7 +14,7 @@ app = typer.Typer(help="Manage Arraylake compute services", no_args_is_help=True
 logger = get_logger(__name__)
 
 
-class ListOutputType(str, Enum):
+class ListOutputType(StrEnum):
     rich = "rich"
     json = "json"
 

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/cli/config.py

@@ -1,5 +1,5 @@
 import io
-from enum import Enum
+from enum import StrEnum
 from pathlib import Path
 
 import typer
@@ -17,7 +17,7 @@ yaml = YAML(typ="safe", pure=True)
 yaml.default_flow_style = False
 
 
-class ListOutputType(str, Enum):
+class ListOutputType(StrEnum):
     json = "json"
     yaml = "yaml"
 

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/cli/repo.py

@@ -1,6 +1,6 @@
 import json
 import warnings
-from enum import Enum
+from enum import Enum, StrEnum
 from typing import TYPE_CHECKING, Optional
 from uuid import UUID
 
@@ -30,7 +30,7 @@ app = typer.Typer(help="Manage Arraylake repositories", no_args_is_help=True)
 logger = get_logger(__name__)
 
 
-class ListOutputType(str, Enum):
+class ListOutputType(StrEnum):
     rich = "rich"
     json = "json"
 
@@ -142,7 +142,6 @@ async def create(
     repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
     bucket_config_nickname: str | None = typer.Option(None, help="Bucket config nickname"),
     prefix: str = typer.Option(None, help="Optional prefix for Icechunk repo. If not provided, a random ID + the repo name will be used."),
-    import_existing: bool = typer.Option(False, help="Import existing Icechunk data into the new repo"),
     description: str | None = typer.Option(None, help="Description of the repo"),
     metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
 ):
@@ -162,7 +161,6 @@ async def create(
             repo_name,
             bucket_config_nickname=bucket_config_nickname,
             prefix=prefix,
-            import_existing=import_existing,
             description=description,
             metadata=metadata_dict,
         )
@@ -173,7 +171,7 @@ async def create(
 async def import_repo(
     repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
     bucket_config_nickname: str = typer.Argument(..., help="Bucket config nickname"),
-    prefix: str = typer.Option(None, help="Sub-prefix in which the Icechunk repo exists"),
+    prefix: str = typer.Argument(..., help="Prefix in which the Icechunk repo exists in the bucket"),
     description: str | None = typer.Option(None, help="Description of the repo"),
     metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
 ):
@@ -184,15 +182,15 @@ async def import_repo(
     - Import existing Icechunk repository
 
         ```
-        $ arraylake repo import my-org/example-repo --bucket-config-nickname icechunk-bucket --prefix my-icechunk-prefix --metadata '{"key": "value"}'
+        $ arraylake repo import my-org/example-repo icechunk-bucket my-icechunk-prefix --metadata '{"key": "value"}'
         ```
     """
     metadata_dict = json.loads(metadata) if metadata else None
     with simple_progress(f"Importing repo [bold]{repo_name}[/bold]..."):
         await AsyncClient().import_repo(
             repo_name,
-            bucket_config_nickname=bucket_config_nickname,
-            prefix=prefix,
+            bucket_config_nickname,
+            prefix,
             description=description,
             metadata=metadata_dict,
         )

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/client.py

@@ -46,6 +46,7 @@ from arraylake.repos.icechunk.storage import (
     _get_icechunk_storage_obj,
 )
 from arraylake.repos.icechunk.virtual import (
+    create_vcc_store_config_for_platform,
     create_virtual_chunk_container,
     forbid_unsafe_virtual_bucket_configs,
     forbid_unsafe_virtual_chunk_containers,
@@ -558,6 +559,16 @@ class AsyncClient:
             if response.virtual_chunk_credentials:
                 credentials_map: dict[str, icechunk.AnyS3Credential | icechunk.AnyGcsCredential | icechunk.AnyAzureCredential | None] = {}
                 for vcc_prefix, vcc_creds in response.virtual_chunk_credentials.items():
+                    # Handle runtime VCCs (e.g., filtered subscriptions) - create VCC on the fly
+                    if vcc_creds.runtime_vcc_name is not None:
+                        store_config = create_vcc_store_config_for_platform(vcc_creds.platform)
+                        container = icechunk.VirtualChunkContainer(
+                            url_prefix=vcc_prefix,
+                            store=store_config,
+                        )
+                        container.name = vcc_creds.runtime_vcc_name
+                        config.set_virtual_chunk_container(container)
+
                     # TODO: build refresh closures here using vcc_creds.org, vcc_creds.bucket_nickname,
                     # and vcc_creds.platform so that credentials can be refreshed when they expire. (See https://github.com/earth-mover/arraylake/issues/5122)
                     # TODO: support Azure virtual chunk credentials
@@ -644,7 +655,6 @@ class AsyncClient:
         *,
         bucket_config_nickname: BucketNickname | None = None,
         prefix: str | None = None,
-        import_existing: bool = False,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
         config: icechunk.RepositoryConfig | None = None,
@@ -658,7 +668,6 @@ class AsyncClient:
             bucket_config_nickname: Bucket in which the underlying Icechunk repo will be created.
                 If the repo exists, bucket_config_nickname is ignored.
             prefix: Optional prefix for Icechunk store. If not provided, a random ID + the repo name will be used.
-            import_existing: If True, the Icechunk repo will be imported if it already exists.
             description: Optional description for the repo.
             metadata: Optional dictionary of metadata to tag the repo with.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
@@ -707,7 +716,6 @@ class AsyncClient:
                 name,
                 bucket_config_nickname=bucket_config_nickname,
                 prefix=prefix,
-                import_existing=import_existing,
                 description=description,
                 metadata=metadata,
                 config=config,
@@ -721,7 +729,6 @@ class AsyncClient:
         *,
         bucket_config_nickname: BucketNickname | None = None,
         prefix: str | None = None,
-        import_existing: bool = False,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
         config: icechunk.RepositoryConfig | None = None,
@@ -734,7 +741,6 @@ class AsyncClient:
             name: Full name of the repo to create (of the form [ORG]/[REPO]).
             bucket_config_nickname: Bucket in which the underlying Icechunk repo will be created.
             prefix: Optional prefix for Icechunk store. If not provided, a random ID + the repo name will be used.
-            import_existing: If True, the Icechunk repo will be imported if it already exists.
             description: Optional description for the repo.
             metadata: Optional dictionary of metadata to tag the repo with.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
@@ -761,13 +767,12 @@ class AsyncClient:
             bucket_config_nickname,
             kind=RepoKind.Icechunk,
             prefix=prefix,
-            import_existing=import_existing,
             description=description,
             metadata=metadata,
         )
 
         # Raise a warning about token scoping for Azure delegated credentials
-        if repo_model.bucket and repo_model.bucket.auth_config.method == "azure_credential_delegation":
+        if repo_model.bucket and repo_model.bucket.auth_config and repo_model.bucket.auth_config.method == "azure_credential_delegation":
             warnings.warn(
                 "The bucket is configured to use Azure Customer Managed Role authentication. "
                 "Credentials will be scoped to the container level only for this repo. "
@@ -776,6 +781,13 @@ class AsyncClient:
             )
 
         try:
+            # Throw error if the bucket is configured for anonymous access, this is only supported for imported repo
+            if _use_anonymous_credentials(repo_model.bucket):
+                bucket_name = f"{repo_model.bucket.nickname}" if repo_model.bucket else ""
+                raise ValueError(
+                    f"The bucket {bucket_name} is configured for anonymous access and cannot be written to. Use `import_repo` to import existing repositories from this bucket."
+                )
+
             if not isinstance(config, RepositoryConfig) and config is not None:
                 raise ValueError(f"config must be an icechunk.RepositoryConfig object or None: {config}.")
 
@@ -796,41 +808,11 @@ class AsyncClient:
                 # TODO: make icechunk accept an empty dict instead of None
                 prefixes_credential_mapping = None
 
-            # Throws a specific error if the bucket is configured for anonymous access and the repo is not being imported
-            if _use_anonymous_credentials(repo_model.bucket) and not import_existing:
-                bucket_name = f"{repo_model.bucket.nickname}" if repo_model.bucket else ""
-                raise ValueError(
-                    f"The bucket {bucket_name} is configured for anonymous access and cannot be written to. Import existing repositories from this bucket with the `import_existing` parameter."
-                )
-
-            if import_existing:
-                warnings.warn(
-                    "The `import_existing` keyword argument to `create_repo` is deprecated. Please use the dedicated `import_repo` method instead of using `import_existing=True.",
-                    DeprecationWarning,
-                )
-
-                ic_repo = IcechunkRepository.open(
-                    icechunk_storage,
-                    config=config,
-                    authorize_virtual_chunk_access=prefixes_credential_mapping,
-                )
-
-                # Note: We decided this was better than silently attempting to modify the IC repo config upon import by saving the config
-                if authorize_virtual_chunk_access:
-                    warnings.warn(
-                        "New virtual chunk containers will not be persisted to the icechunk repo config. "
-                        "If you want future `client.get_repo` calls to be authorized to access the same virtual chunks, please now modify the config explicitly. "
-                        "You can do this using `client.set_virtual_chunk_containers()` with the same `authorize_virtual_chunk_access`, i.e: "
-                        f"`client.set_virtual_chunk_containers({name}, authorize_virtual_chunk_access={authorize_virtual_chunk_access})`. "
-                        "Note that this requires write access to the location of the icechunk repository, so will not work for anonymous access buckets, for example.",
-                        UserWarning,
-                    )
-            else:
-                ic_repo = IcechunkRepository.create(
-                    icechunk_storage,
-                    config=config,
-                    authorize_virtual_chunk_access=prefixes_credential_mapping,
-                )
+            ic_repo = IcechunkRepository.create(
+                icechunk_storage,
+                config=config,
+                authorize_virtual_chunk_access=prefixes_credential_mapping,
+            )
 
             ic_repo.set_default_commit_metadata({"author_name": author.name, "author_email": author.email})
 
@@ -845,10 +827,11 @@ class AsyncClient:
         self,
         name: OrgAndRepoName,
         bucket_config_nickname: BucketNickname,
+        prefix: str,
         *,
-        prefix: str | None = None,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
+        authorize_virtual_chunk_access: Mapping[BucketPrefix, BucketNickname] | None = None,
         storage_options: StorageOptions | None = None,
     ) -> IcechunkRepository:
         """Create a new Arraylake Repo by importing an existing Icechunk Repository.
@@ -856,35 +839,36 @@ class AsyncClient:
         Args:
             name: Full name of the repo to create (of the form [ORG]/[REPO]).
             bucket_config_nickname: Bucket in which the underlying Icechunk repo exists.
-            prefix: Sub-prefix in which the Icechunk repo exists. If not provided, repo is assumed to be located at the root of the bucket.
+            prefix: Prefix in which the Icechunk repo exists in the bucket.
             description: Optional description for the repo.
             metadata: Optional dictionary of metadata to tag the repo with.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
+            authorize_virtual_chunk_access: A mapping from virtual chunk container prefixes to bucket nicknames.
+                Credentials will be fetched for these buckets based on the auth configuration
+                and used for the containers. Must be supplied as complete urls.
             storage_options: Optional storage options for the underlying Icechunk storage.
                 Currently supports `network_stream_timeout_seconds` for S3, Tigris, and R2 storage.
 
         Returns:
             icechunk.Repository object for the repo.
         """
-
         org, repo_name = _parse_org_and_repo(name)
         mstore = self._metastore_for_org(org)
 
         user = await mstore.get_user()
         author: Author = user.as_author()
 
-        repo_model = await mstore.create_database(
+        repo_model = await mstore.import_database(
             repo_name,
             bucket_config_nickname,
-            kind=RepoKind.Icechunk,
             prefix=prefix,
-            import_existing=True,
+            kind=RepoKind.Icechunk,
             description=description,
             metadata=metadata,
         )
 
         # Raise a warning about token scoping for Azure delegated credentials
-        if repo_model.bucket and repo_model.bucket.auth_config.method == "azure_credential_delegation":
+        if repo_model.bucket and repo_model.bucket.auth_config and repo_model.bucket.auth_config.method == "azure_credential_delegation":
             warnings.warn(
                 "The bucket is configured to use Azure Customer Managed Role authentication. "
                 "Credentials will be scoped to the container level only for this repo. "
@@ -897,7 +881,34 @@ class AsyncClient:
                 repo_model, user_id=user.id, storage_options=storage_options
             )
 
-            ic_repo = IcechunkRepository.open(icechunk_storage)
+            if authorize_virtual_chunk_access is not None:
+                config = await self._add_virtual_chunk_containers(
+                    org=org, user=user, config=None, authorize_virtual_chunk_access=authorize_virtual_chunk_access
+                )
+
+                # fetch credentials to access buckets containing virtual chunks
+                prefixes_credential_mapping = await self.containers_credentials_for_buckets(
+                    org, containers_to_buckets_map=dict(authorize_virtual_chunk_access)
+                )
+
+                # Warn that VCCs won't be persisted when importing
+                warnings.warn(
+                    "New virtual chunk containers will not be persisted to the icechunk repo config. "
+                    "If you want future `client.get_repo` calls to be authorized to access the same virtual chunks, please now modify the config explicitly. "
+                    "You can do this using `client.set_virtual_chunk_containers()` with the same `authorize_virtual_chunk_access`, i.e: "
+                    f"`client.set_virtual_chunk_containers({name}, authorize_virtual_chunk_access={authorize_virtual_chunk_access})`. "
+                    "Note that this requires write access to the location of the icechunk repository, so will not work for anonymous access buckets, for example.",
+                    UserWarning,
+                )
+            else:
+                config = None
+                prefixes_credential_mapping = None
+
+            ic_repo = IcechunkRepository.open(
+                icechunk_storage,
+                config=config,
+                authorize_virtual_chunk_access=prefixes_credential_mapping,
+            )
 
             ic_repo.set_default_commit_metadata({"author_name": author.name, "author_email": author.email})
 
@@ -1600,7 +1611,6 @@ class Client:
         *,
         bucket_config_nickname: BucketNickname | None = None,
         prefix: str | None = None,
-        import_existing: bool = False,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
         config: icechunk.RepositoryConfig | None = None,
@@ -1614,7 +1624,6 @@ class Client:
             bucket_config_nickname: The created repo will use this bucket for its chunks.
                 If the repo exists, bucket_config_nickname is ignored.
             prefix: Optional prefix for Icechunk store. If not provided, a random ID + the repo name will be used.
-            import_existing: If True, the Icechunk repo will be imported if it already exists.
             description: Optional description for the repo.
             metadata: Optional dictionary of metadata to tag the repo with.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
@@ -1640,7 +1649,6 @@ class Client:
             name,
             bucket_config_nickname=bucket_config_nickname,
             prefix=prefix,
-            import_existing=import_existing,
             description=description,
             metadata=metadata,
             config=config,
@@ -1654,7 +1662,6 @@ class Client:
         *,
         bucket_config_nickname: BucketNickname | None = None,
         prefix: str | None = None,
-        import_existing: bool = False,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
         config: icechunk.RepositoryConfig | None = None,
@@ -1667,7 +1674,6 @@ class Client:
             name: Full name of the repo to create (of the form [ORG]/[REPO])
             bucket_config_nickname: An optional bucket to use for the chunkstore
             prefix: Optional prefix for Icechunk store. If not provided, a random ID + the repo name will be used.
-            import_existing: If True, the Icechunk repo will be imported if it already exists.
             description: Optional description for the repo.
             metadata: Optional dictionary of metadata to tag the repo with.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
@@ -1688,7 +1694,6 @@ class Client:
             name,
             bucket_config_nickname=bucket_config_nickname,
             prefix=prefix,
-            import_existing=import_existing,
             description=description,
             metadata=metadata,
             config=config,
@@ -1700,10 +1705,11 @@ class Client:
         self,
         name: OrgAndRepoName,
         bucket_config_nickname: BucketNickname,
+        prefix: str,
         *,
-        prefix: str | None = None,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
+        authorize_virtual_chunk_access: Mapping[BucketPrefix, BucketNickname] | None = None,
         storage_options: StorageOptions | None = None,
     ) -> IcechunkRepository:
         """Create a new Arraylake Repo by importing an existing Icechunk Repository.
@@ -1711,10 +1717,13 @@ class Client:
         Args:
             name: Full name of the repo to create (of the form [ORG]/[REPO]).
             bucket_config_nickname: Bucket in which the underlying Icechunk repo exists.
-            prefix: Sub-prefix in which the Icechunk repo exists. If not provided, repo is assumed to be located at the root of the bucket.
+            prefix: Prefix in which the Icechunk repo exists in the bucket.
             description: Optional description for the repo.
             metadata: Optional dictionary of metadata to tag the repo with.
                 Dictionary values can be a scalar (string, int, float, bool, or None) or a list of scalars.
+            authorize_virtual_chunk_access: A mapping from virtual chunk container prefixes to bucket nicknames.
+                Credentials will be fetched for these buckets based on the auth configuration
+                and used for the containers. Must be supplied as complete urls.
             storage_options: Optional storage options for the underlying Icechunk storage.
                 Currently supports `network_stream_timeout_seconds` for S3, Tigris, and R2 storage.
 
@@ -1724,10 +1733,11 @@ class Client:
         return sync(
             self.aclient.import_repo,
             name,
-            bucket_config_nickname=bucket_config_nickname,
-            prefix=prefix,
+            bucket_config_nickname,
+            prefix,
             description=description,
             metadata=metadata,
+            authorize_virtual_chunk_access=authorize_virtual_chunk_access,
             storage_options=storage_options,
         )
 

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/compute/types.py

@@ -1,11 +1,11 @@
 import re
 from datetime import datetime
-from enum import Enum
+from enum import StrEnum
 
 from pydantic import BaseModel, field_validator
 
 
-class ServiceType(str, Enum):
+class ServiceType(StrEnum):
     dap = "dap2"
     edr = "edr"
     wms = "wms"
@@ -13,14 +13,14 @@ class ServiceType(str, Enum):
     zarr = "zarr"
 
 
-class ServiceStatus(str, Enum):
+class ServiceStatus(StrEnum):
     available = "available"
     progressing = "progressing"
     unknown = "unknown"
     error = "error"
 
 
-class LogLevel(str, Enum):
+class LogLevel(StrEnum):
     debug = "DEBUG"
     info = "INFO"
     warning = "WARNING"

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/metastore/http_metastore.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from dataclasses import dataclass, field
-from typing import Any, Literal
+from typing import Any
 from uuid import UUID
 
 from pydantic import TypeAdapter
@@ -26,6 +26,7 @@ from arraylake.types import (
     Repo,
     RepoActions,
     RepoCreateBody,
+    RepoImportBody,
     RepoKind,
     RepoMetadataT,
     RepoModifyRequest,
@@ -132,10 +133,9 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
         bucket_nickname: str | None = None,
         kind: RepoKind = RepoKind.Icechunk,
         prefix: str | None = None,
-        import_existing: bool = False,
         description: str | None = None,
         metadata: RepoMetadataT | None = None,
-    ):
+    ) -> Repo:
         """
         Creates a repo database entry in the metastore.
 
@@ -144,17 +144,18 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
             bucket_nickname: Optional nickname of a bucket already existing in the org.
             kind: Kind of repo to create
             prefix: Optional prefix for the icechunk repo
-            import_existing: Whether to import an existing icechunk repo
             description: Optional description for the repo
             metadata: Optional metadata for the repo
+
+        Returns:
+            The created Repo object
         """
-        create_mode: Literal["create", "register", "import"] = "import" if import_existing else "register"
         body = RepoCreateBody(
             name=name,
             bucket_nickname=bucket_nickname,
             kind=kind,
             prefix=prefix,
-            create_mode=create_mode,
+            create_mode="register",
             description=description,
             metadata=metadata,
         )
@@ -163,7 +164,46 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
         repo = Repo.model_validate_json(response.content)
 
         if repo.kind == RepoKind.Icechunk:
-            # TODO: should I wrap this response in an IcechunkV2Database object?
+            return repo
+
+        raise ValueError(f"Unknown repo kind: {repo.kind}")
+
+    async def import_database(
+        self,
+        name: str,
+        bucket_nickname: str,
+        prefix: str,
+        kind: RepoKind = RepoKind.Icechunk,
+        description: str | None = None,
+        metadata: RepoMetadataT | None = None,
+    ) -> Repo:
+        """
+        Imports an existing icechunk repo from storage.
+
+        Args:
+            name: Name of the repo to create
+            bucket_nickname: Nickname of the bucket containing the existing icechunk repo.
+            prefix: Prefix where the icechunk repo exists in the bucket.
+            kind: Kind of repo to create
+            description: Optional description for the repo
+            metadata: Optional metadata for the repo
+
+        Returns:
+            The imported Repo object
+        """
+        body = RepoImportBody(
+            name=name,
+            bucket_nickname=bucket_nickname,
+            kind=kind,
+            prefix=prefix,
+            description=description,
+            metadata=metadata,
+        )
+        response = await self._request("POST", f"/orgs/{self._config.org}/repos/import", content=body.model_dump_json())
+        handle_response(response)
+        repo = Repo.model_validate_json(response.content)
+
+        if repo.kind == RepoKind.Icechunk:
             return repo
 
         raise ValueError(f"Unknown repo kind: {repo.kind}")

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/repos/icechunk/types.py

@@ -1,10 +1,10 @@
 import json
 from datetime import datetime
-from enum import Enum
+from enum import StrEnum
 from typing import Any, NewType
 
 from icechunk import Diff, SnapshotInfo
-from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_serializer
+from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_serializer, field_validator
 from zarr import AsyncArray, AsyncGroup
 from zarr.abc.metadata import Metadata
 
@@ -153,7 +153,7 @@ class ExpirationResult(BaseModel):
 # Dataset Tree types for xarray-style representation
 
 
-class ArrayClassification(str, Enum):
+class ArrayClassification(StrEnum):
     """Classification of an array as coordinate or data variable (xarray convention)."""
 
     COORDINATE = "coordinate"
@@ -198,4 +198,24 @@ class DatasetNode(BaseModel):
     child_groups: list[str]  # Names of direct child groups (for navigation)
     is_xarray_compatible: bool = True  # False if dimensions conflict or arrays lack dimension_names
     nbytes: int = 0  # Total bytes consumed by all arrays (uncompressed)
+    # Returning spec_version in this response to avoid another roundtrip in the web app for IC spec version
+    spec_version: int | None = None  # Icechunk spec version (1 or 2)
     model_config = {"arbitrary_types_allowed": True}
+
+    @field_validator("spec_version", mode="before")
+    @classmethod
+    def validate_spec_version(cls, value: Any) -> int | None:
+        """Convert SpecVersion enum to int during validation."""
+        if value is None:
+            return None
+        # Handle icechunk.SpecVersion which has __int__ method
+        return int(value)
+
+    @field_serializer("spec_version")
+    @classmethod
+    def serialize_spec_version(cls, value: Any) -> int | None:
+        """Serialize SpecVersion to int for API responses."""
+        if value is None:
+            return None
+        # Handle icechunk.SpecVersion which has __int__ method
+        return int(value)

{arraylake-0.32.0 → arraylake-0.33.0}/arraylake/repos/icechunk/virtual.py

@@ -19,6 +19,28 @@ from arraylake.types import (
 FORBIDDEN_VIRTUAL_CHUNK_PREFIXES = {"file://", "memory://", "http://"}
 
 
+def create_vcc_store_config_for_platform(platform: str) -> icechunk.AnyObjectStoreConfig:
+    """Create an Icechunk ObjectStoreConfig for a virtual chunk container based on platform.
+
+    This is used for runtime VCCs (e.g., filtered subscriptions) where we need to create
+    a VCC dynamically based on the server-provided platform type.
+
+    Args:
+        platform: The storage platform (s3, s3-compatible, minio, gs, azure)
+
+    Returns:
+        An icechunk ObjectStoreConfig for the given platform
+    """
+    if platform in ("s3", "s3-compatible", "minio"):
+        return icechunk.ObjectStoreConfig.S3(icechunk.S3Options())
+    elif platform == "gs":
+        return icechunk.ObjectStoreConfig.Gcs({})
+    elif platform == "azure":
+        return icechunk.ObjectStoreConfig.Azure({})
+    else:
+        raise ValueError(f"Unsupported platform for VCC store config: {platform}")
+
+
 def get_icechunk_container_credentials(
     bucket_platform: str,
     credentials: S3Credentials | GSCredentials | None,