arraylake 1.5.0__tar.gz → 1.7.0__tar.gz

{arraylake-1.5.0 → arraylake-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: arraylake
-Version: 1.5.0
+Version: 1.7.0
 Summary: Python client for ArrayLake
 Project-URL: Documentation, https://docs.earthmover.io/
 Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -15,23 +15,23 @@ Requires-Dist: click<9.0,>=8.1.3
 Requires-Dist: donfig<1.0,>=0.7
 Requires-Dist: httpx<0.29,>=0.23
 Requires-Dist: icechunk<3.0.0,>=2.2.2
-Requires-Dist: numpy<3.0,>=1.23
 Requires-Dist: packaging<27.0,>=23.0
 Requires-Dist: pydantic[email]<3.0,>=2.9
 Requires-Dist: python-dateutil<3.0,>=2.8
+Requires-Dist: pyyaml<7.0,>=6.0.1
 Requires-Dist: rich<16.0,>=12.6
 Requires-Dist: ruamel-yaml<1.0,>=0.17
 Requires-Dist: structlog<27.0,>=24.1
 Requires-Dist: typer<1.0,>=0.12.0
 Requires-Dist: zarr<4.0,>=3.1.0
 Provides-Extra: iceberg
-Requires-Dist: pyiceberg<0.13,>=0.11; extra == 'iceberg'
+Requires-Dist: pyiceberg[pyarrow]<0.13,>=0.11; extra == 'iceberg'
 Provides-Extra: maximal
 Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
 Provides-Extra: obstore
 Requires-Dist: obstore>=0.9.2; extra == 'obstore'
 Provides-Extra: openeo
-Requires-Dist: openeo<0.52,>=0.35.0; extra == 'openeo'
+Requires-Dist: openeo<0.53,>=0.35.0; extra == 'openeo'
 Provides-Extra: xarray
 Requires-Dist: xarray>=2024.10.0; extra == 'xarray'
 Description-Content-Type: text/markdown

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.5.0'
-__version_tuple__ = version_tuple = (1, 5, 0)
+__version__ = version = '1.7.0'
+__version_tuple__ = version_tuple = (1, 7, 0)
 
 __commit_id__ = commit_id = None

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/cli/compute.py

@@ -3,6 +3,7 @@ from enum import StrEnum
 
 import typer
 from rich import print_json
+from rich.markup import escape
 from rich.table import Table
 
 from arraylake import Client
@@ -25,6 +26,7 @@ def _services_table(services, org):
     table.add_column("Service Type", justify="right", style="green", min_width=10)
     table.add_column("Org", justify="right", style="green", min_width=10)
     table.add_column("Public", justify="right", style="green", min_width=10)
+    table.add_column("Region", justify="right", style="green", min_width=10)
     table.add_column("URL", justify="right", style="green", min_width=10)
     table.add_column("Status", justify="right", style="green", min_width=10)
 
@@ -45,6 +47,7 @@ def _services_table(services, org):
             service.config.service_type,
             service.config.org,
             str(service.config.is_public),
+            service.region or "",
             service.url,
             service.status,
             # .get, because the API owns this vocabulary and adds to it: an older client must
@@ -67,6 +70,12 @@ def enable_service(
         "-n",
         help="Custom deployment name (defaults to protocol name). Must be lowercase alphanumeric with dashes, max 16 chars.",
     ),
+    region: str | None = typer.Option(
+        None,
+        "--region",
+        "-r",
+        help="The region to run the service in, e.g. eu-west-1. Defaults to the environment's default region.",
+    ),
 ):
     """**Enable** a new service
 
@@ -83,13 +92,19 @@ def enable_service(
         ```
         $ arraylake compute enable my-org tiles --sku-code s --deployment-name my-tiles
         ```
+
+    - Enable an EDR service in Europe
+
+        ```
+        $ arraylake compute enable my-org edr --sku-code xs --region eu-west-1
+        ```
     """
-    kwargs = {}
+    kwargs: dict[str, str] = {}
     if deployment_name is not None:
         kwargs["deployment_name"] = deployment_name
 
     with simple_progress(f"Creating [bold]{protocol}[/bold] service for [bold]{org}[/bold]..."):
-        status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, **kwargs)
+        status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, region=region, **kwargs)
     rich_console.print(status["message"])
 
 
@@ -190,8 +205,8 @@ async def get_service_logs(
 
         async for log in log_stream:
             rich_console.print(log)
-    except RuntimeError:
-        rich_console.print(f"[bold red]No logs found for {org}/{service_id}. Please try again later.[/bold red]")
+    except RuntimeError as err:
+        rich_console.print(f"[bold red]{escape(str(err))}[/bold red]")
         raise typer.Exit(code=1)
 
 
@@ -210,7 +225,7 @@ def update_service(
     - Resize a service with ID edr-1234567 to a different compute shape
 
         ```
-        $ arraylake compute update my-org edr-1234567 --sku medium
+        $ arraylake compute update my-org edr-1234567 --sku m
         ```
 
     - Change the scaling bounds of a service

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/cli/config.py

@@ -162,7 +162,7 @@ def unset(
 @app.command()
 def get(
     key: str = typer.Argument(..., help="key to get in config file"),
-    path: Path = typer.Option(user_config_file, help="Config file to update"),
+    path: Path | None = typer.Option(None, help="Config file to read. Defaults to the active config."),
     output: ListOutputType = typer.Option("yaml", help="Output formatting"),
 ):
     """**Get** Arraylake config value
@@ -191,11 +191,11 @@ def get(
 def init(path: Path = typer.Option(user_config_file, help="Config file to initialize")):
     """**Initialize** Arraylake config
 
-    This command will walk you through the steps to configure Arraylake.
+    Creates the config file if it does not exist.
     """
 
     c = get_user_config(path=path)
 
     # write back to file
     write_user_config(c, path=path)
-    rich_console.print(f"✅ Config file updated at {path}")
+    rich_console.print(f"Config file: {path}")

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/cli/iceberg.py

@@ -109,7 +109,7 @@ def list_namespaces(
     filter_metadata: str | None = typer.Option(None, help="JSON metadata filter"),
     sort: NamespaceSort | None = None,
     direction: Direction = Direction.desc,
-    include_ghosts: bool = False,
+    include_ghosts: bool = typer.Option(False, help="Also list deleted namespaces that you can still restore"),
     page_size: int = typer.Option(50, min=1, max=100),
     output: Output = Output.rich,
 ):
@@ -195,10 +195,22 @@ def delete_namespace(
 ):
     """Delete an empty namespace. A soft-delete by default: restore it during its recovery period."""
     if not confirm:
-        verb = "Permanently delete" if immediate else "Delete"
-        typer.confirm(f"{verb} namespace {org}/{name}?", abort=True)
+        typer.confirm(
+            f"{_namespace_delete_outcome(org, name, immediate=immediate, retain_data=retain_data, done=False)} Continue?", abort=True
+        )
     Client().delete_iceberg_namespace(org, name, imsure=True, immediate=immediate, retain_data=retain_data)
-    typer.echo(f"Deleted {org}/{name}")
+    typer.echo(_namespace_delete_outcome(org, name, immediate=immediate, retain_data=retain_data, done=True))
+
+
+def _namespace_delete_outcome(org: str, name: str, *, immediate: bool, retain_data: bool, done: bool) -> str:
+    verb = "Deleted namespace" if done else "This will delete namespace"
+    if not immediate:
+        return (
+            f"{verb} {org}/{name}. Until the grace period ends, you can restore it with `arraylake iceberg namespace restore {org} {name}`."
+        )
+    if retain_data:
+        return f"{verb} {org}/{name} from Arraylake. Its data stays in your storage bucket."
+    return f"{verb} {org}/{name} and its data. You cannot restore it."
 
 
 @namespace_app.command("restore")

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/cli/repo.py

@@ -253,7 +253,7 @@ async def delete(
     """
     if not confirm:
         confirm = typer.confirm(
-            f"This will permanently remove the {repo_name} repo. Are you sure you want to continue?",
+            f"{_delete_outcome(repo_name, immediate=immediate, retain_data=retain_data, done=False)} Continue?",
             abort=True,
         )
 
@@ -262,8 +262,16 @@ async def delete(
     with simple_progress(f"Deleting repo [bold]{repo_name}[/bold]..."):
         await client.delete_repo(repo_name, imsure=confirm, imreallysure=confirm, immediate=immediate, retain_data=retain_data)
 
-    # If the repo is a icechunk repo, print message that the bucket must be deleted manually
-    rich_console.print(f"Repo [bold]{repo_name}[/bold] removed from Arraylake. \nThe underlying Icechunk bucket must be deleted manually.")
+    rich_console.print(_delete_outcome(repo_name, immediate=immediate, retain_data=retain_data, done=True))
+
+
+def _delete_outcome(repo_name: str, *, immediate: bool, retain_data: bool, done: bool) -> str:
+    verb = "Deleted" if done else "This will delete"
+    if not immediate:
+        return f"{verb} {repo_name}. You can restore it from the web app until the grace period ends."
+    if retain_data:
+        return f"{verb} {repo_name} from Arraylake. Its data stays in your storage bucket."
+    return f"{verb} {repo_name} and its data. You cannot restore it."
 
 
 @app.command()
@@ -339,7 +347,7 @@ async def set_status(
 @coro  # type: ignore
 async def tune(
     org_name: str = typer.Argument(..., help="Name of organization"),
-    bucket_config_nickname: str | None = typer.Option(None, help="Chunkstore bucket config nickname"),
+    bucket_config_nickname: str | None = typer.Option(None, help="Bucket config to benchmark. Defaults to the org's default bucket."),
 ):
     """Diagnose I/O configuration for optimal performance
 

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/client.py

@@ -119,6 +119,12 @@ def _parse_org_and_repo(org_and_repo: OrgAndRepoName) -> tuple[OrgName, RepoName
     return (parts[0], parts[1])
 
 
+def _repo_status_text(headline: str, status: RepoOperationStatusResponse) -> str:
+    until = f" until {status.estimated_end_time}" if status.estimated_end_time else ""
+    reason = f" {status.message}" if status.message else ""
+    return f"{headline}{until}.{reason}"
+
+
 def _validate_service_uri(service_uri: str) -> None:
     if not service_uri.startswith("http"):
         raise ValueError("service uri must start with http")
@@ -732,11 +738,9 @@ class AsyncClient:
         response = await mstore.open_repo(repo_name)
 
         if response.status.mode == RepoOperationMode.OFFLINE:
-            raise ValueError(f"Repository is offline, check back after {response.status.estimated_end_time}")
+            raise ValueError(_repo_status_text(f"Repo {name} is offline", response.status))
         elif response.status.mode == RepoOperationMode.MAINTENANCE:
-            warnings.warn(
-                f"Repository is in maintenance mode until {response.status.estimated_end_time}, do not attempt to write to the store"
-            )
+            warnings.warn(_repo_status_text(f"Repo {name} is read-only for maintenance", response.status))
 
         return org, repo_name, response
 
@@ -1293,9 +1297,9 @@ class AsyncClient:
         # Raise a warning about token scoping for Azure delegated credentials
         if repo_model.bucket and repo_model.bucket.auth_config and repo_model.bucket.auth_config.method == "azure_credential_delegation":
             warnings.warn(
-                "The bucket is configured to use Azure Customer Managed Role authentication. "
-                "Credentials will be scoped to the container level only for this repo. "
-                "Ensure that no other sensitive data is stored in the same container, as the credentials will have read access to all data in the container.",
+                "This bucket uses Azure credential delegation, so credentials cover the whole container, not only this repo. "
+                "Users who can write to this repo can also write to every other blob in the container. "
+                "Do not keep other data in this container.",
                 UserWarning,
             )
 
@@ -1397,9 +1401,9 @@ class AsyncClient:
         # Raise a warning about token scoping for Azure delegated credentials
         if repo_model.bucket and repo_model.bucket.auth_config and repo_model.bucket.auth_config.method == "azure_credential_delegation":
             warnings.warn(
-                "The bucket is configured to use Azure Customer Managed Role authentication. "
-                "Credentials will be scoped to the container level only for this repo. "
-                "Ensure that no other sensitive data is stored in the same container, as the credentials will have read access to all data in the container.",
+                "This bucket uses Azure credential delegation, so credentials cover the whole container, not only this repo. "
+                "Users who can write to this repo can also write to every other blob in the container. "
+                "Do not keep other data in this container.",
                 UserWarning,
             )
 
@@ -1471,7 +1475,7 @@ class AsyncClient:
         from arraylake import __version__ as arraylake_version
 
         if not repo_model.subscription and not repo_model.bucket:
-            raise ValueError("The bucket on the catalog object cannot be None for Icechunk V2 repos!")
+            raise ValueError(f"Repo {repo_model.org}/{repo_model.name} has no storage bucket.")
 
         effective_bucket = repo_model.bucket
         effective_prefix = repo_model.prefix
@@ -1481,7 +1485,7 @@ class AsyncClient:
                 effective_prefix = repo_model.subscription.parent_repo.prefix
 
         if effective_bucket is None:
-            raise ValueError("Repo has no bucket configured")
+            raise ValueError(f"Repo {repo_model.org}/{repo_model.name} has no storage bucket.")
 
         credential_refresh_func = self._maybe_get_credential_refresh_func_for_icechunk(
             bucket=effective_bucket, org=repo_model.org, repo_name=repo_model.name
@@ -3129,7 +3133,7 @@ class Client:
         Resolves the URL of the org's enabled openEO compute service and returns
         an authenticated ``openeo.Connection``. Requires the optional ``openeo``
         dependency (``pip install 'arraylake[openeo]'``) and an enabled openEO
-        service (``Client().get_services(org).enable("openeo")``).
+        service (``Client().get_services(org).enable("openeo", sku_code="s")``).
 
         Args:
             org: Name of the org.
@@ -3146,9 +3150,11 @@ class Client:
         deployments = [d for d in self.get_services(org).list_enabled() if d.config.service_type == ServiceType.openeo]
         if deployment_name is not None:
             deployments = [d for d in deployments if (d.config.deployment_name or ServiceType.openeo.value) == deployment_name]
+        if not deployments and deployment_name is not None:
+            raise ValueError(f"Org {org!r} has no openEO service named {deployment_name!r}.")
         if not deployments:
             raise ValueError(
-                f"No openEO service is enabled for org {org!r}. Enable one with Client().get_services({org!r}).enable('openeo')."
+                f"Org {org!r} has no openEO service. Enable one with Client().get_services({org!r}).enable('openeo', sku_code='s')."
             )
 
         token = self.aclient.token

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/compute/http_client.py

@@ -11,6 +11,7 @@ from pydantic import TypeAdapter
 from arraylake.api_utils import ArraylakeHttpClient, handle_response
 from arraylake.compute.types import (
     ComputeConfig,
+    ComputeRegion,
     DeploymentInfo,
     LoadResults,
     ServiceConfig,
@@ -18,6 +19,7 @@ from arraylake.compute.types import (
 
 # type adapters
 LIST_SERVICES_ADAPTER = TypeAdapter(list[DeploymentInfo])
+LIST_REGIONS_ADAPTER = TypeAdapter(list[ComputeRegion])
 DUMP_CONFIGS_ADAPTER = TypeAdapter(dict[str, list[ServiceConfig]])
 
 
@@ -90,6 +92,12 @@ class ComputeHttpClient(ArraylakeHttpClient):
         handle_response(response)
         return response.json()
 
+    async def list_compute_regions(self) -> list[ComputeRegion]:
+        """Lists the regions where this organization can run compute services."""
+        response = await self._request("GET", f"/compute/orgs/{self._config.org}/regions")
+        handle_response(response)
+        return LIST_REGIONS_ADAPTER.validate_json(response.content)
+
     async def list_compute_services(self) -> list[DeploymentInfo]:
         """Lists all of the Arraylake compute services for a given organization."""
         response = await self._request("GET", f"/compute/orgs/{self._config.org}/services")
@@ -130,10 +138,13 @@ class ComputeHttpClient(ArraylakeHttpClient):
                 url = url + "?" + urlencode(params)
 
             async with await self._stream_request("GET", url) as response:
+                if response.is_error:
+                    await response.aread()
+                    handle_response(response)
                 async for line in response.aiter_lines():
                     yield line
         except Exception as e:
-            raise RuntimeError(f"Error getting logs for {service_id}: {e}") from e
+            raise RuntimeError(f"Could not read logs for {service_id}: {e}") from e
 
     async def update_compute_service(self, service_id: str, changes: dict[str, Any]) -> DeploymentInfo:
         """Changes a running Arraylake compute service's shape or scaling.

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/compute/services.py

@@ -20,7 +20,7 @@ from typing import Any
 
 from arraylake.asyn import sync
 from arraylake.compute.http_client import ComputeHttpClient, HttpComputeConfig
-from arraylake.compute.types import DeploymentInfo
+from arraylake.compute.types import ComputeRegion, DeploymentInfo
 
 _VALID_NAME = r"(\w[\w\.\-_]+)"
 
@@ -61,10 +61,18 @@ class AsyncComputeClient:
         """Get a new HTTP client for the organization."""
         return ComputeHttpClient(HttpComputeConfig(self.service_uri, self.org, self.token))
 
-    async def enable(self, protocol: str, is_public: bool = False, *, sku_code: str, **kwargs) -> dict[str, Any]:
+    async def enable(
+        self, protocol: str, is_public: bool = False, *, sku_code: str, region: str | None = None, **kwargs: Any
+    ) -> dict[str, Any]:
         """Enables a new Arraylake compute service."""
+        if region is not None:
+            kwargs["region"] = region
         return await self.http_client.create_compute_service(protocol, is_public, sku_code=sku_code, **kwargs)
 
+    async def regions(self) -> list[ComputeRegion]:
+        """Lists the regions where the organization can run compute services."""
+        return await self.http_client.list_compute_regions()
+
     async def list_enabled(self) -> list[DeploymentInfo]:
         """Lists all of the Arraylake compute services for the organization."""
         return await self.http_client.list_compute_services()
@@ -144,7 +152,7 @@ class ComputeClient:
 
         return wrap(*args, **kwargs)
 
-    def enable(self, protocol: str, is_public: bool = False, *, sku_code: str, **kwargs) -> dict[str, Any]:
+    def enable(self, protocol: str, is_public: bool = False, *, sku_code: str, region: str | None = None, **kwargs: Any) -> dict[str, Any]:
         """Enables a new Arraylake compute service.
 
         Args:
@@ -152,12 +160,23 @@ class ComputeClient:
             is_public: Whether the service should be public or not. Defaults to False.
             sku_code: The compute shape to run the service on. Every service runs on a priced
                 shape, so this is required.
+            region: The region to run the service in, such as "eu-west-1". `regions()` lists the
+                choices. Defaults to the environment's default region. The region cannot change
+                after creation.
             **kwargs: Additional arguments to pass to the service.
 
         Returns:
             Confirmation of the service being enabled.
         """
-        return self._synchronize(self._aclient.enable, protocol, is_public, sku_code=sku_code, **kwargs)
+        return self._synchronize(self._aclient.enable, protocol, is_public, sku_code=sku_code, region=region, **kwargs)
+
+    def regions(self) -> list[ComputeRegion]:
+        """Lists the regions where the organization can run compute services.
+
+        Returns:
+            The available regions. The one marked `default` is used when `enable` gets no region.
+        """
+        return self._synchronize(self._aclient.regions)
 
     def list_enabled(self) -> list[DeploymentInfo]:
         """Lists all of the Arraylake compute services for the organization.

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/compute/types.py

@@ -141,6 +141,9 @@ class ServiceConfig(BaseModel):
     memory_request: str | None = None
     memory_limit: str | None = None
 
+    # Fixed at creation. To move a service, disable it and enable it again in the other region.
+    region: str | None = None
+
     @field_validator("deployment_name")
     @classmethod
     def validate_deployment_name(cls, v: str | None) -> str | None:
@@ -170,6 +173,14 @@ class DeploymentInfo(BaseModel):
     # Pods serving right now. A service that scales to zero sits at 0 while still being available,
     # so this is what separates "ready to answer" from "the next request has to start it".
     ready_replicas: int = 0
+    # An older server sends no region.
+    region: str | None = None
+
+
+class ComputeRegion(BaseModel):
+    name: str
+    location: str | None = None
+    default: bool = False
 
 
 class LoadResults(BaseModel):
@@ -184,9 +195,8 @@ class ComputeConfig(RedactedReprModel):
     container_repository: str
     kube_config: dict
     legacy_domain: str | None = None
-    openmeter_api_key: str | None = None
 
-    __redacted_repr_fields__ = frozenset({"kube_config", "openmeter_api_key"})
+    __redacted_repr_fields__ = frozenset({"kube_config"})
 
 
 class LogMessage(BaseModel):

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/metastore/http_metastore.py

@@ -269,7 +269,7 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
         retain_data: bool = False,
     ) -> None:
         if not (imsure and imreallysure):
-            raise ValueError("Don't do this unless you're really sure. Once the database has been deleted, it's gone forever.")
+            raise ValueError("Pass imsure=True and imreallysure=True to confirm that you want to delete this repo.")
 
         # Default behavior is a soft-delete: the repo enters a recoverable
         # "ghost" state for the grace period and the daily cleanup job reclaims

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/repos/icechunk/virtual.py

@@ -25,9 +25,12 @@ FORBIDDEN_VIRTUAL_CHUNK_PREFIXES = {"file://", "memory://", "http://", "https://
 
 def _ensure_utc(expiration: datetime | None) -> datetime | None:
     """icechunk's Rust layer rejects a tz-naive expiration; treat naive as UTC."""
-    if expiration is not None and expiration.tzinfo is None:
+    if expiration is None:
+        return None
+    if expiration.tzinfo is None:
         return expiration.replace(tzinfo=UTC)
-    return expiration
+    # icechunk also rejects botocore's dateutil tzutc() on fresh STS expirations.
+    return expiration.astimezone(UTC)
 
 
 def get_icechunk_container_credentials(
@@ -174,10 +177,7 @@ def forbid_unsafe_virtual_bucket_configs(bucket: BucketResponse, bucket_nickname
 
     # exclude HMAC buckets specifically as these will never be safe
     if isinstance(auth_method, HmacAuth):
-        raise ValueError(
-            "Cannot use virtual chunk references that refer to a bucket which uses HMAC credentials, "
-            f"But bucket {bucket_nickname} uses HMAC credentials."
-        )
+        raise ValueError(f"Bucket {bucket_nickname} uses HMAC credentials, so virtual chunk references cannot point to it.")
 
 
 def forbid_unsafe_virtual_chunk_containers(

{arraylake-1.5.0 → arraylake-1.7.0}/arraylake/tuning.py

@@ -241,8 +241,8 @@ async def tune_org(org: str, bucket_config_nickname: str | None = None) -> Optim
         results = await tune_repo(repo)
         optimal_config = OptimalIOConfig(read=results.best_read(), write=results.best_write())
     finally:
-        with CONSOLE.status("Cleaning up temporary repo"):
-            client.delete_repo(repo_name, imsure=True, imreallysure=True)
+        with CONSOLE.status(f"Deleting temporary repo [yellow]{repo_name}"):
+            client.delete_repo(repo_name, imsure=True, imreallysure=True, immediate=True)
     CONSOLE.print("Optimal I/O configuration found:")
     CONSOLE.print(optimal_config)
     return optimal_config