arraylake 1.1.1__tar.gz → 1.2.0__tar.gz

{arraylake-1.1.1 → arraylake-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arraylake
-Version: 1.1.1
+Version: 1.2.0
 Summary: Python client for ArrayLake
 Project-URL: Documentation, https://docs.earthmover.io/
 Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -29,6 +29,8 @@ Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
 Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
 Provides-Extra: obstore
 Requires-Dist: obstore>=0.9.2; extra == 'obstore'
+Provides-Extra: openeo
+Requires-Dist: openeo<0.40,>=0.35.0; extra == 'openeo'
 Provides-Extra: xarray
 Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
 Requires-Dist: xarray>=2024.10.0; extra == 'xarray'

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.1.1'
-__version_tuple__ = version_tuple = (1, 1, 1)
+__version__ = version = '1.2.0'
+__version_tuple__ = version_tuple = (1, 2, 0)
 
 __commit_id__ = commit_id = None

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/api_utils.py

@@ -542,17 +542,21 @@ class UserAuth(httpx.Auth):
         return f"Bearer {token}"
 
     async def async_auth_flow(self, request: httpx.Request) -> AsyncGenerator[httpx.Request, httpx.Response]:
-        request.headers["Authorization"] = self._bearer_token
+        token_at_send = self._bearer_token
+        request.headers["Authorization"] = token_at_send
         response = yield request
         if response.status_code == httpx.codes.UNAUTHORIZED:
             # If the server issues a 401 response, then issue a request to
             # refresh tokens, and resend the request.
             async with self._async_lock:
-                refresh_response = yield self._token_handler.refresh_request
-                await refresh_response.aread()
-                handle_response(refresh_response)
-                new_tokens = OauthTokensResponse.model_validate_json(refresh_response.content)
-                self._token_handler.update(new_tokens)
+                # A concurrently-401'd request may have refreshed while we waited for
+                # the lock; refreshing again would burn a redundant Auth0 rotation.
+                if self._bearer_token == token_at_send:
+                    refresh_response = yield await self._token_handler.build_refresh_request()
+                    await refresh_response.aread()
+                    handle_response(refresh_response)
+                    new_tokens = OauthTokensResponse.model_validate_json(refresh_response.content)
+                    self._token_handler.update(new_tokens)
 
             request.headers["Authorization"] = self._bearer_token
             yield request

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/cli/repo.py

@@ -239,9 +239,14 @@ async def modify(
 async def delete(
     repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
     confirm: bool = typer.Option(False, help="confirm deletion without prompting"),
+    immediate: bool = typer.Option(False, help="skip the recovery grace period and move straight to cleanup (cannot be restored)"),
+    retain_data: bool = typer.Option(False, help="leave the bucket bytes in place; only remove Arraylake metadata"),
 ):
     """**Delete** a repository
 
+    By default the repo enters a recoverable state for a grace period before its
+    data is cleaned up. Pass --immediate to skip the grace period.
+
     **Examples**
 
     - Delete repository without confirmation prompt
@@ -249,6 +254,12 @@ async def delete(
         ```
         $ arraylake repo delete my-org/example-repo --confirm
         ```
+
+    - Delete immediately, freeing the name right away
+
+        ```
+        $ arraylake repo delete my-org/example-repo --confirm --immediate
+        ```
     """
     if not confirm:
         confirm = typer.confirm(
@@ -259,7 +270,7 @@ async def delete(
     client = AsyncClient()
 
     with simple_progress(f"Deleting repo [bold]{repo_name}[/bold]..."):
-        await client.delete_repo(repo_name, imsure=confirm, imreallysure=confirm)
+        await client.delete_repo(repo_name, imsure=confirm, imreallysure=confirm, immediate=immediate, retain_data=retain_data)
 
     # If the repo is a icechunk repo, print message that the bucket must be deleted manually
     rich_console.print(f"Repo [bold]{repo_name}[/bold] removed from Arraylake. \nThe underlying Icechunk bucket must be deleted manually.")

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/cli/skills.py

@@ -18,6 +18,7 @@ required.
 
 from __future__ import annotations
 
+import asyncio
 import shutil
 import tempfile
 from dataclasses import dataclass
@@ -81,14 +82,18 @@ def _parse_manifest(payload: dict) -> list[SkillEntry]:
     return entries
 
 
-def _fetch_manifest(uri: str) -> list[SkillEntry]:
+async def _async_fetch_manifest(uri: str) -> list[SkillEntry]:
     url = f"{uri}/manifest.json"
-    with httpx.Client(timeout=20.0, follow_redirects=True) as client:
-        resp = client.get(url)
+    async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+        resp = await client.get(url)
         resp.raise_for_status()
         return _parse_manifest(resp.json())
 
 
+def _fetch_manifest(uri: str) -> list[SkillEntry]:
+    return asyncio.run(_async_fetch_manifest(uri))
+
+
 def _empty_temporary_sibling(dest: Path, prefix: str) -> Path:
     path = Path(tempfile.mkdtemp(prefix=prefix, dir=dest))
     path.rmdir()
@@ -102,6 +107,19 @@ def _remove_path(path: Path) -> None:
         shutil.rmtree(path)
 
 
+async def _download_skill_files(uri: str, entry: SkillEntry, tmp_root: Path) -> None:
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        for rel in entry.files:
+            if ".." in rel.split("/") or rel.startswith("/"):
+                raise RuntimeError(f"Refusing to write file outside skill root: {rel}")
+            file_url = f"{uri}/{entry.name}/{rel}"
+            resp = await client.get(file_url)
+            resp.raise_for_status()
+            target = tmp_root / rel
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(resp.content)
+
+
 def _install_skill(uri: str, entry: SkillEntry, dest: Path) -> str:
     """Download all of `entry.files` into `dest/<name>/...`.
 
@@ -115,16 +133,7 @@ def _install_skill(uri: str, entry: SkillEntry, dest: Path) -> str:
     backup_root: Path | None = None
 
     try:
-        with httpx.Client(timeout=30.0, follow_redirects=True) as client:
-            for rel in entry.files:
-                if ".." in rel.split("/") or rel.startswith("/"):
-                    raise RuntimeError(f"Refusing to write file outside skill root: {rel}")
-                file_url = f"{uri}/{entry.name}/{rel}"
-                resp = client.get(file_url)
-                resp.raise_for_status()
-                target = tmp_root / rel
-                target.parent.mkdir(parents=True, exist_ok=True)
-                target.write_bytes(resp.content)
+        asyncio.run(_download_skill_files(uri, entry, tmp_root))
 
         if skill_root.exists() or skill_root.is_symlink():
             # Pre-existing install, including legacy wheel-bundled symlinks.

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/client.py

@@ -29,6 +29,7 @@ from icechunk import Repository as IcechunkRepository
 from arraylake._credential_cache import CredentialCacheKey, get_or_refresh, populate
 from arraylake.asyn import async_gather_tasks, asyncio_run, sync
 from arraylake.compute.services import AsyncComputeClient, ComputeClient
+from arraylake.compute.types import ServiceType
 from arraylake.config import config as arraylake_config
 from arraylake.config import default_service_uri
 from arraylake.credentials import (
@@ -90,6 +91,7 @@ from arraylake.types import Repo as RepoModel
 
 if TYPE_CHECKING:
     import obstore as obs
+    import openeo
 
 logger = get_logger(__name__)
 
@@ -1061,7 +1063,7 @@ class AsyncClient:
 
         except (IcechunkError, ValueError):
             # If the repo fails to create, we need to delete the repo model
-            await mstore.delete_database(repo_name, imsure=True, imreallysure=True)
+            await mstore.delete_database(repo_name, imsure=True, imreallysure=True, immediate=True, retain_data=True)
             raise
 
     async def import_repo(
@@ -1160,7 +1162,7 @@ class AsyncClient:
 
         except (IcechunkError, ValueError):
             # If the repo fails to create, we need to delete the repo model
-            await mstore.delete_database(repo_name, imsure=True, imreallysure=True)
+            await mstore.delete_database(repo_name, imsure=True, imreallysure=True, immediate=True, retain_data=True)
             raise
 
     async def _get_icechunk_storage_from_repo_model(
@@ -1466,16 +1468,38 @@ class AsyncClient:
             optimization_config=optimization_config,
         )
 
-    async def delete_repo(self, name: OrgAndRepoName, *, imsure: bool = False, imreallysure: bool = False) -> None:
-        """Delete a repo
+    async def delete_repo(
+        self,
+        name: OrgAndRepoName,
+        *,
+        imsure: bool = False,
+        imreallysure: bool = False,
+        immediate: bool = False,
+        retain_data: bool = False,
+    ) -> None:
+        """Delete a repo.
+
+        By default the repo enters a recoverable "ghost" state for a grace
+        period (currently 7 days) and the bucket bytes are reclaimed by the
+        daily cleanup job after that. A repo in the ghost state can be brought
+        back through the web app.
 
         Args:
             name: Full name of the repo to delete (of the form [ORG]/[REPO])
-            imsure, imreallysure: confirm you intend to delete this bucket config
+            imsure, imreallysure: confirm you intend to delete this repo
+            immediate: skip the grace period and move straight to cleanup
+            retain_data: leave the bucket bytes alone; only remove Arraylake
+                metadata
         """
         org, repo_name = _parse_org_and_repo(name)
         mstore = self._metastore_for_org(org)
-        await mstore.delete_database(repo_name, imsure=imsure, imreallysure=imreallysure)
+        await mstore.delete_database(
+            repo_name,
+            imsure=imsure,
+            imreallysure=imreallysure,
+            immediate=immediate,
+            retain_data=retain_data,
+        )
 
     async def _set_repo_status(
         self, qualified_repo_name: OrgAndRepoName, mode: RepoOperationMode, message: str | None = None
@@ -2239,14 +2263,38 @@ class Client:
             optimization_config=optimization_config,
         )
 
-    def delete_repo(self, name: OrgAndRepoName, *, imsure: bool = False, imreallysure: bool = False) -> None:
-        """Delete a repo
+    def delete_repo(
+        self,
+        name: OrgAndRepoName,
+        *,
+        imsure: bool = False,
+        imreallysure: bool = False,
+        immediate: bool = False,
+        retain_data: bool = False,
+    ) -> None:
+        """Delete a repo.
+
+        By default the repo enters a recoverable "ghost" state for a grace
+        period (currently 7 days) and the bucket bytes are reclaimed by the
+        daily cleanup job after that. A repo in the ghost state can be brought
+        back through the web app.
 
         Args:
             name: Full name of the repo to delete (of the form [ORG]/[REPO])
+            imsure, imreallysure: confirm you intend to delete this repo
+            immediate: skip the grace period and move straight to cleanup
+            retain_data: leave the bucket bytes alone; only remove Arraylake
+                metadata
         """
 
-        return sync(self.aclient.delete_repo, name, imsure=imsure, imreallysure=imreallysure)
+        return sync(
+            self.aclient.delete_repo,
+            name,
+            imsure=imsure,
+            imreallysure=imreallysure,
+            immediate=immediate,
+            retain_data=retain_data,
+        )
 
     def create_bucket_config(
         self,
@@ -2435,3 +2483,46 @@ class Client:
             org: Name of the org
         """
         return self.aclient.get_services(org).to_sync_client()
+
+    def get_openeo_connection(self, org: OrgName, deployment_name: str | None = None) -> openeo.Connection:
+        """Connect to this org's deployed openEO service with the openeo client.
+
+        Resolves the URL of the org's enabled openEO compute service and returns
+        an authenticated ``openeo.Connection``. Requires the optional ``openeo``
+        dependency (``pip install 'arraylake[openeo]'``) and an enabled openEO
+        service (``Client().get_services(org).enable("openeo")``).
+
+        Args:
+            org: Name of the org.
+            deployment_name: Select a specific openEO deployment by name when the
+                org has more than one. Defaults to the only/first openEO service.
+        """
+        try:
+            import openeo
+        except ImportError as e:
+            raise ImportError(
+                "The 'openeo' package is required for get_openeo_connection. Install it with `pip install 'arraylake[openeo]'`."
+            ) from e
+
+        deployments = [d for d in self.get_services(org).list_enabled() if d.config.service_type == ServiceType.openeo]
+        if deployment_name is not None:
+            deployments = [d for d in deployments if (d.config.deployment_name or ServiceType.openeo.value) == deployment_name]
+        if not deployments:
+            raise ValueError(
+                f"No openEO service is enabled for org {org!r}. Enable one with Client().get_services({org!r}).enable('openeo')."
+            )
+
+        token = self.aclient.token
+        if not token:
+            handler = get_auth_handler(self.aclient.service_uri)
+            if handler.tokens is not None:
+                token = handler.tokens.id_token.get_secret_value()
+        if not token:
+            raise ValueError(
+                "An Arraylake API token is required to authenticate to openEO. "
+                "Provide one via Client(token=...) or log in with `arraylake auth login`."
+            )
+
+        connection = openeo.connect(deployments[0].url)
+        connection.authenticate_bearer_token(token)
+        return connection

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/compute/types.py

@@ -11,6 +11,7 @@ class ServiceType(StrEnum):
     wms = "wms"
     tiles = "tiles"
     zarr = "zarr"
+    openeo = "openeo"
 
 
 class ServiceStatus(StrEnum):
@@ -28,6 +29,23 @@ class LogLevel(StrEnum):
     critical = "CRITICAL"
 
 
+class InstanceSize(StrEnum):
+    xsmall = "xsmall"
+    small = "small"
+    medium = "medium"
+    large = "large"
+
+
+# Mapping of instance sizes to resource specifications
+# Based on AWS c7g instance family
+INSTANCE_SIZE_SPECS: dict[InstanceSize, dict[str, int]] = {
+    InstanceSize.xsmall: {"vcpus": 1, "memory_gib": 2},  # c7g.medium
+    InstanceSize.small: {"vcpus": 4, "memory_gib": 8},  # c7g.xlarge
+    InstanceSize.medium: {"vcpus": 16, "memory_gib": 32},  # c7g.4xlarge
+    InstanceSize.large: {"vcpus": 64, "memory_gib": 128},  # c7g.16xlarge
+}
+
+
 class ServiceConfig(BaseModel):
     service_type: ServiceType
     org: str
@@ -83,6 +101,10 @@ class ServiceConfig(BaseModel):
     # other than DAP2. DAP2 always uses Dask regardless of this setting.
     use_dask: bool = False
 
+    # Optional instance size selection. When set, this maps to predefined
+    # cpu_request, cpu_limit, memory_request, and memory_limit values.
+    instance_size: InstanceSize | None = None
+
     # Optional container resource overrides. When unset, the per-service-type
     # defaults from the server's ImageConfig are used. Setting any of these
     # requires the caller to have the CAN_MANAGE_ALL_ORGS service permission;

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/diagnostics.py

@@ -17,6 +17,7 @@
 #   limitations under the License.
 """Utility functions for printing version information."""
 
+import asyncio
 import importlib
 import locale
 import os
@@ -107,12 +108,15 @@ def get_versions() -> dict[str, str]:
     return versions
 
 
-def get_service_info(service_uri: str) -> dict[str, str]:
-    with httpx.Client() as client:
-        response = client.get(service_uri)
+async def _async_get_service_info(service_uri: str) -> dict[str, str]:
+    async with httpx.AsyncClient() as client:
+        response = await client.get(service_uri)
     data = response.json()
-    info = {"service_uri": service_uri, "service_version": data["arraylake_api_version"]}
-    return info
+    return {"service_uri": service_uri, "service_version": data["arraylake_api_version"]}
+
+
+def get_service_info(service_uri: str) -> dict[str, str]:
+    return asyncio.run(_async_get_service_info(service_uri))
 
 
 def get_config_info() -> dict[str, str]:

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/metastore/abc.py

@@ -32,14 +32,30 @@ class Metastore(ABC):  # pragma: no cover
         ...
 
     @abstractmethod
-    async def delete_database(self, name: str, *, imsure: bool = False, imreallysure: bool = False) -> None:
+    async def delete_database(
+        self,
+        name: str,
+        *,
+        imsure: bool = False,
+        imreallysure: bool = False,
+        immediate: bool = False,
+        retain_data: bool = False,
+    ) -> None:
         """Delete an existing metastore database.
 
         Parameters
         ----------
         name : str
             Name of repo
-        imsure, imreallsure : bool
-            Confirm permanent deletion.
+        imsure, imreallysure : bool
+            Confirm intent to delete.
+        immediate : bool, default False
+            Skip the soft-delete grace period and move straight to cleanup.
+            With the default ``immediate=False`` the repo enters a recoverable
+            "ghost" state for the grace period before the daily cleanup job
+            takes it.
+        retain_data : bool, default False
+            Leave the bucket bytes alone when the cleanup runs; only the
+            Arraylake metadata is removed.
         """
         ...

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/metastore/http_metastore.py

@@ -234,11 +234,37 @@ class HttpMetastore(ArraylakeHttpClient, Metastore):
         response = await self._request("PATCH", f"/orgs/{self._config.org}/{name}", content=repo_modify_request.model_dump_json())
         handle_response(response)
 
-    async def delete_database(self, name: str, *, imsure: bool = False, imreallysure: bool = False) -> None:
+    async def delete_database(
+        self,
+        name: str,
+        *,
+        imsure: bool = False,
+        imreallysure: bool = False,
+        immediate: bool = False,
+        retain_data: bool = False,
+    ) -> None:
         if not (imsure and imreallysure):
             raise ValueError("Don't do this unless you're really sure. Once the database has been deleted, it's gone forever.")
 
-        response = await self._request("DELETE", f"/orgs/{self._config.org}/{name}")
+        # Default behavior is a soft-delete: the repo enters a recoverable
+        # "ghost" state for the grace period and the daily cleanup job reclaims
+        # the bucket bytes afterward. Callers can opt out:
+        #   immediate=True   - skip the grace period; ghost vanishes immediately
+        #                      and the subscription cascade runs now
+        #   retain_data=True - leave bucket bytes alone when the cleanup job
+        #                      finalizes (only the Arraylake metadata is removed)
+        # Setting both together makes the server drop the row inline so the
+        # (org, name) slot is immediately reusable.
+        params: dict[str, str] = {}
+        if immediate:
+            params["immediate"] = "true"
+        if retain_data:
+            params["retain_data"] = "true"
+        response = await self._request(
+            "DELETE",
+            f"/orgs/{self._config.org}/{name}",
+            params=params,
+        )
         handle_response(response)
 
     async def create_bucket_config(self, bucket_config: NewBucket) -> BucketResponse:

{arraylake-1.1.1 → arraylake-1.2.0}/arraylake/repos/icechunk/types.py

@@ -98,6 +98,25 @@ class CreateBranchRequest(BaseModel):
     commit_id: CommitId
 
 
+class AttributeEdit(BaseModel):
+    """A merge patch for one node's attributes (path "" targets the root group)."""
+
+    path: str = ""
+    attributes: dict[str, Any]
+
+
+class UpdateNodeAttributesRequest(BaseModel):
+    """Merge attributes into one or more nodes (groups or arrays), applying every
+    edit to a branch as a single commit."""
+
+    branch: BranchName
+    edits: list[AttributeEdit]
+    message: str | None = None
+    # Optimistic concurrency: the commit the caller expects the new commit to be
+    # parented on (i.e. the current branch tip). Reject if the branch has moved.
+    expected_parent_commit_id: CommitId | None = None
+
+
 class Tag(BaseModel):
     id: TagName
     commit_id: CommitId