arraylake 1.6.0__tar.gz → 1.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-1.6.0 → arraylake-1.8.0}/PKG-INFO +6 -5
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/_version.py +2 -2
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/api_utils.py +20 -6
- arraylake-1.8.0/arraylake/cli/auth.py +195 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/compute.py +20 -5
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/config.py +3 -3
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/iceberg.py +16 -4
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/repo.py +96 -5
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/client.py +291 -33
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/compute/http_client.py +12 -1
- arraylake-1.8.0/arraylake/compute/openeo_connection.py +47 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/compute/services.py +23 -4
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/compute/types.py +12 -2
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/config.py +8 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/metastore/http_metastore.py +14 -2
- arraylake-1.8.0/arraylake/repos/iceberg/auth.py +19 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/icechunk/virtual.py +6 -6
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/token.py +103 -3
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/tuning.py +2 -2
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/types.py +40 -7
- {arraylake-1.6.0 → arraylake-1.8.0}/pyproject.toml +14 -28
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/__init__.py +8 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/compute/test_client.py +43 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/icechunk/test_virtual.py +20 -4
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_api_utils.py +55 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_cli.py +175 -8
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_client.py +83 -7
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_http_metastore.py +5 -5
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_iceberg_client.py +10 -7
- arraylake-1.8.0/tests/test_openeo_connection.py +110 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_subscriptions.py +4 -46
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_token_handler.py +90 -2
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_types.py +16 -2
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_virtual_chunks.py +24 -21
- {arraylake-1.6.0 → arraylake-1.8.0}/uv.lock +719 -488
- arraylake-1.6.0/arraylake/cli/auth.py +0 -134
- arraylake-1.6.0/arraylake/repos/iceberg/auth.py +0 -54
- {arraylake-1.6.0 → arraylake-1.8.0}/.gitignore +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/README.md +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/__main__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/_credential_cache.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/asyn.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/main.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/org.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/skills.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/cli/utils.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/config.yaml +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/credentials.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/diagnostics.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/display/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/display/repo.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/display/repolist.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/exceptions.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/log_util.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/metastore/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/metastore/abc.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/obstore.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/py.typed +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/iceberg/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/iceberg/catalog.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/icechunk/storage.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/arraylake/repos/icechunk/types.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/compute/test_cli_table.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/config.yaml +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/conftest.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/icechunk/__init__.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/icechunk/conftest.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/icechunk/test_storage.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_asyn.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_config.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_credential_cache.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_credentials.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_diagnostics.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_obstore.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_performance.py +0 -0
- {arraylake-1.6.0 → arraylake-1.8.0}/tests/test_skills.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: arraylake
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.0
|
|
4
4
|
Summary: Python client for ArrayLake
|
|
5
5
|
Project-URL: Documentation, https://docs.earthmover.io/
|
|
6
6
|
Project-URL: Changelog, https://docs.earthmover.io/changelog
|
|
@@ -15,10 +15,10 @@ Requires-Dist: click<9.0,>=8.1.3
|
|
|
15
15
|
Requires-Dist: donfig<1.0,>=0.7
|
|
16
16
|
Requires-Dist: httpx<0.29,>=0.23
|
|
17
17
|
Requires-Dist: icechunk<3.0.0,>=2.2.2
|
|
18
|
-
Requires-Dist: numpy<3.0,>=1.23
|
|
19
18
|
Requires-Dist: packaging<27.0,>=23.0
|
|
20
19
|
Requires-Dist: pydantic[email]<3.0,>=2.9
|
|
21
20
|
Requires-Dist: python-dateutil<3.0,>=2.8
|
|
21
|
+
Requires-Dist: pyyaml<7.0,>=6.0.1
|
|
22
22
|
Requires-Dist: rich<16.0,>=12.6
|
|
23
23
|
Requires-Dist: ruamel-yaml<1.0,>=0.17
|
|
24
24
|
Requires-Dist: structlog<27.0,>=24.1
|
|
@@ -27,13 +27,14 @@ Requires-Dist: zarr<4.0,>=3.1.0
|
|
|
27
27
|
Provides-Extra: iceberg
|
|
28
28
|
Requires-Dist: pyiceberg[pyarrow]<0.13,>=0.11; extra == 'iceberg'
|
|
29
29
|
Provides-Extra: maximal
|
|
30
|
-
Requires-Dist: xarray[accel,etc,io,parallel]>=
|
|
30
|
+
Requires-Dist: xarray[accel,etc,io,parallel]>=2025.6.0; extra == 'maximal'
|
|
31
31
|
Provides-Extra: obstore
|
|
32
32
|
Requires-Dist: obstore>=0.9.2; extra == 'obstore'
|
|
33
33
|
Provides-Extra: openeo
|
|
34
|
-
Requires-Dist: openeo<0.
|
|
34
|
+
Requires-Dist: openeo<0.54,>=0.35.0; extra == 'openeo'
|
|
35
|
+
Requires-Dist: xarray>=2025.6.0; extra == 'openeo'
|
|
35
36
|
Provides-Extra: xarray
|
|
36
|
-
Requires-Dist: xarray>=
|
|
37
|
+
Requires-Dist: xarray>=2025.6.0; extra == 'xarray'
|
|
37
38
|
Description-Content-Type: text/markdown
|
|
38
39
|
|
|
39
40
|
<p align="center">
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1,
|
|
21
|
+
__version__ = version = '1.8.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 8, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -30,7 +30,7 @@ from arraylake.exceptions import (
|
|
|
30
30
|
ArraylakeValidationError,
|
|
31
31
|
)
|
|
32
32
|
from arraylake.log_util import get_logger
|
|
33
|
-
from arraylake.types import ApiTokenInfo,
|
|
33
|
+
from arraylake.types import ApiTokenInfo, UserInfo
|
|
34
34
|
|
|
35
35
|
logger = get_logger(__name__)
|
|
36
36
|
|
|
@@ -484,6 +484,9 @@ def _exception_log_debug(request: httpx.Request, response: httpx.Response):
|
|
|
484
484
|
)
|
|
485
485
|
|
|
486
486
|
|
|
487
|
+
# Request extension set by our auth flows on a 401, describing the credential that was rejected.
|
|
488
|
+
_CREDENTIAL_EXTENSION = "arraylake_credential"
|
|
489
|
+
|
|
487
490
|
_emitted_deprecation_warnings: set[str] = set()
|
|
488
491
|
_emitted_user_warnings: set[str] = set()
|
|
489
492
|
|
|
@@ -526,6 +529,8 @@ def handle_response(response: httpx.Response):
|
|
|
526
529
|
else:
|
|
527
530
|
# Handle other HTTP status errors
|
|
528
531
|
error_msg = f"Error response {exc.response.status_code} while requesting {exc.request.url!r}. {response}: {response.text}"
|
|
532
|
+
if credential := exc.request.extensions.get(_CREDENTIAL_EXTENSION):
|
|
533
|
+
error_msg = f"{error_msg} (Authenticated {credential}.)"
|
|
529
534
|
if 400 <= exc.response.status_code < 500:
|
|
530
535
|
raise ArraylakeClientError(error_msg, request_id=request_id, status_code=exc.response.status_code) from exc
|
|
531
536
|
else:
|
|
@@ -550,7 +555,12 @@ class TokenAuth(httpx.Auth):
|
|
|
550
555
|
def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
|
|
551
556
|
# Send the request, with a bearer token header
|
|
552
557
|
request.headers["Authorization"] = f"Bearer {self.token}"
|
|
553
|
-
yield request
|
|
558
|
+
response = yield request
|
|
559
|
+
if response.status_code == httpx.codes.UNAUTHORIZED:
|
|
560
|
+
# Import locally to avoid circular imports
|
|
561
|
+
from arraylake.token import describe_token_source
|
|
562
|
+
|
|
563
|
+
request.extensions[_CREDENTIAL_EXTENSION] = describe_token_source(self.token)
|
|
554
564
|
|
|
555
565
|
|
|
556
566
|
class UserAuth(httpx.Auth):
|
|
@@ -579,11 +589,14 @@ class UserAuth(httpx.Auth):
|
|
|
579
589
|
# Import locally to avoid circular imports
|
|
580
590
|
from arraylake.token import AuthException
|
|
581
591
|
|
|
582
|
-
raise AuthException("Not logged in, please log in with `
|
|
592
|
+
raise AuthException("Not logged in, please log in with `al auth login`")
|
|
583
593
|
token = self._token_handler.tokens.id_token.get_secret_value()
|
|
584
594
|
return f"Bearer {token}"
|
|
585
595
|
|
|
586
596
|
async def async_auth_flow(self, request: httpx.Request) -> AsyncGenerator[httpx.Request, httpx.Response]:
|
|
597
|
+
# Import locally to avoid circular imports
|
|
598
|
+
from arraylake.token import describe_cached_login, parse_refreshed_tokens
|
|
599
|
+
|
|
587
600
|
token_at_send = self._bearer_token
|
|
588
601
|
request.headers["Authorization"] = token_at_send
|
|
589
602
|
response = yield request
|
|
@@ -597,11 +610,12 @@ class UserAuth(httpx.Auth):
|
|
|
597
610
|
refresh_response = yield await self._token_handler.build_refresh_request()
|
|
598
611
|
await refresh_response.aread()
|
|
599
612
|
handle_response(refresh_response)
|
|
600
|
-
|
|
601
|
-
self._token_handler.update(new_tokens)
|
|
613
|
+
self._token_handler.update(parse_refreshed_tokens(refresh_response.content))
|
|
602
614
|
|
|
603
615
|
request.headers["Authorization"] = self._bearer_token
|
|
604
|
-
yield request
|
|
616
|
+
response = yield request
|
|
617
|
+
if response.status_code == httpx.codes.UNAUTHORIZED:
|
|
618
|
+
request.extensions[_CREDENTIAL_EXTENSION] = describe_cached_login(self.api_endpoint)
|
|
605
619
|
|
|
606
620
|
def sync_auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
|
|
607
621
|
# we can implement this method if we end up needing the sync httpx.Client
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
import typer
|
|
5
|
+
from rich.align import Align
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
from rich.markup import escape
|
|
8
|
+
from rich.panel import Panel
|
|
9
|
+
|
|
10
|
+
from arraylake.cli.utils import (
|
|
11
|
+
coro,
|
|
12
|
+
error_console,
|
|
13
|
+
print_logo,
|
|
14
|
+
print_logo_mini,
|
|
15
|
+
rich_console,
|
|
16
|
+
)
|
|
17
|
+
from arraylake.config import default_service_uri, default_token
|
|
18
|
+
from arraylake.token import AuthException, TokenHandler, describe_token, get_auth_handler
|
|
19
|
+
|
|
20
|
+
auth = typer.Typer(help="Manage Arraylake authentication")
|
|
21
|
+
|
|
22
|
+
NOT_LOGGED_IN_MESSAGE = "⚠️ Not logged in, please log in with: `al auth login`"
|
|
23
|
+
|
|
24
|
+
# `al auth token` refreshes tokens that expire sooner than this, so the printed token stays usable for a while.
|
|
25
|
+
TOKEN_MIN_VALID_SECONDS = 300
|
|
26
|
+
|
|
27
|
+
# Uncolored notices on stderr, keeping stdout machine-readable.
|
|
28
|
+
notice_console = Console(stderr=True)
|
|
29
|
+
|
|
30
|
+
SERVICE_URI_OPTION = typer.Option(
|
|
31
|
+
None,
|
|
32
|
+
"--service-uri",
|
|
33
|
+
help=(
|
|
34
|
+
"Arraylake API endpoint to act on, e.g. **https://dev.api.earthmover.io**. Defaults to the configured "
|
|
35
|
+
"**service.uri**. Tokens are stored per endpoint, so this need not be persisted to config — unless "
|
|
36
|
+
"**service.token_path** is set, which makes all endpoints share one token file."
|
|
37
|
+
),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _resolve_service_uri(service_uri: str | None) -> str:
|
|
42
|
+
return service_uri if service_uri else default_service_uri()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@auth.command()
|
|
46
|
+
@coro # type: ignore
|
|
47
|
+
async def login(
|
|
48
|
+
browser: bool = typer.Option(True, "--browser/--no-browser", " /--nobrowser", help="Whether to automatically open a browser window."),
|
|
49
|
+
service_uri: str | None = SERVICE_URI_OPTION,
|
|
50
|
+
):
|
|
51
|
+
"""**Log in** to Arraylake
|
|
52
|
+
|
|
53
|
+
This will automatically open a browser window. If **--no-browser** is specified, a link will be printed.
|
|
54
|
+
|
|
55
|
+
**Examples**
|
|
56
|
+
|
|
57
|
+
- Log in without automatically opening a browser window
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
$ al auth login --no-browser
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
- Log in to a non-default Arraylake deployment
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
$ al auth login --service-uri https://dev.api.earthmover.io
|
|
67
|
+
```
|
|
68
|
+
"""
|
|
69
|
+
print_logo()
|
|
70
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
71
|
+
await handler.login(browser=browser)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@auth.command()
|
|
75
|
+
@coro # type: ignore
|
|
76
|
+
async def refresh(service_uri: str | None = SERVICE_URI_OPTION) -> None:
|
|
77
|
+
"""**Refresh** Arraylake's auth token"""
|
|
78
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
79
|
+
if handler.tokens is not None:
|
|
80
|
+
try:
|
|
81
|
+
await handler.refresh_token()
|
|
82
|
+
except Exception as e:
|
|
83
|
+
error_console.print(f"\n[bold red]❌ {e}[/bold red]")
|
|
84
|
+
raise typer.Exit(code=1)
|
|
85
|
+
else:
|
|
86
|
+
rich_console.print(Align(f"[yellow]{NOT_LOGGED_IN_MESSAGE}[/yellow]", align="center"))
|
|
87
|
+
raise typer.Exit(code=1)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@auth.command()
|
|
91
|
+
@coro # type: ignore
|
|
92
|
+
async def logout(service_uri: str | None = SERVICE_URI_OPTION) -> None:
|
|
93
|
+
"""**Logout** of Arraylake
|
|
94
|
+
|
|
95
|
+
**Examples**
|
|
96
|
+
|
|
97
|
+
- Logout of arraylake
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
$ al auth logout
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
- Log out of a non-default Arraylake deployment
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
$ al auth logout --service-uri https://dev.api.earthmover.io
|
|
107
|
+
```
|
|
108
|
+
"""
|
|
109
|
+
try:
|
|
110
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
111
|
+
await handler.logout()
|
|
112
|
+
except AuthException as e:
|
|
113
|
+
error_console.print(e)
|
|
114
|
+
raise typer.Exit(code=1)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _configured_token_name() -> str:
|
|
118
|
+
return "ARRAYLAKE_TOKEN" if os.environ.get("ARRAYLAKE_TOKEN", "").strip() else "The `token` arraylake config setting"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
async def _check_configured_token(handler: TokenHandler, token: str) -> bool:
|
|
122
|
+
"""Report on a configured token, which clients use instead of the cached login. Returns whether it is valid."""
|
|
123
|
+
name = _configured_token_name()
|
|
124
|
+
rich_console.print(
|
|
125
|
+
Align(f"[yellow]{name} is set ({describe_token(token)}) and takes precedence over your cached login.[/yellow]", align="center")
|
|
126
|
+
)
|
|
127
|
+
try:
|
|
128
|
+
async with handler._create_async_client() as client:
|
|
129
|
+
response = await client.get(
|
|
130
|
+
f"{handler.api_endpoint}/user", headers={"Authorization": f"Bearer {token}", **handler.arraylake_headers}
|
|
131
|
+
)
|
|
132
|
+
except httpx.HTTPError as e:
|
|
133
|
+
rich_console.print(Align(f"[yellow]⚠️ Could not verify {name}: {escape(str(e))}[/yellow]", align="center"))
|
|
134
|
+
return False
|
|
135
|
+
if response.status_code != httpx.codes.OK:
|
|
136
|
+
hint = "`unset ARRAYLAKE_TOKEN`" if name == "ARRAYLAKE_TOKEN" else "remove it"
|
|
137
|
+
rich_console.print(
|
|
138
|
+
Align(f"[red]❌ {name} was rejected ({response.status_code}); {hint} to use your cached login.[/red]", align="center")
|
|
139
|
+
)
|
|
140
|
+
return False
|
|
141
|
+
rich_console.print(Align(f"[green][bold]🔓 {name} is valid ({response.json().get('email', 'unknown')})[/bold][/green]", align="center"))
|
|
142
|
+
return True
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@auth.command()
|
|
146
|
+
@coro # type: ignore
|
|
147
|
+
async def status(service_uri: str | None = SERVICE_URI_OPTION):
|
|
148
|
+
"""Verify and display information about your authentication **status**
|
|
149
|
+
|
|
150
|
+
Reports **ARRAYLAKE_TOKEN** (or the **token** config setting) first when set, since clients use it instead of
|
|
151
|
+
the cached login.
|
|
152
|
+
"""
|
|
153
|
+
endpoint = _resolve_service_uri(service_uri)
|
|
154
|
+
print_logo_mini()
|
|
155
|
+
rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {endpoint}"), align="center"))
|
|
156
|
+
|
|
157
|
+
handler = get_auth_handler(api_endpoint=endpoint)
|
|
158
|
+
configured_token = default_token()
|
|
159
|
+
configured_token_ok = await _check_configured_token(handler, configured_token) if configured_token else None
|
|
160
|
+
try:
|
|
161
|
+
user = await handler._get_user() # checks that the new tokens are valid
|
|
162
|
+
rich_console.print(Align(f"[green][bold]🔓 Logged in as {user.email}[/green][/bold]", align="center"))
|
|
163
|
+
except AuthException:
|
|
164
|
+
if configured_token_ok is None:
|
|
165
|
+
rich_console.print(Align(f"[yellow]{NOT_LOGGED_IN_MESSAGE}[/yellow]", align="center"))
|
|
166
|
+
raise typer.Exit(code=1)
|
|
167
|
+
rich_console.print(Align("[dim]No valid cached login (`al auth login`) for this endpoint.[/dim]", align="center"))
|
|
168
|
+
if configured_token_ok is False:
|
|
169
|
+
raise typer.Exit(code=1)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@auth.command()
|
|
173
|
+
@coro # type: ignore
|
|
174
|
+
async def token(service_uri: str | None = SERVICE_URI_OPTION):
|
|
175
|
+
"""Print your login's ID **token**, refreshing it first if it has expired or is about to
|
|
176
|
+
|
|
177
|
+
Only the token goes to stdout, so it can be captured, e.g. **PGPASSWORD="$(al auth token)"**. Make sure
|
|
178
|
+
**--service-uri** matches the deployment you will send the token to.
|
|
179
|
+
"""
|
|
180
|
+
endpoint = _resolve_service_uri(service_uri)
|
|
181
|
+
handler = get_auth_handler(api_endpoint=endpoint)
|
|
182
|
+
if handler.tokens is None:
|
|
183
|
+
error_console.print(NOT_LOGGED_IN_MESSAGE)
|
|
184
|
+
raise typer.Exit(code=1)
|
|
185
|
+
try:
|
|
186
|
+
token_value = await handler.get_fresh_id_token(min_valid_seconds=TOKEN_MIN_VALID_SECONDS)
|
|
187
|
+
except AuthException as e:
|
|
188
|
+
error_console.print(f"❌ {escape(str(e))}")
|
|
189
|
+
raise typer.Exit(code=1)
|
|
190
|
+
notice_console.print(f"[dim]ID token for {endpoint} ({describe_token(token_value)})[/dim]", highlight=False)
|
|
191
|
+
if default_token():
|
|
192
|
+
notice_console.print(
|
|
193
|
+
f"[yellow]Note: {_configured_token_name()} is set and takes precedence over this login in Arraylake clients.[/yellow]"
|
|
194
|
+
)
|
|
195
|
+
typer.echo(token_value)
|
|
@@ -3,6 +3,7 @@ from enum import StrEnum
|
|
|
3
3
|
|
|
4
4
|
import typer
|
|
5
5
|
from rich import print_json
|
|
6
|
+
from rich.markup import escape
|
|
6
7
|
from rich.table import Table
|
|
7
8
|
|
|
8
9
|
from arraylake import Client
|
|
@@ -25,6 +26,7 @@ def _services_table(services, org):
|
|
|
25
26
|
table.add_column("Service Type", justify="right", style="green", min_width=10)
|
|
26
27
|
table.add_column("Org", justify="right", style="green", min_width=10)
|
|
27
28
|
table.add_column("Public", justify="right", style="green", min_width=10)
|
|
29
|
+
table.add_column("Region", justify="right", style="green", min_width=10)
|
|
28
30
|
table.add_column("URL", justify="right", style="green", min_width=10)
|
|
29
31
|
table.add_column("Status", justify="right", style="green", min_width=10)
|
|
30
32
|
|
|
@@ -45,6 +47,7 @@ def _services_table(services, org):
|
|
|
45
47
|
service.config.service_type,
|
|
46
48
|
service.config.org,
|
|
47
49
|
str(service.config.is_public),
|
|
50
|
+
service.region or "",
|
|
48
51
|
service.url,
|
|
49
52
|
service.status,
|
|
50
53
|
# .get, because the API owns this vocabulary and adds to it: an older client must
|
|
@@ -67,6 +70,12 @@ def enable_service(
|
|
|
67
70
|
"-n",
|
|
68
71
|
help="Custom deployment name (defaults to protocol name). Must be lowercase alphanumeric with dashes, max 16 chars.",
|
|
69
72
|
),
|
|
73
|
+
region: str | None = typer.Option(
|
|
74
|
+
None,
|
|
75
|
+
"--region",
|
|
76
|
+
"-r",
|
|
77
|
+
help="The region to run the service in, e.g. eu-west-1. Defaults to the environment's default region.",
|
|
78
|
+
),
|
|
70
79
|
):
|
|
71
80
|
"""**Enable** a new service
|
|
72
81
|
|
|
@@ -83,13 +92,19 @@ def enable_service(
|
|
|
83
92
|
```
|
|
84
93
|
$ arraylake compute enable my-org tiles --sku-code s --deployment-name my-tiles
|
|
85
94
|
```
|
|
95
|
+
|
|
96
|
+
- Enable an EDR service in Europe
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
$ arraylake compute enable my-org edr --sku-code xs --region eu-west-1
|
|
100
|
+
```
|
|
86
101
|
"""
|
|
87
|
-
kwargs = {}
|
|
102
|
+
kwargs: dict[str, str] = {}
|
|
88
103
|
if deployment_name is not None:
|
|
89
104
|
kwargs["deployment_name"] = deployment_name
|
|
90
105
|
|
|
91
106
|
with simple_progress(f"Creating [bold]{protocol}[/bold] service for [bold]{org}[/bold]..."):
|
|
92
|
-
status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, **kwargs)
|
|
107
|
+
status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, region=region, **kwargs)
|
|
93
108
|
rich_console.print(status["message"])
|
|
94
109
|
|
|
95
110
|
|
|
@@ -190,8 +205,8 @@ async def get_service_logs(
|
|
|
190
205
|
|
|
191
206
|
async for log in log_stream:
|
|
192
207
|
rich_console.print(log)
|
|
193
|
-
except RuntimeError:
|
|
194
|
-
rich_console.print(f"[bold red]
|
|
208
|
+
except RuntimeError as err:
|
|
209
|
+
rich_console.print(f"[bold red]{escape(str(err))}[/bold red]")
|
|
195
210
|
raise typer.Exit(code=1)
|
|
196
211
|
|
|
197
212
|
|
|
@@ -210,7 +225,7 @@ def update_service(
|
|
|
210
225
|
- Resize a service with ID edr-1234567 to a different compute shape
|
|
211
226
|
|
|
212
227
|
```
|
|
213
|
-
$ arraylake compute update my-org edr-1234567 --sku
|
|
228
|
+
$ arraylake compute update my-org edr-1234567 --sku m
|
|
214
229
|
```
|
|
215
230
|
|
|
216
231
|
- Change the scaling bounds of a service
|
|
@@ -162,7 +162,7 @@ def unset(
|
|
|
162
162
|
@app.command()
|
|
163
163
|
def get(
|
|
164
164
|
key: str = typer.Argument(..., help="key to get in config file"),
|
|
165
|
-
path: Path = typer.Option(
|
|
165
|
+
path: Path | None = typer.Option(None, help="Config file to read. Defaults to the active config."),
|
|
166
166
|
output: ListOutputType = typer.Option("yaml", help="Output formatting"),
|
|
167
167
|
):
|
|
168
168
|
"""**Get** Arraylake config value
|
|
@@ -191,11 +191,11 @@ def get(
|
|
|
191
191
|
def init(path: Path = typer.Option(user_config_file, help="Config file to initialize")):
|
|
192
192
|
"""**Initialize** Arraylake config
|
|
193
193
|
|
|
194
|
-
|
|
194
|
+
Creates the config file if it does not exist.
|
|
195
195
|
"""
|
|
196
196
|
|
|
197
197
|
c = get_user_config(path=path)
|
|
198
198
|
|
|
199
199
|
# write back to file
|
|
200
200
|
write_user_config(c, path=path)
|
|
201
|
-
rich_console.print(f"
|
|
201
|
+
rich_console.print(f"Config file: {path}")
|
|
@@ -109,7 +109,7 @@ def list_namespaces(
|
|
|
109
109
|
filter_metadata: str | None = typer.Option(None, help="JSON metadata filter"),
|
|
110
110
|
sort: NamespaceSort | None = None,
|
|
111
111
|
direction: Direction = Direction.desc,
|
|
112
|
-
include_ghosts: bool = False,
|
|
112
|
+
include_ghosts: bool = typer.Option(False, help="Also list deleted namespaces that you can still restore"),
|
|
113
113
|
page_size: int = typer.Option(50, min=1, max=100),
|
|
114
114
|
output: Output = Output.rich,
|
|
115
115
|
):
|
|
@@ -195,10 +195,22 @@ def delete_namespace(
|
|
|
195
195
|
):
|
|
196
196
|
"""Delete an empty namespace. A soft-delete by default: restore it during its recovery period."""
|
|
197
197
|
if not confirm:
|
|
198
|
-
|
|
199
|
-
|
|
198
|
+
typer.confirm(
|
|
199
|
+
f"{_namespace_delete_outcome(org, name, immediate=immediate, retain_data=retain_data, done=False)} Continue?", abort=True
|
|
200
|
+
)
|
|
200
201
|
Client().delete_iceberg_namespace(org, name, imsure=True, immediate=immediate, retain_data=retain_data)
|
|
201
|
-
typer.echo(
|
|
202
|
+
typer.echo(_namespace_delete_outcome(org, name, immediate=immediate, retain_data=retain_data, done=True))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _namespace_delete_outcome(org: str, name: str, *, immediate: bool, retain_data: bool, done: bool) -> str:
|
|
206
|
+
verb = "Deleted namespace" if done else "This will delete namespace"
|
|
207
|
+
if not immediate:
|
|
208
|
+
return (
|
|
209
|
+
f"{verb} {org}/{name}. Until the grace period ends, you can restore it with `arraylake iceberg namespace restore {org} {name}`."
|
|
210
|
+
)
|
|
211
|
+
if retain_data:
|
|
212
|
+
return f"{verb} {org}/{name} from Arraylake. Its data stays in your storage bucket."
|
|
213
|
+
return f"{verb} {org}/{name} and its data. You cannot restore it."
|
|
202
214
|
|
|
203
215
|
|
|
204
216
|
@namespace_app.command("restore")
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import json
|
|
2
|
+
from datetime import datetime
|
|
2
3
|
from enum import Enum, StrEnum
|
|
3
4
|
from uuid import UUID
|
|
4
5
|
|
|
@@ -12,7 +13,7 @@ from rich.text import Text
|
|
|
12
13
|
from arraylake import AsyncClient
|
|
13
14
|
from arraylake.cli.utils import coro, rich_console, simple_progress
|
|
14
15
|
from arraylake.log_util import get_logger
|
|
15
|
-
from arraylake.types import RepoOperationMode
|
|
16
|
+
from arraylake.types import DatasetFilter, NodeFilter, RepoOperationMode, SubscriptionType
|
|
16
17
|
|
|
17
18
|
app = typer.Typer(help="Manage Arraylake repositories", no_args_is_help=True)
|
|
18
19
|
logger = get_logger(__name__)
|
|
@@ -85,6 +86,8 @@ def _make_json_safe(obj):
|
|
|
85
86
|
return "[REDACTED]"
|
|
86
87
|
elif isinstance(obj, UUID):
|
|
87
88
|
return str(obj)
|
|
89
|
+
elif isinstance(obj, datetime):
|
|
90
|
+
return obj.isoformat()
|
|
88
91
|
else:
|
|
89
92
|
return obj
|
|
90
93
|
|
|
@@ -186,6 +189,86 @@ async def import_repo(
|
|
|
186
189
|
)
|
|
187
190
|
|
|
188
191
|
|
|
192
|
+
@app.command()
|
|
193
|
+
@coro # type: ignore
|
|
194
|
+
async def subscribe(
|
|
195
|
+
repo_name: str = typer.Argument(..., help="Name of the subscriber repository {ORG}/{REPO_NAME}"),
|
|
196
|
+
listing_id: str = typer.Argument(..., help="ID of the exchange listing"),
|
|
197
|
+
subscription_type: SubscriptionType = typer.Option(SubscriptionType.DIRECT, help="Subscription type"),
|
|
198
|
+
include_path: list[str] | None = typer.Option(None, help="Group or array to copy (filtered/private). Repeat for more paths."),
|
|
199
|
+
exclude_path: list[str] | None = typer.Option(None, help="Group or array to leave out (filtered/private). Repeat for more paths."),
|
|
200
|
+
bucket_config_nickname: str | None = typer.Option(None, help="Bucket for the copied data (filtered/private)"),
|
|
201
|
+
prefix: str | None = typer.Option(None, help="Optional prefix in the bucket. If not provided, one is generated."),
|
|
202
|
+
description: str | None = typer.Option(None, help="Description of the repo"),
|
|
203
|
+
metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
|
|
204
|
+
):
|
|
205
|
+
"""**Subscribe** to an exchange listing
|
|
206
|
+
|
|
207
|
+
**Examples**
|
|
208
|
+
|
|
209
|
+
- Subscribe to a free listing
|
|
210
|
+
|
|
211
|
+
```
|
|
212
|
+
$ arraylake repo subscribe my-org/era5 6720f1c2a9e3b4d5e6f70812
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
- Copy two arrays from a paid listing into your bucket
|
|
216
|
+
|
|
217
|
+
```
|
|
218
|
+
$ arraylake repo subscribe my-org/era5-subset 6720f1c2a9e3b4d5e6f70812 --subscription-type filtered --include-path /t2m --include-path /u10 --bucket-config-nickname my-bucket
|
|
219
|
+
```
|
|
220
|
+
"""
|
|
221
|
+
target_filter = (
|
|
222
|
+
DatasetFilter(nodes=NodeFilter(include_paths=include_path or [], exclude_paths=exclude_path or []))
|
|
223
|
+
if include_path or exclude_path
|
|
224
|
+
else None
|
|
225
|
+
)
|
|
226
|
+
metadata_dict = json.loads(metadata) if metadata else None
|
|
227
|
+
with simple_progress(f"Subscribing [bold]{repo_name}[/bold]..."):
|
|
228
|
+
await AsyncClient().subscribe_to_listing(
|
|
229
|
+
repo_name,
|
|
230
|
+
listing_id,
|
|
231
|
+
subscription_type=subscription_type,
|
|
232
|
+
target_filter=target_filter,
|
|
233
|
+
bucket_config_nickname=bucket_config_nickname,
|
|
234
|
+
prefix=prefix,
|
|
235
|
+
description=description,
|
|
236
|
+
metadata=metadata_dict,
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@app.command()
|
|
241
|
+
@coro # type: ignore
|
|
242
|
+
async def claim(
|
|
243
|
+
repo_name: str = typer.Argument(..., help="Name of the subscriber repository {ORG}/{REPO_NAME}"),
|
|
244
|
+
claim_code: str = typer.Argument(..., help="Claim code from the provider"),
|
|
245
|
+
bucket_config_nickname: str | None = typer.Option(None, help="Bucket for the copied data (filtered/private)"),
|
|
246
|
+
prefix: str | None = typer.Option(None, help="Optional prefix in the bucket. If not provided, one is generated."),
|
|
247
|
+
description: str | None = typer.Option(None, help="Description of the repo"),
|
|
248
|
+
metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
|
|
249
|
+
):
|
|
250
|
+
"""**Claim** a subscription with a claim code
|
|
251
|
+
|
|
252
|
+
**Examples**
|
|
253
|
+
|
|
254
|
+
- Claim a subscription
|
|
255
|
+
|
|
256
|
+
```
|
|
257
|
+
$ arraylake repo claim my-org/era5 ABCD-1234
|
|
258
|
+
```
|
|
259
|
+
"""
|
|
260
|
+
metadata_dict = json.loads(metadata) if metadata else None
|
|
261
|
+
with simple_progress(f"Claiming [bold]{repo_name}[/bold]..."):
|
|
262
|
+
await AsyncClient().claim_subscription(
|
|
263
|
+
repo_name,
|
|
264
|
+
claim_code,
|
|
265
|
+
bucket_config_nickname=bucket_config_nickname,
|
|
266
|
+
prefix=prefix,
|
|
267
|
+
description=description,
|
|
268
|
+
metadata=metadata_dict,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
189
272
|
@app.command()
|
|
190
273
|
@coro # type: ignore
|
|
191
274
|
async def modify(
|
|
@@ -253,7 +336,7 @@ async def delete(
|
|
|
253
336
|
"""
|
|
254
337
|
if not confirm:
|
|
255
338
|
confirm = typer.confirm(
|
|
256
|
-
f"
|
|
339
|
+
f"{_delete_outcome(repo_name, immediate=immediate, retain_data=retain_data, done=False)} Continue?",
|
|
257
340
|
abort=True,
|
|
258
341
|
)
|
|
259
342
|
|
|
@@ -262,8 +345,16 @@ async def delete(
|
|
|
262
345
|
with simple_progress(f"Deleting repo [bold]{repo_name}[/bold]..."):
|
|
263
346
|
await client.delete_repo(repo_name, imsure=confirm, imreallysure=confirm, immediate=immediate, retain_data=retain_data)
|
|
264
347
|
|
|
265
|
-
|
|
266
|
-
|
|
348
|
+
rich_console.print(_delete_outcome(repo_name, immediate=immediate, retain_data=retain_data, done=True))
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _delete_outcome(repo_name: str, *, immediate: bool, retain_data: bool, done: bool) -> str:
|
|
352
|
+
verb = "Deleted" if done else "This will delete"
|
|
353
|
+
if not immediate:
|
|
354
|
+
return f"{verb} {repo_name}. You can restore it from the web app until the grace period ends."
|
|
355
|
+
if retain_data:
|
|
356
|
+
return f"{verb} {repo_name} from Arraylake. Its data stays in your storage bucket."
|
|
357
|
+
return f"{verb} {repo_name} and its data. You cannot restore it."
|
|
267
358
|
|
|
268
359
|
|
|
269
360
|
@app.command()
|
|
@@ -339,7 +430,7 @@ async def set_status(
|
|
|
339
430
|
@coro # type: ignore
|
|
340
431
|
async def tune(
|
|
341
432
|
org_name: str = typer.Argument(..., help="Name of organization"),
|
|
342
|
-
bucket_config_nickname: str | None = typer.Option(None, help="
|
|
433
|
+
bucket_config_nickname: str | None = typer.Option(None, help="Bucket config to benchmark. Defaults to the org's default bucket."),
|
|
343
434
|
):
|
|
344
435
|
"""Diagnose I/O configuration for optimal performance
|
|
345
436
|
|