arraylake 1.2.0__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-1.2.0 → arraylake-1.5.0}/PKG-INFO +7 -7
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/_version.py +2 -2
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/api_utils.py +52 -10
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/auth.py +39 -12
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/compute.py +56 -6
- arraylake-1.5.0/arraylake/cli/iceberg.py +255 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/main.py +13 -1
- arraylake-1.5.0/arraylake/cli/org.py +41 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/repo.py +4 -160
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/client.py +752 -113
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/compute/http_client.py +13 -1
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/compute/services.py +53 -5
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/compute/types.py +65 -34
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/display/repo.py +3 -1
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/metastore/http_metastore.py +174 -11
- arraylake-1.5.0/arraylake/obstore.py +132 -0
- arraylake-1.5.0/arraylake/repos/iceberg/__init__.py +3 -0
- arraylake-1.5.0/arraylake/repos/iceberg/auth.py +54 -0
- arraylake-1.5.0/arraylake/repos/iceberg/catalog.py +52 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/types.py +20 -2
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/virtual.py +53 -2
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/token.py +24 -2
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/types.py +364 -52
- {arraylake-1.2.0 → arraylake-1.5.0}/pyproject.toml +8 -6
- arraylake-1.5.0/tests/compute/test_cli_table.py +33 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/compute/test_client.py +33 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/test_virtual.py +26 -1
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_api_utils.py +98 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_cli.py +106 -1
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_client.py +124 -2
- arraylake-1.5.0/tests/test_http_metastore.py +97 -0
- arraylake-1.5.0/tests/test_iceberg_client.py +419 -0
- arraylake-1.5.0/tests/test_obstore.py +182 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_performance.py +4 -14
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_subscriptions.py +4 -4
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_token_handler.py +44 -1
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_types.py +246 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_virtual_chunks.py +134 -63
- {arraylake-1.2.0 → arraylake-1.5.0}/uv.lock +943 -571
- arraylake-1.2.0/arraylake/compute/doctor.py +0 -266
- arraylake-1.2.0/tests/compute/test_doctor.py +0 -173
- {arraylake-1.2.0 → arraylake-1.5.0}/.gitignore +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/README.md +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/__main__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/_credential_cache.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/asyn.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/config.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/skills.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/utils.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/config.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/config.yaml +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/credentials.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/diagnostics.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/display/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/display/repolist.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/exceptions.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/log_util.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/metastore/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/metastore/abc.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/py.typed +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/storage.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/tuning.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/config.yaml +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/conftest.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/__init__.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/conftest.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/test_storage.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_asyn.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_config.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_credential_cache.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_credentials.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_diagnostics.py +0 -0
- {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_skills.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: arraylake
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: Python client for ArrayLake
|
|
5
5
|
Project-URL: Documentation, https://docs.earthmover.io/
|
|
6
6
|
Project-URL: Changelog, https://docs.earthmover.io/changelog
|
|
@@ -14,25 +14,25 @@ Requires-Python: <3.15,>=3.12
|
|
|
14
14
|
Requires-Dist: click<9.0,>=8.1.3
|
|
15
15
|
Requires-Dist: donfig<1.0,>=0.7
|
|
16
16
|
Requires-Dist: httpx<0.29,>=0.23
|
|
17
|
-
Requires-Dist: icechunk<3.0.0,>=2.
|
|
17
|
+
Requires-Dist: icechunk<3.0.0,>=2.2.2
|
|
18
18
|
Requires-Dist: numpy<3.0,>=1.23
|
|
19
19
|
Requires-Dist: packaging<27.0,>=23.0
|
|
20
20
|
Requires-Dist: pydantic[email]<3.0,>=2.9
|
|
21
21
|
Requires-Dist: python-dateutil<3.0,>=2.8
|
|
22
22
|
Requires-Dist: rich<16.0,>=12.6
|
|
23
23
|
Requires-Dist: ruamel-yaml<1.0,>=0.17
|
|
24
|
-
Requires-Dist: structlog<
|
|
24
|
+
Requires-Dist: structlog<27.0,>=24.1
|
|
25
25
|
Requires-Dist: typer<1.0,>=0.12.0
|
|
26
26
|
Requires-Dist: zarr<4.0,>=3.1.0
|
|
27
|
+
Provides-Extra: iceberg
|
|
28
|
+
Requires-Dist: pyiceberg<0.13,>=0.11; extra == 'iceberg'
|
|
27
29
|
Provides-Extra: maximal
|
|
28
|
-
Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
|
|
29
30
|
Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
|
|
30
31
|
Provides-Extra: obstore
|
|
31
32
|
Requires-Dist: obstore>=0.9.2; extra == 'obstore'
|
|
32
33
|
Provides-Extra: openeo
|
|
33
|
-
Requires-Dist: openeo<0.
|
|
34
|
+
Requires-Dist: openeo<0.52,>=0.35.0; extra == 'openeo'
|
|
34
35
|
Provides-Extra: xarray
|
|
35
|
-
Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
|
|
36
36
|
Requires-Dist: xarray>=2024.10.0; extra == 'xarray'
|
|
37
37
|
Description-Content-Type: text/markdown
|
|
38
38
|
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1,
|
|
21
|
+
__version__ = version = '1.5.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 5, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -84,6 +84,25 @@ def _get_proxy_from_env() -> str | None:
|
|
|
84
84
|
return None
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
# Client name reported in the X-Arraylake-Client header. Defaults to the Python
|
|
88
|
+
# library; the CLI entry point overrides it so server logs can tell them apart.
|
|
89
|
+
_client_name: str = "python"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def set_client_name(name: str) -> None:
|
|
93
|
+
"""Override the client name sent in the ``X-Arraylake-Client`` header.
|
|
94
|
+
|
|
95
|
+
Must be called before any ``ArraylakeHttpClient`` is constructed to take effect,
|
|
96
|
+
since headers are baked in at construction time.
|
|
97
|
+
"""
|
|
98
|
+
global _client_name
|
|
99
|
+
_client_name = name
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def get_client_name() -> str:
|
|
103
|
+
return _client_name
|
|
104
|
+
|
|
105
|
+
|
|
87
106
|
@functools.cache
|
|
88
107
|
def _get_session_id() -> str:
|
|
89
108
|
"""Session ID for this Python process.
|
|
@@ -116,10 +135,20 @@ class ClientKey:
|
|
|
116
135
|
# the global cache of httpx clients
|
|
117
136
|
_GLOBAL_CLIENTS: dict[ClientKey, httpx.AsyncClient] = {}
|
|
118
137
|
|
|
138
|
+
# one lock per event loop, so concurrent callers on a cold cache build one client, not two
|
|
139
|
+
_CLIENT_CREATION_LOCKS: dict[asyncio.AbstractEventLoop, asyncio.Lock] = {}
|
|
140
|
+
|
|
119
141
|
# this is a cache to use hold asyncio tasks so they are not garbage collected before finishing
|
|
120
142
|
background_tasks: set[asyncio.Task] = set()
|
|
121
143
|
|
|
122
144
|
|
|
145
|
+
def _creation_lock(loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
|
|
146
|
+
lock = _CLIENT_CREATION_LOCKS.get(loop)
|
|
147
|
+
if lock is None:
|
|
148
|
+
lock = _CLIENT_CREATION_LOCKS[loop] = asyncio.Lock()
|
|
149
|
+
return lock
|
|
150
|
+
|
|
151
|
+
|
|
123
152
|
async def get_client(
|
|
124
153
|
loop: asyncio.AbstractEventLoop,
|
|
125
154
|
api_url: str,
|
|
@@ -162,7 +191,13 @@ async def get_client(
|
|
|
162
191
|
proxy,
|
|
163
192
|
)
|
|
164
193
|
await logger.adebug("%i httpx clients present in cache.", len(_GLOBAL_CLIENTS))
|
|
165
|
-
if key not
|
|
194
|
+
if (client := _GLOBAL_CLIENTS.get(key)) is not None:
|
|
195
|
+
await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
|
|
196
|
+
return client
|
|
197
|
+
|
|
198
|
+
async with _creation_lock(loop):
|
|
199
|
+
if (client := _GLOBAL_CLIENTS.get(key)) is not None:
|
|
200
|
+
return client
|
|
166
201
|
await logger.adebug("Creating new httpx client %s. Loop id %s.", key, id(loop))
|
|
167
202
|
transport = AsyncRetryTransport(
|
|
168
203
|
max_keepalive_connections=max_keepalive_connections,
|
|
@@ -180,15 +215,12 @@ async def get_client(
|
|
|
180
215
|
verify=verify_ssl,
|
|
181
216
|
cert=ssl_cafile,
|
|
182
217
|
).__aenter__()
|
|
183
|
-
weakref.finalize(client, close_client, key)
|
|
218
|
+
weakref.finalize(client, close_client, key, id(client))
|
|
184
219
|
_GLOBAL_CLIENTS[key] = client
|
|
185
|
-
|
|
186
|
-
await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
|
|
187
|
-
assert key in _GLOBAL_CLIENTS
|
|
188
|
-
return _GLOBAL_CLIENTS[key]
|
|
220
|
+
return client
|
|
189
221
|
|
|
190
222
|
|
|
191
|
-
def close_client(key: ClientKey):
|
|
223
|
+
def close_client(key: ClientKey, client_id: int) -> None:
|
|
192
224
|
"""
|
|
193
225
|
This is a finalizer function that is called when a global client is
|
|
194
226
|
garbage collected. It cleanly closes the client for the specified key.
|
|
@@ -205,11 +237,13 @@ def close_client(key: ClientKey):
|
|
|
205
237
|
Note: logging in this function runs the risk of conflicting with pytest#5502. For
|
|
206
238
|
this reason, we have removed debug log statements.
|
|
207
239
|
"""
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
if
|
|
240
|
+
cached = _GLOBAL_CLIENTS.get(key)
|
|
241
|
+
# A newer client can own this key. Only the client that registered this finalizer may evict it.
|
|
242
|
+
if cached is None or id(cached) != client_id:
|
|
211
243
|
return
|
|
212
244
|
|
|
245
|
+
client = _GLOBAL_CLIENTS.pop(key)
|
|
246
|
+
|
|
213
247
|
if client.is_closed:
|
|
214
248
|
return
|
|
215
249
|
|
|
@@ -317,6 +351,9 @@ class ArraylakeHttpClient:
|
|
|
317
351
|
# technically we don't enforce this, for now it's for debugging purposes
|
|
318
352
|
"client-name": "arraylake-python-client",
|
|
319
353
|
"client-version": arraylake.__version__,
|
|
354
|
+
"x-arraylake-client": f"{get_client_name()}/{arraylake.__version__}",
|
|
355
|
+
"x-arraylake-source": get_client_name(),
|
|
356
|
+
"user-agent": f"arraylake-{get_client_name()}/{arraylake.__version__}",
|
|
320
357
|
"arraylake-session-id": _get_session_id(),
|
|
321
358
|
}
|
|
322
359
|
|
|
@@ -448,6 +485,7 @@ def _exception_log_debug(request: httpx.Request, response: httpx.Response):
|
|
|
448
485
|
|
|
449
486
|
|
|
450
487
|
_emitted_deprecation_warnings: set[str] = set()
|
|
488
|
+
_emitted_user_warnings: set[str] = set()
|
|
451
489
|
|
|
452
490
|
|
|
453
491
|
def handle_response(response: httpx.Response):
|
|
@@ -456,6 +494,10 @@ def handle_response(response: httpx.Response):
|
|
|
456
494
|
if deprecation_msg and deprecation_msg not in _emitted_deprecation_warnings:
|
|
457
495
|
_emitted_deprecation_warnings.add(deprecation_msg)
|
|
458
496
|
warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2)
|
|
497
|
+
warning_msg = response.headers.get("X-Arraylake-Warning") # Service-sent user-facing warnings
|
|
498
|
+
if warning_msg and warning_msg not in _emitted_user_warnings:
|
|
499
|
+
_emitted_user_warnings.add(warning_msg)
|
|
500
|
+
warnings.warn(warning_msg, UserWarning, stacklevel=2)
|
|
459
501
|
try:
|
|
460
502
|
response.raise_for_status()
|
|
461
503
|
except httpx.RequestError as exc:
|
|
@@ -2,7 +2,6 @@ import typer
|
|
|
2
2
|
from rich.align import Align
|
|
3
3
|
from rich.panel import Panel
|
|
4
4
|
|
|
5
|
-
from arraylake import config
|
|
6
5
|
from arraylake.cli.utils import (
|
|
7
6
|
coro,
|
|
8
7
|
error_console,
|
|
@@ -17,11 +16,26 @@ auth = typer.Typer(help="Manage Arraylake authentication")
|
|
|
17
16
|
|
|
18
17
|
NOT_LOGGED_IN_MESSAGE = "⚠️ Not logged in, please log in with: `arraylake auth login`"
|
|
19
18
|
|
|
19
|
+
SERVICE_URI_OPTION = typer.Option(
|
|
20
|
+
None,
|
|
21
|
+
"--service-uri",
|
|
22
|
+
help=(
|
|
23
|
+
"Arraylake API endpoint to act on, e.g. **https://api.dev.earthmover.io**. Defaults to the configured "
|
|
24
|
+
"**service.uri**. Tokens are stored per endpoint, so this need not be persisted to config — unless "
|
|
25
|
+
"**service.token_path** is set, which makes all endpoints share one token file."
|
|
26
|
+
),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _resolve_service_uri(service_uri: str | None) -> str:
|
|
31
|
+
return service_uri if service_uri else default_service_uri()
|
|
32
|
+
|
|
20
33
|
|
|
21
34
|
@auth.command()
|
|
22
35
|
@coro # type: ignore
|
|
23
36
|
async def login(
|
|
24
37
|
browser: bool = typer.Option(True, "--browser/--no-browser", " /--nobrowser", help="Whether to automatically open a browser window."),
|
|
38
|
+
service_uri: str | None = SERVICE_URI_OPTION,
|
|
25
39
|
):
|
|
26
40
|
"""**Log in** to Arraylake
|
|
27
41
|
|
|
@@ -34,17 +48,23 @@ async def login(
|
|
|
34
48
|
```
|
|
35
49
|
$ arraylake auth login --no-browser
|
|
36
50
|
```
|
|
51
|
+
|
|
52
|
+
- Log in to a non-default Arraylake deployment
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
$ arraylake auth login --service-uri https://api.dev.earthmover.io
|
|
56
|
+
```
|
|
37
57
|
"""
|
|
38
58
|
print_logo()
|
|
39
|
-
handler = get_auth_handler(api_endpoint=
|
|
59
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
40
60
|
await handler.login(browser=browser)
|
|
41
61
|
|
|
42
62
|
|
|
43
63
|
@auth.command()
|
|
44
64
|
@coro # type: ignore
|
|
45
|
-
async def refresh() -> None:
|
|
65
|
+
async def refresh(service_uri: str | None = SERVICE_URI_OPTION) -> None:
|
|
46
66
|
"""**Refresh** Arraylake's auth token"""
|
|
47
|
-
handler = get_auth_handler(api_endpoint=
|
|
67
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
48
68
|
if handler.tokens is not None:
|
|
49
69
|
try:
|
|
50
70
|
await handler.refresh_token()
|
|
@@ -57,7 +77,7 @@ async def refresh() -> None:
|
|
|
57
77
|
|
|
58
78
|
@auth.command()
|
|
59
79
|
@coro # type: ignore
|
|
60
|
-
async def logout() -> None:
|
|
80
|
+
async def logout(service_uri: str | None = SERVICE_URI_OPTION) -> None:
|
|
61
81
|
"""**Logout** of Arraylake
|
|
62
82
|
|
|
63
83
|
**Examples**
|
|
@@ -67,9 +87,15 @@ async def logout() -> None:
|
|
|
67
87
|
```
|
|
68
88
|
$ arraylake auth logout
|
|
69
89
|
```
|
|
90
|
+
|
|
91
|
+
- Log out of a non-default Arraylake deployment
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
$ arraylake auth logout --service-uri https://api.dev.earthmover.io
|
|
95
|
+
```
|
|
70
96
|
"""
|
|
71
97
|
try:
|
|
72
|
-
handler = get_auth_handler(api_endpoint=
|
|
98
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
73
99
|
await handler.logout()
|
|
74
100
|
except AuthException as e:
|
|
75
101
|
error_console.print(e)
|
|
@@ -77,14 +103,15 @@ async def logout() -> None:
|
|
|
77
103
|
|
|
78
104
|
|
|
79
105
|
@auth.command()
|
|
80
|
-
@coro
|
|
81
|
-
async def status():
|
|
106
|
+
@coro # type: ignore
|
|
107
|
+
async def status(service_uri: str | None = SERVICE_URI_OPTION):
|
|
82
108
|
"""Verify and display information about your authentication **status**"""
|
|
109
|
+
endpoint = _resolve_service_uri(service_uri)
|
|
83
110
|
print_logo_mini()
|
|
84
|
-
rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {
|
|
111
|
+
rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {endpoint}"), align="center"))
|
|
85
112
|
|
|
86
113
|
try:
|
|
87
|
-
handler = get_auth_handler(api_endpoint=
|
|
114
|
+
handler = get_auth_handler(api_endpoint=endpoint)
|
|
88
115
|
user = await handler._get_user() # checks that the new tokens are valid
|
|
89
116
|
rich_console.print(Align(f"[green][bold]🔓 Logged in as {user.email}[/green][/bold]", align="center"))
|
|
90
117
|
except AuthException:
|
|
@@ -93,9 +120,9 @@ async def status():
|
|
|
93
120
|
|
|
94
121
|
|
|
95
122
|
@auth.command()
|
|
96
|
-
def token():
|
|
123
|
+
def token(service_uri: str | None = SERVICE_URI_OPTION):
|
|
97
124
|
"""Display your authentication **token**"""
|
|
98
|
-
handler = get_auth_handler(api_endpoint=
|
|
125
|
+
handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
|
|
99
126
|
if handler.tokens is None:
|
|
100
127
|
rich_console.print(Align(f"[yellow]{NOT_LOGGED_IN_MESSAGE}[/yellow]", align="center"))
|
|
101
128
|
raise typer.Exit(code=1)
|
|
@@ -28,7 +28,16 @@ def _services_table(services, org):
|
|
|
28
28
|
table.add_column("URL", justify="right", style="green", min_width=10)
|
|
29
29
|
table.add_column("Status", justify="right", style="green", min_width=10)
|
|
30
30
|
|
|
31
|
-
status_colors = {
|
|
31
|
+
status_colors = {
|
|
32
|
+
"available": "green",
|
|
33
|
+
"progressing": "yellow",
|
|
34
|
+
"error": "red",
|
|
35
|
+
"quota_exceeded": "red",
|
|
36
|
+
"sleeping": "cyan",
|
|
37
|
+
"stopped": "grey",
|
|
38
|
+
"suspended": "grey",
|
|
39
|
+
"unknown": "grey",
|
|
40
|
+
}
|
|
32
41
|
|
|
33
42
|
for service in services:
|
|
34
43
|
table.add_row(
|
|
@@ -38,7 +47,9 @@ def _services_table(services, org):
|
|
|
38
47
|
str(service.config.is_public),
|
|
39
48
|
service.url,
|
|
40
49
|
service.status,
|
|
41
|
-
|
|
50
|
+
# .get, because the API owns this vocabulary and adds to it: an older client must
|
|
51
|
+
# render a status it has never heard of, not refuse to list anything at all.
|
|
52
|
+
style=status_colors.get(service.status, "grey"),
|
|
42
53
|
)
|
|
43
54
|
|
|
44
55
|
return table
|
|
@@ -47,7 +58,8 @@ def _services_table(services, org):
|
|
|
47
58
|
@app.command(name="enable")
|
|
48
59
|
def enable_service(
|
|
49
60
|
org: str = typer.Argument(..., help="The organization to enable the service for"),
|
|
50
|
-
protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr"),
|
|
61
|
+
protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr, openeo, sql"),
|
|
62
|
+
sku_code: str = typer.Option(..., "--sku-code", "-s", help="The compute shape to run the service on, e.g. xs"),
|
|
51
63
|
is_public: bool = typer.Option(False, help="Whether the service is public"),
|
|
52
64
|
deployment_name: str | None = typer.Option(
|
|
53
65
|
None,
|
|
@@ -63,13 +75,13 @@ def enable_service(
|
|
|
63
75
|
- Enable a new DAP service for _my-org_
|
|
64
76
|
|
|
65
77
|
```
|
|
66
|
-
$ arraylake compute enable my-org dap2
|
|
78
|
+
$ arraylake compute enable my-org dap2 --sku-code xs
|
|
67
79
|
```
|
|
68
80
|
|
|
69
81
|
- Enable a tiles service with a custom deployment name
|
|
70
82
|
|
|
71
83
|
```
|
|
72
|
-
$ arraylake compute enable my-org tiles --deployment-name my-tiles
|
|
84
|
+
$ arraylake compute enable my-org tiles --sku-code s --deployment-name my-tiles
|
|
73
85
|
```
|
|
74
86
|
"""
|
|
75
87
|
kwargs = {}
|
|
@@ -77,7 +89,7 @@ def enable_service(
|
|
|
77
89
|
kwargs["deployment_name"] = deployment_name
|
|
78
90
|
|
|
79
91
|
with simple_progress(f"Creating [bold]{protocol}[/bold] service for [bold]{org}[/bold]..."):
|
|
80
|
-
status = Client().get_services(org).enable(protocol, is_public, **kwargs)
|
|
92
|
+
status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, **kwargs)
|
|
81
93
|
rich_console.print(status["message"])
|
|
82
94
|
|
|
83
95
|
|
|
@@ -183,6 +195,44 @@ async def get_service_logs(
|
|
|
183
195
|
raise typer.Exit(code=1)
|
|
184
196
|
|
|
185
197
|
|
|
198
|
+
@app.command(name="update")
|
|
199
|
+
def update_service(
|
|
200
|
+
org: str = typer.Argument(..., help="The organization that the service belongs to"),
|
|
201
|
+
service_id: str = typer.Argument(..., help="The service ID to update"),
|
|
202
|
+
sku: str | None = typer.Option(None, "--sku", help="The compute shape (SKU code) to resize the service to"),
|
|
203
|
+
min_replicas: int | None = typer.Option(None, min=0, help="The minimum number of replicas the service scales down to"),
|
|
204
|
+
max_replicas: int | None = typer.Option(None, min=1, help="The maximum number of replicas the service scales up to"),
|
|
205
|
+
):
|
|
206
|
+
"""**Update** a running service's shape or scaling. Omitted options keep their current values.
|
|
207
|
+
|
|
208
|
+
**Examples**
|
|
209
|
+
|
|
210
|
+
- Resize a service with ID edr-1234567 to a different compute shape
|
|
211
|
+
|
|
212
|
+
```
|
|
213
|
+
$ arraylake compute update my-org edr-1234567 --sku medium
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
- Change the scaling bounds of a service
|
|
217
|
+
|
|
218
|
+
```
|
|
219
|
+
$ arraylake compute update my-org edr-1234567 --min-replicas 2 --max-replicas 10
|
|
220
|
+
```
|
|
221
|
+
"""
|
|
222
|
+
if sku is None and min_replicas is None and max_replicas is None:
|
|
223
|
+
rich_console.print("[bold red]Nothing to update: pass at least one of --sku, --min-replicas, --max-replicas[/bold red]")
|
|
224
|
+
raise typer.Exit(code=1)
|
|
225
|
+
|
|
226
|
+
with simple_progress(f"Updating service [bold]{service_id}[/bold]..."):
|
|
227
|
+
deployment = Client().get_services(org).update(service_id, sku_code=sku, min_replicas=min_replicas, max_replicas=max_replicas)
|
|
228
|
+
|
|
229
|
+
config = deployment.config
|
|
230
|
+
rich_console.print(
|
|
231
|
+
f"Service [bold]{deployment.name}[/bold] updated: "
|
|
232
|
+
f"sku={config.sku_code}, min_replicas={config.min_replicas}, max_replicas={config.max_replicas}"
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
186
236
|
@app.command(name="disable")
|
|
187
237
|
def disable_service(
|
|
188
238
|
org: str = typer.Argument(..., help="The organization that the service belongs to"),
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Namespace administration and table discovery for Arraylake's Iceberg catalog."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from enum import StrEnum
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from arraylake import Client
|
|
12
|
+
from arraylake.cli.utils import rich_console
|
|
13
|
+
from arraylake.types import RepoVisibility
|
|
14
|
+
|
|
15
|
+
app = typer.Typer(help="Manage Iceberg namespaces and discover tables", no_args_is_help=True)
|
|
16
|
+
namespace_app = typer.Typer(help="Manage Iceberg namespaces", no_args_is_help=True)
|
|
17
|
+
table_app = typer.Typer(help="Discover and describe Iceberg tables", no_args_is_help=True)
|
|
18
|
+
app.add_typer(namespace_app, name="namespace")
|
|
19
|
+
app.add_typer(table_app, name="table")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Output(StrEnum):
|
|
23
|
+
rich = "rich"
|
|
24
|
+
json = "json"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class NamespaceSort(StrEnum):
|
|
28
|
+
relevance = "relevance"
|
|
29
|
+
updated = "updated"
|
|
30
|
+
created = "created"
|
|
31
|
+
name = "name"
|
|
32
|
+
tables = "tables"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TableSort(StrEnum):
|
|
36
|
+
relevance = "relevance"
|
|
37
|
+
updated = "updated"
|
|
38
|
+
created = "created"
|
|
39
|
+
name = "name"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Direction(StrEnum):
|
|
43
|
+
asc = "asc"
|
|
44
|
+
desc = "desc"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _table_identifier(namespace: str, table: str) -> str:
|
|
48
|
+
"""Format exact names; quoted components escape double quotes by doubling them."""
|
|
49
|
+
|
|
50
|
+
def quote(name: str) -> str:
|
|
51
|
+
return name if re.fullmatch(r"[A-Za-z0-9_-]+", name) else '"' + name.replace('"', '""') + '"'
|
|
52
|
+
|
|
53
|
+
return f"{quote(namespace)}.{quote(table)}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _parse_table_identifier(identifier: str) -> tuple[str, str]:
|
|
57
|
+
# Keep this grammar in sync with the app's iceberg-identifier.ts formatter.
|
|
58
|
+
component = r'(?:[A-Za-z0-9_-]+|"(?:[^"]|"")+")'
|
|
59
|
+
match = re.fullmatch(rf"({component})\.({component})", identifier)
|
|
60
|
+
if match is None:
|
|
61
|
+
raise typer.BadParameter(
|
|
62
|
+
"Expected NAMESPACE.TABLE. Double-quote names containing dots or special characters, "
|
|
63
|
+
"and wrap the identifier in single quotes in your shell, e.g. '\"namespace.v2\".table'.",
|
|
64
|
+
param_hint="IDENTIFIER",
|
|
65
|
+
)
|
|
66
|
+
namespace, table = (part[1:-1].replace('""', '"') if part.startswith('"') else part for part in match.groups())
|
|
67
|
+
return namespace, table
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _json_object(value: str | None, option: str) -> dict[str, Any] | None:
|
|
71
|
+
if value is None:
|
|
72
|
+
return None
|
|
73
|
+
try:
|
|
74
|
+
result = json.loads(value)
|
|
75
|
+
except ValueError as exc:
|
|
76
|
+
raise typer.BadParameter("Expected a JSON object", param_hint=option) from exc
|
|
77
|
+
if not isinstance(result, dict):
|
|
78
|
+
raise typer.BadParameter("Expected a JSON object", param_hint=option)
|
|
79
|
+
return result
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _show(value: Any, output: Output) -> None:
|
|
83
|
+
if output == Output.json:
|
|
84
|
+
typer.echo(json.dumps(value))
|
|
85
|
+
else:
|
|
86
|
+
rich_console.print(value)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _show_list(rows: list[dict[str, Any]], output: Output, *, namespaces: bool) -> None:
|
|
90
|
+
if not namespaces:
|
|
91
|
+
rows = [{**row, "identifier": _table_identifier(row["namespace"], row["name"])} for row in rows]
|
|
92
|
+
if output == Output.json:
|
|
93
|
+
_show(rows, output)
|
|
94
|
+
return
|
|
95
|
+
table = Table("Name" if namespaces else "Identifier", "Updated", "Tables" if namespaces else "Storage override")
|
|
96
|
+
for row in rows:
|
|
97
|
+
table.add_row(
|
|
98
|
+
row["name"] if namespaces else row["identifier"],
|
|
99
|
+
row["updated"],
|
|
100
|
+
str(row["table_count"] if namespaces else row["has_storage_override"]),
|
|
101
|
+
)
|
|
102
|
+
rich_console.print(table)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@namespace_app.command("list")
|
|
106
|
+
def list_namespaces(
|
|
107
|
+
org: str,
|
|
108
|
+
search: str | None = None,
|
|
109
|
+
filter_metadata: str | None = typer.Option(None, help="JSON metadata filter"),
|
|
110
|
+
sort: NamespaceSort | None = None,
|
|
111
|
+
direction: Direction = Direction.desc,
|
|
112
|
+
include_ghosts: bool = False,
|
|
113
|
+
page_size: int = typer.Option(50, min=1, max=100),
|
|
114
|
+
output: Output = Output.rich,
|
|
115
|
+
):
|
|
116
|
+
"""List namespaces, with optional search, metadata filters, and sorting."""
|
|
117
|
+
rows = Client().list_iceberg_namespaces_paginated(
|
|
118
|
+
org,
|
|
119
|
+
search=search,
|
|
120
|
+
filter_metadata=_json_object(filter_metadata, "--filter-metadata"),
|
|
121
|
+
sort=sort.value if sort else None,
|
|
122
|
+
direction=direction.value,
|
|
123
|
+
include_ghosts=include_ghosts,
|
|
124
|
+
page_size=page_size,
|
|
125
|
+
)
|
|
126
|
+
_show_list([row.model_dump(mode="json") for row in rows], output, namespaces=True)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@namespace_app.command("get")
|
|
130
|
+
def get_namespace(org: str, name: str, output: Output = Output.rich):
|
|
131
|
+
"""Show namespace storage, description, metadata, and visibility."""
|
|
132
|
+
_show(Client().get_iceberg_namespace(org, name).model_dump(mode="json"), output)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@namespace_app.command("create")
|
|
136
|
+
def create_namespace(
|
|
137
|
+
org: str,
|
|
138
|
+
name: str,
|
|
139
|
+
bucket_config_nickname: str | None = None,
|
|
140
|
+
description: str | None = None,
|
|
141
|
+
metadata: str | None = typer.Option(None, help="JSON discovery metadata"),
|
|
142
|
+
visibility: RepoVisibility = RepoVisibility.PRIVATE,
|
|
143
|
+
properties: str | None = typer.Option(None, help="JSON Iceberg properties (string values)"),
|
|
144
|
+
output: Output = Output.rich,
|
|
145
|
+
):
|
|
146
|
+
"""Create a namespace on the chosen or default bucket."""
|
|
147
|
+
props = _json_object(properties, "--properties")
|
|
148
|
+
if props is not None and any(not isinstance(v, str) for v in props.values()):
|
|
149
|
+
raise typer.BadParameter("Property values must be strings", param_hint="--properties")
|
|
150
|
+
result = Client().create_iceberg_namespace(
|
|
151
|
+
org,
|
|
152
|
+
name,
|
|
153
|
+
bucket_config_nickname=bucket_config_nickname,
|
|
154
|
+
description=description,
|
|
155
|
+
metadata=_json_object(metadata, "--metadata"),
|
|
156
|
+
visibility=visibility,
|
|
157
|
+
properties=props,
|
|
158
|
+
)
|
|
159
|
+
_show(result.model_dump(mode="json"), output)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@namespace_app.command("update")
|
|
163
|
+
def update_namespace(
|
|
164
|
+
org: str,
|
|
165
|
+
name: str,
|
|
166
|
+
description: str | None = None,
|
|
167
|
+
add_metadata: str | None = typer.Option(None, help="JSON metadata to add"),
|
|
168
|
+
remove_metadata: list[str] | None = typer.Option(None, help="Metadata key to remove; repeat for multiple keys"),
|
|
169
|
+
update_metadata: str | None = typer.Option(None, help="JSON metadata to update"),
|
|
170
|
+
visibility: RepoVisibility | None = None,
|
|
171
|
+
output: Output = Output.rich,
|
|
172
|
+
):
|
|
173
|
+
"""Update description, discovery metadata, or visibility. Use --description '' to clear."""
|
|
174
|
+
result = Client().modify_iceberg_namespace(
|
|
175
|
+
org,
|
|
176
|
+
name,
|
|
177
|
+
description=description,
|
|
178
|
+
add_metadata=_json_object(add_metadata, "--add-metadata"),
|
|
179
|
+
remove_metadata=remove_metadata,
|
|
180
|
+
update_metadata=_json_object(update_metadata, "--update-metadata"),
|
|
181
|
+
visibility=visibility,
|
|
182
|
+
)
|
|
183
|
+
_show(result.model_dump(mode="json"), output)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@namespace_app.command("delete")
|
|
187
|
+
def delete_namespace(
|
|
188
|
+
org: str,
|
|
189
|
+
name: str,
|
|
190
|
+
confirm: bool = typer.Option(False, "--confirm", help="Skip the confirmation prompt"),
|
|
191
|
+
immediate: bool = typer.Option(
|
|
192
|
+
False, "--immediate", help="Skip the recovery period: the namespace cannot be restored and its name is reusable at once"
|
|
193
|
+
),
|
|
194
|
+
retain_data: bool = typer.Option(False, "--retain-data", help="Leave the bytes under the namespace's storage prefix in the bucket"),
|
|
195
|
+
):
|
|
196
|
+
"""Delete an empty namespace. A soft-delete by default: restore it during its recovery period."""
|
|
197
|
+
if not confirm:
|
|
198
|
+
verb = "Permanently delete" if immediate else "Delete"
|
|
199
|
+
typer.confirm(f"{verb} namespace {org}/{name}?", abort=True)
|
|
200
|
+
Client().delete_iceberg_namespace(org, name, imsure=True, immediate=immediate, retain_data=retain_data)
|
|
201
|
+
typer.echo(f"Deleted {org}/{name}")
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@namespace_app.command("restore")
|
|
205
|
+
def restore_namespace(org: str, name: str):
|
|
206
|
+
"""Restore a soft-deleted namespace."""
|
|
207
|
+
Client().restore_iceberg_namespace(org, name)
|
|
208
|
+
typer.echo(f"Restored {org}/{name}")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@table_app.command("list")
|
|
212
|
+
def list_tables(
|
|
213
|
+
org: str,
|
|
214
|
+
namespace: str,
|
|
215
|
+
search: str | None = None,
|
|
216
|
+
sort: TableSort | None = None,
|
|
217
|
+
direction: Direction = Direction.desc,
|
|
218
|
+
page_size: int = typer.Option(50, min=1, max=100),
|
|
219
|
+
output: Output = Output.rich,
|
|
220
|
+
):
|
|
221
|
+
"""List table summaries without fetching table metadata or recording accesses."""
|
|
222
|
+
rows = Client().list_iceberg_tables_paginated(
|
|
223
|
+
org,
|
|
224
|
+
namespace,
|
|
225
|
+
search=search,
|
|
226
|
+
sort=sort.value if sort else None,
|
|
227
|
+
direction=direction.value,
|
|
228
|
+
page_size=page_size,
|
|
229
|
+
)
|
|
230
|
+
_show_list([row.model_dump(mode="json") for row in rows], output, namespaces=False)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@table_app.command("describe")
|
|
234
|
+
def describe_table(
|
|
235
|
+
org: str,
|
|
236
|
+
identifier: str = typer.Argument(..., help="NAMESPACE.TABLE; double-quote components containing dots or special characters."),
|
|
237
|
+
output: Output = Output.rich,
|
|
238
|
+
):
|
|
239
|
+
"""Load a table's schema, snapshots, partitioning, and properties. Requires arraylake[iceberg]."""
|
|
240
|
+
namespace, table = _parse_table_identifier(identifier)
|
|
241
|
+
catalog = Client().get_iceberg(org)
|
|
242
|
+
try:
|
|
243
|
+
loaded = catalog.load_table((namespace, table))
|
|
244
|
+
_show(
|
|
245
|
+
{
|
|
246
|
+
"name": table,
|
|
247
|
+
"namespace": namespace,
|
|
248
|
+
"identifier": _table_identifier(namespace, table),
|
|
249
|
+
"metadata_location": loaded.metadata_location,
|
|
250
|
+
"metadata": loaded.metadata.model_dump(mode="json", by_alias=True),
|
|
251
|
+
},
|
|
252
|
+
output,
|
|
253
|
+
)
|
|
254
|
+
finally:
|
|
255
|
+
catalog.close()
|