arraylake 1.3.0__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-1.3.0 → arraylake-1.5.0}/PKG-INFO +6 -6
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/_version.py +2 -2
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/api_utils.py +25 -10
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/compute.py +14 -3
- arraylake-1.5.0/arraylake/cli/iceberg.py +255 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/main.py +8 -1
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/repo.py +4 -160
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/client.py +645 -41
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/compute/services.py +1 -1
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/compute/types.py +26 -13
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/metastore/http_metastore.py +147 -2
- arraylake-1.5.0/arraylake/repos/iceberg/__init__.py +3 -0
- arraylake-1.5.0/arraylake/repos/iceberg/auth.py +54 -0
- arraylake-1.5.0/arraylake/repos/iceberg/catalog.py +52 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/types.py +2 -2
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/virtual.py +28 -4
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/types.py +111 -16
- {arraylake-1.3.0 → arraylake-1.5.0}/pyproject.toml +7 -5
- arraylake-1.5.0/tests/compute/test_cli_table.py +33 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/test_virtual.py +26 -1
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_api_utils.py +73 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_client.py +120 -1
- arraylake-1.5.0/tests/test_iceberg_client.py +419 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_subscriptions.py +4 -4
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_virtual_chunks.py +81 -24
- {arraylake-1.3.0 → arraylake-1.5.0}/uv.lock +741 -605
- arraylake-1.3.0/arraylake/compute/doctor.py +0 -266
- arraylake-1.3.0/tests/compute/test_doctor.py +0 -173
- {arraylake-1.3.0 → arraylake-1.5.0}/.gitignore +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/README.md +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/__main__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/_credential_cache.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/asyn.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/auth.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/config.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/org.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/skills.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/utils.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/compute/http_client.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/config.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/config.yaml +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/credentials.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/diagnostics.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/display/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/display/repo.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/display/repolist.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/exceptions.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/log_util.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/metastore/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/metastore/abc.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/obstore.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/py.typed +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/storage.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/token.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/tuning.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/compute/test_client.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/config.yaml +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/conftest.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/__init__.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/conftest.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/test_storage.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_asyn.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_cli.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_config.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_credential_cache.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_credentials.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_diagnostics.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_http_metastore.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_obstore.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_performance.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_skills.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_token_handler.py +0 -0
- {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: arraylake
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: Python client for ArrayLake
|
|
5
5
|
Project-URL: Documentation, https://docs.earthmover.io/
|
|
6
6
|
Project-URL: Changelog, https://docs.earthmover.io/changelog
|
|
@@ -14,7 +14,7 @@ Requires-Python: <3.15,>=3.12
|
|
|
14
14
|
Requires-Dist: click<9.0,>=8.1.3
|
|
15
15
|
Requires-Dist: donfig<1.0,>=0.7
|
|
16
16
|
Requires-Dist: httpx<0.29,>=0.23
|
|
17
|
-
Requires-Dist: icechunk<3.0.0,>=2.
|
|
17
|
+
Requires-Dist: icechunk<3.0.0,>=2.2.2
|
|
18
18
|
Requires-Dist: numpy<3.0,>=1.23
|
|
19
19
|
Requires-Dist: packaging<27.0,>=23.0
|
|
20
20
|
Requires-Dist: pydantic[email]<3.0,>=2.9
|
|
@@ -24,15 +24,15 @@ Requires-Dist: ruamel-yaml<1.0,>=0.17
|
|
|
24
24
|
Requires-Dist: structlog<27.0,>=24.1
|
|
25
25
|
Requires-Dist: typer<1.0,>=0.12.0
|
|
26
26
|
Requires-Dist: zarr<4.0,>=3.1.0
|
|
27
|
+
Provides-Extra: iceberg
|
|
28
|
+
Requires-Dist: pyiceberg<0.13,>=0.11; extra == 'iceberg'
|
|
27
29
|
Provides-Extra: maximal
|
|
28
|
-
Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
|
|
29
30
|
Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
|
|
30
31
|
Provides-Extra: obstore
|
|
31
32
|
Requires-Dist: obstore>=0.9.2; extra == 'obstore'
|
|
32
33
|
Provides-Extra: openeo
|
|
33
|
-
Requires-Dist: openeo<0.
|
|
34
|
+
Requires-Dist: openeo<0.52,>=0.35.0; extra == 'openeo'
|
|
34
35
|
Provides-Extra: xarray
|
|
35
|
-
Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
|
|
36
36
|
Requires-Dist: xarray>=2024.10.0; extra == 'xarray'
|
|
37
37
|
Description-Content-Type: text/markdown
|
|
38
38
|
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1,
|
|
21
|
+
__version__ = version = '1.5.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 5, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -135,10 +135,20 @@ class ClientKey:
|
|
|
135
135
|
# the global cache of httpx clients
|
|
136
136
|
_GLOBAL_CLIENTS: dict[ClientKey, httpx.AsyncClient] = {}
|
|
137
137
|
|
|
138
|
+
# one lock per event loop, so concurrent callers on a cold cache build one client, not two
|
|
139
|
+
_CLIENT_CREATION_LOCKS: dict[asyncio.AbstractEventLoop, asyncio.Lock] = {}
|
|
140
|
+
|
|
138
141
|
# this is a cache to use hold asyncio tasks so they are not garbage collected before finishing
|
|
139
142
|
background_tasks: set[asyncio.Task] = set()
|
|
140
143
|
|
|
141
144
|
|
|
145
|
+
def _creation_lock(loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
|
|
146
|
+
lock = _CLIENT_CREATION_LOCKS.get(loop)
|
|
147
|
+
if lock is None:
|
|
148
|
+
lock = _CLIENT_CREATION_LOCKS[loop] = asyncio.Lock()
|
|
149
|
+
return lock
|
|
150
|
+
|
|
151
|
+
|
|
142
152
|
async def get_client(
|
|
143
153
|
loop: asyncio.AbstractEventLoop,
|
|
144
154
|
api_url: str,
|
|
@@ -181,7 +191,13 @@ async def get_client(
|
|
|
181
191
|
proxy,
|
|
182
192
|
)
|
|
183
193
|
await logger.adebug("%i httpx clients present in cache.", len(_GLOBAL_CLIENTS))
|
|
184
|
-
if key not
|
|
194
|
+
if (client := _GLOBAL_CLIENTS.get(key)) is not None:
|
|
195
|
+
await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
|
|
196
|
+
return client
|
|
197
|
+
|
|
198
|
+
async with _creation_lock(loop):
|
|
199
|
+
if (client := _GLOBAL_CLIENTS.get(key)) is not None:
|
|
200
|
+
return client
|
|
185
201
|
await logger.adebug("Creating new httpx client %s. Loop id %s.", key, id(loop))
|
|
186
202
|
transport = AsyncRetryTransport(
|
|
187
203
|
max_keepalive_connections=max_keepalive_connections,
|
|
@@ -199,15 +215,12 @@ async def get_client(
|
|
|
199
215
|
verify=verify_ssl,
|
|
200
216
|
cert=ssl_cafile,
|
|
201
217
|
).__aenter__()
|
|
202
|
-
weakref.finalize(client, close_client, key)
|
|
218
|
+
weakref.finalize(client, close_client, key, id(client))
|
|
203
219
|
_GLOBAL_CLIENTS[key] = client
|
|
204
|
-
|
|
205
|
-
await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
|
|
206
|
-
assert key in _GLOBAL_CLIENTS
|
|
207
|
-
return _GLOBAL_CLIENTS[key]
|
|
220
|
+
return client
|
|
208
221
|
|
|
209
222
|
|
|
210
|
-
def close_client(key: ClientKey):
|
|
223
|
+
def close_client(key: ClientKey, client_id: int) -> None:
|
|
211
224
|
"""
|
|
212
225
|
This is a finalizer function that is called when a global client is
|
|
213
226
|
garbage collected. It cleanly closes the client for the specified key.
|
|
@@ -224,11 +237,13 @@ def close_client(key: ClientKey):
|
|
|
224
237
|
Note: logging in this function runs the risk of conflicting with pytest#5502. For
|
|
225
238
|
this reason, we have removed debug log statements.
|
|
226
239
|
"""
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
if
|
|
240
|
+
cached = _GLOBAL_CLIENTS.get(key)
|
|
241
|
+
# A newer client can own this key. Only the client that registered this finalizer may evict it.
|
|
242
|
+
if cached is None or id(cached) != client_id:
|
|
230
243
|
return
|
|
231
244
|
|
|
245
|
+
client = _GLOBAL_CLIENTS.pop(key)
|
|
246
|
+
|
|
232
247
|
if client.is_closed:
|
|
233
248
|
return
|
|
234
249
|
|
|
@@ -28,7 +28,16 @@ def _services_table(services, org):
|
|
|
28
28
|
table.add_column("URL", justify="right", style="green", min_width=10)
|
|
29
29
|
table.add_column("Status", justify="right", style="green", min_width=10)
|
|
30
30
|
|
|
31
|
-
status_colors = {
|
|
31
|
+
status_colors = {
|
|
32
|
+
"available": "green",
|
|
33
|
+
"progressing": "yellow",
|
|
34
|
+
"error": "red",
|
|
35
|
+
"quota_exceeded": "red",
|
|
36
|
+
"sleeping": "cyan",
|
|
37
|
+
"stopped": "grey",
|
|
38
|
+
"suspended": "grey",
|
|
39
|
+
"unknown": "grey",
|
|
40
|
+
}
|
|
32
41
|
|
|
33
42
|
for service in services:
|
|
34
43
|
table.add_row(
|
|
@@ -38,7 +47,9 @@ def _services_table(services, org):
|
|
|
38
47
|
str(service.config.is_public),
|
|
39
48
|
service.url,
|
|
40
49
|
service.status,
|
|
41
|
-
|
|
50
|
+
# .get, because the API owns this vocabulary and adds to it: an older client must
|
|
51
|
+
# render a status it has never heard of, not refuse to list anything at all.
|
|
52
|
+
style=status_colors.get(service.status, "grey"),
|
|
42
53
|
)
|
|
43
54
|
|
|
44
55
|
return table
|
|
@@ -47,7 +58,7 @@ def _services_table(services, org):
|
|
|
47
58
|
@app.command(name="enable")
|
|
48
59
|
def enable_service(
|
|
49
60
|
org: str = typer.Argument(..., help="The organization to enable the service for"),
|
|
50
|
-
protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr"),
|
|
61
|
+
protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr, openeo, sql"),
|
|
51
62
|
sku_code: str = typer.Option(..., "--sku-code", "-s", help="The compute shape to run the service on, e.g. xs"),
|
|
52
63
|
is_public: bool = typer.Option(False, help="Whether the service is public"),
|
|
53
64
|
deployment_name: str | None = typer.Option(
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Namespace administration and table discovery for Arraylake's Iceberg catalog."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from enum import StrEnum
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from arraylake import Client
|
|
12
|
+
from arraylake.cli.utils import rich_console
|
|
13
|
+
from arraylake.types import RepoVisibility
|
|
14
|
+
|
|
15
|
+
app = typer.Typer(help="Manage Iceberg namespaces and discover tables", no_args_is_help=True)
|
|
16
|
+
namespace_app = typer.Typer(help="Manage Iceberg namespaces", no_args_is_help=True)
|
|
17
|
+
table_app = typer.Typer(help="Discover and describe Iceberg tables", no_args_is_help=True)
|
|
18
|
+
app.add_typer(namespace_app, name="namespace")
|
|
19
|
+
app.add_typer(table_app, name="table")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Output(StrEnum):
|
|
23
|
+
rich = "rich"
|
|
24
|
+
json = "json"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class NamespaceSort(StrEnum):
|
|
28
|
+
relevance = "relevance"
|
|
29
|
+
updated = "updated"
|
|
30
|
+
created = "created"
|
|
31
|
+
name = "name"
|
|
32
|
+
tables = "tables"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TableSort(StrEnum):
|
|
36
|
+
relevance = "relevance"
|
|
37
|
+
updated = "updated"
|
|
38
|
+
created = "created"
|
|
39
|
+
name = "name"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Direction(StrEnum):
|
|
43
|
+
asc = "asc"
|
|
44
|
+
desc = "desc"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _table_identifier(namespace: str, table: str) -> str:
|
|
48
|
+
"""Format exact names; quoted components escape double quotes by doubling them."""
|
|
49
|
+
|
|
50
|
+
def quote(name: str) -> str:
|
|
51
|
+
return name if re.fullmatch(r"[A-Za-z0-9_-]+", name) else '"' + name.replace('"', '""') + '"'
|
|
52
|
+
|
|
53
|
+
return f"{quote(namespace)}.{quote(table)}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _parse_table_identifier(identifier: str) -> tuple[str, str]:
|
|
57
|
+
# Keep this grammar in sync with the app's iceberg-identifier.ts formatter.
|
|
58
|
+
component = r'(?:[A-Za-z0-9_-]+|"(?:[^"]|"")+")'
|
|
59
|
+
match = re.fullmatch(rf"({component})\.({component})", identifier)
|
|
60
|
+
if match is None:
|
|
61
|
+
raise typer.BadParameter(
|
|
62
|
+
"Expected NAMESPACE.TABLE. Double-quote names containing dots or special characters, "
|
|
63
|
+
"and wrap the identifier in single quotes in your shell, e.g. '\"namespace.v2\".table'.",
|
|
64
|
+
param_hint="IDENTIFIER",
|
|
65
|
+
)
|
|
66
|
+
namespace, table = (part[1:-1].replace('""', '"') if part.startswith('"') else part for part in match.groups())
|
|
67
|
+
return namespace, table
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _json_object(value: str | None, option: str) -> dict[str, Any] | None:
|
|
71
|
+
if value is None:
|
|
72
|
+
return None
|
|
73
|
+
try:
|
|
74
|
+
result = json.loads(value)
|
|
75
|
+
except ValueError as exc:
|
|
76
|
+
raise typer.BadParameter("Expected a JSON object", param_hint=option) from exc
|
|
77
|
+
if not isinstance(result, dict):
|
|
78
|
+
raise typer.BadParameter("Expected a JSON object", param_hint=option)
|
|
79
|
+
return result
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _show(value: Any, output: Output) -> None:
|
|
83
|
+
if output == Output.json:
|
|
84
|
+
typer.echo(json.dumps(value))
|
|
85
|
+
else:
|
|
86
|
+
rich_console.print(value)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _show_list(rows: list[dict[str, Any]], output: Output, *, namespaces: bool) -> None:
|
|
90
|
+
if not namespaces:
|
|
91
|
+
rows = [{**row, "identifier": _table_identifier(row["namespace"], row["name"])} for row in rows]
|
|
92
|
+
if output == Output.json:
|
|
93
|
+
_show(rows, output)
|
|
94
|
+
return
|
|
95
|
+
table = Table("Name" if namespaces else "Identifier", "Updated", "Tables" if namespaces else "Storage override")
|
|
96
|
+
for row in rows:
|
|
97
|
+
table.add_row(
|
|
98
|
+
row["name"] if namespaces else row["identifier"],
|
|
99
|
+
row["updated"],
|
|
100
|
+
str(row["table_count"] if namespaces else row["has_storage_override"]),
|
|
101
|
+
)
|
|
102
|
+
rich_console.print(table)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@namespace_app.command("list")
|
|
106
|
+
def list_namespaces(
|
|
107
|
+
org: str,
|
|
108
|
+
search: str | None = None,
|
|
109
|
+
filter_metadata: str | None = typer.Option(None, help="JSON metadata filter"),
|
|
110
|
+
sort: NamespaceSort | None = None,
|
|
111
|
+
direction: Direction = Direction.desc,
|
|
112
|
+
include_ghosts: bool = False,
|
|
113
|
+
page_size: int = typer.Option(50, min=1, max=100),
|
|
114
|
+
output: Output = Output.rich,
|
|
115
|
+
):
|
|
116
|
+
"""List namespaces, with optional search, metadata filters, and sorting."""
|
|
117
|
+
rows = Client().list_iceberg_namespaces_paginated(
|
|
118
|
+
org,
|
|
119
|
+
search=search,
|
|
120
|
+
filter_metadata=_json_object(filter_metadata, "--filter-metadata"),
|
|
121
|
+
sort=sort.value if sort else None,
|
|
122
|
+
direction=direction.value,
|
|
123
|
+
include_ghosts=include_ghosts,
|
|
124
|
+
page_size=page_size,
|
|
125
|
+
)
|
|
126
|
+
_show_list([row.model_dump(mode="json") for row in rows], output, namespaces=True)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@namespace_app.command("get")
|
|
130
|
+
def get_namespace(org: str, name: str, output: Output = Output.rich):
|
|
131
|
+
"""Show namespace storage, description, metadata, and visibility."""
|
|
132
|
+
_show(Client().get_iceberg_namespace(org, name).model_dump(mode="json"), output)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@namespace_app.command("create")
|
|
136
|
+
def create_namespace(
|
|
137
|
+
org: str,
|
|
138
|
+
name: str,
|
|
139
|
+
bucket_config_nickname: str | None = None,
|
|
140
|
+
description: str | None = None,
|
|
141
|
+
metadata: str | None = typer.Option(None, help="JSON discovery metadata"),
|
|
142
|
+
visibility: RepoVisibility = RepoVisibility.PRIVATE,
|
|
143
|
+
properties: str | None = typer.Option(None, help="JSON Iceberg properties (string values)"),
|
|
144
|
+
output: Output = Output.rich,
|
|
145
|
+
):
|
|
146
|
+
"""Create a namespace on the chosen or default bucket."""
|
|
147
|
+
props = _json_object(properties, "--properties")
|
|
148
|
+
if props is not None and any(not isinstance(v, str) for v in props.values()):
|
|
149
|
+
raise typer.BadParameter("Property values must be strings", param_hint="--properties")
|
|
150
|
+
result = Client().create_iceberg_namespace(
|
|
151
|
+
org,
|
|
152
|
+
name,
|
|
153
|
+
bucket_config_nickname=bucket_config_nickname,
|
|
154
|
+
description=description,
|
|
155
|
+
metadata=_json_object(metadata, "--metadata"),
|
|
156
|
+
visibility=visibility,
|
|
157
|
+
properties=props,
|
|
158
|
+
)
|
|
159
|
+
_show(result.model_dump(mode="json"), output)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@namespace_app.command("update")
|
|
163
|
+
def update_namespace(
|
|
164
|
+
org: str,
|
|
165
|
+
name: str,
|
|
166
|
+
description: str | None = None,
|
|
167
|
+
add_metadata: str | None = typer.Option(None, help="JSON metadata to add"),
|
|
168
|
+
remove_metadata: list[str] | None = typer.Option(None, help="Metadata key to remove; repeat for multiple keys"),
|
|
169
|
+
update_metadata: str | None = typer.Option(None, help="JSON metadata to update"),
|
|
170
|
+
visibility: RepoVisibility | None = None,
|
|
171
|
+
output: Output = Output.rich,
|
|
172
|
+
):
|
|
173
|
+
"""Update description, discovery metadata, or visibility. Use --description '' to clear."""
|
|
174
|
+
result = Client().modify_iceberg_namespace(
|
|
175
|
+
org,
|
|
176
|
+
name,
|
|
177
|
+
description=description,
|
|
178
|
+
add_metadata=_json_object(add_metadata, "--add-metadata"),
|
|
179
|
+
remove_metadata=remove_metadata,
|
|
180
|
+
update_metadata=_json_object(update_metadata, "--update-metadata"),
|
|
181
|
+
visibility=visibility,
|
|
182
|
+
)
|
|
183
|
+
_show(result.model_dump(mode="json"), output)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@namespace_app.command("delete")
|
|
187
|
+
def delete_namespace(
|
|
188
|
+
org: str,
|
|
189
|
+
name: str,
|
|
190
|
+
confirm: bool = typer.Option(False, "--confirm", help="Skip the confirmation prompt"),
|
|
191
|
+
immediate: bool = typer.Option(
|
|
192
|
+
False, "--immediate", help="Skip the recovery period: the namespace cannot be restored and its name is reusable at once"
|
|
193
|
+
),
|
|
194
|
+
retain_data: bool = typer.Option(False, "--retain-data", help="Leave the bytes under the namespace's storage prefix in the bucket"),
|
|
195
|
+
):
|
|
196
|
+
"""Delete an empty namespace. A soft-delete by default: restore it during its recovery period."""
|
|
197
|
+
if not confirm:
|
|
198
|
+
verb = "Permanently delete" if immediate else "Delete"
|
|
199
|
+
typer.confirm(f"{verb} namespace {org}/{name}?", abort=True)
|
|
200
|
+
Client().delete_iceberg_namespace(org, name, imsure=True, immediate=immediate, retain_data=retain_data)
|
|
201
|
+
typer.echo(f"Deleted {org}/{name}")
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@namespace_app.command("restore")
|
|
205
|
+
def restore_namespace(org: str, name: str):
|
|
206
|
+
"""Restore a soft-deleted namespace."""
|
|
207
|
+
Client().restore_iceberg_namespace(org, name)
|
|
208
|
+
typer.echo(f"Restored {org}/{name}")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@table_app.command("list")
|
|
212
|
+
def list_tables(
|
|
213
|
+
org: str,
|
|
214
|
+
namespace: str,
|
|
215
|
+
search: str | None = None,
|
|
216
|
+
sort: TableSort | None = None,
|
|
217
|
+
direction: Direction = Direction.desc,
|
|
218
|
+
page_size: int = typer.Option(50, min=1, max=100),
|
|
219
|
+
output: Output = Output.rich,
|
|
220
|
+
):
|
|
221
|
+
"""List table summaries without fetching table metadata or recording accesses."""
|
|
222
|
+
rows = Client().list_iceberg_tables_paginated(
|
|
223
|
+
org,
|
|
224
|
+
namespace,
|
|
225
|
+
search=search,
|
|
226
|
+
sort=sort.value if sort else None,
|
|
227
|
+
direction=direction.value,
|
|
228
|
+
page_size=page_size,
|
|
229
|
+
)
|
|
230
|
+
_show_list([row.model_dump(mode="json") for row in rows], output, namespaces=False)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@table_app.command("describe")
|
|
234
|
+
def describe_table(
|
|
235
|
+
org: str,
|
|
236
|
+
identifier: str = typer.Argument(..., help="NAMESPACE.TABLE; double-quote components containing dots or special characters."),
|
|
237
|
+
output: Output = Output.rich,
|
|
238
|
+
):
|
|
239
|
+
"""Load a table's schema, snapshots, partitioning, and properties. Requires arraylake[iceberg]."""
|
|
240
|
+
namespace, table = _parse_table_identifier(identifier)
|
|
241
|
+
catalog = Client().get_iceberg(org)
|
|
242
|
+
try:
|
|
243
|
+
loaded = catalog.load_table((namespace, table))
|
|
244
|
+
_show(
|
|
245
|
+
{
|
|
246
|
+
"name": table,
|
|
247
|
+
"namespace": namespace,
|
|
248
|
+
"identifier": _table_identifier(namespace, table),
|
|
249
|
+
"metadata_location": loaded.metadata_location,
|
|
250
|
+
"metadata": loaded.metadata.model_dump(mode="json", by_alias=True),
|
|
251
|
+
},
|
|
252
|
+
output,
|
|
253
|
+
)
|
|
254
|
+
finally:
|
|
255
|
+
catalog.close()
|
|
@@ -6,13 +6,19 @@ from arraylake.api_utils import set_client_name
|
|
|
6
6
|
from arraylake.cli.auth import auth
|
|
7
7
|
from arraylake.cli.compute import app as compute_app
|
|
8
8
|
from arraylake.cli.config import app as config_app
|
|
9
|
+
from arraylake.cli.iceberg import app as iceberg_app
|
|
9
10
|
from arraylake.cli.org import app as org_app
|
|
10
11
|
from arraylake.cli.repo import app as repo_app
|
|
11
12
|
from arraylake.cli.skills import app as skills_app
|
|
12
13
|
from arraylake.cli.utils import rich_console
|
|
13
14
|
from arraylake.diagnostics import get_diagnostics
|
|
14
15
|
|
|
15
|
-
app = typer.Typer(
|
|
16
|
+
app = typer.Typer(
|
|
17
|
+
no_args_is_help=True,
|
|
18
|
+
context_settings={"help_option_names": ["-h", "--help"]},
|
|
19
|
+
rich_markup_mode="markdown",
|
|
20
|
+
pretty_exceptions_show_locals=False,
|
|
21
|
+
)
|
|
16
22
|
|
|
17
23
|
|
|
18
24
|
def version_callback(value: bool):
|
|
@@ -48,6 +54,7 @@ def main(
|
|
|
48
54
|
app.add_typer(auth, name="auth")
|
|
49
55
|
app.add_typer(org_app, name="org")
|
|
50
56
|
app.add_typer(repo_app, name="repo")
|
|
57
|
+
app.add_typer(iceberg_app, name="iceberg")
|
|
51
58
|
app.add_typer(config_app, name="config")
|
|
52
59
|
app.add_typer(compute_app, name="compute")
|
|
53
60
|
app.add_typer(skills_app, name="skills")
|
|
@@ -1,31 +1,19 @@
|
|
|
1
1
|
import json
|
|
2
|
-
import warnings
|
|
3
2
|
from enum import Enum, StrEnum
|
|
4
|
-
from typing import TYPE_CHECKING, Optional
|
|
5
3
|
from uuid import UUID
|
|
6
4
|
|
|
7
5
|
import typer
|
|
8
6
|
import zarr
|
|
9
7
|
from pydantic import SecretStr
|
|
10
8
|
from rich import print_json
|
|
11
|
-
from rich.prompt import Confirm
|
|
12
9
|
from rich.table import Table
|
|
13
10
|
from rich.text import Text
|
|
14
11
|
|
|
15
|
-
from arraylake import AsyncClient
|
|
12
|
+
from arraylake import AsyncClient
|
|
16
13
|
from arraylake.cli.utils import coro, rich_console, simple_progress
|
|
17
|
-
from arraylake.compute.doctor import (
|
|
18
|
-
CFDiagnosisSummary,
|
|
19
|
-
check_cf_attribute_completeness,
|
|
20
|
-
fix_cf_noncompliance,
|
|
21
|
-
)
|
|
22
14
|
from arraylake.log_util import get_logger
|
|
23
15
|
from arraylake.types import RepoOperationMode
|
|
24
16
|
|
|
25
|
-
if TYPE_CHECKING:
|
|
26
|
-
import xarray as xr # noqa: F401
|
|
27
|
-
from xarray import Dataset
|
|
28
|
-
|
|
29
17
|
app = typer.Typer(help="Manage Arraylake repositories", no_args_is_help=True)
|
|
30
18
|
logger = get_logger(__name__)
|
|
31
19
|
|
|
@@ -141,7 +129,9 @@ async def list_repos(
|
|
|
141
129
|
async def create(
|
|
142
130
|
repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
|
|
143
131
|
bucket_config_nickname: str | None = typer.Option(None, help="Bucket config nickname"),
|
|
144
|
-
prefix: str = typer.Option(
|
|
132
|
+
prefix: str = typer.Option(
|
|
133
|
+
None, help="Optional prefix for the repo's storage. If not provided, a random ID + the repo name will be used."
|
|
134
|
+
),
|
|
145
135
|
description: str | None = typer.Option(None, help="Description of the repo"),
|
|
146
136
|
metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
|
|
147
137
|
):
|
|
@@ -315,152 +305,6 @@ async def tree(
|
|
|
315
305
|
print(_tree)
|
|
316
306
|
|
|
317
307
|
|
|
318
|
-
def diagnose_dataset(ds: "Dataset") -> CFDiagnosisSummary:
|
|
319
|
-
"""Checks if a dataset is CF compliant."""
|
|
320
|
-
rich_console.print("[bold]Diagnosing dataset...[/bold] :stethoscope:")
|
|
321
|
-
# TODO: pause the spinner while this is running
|
|
322
|
-
diagnosis: CFDiagnosisSummary = check_cf_attribute_completeness(ds)
|
|
323
|
-
|
|
324
|
-
rich_console.print("[bold]Checking compatibilty with compute services...[/bold] :stethoscope:")
|
|
325
|
-
if diagnosis.compatible_services:
|
|
326
|
-
for service in diagnosis.compatible_services:
|
|
327
|
-
rich_console.print(f" :white_check_mark: {service}")
|
|
328
|
-
if diagnosis.incompatible_services:
|
|
329
|
-
for service in diagnosis.incompatible_services:
|
|
330
|
-
rich_console.print(f" :x: {service}")
|
|
331
|
-
|
|
332
|
-
if diagnosis.is_healthy:
|
|
333
|
-
# Dataset is already compliant
|
|
334
|
-
rich_console.print("Dataset is healthy! :apple:")
|
|
335
|
-
else:
|
|
336
|
-
rich_console.print("[bold]Dataset is unhealthy[/bold] :face_with_thermometer:")
|
|
337
|
-
rich_console.print("[bold]Diagnosis:[/bold] :woman_health_worker::clipboard:")
|
|
338
|
-
if diagnosis.missing_required_keys:
|
|
339
|
-
rich_console.print(" Missing CF Attributes:")
|
|
340
|
-
for missing_key in diagnosis.sorted_missing_keys:
|
|
341
|
-
missing_required = " (required)" if missing_key.required else ""
|
|
342
|
-
color = "default"
|
|
343
|
-
if missing_key.required:
|
|
344
|
-
color = "red"
|
|
345
|
-
elif missing_key.proposed_variable:
|
|
346
|
-
color = "yellow"
|
|
347
|
-
rich_console.print(f" [bold][{color}]- {missing_key.attr_name}: {missing_key.name}{missing_required}[/{color}][/bold]")
|
|
348
|
-
if missing_key.proposed_variable:
|
|
349
|
-
rich_console.print(f" Possible variable: {missing_key.proposed_variable}")
|
|
350
|
-
if diagnosis.has_invalid_keys:
|
|
351
|
-
rich_console.print(" Invalid CF Attributes:")
|
|
352
|
-
for invalid_key in diagnosis.invalid:
|
|
353
|
-
rich_console.print(f" [bold][red]- {invalid_key.attr_name}: {invalid_key.name}: {invalid_key.issue}[/red][/bold]")
|
|
354
|
-
return diagnosis
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
def treat_dataset(diagnosis: CFDiagnosisSummary, store, group) -> tuple[CFDiagnosisSummary, Optional["Dataset"]]:
|
|
358
|
-
"""Treats a dataset to make it CF compliant."""
|
|
359
|
-
if diagnosis.is_healthy:
|
|
360
|
-
rich_console.print("Dataset does not need to be treated :muscle:")
|
|
361
|
-
return diagnosis, None
|
|
362
|
-
else:
|
|
363
|
-
rich_console.print("[bold]Treating dataset... [/bold]:ambulance:")
|
|
364
|
-
fix_cf_noncompliance(diagnosis, store, group)
|
|
365
|
-
# Read the data back in
|
|
366
|
-
import xarray as xr # noqa: F811
|
|
367
|
-
|
|
368
|
-
with warnings.catch_warnings():
|
|
369
|
-
warnings.simplefilter("ignore")
|
|
370
|
-
treated_ds = xr.open_zarr(store, zarr_format=3, group=group, consolidated=False)
|
|
371
|
-
# Run the compliance check again to see if we actually fixed it
|
|
372
|
-
new_diagnosis = check_cf_attribute_completeness(treated_ds)
|
|
373
|
-
if not new_diagnosis.is_healthy:
|
|
374
|
-
raise ValueError(
|
|
375
|
-
"[bold]Failed to treat dataset![/bold] :coffin: \n "
|
|
376
|
-
"Please use the diagnosis to manually update the necessary attributes :hammer:"
|
|
377
|
-
)
|
|
378
|
-
return new_diagnosis, treated_ds
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
def get_treatment_plan(og_ds: "Dataset", treated_ds: "Dataset"):
|
|
382
|
-
"""Prints the treatment plan for a dataset by comparing variable attrs."""
|
|
383
|
-
all_vars = set(og_ds.variables).union(treated_ds.variables)
|
|
384
|
-
|
|
385
|
-
for var in all_vars:
|
|
386
|
-
og_attrs = og_ds[var].attrs if var in og_ds else {}
|
|
387
|
-
treated_attrs = treated_ds[var].attrs if var in treated_ds else {}
|
|
388
|
-
|
|
389
|
-
if og_attrs != treated_attrs:
|
|
390
|
-
rich_console.print(f"\n [bold]Treatment plan for variable '{var}' metadata: [/bold]")
|
|
391
|
-
|
|
392
|
-
for key in set(og_attrs.keys()).union(treated_attrs.keys()):
|
|
393
|
-
og_val = og_attrs.get(key, "MISSING")
|
|
394
|
-
new_val = treated_attrs.get(key, "MISSING")
|
|
395
|
-
|
|
396
|
-
if og_val != new_val:
|
|
397
|
-
color = "green" if key in treated_attrs else "red"
|
|
398
|
-
rich_console.print(f" - [bold][{color}]{key}:[/{color}][/bold]")
|
|
399
|
-
rich_console.print(f" [bold]Original:[/bold] [red]{og_val}[/red]")
|
|
400
|
-
rich_console.print(f" [bold]Treated:[/bold] [green]{new_val}[/green]")
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
@app.command(name="doctor")
|
|
404
|
-
@coro # type: ignore
|
|
405
|
-
async def doctor_dataset(
|
|
406
|
-
repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
|
|
407
|
-
group: str | None = typer.Option(None, help="The path to the zarr group"),
|
|
408
|
-
treat: bool = typer.Option(default=False, hidden=True, help="Fix and commit the changes automatically"),
|
|
409
|
-
dry_run: bool = typer.Option(default=False, hidden=True, help="Run the doctor without committing changes"),
|
|
410
|
-
):
|
|
411
|
-
"""**Doctor** a dataset to make it usable by the Arraylake compute engine.
|
|
412
|
-
|
|
413
|
-
**Examples**
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
"""
|
|
417
|
-
if not treat and dry_run:
|
|
418
|
-
rich_console.print("[bold][yellow]Warning: [/bold][/yellow] Treat must be enabled for dry run mode.")
|
|
419
|
-
|
|
420
|
-
group_str = group if group is not None else ""
|
|
421
|
-
rich_console.print(f"Admitting dataset [bold]{repo_name}/{group_str}[/bold] to the Arraylake hospital :hospital:")
|
|
422
|
-
client = Client()
|
|
423
|
-
repo = client.get_repo(repo_name)
|
|
424
|
-
|
|
425
|
-
session = repo.writable_session(branch="main") if treat else repo.readonly_session(branch="main")
|
|
426
|
-
store = session.store
|
|
427
|
-
|
|
428
|
-
# with ekg_progress(""):
|
|
429
|
-
import xarray as xr # noqa: F811
|
|
430
|
-
|
|
431
|
-
with warnings.catch_warnings():
|
|
432
|
-
warnings.simplefilter("ignore")
|
|
433
|
-
ds = xr.open_zarr(store, zarr_format=3, group=group, consolidated=False)
|
|
434
|
-
diagnosis = diagnose_dataset(ds)
|
|
435
|
-
|
|
436
|
-
if treat:
|
|
437
|
-
updated_diagnosis, treated_ds = treat_dataset(diagnosis, store, group=group)
|
|
438
|
-
# If the dataset was already healthy, treatment is not needed
|
|
439
|
-
if treated_ds is None:
|
|
440
|
-
return
|
|
441
|
-
# Check that the only the dataset was modified using session status
|
|
442
|
-
diff = session.status()
|
|
443
|
-
assert len(diff.new_groups) == 0
|
|
444
|
-
assert len(diff.new_arrays) == 0
|
|
445
|
-
assert len(diff.deleted_groups) == 0
|
|
446
|
-
assert len(diff.deleted_arrays) == 0
|
|
447
|
-
assert len(diff.updated_chunks) == 0
|
|
448
|
-
assert diff.updated_arrays # type: ignore[unused-ignore,attr-defined]
|
|
449
|
-
rich_console.print("[bold]Proposed treatment:[/bold] :pill:")
|
|
450
|
-
get_treatment_plan(ds, treated_ds)
|
|
451
|
-
|
|
452
|
-
if updated_diagnosis.is_healthy and not dry_run:
|
|
453
|
-
lets_commit = Confirm.ask("Do you want to commit the proposed changes?")
|
|
454
|
-
if lets_commit:
|
|
455
|
-
rich_console.print("[bold]Committing changes...[/bold] :sparkles:")
|
|
456
|
-
message = "Updated metadata attributes for use with Arraylake Flux using repo doctor"
|
|
457
|
-
snapshot_id = session.commit(message)
|
|
458
|
-
rich_console.print(f"Dataset successfully treated and committed to the repo with ID {snapshot_id} :dizzy:")
|
|
459
|
-
else:
|
|
460
|
-
if not diagnosis.is_healthy:
|
|
461
|
-
rich_console.print("\nTo manually treat the dataset, use the diagnosis provided above to update the dataset metadata")
|
|
462
|
-
|
|
463
|
-
|
|
464
308
|
@app.command(hidden=True)
|
|
465
309
|
@coro # type: ignore
|
|
466
310
|
async def get_status(
|