arraylake 1.2.0__tar.gz → 1.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {arraylake-1.2.0 → arraylake-1.5.0}/PKG-INFO +7 -7
  2. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/_version.py +2 -2
  3. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/api_utils.py +52 -10
  4. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/auth.py +39 -12
  5. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/compute.py +56 -6
  6. arraylake-1.5.0/arraylake/cli/iceberg.py +255 -0
  7. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/main.py +13 -1
  8. arraylake-1.5.0/arraylake/cli/org.py +41 -0
  9. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/repo.py +4 -160
  10. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/client.py +752 -113
  11. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/compute/http_client.py +13 -1
  12. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/compute/services.py +53 -5
  13. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/compute/types.py +65 -34
  14. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/display/repo.py +3 -1
  15. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/metastore/http_metastore.py +174 -11
  16. arraylake-1.5.0/arraylake/obstore.py +132 -0
  17. arraylake-1.5.0/arraylake/repos/iceberg/__init__.py +3 -0
  18. arraylake-1.5.0/arraylake/repos/iceberg/auth.py +54 -0
  19. arraylake-1.5.0/arraylake/repos/iceberg/catalog.py +52 -0
  20. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/types.py +20 -2
  21. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/virtual.py +53 -2
  22. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/token.py +24 -2
  23. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/types.py +364 -52
  24. {arraylake-1.2.0 → arraylake-1.5.0}/pyproject.toml +8 -6
  25. arraylake-1.5.0/tests/compute/test_cli_table.py +33 -0
  26. {arraylake-1.2.0 → arraylake-1.5.0}/tests/compute/test_client.py +33 -0
  27. {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/test_virtual.py +26 -1
  28. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_api_utils.py +98 -0
  29. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_cli.py +106 -1
  30. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_client.py +124 -2
  31. arraylake-1.5.0/tests/test_http_metastore.py +97 -0
  32. arraylake-1.5.0/tests/test_iceberg_client.py +419 -0
  33. arraylake-1.5.0/tests/test_obstore.py +182 -0
  34. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_performance.py +4 -14
  35. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_subscriptions.py +4 -4
  36. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_token_handler.py +44 -1
  37. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_types.py +246 -0
  38. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_virtual_chunks.py +134 -63
  39. {arraylake-1.2.0 → arraylake-1.5.0}/uv.lock +943 -571
  40. arraylake-1.2.0/arraylake/compute/doctor.py +0 -266
  41. arraylake-1.2.0/tests/compute/test_doctor.py +0 -173
  42. {arraylake-1.2.0 → arraylake-1.5.0}/.gitignore +0 -0
  43. {arraylake-1.2.0 → arraylake-1.5.0}/README.md +0 -0
  44. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/__init__.py +0 -0
  45. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/__main__.py +0 -0
  46. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/_credential_cache.py +0 -0
  47. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/asyn.py +0 -0
  48. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/__init__.py +0 -0
  49. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/config.py +0 -0
  50. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/skills.py +0 -0
  51. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/cli/utils.py +0 -0
  52. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/config.py +0 -0
  53. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/config.yaml +0 -0
  54. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/credentials.py +0 -0
  55. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/diagnostics.py +0 -0
  56. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/display/__init__.py +0 -0
  57. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/display/repolist.py +0 -0
  58. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/exceptions.py +0 -0
  59. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/log_util.py +0 -0
  60. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/metastore/__init__.py +0 -0
  61. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/metastore/abc.py +0 -0
  62. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/py.typed +0 -0
  63. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/__init__.py +0 -0
  64. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/__init__.py +0 -0
  65. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/repos/icechunk/storage.py +0 -0
  66. {arraylake-1.2.0 → arraylake-1.5.0}/arraylake/tuning.py +0 -0
  67. {arraylake-1.2.0 → arraylake-1.5.0}/tests/__init__.py +0 -0
  68. {arraylake-1.2.0 → arraylake-1.5.0}/tests/config.yaml +0 -0
  69. {arraylake-1.2.0 → arraylake-1.5.0}/tests/conftest.py +0 -0
  70. {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/__init__.py +0 -0
  71. {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/conftest.py +0 -0
  72. {arraylake-1.2.0 → arraylake-1.5.0}/tests/icechunk/test_storage.py +0 -0
  73. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_asyn.py +0 -0
  74. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_config.py +0 -0
  75. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_credential_cache.py +0 -0
  76. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_credentials.py +0 -0
  77. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_diagnostics.py +0 -0
  78. {arraylake-1.2.0 → arraylake-1.5.0}/tests/test_skills.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: arraylake
3
- Version: 1.2.0
3
+ Version: 1.5.0
4
4
  Summary: Python client for ArrayLake
5
5
  Project-URL: Documentation, https://docs.earthmover.io/
6
6
  Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -14,25 +14,25 @@ Requires-Python: <3.15,>=3.12
14
14
  Requires-Dist: click<9.0,>=8.1.3
15
15
  Requires-Dist: donfig<1.0,>=0.7
16
16
  Requires-Dist: httpx<0.29,>=0.23
17
- Requires-Dist: icechunk<3.0.0,>=2.0.6
17
+ Requires-Dist: icechunk<3.0.0,>=2.2.2
18
18
  Requires-Dist: numpy<3.0,>=1.23
19
19
  Requires-Dist: packaging<27.0,>=23.0
20
20
  Requires-Dist: pydantic[email]<3.0,>=2.9
21
21
  Requires-Dist: python-dateutil<3.0,>=2.8
22
22
  Requires-Dist: rich<16.0,>=12.6
23
23
  Requires-Dist: ruamel-yaml<1.0,>=0.17
24
- Requires-Dist: structlog<26.0,>=24.1
24
+ Requires-Dist: structlog<27.0,>=24.1
25
25
  Requires-Dist: typer<1.0,>=0.12.0
26
26
  Requires-Dist: zarr<4.0,>=3.1.0
27
+ Provides-Extra: iceberg
28
+ Requires-Dist: pyiceberg<0.13,>=0.11; extra == 'iceberg'
27
29
  Provides-Extra: maximal
28
- Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
29
30
  Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
30
31
  Provides-Extra: obstore
31
32
  Requires-Dist: obstore>=0.9.2; extra == 'obstore'
32
33
  Provides-Extra: openeo
33
- Requires-Dist: openeo<0.40,>=0.35.0; extra == 'openeo'
34
+ Requires-Dist: openeo<0.52,>=0.35.0; extra == 'openeo'
34
35
  Provides-Extra: xarray
35
- Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
36
36
  Requires-Dist: xarray>=2024.10.0; extra == 'xarray'
37
37
  Description-Content-Type: text/markdown
38
38
 
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.2.0'
22
- __version_tuple__ = version_tuple = (1, 2, 0)
21
+ __version__ = version = '1.5.0'
22
+ __version_tuple__ = version_tuple = (1, 5, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -84,6 +84,25 @@ def _get_proxy_from_env() -> str | None:
84
84
  return None
85
85
 
86
86
 
87
+ # Client name reported in the X-Arraylake-Client header. Defaults to the Python
88
+ # library; the CLI entry point overrides it so server logs can tell them apart.
89
+ _client_name: str = "python"
90
+
91
+
92
+ def set_client_name(name: str) -> None:
93
+ """Override the client name sent in the ``X-Arraylake-Client`` header.
94
+
95
+ Must be called before any ``ArraylakeHttpClient`` is constructed to take effect,
96
+ since headers are baked in at construction time.
97
+ """
98
+ global _client_name
99
+ _client_name = name
100
+
101
+
102
+ def get_client_name() -> str:
103
+ return _client_name
104
+
105
+
87
106
  @functools.cache
88
107
  def _get_session_id() -> str:
89
108
  """Session ID for this Python process.
@@ -116,10 +135,20 @@ class ClientKey:
116
135
  # the global cache of httpx clients
117
136
  _GLOBAL_CLIENTS: dict[ClientKey, httpx.AsyncClient] = {}
118
137
 
138
+ # one lock per event loop, so concurrent callers on a cold cache build one client, not two
139
+ _CLIENT_CREATION_LOCKS: dict[asyncio.AbstractEventLoop, asyncio.Lock] = {}
140
+
119
141
  # this is a cache to use hold asyncio tasks so they are not garbage collected before finishing
120
142
  background_tasks: set[asyncio.Task] = set()
121
143
 
122
144
 
145
+ def _creation_lock(loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
146
+ lock = _CLIENT_CREATION_LOCKS.get(loop)
147
+ if lock is None:
148
+ lock = _CLIENT_CREATION_LOCKS[loop] = asyncio.Lock()
149
+ return lock
150
+
151
+
123
152
  async def get_client(
124
153
  loop: asyncio.AbstractEventLoop,
125
154
  api_url: str,
@@ -162,7 +191,13 @@ async def get_client(
162
191
  proxy,
163
192
  )
164
193
  await logger.adebug("%i httpx clients present in cache.", len(_GLOBAL_CLIENTS))
165
- if key not in _GLOBAL_CLIENTS:
194
+ if (client := _GLOBAL_CLIENTS.get(key)) is not None:
195
+ await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
196
+ return client
197
+
198
+ async with _creation_lock(loop):
199
+ if (client := _GLOBAL_CLIENTS.get(key)) is not None:
200
+ return client
166
201
  await logger.adebug("Creating new httpx client %s. Loop id %s.", key, id(loop))
167
202
  transport = AsyncRetryTransport(
168
203
  max_keepalive_connections=max_keepalive_connections,
@@ -180,15 +215,12 @@ async def get_client(
180
215
  verify=verify_ssl,
181
216
  cert=ssl_cafile,
182
217
  ).__aenter__()
183
- weakref.finalize(client, close_client, key)
218
+ weakref.finalize(client, close_client, key, id(client))
184
219
  _GLOBAL_CLIENTS[key] = client
185
- else:
186
- await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
187
- assert key in _GLOBAL_CLIENTS
188
- return _GLOBAL_CLIENTS[key]
220
+ return client
189
221
 
190
222
 
191
- def close_client(key: ClientKey):
223
+ def close_client(key: ClientKey, client_id: int) -> None:
192
224
  """
193
225
  This is a finalizer function that is called when a global client is
194
226
  garbage collected. It cleanly closes the client for the specified key.
@@ -205,11 +237,13 @@ def close_client(key: ClientKey):
205
237
  Note: logging in this function runs the risk of conflicting with pytest#5502. For
206
238
  this reason, we have removed debug log statements.
207
239
  """
208
- client = _GLOBAL_CLIENTS.pop(key, None)
209
-
210
- if client is None:
240
+ cached = _GLOBAL_CLIENTS.get(key)
241
+ # A newer client can own this key. Only the client that registered this finalizer may evict it.
242
+ if cached is None or id(cached) != client_id:
211
243
  return
212
244
 
245
+ client = _GLOBAL_CLIENTS.pop(key)
246
+
213
247
  if client.is_closed:
214
248
  return
215
249
 
@@ -317,6 +351,9 @@ class ArraylakeHttpClient:
317
351
  # technically we don't enforce this, for now it's for debugging purposes
318
352
  "client-name": "arraylake-python-client",
319
353
  "client-version": arraylake.__version__,
354
+ "x-arraylake-client": f"{get_client_name()}/{arraylake.__version__}",
355
+ "x-arraylake-source": get_client_name(),
356
+ "user-agent": f"arraylake-{get_client_name()}/{arraylake.__version__}",
320
357
  "arraylake-session-id": _get_session_id(),
321
358
  }
322
359
 
@@ -448,6 +485,7 @@ def _exception_log_debug(request: httpx.Request, response: httpx.Response):
448
485
 
449
486
 
450
487
  _emitted_deprecation_warnings: set[str] = set()
488
+ _emitted_user_warnings: set[str] = set()
451
489
 
452
490
 
453
491
  def handle_response(response: httpx.Response):
@@ -456,6 +494,10 @@ def handle_response(response: httpx.Response):
456
494
  if deprecation_msg and deprecation_msg not in _emitted_deprecation_warnings:
457
495
  _emitted_deprecation_warnings.add(deprecation_msg)
458
496
  warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2)
497
+ warning_msg = response.headers.get("X-Arraylake-Warning") # Service-sent user-facing warnings
498
+ if warning_msg and warning_msg not in _emitted_user_warnings:
499
+ _emitted_user_warnings.add(warning_msg)
500
+ warnings.warn(warning_msg, UserWarning, stacklevel=2)
459
501
  try:
460
502
  response.raise_for_status()
461
503
  except httpx.RequestError as exc:
@@ -2,7 +2,6 @@ import typer
2
2
  from rich.align import Align
3
3
  from rich.panel import Panel
4
4
 
5
- from arraylake import config
6
5
  from arraylake.cli.utils import (
7
6
  coro,
8
7
  error_console,
@@ -17,11 +16,26 @@ auth = typer.Typer(help="Manage Arraylake authentication")
17
16
 
18
17
  NOT_LOGGED_IN_MESSAGE = "⚠️ Not logged in, please log in with: `arraylake auth login`"
19
18
 
19
+ SERVICE_URI_OPTION = typer.Option(
20
+ None,
21
+ "--service-uri",
22
+ help=(
23
+ "Arraylake API endpoint to act on, e.g. **https://api.dev.earthmover.io**. Defaults to the configured "
24
+ "**service.uri**. Tokens are stored per endpoint, so this need not be persisted to config — unless "
25
+ "**service.token_path** is set, which makes all endpoints share one token file."
26
+ ),
27
+ )
28
+
29
+
30
+ def _resolve_service_uri(service_uri: str | None) -> str:
31
+ return service_uri if service_uri else default_service_uri()
32
+
20
33
 
21
34
  @auth.command()
22
35
  @coro # type: ignore
23
36
  async def login(
24
37
  browser: bool = typer.Option(True, "--browser/--no-browser", " /--nobrowser", help="Whether to automatically open a browser window."),
38
+ service_uri: str | None = SERVICE_URI_OPTION,
25
39
  ):
26
40
  """**Log in** to Arraylake
27
41
 
@@ -34,17 +48,23 @@ async def login(
34
48
  ```
35
49
  $ arraylake auth login --no-browser
36
50
  ```
51
+
52
+ - Log in to a non-default Arraylake deployment
53
+
54
+ ```
55
+ $ arraylake auth login --service-uri https://api.dev.earthmover.io
56
+ ```
37
57
  """
38
58
  print_logo()
39
- handler = get_auth_handler(api_endpoint=default_service_uri())
59
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
40
60
  await handler.login(browser=browser)
41
61
 
42
62
 
43
63
  @auth.command()
44
64
  @coro # type: ignore
45
- async def refresh() -> None:
65
+ async def refresh(service_uri: str | None = SERVICE_URI_OPTION) -> None:
46
66
  """**Refresh** Arraylake's auth token"""
47
- handler = get_auth_handler(api_endpoint=default_service_uri())
67
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
48
68
  if handler.tokens is not None:
49
69
  try:
50
70
  await handler.refresh_token()
@@ -57,7 +77,7 @@ async def refresh() -> None:
57
77
 
58
78
  @auth.command()
59
79
  @coro # type: ignore
60
- async def logout() -> None:
80
+ async def logout(service_uri: str | None = SERVICE_URI_OPTION) -> None:
61
81
  """**Logout** of Arraylake
62
82
 
63
83
  **Examples**
@@ -67,9 +87,15 @@ async def logout() -> None:
67
87
  ```
68
88
  $ arraylake auth logout
69
89
  ```
90
+
91
+ - Log out of a non-default Arraylake deployment
92
+
93
+ ```
94
+ $ arraylake auth logout --service-uri https://api.dev.earthmover.io
95
+ ```
70
96
  """
71
97
  try:
72
- handler = get_auth_handler(api_endpoint=default_service_uri())
98
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
73
99
  await handler.logout()
74
100
  except AuthException as e:
75
101
  error_console.print(e)
@@ -77,14 +103,15 @@ async def logout() -> None:
77
103
 
78
104
 
79
105
  @auth.command()
80
- @coro
81
- async def status():
106
+ @coro # type: ignore
107
+ async def status(service_uri: str | None = SERVICE_URI_OPTION):
82
108
  """Verify and display information about your authentication **status**"""
109
+ endpoint = _resolve_service_uri(service_uri)
83
110
  print_logo_mini()
84
- rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {config.get('service.uri')}"), align="center"))
111
+ rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {endpoint}"), align="center"))
85
112
 
86
113
  try:
87
- handler = get_auth_handler(api_endpoint=default_service_uri())
114
+ handler = get_auth_handler(api_endpoint=endpoint)
88
115
  user = await handler._get_user() # checks that the new tokens are valid
89
116
  rich_console.print(Align(f"[green][bold]🔓 Logged in as {user.email}[/green][/bold]", align="center"))
90
117
  except AuthException:
@@ -93,9 +120,9 @@ async def status():
93
120
 
94
121
 
95
122
  @auth.command()
96
- def token():
123
+ def token(service_uri: str | None = SERVICE_URI_OPTION):
97
124
  """Display your authentication **token**"""
98
- handler = get_auth_handler(api_endpoint=default_service_uri())
125
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
99
126
  if handler.tokens is None:
100
127
  rich_console.print(Align(f"[yellow]{NOT_LOGGED_IN_MESSAGE}[/yellow]", align="center"))
101
128
  raise typer.Exit(code=1)
@@ -28,7 +28,16 @@ def _services_table(services, org):
28
28
  table.add_column("URL", justify="right", style="green", min_width=10)
29
29
  table.add_column("Status", justify="right", style="green", min_width=10)
30
30
 
31
- status_colors = {"available": "green", "progressing": "yellow", "error": "red", "unknown": "grey"}
31
+ status_colors = {
32
+ "available": "green",
33
+ "progressing": "yellow",
34
+ "error": "red",
35
+ "quota_exceeded": "red",
36
+ "sleeping": "cyan",
37
+ "stopped": "grey",
38
+ "suspended": "grey",
39
+ "unknown": "grey",
40
+ }
32
41
 
33
42
  for service in services:
34
43
  table.add_row(
@@ -38,7 +47,9 @@ def _services_table(services, org):
38
47
  str(service.config.is_public),
39
48
  service.url,
40
49
  service.status,
41
- style=status_colors[service.status],
50
+ # .get, because the API owns this vocabulary and adds to it: an older client must
51
+ # render a status it has never heard of, not refuse to list anything at all.
52
+ style=status_colors.get(service.status, "grey"),
42
53
  )
43
54
 
44
55
  return table
@@ -47,7 +58,8 @@ def _services_table(services, org):
47
58
  @app.command(name="enable")
48
59
  def enable_service(
49
60
  org: str = typer.Argument(..., help="The organization to enable the service for"),
50
- protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr"),
61
+ protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr, openeo, sql"),
62
+ sku_code: str = typer.Option(..., "--sku-code", "-s", help="The compute shape to run the service on, e.g. xs"),
51
63
  is_public: bool = typer.Option(False, help="Whether the service is public"),
52
64
  deployment_name: str | None = typer.Option(
53
65
  None,
@@ -63,13 +75,13 @@ def enable_service(
63
75
  - Enable a new DAP service for _my-org_
64
76
 
65
77
  ```
66
- $ arraylake compute enable my-org dap2
78
+ $ arraylake compute enable my-org dap2 --sku-code xs
67
79
  ```
68
80
 
69
81
  - Enable a tiles service with a custom deployment name
70
82
 
71
83
  ```
72
- $ arraylake compute enable my-org tiles --deployment-name my-tiles
84
+ $ arraylake compute enable my-org tiles --sku-code s --deployment-name my-tiles
73
85
  ```
74
86
  """
75
87
  kwargs = {}
@@ -77,7 +89,7 @@ def enable_service(
77
89
  kwargs["deployment_name"] = deployment_name
78
90
 
79
91
  with simple_progress(f"Creating [bold]{protocol}[/bold] service for [bold]{org}[/bold]..."):
80
- status = Client().get_services(org).enable(protocol, is_public, **kwargs)
92
+ status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, **kwargs)
81
93
  rich_console.print(status["message"])
82
94
 
83
95
 
@@ -183,6 +195,44 @@ async def get_service_logs(
183
195
  raise typer.Exit(code=1)
184
196
 
185
197
 
198
+ @app.command(name="update")
199
+ def update_service(
200
+ org: str = typer.Argument(..., help="The organization that the service belongs to"),
201
+ service_id: str = typer.Argument(..., help="The service ID to update"),
202
+ sku: str | None = typer.Option(None, "--sku", help="The compute shape (SKU code) to resize the service to"),
203
+ min_replicas: int | None = typer.Option(None, min=0, help="The minimum number of replicas the service scales down to"),
204
+ max_replicas: int | None = typer.Option(None, min=1, help="The maximum number of replicas the service scales up to"),
205
+ ):
206
+ """**Update** a running service's shape or scaling. Omitted options keep their current values.
207
+
208
+ **Examples**
209
+
210
+ - Resize a service with ID edr-1234567 to a different compute shape
211
+
212
+ ```
213
+ $ arraylake compute update my-org edr-1234567 --sku medium
214
+ ```
215
+
216
+ - Change the scaling bounds of a service
217
+
218
+ ```
219
+ $ arraylake compute update my-org edr-1234567 --min-replicas 2 --max-replicas 10
220
+ ```
221
+ """
222
+ if sku is None and min_replicas is None and max_replicas is None:
223
+ rich_console.print("[bold red]Nothing to update: pass at least one of --sku, --min-replicas, --max-replicas[/bold red]")
224
+ raise typer.Exit(code=1)
225
+
226
+ with simple_progress(f"Updating service [bold]{service_id}[/bold]..."):
227
+ deployment = Client().get_services(org).update(service_id, sku_code=sku, min_replicas=min_replicas, max_replicas=max_replicas)
228
+
229
+ config = deployment.config
230
+ rich_console.print(
231
+ f"Service [bold]{deployment.name}[/bold] updated: "
232
+ f"sku={config.sku_code}, min_replicas={config.min_replicas}, max_replicas={config.max_replicas}"
233
+ )
234
+
235
+
186
236
  @app.command(name="disable")
187
237
  def disable_service(
188
238
  org: str = typer.Argument(..., help="The organization that the service belongs to"),
@@ -0,0 +1,255 @@
1
+ """Namespace administration and table discovery for Arraylake's Iceberg catalog."""
2
+
3
+ import json
4
+ import re
5
+ from enum import StrEnum
6
+ from typing import Any
7
+
8
+ import typer
9
+ from rich.table import Table
10
+
11
+ from arraylake import Client
12
+ from arraylake.cli.utils import rich_console
13
+ from arraylake.types import RepoVisibility
14
+
15
+ app = typer.Typer(help="Manage Iceberg namespaces and discover tables", no_args_is_help=True)
16
+ namespace_app = typer.Typer(help="Manage Iceberg namespaces", no_args_is_help=True)
17
+ table_app = typer.Typer(help="Discover and describe Iceberg tables", no_args_is_help=True)
18
+ app.add_typer(namespace_app, name="namespace")
19
+ app.add_typer(table_app, name="table")
20
+
21
+
22
+ class Output(StrEnum):
23
+ rich = "rich"
24
+ json = "json"
25
+
26
+
27
+ class NamespaceSort(StrEnum):
28
+ relevance = "relevance"
29
+ updated = "updated"
30
+ created = "created"
31
+ name = "name"
32
+ tables = "tables"
33
+
34
+
35
+ class TableSort(StrEnum):
36
+ relevance = "relevance"
37
+ updated = "updated"
38
+ created = "created"
39
+ name = "name"
40
+
41
+
42
+ class Direction(StrEnum):
43
+ asc = "asc"
44
+ desc = "desc"
45
+
46
+
47
+ def _table_identifier(namespace: str, table: str) -> str:
48
+ """Format exact names; quoted components escape double quotes by doubling them."""
49
+
50
+ def quote(name: str) -> str:
51
+ return name if re.fullmatch(r"[A-Za-z0-9_-]+", name) else '"' + name.replace('"', '""') + '"'
52
+
53
+ return f"{quote(namespace)}.{quote(table)}"
54
+
55
+
56
+ def _parse_table_identifier(identifier: str) -> tuple[str, str]:
57
+ # Keep this grammar in sync with the app's iceberg-identifier.ts formatter.
58
+ component = r'(?:[A-Za-z0-9_-]+|"(?:[^"]|"")+")'
59
+ match = re.fullmatch(rf"({component})\.({component})", identifier)
60
+ if match is None:
61
+ raise typer.BadParameter(
62
+ "Expected NAMESPACE.TABLE. Double-quote names containing dots or special characters, "
63
+ "and wrap the identifier in single quotes in your shell, e.g. '\"namespace.v2\".table'.",
64
+ param_hint="IDENTIFIER",
65
+ )
66
+ namespace, table = (part[1:-1].replace('""', '"') if part.startswith('"') else part for part in match.groups())
67
+ return namespace, table
68
+
69
+
70
+ def _json_object(value: str | None, option: str) -> dict[str, Any] | None:
71
+ if value is None:
72
+ return None
73
+ try:
74
+ result = json.loads(value)
75
+ except ValueError as exc:
76
+ raise typer.BadParameter("Expected a JSON object", param_hint=option) from exc
77
+ if not isinstance(result, dict):
78
+ raise typer.BadParameter("Expected a JSON object", param_hint=option)
79
+ return result
80
+
81
+
82
+ def _show(value: Any, output: Output) -> None:
83
+ if output == Output.json:
84
+ typer.echo(json.dumps(value))
85
+ else:
86
+ rich_console.print(value)
87
+
88
+
89
+ def _show_list(rows: list[dict[str, Any]], output: Output, *, namespaces: bool) -> None:
90
+ if not namespaces:
91
+ rows = [{**row, "identifier": _table_identifier(row["namespace"], row["name"])} for row in rows]
92
+ if output == Output.json:
93
+ _show(rows, output)
94
+ return
95
+ table = Table("Name" if namespaces else "Identifier", "Updated", "Tables" if namespaces else "Storage override")
96
+ for row in rows:
97
+ table.add_row(
98
+ row["name"] if namespaces else row["identifier"],
99
+ row["updated"],
100
+ str(row["table_count"] if namespaces else row["has_storage_override"]),
101
+ )
102
+ rich_console.print(table)
103
+
104
+
105
+ @namespace_app.command("list")
106
+ def list_namespaces(
107
+ org: str,
108
+ search: str | None = None,
109
+ filter_metadata: str | None = typer.Option(None, help="JSON metadata filter"),
110
+ sort: NamespaceSort | None = None,
111
+ direction: Direction = Direction.desc,
112
+ include_ghosts: bool = False,
113
+ page_size: int = typer.Option(50, min=1, max=100),
114
+ output: Output = Output.rich,
115
+ ):
116
+ """List namespaces, with optional search, metadata filters, and sorting."""
117
+ rows = Client().list_iceberg_namespaces_paginated(
118
+ org,
119
+ search=search,
120
+ filter_metadata=_json_object(filter_metadata, "--filter-metadata"),
121
+ sort=sort.value if sort else None,
122
+ direction=direction.value,
123
+ include_ghosts=include_ghosts,
124
+ page_size=page_size,
125
+ )
126
+ _show_list([row.model_dump(mode="json") for row in rows], output, namespaces=True)
127
+
128
+
129
+ @namespace_app.command("get")
130
+ def get_namespace(org: str, name: str, output: Output = Output.rich):
131
+ """Show namespace storage, description, metadata, and visibility."""
132
+ _show(Client().get_iceberg_namespace(org, name).model_dump(mode="json"), output)
133
+
134
+
135
+ @namespace_app.command("create")
136
+ def create_namespace(
137
+ org: str,
138
+ name: str,
139
+ bucket_config_nickname: str | None = None,
140
+ description: str | None = None,
141
+ metadata: str | None = typer.Option(None, help="JSON discovery metadata"),
142
+ visibility: RepoVisibility = RepoVisibility.PRIVATE,
143
+ properties: str | None = typer.Option(None, help="JSON Iceberg properties (string values)"),
144
+ output: Output = Output.rich,
145
+ ):
146
+ """Create a namespace on the chosen or default bucket."""
147
+ props = _json_object(properties, "--properties")
148
+ if props is not None and any(not isinstance(v, str) for v in props.values()):
149
+ raise typer.BadParameter("Property values must be strings", param_hint="--properties")
150
+ result = Client().create_iceberg_namespace(
151
+ org,
152
+ name,
153
+ bucket_config_nickname=bucket_config_nickname,
154
+ description=description,
155
+ metadata=_json_object(metadata, "--metadata"),
156
+ visibility=visibility,
157
+ properties=props,
158
+ )
159
+ _show(result.model_dump(mode="json"), output)
160
+
161
+
162
+ @namespace_app.command("update")
163
+ def update_namespace(
164
+ org: str,
165
+ name: str,
166
+ description: str | None = None,
167
+ add_metadata: str | None = typer.Option(None, help="JSON metadata to add"),
168
+ remove_metadata: list[str] | None = typer.Option(None, help="Metadata key to remove; repeat for multiple keys"),
169
+ update_metadata: str | None = typer.Option(None, help="JSON metadata to update"),
170
+ visibility: RepoVisibility | None = None,
171
+ output: Output = Output.rich,
172
+ ):
173
+ """Update description, discovery metadata, or visibility. Use --description '' to clear."""
174
+ result = Client().modify_iceberg_namespace(
175
+ org,
176
+ name,
177
+ description=description,
178
+ add_metadata=_json_object(add_metadata, "--add-metadata"),
179
+ remove_metadata=remove_metadata,
180
+ update_metadata=_json_object(update_metadata, "--update-metadata"),
181
+ visibility=visibility,
182
+ )
183
+ _show(result.model_dump(mode="json"), output)
184
+
185
+
186
+ @namespace_app.command("delete")
187
+ def delete_namespace(
188
+ org: str,
189
+ name: str,
190
+ confirm: bool = typer.Option(False, "--confirm", help="Skip the confirmation prompt"),
191
+ immediate: bool = typer.Option(
192
+ False, "--immediate", help="Skip the recovery period: the namespace cannot be restored and its name is reusable at once"
193
+ ),
194
+ retain_data: bool = typer.Option(False, "--retain-data", help="Leave the bytes under the namespace's storage prefix in the bucket"),
195
+ ):
196
+ """Delete an empty namespace. A soft-delete by default: restore it during its recovery period."""
197
+ if not confirm:
198
+ verb = "Permanently delete" if immediate else "Delete"
199
+ typer.confirm(f"{verb} namespace {org}/{name}?", abort=True)
200
+ Client().delete_iceberg_namespace(org, name, imsure=True, immediate=immediate, retain_data=retain_data)
201
+ typer.echo(f"Deleted {org}/{name}")
202
+
203
+
204
+ @namespace_app.command("restore")
205
+ def restore_namespace(org: str, name: str):
206
+ """Restore a soft-deleted namespace."""
207
+ Client().restore_iceberg_namespace(org, name)
208
+ typer.echo(f"Restored {org}/{name}")
209
+
210
+
211
+ @table_app.command("list")
212
+ def list_tables(
213
+ org: str,
214
+ namespace: str,
215
+ search: str | None = None,
216
+ sort: TableSort | None = None,
217
+ direction: Direction = Direction.desc,
218
+ page_size: int = typer.Option(50, min=1, max=100),
219
+ output: Output = Output.rich,
220
+ ):
221
+ """List table summaries without fetching table metadata or recording accesses."""
222
+ rows = Client().list_iceberg_tables_paginated(
223
+ org,
224
+ namespace,
225
+ search=search,
226
+ sort=sort.value if sort else None,
227
+ direction=direction.value,
228
+ page_size=page_size,
229
+ )
230
+ _show_list([row.model_dump(mode="json") for row in rows], output, namespaces=False)
231
+
232
+
233
+ @table_app.command("describe")
234
+ def describe_table(
235
+ org: str,
236
+ identifier: str = typer.Argument(..., help="NAMESPACE.TABLE; double-quote components containing dots or special characters."),
237
+ output: Output = Output.rich,
238
+ ):
239
+ """Load a table's schema, snapshots, partitioning, and properties. Requires arraylake[iceberg]."""
240
+ namespace, table = _parse_table_identifier(identifier)
241
+ catalog = Client().get_iceberg(org)
242
+ try:
243
+ loaded = catalog.load_table((namespace, table))
244
+ _show(
245
+ {
246
+ "name": table,
247
+ "namespace": namespace,
248
+ "identifier": _table_identifier(namespace, table),
249
+ "metadata_location": loaded.metadata_location,
250
+ "metadata": loaded.metadata.model_dump(mode="json", by_alias=True),
251
+ },
252
+ output,
253
+ )
254
+ finally:
255
+ catalog.close()