arraylake 1.3.0__tar.gz → 1.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {arraylake-1.3.0 → arraylake-1.5.0}/PKG-INFO +6 -6
  2. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/_version.py +2 -2
  3. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/api_utils.py +25 -10
  4. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/compute.py +14 -3
  5. arraylake-1.5.0/arraylake/cli/iceberg.py +255 -0
  6. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/main.py +8 -1
  7. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/repo.py +4 -160
  8. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/client.py +645 -41
  9. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/compute/services.py +1 -1
  10. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/compute/types.py +26 -13
  11. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/metastore/http_metastore.py +147 -2
  12. arraylake-1.5.0/arraylake/repos/iceberg/__init__.py +3 -0
  13. arraylake-1.5.0/arraylake/repos/iceberg/auth.py +54 -0
  14. arraylake-1.5.0/arraylake/repos/iceberg/catalog.py +52 -0
  15. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/types.py +2 -2
  16. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/virtual.py +28 -4
  17. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/types.py +111 -16
  18. {arraylake-1.3.0 → arraylake-1.5.0}/pyproject.toml +7 -5
  19. arraylake-1.5.0/tests/compute/test_cli_table.py +33 -0
  20. {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/test_virtual.py +26 -1
  21. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_api_utils.py +73 -0
  22. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_client.py +120 -1
  23. arraylake-1.5.0/tests/test_iceberg_client.py +419 -0
  24. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_subscriptions.py +4 -4
  25. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_virtual_chunks.py +81 -24
  26. {arraylake-1.3.0 → arraylake-1.5.0}/uv.lock +741 -605
  27. arraylake-1.3.0/arraylake/compute/doctor.py +0 -266
  28. arraylake-1.3.0/tests/compute/test_doctor.py +0 -173
  29. {arraylake-1.3.0 → arraylake-1.5.0}/.gitignore +0 -0
  30. {arraylake-1.3.0 → arraylake-1.5.0}/README.md +0 -0
  31. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/__init__.py +0 -0
  32. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/__main__.py +0 -0
  33. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/_credential_cache.py +0 -0
  34. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/asyn.py +0 -0
  35. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/__init__.py +0 -0
  36. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/auth.py +0 -0
  37. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/config.py +0 -0
  38. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/org.py +0 -0
  39. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/skills.py +0 -0
  40. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/cli/utils.py +0 -0
  41. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/compute/http_client.py +0 -0
  42. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/config.py +0 -0
  43. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/config.yaml +0 -0
  44. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/credentials.py +0 -0
  45. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/diagnostics.py +0 -0
  46. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/display/__init__.py +0 -0
  47. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/display/repo.py +0 -0
  48. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/display/repolist.py +0 -0
  49. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/exceptions.py +0 -0
  50. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/log_util.py +0 -0
  51. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/metastore/__init__.py +0 -0
  52. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/metastore/abc.py +0 -0
  53. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/obstore.py +0 -0
  54. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/py.typed +0 -0
  55. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/__init__.py +0 -0
  56. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/__init__.py +0 -0
  57. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/repos/icechunk/storage.py +0 -0
  58. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/token.py +0 -0
  59. {arraylake-1.3.0 → arraylake-1.5.0}/arraylake/tuning.py +0 -0
  60. {arraylake-1.3.0 → arraylake-1.5.0}/tests/__init__.py +0 -0
  61. {arraylake-1.3.0 → arraylake-1.5.0}/tests/compute/test_client.py +0 -0
  62. {arraylake-1.3.0 → arraylake-1.5.0}/tests/config.yaml +0 -0
  63. {arraylake-1.3.0 → arraylake-1.5.0}/tests/conftest.py +0 -0
  64. {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/__init__.py +0 -0
  65. {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/conftest.py +0 -0
  66. {arraylake-1.3.0 → arraylake-1.5.0}/tests/icechunk/test_storage.py +0 -0
  67. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_asyn.py +0 -0
  68. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_cli.py +0 -0
  69. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_config.py +0 -0
  70. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_credential_cache.py +0 -0
  71. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_credentials.py +0 -0
  72. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_diagnostics.py +0 -0
  73. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_http_metastore.py +0 -0
  74. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_obstore.py +0 -0
  75. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_performance.py +0 -0
  76. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_skills.py +0 -0
  77. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_token_handler.py +0 -0
  78. {arraylake-1.3.0 → arraylake-1.5.0}/tests/test_types.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: arraylake
3
- Version: 1.3.0
3
+ Version: 1.5.0
4
4
  Summary: Python client for ArrayLake
5
5
  Project-URL: Documentation, https://docs.earthmover.io/
6
6
  Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -14,7 +14,7 @@ Requires-Python: <3.15,>=3.12
14
14
  Requires-Dist: click<9.0,>=8.1.3
15
15
  Requires-Dist: donfig<1.0,>=0.7
16
16
  Requires-Dist: httpx<0.29,>=0.23
17
- Requires-Dist: icechunk<3.0.0,>=2.1.2
17
+ Requires-Dist: icechunk<3.0.0,>=2.2.2
18
18
  Requires-Dist: numpy<3.0,>=1.23
19
19
  Requires-Dist: packaging<27.0,>=23.0
20
20
  Requires-Dist: pydantic[email]<3.0,>=2.9
@@ -24,15 +24,15 @@ Requires-Dist: ruamel-yaml<1.0,>=0.17
24
24
  Requires-Dist: structlog<27.0,>=24.1
25
25
  Requires-Dist: typer<1.0,>=0.12.0
26
26
  Requires-Dist: zarr<4.0,>=3.1.0
27
+ Provides-Extra: iceberg
28
+ Requires-Dist: pyiceberg<0.13,>=0.11; extra == 'iceberg'
27
29
  Provides-Extra: maximal
28
- Requires-Dist: cf-xarray>=0.10.4; extra == 'maximal'
29
30
  Requires-Dist: xarray[accel,etc,io,parallel]>=2024.10.0; extra == 'maximal'
30
31
  Provides-Extra: obstore
31
32
  Requires-Dist: obstore>=0.9.2; extra == 'obstore'
32
33
  Provides-Extra: openeo
33
- Requires-Dist: openeo<0.40,>=0.35.0; extra == 'openeo'
34
+ Requires-Dist: openeo<0.52,>=0.35.0; extra == 'openeo'
34
35
  Provides-Extra: xarray
35
- Requires-Dist: cf-xarray>=0.10.4; extra == 'xarray'
36
36
  Requires-Dist: xarray>=2024.10.0; extra == 'xarray'
37
37
  Description-Content-Type: text/markdown
38
38
 
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.3.0'
22
- __version_tuple__ = version_tuple = (1, 3, 0)
21
+ __version__ = version = '1.5.0'
22
+ __version_tuple__ = version_tuple = (1, 5, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -135,10 +135,20 @@ class ClientKey:
135
135
  # the global cache of httpx clients
136
136
  _GLOBAL_CLIENTS: dict[ClientKey, httpx.AsyncClient] = {}
137
137
 
138
+ # one lock per event loop, so concurrent callers on a cold cache build one client, not two
139
+ _CLIENT_CREATION_LOCKS: dict[asyncio.AbstractEventLoop, asyncio.Lock] = {}
140
+
138
141
  # this is a cache to use hold asyncio tasks so they are not garbage collected before finishing
139
142
  background_tasks: set[asyncio.Task] = set()
140
143
 
141
144
 
145
+ def _creation_lock(loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
146
+ lock = _CLIENT_CREATION_LOCKS.get(loop)
147
+ if lock is None:
148
+ lock = _CLIENT_CREATION_LOCKS[loop] = asyncio.Lock()
149
+ return lock
150
+
151
+
142
152
  async def get_client(
143
153
  loop: asyncio.AbstractEventLoop,
144
154
  api_url: str,
@@ -181,7 +191,13 @@ async def get_client(
181
191
  proxy,
182
192
  )
183
193
  await logger.adebug("%i httpx clients present in cache.", len(_GLOBAL_CLIENTS))
184
- if key not in _GLOBAL_CLIENTS:
194
+ if (client := _GLOBAL_CLIENTS.get(key)) is not None:
195
+ await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
196
+ return client
197
+
198
+ async with _creation_lock(loop):
199
+ if (client := _GLOBAL_CLIENTS.get(key)) is not None:
200
+ return client
185
201
  await logger.adebug("Creating new httpx client %s. Loop id %s.", key, id(loop))
186
202
  transport = AsyncRetryTransport(
187
203
  max_keepalive_connections=max_keepalive_connections,
@@ -199,15 +215,12 @@ async def get_client(
199
215
  verify=verify_ssl,
200
216
  cert=ssl_cafile,
201
217
  ).__aenter__()
202
- weakref.finalize(client, close_client, key)
218
+ weakref.finalize(client, close_client, key, id(client))
203
219
  _GLOBAL_CLIENTS[key] = client
204
- else:
205
- await logger.adebug("Client already present %s. Loop id %s.", key, id(loop))
206
- assert key in _GLOBAL_CLIENTS
207
- return _GLOBAL_CLIENTS[key]
220
+ return client
208
221
 
209
222
 
210
- def close_client(key: ClientKey):
223
+ def close_client(key: ClientKey, client_id: int) -> None:
211
224
  """
212
225
  This is a finalizer function that is called when a global client is
213
226
  garbage collected. It cleanly closes the client for the specified key.
@@ -224,11 +237,13 @@ def close_client(key: ClientKey):
224
237
  Note: logging in this function runs the risk of conflicting with pytest#5502. For
225
238
  this reason, we have removed debug log statements.
226
239
  """
227
- client = _GLOBAL_CLIENTS.pop(key, None)
228
-
229
- if client is None:
240
+ cached = _GLOBAL_CLIENTS.get(key)
241
+ # A newer client can own this key. Only the client that registered this finalizer may evict it.
242
+ if cached is None or id(cached) != client_id:
230
243
  return
231
244
 
245
+ client = _GLOBAL_CLIENTS.pop(key)
246
+
232
247
  if client.is_closed:
233
248
  return
234
249
 
@@ -28,7 +28,16 @@ def _services_table(services, org):
28
28
  table.add_column("URL", justify="right", style="green", min_width=10)
29
29
  table.add_column("Status", justify="right", style="green", min_width=10)
30
30
 
31
- status_colors = {"available": "green", "progressing": "yellow", "error": "red", "unknown": "grey"}
31
+ status_colors = {
32
+ "available": "green",
33
+ "progressing": "yellow",
34
+ "error": "red",
35
+ "quota_exceeded": "red",
36
+ "sleeping": "cyan",
37
+ "stopped": "grey",
38
+ "suspended": "grey",
39
+ "unknown": "grey",
40
+ }
32
41
 
33
42
  for service in services:
34
43
  table.add_row(
@@ -38,7 +47,9 @@ def _services_table(services, org):
38
47
  str(service.config.is_public),
39
48
  service.url,
40
49
  service.status,
41
- style=status_colors[service.status],
50
+ # .get, because the API owns this vocabulary and adds to it: an older client must
51
+ # render a status it has never heard of, not refuse to list anything at all.
52
+ style=status_colors.get(service.status, "grey"),
42
53
  )
43
54
 
44
55
  return table
@@ -47,7 +58,7 @@ def _services_table(services, org):
47
58
  @app.command(name="enable")
48
59
  def enable_service(
49
60
  org: str = typer.Argument(..., help="The organization to enable the service for"),
50
- protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr"),
61
+ protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr, openeo, sql"),
51
62
  sku_code: str = typer.Option(..., "--sku-code", "-s", help="The compute shape to run the service on, e.g. xs"),
52
63
  is_public: bool = typer.Option(False, help="Whether the service is public"),
53
64
  deployment_name: str | None = typer.Option(
@@ -0,0 +1,255 @@
1
+ """Namespace administration and table discovery for Arraylake's Iceberg catalog."""
2
+
3
+ import json
4
+ import re
5
+ from enum import StrEnum
6
+ from typing import Any
7
+
8
+ import typer
9
+ from rich.table import Table
10
+
11
+ from arraylake import Client
12
+ from arraylake.cli.utils import rich_console
13
+ from arraylake.types import RepoVisibility
14
+
15
+ app = typer.Typer(help="Manage Iceberg namespaces and discover tables", no_args_is_help=True)
16
+ namespace_app = typer.Typer(help="Manage Iceberg namespaces", no_args_is_help=True)
17
+ table_app = typer.Typer(help="Discover and describe Iceberg tables", no_args_is_help=True)
18
+ app.add_typer(namespace_app, name="namespace")
19
+ app.add_typer(table_app, name="table")
20
+
21
+
22
+ class Output(StrEnum):
23
+ rich = "rich"
24
+ json = "json"
25
+
26
+
27
+ class NamespaceSort(StrEnum):
28
+ relevance = "relevance"
29
+ updated = "updated"
30
+ created = "created"
31
+ name = "name"
32
+ tables = "tables"
33
+
34
+
35
+ class TableSort(StrEnum):
36
+ relevance = "relevance"
37
+ updated = "updated"
38
+ created = "created"
39
+ name = "name"
40
+
41
+
42
+ class Direction(StrEnum):
43
+ asc = "asc"
44
+ desc = "desc"
45
+
46
+
47
+ def _table_identifier(namespace: str, table: str) -> str:
48
+ """Format exact names; quoted components escape double quotes by doubling them."""
49
+
50
+ def quote(name: str) -> str:
51
+ return name if re.fullmatch(r"[A-Za-z0-9_-]+", name) else '"' + name.replace('"', '""') + '"'
52
+
53
+ return f"{quote(namespace)}.{quote(table)}"
54
+
55
+
56
+ def _parse_table_identifier(identifier: str) -> tuple[str, str]:
57
+ # Keep this grammar in sync with the app's iceberg-identifier.ts formatter.
58
+ component = r'(?:[A-Za-z0-9_-]+|"(?:[^"]|"")+")'
59
+ match = re.fullmatch(rf"({component})\.({component})", identifier)
60
+ if match is None:
61
+ raise typer.BadParameter(
62
+ "Expected NAMESPACE.TABLE. Double-quote names containing dots or special characters, "
63
+ "and wrap the identifier in single quotes in your shell, e.g. '\"namespace.v2\".table'.",
64
+ param_hint="IDENTIFIER",
65
+ )
66
+ namespace, table = (part[1:-1].replace('""', '"') if part.startswith('"') else part for part in match.groups())
67
+ return namespace, table
68
+
69
+
70
+ def _json_object(value: str | None, option: str) -> dict[str, Any] | None:
71
+ if value is None:
72
+ return None
73
+ try:
74
+ result = json.loads(value)
75
+ except ValueError as exc:
76
+ raise typer.BadParameter("Expected a JSON object", param_hint=option) from exc
77
+ if not isinstance(result, dict):
78
+ raise typer.BadParameter("Expected a JSON object", param_hint=option)
79
+ return result
80
+
81
+
82
+ def _show(value: Any, output: Output) -> None:
83
+ if output == Output.json:
84
+ typer.echo(json.dumps(value))
85
+ else:
86
+ rich_console.print(value)
87
+
88
+
89
+ def _show_list(rows: list[dict[str, Any]], output: Output, *, namespaces: bool) -> None:
90
+ if not namespaces:
91
+ rows = [{**row, "identifier": _table_identifier(row["namespace"], row["name"])} for row in rows]
92
+ if output == Output.json:
93
+ _show(rows, output)
94
+ return
95
+ table = Table("Name" if namespaces else "Identifier", "Updated", "Tables" if namespaces else "Storage override")
96
+ for row in rows:
97
+ table.add_row(
98
+ row["name"] if namespaces else row["identifier"],
99
+ row["updated"],
100
+ str(row["table_count"] if namespaces else row["has_storage_override"]),
101
+ )
102
+ rich_console.print(table)
103
+
104
+
105
+ @namespace_app.command("list")
106
+ def list_namespaces(
107
+ org: str,
108
+ search: str | None = None,
109
+ filter_metadata: str | None = typer.Option(None, help="JSON metadata filter"),
110
+ sort: NamespaceSort | None = None,
111
+ direction: Direction = Direction.desc,
112
+ include_ghosts: bool = False,
113
+ page_size: int = typer.Option(50, min=1, max=100),
114
+ output: Output = Output.rich,
115
+ ):
116
+ """List namespaces, with optional search, metadata filters, and sorting."""
117
+ rows = Client().list_iceberg_namespaces_paginated(
118
+ org,
119
+ search=search,
120
+ filter_metadata=_json_object(filter_metadata, "--filter-metadata"),
121
+ sort=sort.value if sort else None,
122
+ direction=direction.value,
123
+ include_ghosts=include_ghosts,
124
+ page_size=page_size,
125
+ )
126
+ _show_list([row.model_dump(mode="json") for row in rows], output, namespaces=True)
127
+
128
+
129
+ @namespace_app.command("get")
130
+ def get_namespace(org: str, name: str, output: Output = Output.rich):
131
+ """Show namespace storage, description, metadata, and visibility."""
132
+ _show(Client().get_iceberg_namespace(org, name).model_dump(mode="json"), output)
133
+
134
+
135
+ @namespace_app.command("create")
136
+ def create_namespace(
137
+ org: str,
138
+ name: str,
139
+ bucket_config_nickname: str | None = None,
140
+ description: str | None = None,
141
+ metadata: str | None = typer.Option(None, help="JSON discovery metadata"),
142
+ visibility: RepoVisibility = RepoVisibility.PRIVATE,
143
+ properties: str | None = typer.Option(None, help="JSON Iceberg properties (string values)"),
144
+ output: Output = Output.rich,
145
+ ):
146
+ """Create a namespace on the chosen or default bucket."""
147
+ props = _json_object(properties, "--properties")
148
+ if props is not None and any(not isinstance(v, str) for v in props.values()):
149
+ raise typer.BadParameter("Property values must be strings", param_hint="--properties")
150
+ result = Client().create_iceberg_namespace(
151
+ org,
152
+ name,
153
+ bucket_config_nickname=bucket_config_nickname,
154
+ description=description,
155
+ metadata=_json_object(metadata, "--metadata"),
156
+ visibility=visibility,
157
+ properties=props,
158
+ )
159
+ _show(result.model_dump(mode="json"), output)
160
+
161
+
162
+ @namespace_app.command("update")
163
+ def update_namespace(
164
+ org: str,
165
+ name: str,
166
+ description: str | None = None,
167
+ add_metadata: str | None = typer.Option(None, help="JSON metadata to add"),
168
+ remove_metadata: list[str] | None = typer.Option(None, help="Metadata key to remove; repeat for multiple keys"),
169
+ update_metadata: str | None = typer.Option(None, help="JSON metadata to update"),
170
+ visibility: RepoVisibility | None = None,
171
+ output: Output = Output.rich,
172
+ ):
173
+ """Update description, discovery metadata, or visibility. Use --description '' to clear."""
174
+ result = Client().modify_iceberg_namespace(
175
+ org,
176
+ name,
177
+ description=description,
178
+ add_metadata=_json_object(add_metadata, "--add-metadata"),
179
+ remove_metadata=remove_metadata,
180
+ update_metadata=_json_object(update_metadata, "--update-metadata"),
181
+ visibility=visibility,
182
+ )
183
+ _show(result.model_dump(mode="json"), output)
184
+
185
+
186
+ @namespace_app.command("delete")
187
+ def delete_namespace(
188
+ org: str,
189
+ name: str,
190
+ confirm: bool = typer.Option(False, "--confirm", help="Skip the confirmation prompt"),
191
+ immediate: bool = typer.Option(
192
+ False, "--immediate", help="Skip the recovery period: the namespace cannot be restored and its name is reusable at once"
193
+ ),
194
+ retain_data: bool = typer.Option(False, "--retain-data", help="Leave the bytes under the namespace's storage prefix in the bucket"),
195
+ ):
196
+ """Delete an empty namespace. A soft-delete by default: restore it during its recovery period."""
197
+ if not confirm:
198
+ verb = "Permanently delete" if immediate else "Delete"
199
+ typer.confirm(f"{verb} namespace {org}/{name}?", abort=True)
200
+ Client().delete_iceberg_namespace(org, name, imsure=True, immediate=immediate, retain_data=retain_data)
201
+ typer.echo(f"Deleted {org}/{name}")
202
+
203
+
204
+ @namespace_app.command("restore")
205
+ def restore_namespace(org: str, name: str):
206
+ """Restore a soft-deleted namespace."""
207
+ Client().restore_iceberg_namespace(org, name)
208
+ typer.echo(f"Restored {org}/{name}")
209
+
210
+
211
+ @table_app.command("list")
212
+ def list_tables(
213
+ org: str,
214
+ namespace: str,
215
+ search: str | None = None,
216
+ sort: TableSort | None = None,
217
+ direction: Direction = Direction.desc,
218
+ page_size: int = typer.Option(50, min=1, max=100),
219
+ output: Output = Output.rich,
220
+ ):
221
+ """List table summaries without fetching table metadata or recording accesses."""
222
+ rows = Client().list_iceberg_tables_paginated(
223
+ org,
224
+ namespace,
225
+ search=search,
226
+ sort=sort.value if sort else None,
227
+ direction=direction.value,
228
+ page_size=page_size,
229
+ )
230
+ _show_list([row.model_dump(mode="json") for row in rows], output, namespaces=False)
231
+
232
+
233
+ @table_app.command("describe")
234
+ def describe_table(
235
+ org: str,
236
+ identifier: str = typer.Argument(..., help="NAMESPACE.TABLE; double-quote components containing dots or special characters."),
237
+ output: Output = Output.rich,
238
+ ):
239
+ """Load a table's schema, snapshots, partitioning, and properties. Requires arraylake[iceberg]."""
240
+ namespace, table = _parse_table_identifier(identifier)
241
+ catalog = Client().get_iceberg(org)
242
+ try:
243
+ loaded = catalog.load_table((namespace, table))
244
+ _show(
245
+ {
246
+ "name": table,
247
+ "namespace": namespace,
248
+ "identifier": _table_identifier(namespace, table),
249
+ "metadata_location": loaded.metadata_location,
250
+ "metadata": loaded.metadata.model_dump(mode="json", by_alias=True),
251
+ },
252
+ output,
253
+ )
254
+ finally:
255
+ catalog.close()
@@ -6,13 +6,19 @@ from arraylake.api_utils import set_client_name
6
6
  from arraylake.cli.auth import auth
7
7
  from arraylake.cli.compute import app as compute_app
8
8
  from arraylake.cli.config import app as config_app
9
+ from arraylake.cli.iceberg import app as iceberg_app
9
10
  from arraylake.cli.org import app as org_app
10
11
  from arraylake.cli.repo import app as repo_app
11
12
  from arraylake.cli.skills import app as skills_app
12
13
  from arraylake.cli.utils import rich_console
13
14
  from arraylake.diagnostics import get_diagnostics
14
15
 
15
- app = typer.Typer(no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]}, rich_markup_mode="markdown")
16
+ app = typer.Typer(
17
+ no_args_is_help=True,
18
+ context_settings={"help_option_names": ["-h", "--help"]},
19
+ rich_markup_mode="markdown",
20
+ pretty_exceptions_show_locals=False,
21
+ )
16
22
 
17
23
 
18
24
  def version_callback(value: bool):
@@ -48,6 +54,7 @@ def main(
48
54
  app.add_typer(auth, name="auth")
49
55
  app.add_typer(org_app, name="org")
50
56
  app.add_typer(repo_app, name="repo")
57
+ app.add_typer(iceberg_app, name="iceberg")
51
58
  app.add_typer(config_app, name="config")
52
59
  app.add_typer(compute_app, name="compute")
53
60
  app.add_typer(skills_app, name="skills")
@@ -1,31 +1,19 @@
1
1
  import json
2
- import warnings
3
2
  from enum import Enum, StrEnum
4
- from typing import TYPE_CHECKING, Optional
5
3
  from uuid import UUID
6
4
 
7
5
  import typer
8
6
  import zarr
9
7
  from pydantic import SecretStr
10
8
  from rich import print_json
11
- from rich.prompt import Confirm
12
9
  from rich.table import Table
13
10
  from rich.text import Text
14
11
 
15
- from arraylake import AsyncClient, Client
12
+ from arraylake import AsyncClient
16
13
  from arraylake.cli.utils import coro, rich_console, simple_progress
17
- from arraylake.compute.doctor import (
18
- CFDiagnosisSummary,
19
- check_cf_attribute_completeness,
20
- fix_cf_noncompliance,
21
- )
22
14
  from arraylake.log_util import get_logger
23
15
  from arraylake.types import RepoOperationMode
24
16
 
25
- if TYPE_CHECKING:
26
- import xarray as xr # noqa: F401
27
- from xarray import Dataset
28
-
29
17
  app = typer.Typer(help="Manage Arraylake repositories", no_args_is_help=True)
30
18
  logger = get_logger(__name__)
31
19
 
@@ -141,7 +129,9 @@ async def list_repos(
141
129
  async def create(
142
130
  repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
143
131
  bucket_config_nickname: str | None = typer.Option(None, help="Bucket config nickname"),
144
- prefix: str = typer.Option(None, help="Optional prefix for Icechunk repo. If not provided, a random ID + the repo name will be used."),
132
+ prefix: str = typer.Option(
133
+ None, help="Optional prefix for the repo's storage. If not provided, a random ID + the repo name will be used."
134
+ ),
145
135
  description: str | None = typer.Option(None, help="Description of the repo"),
146
136
  metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
147
137
  ):
@@ -315,152 +305,6 @@ async def tree(
315
305
  print(_tree)
316
306
 
317
307
 
318
- def diagnose_dataset(ds: "Dataset") -> CFDiagnosisSummary:
319
- """Checks if a dataset is CF compliant."""
320
- rich_console.print("[bold]Diagnosing dataset...[/bold] :stethoscope:")
321
- # TODO: pause the spinner while this is running
322
- diagnosis: CFDiagnosisSummary = check_cf_attribute_completeness(ds)
323
-
324
- rich_console.print("[bold]Checking compatibilty with compute services...[/bold] :stethoscope:")
325
- if diagnosis.compatible_services:
326
- for service in diagnosis.compatible_services:
327
- rich_console.print(f" :white_check_mark: {service}")
328
- if diagnosis.incompatible_services:
329
- for service in diagnosis.incompatible_services:
330
- rich_console.print(f" :x: {service}")
331
-
332
- if diagnosis.is_healthy:
333
- # Dataset is already compliant
334
- rich_console.print("Dataset is healthy! :apple:")
335
- else:
336
- rich_console.print("[bold]Dataset is unhealthy[/bold] :face_with_thermometer:")
337
- rich_console.print("[bold]Diagnosis:[/bold] :woman_health_worker::clipboard:")
338
- if diagnosis.missing_required_keys:
339
- rich_console.print(" Missing CF Attributes:")
340
- for missing_key in diagnosis.sorted_missing_keys:
341
- missing_required = " (required)" if missing_key.required else ""
342
- color = "default"
343
- if missing_key.required:
344
- color = "red"
345
- elif missing_key.proposed_variable:
346
- color = "yellow"
347
- rich_console.print(f" [bold][{color}]- {missing_key.attr_name}: {missing_key.name}{missing_required}[/{color}][/bold]")
348
- if missing_key.proposed_variable:
349
- rich_console.print(f" Possible variable: {missing_key.proposed_variable}")
350
- if diagnosis.has_invalid_keys:
351
- rich_console.print(" Invalid CF Attributes:")
352
- for invalid_key in diagnosis.invalid:
353
- rich_console.print(f" [bold][red]- {invalid_key.attr_name}: {invalid_key.name}: {invalid_key.issue}[/red][/bold]")
354
- return diagnosis
355
-
356
-
357
- def treat_dataset(diagnosis: CFDiagnosisSummary, store, group) -> tuple[CFDiagnosisSummary, Optional["Dataset"]]:
358
- """Treats a dataset to make it CF compliant."""
359
- if diagnosis.is_healthy:
360
- rich_console.print("Dataset does not need to be treated :muscle:")
361
- return diagnosis, None
362
- else:
363
- rich_console.print("[bold]Treating dataset... [/bold]:ambulance:")
364
- fix_cf_noncompliance(diagnosis, store, group)
365
- # Read the data back in
366
- import xarray as xr # noqa: F811
367
-
368
- with warnings.catch_warnings():
369
- warnings.simplefilter("ignore")
370
- treated_ds = xr.open_zarr(store, zarr_format=3, group=group, consolidated=False)
371
- # Run the compliance check again to see if we actually fixed it
372
- new_diagnosis = check_cf_attribute_completeness(treated_ds)
373
- if not new_diagnosis.is_healthy:
374
- raise ValueError(
375
- "[bold]Failed to treat dataset![/bold] :coffin: \n "
376
- "Please use the diagnosis to manually update the necessary attributes :hammer:"
377
- )
378
- return new_diagnosis, treated_ds
379
-
380
-
381
- def get_treatment_plan(og_ds: "Dataset", treated_ds: "Dataset"):
382
- """Prints the treatment plan for a dataset by comparing variable attrs."""
383
- all_vars = set(og_ds.variables).union(treated_ds.variables)
384
-
385
- for var in all_vars:
386
- og_attrs = og_ds[var].attrs if var in og_ds else {}
387
- treated_attrs = treated_ds[var].attrs if var in treated_ds else {}
388
-
389
- if og_attrs != treated_attrs:
390
- rich_console.print(f"\n [bold]Treatment plan for variable '{var}' metadata: [/bold]")
391
-
392
- for key in set(og_attrs.keys()).union(treated_attrs.keys()):
393
- og_val = og_attrs.get(key, "MISSING")
394
- new_val = treated_attrs.get(key, "MISSING")
395
-
396
- if og_val != new_val:
397
- color = "green" if key in treated_attrs else "red"
398
- rich_console.print(f" - [bold][{color}]{key}:[/{color}][/bold]")
399
- rich_console.print(f" [bold]Original:[/bold] [red]{og_val}[/red]")
400
- rich_console.print(f" [bold]Treated:[/bold] [green]{new_val}[/green]")
401
-
402
-
403
- @app.command(name="doctor")
404
- @coro # type: ignore
405
- async def doctor_dataset(
406
- repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
407
- group: str | None = typer.Option(None, help="The path to the zarr group"),
408
- treat: bool = typer.Option(default=False, hidden=True, help="Fix and commit the changes automatically"),
409
- dry_run: bool = typer.Option(default=False, hidden=True, help="Run the doctor without committing changes"),
410
- ):
411
- """**Doctor** a dataset to make it usable by the Arraylake compute engine.
412
-
413
- **Examples**
414
-
415
-
416
- """
417
- if not treat and dry_run:
418
- rich_console.print("[bold][yellow]Warning: [/bold][/yellow] Treat must be enabled for dry run mode.")
419
-
420
- group_str = group if group is not None else ""
421
- rich_console.print(f"Admitting dataset [bold]{repo_name}/{group_str}[/bold] to the Arraylake hospital :hospital:")
422
- client = Client()
423
- repo = client.get_repo(repo_name)
424
-
425
- session = repo.writable_session(branch="main") if treat else repo.readonly_session(branch="main")
426
- store = session.store
427
-
428
- # with ekg_progress(""):
429
- import xarray as xr # noqa: F811
430
-
431
- with warnings.catch_warnings():
432
- warnings.simplefilter("ignore")
433
- ds = xr.open_zarr(store, zarr_format=3, group=group, consolidated=False)
434
- diagnosis = diagnose_dataset(ds)
435
-
436
- if treat:
437
- updated_diagnosis, treated_ds = treat_dataset(diagnosis, store, group=group)
438
- # If the dataset was already healthy, treatment is not needed
439
- if treated_ds is None:
440
- return
441
- # Check that the only the dataset was modified using session status
442
- diff = session.status()
443
- assert len(diff.new_groups) == 0
444
- assert len(diff.new_arrays) == 0
445
- assert len(diff.deleted_groups) == 0
446
- assert len(diff.deleted_arrays) == 0
447
- assert len(diff.updated_chunks) == 0
448
- assert diff.updated_arrays # type: ignore[unused-ignore,attr-defined]
449
- rich_console.print("[bold]Proposed treatment:[/bold] :pill:")
450
- get_treatment_plan(ds, treated_ds)
451
-
452
- if updated_diagnosis.is_healthy and not dry_run:
453
- lets_commit = Confirm.ask("Do you want to commit the proposed changes?")
454
- if lets_commit:
455
- rich_console.print("[bold]Committing changes...[/bold] :sparkles:")
456
- message = "Updated metadata attributes for use with Arraylake Flux using repo doctor"
457
- snapshot_id = session.commit(message)
458
- rich_console.print(f"Dataset successfully treated and committed to the repo with ID {snapshot_id} :dizzy:")
459
- else:
460
- if not diagnosis.is_healthy:
461
- rich_console.print("\nTo manually treat the dataset, use the diagnosis provided above to update the dataset metadata")
462
-
463
-
464
308
  @app.command(hidden=True)
465
309
  @coro # type: ignore
466
310
  async def get_status(