arraylake 0.27.0__tar.gz → 0.31.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arraylake-0.27.0 → arraylake-0.31.0}/PKG-INFO +1 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/__init__.py +2 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/_version.py +2 -2
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/api_utils.py +4 -4
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/asyn.py +6 -4
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/compute.py +17 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/repo.py +41 -9
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/utils.py +1 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/client.py +469 -117
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/compute/doctor.py +12 -8
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/compute/services.py +5 -2
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/compute/types.py +21 -6
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/credentials.py +5 -6
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/display/repolist.py +8 -2
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/metastore/http_metastore.py +143 -7
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/repos/icechunk/storage.py +35 -11
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/repos/icechunk/types.py +65 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/repos/icechunk/virtual.py +30 -7
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/token.py +35 -45
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/tuning.py +8 -6
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/types.py +290 -15
- {arraylake-0.27.0 → arraylake-0.31.0}/pyproject.toml +2 -2
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/conftest.py +4 -2
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/icechunk/test_virtual.py +1 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_cli.py +44 -5
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_client.py +194 -46
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_credentials.py +1 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_diagnostics.py +3 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_performance.py +4 -7
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_subscriptions.py +1 -1
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_token_handler.py +7 -8
- arraylake-0.31.0/tests/test_types.py +404 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_virtual_chunks.py +218 -7
- arraylake-0.31.0/uv.lock +3090 -0
- arraylake-0.27.0/tests/test_types.py +0 -119
- arraylake-0.27.0/uv.lock +0 -3079
- {arraylake-0.27.0 → arraylake-0.31.0}/.gitignore +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/README.md +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/__main__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/auth.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/config.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/cli/main.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/compute/http_client.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/config.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/config.yaml +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/diagnostics.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/display/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/display/repo.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/exceptions.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/log_util.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/metastore/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/metastore/abc.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/py.typed +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/repos/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/arraylake/repos/icechunk/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/compute/test_client.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/compute/test_doctor.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/config.yaml +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/icechunk/__init__.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/icechunk/conftest.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/icechunk/test_storage.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_api_utils.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_asyn.py +0 -0
- {arraylake-0.27.0 → arraylake-0.31.0}/tests/test_config.py +0 -0
|
@@ -8,6 +8,7 @@ from arraylake.exceptions import (
|
|
|
8
8
|
ArraylakeServerError,
|
|
9
9
|
ArraylakeValidationError,
|
|
10
10
|
)
|
|
11
|
+
from arraylake.types import StorageOptions
|
|
11
12
|
|
|
12
13
|
__version__ = importlib.metadata.version("arraylake")
|
|
13
14
|
|
|
@@ -22,4 +23,5 @@ __all__ = [
|
|
|
22
23
|
"ArraylakeClientError",
|
|
23
24
|
"ArraylakeServerError",
|
|
24
25
|
"ArraylakeValidationError",
|
|
26
|
+
"StorageOptions",
|
|
25
27
|
]
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.31.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 31, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -402,8 +402,7 @@ class ArraylakeHttpClient:
|
|
|
402
402
|
if config.get("user.diagnostics", True):
|
|
403
403
|
try:
|
|
404
404
|
diagnostics = get_diagnostics()
|
|
405
|
-
response =
|
|
406
|
-
handle_response(response)
|
|
405
|
+
response = asyncio.create_task(self._request("POST", "user/diagnostics", content=diagnostics.model_dump_json()))
|
|
407
406
|
except Exception as e:
|
|
408
407
|
await logger.adebug("failed to send diagnostics", exception=str(e))
|
|
409
408
|
|
|
@@ -617,7 +616,7 @@ class AsyncRetryTransport(httpx.AsyncBaseTransport):
|
|
|
617
616
|
"""
|
|
618
617
|
remaining_attempts = self.max_attempts
|
|
619
618
|
attempts_made = 0
|
|
620
|
-
headers: httpx.Headers =
|
|
619
|
+
headers: httpx.Headers = httpx.Headers()
|
|
621
620
|
while True:
|
|
622
621
|
if attempts_made > 0:
|
|
623
622
|
await asyncio.sleep(self._calculate_sleep(attempts_made, headers))
|
|
@@ -627,7 +626,8 @@ class AsyncRetryTransport(httpx.AsyncBaseTransport):
|
|
|
627
626
|
return response
|
|
628
627
|
if response.status_code not in self.retryable_mapping:
|
|
629
628
|
return response
|
|
630
|
-
|
|
629
|
+
status_code_http = HTTPStatus(response.status_code)
|
|
630
|
+
if request.method not in self.retryable_mapping[status_code_http]:
|
|
631
631
|
return response
|
|
632
632
|
await response.aclose()
|
|
633
633
|
attempts_made += 1
|
|
@@ -63,7 +63,9 @@ T = TypeVar("T")
|
|
|
63
63
|
PageableFunction = Callable[..., Awaitable[tuple[Iterable[T], "PageableFunction"]]]
|
|
64
64
|
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
# NOTE: Properly type hinting this is impossible at the moment
|
|
67
|
+
# https://discuss.python.org/t/pre-pep-considerations-and-feedback-type-transformations-on-variadic-generics/50605
|
|
68
|
+
async def async_gather_tasks(*coroutines: Coroutine[Any, Any, T]) -> list[T]:
|
|
67
69
|
"""
|
|
68
70
|
Run coroutines concurrently using TaskGroup, returning ordered results.
|
|
69
71
|
|
|
@@ -74,7 +76,7 @@ async def async_gather_tasks(*coroutines: Awaitable[T]) -> list[T]:
|
|
|
74
76
|
"""
|
|
75
77
|
try:
|
|
76
78
|
async with asyncio.TaskGroup() as tg:
|
|
77
|
-
tasks = [tg.create_task(coro) for coro in coroutines]
|
|
79
|
+
tasks: list[asyncio.Task[T]] = [tg.create_task(coro) for coro in coroutines]
|
|
78
80
|
|
|
79
81
|
return [task.result() for task in tasks]
|
|
80
82
|
except BaseException as e:
|
|
@@ -227,7 +229,7 @@ def get_background_loop() -> asyncio.AbstractEventLoop:
|
|
|
227
229
|
return new_loop
|
|
228
230
|
|
|
229
231
|
|
|
230
|
-
def asyncio_run(coro:
|
|
232
|
+
def asyncio_run(coro: Coroutine[Any, Any, T], timeout=30) -> T:
|
|
231
233
|
"""
|
|
232
234
|
Runs the coroutine in an event loop running on a background thread,
|
|
233
235
|
and blocks the current thread until it returns a result.
|
|
@@ -238,7 +240,7 @@ def asyncio_run(coro: Awaitable[T], timeout=30) -> T:
|
|
|
238
240
|
Args:
|
|
239
241
|
coro: A coroutine, typically an async method
|
|
240
242
|
timeout: How many seconds we should wait for a result before raising an error
|
|
241
|
-
"""
|
|
243
|
+
"""
|
|
242
244
|
return asyncio.run_coroutine_threadsafe(coro, get_background_loop()).result(timeout=timeout)
|
|
243
245
|
|
|
244
246
|
|
|
@@ -49,6 +49,12 @@ def enable_service(
|
|
|
49
49
|
org: str = typer.Argument(..., help="The organization to enable the service for"),
|
|
50
50
|
protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr"),
|
|
51
51
|
is_public: bool = typer.Option(False, help="Whether the service is public"),
|
|
52
|
+
deployment_name: str | None = typer.Option(
|
|
53
|
+
None,
|
|
54
|
+
"--deployment-name",
|
|
55
|
+
"-n",
|
|
56
|
+
help="Custom deployment name (defaults to protocol name). Must be lowercase alphanumeric with dashes, max 16 chars.",
|
|
57
|
+
),
|
|
52
58
|
):
|
|
53
59
|
"""**Enable** a new service
|
|
54
60
|
|
|
@@ -59,9 +65,19 @@ def enable_service(
|
|
|
59
65
|
```
|
|
60
66
|
$ arraylake compute enable my-org dap2
|
|
61
67
|
```
|
|
68
|
+
|
|
69
|
+
- Enable a tiles service with a custom deployment name
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
$ arraylake compute enable my-org tiles --deployment-name my-tiles
|
|
73
|
+
```
|
|
62
74
|
"""
|
|
75
|
+
kwargs = {}
|
|
76
|
+
if deployment_name is not None:
|
|
77
|
+
kwargs["deployment_name"] = deployment_name
|
|
78
|
+
|
|
63
79
|
with simple_progress(f"Creating [bold]{protocol}[/bold] service for [bold]{org}[/bold]..."):
|
|
64
|
-
status = Client().get_services(org).enable(protocol, is_public)
|
|
80
|
+
status = Client().get_services(org).enable(protocol, is_public, **kwargs)
|
|
65
81
|
rich_console.print(status["message"])
|
|
66
82
|
|
|
67
83
|
|
|
@@ -128,8 +128,8 @@ async def list_repos(
|
|
|
128
128
|
repos = await AsyncClient().list_repos(org, filter_metadata=metadata_filter_dict)
|
|
129
129
|
|
|
130
130
|
if output == "json":
|
|
131
|
-
|
|
132
|
-
print_json(data=
|
|
131
|
+
repos_json = [_make_json_safe(r._asdict()) for r in repos]
|
|
132
|
+
print_json(data=repos_json)
|
|
133
133
|
elif repos:
|
|
134
134
|
rich_console.print(_repos_table(repos, org))
|
|
135
135
|
else:
|
|
@@ -141,6 +141,7 @@ async def list_repos(
|
|
|
141
141
|
async def create(
|
|
142
142
|
repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
|
|
143
143
|
bucket_config_nickname: str | None = typer.Option(None, help="Bucket config nickname"),
|
|
144
|
+
prefix: str = typer.Option(None, help="Optional prefix for Icechunk repo. If not provided, a random ID + the repo name will be used."),
|
|
144
145
|
import_existing: bool = typer.Option(False, help="Import existing Icechunk data into the new repo"),
|
|
145
146
|
description: str | None = typer.Option(None, help="Description of the repo"),
|
|
146
147
|
metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
|
|
@@ -157,13 +158,44 @@ async def create(
|
|
|
157
158
|
"""
|
|
158
159
|
metadata_dict = json.loads(metadata) if metadata else None
|
|
159
160
|
with simple_progress(f"Creating repo [bold]{repo_name}[/bold]..."):
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
161
|
+
await AsyncClient().create_repo(
|
|
162
|
+
repo_name,
|
|
163
|
+
bucket_config_nickname=bucket_config_nickname,
|
|
164
|
+
prefix=prefix,
|
|
165
|
+
import_existing=import_existing,
|
|
166
|
+
description=description,
|
|
167
|
+
metadata=metadata_dict,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@app.command(name="import")
|
|
172
|
+
@coro # type: ignore
|
|
173
|
+
async def import_repo(
|
|
174
|
+
repo_name: str = typer.Argument(..., help="Name of repository {ORG}/{REPO_NAME}"),
|
|
175
|
+
bucket_config_nickname: str = typer.Argument(..., help="Bucket config nickname"),
|
|
176
|
+
prefix: str = typer.Option(None, help="Sub-prefix in which the Icechunk repo exists"),
|
|
177
|
+
description: str | None = typer.Option(None, help="Description of the repo"),
|
|
178
|
+
metadata: str | None = typer.Option(None, help="Optional metadata for the repo"),
|
|
179
|
+
):
|
|
180
|
+
"""**Import** an existing Icechunk repository
|
|
181
|
+
|
|
182
|
+
**Examples**
|
|
183
|
+
|
|
184
|
+
- Import existing Icechunk repository
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
$ arraylake repo import my-org/example-repo --bucket-config-nickname icechunk-bucket --prefix my-icechunk-prefix --metadata '{"key": "value"}'
|
|
188
|
+
```
|
|
189
|
+
"""
|
|
190
|
+
metadata_dict = json.loads(metadata) if metadata else None
|
|
191
|
+
with simple_progress(f"Importing repo [bold]{repo_name}[/bold]..."):
|
|
192
|
+
await AsyncClient().import_repo(
|
|
193
|
+
repo_name,
|
|
194
|
+
bucket_config_nickname=bucket_config_nickname,
|
|
195
|
+
prefix=prefix,
|
|
196
|
+
description=description,
|
|
197
|
+
metadata=metadata_dict,
|
|
198
|
+
)
|
|
167
199
|
|
|
168
200
|
|
|
169
201
|
@app.command()
|
|
@@ -162,7 +162,7 @@ def print_user_details(user: UserInfo):
|
|
|
162
162
|
rich_console.print(
|
|
163
163
|
Panel(
|
|
164
164
|
f"Name: [dim]{user.first_name} {user.family_name}[/dim]\nEmail: [dim]{user.email}[/dim]\nId: [dim]{user.id}[/dim]",
|
|
165
|
-
title="
|
|
165
|
+
title="[bold]User Details[/bold]",
|
|
166
166
|
)
|
|
167
167
|
)
|
|
168
168
|
|