arraylake 1.2.0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {arraylake-1.2.0 → arraylake-1.3.0}/PKG-INFO +3 -3
  2. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/_version.py +2 -2
  3. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/api_utils.py +27 -0
  4. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/auth.py +39 -12
  5. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/compute.py +42 -3
  6. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/main.py +5 -0
  7. arraylake-1.3.0/arraylake/cli/org.py +41 -0
  8. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/client.py +112 -77
  9. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/compute/http_client.py +13 -1
  10. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/compute/services.py +52 -4
  11. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/compute/types.py +43 -25
  12. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/display/repo.py +3 -1
  13. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/metastore/http_metastore.py +27 -9
  14. arraylake-1.3.0/arraylake/obstore.py +132 -0
  15. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/repos/icechunk/types.py +18 -0
  16. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/repos/icechunk/virtual.py +27 -0
  17. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/token.py +24 -2
  18. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/types.py +255 -38
  19. {arraylake-1.2.0 → arraylake-1.3.0}/pyproject.toml +3 -3
  20. {arraylake-1.2.0 → arraylake-1.3.0}/tests/compute/test_client.py +33 -0
  21. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_api_utils.py +25 -0
  22. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_cli.py +106 -1
  23. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_client.py +4 -1
  24. arraylake-1.3.0/tests/test_http_metastore.py +97 -0
  25. arraylake-1.3.0/tests/test_obstore.py +182 -0
  26. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_performance.py +4 -14
  27. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_token_handler.py +44 -1
  28. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_types.py +246 -0
  29. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_virtual_chunks.py +53 -39
  30. {arraylake-1.2.0 → arraylake-1.3.0}/uv.lock +561 -325
  31. {arraylake-1.2.0 → arraylake-1.3.0}/.gitignore +0 -0
  32. {arraylake-1.2.0 → arraylake-1.3.0}/README.md +0 -0
  33. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/__init__.py +0 -0
  34. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/__main__.py +0 -0
  35. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/_credential_cache.py +0 -0
  36. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/asyn.py +0 -0
  37. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/__init__.py +0 -0
  38. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/config.py +0 -0
  39. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/repo.py +0 -0
  40. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/skills.py +0 -0
  41. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/cli/utils.py +0 -0
  42. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/compute/doctor.py +0 -0
  43. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/config.py +0 -0
  44. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/config.yaml +0 -0
  45. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/credentials.py +0 -0
  46. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/diagnostics.py +0 -0
  47. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/display/__init__.py +0 -0
  48. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/display/repolist.py +0 -0
  49. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/exceptions.py +0 -0
  50. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/log_util.py +0 -0
  51. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/metastore/__init__.py +0 -0
  52. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/metastore/abc.py +0 -0
  53. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/py.typed +0 -0
  54. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/repos/__init__.py +0 -0
  55. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/repos/icechunk/__init__.py +0 -0
  56. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/repos/icechunk/storage.py +0 -0
  57. {arraylake-1.2.0 → arraylake-1.3.0}/arraylake/tuning.py +0 -0
  58. {arraylake-1.2.0 → arraylake-1.3.0}/tests/__init__.py +0 -0
  59. {arraylake-1.2.0 → arraylake-1.3.0}/tests/compute/test_doctor.py +0 -0
  60. {arraylake-1.2.0 → arraylake-1.3.0}/tests/config.yaml +0 -0
  61. {arraylake-1.2.0 → arraylake-1.3.0}/tests/conftest.py +0 -0
  62. {arraylake-1.2.0 → arraylake-1.3.0}/tests/icechunk/__init__.py +0 -0
  63. {arraylake-1.2.0 → arraylake-1.3.0}/tests/icechunk/conftest.py +0 -0
  64. {arraylake-1.2.0 → arraylake-1.3.0}/tests/icechunk/test_storage.py +0 -0
  65. {arraylake-1.2.0 → arraylake-1.3.0}/tests/icechunk/test_virtual.py +0 -0
  66. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_asyn.py +0 -0
  67. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_config.py +0 -0
  68. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_credential_cache.py +0 -0
  69. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_credentials.py +0 -0
  70. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_diagnostics.py +0 -0
  71. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_skills.py +0 -0
  72. {arraylake-1.2.0 → arraylake-1.3.0}/tests/test_subscriptions.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arraylake
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary: Python client for ArrayLake
5
5
  Project-URL: Documentation, https://docs.earthmover.io/
6
6
  Project-URL: Changelog, https://docs.earthmover.io/changelog
@@ -14,14 +14,14 @@ Requires-Python: <3.15,>=3.12
14
14
  Requires-Dist: click<9.0,>=8.1.3
15
15
  Requires-Dist: donfig<1.0,>=0.7
16
16
  Requires-Dist: httpx<0.29,>=0.23
17
- Requires-Dist: icechunk<3.0.0,>=2.0.6
17
+ Requires-Dist: icechunk<3.0.0,>=2.1.2
18
18
  Requires-Dist: numpy<3.0,>=1.23
19
19
  Requires-Dist: packaging<27.0,>=23.0
20
20
  Requires-Dist: pydantic[email]<3.0,>=2.9
21
21
  Requires-Dist: python-dateutil<3.0,>=2.8
22
22
  Requires-Dist: rich<16.0,>=12.6
23
23
  Requires-Dist: ruamel-yaml<1.0,>=0.17
24
- Requires-Dist: structlog<26.0,>=24.1
24
+ Requires-Dist: structlog<27.0,>=24.1
25
25
  Requires-Dist: typer<1.0,>=0.12.0
26
26
  Requires-Dist: zarr<4.0,>=3.1.0
27
27
  Provides-Extra: maximal
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.2.0'
22
- __version_tuple__ = version_tuple = (1, 2, 0)
21
+ __version__ = version = '1.3.0'
22
+ __version_tuple__ = version_tuple = (1, 3, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -84,6 +84,25 @@ def _get_proxy_from_env() -> str | None:
84
84
  return None
85
85
 
86
86
 
87
+ # Client name reported in the X-Arraylake-Client header. Defaults to the Python
88
+ # library; the CLI entry point overrides it so server logs can tell them apart.
89
+ _client_name: str = "python"
90
+
91
+
92
+ def set_client_name(name: str) -> None:
93
+ """Override the client name sent in the ``X-Arraylake-Client`` header.
94
+
95
+ Must be called before any ``ArraylakeHttpClient`` is constructed to take effect,
96
+ since headers are baked in at construction time.
97
+ """
98
+ global _client_name
99
+ _client_name = name
100
+
101
+
102
+ def get_client_name() -> str:
103
+ return _client_name
104
+
105
+
87
106
  @functools.cache
88
107
  def _get_session_id() -> str:
89
108
  """Session ID for this Python process.
@@ -317,6 +336,9 @@ class ArraylakeHttpClient:
317
336
  # technically we don't enforce this, for now it's for debugging purposes
318
337
  "client-name": "arraylake-python-client",
319
338
  "client-version": arraylake.__version__,
339
+ "x-arraylake-client": f"{get_client_name()}/{arraylake.__version__}",
340
+ "x-arraylake-source": get_client_name(),
341
+ "user-agent": f"arraylake-{get_client_name()}/{arraylake.__version__}",
320
342
  "arraylake-session-id": _get_session_id(),
321
343
  }
322
344
 
@@ -448,6 +470,7 @@ def _exception_log_debug(request: httpx.Request, response: httpx.Response):
448
470
 
449
471
 
450
472
  _emitted_deprecation_warnings: set[str] = set()
473
+ _emitted_user_warnings: set[str] = set()
451
474
 
452
475
 
453
476
  def handle_response(response: httpx.Response):
@@ -456,6 +479,10 @@ def handle_response(response: httpx.Response):
456
479
  if deprecation_msg and deprecation_msg not in _emitted_deprecation_warnings:
457
480
  _emitted_deprecation_warnings.add(deprecation_msg)
458
481
  warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2)
482
+ warning_msg = response.headers.get("X-Arraylake-Warning") # Service-sent user-facing warnings
483
+ if warning_msg and warning_msg not in _emitted_user_warnings:
484
+ _emitted_user_warnings.add(warning_msg)
485
+ warnings.warn(warning_msg, UserWarning, stacklevel=2)
459
486
  try:
460
487
  response.raise_for_status()
461
488
  except httpx.RequestError as exc:
@@ -2,7 +2,6 @@ import typer
2
2
  from rich.align import Align
3
3
  from rich.panel import Panel
4
4
 
5
- from arraylake import config
6
5
  from arraylake.cli.utils import (
7
6
  coro,
8
7
  error_console,
@@ -17,11 +16,26 @@ auth = typer.Typer(help="Manage Arraylake authentication")
17
16
 
18
17
  NOT_LOGGED_IN_MESSAGE = "⚠️ Not logged in, please log in with: `arraylake auth login`"
19
18
 
19
+ SERVICE_URI_OPTION = typer.Option(
20
+ None,
21
+ "--service-uri",
22
+ help=(
23
+ "Arraylake API endpoint to act on, e.g. **https://api.dev.earthmover.io**. Defaults to the configured "
24
+ "**service.uri**. Tokens are stored per endpoint, so this need not be persisted to config — unless "
25
+ "**service.token_path** is set, which makes all endpoints share one token file."
26
+ ),
27
+ )
28
+
29
+
30
+ def _resolve_service_uri(service_uri: str | None) -> str:
31
+ return service_uri if service_uri else default_service_uri()
32
+
20
33
 
21
34
  @auth.command()
22
35
  @coro # type: ignore
23
36
  async def login(
24
37
  browser: bool = typer.Option(True, "--browser/--no-browser", " /--nobrowser", help="Whether to automatically open a browser window."),
38
+ service_uri: str | None = SERVICE_URI_OPTION,
25
39
  ):
26
40
  """**Log in** to Arraylake
27
41
 
@@ -34,17 +48,23 @@ async def login(
34
48
  ```
35
49
  $ arraylake auth login --no-browser
36
50
  ```
51
+
52
+ - Log in to a non-default Arraylake deployment
53
+
54
+ ```
55
+ $ arraylake auth login --service-uri https://api.dev.earthmover.io
56
+ ```
37
57
  """
38
58
  print_logo()
39
- handler = get_auth_handler(api_endpoint=default_service_uri())
59
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
40
60
  await handler.login(browser=browser)
41
61
 
42
62
 
43
63
  @auth.command()
44
64
  @coro # type: ignore
45
- async def refresh() -> None:
65
+ async def refresh(service_uri: str | None = SERVICE_URI_OPTION) -> None:
46
66
  """**Refresh** Arraylake's auth token"""
47
- handler = get_auth_handler(api_endpoint=default_service_uri())
67
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
48
68
  if handler.tokens is not None:
49
69
  try:
50
70
  await handler.refresh_token()
@@ -57,7 +77,7 @@ async def refresh() -> None:
57
77
 
58
78
  @auth.command()
59
79
  @coro # type: ignore
60
- async def logout() -> None:
80
+ async def logout(service_uri: str | None = SERVICE_URI_OPTION) -> None:
61
81
  """**Logout** of Arraylake
62
82
 
63
83
  **Examples**
@@ -67,9 +87,15 @@ async def logout() -> None:
67
87
  ```
68
88
  $ arraylake auth logout
69
89
  ```
90
+
91
+ - Log out of a non-default Arraylake deployment
92
+
93
+ ```
94
+ $ arraylake auth logout --service-uri https://api.dev.earthmover.io
95
+ ```
70
96
  """
71
97
  try:
72
- handler = get_auth_handler(api_endpoint=default_service_uri())
98
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
73
99
  await handler.logout()
74
100
  except AuthException as e:
75
101
  error_console.print(e)
@@ -77,14 +103,15 @@ async def logout() -> None:
77
103
 
78
104
 
79
105
  @auth.command()
80
- @coro
81
- async def status():
106
+ @coro # type: ignore
107
+ async def status(service_uri: str | None = SERVICE_URI_OPTION):
82
108
  """Verify and display information about your authentication **status**"""
109
+ endpoint = _resolve_service_uri(service_uri)
83
110
  print_logo_mini()
84
- rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {config.get('service.uri')}"), align="center"))
111
+ rich_console.print(Align(Panel.fit(f"[bold]Arraylake API Endpoint[/bold]: {endpoint}"), align="center"))
85
112
 
86
113
  try:
87
- handler = get_auth_handler(api_endpoint=default_service_uri())
114
+ handler = get_auth_handler(api_endpoint=endpoint)
88
115
  user = await handler._get_user() # checks that the new tokens are valid
89
116
  rich_console.print(Align(f"[green][bold]🔓 Logged in as {user.email}[/green][/bold]", align="center"))
90
117
  except AuthException:
@@ -93,9 +120,9 @@ async def status():
93
120
 
94
121
 
95
122
  @auth.command()
96
- def token():
123
+ def token(service_uri: str | None = SERVICE_URI_OPTION):
97
124
  """Display your authentication **token**"""
98
- handler = get_auth_handler(api_endpoint=default_service_uri())
125
+ handler = get_auth_handler(api_endpoint=_resolve_service_uri(service_uri))
99
126
  if handler.tokens is None:
100
127
  rich_console.print(Align(f"[yellow]{NOT_LOGGED_IN_MESSAGE}[/yellow]", align="center"))
101
128
  raise typer.Exit(code=1)
@@ -48,6 +48,7 @@ def _services_table(services, org):
48
48
  def enable_service(
49
49
  org: str = typer.Argument(..., help="The organization to enable the service for"),
50
50
  protocol: str = typer.Argument(..., help="The protocol to enable. Must be one of: dap2, edr, wms, tiles, zarr"),
51
+ sku_code: str = typer.Option(..., "--sku-code", "-s", help="The compute shape to run the service on, e.g. xs"),
51
52
  is_public: bool = typer.Option(False, help="Whether the service is public"),
52
53
  deployment_name: str | None = typer.Option(
53
54
  None,
@@ -63,13 +64,13 @@ def enable_service(
63
64
  - Enable a new DAP service for _my-org_
64
65
 
65
66
  ```
66
- $ arraylake compute enable my-org dap2
67
+ $ arraylake compute enable my-org dap2 --sku-code xs
67
68
  ```
68
69
 
69
70
  - Enable a tiles service with a custom deployment name
70
71
 
71
72
  ```
72
- $ arraylake compute enable my-org tiles --deployment-name my-tiles
73
+ $ arraylake compute enable my-org tiles --sku-code s --deployment-name my-tiles
73
74
  ```
74
75
  """
75
76
  kwargs = {}
@@ -77,7 +78,7 @@ def enable_service(
77
78
  kwargs["deployment_name"] = deployment_name
78
79
 
79
80
  with simple_progress(f"Creating [bold]{protocol}[/bold] service for [bold]{org}[/bold]..."):
80
- status = Client().get_services(org).enable(protocol, is_public, **kwargs)
81
+ status = Client().get_services(org).enable(protocol, is_public, sku_code=sku_code, **kwargs)
81
82
  rich_console.print(status["message"])
82
83
 
83
84
 
@@ -183,6 +184,44 @@ async def get_service_logs(
183
184
  raise typer.Exit(code=1)
184
185
 
185
186
 
187
+ @app.command(name="update")
188
+ def update_service(
189
+ org: str = typer.Argument(..., help="The organization that the service belongs to"),
190
+ service_id: str = typer.Argument(..., help="The service ID to update"),
191
+ sku: str | None = typer.Option(None, "--sku", help="The compute shape (SKU code) to resize the service to"),
192
+ min_replicas: int | None = typer.Option(None, min=0, help="The minimum number of replicas the service scales down to"),
193
+ max_replicas: int | None = typer.Option(None, min=1, help="The maximum number of replicas the service scales up to"),
194
+ ):
195
+ """**Update** a running service's shape or scaling. Omitted options keep their current values.
196
+
197
+ **Examples**
198
+
199
+ - Resize a service with ID edr-1234567 to a different compute shape
200
+
201
+ ```
202
+ $ arraylake compute update my-org edr-1234567 --sku medium
203
+ ```
204
+
205
+ - Change the scaling bounds of a service
206
+
207
+ ```
208
+ $ arraylake compute update my-org edr-1234567 --min-replicas 2 --max-replicas 10
209
+ ```
210
+ """
211
+ if sku is None and min_replicas is None and max_replicas is None:
212
+ rich_console.print("[bold red]Nothing to update: pass at least one of --sku, --min-replicas, --max-replicas[/bold red]")
213
+ raise typer.Exit(code=1)
214
+
215
+ with simple_progress(f"Updating service [bold]{service_id}[/bold]..."):
216
+ deployment = Client().get_services(org).update(service_id, sku_code=sku, min_replicas=min_replicas, max_replicas=max_replicas)
217
+
218
+ config = deployment.config
219
+ rich_console.print(
220
+ f"Service [bold]{deployment.name}[/bold] updated: "
221
+ f"sku={config.sku_code}, min_replicas={config.min_replicas}, max_replicas={config.max_replicas}"
222
+ )
223
+
224
+
186
225
  @app.command(name="disable")
187
226
  def disable_service(
188
227
  org: str = typer.Argument(..., help="The organization that the service belongs to"),
@@ -2,9 +2,11 @@ import typer
2
2
 
3
3
  from arraylake import __version__
4
4
  from arraylake import config as config_obj
5
+ from arraylake.api_utils import set_client_name
5
6
  from arraylake.cli.auth import auth
6
7
  from arraylake.cli.compute import app as compute_app
7
8
  from arraylake.cli.config import app as config_app
9
+ from arraylake.cli.org import app as org_app
8
10
  from arraylake.cli.repo import app as repo_app
9
11
  from arraylake.cli.skills import app as skills_app
10
12
  from arraylake.cli.utils import rich_console
@@ -36,12 +38,15 @@ def main(
36
38
  ),
37
39
  ):
38
40
  """Manage ArrayLake from the command line."""
41
+ # Distinguish CLI usage from library usage in server access logs
42
+ set_client_name("python-cli")
39
43
  if config:
40
44
  opts = dict(map(lambda x: x.split("="), config))
41
45
  config_obj.set(opts, priority="new")
42
46
 
43
47
 
44
48
  app.add_typer(auth, name="auth")
49
+ app.add_typer(org_app, name="org")
45
50
  app.add_typer(repo_app, name="repo")
46
51
  app.add_typer(config_app, name="config")
47
52
  app.add_typer(compute_app, name="compute")
@@ -0,0 +1,41 @@
1
+ import typer
2
+ from rich.table import Table
3
+
4
+ from arraylake import AsyncClient
5
+ from arraylake.cli.utils import coro, rich_console, simple_progress
6
+
7
+ app = typer.Typer(help="Manage Arraylake organizations", no_args_is_help=True)
8
+
9
+
10
+ def _orgs_table(orgs) -> Table:
11
+ table = Table(title="Arraylake Organizations", min_width=60)
12
+ table.add_column("Name", justify="left", style="cyan", no_wrap=True, min_width=30)
13
+ table.add_column("Display Name", justify="left", style="green", min_width=20)
14
+ table.add_column("Status", justify="right", style="green", min_width=10)
15
+
16
+ for org in orgs:
17
+ table.add_row(org.name, org.display_name or org.name, org.status)
18
+
19
+ return table
20
+
21
+
22
+ @app.command(name="list")
23
+ @coro
24
+ async def list_orgs():
25
+ """**List** all organizations for the authenticated user
26
+
27
+ **Examples**
28
+
29
+ - List all orgs
30
+
31
+ ```
32
+ $ arraylake org list
33
+ ```
34
+ """
35
+ with simple_progress("Listing orgs..."):
36
+ orgs = await AsyncClient().get_orgs()
37
+
38
+ if orgs:
39
+ rich_console.print(_orgs_table(orgs))
40
+ else:
41
+ rich_console.print("\nNo organizations found")