datalab-api 0.5.2__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalab_api-0.5.2/src/datalab_api.egg-info → datalab_api-0.6.0}/PKG-INFO +30 -7
- {datalab_api-0.5.2 → datalab_api-0.6.0}/README.md +22 -1
- {datalab_api-0.5.2 → datalab_api-0.6.0}/pyproject.toml +15 -7
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api/__init__.py +65 -4
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api/_base.py +46 -14
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api/cli.py +6 -7
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api/utils.py +37 -1
- {datalab_api-0.5.2 → datalab_api-0.6.0/src/datalab_api.egg-info}/PKG-INFO +30 -7
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api.egg-info/requires.txt +10 -5
- {datalab_api-0.5.2 → datalab_api-0.6.0}/tests/test_client.py +81 -2
- {datalab_api-0.5.2 → datalab_api-0.6.0}/LICENSE +0 -0
- {datalab_api-0.5.2 → datalab_api-0.6.0}/setup.cfg +0 -0
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api.egg-info/SOURCES.txt +0 -0
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api.egg-info/dependency_links.txt +0 -0
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api.egg-info/entry_points.txt +0 -0
- {datalab_api-0.5.2 → datalab_api-0.6.0}/src/datalab_api.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datalab-api
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A Python API for instances of the datalab data management platform (https://github.com/datalab-org/datalab).
|
|
5
5
|
Author-email: Matthew Evans <matthew@ml-evs.science>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -9,7 +9,6 @@ Project-URL: repository, https://github.com/datalab-org/datalab-api
|
|
|
9
9
|
Project-URL: documentation, https://datalab-python-api.readthedocs.io
|
|
10
10
|
Project-URL: changelog, https://github.com/datalab-org/datalab-api/releases
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
@@ -18,13 +17,16 @@ Classifier: Intended Audience :: Science/Research
|
|
|
18
17
|
Classifier: Intended Audience :: Information Technology
|
|
19
18
|
Classifier: Topic :: Other/Nonlisted Topic
|
|
20
19
|
Classifier: Topic :: Scientific/Engineering
|
|
21
|
-
Requires-Python: >=3.
|
|
20
|
+
Requires-Python: >=3.10
|
|
22
21
|
Description-Content-Type: text/markdown
|
|
23
22
|
License-File: LICENSE
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist: bokeh~=2.4; platform_system != "Emscripten"
|
|
23
|
+
Requires-Dist: httpx2~=2.13
|
|
26
24
|
Requires-Dist: rich<15,>=13
|
|
27
|
-
|
|
25
|
+
Provides-Extra: plotting
|
|
26
|
+
Requires-Dist: bokeh~=2.4; platform_system != "Emscripten" and extra == "plotting"
|
|
27
|
+
Requires-Dist: numpy<2; extra == "plotting"
|
|
28
|
+
Provides-Extra: networkx
|
|
29
|
+
Requires-Dist: networkx>=3; extra == "networkx"
|
|
28
30
|
Provides-Extra: dev
|
|
29
31
|
Requires-Dist: pre-commit<5,>=3; extra == "dev"
|
|
30
32
|
Requires-Dist: pytest~=8.0; extra == "dev"
|
|
@@ -108,7 +110,6 @@ This package implements basic functionality for displaying and manipulating entr
|
|
|
108
110
|
from datalab_api import DatalabClient
|
|
109
111
|
|
|
110
112
|
with DatalabClient("https://demo-api.datalab-org.io") as client:
|
|
111
|
-
|
|
112
113
|
# List all items of a given type
|
|
113
114
|
items = client.get_items()
|
|
114
115
|
|
|
@@ -117,9 +118,31 @@ with DatalabClient("https://demo-api.datalab-org.io") as client:
|
|
|
117
118
|
|
|
118
119
|
# Upload a file to an item
|
|
119
120
|
file_response = client.upload_file(filepath="my_echem_data.mpr", item_id="test")
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Elevated (admin) permissions
|
|
120
124
|
|
|
125
|
+
By default a client can only see items its account owns or has been shared on.
|
|
126
|
+
Admins can opt in to datalab's super-user mode to read items belonging to other users, by passing `elevate_permissions`:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
with DatalabClient("https://demo-api.datalab-org.io", elevate_permissions=True) as client:
|
|
130
|
+
# Finds the item even if it belongs to another user
|
|
131
|
+
item = client.get_item(item_id="someone-elses-sample")
|
|
121
132
|
```
|
|
122
133
|
|
|
134
|
+
This sends datalab's `sudo=1` flag on read requests.
|
|
135
|
+
It requires the API key in use to belong to an **admin** account; for any other account the server ignores
|
|
136
|
+
it and normal permissions apply.
|
|
137
|
+
|
|
138
|
+
The attribute can also be toggled at any point in the client's lifetime:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
client.elevate_permissions = True
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Note that objects created while elevated (e.g. uploaded files) are still recorded as belonging to the admin's own account, not the item owner's.
|
|
145
|
+
|
|
123
146
|
### Command-line interface (CLI)
|
|
124
147
|
|
|
125
148
|
There is also an EXPERIMENTAL CLI, exposed via the `datalab` command.
|
|
@@ -63,7 +63,6 @@ This package implements basic functionality for displaying and manipulating entr
|
|
|
63
63
|
from datalab_api import DatalabClient
|
|
64
64
|
|
|
65
65
|
with DatalabClient("https://demo-api.datalab-org.io") as client:
|
|
66
|
-
|
|
67
66
|
# List all items of a given type
|
|
68
67
|
items = client.get_items()
|
|
69
68
|
|
|
@@ -72,9 +71,31 @@ with DatalabClient("https://demo-api.datalab-org.io") as client:
|
|
|
72
71
|
|
|
73
72
|
# Upload a file to an item
|
|
74
73
|
file_response = client.upload_file(filepath="my_echem_data.mpr", item_id="test")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Elevated (admin) permissions
|
|
75
77
|
|
|
78
|
+
By default a client can only see items its account owns or has been shared on.
|
|
79
|
+
Admins can opt in to datalab's super-user mode to read items belonging to other users, by passing `elevate_permissions`:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
with DatalabClient("https://demo-api.datalab-org.io", elevate_permissions=True) as client:
|
|
83
|
+
# Finds the item even if it belongs to another user
|
|
84
|
+
item = client.get_item(item_id="someone-elses-sample")
|
|
76
85
|
```
|
|
77
86
|
|
|
87
|
+
This sends datalab's `sudo=1` flag on read requests.
|
|
88
|
+
It requires the API key in use to belong to an **admin** account; for any other account the server ignores
|
|
89
|
+
it and normal permissions apply.
|
|
90
|
+
|
|
91
|
+
The attribute can also be toggled at any point in the client's lifetime:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
client.elevate_permissions = True
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Note that objects created while elevated (e.g. uploaded files) are still recorded as belonging to the admin's own account, not the item owner's.
|
|
98
|
+
|
|
78
99
|
### Command-line interface (CLI)
|
|
79
100
|
|
|
80
101
|
There is also an EXPERIMENTAL CLI, exposed via the `datalab` command.
|
|
@@ -12,7 +12,6 @@ authors = [{ name = "Matthew Evans", email = "matthew@ml-evs.science" }]
|
|
|
12
12
|
dynamic = ["version"]
|
|
13
13
|
classifiers = [
|
|
14
14
|
"Programming Language :: Python :: 3",
|
|
15
|
-
"Programming Language :: Python :: 3.9",
|
|
16
15
|
"Programming Language :: Python :: 3.10",
|
|
17
16
|
"Programming Language :: Python :: 3.11",
|
|
18
17
|
"Programming Language :: Python :: 3.12",
|
|
@@ -23,16 +22,23 @@ classifiers = [
|
|
|
23
22
|
"Topic :: Scientific/Engineering",
|
|
24
23
|
]
|
|
25
24
|
|
|
26
|
-
requires-python = ">=3.
|
|
25
|
+
requires-python = ">=3.10"
|
|
27
26
|
dependencies = [
|
|
28
|
-
"
|
|
29
|
-
'bokeh ~= 2.4; platform_system != "Emscripten"', # interactive plots from datalab directly
|
|
27
|
+
"httpx2 ~= 2.13", # better HTTP requests
|
|
30
28
|
"rich >= 13,< 15", # nicer terminal output
|
|
31
|
-
"numpy < 2", # upper pin for numpy due to bad pins in bokeh
|
|
32
29
|
]
|
|
33
30
|
|
|
34
31
|
|
|
35
32
|
[project.optional-dependencies]
|
|
33
|
+
plotting = [
|
|
34
|
+
"bokeh ~= 2.4; platform_system != 'Emscripten'", # interactive plots from datalab directly
|
|
35
|
+
"numpy < 2", # upper pin for numpy due to bad pins in bokeh
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
networkx = [
|
|
39
|
+
"networkx >= 3", # build graphs of item relationships
|
|
40
|
+
]
|
|
41
|
+
|
|
36
42
|
dev = [
|
|
37
43
|
"pre-commit >= 3,< 5",
|
|
38
44
|
"pytest ~= 8.0",
|
|
@@ -78,7 +84,7 @@ follow_imports = "skip"
|
|
|
78
84
|
|
|
79
85
|
[tool.ruff]
|
|
80
86
|
line-length = 100
|
|
81
|
-
target-version = "
|
|
87
|
+
target-version = "py310"
|
|
82
88
|
|
|
83
89
|
[tool.ruff.lint]
|
|
84
90
|
select = ["ALL"]
|
|
@@ -96,6 +102,7 @@ ignore = [
|
|
|
96
102
|
"C419",
|
|
97
103
|
"C901", # function too complex
|
|
98
104
|
"COM812", # trailing comma missing
|
|
105
|
+
"CPY001", # missing copyright notice
|
|
99
106
|
"D",
|
|
100
107
|
"D205",
|
|
101
108
|
"DTZ", # datetime-tz-now
|
|
@@ -120,6 +127,7 @@ ignore = [
|
|
|
120
127
|
"PERF401",
|
|
121
128
|
"PGH003",
|
|
122
129
|
"PGH004",
|
|
130
|
+
"PLC0415", # import not at top-level (used for lazy optional deps)
|
|
123
131
|
"PLR", # pylint-refactor
|
|
124
132
|
"PLW0602",
|
|
125
133
|
"PLW0603",
|
|
@@ -157,7 +165,7 @@ ignore = [
|
|
|
157
165
|
"UP031",
|
|
158
166
|
]
|
|
159
167
|
pydocstyle.convention = "numpy"
|
|
160
|
-
isort.known-first-party = ["
|
|
168
|
+
isort.known-first-party = ["datalab_api"]
|
|
161
169
|
isort.split-on-trailing-comma = false
|
|
162
170
|
fixable = ["A", "B", "C", "D", "E", "F", "I", "PT", "RUF"]
|
|
163
171
|
unfixable = []
|
|
@@ -3,11 +3,15 @@ from __future__ import annotations
|
|
|
3
3
|
import warnings
|
|
4
4
|
from collections.abc import Iterable
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Any, Optional, Union
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
7
7
|
|
|
8
8
|
import httpx
|
|
9
9
|
|
|
10
10
|
from ._base import BaseDatalabClient, DatalabAPIError, DuplicateItemError, __version__
|
|
11
|
+
from .utils import networkx_from_item_graph
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
import networkx as nx
|
|
11
15
|
|
|
12
16
|
__all__ = ("DatalabClient", "DuplicateItemError", "__version__")
|
|
13
17
|
|
|
@@ -253,6 +257,59 @@ class DatalabClient(BaseDatalabClient):
|
|
|
253
257
|
|
|
254
258
|
return item["item_data"]
|
|
255
259
|
|
|
260
|
+
def get_item_graph(
|
|
261
|
+
self,
|
|
262
|
+
item_id: str | None = None,
|
|
263
|
+
collection_id: str | None = None,
|
|
264
|
+
hide_collections: bool | None = None,
|
|
265
|
+
max_depth: int | None = None,
|
|
266
|
+
as_networkx: bool = False,
|
|
267
|
+
) -> dict[str, list[dict[str, Any]]] | nx.DiGraph:
|
|
268
|
+
"""Get the graph of relationships between items.
|
|
269
|
+
|
|
270
|
+
With no arguments, the global item graph for all items visible to the
|
|
271
|
+
authenticated user is returned. If `item_id` is provided, only the
|
|
272
|
+
neighbourhood of that item is returned.
|
|
273
|
+
|
|
274
|
+
Parameters:
|
|
275
|
+
item_id: The ID of the item to centre the graph on (optional).
|
|
276
|
+
If not provided, the global graph is returned.
|
|
277
|
+
collection_id: Restrict the global graph to members of this
|
|
278
|
+
collection (optional; ignored when `item_id` is provided).
|
|
279
|
+
hide_collections: Whether to omit collections as nodes in the graph.
|
|
280
|
+
If `None`, the server default is used (hidden, for recent versions).
|
|
281
|
+
max_depth: The number of relationship hops to follow from `item_id`.
|
|
282
|
+
If `None`, the server default is used (1). Requires a recent
|
|
283
|
+
version of datalab.
|
|
284
|
+
as_networkx: Whether to return the graph as a `networkx.DiGraph`
|
|
285
|
+
(requires the optional `networkx` dependency).
|
|
286
|
+
|
|
287
|
+
Returns:
|
|
288
|
+
A dictionary with `nodes` and `edges` keys, in the Cytoscape.js
|
|
289
|
+
format used by the datalab UI, or a `networkx.DiGraph` if
|
|
290
|
+
`as_networkx` is `True`.
|
|
291
|
+
|
|
292
|
+
"""
|
|
293
|
+
graph_url = f"{self.datalab_api_url}/item-graph"
|
|
294
|
+
if item_id is not None:
|
|
295
|
+
graph_url += f"/{item_id}"
|
|
296
|
+
|
|
297
|
+
params: dict[str, Any] = {}
|
|
298
|
+
if collection_id is not None:
|
|
299
|
+
params["collection_id"] = collection_id
|
|
300
|
+
if hide_collections is not None:
|
|
301
|
+
params["hide_collections"] = str(hide_collections).lower()
|
|
302
|
+
if max_depth is not None:
|
|
303
|
+
params["max_depth"] = max_depth
|
|
304
|
+
|
|
305
|
+
graph = self._get(graph_url, params=params)
|
|
306
|
+
graph = {"nodes": graph.get("nodes", []), "edges": graph.get("edges", [])}
|
|
307
|
+
|
|
308
|
+
if as_networkx:
|
|
309
|
+
return networkx_from_item_graph(graph)
|
|
310
|
+
|
|
311
|
+
return graph
|
|
312
|
+
|
|
256
313
|
def get_item_files(self, item_id: str) -> None:
|
|
257
314
|
"""Download all the files for a given item and save them locally
|
|
258
315
|
in the current working directory.
|
|
@@ -268,10 +325,14 @@ class DatalabClient(BaseDatalabClient):
|
|
|
268
325
|
if Path(f["name"]).exists():
|
|
269
326
|
warnings.warn(f"Will not overwrite existing file {f['name']}")
|
|
270
327
|
continue
|
|
328
|
+
# This streams directly from the session rather than going through
|
|
329
|
+
# `_request`, so the elevation flag has to be applied by hand.
|
|
330
|
+
stream_kwargs = self._apply_elevation("GET", {})
|
|
271
331
|
with open(f["name"], "wb") as file:
|
|
272
|
-
with self.session.stream(
|
|
273
|
-
|
|
274
|
-
|
|
332
|
+
with self.session.stream(
|
|
333
|
+
"GET", url, follow_redirects=True, **stream_kwargs
|
|
334
|
+
) as response:
|
|
335
|
+
file.writelines(response.iter_bytes(chunk_size=1024))
|
|
275
336
|
|
|
276
337
|
def get_block(
|
|
277
338
|
self,
|
|
@@ -4,7 +4,7 @@ import re
|
|
|
4
4
|
import warnings
|
|
5
5
|
from getpass import getpass
|
|
6
6
|
from importlib.metadata import version
|
|
7
|
-
from typing import Any
|
|
7
|
+
from typing import Any
|
|
8
8
|
|
|
9
9
|
import httpx
|
|
10
10
|
from rich.logging import RichHandler
|
|
@@ -33,8 +33,8 @@ class BaseDatalabClient(metaclass=AutoPrettyPrint):
|
|
|
33
33
|
readable by users.
|
|
34
34
|
"""
|
|
35
35
|
|
|
36
|
-
_api_key:
|
|
37
|
-
_session:
|
|
36
|
+
_api_key: str | None = None
|
|
37
|
+
_session: httpx.Client | None = None
|
|
38
38
|
_headers: dict[str, str]
|
|
39
39
|
_timeout: httpx.Timeout = httpx.Timeout(10.0, read=60.0)
|
|
40
40
|
|
|
@@ -52,13 +52,28 @@ class BaseDatalabClient(metaclass=AutoPrettyPrint):
|
|
|
52
52
|
block_info: list[dict[str, Any]] = []
|
|
53
53
|
"""The `data` response from the `/info/blocks` endpoint of the Datalab API."""
|
|
54
54
|
|
|
55
|
-
bad_server_versions:
|
|
55
|
+
bad_server_versions: tuple[tuple[int, int, int]] | None = ((0, 2, 0),)
|
|
56
56
|
"""Any known server versions that are not supported by this client."""
|
|
57
57
|
|
|
58
58
|
min_server_version: tuple[int, int, int] = (0, 1, 0)
|
|
59
59
|
"""The minimum supported server version that this client supports."""
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
elevate_permissions: bool = False
|
|
62
|
+
"""Whether to make requests in datalab's admin super-user mode, i.e., with
|
|
63
|
+
access to items belonging to other users.
|
|
64
|
+
|
|
65
|
+
This only has an effect if the API key in use belongs to an admin account;
|
|
66
|
+
for any other account the server ignores the request and normal permissions
|
|
67
|
+
apply. Can be set at any point in the client's lifetime to toggle the
|
|
68
|
+
behaviour.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(
|
|
72
|
+
self,
|
|
73
|
+
datalab_api_url: str,
|
|
74
|
+
log_level: str = "WARNING",
|
|
75
|
+
elevate_permissions: bool = False,
|
|
76
|
+
):
|
|
62
77
|
"""Creates an authenticated client.
|
|
63
78
|
|
|
64
79
|
An API key is required to authenticate requests. The client will attempt to load it from a
|
|
@@ -72,10 +87,14 @@ class BaseDatalabClient(metaclass=AutoPrettyPrint):
|
|
|
72
87
|
to resolve the underlying API URL (e.g., `https://demo-api.datalab-org.io`
|
|
73
88
|
will 'redirect' to `https://demo-api.datalab-org.io`).
|
|
74
89
|
log_level: The logging level to use for the client. Defaults to "WARNING".
|
|
90
|
+
elevate_permissions: Whether to read items belonging to other users, using
|
|
91
|
+
datalab's admin super-user mode. Requires the API key in use to belong
|
|
92
|
+
to an admin account; ignored by the server otherwise. Defaults to False.
|
|
75
93
|
|
|
76
94
|
|
|
77
95
|
"""
|
|
78
96
|
|
|
97
|
+
self.elevate_permissions = elevate_permissions
|
|
79
98
|
self.datalab_api_url = datalab_api_url
|
|
80
99
|
if not self.datalab_api_url:
|
|
81
100
|
raise ValueError("No Datalab API URL provided.")
|
|
@@ -94,9 +113,7 @@ class BaseDatalabClient(metaclass=AutoPrettyPrint):
|
|
|
94
113
|
|
|
95
114
|
self._datalab_api_versions: list[str] = self.info["attributes"]["available_api_versions"]
|
|
96
115
|
self._datalab_server_version: str = self.info["attributes"]["server_version"]
|
|
97
|
-
self._datalab_instance_prefix:
|
|
98
|
-
"identifier_prefix"
|
|
99
|
-
)
|
|
116
|
+
self._datalab_instance_prefix: str | None = self.info["attributes"].get("identifier_prefix")
|
|
100
117
|
|
|
101
118
|
self._find_api_key()
|
|
102
119
|
|
|
@@ -184,7 +201,7 @@ class BaseDatalabClient(metaclass=AutoPrettyPrint):
|
|
|
184
201
|
if self._api_key is None:
|
|
185
202
|
key_env_var = "DATALAB_API_KEY"
|
|
186
203
|
|
|
187
|
-
api_key:
|
|
204
|
+
api_key: str | None = None
|
|
188
205
|
|
|
189
206
|
# probe the prefixed environment variable first
|
|
190
207
|
if self._datalab_instance_prefix is not None:
|
|
@@ -229,7 +246,7 @@ class BaseDatalabClient(metaclass=AutoPrettyPrint):
|
|
|
229
246
|
self._session.close()
|
|
230
247
|
|
|
231
248
|
def _handle_response(
|
|
232
|
-
self, response: httpx.Response, url: str, expected_status:
|
|
249
|
+
self, response: httpx.Response, url: str, expected_status: int | list[int] = 200
|
|
233
250
|
) -> dict[str, Any]:
|
|
234
251
|
"""Handle HTTP response with consistent error handling.
|
|
235
252
|
|
|
@@ -353,7 +370,7 @@ This is likely a server-side bug. Please report this issue to the datalab develo
|
|
|
353
370
|
return completed_tasks, error_tasks
|
|
354
371
|
|
|
355
372
|
def _request(
|
|
356
|
-
self, method: str, url: str, expected_status:
|
|
373
|
+
self, method: str, url: str, expected_status: int | list[int] = 200, **kwargs
|
|
357
374
|
) -> dict[str, Any]:
|
|
358
375
|
"""Make an HTTP request with consistent error handling.
|
|
359
376
|
|
|
@@ -367,6 +384,7 @@ This is likely a server-side bug. Please report this issue to the datalab develo
|
|
|
367
384
|
The JSON response data
|
|
368
385
|
"""
|
|
369
386
|
timeout = kwargs.pop("timeout", None)
|
|
387
|
+
kwargs = self._apply_elevation(method, kwargs)
|
|
370
388
|
try:
|
|
371
389
|
response = self.session.request(
|
|
372
390
|
method, url, follow_redirects=True, timeout=timeout, **kwargs
|
|
@@ -375,13 +393,27 @@ This is likely a server-side bug. Please report this issue to the datalab develo
|
|
|
375
393
|
except (httpx.RequestError, httpx.HTTPStatusError) as e:
|
|
376
394
|
raise DatalabAPIError(f"Request failed for {url}: {e}")
|
|
377
395
|
|
|
396
|
+
def _apply_elevation(self, method: str, kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
397
|
+
"""Add datalab's super-user flag to a read when running elevated.
|
|
398
|
+
|
|
399
|
+
datalab treats an admin as an ordinary user on GET requests unless the
|
|
400
|
+
request opts in with `sudo=1`, but grants them full access on writes
|
|
401
|
+
with no opt-in at all — so only reads need the flag. Any `params` the
|
|
402
|
+
caller supplied are preserved.
|
|
403
|
+
"""
|
|
404
|
+
if not self.elevate_permissions or method.upper() != "GET":
|
|
405
|
+
return kwargs
|
|
406
|
+
|
|
407
|
+
params = dict(kwargs.pop("params", None) or {})
|
|
408
|
+
params.setdefault("sudo", "1")
|
|
409
|
+
kwargs["params"] = params
|
|
410
|
+
return kwargs
|
|
411
|
+
|
|
378
412
|
def _get(self, url: str, **kwargs) -> dict[str, Any]:
|
|
379
413
|
"""Make a GET request with error handling."""
|
|
380
414
|
return self._request("GET", url, **kwargs)
|
|
381
415
|
|
|
382
|
-
def _post(
|
|
383
|
-
self, url: str, expected_status: Union[int, list[int]] = 200, **kwargs
|
|
384
|
-
) -> dict[str, Any]:
|
|
416
|
+
def _post(self, url: str, expected_status: int | list[int] = 200, **kwargs) -> dict[str, Any]:
|
|
385
417
|
"""Make a POST request with error handling."""
|
|
386
418
|
return self._request("POST", url, expected_status, **kwargs)
|
|
387
419
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import random
|
|
1
2
|
import time
|
|
2
|
-
from typing import Annotated
|
|
3
|
+
from typing import Annotated
|
|
3
4
|
|
|
4
5
|
import typer
|
|
5
6
|
from click_shell import make_click_shell
|
|
@@ -23,7 +24,7 @@ console = Console()
|
|
|
23
24
|
@app.callback(invoke_without_command=True)
|
|
24
25
|
def launch(
|
|
25
26
|
ctx: typer.Context,
|
|
26
|
-
instance_url: Annotated[
|
|
27
|
+
instance_url: Annotated[str | None, typer.Argument()] = None,
|
|
27
28
|
animate_intro: bool = True,
|
|
28
29
|
):
|
|
29
30
|
"""Makes an interactive REPL-style interface using the subcommands below."""
|
|
@@ -68,7 +69,7 @@ def launch(
|
|
|
68
69
|
|
|
69
70
|
def _get_client(
|
|
70
71
|
ctx: typer.Context,
|
|
71
|
-
instance_url:
|
|
72
|
+
instance_url: str | None = None,
|
|
72
73
|
log_level: str = "WARNING",
|
|
73
74
|
):
|
|
74
75
|
client = getattr(ctx, "client", None)
|
|
@@ -91,7 +92,7 @@ def _get_instance_url(ctx: typer.Context):
|
|
|
91
92
|
@app.command()
|
|
92
93
|
def authenticate(
|
|
93
94
|
ctx: typer.Context,
|
|
94
|
-
instance_url: Annotated[
|
|
95
|
+
instance_url: Annotated[str | None, typer.Argument()] = None,
|
|
95
96
|
log_level: str = "WARNING",
|
|
96
97
|
):
|
|
97
98
|
client = _get_client(ctx, instance_url, log_level)
|
|
@@ -105,7 +106,7 @@ def authenticate(
|
|
|
105
106
|
def get(
|
|
106
107
|
ctx: typer.Context,
|
|
107
108
|
item_type: str,
|
|
108
|
-
instance_url: Annotated[
|
|
109
|
+
instance_url: Annotated[str | None, typer.Argument()] = None,
|
|
109
110
|
page_limit: int = 10,
|
|
110
111
|
log_level: str = "WARNING",
|
|
111
112
|
):
|
|
@@ -143,8 +144,6 @@ def info(ctx: typer.Context, instance_url: str, log_level: str = "WARNING"):
|
|
|
143
144
|
|
|
144
145
|
def _make_fancy_intro(animate=True):
|
|
145
146
|
"""Bit of fun, make an animated datalab logo intro to the CLI."""
|
|
146
|
-
import random
|
|
147
|
-
|
|
148
147
|
intro_ascii = """
|
|
149
148
|
oooo o8 o888 oooo
|
|
150
149
|
ooooo888 ooooooo o888oo ooooooo 888 ooooooo 888ooooo
|
|
@@ -4,7 +4,7 @@ from rich.console import Console
|
|
|
4
4
|
from rich.pretty import pprint
|
|
5
5
|
from rich.table import Table
|
|
6
6
|
|
|
7
|
-
__all__ = ("AutoPrettyPrint", "bokeh_from_json", "pretty_displayer")
|
|
7
|
+
__all__ = ("AutoPrettyPrint", "bokeh_from_json", "networkx_from_item_graph", "pretty_displayer")
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def pretty_displayer(method):
|
|
@@ -77,3 +77,39 @@ def bokeh_from_json(block_data, show=True):
|
|
|
77
77
|
bokeh_show(curdoc().roots[0])
|
|
78
78
|
|
|
79
79
|
return curdoc()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def networkx_from_item_graph(graph):
|
|
83
|
+
"""Convert an item graph, as returned by the `/item-graph` endpoint,
|
|
84
|
+
into a `networkx.DiGraph`.
|
|
85
|
+
|
|
86
|
+
Nodes are keyed by their ID (item ID, or `"Collection: <collection_id>"`
|
|
87
|
+
for collections), with all other node data stored as attributes.
|
|
88
|
+
Edges point from parent to child.
|
|
89
|
+
|
|
90
|
+
Parameters:
|
|
91
|
+
graph: A dictionary with `nodes` and `edges` keys in Cytoscape.js format.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
A `networkx.DiGraph` of the item relationships.
|
|
95
|
+
|
|
96
|
+
"""
|
|
97
|
+
try:
|
|
98
|
+
import networkx as nx
|
|
99
|
+
except ImportError as exc:
|
|
100
|
+
raise ImportError(
|
|
101
|
+
"The `networkx` package is required to create a graph; "
|
|
102
|
+
"install it with `pip install datalab-api[networkx]`."
|
|
103
|
+
) from exc
|
|
104
|
+
|
|
105
|
+
nx_graph = nx.DiGraph()
|
|
106
|
+
for node in graph.get("nodes", []):
|
|
107
|
+
data = dict(node["data"])
|
|
108
|
+
nx_graph.add_node(data.pop("id"), **data)
|
|
109
|
+
for edge in graph.get("edges", []):
|
|
110
|
+
data = dict(edge["data"])
|
|
111
|
+
source, target = data.pop("source"), data.pop("target")
|
|
112
|
+
data.pop("id", None)
|
|
113
|
+
nx_graph.add_edge(source, target, **data)
|
|
114
|
+
|
|
115
|
+
return nx_graph
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datalab-api
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A Python API for instances of the datalab data management platform (https://github.com/datalab-org/datalab).
|
|
5
5
|
Author-email: Matthew Evans <matthew@ml-evs.science>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -9,7 +9,6 @@ Project-URL: repository, https://github.com/datalab-org/datalab-api
|
|
|
9
9
|
Project-URL: documentation, https://datalab-python-api.readthedocs.io
|
|
10
10
|
Project-URL: changelog, https://github.com/datalab-org/datalab-api/releases
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
@@ -18,13 +17,16 @@ Classifier: Intended Audience :: Science/Research
|
|
|
18
17
|
Classifier: Intended Audience :: Information Technology
|
|
19
18
|
Classifier: Topic :: Other/Nonlisted Topic
|
|
20
19
|
Classifier: Topic :: Scientific/Engineering
|
|
21
|
-
Requires-Python: >=3.
|
|
20
|
+
Requires-Python: >=3.10
|
|
22
21
|
Description-Content-Type: text/markdown
|
|
23
22
|
License-File: LICENSE
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist: bokeh~=2.4; platform_system != "Emscripten"
|
|
23
|
+
Requires-Dist: httpx2~=2.13
|
|
26
24
|
Requires-Dist: rich<15,>=13
|
|
27
|
-
|
|
25
|
+
Provides-Extra: plotting
|
|
26
|
+
Requires-Dist: bokeh~=2.4; platform_system != "Emscripten" and extra == "plotting"
|
|
27
|
+
Requires-Dist: numpy<2; extra == "plotting"
|
|
28
|
+
Provides-Extra: networkx
|
|
29
|
+
Requires-Dist: networkx>=3; extra == "networkx"
|
|
28
30
|
Provides-Extra: dev
|
|
29
31
|
Requires-Dist: pre-commit<5,>=3; extra == "dev"
|
|
30
32
|
Requires-Dist: pytest~=8.0; extra == "dev"
|
|
@@ -108,7 +110,6 @@ This package implements basic functionality for displaying and manipulating entr
|
|
|
108
110
|
from datalab_api import DatalabClient
|
|
109
111
|
|
|
110
112
|
with DatalabClient("https://demo-api.datalab-org.io") as client:
|
|
111
|
-
|
|
112
113
|
# List all items of a given type
|
|
113
114
|
items = client.get_items()
|
|
114
115
|
|
|
@@ -117,9 +118,31 @@ with DatalabClient("https://demo-api.datalab-org.io") as client:
|
|
|
117
118
|
|
|
118
119
|
# Upload a file to an item
|
|
119
120
|
file_response = client.upload_file(filepath="my_echem_data.mpr", item_id="test")
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Elevated (admin) permissions
|
|
120
124
|
|
|
125
|
+
By default a client can only see items its account owns or has been shared on.
|
|
126
|
+
Admins can opt in to datalab's super-user mode to read items belonging to other users, by passing `elevate_permissions`:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
with DatalabClient("https://demo-api.datalab-org.io", elevate_permissions=True) as client:
|
|
130
|
+
# Finds the item even if it belongs to another user
|
|
131
|
+
item = client.get_item(item_id="someone-elses-sample")
|
|
121
132
|
```
|
|
122
133
|
|
|
134
|
+
This sends datalab's `sudo=1` flag on read requests.
|
|
135
|
+
It requires the API key in use to belong to an **admin** account; for any other account the server ignores
|
|
136
|
+
it and normal permissions apply.
|
|
137
|
+
|
|
138
|
+
The attribute can also be toggled at any point in the client's lifetime:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
client.elevate_permissions = True
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Note that objects created while elevated (e.g. uploaded files) are still recorded as belonging to the admin's own account, not the item owner's.
|
|
145
|
+
|
|
123
146
|
### Command-line interface (CLI)
|
|
124
147
|
|
|
125
148
|
There is also an EXPERIMENTAL CLI, exposed via the `datalab` command.
|
|
@@ -1,9 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
httpx2~=2.13
|
|
2
2
|
rich<15,>=13
|
|
3
|
-
numpy<2
|
|
4
|
-
|
|
5
|
-
[:platform_system != "Emscripten"]
|
|
6
|
-
bokeh~=2.4
|
|
7
3
|
|
|
8
4
|
[all]
|
|
9
5
|
datalab-api[cli,dev,docs]
|
|
@@ -24,3 +20,12 @@ mkdocs-material
|
|
|
24
20
|
mkdocstrings[python-legacy]
|
|
25
21
|
mkdocs-awesome-pages-plugin
|
|
26
22
|
mkdocs-jupyter
|
|
23
|
+
|
|
24
|
+
[networkx]
|
|
25
|
+
networkx>=3
|
|
26
|
+
|
|
27
|
+
[plotting]
|
|
28
|
+
numpy<2
|
|
29
|
+
|
|
30
|
+
[plotting:platform_system != "Emscripten"]
|
|
31
|
+
bokeh~=2.4
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
1
3
|
import pytest
|
|
2
4
|
import respx
|
|
3
5
|
from httpx import Response
|
|
@@ -40,7 +42,9 @@ def test_sample_operations(mocked_api, fake_api_url):
|
|
|
40
42
|
# Check that the full server error message is shown for a missing item
|
|
41
43
|
with pytest.raises(
|
|
42
44
|
DatalabAPIError,
|
|
43
|
-
match=
|
|
45
|
+
match=re.escape(
|
|
46
|
+
"No matching items for match={'item_id': 'TEST'} with current authorization"
|
|
47
|
+
),
|
|
44
48
|
):
|
|
45
49
|
client.get_item("TEST", display=True)
|
|
46
50
|
assert mocked_api["sample-missing-TEST"].called
|
|
@@ -57,7 +61,7 @@ def test_collection_get(fake_api_url, mocked_api):
|
|
|
57
61
|
assert mocked_api["info"].called
|
|
58
62
|
assert mocked_api["info-blocks"].called
|
|
59
63
|
|
|
60
|
-
|
|
64
|
+
_collection, _children = client.get_collection("test_collection", display=True)
|
|
61
65
|
assert mocked_api["collection-test_collection"].called
|
|
62
66
|
|
|
63
67
|
|
|
@@ -123,3 +127,78 @@ def test_upload_file_not_modified(fake_api_url, mocked_api, tmp_path):
|
|
|
123
127
|
assert result["file_id"] == "6810ab6bb2b13a2be6c1e37f"
|
|
124
128
|
assert result["is_update"] is True
|
|
125
129
|
assert result["not_modified"] is True
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_elevate_permissions_adds_sudo_to_reads(mocked_api, fake_api_url):
|
|
133
|
+
"""An elevated client sends `sudo=1` on GETs (and nothing else)."""
|
|
134
|
+
with DatalabClient(fake_api_url, elevate_permissions=True) as client:
|
|
135
|
+
client.get_item("KUVEKJ")
|
|
136
|
+
|
|
137
|
+
request = mocked_api["sample-KUVEKJ"].calls.last.request
|
|
138
|
+
assert request.url.params.get("sudo") == "1"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_elevate_permissions_is_toggleable(mocked_api, fake_api_url):
|
|
142
|
+
"""`elevate_permissions` can be flipped after construction."""
|
|
143
|
+
with DatalabClient(fake_api_url) as client:
|
|
144
|
+
client.get_item("KUVEKJ")
|
|
145
|
+
assert "sudo" not in mocked_api["sample-KUVEKJ"].calls.last.request.url.params
|
|
146
|
+
|
|
147
|
+
client.elevate_permissions = True
|
|
148
|
+
client.get_item("KUVEKJ")
|
|
149
|
+
assert mocked_api["sample-KUVEKJ"].calls.last.request.url.params.get("sudo") == "1"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_elevate_permissions_not_sent_on_writes(mocked_api, fake_api_url):
|
|
153
|
+
"""datalab grants admins full write access with no opt-in, so the flag
|
|
154
|
+
must not leak onto non-GET requests."""
|
|
155
|
+
with DatalabClient(fake_api_url, elevate_permissions=True) as client:
|
|
156
|
+
with pytest.raises(DatalabAPIError):
|
|
157
|
+
# `/save-item/` is mocked as a 500; we only care about the request
|
|
158
|
+
client.update_item("KUVEKJ", {"description": "test"})
|
|
159
|
+
|
|
160
|
+
request = mocked_api["bad-save"].calls.last.request
|
|
161
|
+
assert request.method == "POST"
|
|
162
|
+
assert "sudo" not in request.url.params
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_elevate_permissions_preserves_existing_params(mocked_api, fake_api_url):
|
|
166
|
+
"""A caller-supplied query string must survive elevation."""
|
|
167
|
+
with DatalabClient(fake_api_url, elevate_permissions=True) as client:
|
|
168
|
+
client._get(
|
|
169
|
+
f"{client.datalab_api_url}/get-item-data/KUVEKJ",
|
|
170
|
+
params={"load_blocks": "false"},
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
request = mocked_api["sample-KUVEKJ"].calls.last.request
|
|
174
|
+
assert request.url.params.get("load_blocks") == "false"
|
|
175
|
+
assert request.url.params.get("sudo") == "1"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_get_item_graph(mocked_api, fake_api_url):
|
|
179
|
+
with DatalabClient(fake_api_url) as client:
|
|
180
|
+
graph = client.get_item_graph(display=True)
|
|
181
|
+
assert mocked_api["item-graph"].called
|
|
182
|
+
assert not mocked_api["item-graph"].calls.last.request.url.params
|
|
183
|
+
assert len(graph["nodes"]) == 3
|
|
184
|
+
assert len(graph["edges"]) == 2
|
|
185
|
+
|
|
186
|
+
graph = client.get_item_graph(
|
|
187
|
+
"KUVEKJ", hide_collections=False, max_depth=2, collection_id="test_collection"
|
|
188
|
+
)
|
|
189
|
+
params = mocked_api["item-graph-KUVEKJ"].calls.last.request.url.params
|
|
190
|
+
assert params["hide_collections"] == "false"
|
|
191
|
+
assert params["max_depth"] == "2"
|
|
192
|
+
assert params["collection_id"] == "test_collection"
|
|
193
|
+
assert set(graph) == {"nodes", "edges"}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def test_get_item_graph_networkx(mocked_api, fake_api_url):
|
|
197
|
+
nx = pytest.importorskip("networkx")
|
|
198
|
+
with DatalabClient(fake_api_url) as client:
|
|
199
|
+
graph = client.get_item_graph("KUVEKJ", as_networkx=True)
|
|
200
|
+
assert isinstance(graph, nx.DiGraph)
|
|
201
|
+
assert set(graph.nodes) == {"parent", "KUVEKJ", "sibling"}
|
|
202
|
+
assert set(graph.successors("parent")) == {"KUVEKJ", "sibling"}
|
|
203
|
+
assert graph.nodes["KUVEKJ"]["name"] == "Child"
|
|
204
|
+
assert graph.nodes["KUVEKJ"]["special"] is True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|