sunstone-data-platform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunstone_data/__init__.py +36 -0
- sunstone_data/_version.py +24 -0
- sunstone_data/auth/__init__.py +43 -0
- sunstone_data/auth/client_creds.py +239 -0
- sunstone_data/auth/pkce.py +307 -0
- sunstone_data/auth/store.py +97 -0
- sunstone_data/catalog.py +175 -0
- sunstone_data/cli/__init__.py +5 -0
- sunstone_data/cli/_app.py +26 -0
- sunstone_data/cli/_errors.py +31 -0
- sunstone_data/cli/commands/__init__.py +22 -0
- sunstone_data/cli/commands/data_auth.py +120 -0
- sunstone_data/cli/commands/data_import.py +157 -0
- sunstone_data/cli/commands/data_info.py +166 -0
- sunstone_data/cli/commands/data_publish.py +69 -0
- sunstone_data/cli/commands/ls.py +49 -0
- sunstone_data/cli/commands/prov.py +58 -0
- sunstone_data/cli/commands/refs.py +142 -0
- sunstone_data/cli/commands/show.py +139 -0
- sunstone_data/cli/commands/snapshots.py +77 -0
- sunstone_data/cli/commands/sparql.py +68 -0
- sunstone_data/cli/main.py +30 -0
- sunstone_data/config.py +250 -0
- sunstone_data/discovery.py +317 -0
- sunstone_data/env_commands.py +97 -0
- sunstone_data/env_section.py +86 -0
- sunstone_data/graph/__init__.py +36 -0
- sunstone_data/graph/client.py +328 -0
- sunstone_data/graph/dataset.py +438 -0
- sunstone_data/graph/terms.py +34 -0
- sunstone_data/identifiers.py +155 -0
- sunstone_data/import_/__init__.py +28 -0
- sunstone_data/import_/fetcher.py +259 -0
- sunstone_data/import_/owner.py +130 -0
- sunstone_data/import_/parser.py +492 -0
- sunstone_data/import_/pipeline.py +275 -0
- sunstone_data/import_/storage.py +135 -0
- sunstone_data/metadata/__init__.py +41 -0
- sunstone_data/metadata/builder.py +276 -0
- sunstone_data/metadata/models.py +87 -0
- sunstone_data/metadata/provenance.py +171 -0
- sunstone_data/metadata/provenance_rdf.py +102 -0
- sunstone_data/plugin.py +62 -0
- sunstone_data/publish.py +134 -0
- sunstone_data/query.py +624 -0
- sunstone_data/resolve.py +178 -0
- sunstone_data/url.py +78 -0
- sunstone_data/utils/__init__.py +12 -0
- sunstone_data/utils/checksum.py +14 -0
- sunstone_data/utils/encoding.py +16 -0
- sunstone_data/utils/naming.py +38 -0
- sunstone_data/versioning/__init__.py +17 -0
- sunstone_data/versioning/schema_evolution.py +180 -0
- sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
- sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
- sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
- sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
- sunstone_pipeline/__init__.py +1 -0
- sunstone_pipeline/assets/__init__.py +11 -0
- sunstone_pipeline/assets/mirrors.py +123 -0
- sunstone_pipeline/assets/transforms.py +74 -0
- sunstone_pipeline/definitions.py +25 -0
- sunstone_pipeline/io_manager.py +56 -0
- sunstone_pipeline/resources/__init__.py +6 -0
- sunstone_pipeline/resources/iceberg.py +29 -0
- sunstone_pipeline/resources/refs.py +88 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""HTTP client for graph-server.
|
|
2
|
+
|
|
3
|
+
graph-server speaks the SPARQL 1.1 Graph Store Protocol (GSP) over named
|
|
4
|
+
graphs, plus small REST endpoints for branches, tags and concept maps. Its error
|
|
5
|
+
bodies are bare ``text/plain`` with no JSON envelope, so this client maps
|
|
6
|
+
``(status, route)`` to a typed exception and never reads a body for meaning
|
|
7
|
+
(spec 180 §5).
|
|
8
|
+
|
|
9
|
+
ETags are graph-server's optimistic-concurrency token. The wire form is the
|
|
10
|
+
strong-quoted ``"txn-9"``; callers here see and pass the unquoted ``txn-9``,
|
|
11
|
+
or the sentinel ``*`` for "whatever is current".
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
|
|
20
|
+
from sunstone_data.config import current_environment_name, get_settings
|
|
21
|
+
|
|
22
|
+
#: graph-server rejects request bodies larger than this.
|
|
23
|
+
MAX_REQUEST_BYTES = 64 * 1024 * 1024
|
|
24
|
+
|
|
25
|
+
TURTLE = "text/turtle"
|
|
26
|
+
N_TRIPLES = "application/n-triples"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AuthError(RuntimeError):
|
|
30
|
+
"""Not authenticated: no token, or the token was rejected twice."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, message: str = "Not authenticated; run `sunstone data login`"):
|
|
33
|
+
super().__init__(message)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class PermissionError(RuntimeError): # noqa: A001 - deliberate shadow, see module docstring
|
|
37
|
+
"""Authenticated, but the token lacks the role the route needs.
|
|
38
|
+
|
|
39
|
+
The server's 403 body is the opaque string ``forbidden``, so the missing
|
|
40
|
+
role is worked out client-side from the method and path.
|
|
41
|
+
|
|
42
|
+
Attributes:
|
|
43
|
+
role: Role the route requires ("read" or "readwrite").
|
|
44
|
+
client_id: Client identity this client was configured with, if any.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, role: str, client_id: str | None):
|
|
48
|
+
self.role = role
|
|
49
|
+
self.client_id = client_id
|
|
50
|
+
super().__init__(f"missing role {role!r} for client {client_id!r}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class NotFound(RuntimeError):
|
|
54
|
+
"""The branch, tag, graph or concept map does not exist."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ConflictError(RuntimeError):
|
|
58
|
+
"""An ``If-Match`` precondition failed.
|
|
59
|
+
|
|
60
|
+
Attributes:
|
|
61
|
+
etag: The branch's current ETag, unquoted. Refetch and retry with it.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(self, etag: str, message: str | None = None):
|
|
65
|
+
self.etag = etag
|
|
66
|
+
super().__init__(message or f"precondition failed; current ETag is {etag!r}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class TooLarge(RuntimeError):
|
|
70
|
+
"""Request body exceeded the 64 MiB cap."""
|
|
71
|
+
|
|
72
|
+
def __init__(self, message: str = "request body exceeds the 64 MiB cap"):
|
|
73
|
+
super().__init__(message)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class UnsupportedMedia(RuntimeError):
|
|
77
|
+
"""Unsupported media type: Turtle or N-Triples only."""
|
|
78
|
+
|
|
79
|
+
def __init__(self, message: str = f"unsupported media type; use {TURTLE} or {N_TRIPLES}"):
|
|
80
|
+
super().__init__(message)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class Unavailable(RuntimeError):
|
|
84
|
+
"""graph SoR unavailable; safe to retry."""
|
|
85
|
+
|
|
86
|
+
def __init__(self, message: str = "graph SoR unavailable; safe to retry"):
|
|
87
|
+
super().__init__(message)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _required_role(method: str, path: str) -> str:
|
|
91
|
+
"""Mirror graph-server's own role classifier (graph-core ``auth.rs``)."""
|
|
92
|
+
if path == "/sparql" and method in ("GET", "HEAD", "POST"):
|
|
93
|
+
return "read"
|
|
94
|
+
return "read" if method in ("GET", "HEAD") else "readwrite"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _unquote_etag(value: str | None) -> str:
|
|
98
|
+
"""Strip the strong-quote wrapper from an ETag header value."""
|
|
99
|
+
return (value or "").strip().strip('"')
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class GraphClient:
|
|
103
|
+
"""Thin HTTP client for a graph-server instance.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
base_url: graph-server base URL (e.g. "http://localhost:8080").
|
|
107
|
+
token_provider: Returns a bearer token, or None when unauthenticated.
|
|
108
|
+
Called lazily on first use and again after a 401.
|
|
109
|
+
client_id: Client identity, reported in `PermissionError`. Informational
|
|
110
|
+
only — this client never decodes the token.
|
|
111
|
+
timeout: HTTP request timeout in seconds.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def __init__(
|
|
115
|
+
self,
|
|
116
|
+
base_url: str,
|
|
117
|
+
token_provider: Callable[[], str | None] | None = None,
|
|
118
|
+
client_id: str | None = None,
|
|
119
|
+
timeout: float = 10.0,
|
|
120
|
+
):
|
|
121
|
+
self._base_url = base_url.rstrip("/")
|
|
122
|
+
self._token_provider = token_provider
|
|
123
|
+
self._client_id = client_id
|
|
124
|
+
self._timeout = timeout
|
|
125
|
+
self._token: str | None = None
|
|
126
|
+
self._token_loaded = False
|
|
127
|
+
|
|
128
|
+
# --- Graph Store Protocol ---
|
|
129
|
+
|
|
130
|
+
def get_graph(self, branch: str, graph_iri: str, accept: str = TURTLE) -> str:
|
|
131
|
+
"""Fetch a named graph's contents as text.
|
|
132
|
+
|
|
133
|
+
Args:
|
|
134
|
+
branch: Branch name.
|
|
135
|
+
graph_iri: IRI of the named graph.
|
|
136
|
+
accept: `N_TRIPLES` for N-Triples; anything else yields Turtle.
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
The serialized graph.
|
|
140
|
+
|
|
141
|
+
Raises:
|
|
142
|
+
NotFound: The branch or graph has no visible quads.
|
|
143
|
+
"""
|
|
144
|
+
return self.get_graph_with_etag(branch, graph_iri, accept)[0]
|
|
145
|
+
|
|
146
|
+
def get_graph_with_etag(self, branch: str, graph_iri: str, accept: str = TURTLE) -> tuple[str, str]:
|
|
147
|
+
"""Fetch a named graph's contents and the branch's ETag together.
|
|
148
|
+
|
|
149
|
+
A read-modify-write needs both from the *same* response: the ETag of a
|
|
150
|
+
later, separate read would not cover the bytes being modified.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
(serialized graph, ETag unquoted).
|
|
154
|
+
|
|
155
|
+
Raises:
|
|
156
|
+
NotFound: The branch or graph has no visible quads.
|
|
157
|
+
"""
|
|
158
|
+
resp = self._request(
|
|
159
|
+
"GET",
|
|
160
|
+
f"/branches/{branch}/graphs",
|
|
161
|
+
params={"graph": graph_iri},
|
|
162
|
+
headers={"Accept": accept},
|
|
163
|
+
)
|
|
164
|
+
return resp.text, _unquote_etag(resp.headers.get("ETag"))
|
|
165
|
+
|
|
166
|
+
def put_graph(
|
|
167
|
+
self,
|
|
168
|
+
branch: str,
|
|
169
|
+
graph_iri: str,
|
|
170
|
+
data: str | bytes,
|
|
171
|
+
if_match: str,
|
|
172
|
+
content_type: str = TURTLE,
|
|
173
|
+
) -> str:
|
|
174
|
+
"""Replace a named graph. Returns the new ETag, unquoted."""
|
|
175
|
+
return self._write_graph("PUT", branch, graph_iri, if_match, data, content_type)
|
|
176
|
+
|
|
177
|
+
def post_graph(
|
|
178
|
+
self,
|
|
179
|
+
branch: str,
|
|
180
|
+
graph_iri: str,
|
|
181
|
+
data: str | bytes,
|
|
182
|
+
if_match: str,
|
|
183
|
+
content_type: str = TURTLE,
|
|
184
|
+
) -> str:
|
|
185
|
+
"""Merge triples into a named graph. Returns the new ETag, unquoted."""
|
|
186
|
+
return self._write_graph("POST", branch, graph_iri, if_match, data, content_type)
|
|
187
|
+
|
|
188
|
+
def delete_graph(self, branch: str, graph_iri: str, if_match: str) -> str:
|
|
189
|
+
"""Delete a named graph. Returns the new ETag, unquoted."""
|
|
190
|
+
return self._write_graph("DELETE", branch, graph_iri, if_match)
|
|
191
|
+
|
|
192
|
+
def _write_graph(
|
|
193
|
+
self,
|
|
194
|
+
method: str,
|
|
195
|
+
branch: str,
|
|
196
|
+
graph_iri: str,
|
|
197
|
+
if_match: str,
|
|
198
|
+
data: str | bytes | None = None,
|
|
199
|
+
content_type: str | None = None,
|
|
200
|
+
) -> str:
|
|
201
|
+
headers = {"If-Match": if_match if if_match == "*" else f'"{if_match}"'}
|
|
202
|
+
if content_type is not None:
|
|
203
|
+
headers["Content-Type"] = content_type
|
|
204
|
+
resp = self._request(
|
|
205
|
+
method,
|
|
206
|
+
f"/branches/{branch}/graphs",
|
|
207
|
+
params={"graph": graph_iri},
|
|
208
|
+
headers=headers,
|
|
209
|
+
content=data.encode() if isinstance(data, str) else data,
|
|
210
|
+
)
|
|
211
|
+
return _unquote_etag(resp.headers.get("ETag"))
|
|
212
|
+
|
|
213
|
+
# --- Branches ---
|
|
214
|
+
|
|
215
|
+
def list_branches(self) -> list[dict]:
|
|
216
|
+
"""List branches. Rows carry name, parent, head_txn, created_by, timestamps."""
|
|
217
|
+
return self._request("GET", "/branches").json().get("branches", [])
|
|
218
|
+
|
|
219
|
+
def create_branch(self, name: str, parent: str | None = None) -> None:
|
|
220
|
+
"""Create a branch. Server defaults the parent to "main"."""
|
|
221
|
+
self._request("PUT", f"/branches/{name}", params={"parent": parent} if parent else None)
|
|
222
|
+
|
|
223
|
+
def delete_branch(self, name: str) -> None:
|
|
224
|
+
"""Delete a branch."""
|
|
225
|
+
self._request("DELETE", f"/branches/{name}")
|
|
226
|
+
|
|
227
|
+
# --- Tags ---
|
|
228
|
+
|
|
229
|
+
def list_tags(self) -> list[dict]:
|
|
230
|
+
"""List tags. Rows carry name, branch, at_txn."""
|
|
231
|
+
return self._request("GET", "/tags").json().get("tags", [])
|
|
232
|
+
|
|
233
|
+
def create_tag(self, name: str, branch: str, at_txn: int | None = None) -> dict:
|
|
234
|
+
"""Tag a branch at a txn (default: the branch's current head)."""
|
|
235
|
+
body: dict = {"branch": branch}
|
|
236
|
+
if at_txn is not None:
|
|
237
|
+
body["at_txn"] = at_txn
|
|
238
|
+
return self._request("PUT", f"/tags/{name}", json=body).json()
|
|
239
|
+
|
|
240
|
+
def delete_tag(self, name: str) -> None:
|
|
241
|
+
"""Delete a tag."""
|
|
242
|
+
self._request("DELETE", f"/tags/{name}")
|
|
243
|
+
|
|
244
|
+
# --- Concept maps ---
|
|
245
|
+
|
|
246
|
+
def allocate_conceptmap(self) -> dict:
|
|
247
|
+
"""Allocate a concept map. Returns {"conceptmap": ..., "graph_iri": ...}."""
|
|
248
|
+
return self._request("POST", "/conceptmaps/").json()
|
|
249
|
+
|
|
250
|
+
# --- SPARQL ---
|
|
251
|
+
|
|
252
|
+
def sparql(self, query: str, accept: str = "application/sparql-results+json") -> httpx.Response:
|
|
253
|
+
"""Run a SPARQL query. Pass-through: the raw response is returned."""
|
|
254
|
+
return self._request(
|
|
255
|
+
"POST",
|
|
256
|
+
"/sparql",
|
|
257
|
+
headers={"Content-Type": "application/sparql-query", "Accept": accept},
|
|
258
|
+
content=query.encode(),
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
# --- HTTP ---
|
|
262
|
+
|
|
263
|
+
def _bearer(self, refresh: bool = False) -> str | None:
|
|
264
|
+
if refresh or not self._token_loaded:
|
|
265
|
+
self._token = self._token_provider() if self._token_provider else None
|
|
266
|
+
self._token_loaded = True
|
|
267
|
+
return self._token
|
|
268
|
+
|
|
269
|
+
def _request(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
270
|
+
resp = self._send(method, path, self._bearer(), **kwargs)
|
|
271
|
+
if resp.status_code == 401 and self._token_provider is not None:
|
|
272
|
+
# Spec 180 §5: refresh once, then give up.
|
|
273
|
+
resp = self._send(method, path, self._bearer(refresh=True), **kwargs)
|
|
274
|
+
if resp.status_code >= 400:
|
|
275
|
+
exc = self._error(resp, method, path)
|
|
276
|
+
if exc is not None:
|
|
277
|
+
raise exc
|
|
278
|
+
resp.raise_for_status() # statuses outside the spec table (400, 409, ...)
|
|
279
|
+
return resp
|
|
280
|
+
|
|
281
|
+
def _send(self, method: str, path: str, token: str | None, **kwargs) -> httpx.Response:
|
|
282
|
+
headers = dict(kwargs.pop("headers", None) or {})
|
|
283
|
+
if token:
|
|
284
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
285
|
+
return httpx.request(
|
|
286
|
+
method,
|
|
287
|
+
f"{self._base_url}{path}",
|
|
288
|
+
headers=headers,
|
|
289
|
+
timeout=self._timeout,
|
|
290
|
+
**kwargs,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
def _error(self, resp: httpx.Response, method: str, path: str) -> Exception | None:
|
|
294
|
+
"""Map (status, route) to a typed exception; None for statuses not in spec §5."""
|
|
295
|
+
status = resp.status_code
|
|
296
|
+
if status == 401:
|
|
297
|
+
return AuthError()
|
|
298
|
+
if status == 403:
|
|
299
|
+
return PermissionError(_required_role(method, path), self._client_id)
|
|
300
|
+
if status == 404:
|
|
301
|
+
return NotFound(f"{method} {path} not found")
|
|
302
|
+
if status == 412:
|
|
303
|
+
return ConflictError(_unquote_etag(resp.headers.get("ETag")))
|
|
304
|
+
if status == 413:
|
|
305
|
+
return TooLarge()
|
|
306
|
+
if status == 415:
|
|
307
|
+
return UnsupportedMedia()
|
|
308
|
+
if status == 503:
|
|
309
|
+
return Unavailable()
|
|
310
|
+
return None
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def get_graph_client(env_name: str | None = None) -> GraphClient:
|
|
314
|
+
"""Create a GraphClient from the current settings."""
|
|
315
|
+
settings = get_settings()
|
|
316
|
+
env = env_name or current_environment_name()
|
|
317
|
+
|
|
318
|
+
def token_provider() -> str | None:
|
|
319
|
+
from sunstone_data.auth.client_creds import resolve_access_token
|
|
320
|
+
|
|
321
|
+
with httpx.Client() as client:
|
|
322
|
+
return resolve_access_token(settings, env, client)
|
|
323
|
+
|
|
324
|
+
return GraphClient(
|
|
325
|
+
settings.graph_url,
|
|
326
|
+
token_provider=token_provider,
|
|
327
|
+
client_id=settings.client_id or None,
|
|
328
|
+
)
|