sunstone-data-platform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunstone_data/__init__.py +36 -0
- sunstone_data/_version.py +24 -0
- sunstone_data/auth/__init__.py +43 -0
- sunstone_data/auth/client_creds.py +239 -0
- sunstone_data/auth/pkce.py +307 -0
- sunstone_data/auth/store.py +97 -0
- sunstone_data/catalog.py +175 -0
- sunstone_data/cli/__init__.py +5 -0
- sunstone_data/cli/_app.py +26 -0
- sunstone_data/cli/_errors.py +31 -0
- sunstone_data/cli/commands/__init__.py +22 -0
- sunstone_data/cli/commands/data_auth.py +120 -0
- sunstone_data/cli/commands/data_import.py +157 -0
- sunstone_data/cli/commands/data_info.py +166 -0
- sunstone_data/cli/commands/data_publish.py +69 -0
- sunstone_data/cli/commands/ls.py +49 -0
- sunstone_data/cli/commands/prov.py +58 -0
- sunstone_data/cli/commands/refs.py +142 -0
- sunstone_data/cli/commands/show.py +139 -0
- sunstone_data/cli/commands/snapshots.py +77 -0
- sunstone_data/cli/commands/sparql.py +68 -0
- sunstone_data/cli/main.py +30 -0
- sunstone_data/config.py +250 -0
- sunstone_data/discovery.py +317 -0
- sunstone_data/env_commands.py +97 -0
- sunstone_data/env_section.py +86 -0
- sunstone_data/graph/__init__.py +36 -0
- sunstone_data/graph/client.py +328 -0
- sunstone_data/graph/dataset.py +438 -0
- sunstone_data/graph/terms.py +34 -0
- sunstone_data/identifiers.py +155 -0
- sunstone_data/import_/__init__.py +28 -0
- sunstone_data/import_/fetcher.py +259 -0
- sunstone_data/import_/owner.py +130 -0
- sunstone_data/import_/parser.py +492 -0
- sunstone_data/import_/pipeline.py +275 -0
- sunstone_data/import_/storage.py +135 -0
- sunstone_data/metadata/__init__.py +41 -0
- sunstone_data/metadata/builder.py +276 -0
- sunstone_data/metadata/models.py +87 -0
- sunstone_data/metadata/provenance.py +171 -0
- sunstone_data/metadata/provenance_rdf.py +102 -0
- sunstone_data/plugin.py +62 -0
- sunstone_data/publish.py +134 -0
- sunstone_data/query.py +624 -0
- sunstone_data/resolve.py +178 -0
- sunstone_data/url.py +78 -0
- sunstone_data/utils/__init__.py +12 -0
- sunstone_data/utils/checksum.py +14 -0
- sunstone_data/utils/encoding.py +16 -0
- sunstone_data/utils/naming.py +38 -0
- sunstone_data/versioning/__init__.py +17 -0
- sunstone_data/versioning/schema_evolution.py +180 -0
- sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
- sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
- sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
- sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
- sunstone_pipeline/__init__.py +1 -0
- sunstone_pipeline/assets/__init__.py +11 -0
- sunstone_pipeline/assets/mirrors.py +123 -0
- sunstone_pipeline/assets/transforms.py +74 -0
- sunstone_pipeline/definitions.py +25 -0
- sunstone_pipeline/io_manager.py +56 -0
- sunstone_pipeline/resources/__init__.py +6 -0
- sunstone_pipeline/resources/iceberg.py +29 -0
- sunstone_pipeline/resources/refs.py +88 -0
sunstone_data/publish.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Publish a dataset description into its named graph (spec 180 §6).
|
|
2
|
+
|
|
3
|
+
Two modes, and the difference matters:
|
|
4
|
+
|
|
5
|
+
- The default merges (`POST`). It only ever adds triples, so nothing already
|
|
6
|
+
in the graph — the catalog's own descriptor, another writer's annotations —
|
|
7
|
+
can be lost.
|
|
8
|
+
- `replace=True` retracts the package's previous triples first. graph-server's
|
|
9
|
+
`PUT` computes `deletes = visible - payload` on its side, so "retract only
|
|
10
|
+
what we own" is expressed by sending everything visible *except* our own
|
|
11
|
+
previous triples, plus the new ones.
|
|
12
|
+
|
|
13
|
+
Which triples the package owns is not a hand-maintained predicate list.
|
|
14
|
+
`description_to_graph` and `graph_to_description` are inverses, so reading the
|
|
15
|
+
visible graph back into a description and re-serializing it reproduces exactly
|
|
16
|
+
the triples this mapping wrote before — no more. A predicate list would be
|
|
17
|
+
wrong anyway: `rdf:type` appears on our version nodes *and* on the catalog's
|
|
18
|
+
own descriptor.
|
|
19
|
+
|
|
20
|
+
Every write carries `If-Match` from the read that preceded it. A 412 means
|
|
21
|
+
someone committed in between: refetch once, rebuild, retry. A second 412 is
|
|
22
|
+
raised rather than looped on.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from sunstone_data.graph.client import N_TRIPLES, ConflictError, GraphClient, NotFound, get_graph_client
|
|
28
|
+
from sunstone_data.graph.dataset import (
|
|
29
|
+
DatasetDescription,
|
|
30
|
+
Graph,
|
|
31
|
+
build_graph,
|
|
32
|
+
description_to_graph,
|
|
33
|
+
graph_to_description,
|
|
34
|
+
parse_ntriples,
|
|
35
|
+
serialize,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
_EMPTY: Graph = build_graph([])
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def publish(
|
|
42
|
+
description: DatasetDescription,
|
|
43
|
+
branch: str = "main",
|
|
44
|
+
replace: bool = False,
|
|
45
|
+
client: GraphClient | None = None,
|
|
46
|
+
) -> str:
|
|
47
|
+
"""Write `description` into its dataset graph and return the new ETag.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
description: What to publish. Its `dataset_iri` names the graph.
|
|
51
|
+
branch: Branch to write on.
|
|
52
|
+
replace: Retract the package's previous triples first. Triples written
|
|
53
|
+
by anything else are kept either way.
|
|
54
|
+
client: Graph server client. Defaults to `get_graph_client()`.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
The ETag after the write, unquoted. Republishing unchanged metadata is
|
|
58
|
+
a no-op server-side: the ETag comes back unchanged and no transaction
|
|
59
|
+
is created.
|
|
60
|
+
|
|
61
|
+
Raises:
|
|
62
|
+
ConflictError: Two writes in a row lost the race.
|
|
63
|
+
"""
|
|
64
|
+
client = client or get_graph_client()
|
|
65
|
+
try:
|
|
66
|
+
return _attempt(client, description, branch, replace)
|
|
67
|
+
except ConflictError:
|
|
68
|
+
# One refetch-and-retry, then give up: a caller stuck behind a busy
|
|
69
|
+
# writer should hear about it, not spin.
|
|
70
|
+
return _attempt(client, description, branch, replace)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _attempt(client: GraphClient, description: DatasetDescription, branch: str, replace: bool) -> str:
|
|
74
|
+
iri = description.dataset_iri
|
|
75
|
+
visible, if_match = _read(client, branch, iri)
|
|
76
|
+
ours = description_to_graph(description)
|
|
77
|
+
|
|
78
|
+
if not replace:
|
|
79
|
+
return client.post_graph(branch, iri, serialize(ours), if_match=if_match, content_type=N_TRIPLES)
|
|
80
|
+
|
|
81
|
+
previously_ours = description_to_graph(graph_to_description(visible, iri)) if visible else _EMPTY
|
|
82
|
+
payload = (visible - previously_ours) | ours
|
|
83
|
+
return client.put_graph(branch, iri, serialize(payload), if_match=if_match, content_type=N_TRIPLES)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def publish_graph(
|
|
87
|
+
dataset_iri: str,
|
|
88
|
+
graph: Graph,
|
|
89
|
+
branch: str = "main",
|
|
90
|
+
client: GraphClient | None = None,
|
|
91
|
+
) -> str:
|
|
92
|
+
"""`POST` an arbitrary graph of triples into a dataset's named graph, merging.
|
|
93
|
+
|
|
94
|
+
Same GET -> `If-Match` -> `POST` -> retry-once-on-conflict shape as
|
|
95
|
+
`publish`, for callers that already have a `Graph` (e.g. import
|
|
96
|
+
provenance, spec 180 §9 steps 3-4) rather than a `DatasetDescription`.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
dataset_iri: IRI of the graph to merge into.
|
|
100
|
+
graph: Triples to add. Nothing already in the graph is removed.
|
|
101
|
+
branch: Branch to write on.
|
|
102
|
+
client: Graph server client. Defaults to `get_graph_client()`.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
The ETag after the write, unquoted.
|
|
106
|
+
|
|
107
|
+
Raises:
|
|
108
|
+
ConflictError: Two writes in a row lost the race.
|
|
109
|
+
"""
|
|
110
|
+
client = client or get_graph_client()
|
|
111
|
+
payload = serialize(graph)
|
|
112
|
+
try:
|
|
113
|
+
return _post(client, branch, dataset_iri, payload)
|
|
114
|
+
except ConflictError:
|
|
115
|
+
return _post(client, branch, dataset_iri, payload)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _post(client: GraphClient, branch: str, iri: str, payload: bytes) -> str:
|
|
119
|
+
_, if_match = _read(client, branch, iri)
|
|
120
|
+
return client.post_graph(branch, iri, payload, if_match=if_match, content_type=N_TRIPLES)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _read(client: GraphClient, branch: str, iri: str) -> tuple[Graph, str]:
|
|
124
|
+
"""The graph as currently visible, and the `If-Match` value for the next write.
|
|
125
|
+
|
|
126
|
+
A 404 means nothing is published for this dataset yet, so there is no ETag
|
|
127
|
+
to echo. `*` is the right precondition there: it asks only that the branch
|
|
128
|
+
has a commit at all, not that this graph has one.
|
|
129
|
+
"""
|
|
130
|
+
try:
|
|
131
|
+
text, etag = client.get_graph_with_etag(branch, iri, accept=N_TRIPLES)
|
|
132
|
+
except NotFound:
|
|
133
|
+
return _EMPTY, "*"
|
|
134
|
+
return parse_ntriples(text), etag or "*"
|