sunstone-data-platform 1.2.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/.gitignore +1 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/PKG-INFO +13 -9
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/README.md +9 -7
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/pyproject.toml +6 -1
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/_version.py +2 -2
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/__init__.py +3 -2
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/pkce.py +1 -1
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/stepup.py +4 -2
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/store.py +65 -8
- sunstone_data_platform-1.4.0/src/sunstone_data/branch.py +66 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/_app.py +4 -1
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/__init__.py +1 -0
- sunstone_data_platform-1.4.0/src/sunstone_data/cli/commands/conceptmap.py +81 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/data_auth.py +9 -9
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/data_import.py +11 -3
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/data_publish.py +10 -1
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/refs.py +48 -1
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/show.py +9 -2
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/snapshots.py +10 -2
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/main.py +4 -4
- sunstone_data_platform-1.4.0/src/sunstone_data/conceptmap.py +233 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/config.py +24 -2
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/env_section.py +7 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/graph/client.py +127 -7
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/plugin.py +24 -7
- sunstone_data_platform-1.4.0/src/sunstone_data/push.py +134 -0
- sunstone_data_platform-1.4.0/src/sunstone_data/queries/conceptmap-mappings.rq +41 -0
- sunstone_data_platform-1.4.0/src/sunstone_data/sssom.py +267 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/client_creds.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/catalog.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/_errors.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/data_info.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/ls.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/prov.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/cli/commands/sparql.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/discovery.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/env_commands.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/graph/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/graph/dataset.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/graph/terms.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/identifiers.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/import_/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/import_/fetcher.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/import_/owner.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/import_/parser.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/import_/pipeline.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/import_/storage.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/metadata/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/metadata/builder.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/metadata/models.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/metadata/provenance.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/metadata/provenance_rdf.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/publish.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/query.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/resolve.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/url.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/utils/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/utils/checksum.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/utils/encoding.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/utils/naming.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/versioning/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/versioning/schema_evolution.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/assets/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/assets/mirrors.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/assets/transforms.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/definitions.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/io_manager.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/resources/__init__.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/resources/iceberg.py +0 -0
- {sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_pipeline/resources/refs.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sunstone-data-platform
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Sunstone Data Platform - Data lakehouse with semantic metadata
|
|
5
5
|
Author-email: Sunstone Institute <stig@sunstone.institute>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -9,13 +9,15 @@ Requires-Dist: charset-normalizer>=3.4.0
|
|
|
9
9
|
Requires-Dist: duckdb>=1.5.3
|
|
10
10
|
Requires-Dist: fastexcel>=0.20.2
|
|
11
11
|
Requires-Dist: httpx>=0.27.0
|
|
12
|
+
Requires-Dist: keyring>=25
|
|
12
13
|
Requires-Dist: publicsuffixlist>=1.0.2.20261003
|
|
13
14
|
Requires-Dist: pyiceberg[duckdb,gcsfs,s3fs]>=0.10.0
|
|
14
15
|
Requires-Dist: python-dateutil>=2.9.0.post0
|
|
15
16
|
Requires-Dist: python-dotenv>=1.2.2
|
|
16
17
|
Requires-Dist: python-slugify>=8.0.0
|
|
18
|
+
Requires-Dist: rdflib>=7.6
|
|
17
19
|
Requires-Dist: rich>=14.0.0
|
|
18
|
-
Requires-Dist: sunstone-py>=1.
|
|
20
|
+
Requires-Dist: sunstone-py>=1.19.0
|
|
19
21
|
Requires-Dist: typer>=0.27.1
|
|
20
22
|
Provides-Extra: pipelines
|
|
21
23
|
Requires-Dist: dagster-docker>=0.29.0; extra == 'pipelines'
|
|
@@ -61,7 +63,7 @@ full data flow.
|
|
|
61
63
|
## Endpoints
|
|
62
64
|
|
|
63
65
|
- `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
|
|
64
|
-
(see [
|
|
66
|
+
(see [DP-SPEC-13](https://worklode.dev.sunstoneinstitute.ai/DP-SPEC-13))
|
|
65
67
|
- `/ui/` — the data explorer, a 3D view of the data graph, served by
|
|
66
68
|
graph-server (see [docs/data-explorer.md](docs/data-explorer.md))
|
|
67
69
|
|
|
@@ -199,7 +201,7 @@ uv run pytest tests/unit/test_query.py -v
|
|
|
199
201
|
|
|
200
202
|
| Command | Description |
|
|
201
203
|
|---------|-------------|
|
|
202
|
-
| `sunstone
|
|
204
|
+
| `sunstone auth login`, `logout`, `whoami` | Sign in to graph-server (Keycloak) |
|
|
203
205
|
| `sunstone data ls [namespace]` | List namespaces and tables |
|
|
204
206
|
| `sunstone data show <table>` | Dataset metadata from the graph (table, Turtle or JSON-LD) |
|
|
205
207
|
| `sunstone data info [table]` | Iceberg table properties for one or all datasets |
|
|
@@ -263,8 +265,8 @@ src/sunstone_data/
|
|
|
263
265
|
## Documentation
|
|
264
266
|
|
|
265
267
|
- [docs/README.md](docs/README.md) - Index of all documentation
|
|
266
|
-
- [docs/architecture.md](docs/architecture.md) - System overview and
|
|
267
|
-
- [docs/
|
|
268
|
+
- [docs/architecture.md](docs/architecture.md) - System overview, data flow, and an index of the specs in Worklode. The Python client and CLI are in [DP-SPEC-1](https://worklode.dev.sunstoneinstitute.ai/DP-SPEC-1).
|
|
269
|
+
- [docs/releasing.md](docs/releasing.md) - How to release the server and the client
|
|
268
270
|
|
|
269
271
|
## Resetting the Stack
|
|
270
272
|
|
|
@@ -284,7 +286,7 @@ MIT
|
|
|
284
286
|
`graph-server` requires a Keycloak bearer token on every route except
|
|
285
287
|
`/healthz`/`/readyz`. Access is role-based: the client roles in your token
|
|
286
288
|
determine which environment you can use. See
|
|
287
|
-
|
|
289
|
+
[DP-SPEC-14](https://worklode.dev.sunstoneinstitute.ai/DP-SPEC-14).
|
|
288
290
|
|
|
289
291
|
No hostnames or client ids ship in this package. Each environment's
|
|
290
292
|
`base_iri` comes from config: Sunstone-managed machines get
|
|
@@ -297,11 +299,13 @@ graph-server publishes the OIDC issuer and public client id at
|
|
|
297
299
|
|
|
298
300
|
```bash
|
|
299
301
|
sunstone env init # cache discovery for configured environments
|
|
300
|
-
sunstone
|
|
302
|
+
sunstone auth login # browser sign-in (authorization code + PKCE)
|
|
301
303
|
```
|
|
302
304
|
|
|
303
305
|
A browser window opens for Keycloak SSO; the refresh token renews silently
|
|
304
|
-
until the SSO session expires.
|
|
306
|
+
until the SSO session expires. The refresh token is stored in the OS keyring
|
|
307
|
+
(macOS Keychain, Windows Credential Locker, Linux Secret Service). Without a
|
|
308
|
+
usable keyring it falls back to a mode-0600 file under `~/.cache/sunstone/`.
|
|
305
309
|
|
|
306
310
|
### Services (CI, automation)
|
|
307
311
|
|
|
@@ -35,7 +35,7 @@ full data flow.
|
|
|
35
35
|
## Endpoints
|
|
36
36
|
|
|
37
37
|
- `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
|
|
38
|
-
(see [
|
|
38
|
+
(see [DP-SPEC-13](https://worklode.dev.sunstoneinstitute.ai/DP-SPEC-13))
|
|
39
39
|
- `/ui/` — the data explorer, a 3D view of the data graph, served by
|
|
40
40
|
graph-server (see [docs/data-explorer.md](docs/data-explorer.md))
|
|
41
41
|
|
|
@@ -173,7 +173,7 @@ uv run pytest tests/unit/test_query.py -v
|
|
|
173
173
|
|
|
174
174
|
| Command | Description |
|
|
175
175
|
|---------|-------------|
|
|
176
|
-
| `sunstone
|
|
176
|
+
| `sunstone auth login`, `logout`, `whoami` | Sign in to graph-server (Keycloak) |
|
|
177
177
|
| `sunstone data ls [namespace]` | List namespaces and tables |
|
|
178
178
|
| `sunstone data show <table>` | Dataset metadata from the graph (table, Turtle or JSON-LD) |
|
|
179
179
|
| `sunstone data info [table]` | Iceberg table properties for one or all datasets |
|
|
@@ -237,8 +237,8 @@ src/sunstone_data/
|
|
|
237
237
|
## Documentation
|
|
238
238
|
|
|
239
239
|
- [docs/README.md](docs/README.md) - Index of all documentation
|
|
240
|
-
- [docs/architecture.md](docs/architecture.md) - System overview and
|
|
241
|
-
- [docs/
|
|
240
|
+
- [docs/architecture.md](docs/architecture.md) - System overview, data flow, and an index of the specs in Worklode. The Python client and CLI are in [DP-SPEC-1](https://worklode.dev.sunstoneinstitute.ai/DP-SPEC-1).
|
|
241
|
+
- [docs/releasing.md](docs/releasing.md) - How to release the server and the client
|
|
242
242
|
|
|
243
243
|
## Resetting the Stack
|
|
244
244
|
|
|
@@ -258,7 +258,7 @@ MIT
|
|
|
258
258
|
`graph-server` requires a Keycloak bearer token on every route except
|
|
259
259
|
`/healthz`/`/readyz`. Access is role-based: the client roles in your token
|
|
260
260
|
determine which environment you can use. See
|
|
261
|
-
|
|
261
|
+
[DP-SPEC-14](https://worklode.dev.sunstoneinstitute.ai/DP-SPEC-14).
|
|
262
262
|
|
|
263
263
|
No hostnames or client ids ship in this package. Each environment's
|
|
264
264
|
`base_iri` comes from config: Sunstone-managed machines get
|
|
@@ -271,11 +271,13 @@ graph-server publishes the OIDC issuer and public client id at
|
|
|
271
271
|
|
|
272
272
|
```bash
|
|
273
273
|
sunstone env init # cache discovery for configured environments
|
|
274
|
-
sunstone
|
|
274
|
+
sunstone auth login # browser sign-in (authorization code + PKCE)
|
|
275
275
|
```
|
|
276
276
|
|
|
277
277
|
A browser window opens for Keycloak SSO; the refresh token renews silently
|
|
278
|
-
until the SSO session expires.
|
|
278
|
+
until the SSO session expires. The refresh token is stored in the OS keyring
|
|
279
|
+
(macOS Keychain, Windows Credential Locker, Linux Secret Service). Without a
|
|
280
|
+
usable keyring it falls back to a mode-0600 file under `~/.cache/sunstone/`.
|
|
279
281
|
|
|
280
282
|
### Services (CI, automation)
|
|
281
283
|
|
|
@@ -13,7 +13,7 @@ authors = [
|
|
|
13
13
|
{ name = "Sunstone Institute", email = "stig@sunstone.institute" }
|
|
14
14
|
]
|
|
15
15
|
dependencies = [
|
|
16
|
-
"sunstone-py>=1.
|
|
16
|
+
"sunstone-py>=1.19.0",
|
|
17
17
|
"pyiceberg[s3fs,gcsfs,duckdb]>=0.10.0",
|
|
18
18
|
"duckdb>=1.5.3",
|
|
19
19
|
"python-dotenv>=1.2.2",
|
|
@@ -25,6 +25,8 @@ dependencies = [
|
|
|
25
25
|
"python-slugify>=8.0.0",
|
|
26
26
|
"python-dateutil>=2.9.0.post0",
|
|
27
27
|
"publicsuffixlist>=1.0.2.20261003",
|
|
28
|
+
"keyring>=25",
|
|
29
|
+
"rdflib>=7.6",
|
|
28
30
|
]
|
|
29
31
|
|
|
30
32
|
[project.entry-points."sunstone.plugins"]
|
|
@@ -40,6 +42,9 @@ pipelines = [
|
|
|
40
42
|
|
|
41
43
|
[tool.hatch.version]
|
|
42
44
|
source = "vcs"
|
|
45
|
+
# Only `v*` tags are client versions. Without --match, setuptools-scm takes the
|
|
46
|
+
# nearest tag of any kind and strips a `server-` prefix (DP-REQ-391).
|
|
47
|
+
raw-options = { git_describe_command = "git describe --dirty --tags --long --match v[0-9]*" }
|
|
43
48
|
|
|
44
49
|
[tool.hatch.build.hooks.vcs]
|
|
45
50
|
version-file = "src/sunstone_data/_version.py"
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1,
|
|
21
|
+
__version__ = version = '1.4.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 4, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
{sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/__init__.py
RENAMED
|
@@ -12,8 +12,9 @@ health, the UI and public views. Two ways to get one:
|
|
|
12
12
|
environment credential, so CI is never silently authenticated as whoever
|
|
13
13
|
last logged in on that machine.
|
|
14
14
|
|
|
15
|
-
Access tokens live in memory only. The refresh token is the sole
|
|
16
|
-
|
|
15
|
+
Access tokens live in memory only. The refresh token is the sole credential
|
|
16
|
+
kept between runs: in the OS keyring, or a mode-0600 file when no keyring is
|
|
17
|
+
usable.
|
|
17
18
|
"""
|
|
18
19
|
|
|
19
20
|
from sunstone_data.auth.client_creds import (
|
{sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/pkce.py
RENAMED
|
@@ -13,7 +13,7 @@ wildcard for ports, so the port cannot be ephemeral. The port is bound
|
|
|
13
13
|
*before* the authorization URL is built, because the ``redirect_uri`` must
|
|
14
14
|
name it.
|
|
15
15
|
|
|
16
|
-
This is the flow behind ``sunstone
|
|
16
|
+
This is the flow behind ``sunstone auth login`` (spec 180 §4). The public
|
|
17
17
|
client it signs in as is the one graph-server publishes as ``clientId`` in
|
|
18
18
|
``/ui/config.json``, the same one the graph explorer UI uses.
|
|
19
19
|
"""
|
{sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/stepup.py
RENAMED
|
@@ -83,10 +83,12 @@ def step_up(
|
|
|
83
83
|
"""
|
|
84
84
|
settings = get_settings()
|
|
85
85
|
if os.environ.get("SUNSTONE_ACCESS_TOKEN", "").strip() or (settings.client_id and settings.client_secret):
|
|
86
|
-
raise StepUpRequired("writes to
|
|
86
|
+
raise StepUpRequired("writes to a protected ref need a passkey sign-in; a service credential cannot step up")
|
|
87
87
|
session = store.read_session(env_name)
|
|
88
88
|
if session is None:
|
|
89
|
-
raise StepUpRequired(
|
|
89
|
+
raise StepUpRequired(
|
|
90
|
+
f"writes to a protected ref need a passkey sign-in; run `sunstone auth login` for {env_name!r}"
|
|
91
|
+
)
|
|
90
92
|
issuer = session.issuer or discovered_issuer(env_name, settings.graph_url)
|
|
91
93
|
authorization_endpoint = discover_authorization_endpoint(issuer, client)
|
|
92
94
|
token_endpoint = discover_token_endpoint(issuer, client)
|
{sunstone_data_platform-1.2.0 → sunstone_data_platform-1.4.0}/src/sunstone_data/auth/store.py
RENAMED
|
@@ -1,30 +1,40 @@
|
|
|
1
1
|
"""Per-environment cache for a human login session.
|
|
2
2
|
|
|
3
|
-
Holds the OAuth2 refresh token from ``sunstone
|
|
3
|
+
Holds the OAuth2 refresh token from ``sunstone auth login`` so a user logs in
|
|
4
4
|
once per environment. Access tokens are never stored here — they are minted on
|
|
5
5
|
demand and kept in memory only.
|
|
6
6
|
|
|
7
|
-
The
|
|
8
|
-
|
|
7
|
+
The refresh token goes in the OS keyring (macOS Keychain, Windows Credential
|
|
8
|
+
Locker, Secret Service on Linux) under service ``sunstone-data-platform`` and
|
|
9
|
+
the environment name. The rest of the session (issuer, client id, time) goes
|
|
10
|
+
in a file next to the discovery cache, one level down so the two never
|
|
11
|
+
collide::
|
|
9
12
|
|
|
10
13
|
$XDG_CACHE_HOME/sunstone/data-platform/auth/{env_name}.json
|
|
11
14
|
|
|
12
|
-
|
|
13
|
-
|
|
15
|
+
When no keyring is usable (e.g. a headless Linux box), the refresh token is
|
|
16
|
+
written into that file instead. The file is always written atomically and
|
|
17
|
+
chmod'ed ``0600``, so no other user on the machine can read it.
|
|
14
18
|
"""
|
|
15
19
|
|
|
16
20
|
from __future__ import annotations
|
|
17
21
|
|
|
18
22
|
import json
|
|
23
|
+
import logging
|
|
19
24
|
import os
|
|
20
25
|
import tempfile
|
|
21
26
|
import time
|
|
22
27
|
from dataclasses import asdict, dataclass, field, fields
|
|
23
28
|
from pathlib import Path
|
|
24
29
|
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
25
32
|
#: Mode a session file must end up with — owner read/write only.
|
|
26
33
|
SESSION_FILE_MODE = 0o600
|
|
27
34
|
|
|
35
|
+
#: Keyring service name; the keyring username is the environment name.
|
|
36
|
+
KEYRING_SERVICE = "sunstone-data-platform"
|
|
37
|
+
|
|
28
38
|
|
|
29
39
|
@dataclass(frozen=True)
|
|
30
40
|
class Session:
|
|
@@ -53,18 +63,56 @@ def session_path(env_name: str) -> Path:
|
|
|
53
63
|
return _auth_dir() / f"{env_name}.json"
|
|
54
64
|
|
|
55
65
|
|
|
66
|
+
def _keyring_set(env_name: str, token: str) -> bool:
|
|
67
|
+
"""Store ``token`` in the OS keyring. Returns False if no keyring is usable."""
|
|
68
|
+
try:
|
|
69
|
+
import keyring
|
|
70
|
+
|
|
71
|
+
keyring.set_password(KEYRING_SERVICE, env_name, token)
|
|
72
|
+
except Exception:
|
|
73
|
+
logger.debug("OS keyring not usable", exc_info=True)
|
|
74
|
+
return False
|
|
75
|
+
return True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _keyring_get(env_name: str) -> str | None:
|
|
79
|
+
try:
|
|
80
|
+
import keyring
|
|
81
|
+
|
|
82
|
+
return keyring.get_password(KEYRING_SERVICE, env_name)
|
|
83
|
+
except Exception:
|
|
84
|
+
logger.debug("OS keyring not usable", exc_info=True)
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _keyring_delete(env_name: str) -> None:
|
|
89
|
+
try:
|
|
90
|
+
import keyring
|
|
91
|
+
|
|
92
|
+
keyring.delete_password(KEYRING_SERVICE, env_name)
|
|
93
|
+
except Exception:
|
|
94
|
+
# Missing entry or no usable keyring: nothing to delete either way.
|
|
95
|
+
logger.debug("No keyring entry deleted", exc_info=True)
|
|
96
|
+
|
|
97
|
+
|
|
56
98
|
def write_session(env_name: str, session: Session) -> Path:
|
|
57
99
|
"""Persist a login session, atomically and mode 0600.
|
|
58
100
|
|
|
59
|
-
|
|
101
|
+
The refresh token goes in the OS keyring when one is usable, else in the
|
|
102
|
+
session file. Returns the path written.
|
|
60
103
|
"""
|
|
61
104
|
path = session_path(env_name)
|
|
105
|
+
data = asdict(session)
|
|
106
|
+
if _keyring_set(env_name, session.refresh_token):
|
|
107
|
+
del data["refresh_token"]
|
|
108
|
+
else:
|
|
109
|
+
logger.warning("No usable OS keyring; storing the refresh token in %s (mode 0600)", path)
|
|
62
110
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
111
|
fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
64
112
|
try:
|
|
65
113
|
os.chmod(tmp_name, SESSION_FILE_MODE)
|
|
66
114
|
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
67
|
-
json.dump(
|
|
115
|
+
json.dump(data, f, indent=2)
|
|
68
116
|
os.replace(tmp_name, path)
|
|
69
117
|
except Exception:
|
|
70
118
|
Path(tmp_name).unlink(missing_ok=True)
|
|
@@ -76,7 +124,10 @@ def write_session(env_name: str, session: Session) -> Path:
|
|
|
76
124
|
|
|
77
125
|
|
|
78
126
|
def read_session(env_name: str) -> Session | None:
|
|
79
|
-
"""Load a cached login session. Returns None if missing or unreadable.
|
|
127
|
+
"""Load a cached login session. Returns None if missing or unreadable.
|
|
128
|
+
|
|
129
|
+
A refresh token in the file (keyring fallback) wins over the keyring.
|
|
130
|
+
"""
|
|
80
131
|
try:
|
|
81
132
|
with open(session_path(env_name), encoding="utf-8") as f:
|
|
82
133
|
data = json.load(f)
|
|
@@ -86,6 +137,11 @@ def read_session(env_name: str) -> Session | None:
|
|
|
86
137
|
return None
|
|
87
138
|
field_names = {f.name for f in fields(Session)}
|
|
88
139
|
kwargs = {k: v for k, v in data.items() if k in field_names}
|
|
140
|
+
if not kwargs.get("refresh_token"):
|
|
141
|
+
token = _keyring_get(env_name)
|
|
142
|
+
if token is None:
|
|
143
|
+
return None
|
|
144
|
+
kwargs["refresh_token"] = token
|
|
89
145
|
try:
|
|
90
146
|
return Session(**kwargs)
|
|
91
147
|
except TypeError:
|
|
@@ -95,3 +151,4 @@ def read_session(env_name: str) -> Session | None:
|
|
|
95
151
|
def clear_session(env_name: str) -> None:
|
|
96
152
|
"""Remove the cached session for an environment, if any."""
|
|
97
153
|
session_path(env_name).unlink(missing_ok=True)
|
|
154
|
+
_keyring_delete(env_name)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Default data branch: the current git branch (or CI ref), normalized (DP-SPEC-12 section 12)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
_MAX_LENGTH = 128
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class NoDataBranchError(ValueError):
|
|
15
|
+
"""No data branch could be derived; the caller must pass ``--branch``."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def normalize_branch_name(name: str) -> str:
|
|
19
|
+
"""Turn a git branch name into a valid data branch name.
|
|
20
|
+
|
|
21
|
+
In order: characters outside ``[A-Za-z0-9_.-]`` become ``-``, runs of
|
|
22
|
+
``-`` collapse to one, a name not starting with a letter or ``_`` gets a
|
|
23
|
+
``_`` prefix, and the result is cut to 128 characters.
|
|
24
|
+
"""
|
|
25
|
+
name = re.sub(r"[^A-Za-z0-9_.-]", "-", name)
|
|
26
|
+
name = re.sub(r"-{2,}", "-", name)
|
|
27
|
+
if not re.match(r"[A-Za-z_]", name):
|
|
28
|
+
name = "_" + name
|
|
29
|
+
return name[:_MAX_LENGTH]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _git_branch(cwd: Path | None) -> str | None:
|
|
33
|
+
"""The checked-out git branch, or None (not a checkout, git missing, detached HEAD)."""
|
|
34
|
+
try:
|
|
35
|
+
result = subprocess.run(
|
|
36
|
+
["git", "symbolic-ref", "--short", "-q", "HEAD"],
|
|
37
|
+
cwd=cwd,
|
|
38
|
+
capture_output=True,
|
|
39
|
+
text=True,
|
|
40
|
+
timeout=10,
|
|
41
|
+
check=False,
|
|
42
|
+
)
|
|
43
|
+
except (OSError, subprocess.SubprocessError):
|
|
44
|
+
return None
|
|
45
|
+
name = result.stdout.strip()
|
|
46
|
+
return name if result.returncode == 0 and name else None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def current_data_branch(cwd: Path | None = None, env: Mapping[str, str] | None = None) -> str:
|
|
50
|
+
"""Return the data branch for ``cwd``, normalized.
|
|
51
|
+
|
|
52
|
+
Order: the checked-out git branch (also on a branch with no commits yet),
|
|
53
|
+
then ``GITHUB_HEAD_REF``, then ``GITHUB_REF_NAME``. CI checks out pull
|
|
54
|
+
requests on a detached HEAD, which is why the variables come second.
|
|
55
|
+
|
|
56
|
+
Raises:
|
|
57
|
+
NoDataBranchError: None of these gives a branch. Never falls back to ``main``.
|
|
58
|
+
"""
|
|
59
|
+
env = os.environ if env is None else env
|
|
60
|
+
name = _git_branch(cwd) or env.get("GITHUB_HEAD_REF") or env.get("GITHUB_REF_NAME")
|
|
61
|
+
if not name:
|
|
62
|
+
raise NoDataBranchError(
|
|
63
|
+
"Cannot tell which data branch to use: not on a git branch and GITHUB_HEAD_REF / "
|
|
64
|
+
"GITHUB_REF_NAME are unset. Pass --branch."
|
|
65
|
+
)
|
|
66
|
+
return normalize_branch_name(name)
|
|
@@ -21,6 +21,9 @@ app = typer.Typer(
|
|
|
21
21
|
data_app = typer.Typer(help="Data operations")
|
|
22
22
|
app.add_typer(data_app, name="data")
|
|
23
23
|
|
|
24
|
+
auth_app = typer.Typer(help="Sign in to the data platform")
|
|
25
|
+
app.add_typer(auth_app, name="auth")
|
|
26
|
+
|
|
24
27
|
console = Console()
|
|
25
28
|
|
|
26
|
-
__all__ = ["app", "data_app", "console"]
|
|
29
|
+
__all__ = ["app", "auth_app", "data_app", "console"]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""``sunstone data conceptmap create|update|sssom``: push a map-concepts scope directory, read its mappings (DP-REQ-323)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import asdict
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from sunstone_data.cli._app import console, data_app
|
|
13
|
+
from sunstone_data.cli._errors import cli_errors
|
|
14
|
+
|
|
15
|
+
conceptmap_app = typer.Typer(help="Push concept maps from map-concepts scope directories", no_args_is_help=True)
|
|
16
|
+
data_app.add_typer(conceptmap_app, name="conceptmap")
|
|
17
|
+
|
|
18
|
+
_SCOPE_DIR = typer.Argument(
|
|
19
|
+
..., help="Scope directory written by map-concepts (reads scheme/*.ttl and scheme/mappings.sssom.tsv)"
|
|
20
|
+
)
|
|
21
|
+
# Concept maps are published content, so the default is main, not the git
|
|
22
|
+
# branch of the directory the command runs in (DP-REQ-30).
|
|
23
|
+
_BRANCH = typer.Option("main", "--branch", "-b", help="Branch to write")
|
|
24
|
+
_BRANCH_READ = typer.Option("main", "--branch", "-b", help="Branch to read")
|
|
25
|
+
_JSON = typer.Option(False, "--json", help="Output as JSON")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _report(result, json_output: bool) -> None:
|
|
29
|
+
from urllib.parse import quote
|
|
30
|
+
|
|
31
|
+
from sunstone_data.config import get_settings
|
|
32
|
+
|
|
33
|
+
link = f"{get_settings().graph_url}/ui/#visualise?graph={quote(result.graph_iri, safe='')}"
|
|
34
|
+
if json_output:
|
|
35
|
+
print(json.dumps({**asdict(result), "url": link}))
|
|
36
|
+
return
|
|
37
|
+
console.print(
|
|
38
|
+
f"Concept map [cyan]{result.conceptmap}[/cyan]: {result.graph_iri} on {result.branch}, "
|
|
39
|
+
f"{result.triples} triples (ETag {result.etag})"
|
|
40
|
+
)
|
|
41
|
+
console.print(link)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@conceptmap_app.command("create")
|
|
45
|
+
def create_cmd(scope_dir: Path = _SCOPE_DIR, branch: str = _BRANCH, json_output: bool = _JSON) -> None:
|
|
46
|
+
"""Allocate a new concept map and push the scope directory to it."""
|
|
47
|
+
with cli_errors(json_output):
|
|
48
|
+
from sunstone_data import conceptmap
|
|
49
|
+
|
|
50
|
+
_report(conceptmap.create(scope_dir, branch), json_output)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@conceptmap_app.command("update")
|
|
54
|
+
def update_cmd(
|
|
55
|
+
n: int = typer.Argument(..., help="Concept map id, the <n> in /conceptmap/<n>"),
|
|
56
|
+
scope_dir: Path = _SCOPE_DIR,
|
|
57
|
+
branch: str = _BRANCH,
|
|
58
|
+
json_output: bool = _JSON,
|
|
59
|
+
) -> None:
|
|
60
|
+
"""Replace an existing concept map with the scope directory."""
|
|
61
|
+
with cli_errors(json_output):
|
|
62
|
+
from sunstone_data import conceptmap
|
|
63
|
+
|
|
64
|
+
_report(conceptmap.update(n, scope_dir, branch), json_output)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@conceptmap_app.command("sssom")
|
|
68
|
+
def sssom_cmd(
|
|
69
|
+
n: int = typer.Argument(..., help="Concept map id, the <n> in /conceptmap/<n>"),
|
|
70
|
+
branch: str = _BRANCH_READ,
|
|
71
|
+
output: Path | None = typer.Option(None, "--output", "-o", help="File to write (default: stdout)"),
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Write a concept map's mappings as an SSSOM TSV file."""
|
|
74
|
+
with cli_errors(False):
|
|
75
|
+
from sunstone_data import conceptmap
|
|
76
|
+
|
|
77
|
+
text = conceptmap.sssom(n, branch)
|
|
78
|
+
if output is None:
|
|
79
|
+
sys.stdout.write(text)
|
|
80
|
+
else:
|
|
81
|
+
output.write_text(text, encoding="utf-8")
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
"""``sunstone
|
|
1
|
+
"""``sunstone auth login`` / ``logout`` / ``whoami`` — human sign-in sessions.
|
|
2
2
|
|
|
3
3
|
``login`` runs the PKCE browser flow (:mod:`sunstone_data.auth.pkce`) and
|
|
4
|
-
caches only the refresh token, per environment,
|
|
5
|
-
minted on demand and never
|
|
6
|
-
not live token claims.
|
|
4
|
+
caches only the refresh token, per environment, in the OS keyring (see
|
|
5
|
+
:mod:`sunstone_data.auth.store`). Access tokens are minted on demand and never
|
|
6
|
+
stored, so ``whoami`` reports what was cached, not live token claims.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import typer
|
|
12
12
|
|
|
13
|
-
from sunstone_data.cli._app import
|
|
13
|
+
from sunstone_data.cli._app import auth_app, console
|
|
14
14
|
|
|
15
15
|
_ENV_OPTION_HELP = "Environment to act on (defaults to the active sunstone environment)."
|
|
16
16
|
|
|
@@ -21,7 +21,7 @@ def _resolve_env(env: str | None) -> str:
|
|
|
21
21
|
return env or current_environment_name()
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
@
|
|
24
|
+
@auth_app.command("login")
|
|
25
25
|
def login_cmd(
|
|
26
26
|
env: str | None = typer.Option(None, "--env", help=_ENV_OPTION_HELP),
|
|
27
27
|
) -> None:
|
|
@@ -78,7 +78,7 @@ def login_cmd(
|
|
|
78
78
|
console.print(f"[green]Logged in to '{env_name}'.[/green]")
|
|
79
79
|
|
|
80
80
|
|
|
81
|
-
@
|
|
81
|
+
@auth_app.command("logout")
|
|
82
82
|
def logout_cmd(
|
|
83
83
|
env: str | None = typer.Option(None, "--env", help=_ENV_OPTION_HELP),
|
|
84
84
|
) -> None:
|
|
@@ -94,7 +94,7 @@ def logout_cmd(
|
|
|
94
94
|
console.print(f"Not logged in to '{env_name}'; nothing to do.")
|
|
95
95
|
|
|
96
96
|
|
|
97
|
-
@
|
|
97
|
+
@auth_app.command("whoami")
|
|
98
98
|
def whoami_cmd(
|
|
99
99
|
env: str | None = typer.Option(None, "--env", help=_ENV_OPTION_HELP),
|
|
100
100
|
) -> None:
|
|
@@ -109,7 +109,7 @@ def whoami_cmd(
|
|
|
109
109
|
env_name = _resolve_env(env)
|
|
110
110
|
session = store.read_session(env_name)
|
|
111
111
|
if session is None:
|
|
112
|
-
console.print(f"Not logged in to '{env_name}'. Run `sunstone
|
|
112
|
+
console.print(f"Not logged in to '{env_name}'. Run `sunstone auth login`.")
|
|
113
113
|
return
|
|
114
114
|
|
|
115
115
|
age_hours = max(0.0, time.time() - session.cached_at) / 3600
|
|
@@ -49,8 +49,7 @@ def _format_result(result, as_json: bool = False, branch: str = "main") -> None:
|
|
|
49
49
|
console.print(f" Snapshot: {result.snapshot_id}")
|
|
50
50
|
console.print(f" Dataset: [cyan]{iri}[/cyan]")
|
|
51
51
|
console.print(f" Activity: [cyan]{activity}[/cyan]")
|
|
52
|
-
|
|
53
|
-
console.print(f" Branch: [cyan]{branch}[/cyan]")
|
|
52
|
+
console.print(f" Branch: [cyan]{branch}[/cyan]")
|
|
54
53
|
|
|
55
54
|
|
|
56
55
|
@data_app.command("import")
|
|
@@ -71,7 +70,12 @@ def import_cmd(
|
|
|
71
70
|
help="Publisher organization path, e.g. un/who. Overrides the source-URL rules file. Does not set the namespace.",
|
|
72
71
|
),
|
|
73
72
|
tag: list[str] | None = typer.Option(None, "--tag", "-t", help="Add tags (key:value)"),
|
|
74
|
-
branch: str = typer.Option(
|
|
73
|
+
branch: str | None = typer.Option(
|
|
74
|
+
None,
|
|
75
|
+
"--branch",
|
|
76
|
+
"-b",
|
|
77
|
+
help="Branch to import to (default: the normalized git branch, else the CI ref; fails if neither)",
|
|
78
|
+
),
|
|
75
79
|
force_schema: bool = typer.Option(False, "--force-schema", help="Allow unsafe schema changes"),
|
|
76
80
|
json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
|
|
77
81
|
) -> None:
|
|
@@ -86,11 +90,15 @@ def import_cmd(
|
|
|
86
90
|
TransferSpeedColumn,
|
|
87
91
|
)
|
|
88
92
|
|
|
93
|
+
from sunstone_data.branch import current_data_branch
|
|
89
94
|
from sunstone_data.import_ import run_import
|
|
90
95
|
|
|
91
96
|
is_url = source.startswith(("http://", "https://"))
|
|
92
97
|
|
|
93
98
|
try:
|
|
99
|
+
if branch is None:
|
|
100
|
+
branch = current_data_branch()
|
|
101
|
+
|
|
94
102
|
if sys.stdout.isatty() and not json_output:
|
|
95
103
|
with Progress(
|
|
96
104
|
SpinnerColumn(),
|
|
@@ -10,7 +10,12 @@ from sunstone_data.cli._app import console, data_app
|
|
|
10
10
|
@data_app.command("publish")
|
|
11
11
|
def publish_cmd(
|
|
12
12
|
table: str = typer.Argument(..., help="Table identifier (namespace.table)"),
|
|
13
|
-
branch: str = typer.Option(
|
|
13
|
+
branch: str | None = typer.Option(
|
|
14
|
+
None,
|
|
15
|
+
"--branch",
|
|
16
|
+
"-b",
|
|
17
|
+
help="Branch to publish on (default: the normalized git branch, else the CI ref; fails if neither)",
|
|
18
|
+
),
|
|
14
19
|
version: str | None = typer.Option(
|
|
15
20
|
None, "--version", help="Publish the table's current snapshot under this version label, e.g. v1.0"
|
|
16
21
|
),
|
|
@@ -27,6 +32,7 @@ def publish_cmd(
|
|
|
27
32
|
|
|
28
33
|
from sunstone.lineage import Metadata
|
|
29
34
|
|
|
35
|
+
from sunstone_data.branch import current_data_branch
|
|
30
36
|
from sunstone_data.catalog import get_catalog
|
|
31
37
|
from sunstone_data.graph.dataset import DatasetDescription, DatasetVersion
|
|
32
38
|
from sunstone_data.publish import publish
|
|
@@ -34,6 +40,9 @@ def publish_cmd(
|
|
|
34
40
|
from sunstone_data.resolve import dataset_iri
|
|
35
41
|
|
|
36
42
|
try:
|
|
43
|
+
if branch is None:
|
|
44
|
+
branch = current_data_branch()
|
|
45
|
+
|
|
37
46
|
ref = TableRef.parse(table)
|
|
38
47
|
identifier = f"{ref.namespace}.{ref.table}"
|
|
39
48
|
versions: tuple[DatasetVersion, ...] = ()
|