sunstone-data-platform 1.0.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/PKG-INFO +77 -68
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/README.md +75 -67
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/pyproject.toml +1 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/_version.py +2 -2
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/__init__.py +5 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/pkce.py +26 -11
- sunstone_data_platform-1.2.0/src/sunstone_data/auth/stepup.py +106 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/catalog.py +74 -10
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_import.py +9 -2
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/ls.py +1 -1
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/prov.py +2 -2
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/show.py +3 -3
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/snapshots.py +2 -2
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/config.py +47 -26
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/discovery.py +18 -16
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/env_commands.py +5 -13
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/env_section.py +11 -5
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/client.py +32 -4
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/owner.py +10 -22
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/parser.py +29 -4
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/pipeline.py +4 -6
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/plugin.py +9 -16
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/query.py +1 -1
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/resolve.py +4 -4
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/url.py +5 -5
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/utils/__init__.py +4 -1
- sunstone_data_platform-1.2.0/src/sunstone_data/utils/naming.py +113 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/assets/transforms.py +1 -1
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/resources/iceberg.py +1 -1
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/resources/refs.py +1 -1
- sunstone_data_platform-1.0.0/src/sunstone_data/utils/naming.py +0 -38
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/.gitignore +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/client_creds.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/store.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/_app.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/_errors.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_auth.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_info.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_publish.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/refs.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/sparql.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/main.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/dataset.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/terms.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/identifiers.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/fetcher.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/storage.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/builder.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/models.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/provenance.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/provenance_rdf.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/publish.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/utils/checksum.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/utils/encoding.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/versioning/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/versioning/schema_evolution.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/assets/__init__.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/assets/mirrors.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/definitions.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/io_manager.py +0 -0
- {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/resources/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sunstone-data-platform
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Sunstone Data Platform - Data lakehouse with semantic metadata
|
|
5
5
|
Author-email: Sunstone Institute <stig@sunstone.institute>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -9,6 +9,7 @@ Requires-Dist: charset-normalizer>=3.4.0
|
|
|
9
9
|
Requires-Dist: duckdb>=1.5.3
|
|
10
10
|
Requires-Dist: fastexcel>=0.20.2
|
|
11
11
|
Requires-Dist: httpx>=0.27.0
|
|
12
|
+
Requires-Dist: publicsuffixlist>=1.0.2.20261003
|
|
12
13
|
Requires-Dist: pyiceberg[duckdb,gcsfs,s3fs]>=0.10.0
|
|
13
14
|
Requires-Dist: python-dateutil>=2.9.0.post0
|
|
14
15
|
Requires-Dist: python-dotenv>=1.2.2
|
|
@@ -25,48 +26,44 @@ Description-Content-Type: text/markdown
|
|
|
25
26
|
|
|
26
27
|
# Sunstone Data Platform
|
|
27
28
|
|
|
28
|
-
A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg,
|
|
29
|
+
A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, an Iceberg REST catalog, a versioned RDF graph in Postgres, RustFS, and DuckDB.
|
|
29
30
|
|
|
30
|
-
Data scientists
|
|
31
|
+
Data scientists import CSV/Excel files from local paths or URLs into Iceberg tables, query them with DuckDB using git-like version references, and publish dataset metadata and W3C PROV-DM provenance into the graph. The graph is the system of record for metadata, branches and tags.
|
|
31
32
|
|
|
32
33
|
## Architecture
|
|
33
34
|
|
|
34
35
|
```
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
│ (Iceberg REST API) │
|
|
53
|
-
└──────────┬──────────┘
|
|
54
|
-
│
|
|
55
|
-
┌──────────▼──────────┐
|
|
56
|
-
│ RustFS (S3) │
|
|
57
|
-
│ Iceberg tables + │
|
|
58
|
-
│ original files │
|
|
59
|
-
└─────────────────────┘
|
|
36
|
+
┌─────────────────────────────────┐
|
|
37
|
+
│ sunstone_data (Python) │
|
|
38
|
+
│ sunstone data CLI, sunstone: │
|
|
39
|
+
│ URLs, import pipeline, DuckDB │
|
|
40
|
+
└───────┬─────────────────┬───────┘
|
|
41
|
+
Iceberg REST │ │ GSP, refs, SPARQL
|
|
42
|
+
▼ ▼
|
|
43
|
+
┌─────────────────┐ ┌─────────────────┐
|
|
44
|
+
│ iceberg-catalog │──▶│ graph-server │
|
|
45
|
+
│ :8181 │ │ :8080 │
|
|
46
|
+
└────────┬────────┘ └────────┬────────┘
|
|
47
|
+
▼ ▼
|
|
48
|
+
┌─────────────────┐ ┌─────────────────┐
|
|
49
|
+
│ RustFS (S3) │ │ Postgres graph │
|
|
50
|
+
│ Iceberg tables, │ │ (SoR), copied │
|
|
51
|
+
│ original files │ │ to Oxigraph │
|
|
52
|
+
└─────────────────┘ └─────────────────┘
|
|
60
53
|
```
|
|
61
54
|
|
|
62
|
-
|
|
55
|
+
On every table commit, iceberg-catalog also writes the table's descriptor into
|
|
56
|
+
the graph. See [docs/architecture.md](docs/architecture.md) for the
|
|
57
|
+
full data flow.
|
|
58
|
+
|
|
59
|
+
**Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | graph-server | Oxigraph | RustFS | PostgreSQL 16
|
|
63
60
|
|
|
64
61
|
## Endpoints
|
|
65
62
|
|
|
66
63
|
- `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
|
|
67
64
|
(see [the views design](docs/specs/2026-08-02-graph-server-views-design.md))
|
|
68
|
-
- `/ui/` — the
|
|
69
|
-
graph-server (see [docs/
|
|
65
|
+
- `/ui/` — the data explorer, a 3D view of the data graph, served by
|
|
66
|
+
graph-server (see [docs/data-explorer.md](docs/data-explorer.md))
|
|
70
67
|
|
|
71
68
|
## Local Development Setup
|
|
72
69
|
|
|
@@ -150,14 +147,16 @@ uv run scripts/smoke_test.py
|
|
|
150
147
|
|
|
151
148
|
### 5. Set up your environment
|
|
152
149
|
|
|
153
|
-
The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group
|
|
150
|
+
The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group. You configure one value per environment, the base IRI (`base_iri`). graph-server is served at the base, the Iceberg catalog under `{base_iri}iceberg/`, and datasets and organizations are named under it. The client discovers sign-in settings from graph-server.
|
|
154
151
|
|
|
155
152
|
```bash
|
|
156
153
|
# Add your local Docker stack as an environment
|
|
157
|
-
sunstone env add local
|
|
154
|
+
sunstone env add local data-platform.base_iri=https://data.example.org/
|
|
158
155
|
sunstone env use local
|
|
159
156
|
```
|
|
160
157
|
|
|
158
|
+
`base_iri` must equal the iceberg-catalog service's `BASE_IRI`. In the local Docker stack, graph-server (port 8080) and the catalog (port 8181) listen on different ports, so also set the service overrides `GRAPH_URL=http://localhost:8080` and `CATALOG_URI=http://localhost:8181` (see `.env.example`). They never affect IRIs.
|
|
159
|
+
|
|
161
160
|
### 6. Try it out
|
|
162
161
|
|
|
163
162
|
```bash
|
|
@@ -167,17 +166,17 @@ sunstone data import path/to/data.csv
|
|
|
167
166
|
# Import from a URL
|
|
168
167
|
sunstone data import https://example.com/dataset.csv
|
|
169
168
|
|
|
170
|
-
# List
|
|
171
|
-
sunstone data
|
|
169
|
+
# List namespaces and tables
|
|
170
|
+
sunstone data ls
|
|
172
171
|
|
|
173
|
-
# View dataset metadata
|
|
174
|
-
sunstone data
|
|
172
|
+
# View dataset metadata from the graph
|
|
173
|
+
sunstone data show namespace.table_name
|
|
175
174
|
|
|
176
|
-
#
|
|
175
|
+
# Snapshot history
|
|
177
176
|
sunstone data history namespace.table_name
|
|
178
177
|
|
|
179
|
-
#
|
|
180
|
-
sunstone data
|
|
178
|
+
# Publish the current snapshot as a named version
|
|
179
|
+
sunstone data publish namespace.table_name --version v1.0
|
|
181
180
|
```
|
|
182
181
|
|
|
183
182
|
## Running Tests
|
|
@@ -186,7 +185,7 @@ sunstone data diff namespace.table_name @v1 @v2
|
|
|
186
185
|
# Unit tests (fast, no Docker required)
|
|
187
186
|
uv run pytest tests/unit/
|
|
188
187
|
|
|
189
|
-
# Integration tests (
|
|
188
|
+
# Integration tests (run inside Docker automatically; needs Docker)
|
|
190
189
|
uv run pytest tests/integration/
|
|
191
190
|
|
|
192
191
|
# All tests
|
|
@@ -200,25 +199,33 @@ uv run pytest tests/unit/test_query.py -v
|
|
|
200
199
|
|
|
201
200
|
| Command | Description |
|
|
202
201
|
|---------|-------------|
|
|
203
|
-
| `sunstone data
|
|
204
|
-
| `sunstone data
|
|
205
|
-
| `sunstone data
|
|
206
|
-
| `sunstone data
|
|
207
|
-
| `sunstone data
|
|
208
|
-
| `sunstone data
|
|
209
|
-
| `sunstone data
|
|
210
|
-
| `sunstone data lineage <table>` |
|
|
211
|
-
| `sunstone data
|
|
212
|
-
|
|
213
|
-
|
|
202
|
+
| `sunstone data login`, `logout`, `whoami` | Sign in to graph-server (Keycloak) |
|
|
203
|
+
| `sunstone data ls [namespace]` | List namespaces and tables |
|
|
204
|
+
| `sunstone data show <table>` | Dataset metadata from the graph (table, Turtle or JSON-LD) |
|
|
205
|
+
| `sunstone data info [table]` | Iceberg table properties for one or all datasets |
|
|
206
|
+
| `sunstone data import <source>` | Import CSV/Excel from a path or URL |
|
|
207
|
+
| `sunstone data publish <table>` | Publish dataset metadata to the graph; `--version` names the current snapshot |
|
|
208
|
+
| `sunstone data history <table>` | Iceberg snapshot history |
|
|
209
|
+
| `sunstone data lineage <table>` | Provenance from the published PROV facts |
|
|
210
|
+
| `sunstone data sparql <query>` | Run a SPARQL query through graph-server |
|
|
211
|
+
| `sunstone data branches`, `branch create\|delete` | List, create and delete graph branches |
|
|
212
|
+
| `sunstone data tags`, `tag create\|delete` | List, create and delete graph tags |
|
|
213
|
+
|
|
214
|
+
Every command takes `--json` for machine-readable output. `import`, `publish`, `show`, `history` and `tag create` take `--branch`. Run `sunstone data <command> --help` for full options.
|
|
214
215
|
|
|
215
216
|
## Key Concepts
|
|
216
217
|
|
|
217
|
-
- **Table references** use
|
|
218
|
-
|
|
218
|
+
- **Table references** use the form `namespace.table@ref`. A ref is checked in this order:
|
|
219
|
+
1. a branch name
|
|
220
|
+
2. a snapshot id (13 or more digits)
|
|
221
|
+
3. a date or timestamp (UTC if it has no timezone), which selects the last snapshot at or before that time
|
|
222
|
+
4. a named version created with `sunstone data publish --version`
|
|
223
|
+
|
|
224
|
+
Snapshot ids, timestamps and named versions resolve against `main`. Graph tags (`sunstone data tag create`) are not valid refs.
|
|
225
|
+
- **Deduplication**: Imports are checksum-based (SHA-256). Re-importing the same file is a no-op.
|
|
219
226
|
- **Schema evolution**: Safe type promotions (e.g. int32 to int64) are allowed automatically. Unsafe changes require `--force-schema`.
|
|
220
|
-
- **Provenance**: Every import
|
|
221
|
-
- **Original files**: Source files are stored content-
|
|
227
|
+
- **Provenance**: Every import records source metadata and schema info as Iceberg table properties, and asserts W3C PROV-DM provenance into the dataset's graph.
|
|
228
|
+
- **Original files**: Source files are stored content-addressed in object storage next to the Iceberg data.
|
|
222
229
|
|
|
223
230
|
## sunstone: URLs
|
|
224
231
|
|
|
@@ -230,7 +237,7 @@ import sunstone
|
|
|
230
237
|
df = sunstone.read("sunstone:namespace/table@v1.0")
|
|
231
238
|
```
|
|
232
239
|
|
|
233
|
-
|
|
240
|
+
The `@ref` part follows the same rules as table references above. Here `v1.0` is a named version.
|
|
234
241
|
|
|
235
242
|
## Project Structure
|
|
236
243
|
|
|
@@ -238,24 +245,26 @@ Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
|
|
|
238
245
|
src/sunstone_data/
|
|
239
246
|
├── plugin.py # sunstone-py plugin (CLIProvider + URLHandler)
|
|
240
247
|
├── url.py # sunstone: URL parser
|
|
248
|
+
├── resolve.py # Resolves a @ref to a branch or snapshot
|
|
241
249
|
├── config.py # Settings from DataEnvironment or .env
|
|
250
|
+
├── discovery.py # Sign-in settings from graph-server /ui/config.json
|
|
251
|
+
├── auth/ # Keycloak login and token handling
|
|
252
|
+
├── graph/ # graph-server client and RDF serialization
|
|
253
|
+
├── publish.py # Publishes dataset metadata to the graph
|
|
242
254
|
├── catalog.py # PyIceberg RestCatalog connection to the Iceberg REST catalog
|
|
243
255
|
├── query.py # DuckDB QuerySession with time-travel support
|
|
244
|
-
├──
|
|
245
|
-
├── cli/ # Typer CLI commands + Rich formatters
|
|
256
|
+
├── cli/ # Typer CLI commands
|
|
246
257
|
├── import_/ # Pipeline: fetch → parse → store → metadata
|
|
247
258
|
├── metadata/ # Frozen dataclasses, MetadataBuilder, W3C PROV-DM
|
|
248
|
-
├── versioning/ #
|
|
259
|
+
├── versioning/ # Schema evolution checks
|
|
249
260
|
└── utils/ # Checksums, encoding detection, naming
|
|
250
261
|
```
|
|
251
262
|
|
|
252
263
|
## Documentation
|
|
253
264
|
|
|
254
|
-
- [
|
|
255
|
-
- [
|
|
256
|
-
- [CLI
|
|
257
|
-
- [Import Pipeline](docs/import-pipeline.md) - How imports work
|
|
258
|
-
- [Development Guide](docs/development.md) - Contributing and testing
|
|
265
|
+
- [docs/README.md](docs/README.md) - Index of all documentation
|
|
266
|
+
- [docs/architecture.md](docs/architecture.md) - System overview and data flow
|
|
267
|
+
- [docs/specs2/](docs/specs2/) - Current design, one file per subsystem. The Python client and CLI are in [05-python-client.md](docs/specs2/05-python-client.md).
|
|
259
268
|
|
|
260
269
|
## Resetting the Stack
|
|
261
270
|
|
|
@@ -278,11 +287,11 @@ determine which environment you can use. See
|
|
|
278
287
|
`docs/specs/2026-07-05-graph-server-auth-design.md`.
|
|
279
288
|
|
|
280
289
|
No hostnames or client ids ship in this package. Each environment's
|
|
281
|
-
`
|
|
290
|
+
`base_iri` comes from config: Sunstone-managed machines get
|
|
282
291
|
`/etc/sunstone/data_platform.toml` from the `fleet-config` repo, and other
|
|
283
|
-
machines set `
|
|
292
|
+
machines set `BASE_IRI` or add an environment with `sunstone env add`.
|
|
284
293
|
graph-server publishes the OIDC issuer and public client id at
|
|
285
|
-
|
|
294
|
+
`{base_iri}ui/config.json`; the CLI reads them from there.
|
|
286
295
|
|
|
287
296
|
### Humans (CLI, local agents)
|
|
288
297
|
|
|
@@ -1,47 +1,43 @@
|
|
|
1
1
|
# Sunstone Data Platform
|
|
2
2
|
|
|
3
|
-
A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg,
|
|
3
|
+
A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, an Iceberg REST catalog, a versioned RDF graph in Postgres, RustFS, and DuckDB.
|
|
4
4
|
|
|
5
|
-
Data scientists
|
|
5
|
+
Data scientists import CSV/Excel files from local paths or URLs into Iceberg tables, query them with DuckDB using git-like version references, and publish dataset metadata and W3C PROV-DM provenance into the graph. The graph is the system of record for metadata, branches and tags.
|
|
6
6
|
|
|
7
7
|
## Architecture
|
|
8
8
|
|
|
9
9
|
```
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
│ (Iceberg REST API) │
|
|
28
|
-
└──────────┬──────────┘
|
|
29
|
-
│
|
|
30
|
-
┌──────────▼──────────┐
|
|
31
|
-
│ RustFS (S3) │
|
|
32
|
-
│ Iceberg tables + │
|
|
33
|
-
│ original files │
|
|
34
|
-
└─────────────────────┘
|
|
10
|
+
┌─────────────────────────────────┐
|
|
11
|
+
│ sunstone_data (Python) │
|
|
12
|
+
│ sunstone data CLI, sunstone: │
|
|
13
|
+
│ URLs, import pipeline, DuckDB │
|
|
14
|
+
└───────┬─────────────────┬───────┘
|
|
15
|
+
Iceberg REST │ │ GSP, refs, SPARQL
|
|
16
|
+
▼ ▼
|
|
17
|
+
┌─────────────────┐ ┌─────────────────┐
|
|
18
|
+
│ iceberg-catalog │──▶│ graph-server │
|
|
19
|
+
│ :8181 │ │ :8080 │
|
|
20
|
+
└────────┬────────┘ └────────┬────────┘
|
|
21
|
+
▼ ▼
|
|
22
|
+
┌─────────────────┐ ┌─────────────────┐
|
|
23
|
+
│ RustFS (S3) │ │ Postgres graph │
|
|
24
|
+
│ Iceberg tables, │ │ (SoR), copied │
|
|
25
|
+
│ original files │ │ to Oxigraph │
|
|
26
|
+
└─────────────────┘ └─────────────────┘
|
|
35
27
|
```
|
|
36
28
|
|
|
37
|
-
|
|
29
|
+
On every table commit, iceberg-catalog also writes the table's descriptor into
|
|
30
|
+
the graph. See [docs/architecture.md](docs/architecture.md) for the
|
|
31
|
+
full data flow.
|
|
32
|
+
|
|
33
|
+
**Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | graph-server | Oxigraph | RustFS | PostgreSQL 16
|
|
38
34
|
|
|
39
35
|
## Endpoints
|
|
40
36
|
|
|
41
37
|
- `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
|
|
42
38
|
(see [the views design](docs/specs/2026-08-02-graph-server-views-design.md))
|
|
43
|
-
- `/ui/` — the
|
|
44
|
-
graph-server (see [docs/
|
|
39
|
+
- `/ui/` — the data explorer, a 3D view of the data graph, served by
|
|
40
|
+
graph-server (see [docs/data-explorer.md](docs/data-explorer.md))
|
|
45
41
|
|
|
46
42
|
## Local Development Setup
|
|
47
43
|
|
|
@@ -125,14 +121,16 @@ uv run scripts/smoke_test.py
|
|
|
125
121
|
|
|
126
122
|
### 5. Set up your environment
|
|
127
123
|
|
|
128
|
-
The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group
|
|
124
|
+
The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group. You configure one value per environment, the base IRI (`base_iri`). graph-server is served at the base, the Iceberg catalog under `{base_iri}iceberg/`, and datasets and organizations are named under it. The client discovers sign-in settings from graph-server.
|
|
129
125
|
|
|
130
126
|
```bash
|
|
131
127
|
# Add your local Docker stack as an environment
|
|
132
|
-
sunstone env add local
|
|
128
|
+
sunstone env add local data-platform.base_iri=https://data.example.org/
|
|
133
129
|
sunstone env use local
|
|
134
130
|
```
|
|
135
131
|
|
|
132
|
+
`base_iri` must equal the iceberg-catalog service's `BASE_IRI`. In the local Docker stack, graph-server (port 8080) and the catalog (port 8181) listen on different ports, so also set the service overrides `GRAPH_URL=http://localhost:8080` and `CATALOG_URI=http://localhost:8181` (see `.env.example`). They never affect IRIs.
|
|
133
|
+
|
|
136
134
|
### 6. Try it out
|
|
137
135
|
|
|
138
136
|
```bash
|
|
@@ -142,17 +140,17 @@ sunstone data import path/to/data.csv
|
|
|
142
140
|
# Import from a URL
|
|
143
141
|
sunstone data import https://example.com/dataset.csv
|
|
144
142
|
|
|
145
|
-
# List
|
|
146
|
-
sunstone data
|
|
143
|
+
# List namespaces and tables
|
|
144
|
+
sunstone data ls
|
|
147
145
|
|
|
148
|
-
# View dataset metadata
|
|
149
|
-
sunstone data
|
|
146
|
+
# View dataset metadata from the graph
|
|
147
|
+
sunstone data show namespace.table_name
|
|
150
148
|
|
|
151
|
-
#
|
|
149
|
+
# Snapshot history
|
|
152
150
|
sunstone data history namespace.table_name
|
|
153
151
|
|
|
154
|
-
#
|
|
155
|
-
sunstone data
|
|
152
|
+
# Publish the current snapshot as a named version
|
|
153
|
+
sunstone data publish namespace.table_name --version v1.0
|
|
156
154
|
```
|
|
157
155
|
|
|
158
156
|
## Running Tests
|
|
@@ -161,7 +159,7 @@ sunstone data diff namespace.table_name @v1 @v2
|
|
|
161
159
|
# Unit tests (fast, no Docker required)
|
|
162
160
|
uv run pytest tests/unit/
|
|
163
161
|
|
|
164
|
-
# Integration tests (
|
|
162
|
+
# Integration tests (run inside Docker automatically; needs Docker)
|
|
165
163
|
uv run pytest tests/integration/
|
|
166
164
|
|
|
167
165
|
# All tests
|
|
@@ -175,25 +173,33 @@ uv run pytest tests/unit/test_query.py -v
|
|
|
175
173
|
|
|
176
174
|
| Command | Description |
|
|
177
175
|
|---------|-------------|
|
|
178
|
-
| `sunstone data
|
|
179
|
-
| `sunstone data
|
|
180
|
-
| `sunstone data
|
|
181
|
-
| `sunstone data
|
|
182
|
-
| `sunstone data
|
|
183
|
-
| `sunstone data
|
|
184
|
-
| `sunstone data
|
|
185
|
-
| `sunstone data lineage <table>` |
|
|
186
|
-
| `sunstone data
|
|
187
|
-
|
|
188
|
-
|
|
176
|
+
| `sunstone data login`, `logout`, `whoami` | Sign in to graph-server (Keycloak) |
|
|
177
|
+
| `sunstone data ls [namespace]` | List namespaces and tables |
|
|
178
|
+
| `sunstone data show <table>` | Dataset metadata from the graph (table, Turtle or JSON-LD) |
|
|
179
|
+
| `sunstone data info [table]` | Iceberg table properties for one or all datasets |
|
|
180
|
+
| `sunstone data import <source>` | Import CSV/Excel from a path or URL |
|
|
181
|
+
| `sunstone data publish <table>` | Publish dataset metadata to the graph; `--version` names the current snapshot |
|
|
182
|
+
| `sunstone data history <table>` | Iceberg snapshot history |
|
|
183
|
+
| `sunstone data lineage <table>` | Provenance from the published PROV facts |
|
|
184
|
+
| `sunstone data sparql <query>` | Run a SPARQL query through graph-server |
|
|
185
|
+
| `sunstone data branches`, `branch create\|delete` | List, create and delete graph branches |
|
|
186
|
+
| `sunstone data tags`, `tag create\|delete` | List, create and delete graph tags |
|
|
187
|
+
|
|
188
|
+
Every command takes `--json` for machine-readable output. `import`, `publish`, `show`, `history` and `tag create` take `--branch`. Run `sunstone data <command> --help` for full options.
|
|
189
189
|
|
|
190
190
|
## Key Concepts
|
|
191
191
|
|
|
192
|
-
- **Table references** use
|
|
193
|
-
|
|
192
|
+
- **Table references** use the form `namespace.table@ref`. A ref is checked in this order:
|
|
193
|
+
1. a branch name
|
|
194
|
+
2. a snapshot id (13 or more digits)
|
|
195
|
+
3. a date or timestamp (UTC if it has no timezone), which selects the last snapshot at or before that time
|
|
196
|
+
4. a named version created with `sunstone data publish --version`
|
|
197
|
+
|
|
198
|
+
Snapshot ids, timestamps and named versions resolve against `main`. Graph tags (`sunstone data tag create`) are not valid refs.
|
|
199
|
+
- **Deduplication**: Imports are checksum-based (SHA-256). Re-importing the same file is a no-op.
|
|
194
200
|
- **Schema evolution**: Safe type promotions (e.g. int32 to int64) are allowed automatically. Unsafe changes require `--force-schema`.
|
|
195
|
-
- **Provenance**: Every import
|
|
196
|
-
- **Original files**: Source files are stored content-
|
|
201
|
+
- **Provenance**: Every import records source metadata and schema info as Iceberg table properties, and asserts W3C PROV-DM provenance into the dataset's graph.
|
|
202
|
+
- **Original files**: Source files are stored content-addressed in object storage next to the Iceberg data.
|
|
197
203
|
|
|
198
204
|
## sunstone: URLs
|
|
199
205
|
|
|
@@ -205,7 +211,7 @@ import sunstone
|
|
|
205
211
|
df = sunstone.read("sunstone:namespace/table@v1.0")
|
|
206
212
|
```
|
|
207
213
|
|
|
208
|
-
|
|
214
|
+
The `@ref` part follows the same rules as table references above. Here `v1.0` is a named version.
|
|
209
215
|
|
|
210
216
|
## Project Structure
|
|
211
217
|
|
|
@@ -213,24 +219,26 @@ Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
|
|
|
213
219
|
src/sunstone_data/
|
|
214
220
|
├── plugin.py # sunstone-py plugin (CLIProvider + URLHandler)
|
|
215
221
|
├── url.py # sunstone: URL parser
|
|
222
|
+
├── resolve.py # Resolves a @ref to a branch or snapshot
|
|
216
223
|
├── config.py # Settings from DataEnvironment or .env
|
|
224
|
+
├── discovery.py # Sign-in settings from graph-server /ui/config.json
|
|
225
|
+
├── auth/ # Keycloak login and token handling
|
|
226
|
+
├── graph/ # graph-server client and RDF serialization
|
|
227
|
+
├── publish.py # Publishes dataset metadata to the graph
|
|
217
228
|
├── catalog.py # PyIceberg RestCatalog connection to the Iceberg REST catalog
|
|
218
229
|
├── query.py # DuckDB QuerySession with time-travel support
|
|
219
|
-
├──
|
|
220
|
-
├── cli/ # Typer CLI commands + Rich formatters
|
|
230
|
+
├── cli/ # Typer CLI commands
|
|
221
231
|
├── import_/ # Pipeline: fetch → parse → store → metadata
|
|
222
232
|
├── metadata/ # Frozen dataclasses, MetadataBuilder, W3C PROV-DM
|
|
223
|
-
├── versioning/ #
|
|
233
|
+
├── versioning/ # Schema evolution checks
|
|
224
234
|
└── utils/ # Checksums, encoding detection, naming
|
|
225
235
|
```
|
|
226
236
|
|
|
227
237
|
## Documentation
|
|
228
238
|
|
|
229
|
-
- [
|
|
230
|
-
- [
|
|
231
|
-
- [CLI
|
|
232
|
-
- [Import Pipeline](docs/import-pipeline.md) - How imports work
|
|
233
|
-
- [Development Guide](docs/development.md) - Contributing and testing
|
|
239
|
+
- [docs/README.md](docs/README.md) - Index of all documentation
|
|
240
|
+
- [docs/architecture.md](docs/architecture.md) - System overview and data flow
|
|
241
|
+
- [docs/specs2/](docs/specs2/) - Current design, one file per subsystem. The Python client and CLI are in [05-python-client.md](docs/specs2/05-python-client.md).
|
|
234
242
|
|
|
235
243
|
## Resetting the Stack
|
|
236
244
|
|
|
@@ -253,11 +261,11 @@ determine which environment you can use. See
|
|
|
253
261
|
`docs/specs/2026-07-05-graph-server-auth-design.md`.
|
|
254
262
|
|
|
255
263
|
No hostnames or client ids ship in this package. Each environment's
|
|
256
|
-
`
|
|
264
|
+
`base_iri` comes from config: Sunstone-managed machines get
|
|
257
265
|
`/etc/sunstone/data_platform.toml` from the `fleet-config` repo, and other
|
|
258
|
-
machines set `
|
|
266
|
+
machines set `BASE_IRI` or add an environment with `sunstone env add`.
|
|
259
267
|
graph-server publishes the OIDC issuer and public client id at
|
|
260
|
-
|
|
268
|
+
`{base_iri}ui/config.json`; the CLI reads them from there.
|
|
261
269
|
|
|
262
270
|
### Humans (CLI, local agents)
|
|
263
271
|
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1,
|
|
21
|
+
__version__ = version = '1.2.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 2, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
{sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/__init__.py
RENAMED
|
@@ -22,6 +22,7 @@ from sunstone_data.auth.client_creds import (
|
|
|
22
22
|
refresh_access_token,
|
|
23
23
|
resolve_access_token,
|
|
24
24
|
)
|
|
25
|
+
from sunstone_data.auth.stepup import StepUpChallenge, StepUpRequired, parse_challenge, step_up
|
|
25
26
|
from sunstone_data.auth.store import (
|
|
26
27
|
Session,
|
|
27
28
|
clear_session,
|
|
@@ -33,11 +34,15 @@ from sunstone_data.auth.store import (
|
|
|
33
34
|
__all__ = [
|
|
34
35
|
"ClientCredentialsToken",
|
|
35
36
|
"Session",
|
|
37
|
+
"StepUpChallenge",
|
|
38
|
+
"StepUpRequired",
|
|
36
39
|
"clear_session",
|
|
37
40
|
"discover_token_endpoint",
|
|
38
41
|
"read_session",
|
|
42
|
+
"parse_challenge",
|
|
39
43
|
"refresh_access_token",
|
|
40
44
|
"resolve_access_token",
|
|
41
45
|
"session_path",
|
|
46
|
+
"step_up",
|
|
42
47
|
"write_session",
|
|
43
48
|
]
|
{sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/pkce.py
RENAMED
|
@@ -72,18 +72,27 @@ def build_authorization_url(
|
|
|
72
72
|
redirect_uri: str,
|
|
73
73
|
state: str,
|
|
74
74
|
code_challenge: str,
|
|
75
|
+
acr_values: str | None = None,
|
|
76
|
+
max_age: int | None = None,
|
|
75
77
|
) -> str:
|
|
76
|
-
"""Build the URL the user's browser is sent to in order to sign in.
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
78
|
+
"""Build the URL the user's browser is sent to in order to sign in.
|
|
79
|
+
|
|
80
|
+
``acr_values`` and ``max_age`` are the OIDC step-up parameters (RFC 9470).
|
|
81
|
+
They are sent only when given.
|
|
82
|
+
"""
|
|
83
|
+
params = {
|
|
84
|
+
"response_type": "code",
|
|
85
|
+
"client_id": client_id,
|
|
86
|
+
"redirect_uri": redirect_uri,
|
|
87
|
+
"state": state,
|
|
88
|
+
"code_challenge": code_challenge,
|
|
89
|
+
"code_challenge_method": "S256",
|
|
90
|
+
}
|
|
91
|
+
if acr_values is not None:
|
|
92
|
+
params["acr_values"] = acr_values
|
|
93
|
+
if max_age is not None:
|
|
94
|
+
params["max_age"] = str(max_age)
|
|
95
|
+
query = urlencode(params)
|
|
87
96
|
separator = "&" if urlparse(authorization_endpoint).query else "?"
|
|
88
97
|
return f"{authorization_endpoint}{separator}{query}"
|
|
89
98
|
|
|
@@ -262,6 +271,8 @@ def login(
|
|
|
262
271
|
client_id: str,
|
|
263
272
|
client: httpx.Client,
|
|
264
273
|
open_browser: Callable[[str], object] = webbrowser.open,
|
|
274
|
+
acr_values: str | None = None,
|
|
275
|
+
max_age: int | None = None,
|
|
265
276
|
) -> TokenResponse:
|
|
266
277
|
"""Run the full PKCE sign-in and return the resulting tokens.
|
|
267
278
|
|
|
@@ -276,6 +287,8 @@ def login(
|
|
|
276
287
|
client: HTTP client for the token exchange.
|
|
277
288
|
open_browser: How to show the user the authorization URL. Injected so
|
|
278
289
|
tests need no browser.
|
|
290
|
+
acr_values: Authentication context the sign-in must satisfy (step-up).
|
|
291
|
+
max_age: Newest allowed age, in seconds, of the user's last sign-in.
|
|
279
292
|
|
|
280
293
|
Raises:
|
|
281
294
|
RuntimeError: Sign-in was refused, timed out, or the callback failed
|
|
@@ -292,6 +305,8 @@ def login(
|
|
|
292
305
|
redirect_uri=redirect_uri,
|
|
293
306
|
state=state,
|
|
294
307
|
code_challenge=derive_code_challenge(verifier),
|
|
308
|
+
acr_values=acr_values,
|
|
309
|
+
max_age=max_age,
|
|
295
310
|
)
|
|
296
311
|
open_browser(url)
|
|
297
312
|
code = verify_callback(await_callback(server, sink), state)
|