sunstone-data-platform 1.0.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/PKG-INFO +77 -68
  2. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/README.md +75 -67
  3. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/pyproject.toml +1 -0
  4. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/_version.py +2 -2
  5. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/__init__.py +5 -0
  6. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/pkce.py +26 -11
  7. sunstone_data_platform-1.2.0/src/sunstone_data/auth/stepup.py +106 -0
  8. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/catalog.py +74 -10
  9. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_import.py +9 -2
  10. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/ls.py +1 -1
  11. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/prov.py +2 -2
  12. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/show.py +3 -3
  13. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/snapshots.py +2 -2
  14. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/config.py +47 -26
  15. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/discovery.py +18 -16
  16. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/env_commands.py +5 -13
  17. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/env_section.py +11 -5
  18. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/client.py +32 -4
  19. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/owner.py +10 -22
  20. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/parser.py +29 -4
  21. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/pipeline.py +4 -6
  22. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/plugin.py +9 -16
  23. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/query.py +1 -1
  24. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/resolve.py +4 -4
  25. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/url.py +5 -5
  26. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/utils/__init__.py +4 -1
  27. sunstone_data_platform-1.2.0/src/sunstone_data/utils/naming.py +113 -0
  28. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/assets/transforms.py +1 -1
  29. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/resources/iceberg.py +1 -1
  30. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/resources/refs.py +1 -1
  31. sunstone_data_platform-1.0.0/src/sunstone_data/utils/naming.py +0 -38
  32. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/.gitignore +0 -0
  33. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/__init__.py +0 -0
  34. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/client_creds.py +0 -0
  35. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/auth/store.py +0 -0
  36. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/__init__.py +0 -0
  37. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/_app.py +0 -0
  38. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/_errors.py +0 -0
  39. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/__init__.py +0 -0
  40. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_auth.py +0 -0
  41. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_info.py +0 -0
  42. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/data_publish.py +0 -0
  43. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/refs.py +0 -0
  44. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/commands/sparql.py +0 -0
  45. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/cli/main.py +0 -0
  46. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/__init__.py +0 -0
  47. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/dataset.py +0 -0
  48. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/graph/terms.py +0 -0
  49. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/identifiers.py +0 -0
  50. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/__init__.py +0 -0
  51. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/fetcher.py +0 -0
  52. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/import_/storage.py +0 -0
  53. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/__init__.py +0 -0
  54. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/builder.py +0 -0
  55. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/models.py +0 -0
  56. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/provenance.py +0 -0
  57. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/metadata/provenance_rdf.py +0 -0
  58. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/publish.py +0 -0
  59. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/utils/checksum.py +0 -0
  60. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/utils/encoding.py +0 -0
  61. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/versioning/__init__.py +0 -0
  62. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_data/versioning/schema_evolution.py +0 -0
  63. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/__init__.py +0 -0
  64. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/assets/__init__.py +0 -0
  65. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/assets/mirrors.py +0 -0
  66. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/definitions.py +0 -0
  67. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/io_manager.py +0 -0
  68. {sunstone_data_platform-1.0.0 → sunstone_data_platform-1.2.0}/src/sunstone_pipeline/resources/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sunstone-data-platform
3
- Version: 1.0.0
3
+ Version: 1.2.0
4
4
  Summary: Sunstone Data Platform - Data lakehouse with semantic metadata
5
5
  Author-email: Sunstone Institute <stig@sunstone.institute>
6
6
  License-Expression: MIT
@@ -9,6 +9,7 @@ Requires-Dist: charset-normalizer>=3.4.0
9
9
  Requires-Dist: duckdb>=1.5.3
10
10
  Requires-Dist: fastexcel>=0.20.2
11
11
  Requires-Dist: httpx>=0.27.0
12
+ Requires-Dist: publicsuffixlist>=1.0.2.20261003
12
13
  Requires-Dist: pyiceberg[duckdb,gcsfs,s3fs]>=0.10.0
13
14
  Requires-Dist: python-dateutil>=2.9.0.post0
14
15
  Requires-Dist: python-dotenv>=1.2.2
@@ -25,48 +26,44 @@ Description-Content-Type: text/markdown
25
26
 
26
27
  # Sunstone Data Platform
27
28
 
28
- A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, iceberg-catalog, RustFS, and DuckDB.
29
+ A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, an Iceberg REST catalog, a versioned RDF graph in Postgres, RustFS, and DuckDB.
29
30
 
30
- Data scientists can import CSV/Excel files from local paths or URLs, query them via DuckDB with git-like version references, and track lineage through notebook sessions. All provenance is captured using W3C PROV-DM so metadata can later materialize as a knowledge graph.
31
+ Data scientists import CSV/Excel files from local paths or URLs into Iceberg tables, query them with DuckDB using git-like version references, and publish dataset metadata and W3C PROV-DM provenance into the graph. The graph is the system of record for metadata, branches and tags.
31
32
 
32
33
  ## Architecture
33
34
 
34
35
  ```
35
- ┌──────────────┐
36
- │ sunstone │
37
- │ CLI (Typer) │
38
- └──────┬───────┘
39
- │
40
- ┌────────────────┼────────────────┐
41
- │ │ │
42
- ┌────────▼───────┐ ┌─────▼──────┐ ┌───────▼────────┐
43
- │ Import Pipeline │ │ Query │ │ Versioning │
44
- │ fetch → parse → │ │ (DuckDB) │ │ branches/tags/ │
45
- │ store → metadata│ │ │ │ history/diff │
46
- └────────┬────────┘ └─────┬──────┘ └───────┬────────┘
47
- │ │ │
48
- └────────────────┼────────────────┘
49
- │
50
- ┌──────────▼──────────┐
51
- │ iceberg-catalog │
52
- │ (Iceberg REST API) │
53
- └──────────┬──────────┘
54
- │
55
- ┌──────────▼──────────┐
56
- │ RustFS (S3) │
57
- │ Iceberg tables + │
58
- │ original files │
59
- └─────────────────────┘
36
+ ┌─────────────────────────────────┐
37
+ │ sunstone_data (Python) │
38
+ │ sunstone data CLI, sunstone: │
39
+ │ URLs, import pipeline, DuckDB │
40
+ └───────┬─────────────────┬───────┘
41
+ Iceberg REST │ │ GSP, refs, SPARQL
42
+ ▼ ▼
43
+ ┌─────────────────┐ ┌─────────────────┐
44
+ │ iceberg-catalog │──▶│ graph-server │
45
+ │ :8181 │ │ :8080 │
46
+ └────────┬────────┘ └────────┬────────┘
47
+ ▼ ▼
48
+ ┌─────────────────┐ ┌─────────────────┐
49
+ │ RustFS (S3) │ │ Postgres graph │
50
+ │ Iceberg tables, │ │ (SoR), copied │
51
+ │ original files │ │ to Oxigraph │
52
+ └─────────────────┘ └─────────────────┘
60
53
  ```
61
54
 
62
- **Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | RustFS | PostgreSQL 16
55
+ On every table commit, iceberg-catalog also writes the table's descriptor into
56
+ the graph. See [docs/architecture.md](docs/architecture.md) for the
57
+ full data flow.
58
+
59
+ **Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | graph-server | Oxigraph | RustFS | PostgreSQL 16
63
60
 
64
61
  ## Endpoints
65
62
 
66
63
  - `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
67
64
  (see [the views design](docs/specs/2026-08-02-graph-server-views-design.md))
68
- - `/ui/` — the graph explorer, a 3D view of the data graph, served by
69
- graph-server (see [docs/graph-explorer.md](docs/graph-explorer.md))
65
+ - `/ui/` — the data explorer, a 3D view of the data graph, served by
66
+ graph-server (see [docs/data-explorer.md](docs/data-explorer.md))
70
67
 
71
68
  ## Local Development Setup
72
69
 
@@ -150,14 +147,16 @@ uv run scripts/smoke_test.py
150
147
 
151
148
  ### 5. Set up your environment
152
149
 
153
- The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group:
150
+ The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group. You configure one value per environment, the base IRI (`base_iri`). graph-server is served at the base, the Iceberg catalog under `{base_iri}iceberg/`, and datasets and organizations are named under it. The client discovers sign-in settings from graph-server.
154
151
 
155
152
  ```bash
156
153
  # Add your local Docker stack as an environment
157
- sunstone env add local --catalog-url http://localhost:19120 --s3-endpoint http://localhost:9000
154
+ sunstone env add local data-platform.base_iri=https://data.example.org/
158
155
  sunstone env use local
159
156
  ```
160
157
 
158
+ `base_iri` must equal the iceberg-catalog service's `BASE_IRI`. In the local Docker stack, graph-server (port 8080) and the catalog (port 8181) listen on different ports, so also set the service overrides `GRAPH_URL=http://localhost:8080` and `CATALOG_URI=http://localhost:8181` (see `.env.example`). They never affect IRIs.
159
+
161
160
  ### 6. Try it out
162
161
 
163
162
  ```bash
@@ -167,17 +166,17 @@ sunstone data import path/to/data.csv
167
166
  # Import from a URL
168
167
  sunstone data import https://example.com/dataset.csv
169
168
 
170
- # List datasets
171
- sunstone data info
169
+ # List namespaces and tables
170
+ sunstone data ls
172
171
 
173
- # View dataset metadata
174
- sunstone data info namespace.table_name
172
+ # View dataset metadata from the graph
173
+ sunstone data show namespace.table_name
175
174
 
176
- # Query history
175
+ # Snapshot history
177
176
  sunstone data history namespace.table_name
178
177
 
179
- # Compare versions
180
- sunstone data diff namespace.table_name @v1 @v2
178
+ # Publish the current snapshot as a named version
179
+ sunstone data publish namespace.table_name --version v1.0
181
180
  ```
182
181
 
183
182
  ## Running Tests
@@ -186,7 +185,7 @@ sunstone data diff namespace.table_name @v1 @v2
186
185
  # Unit tests (fast, no Docker required)
187
186
  uv run pytest tests/unit/
188
187
 
189
- # Integration tests (requires Docker stack running)
188
+ # Integration tests (run inside Docker automatically; needs Docker)
190
189
  uv run pytest tests/integration/
191
190
 
192
191
  # All tests
@@ -200,25 +199,33 @@ uv run pytest tests/unit/test_query.py -v
200
199
 
201
200
  | Command | Description |
202
201
  |---------|-------------|
203
- | `sunstone data import <source>` | Import CSV/Excel from path or URL |
204
- | `sunstone data info [table]` | View metadata for one or all datasets |
205
- | `sunstone data history <table>` | Show snapshot history (git-log style) |
206
- | `sunstone data tag <table> <tag>` | Tag a snapshot version |
207
- | `sunstone data untag <table> <tag>` | Remove a tag |
208
- | `sunstone data branches` | List branches and tags |
209
- | `sunstone data diff <table> <from> [to]` | Compare two versions |
210
- | `sunstone data lineage <table>` | Show provenance and lineage |
211
- | `sunstone data original <table>` | Download the original imported source file |
212
-
213
- All commands support `--branch` for working on version branches and `--json` for machine-readable output. Run `sunstone data <command> --help` for full options.
202
+ | `sunstone data login`, `logout`, `whoami` | Sign in to graph-server (Keycloak) |
203
+ | `sunstone data ls [namespace]` | List namespaces and tables |
204
+ | `sunstone data show <table>` | Dataset metadata from the graph (table, Turtle or JSON-LD) |
205
+ | `sunstone data info [table]` | Iceberg table properties for one or all datasets |
206
+ | `sunstone data import <source>` | Import CSV/Excel from a path or URL |
207
+ | `sunstone data publish <table>` | Publish dataset metadata to the graph; `--version` names the current snapshot |
208
+ | `sunstone data history <table>` | Iceberg snapshot history |
209
+ | `sunstone data lineage <table>` | Provenance from the published PROV facts |
210
+ | `sunstone data sparql <query>` | Run a SPARQL query through graph-server |
211
+ | `sunstone data branches`, `branch create\|delete` | List, create and delete graph branches |
212
+ | `sunstone data tags`, `tag create\|delete` | List, create and delete graph tags |
213
+
214
+ Every command takes `--json` for machine-readable output. `import`, `publish`, `show`, `history` and `tag create` take `--branch`. Run `sunstone data <command> --help` for full options.
214
215
 
215
216
  ## Key Concepts
216
217
 
217
- - **Table references** use git-like syntax: `namespace.table@branch`, `@v1.0`, `@2026-01-25`, `@snapshot_id`
218
- - **Deduplication**: Imports are checksum-based (SHA256). Re-importing the same file is a no-op.
218
+ - **Table references** use the form `namespace.table@ref`. A ref is checked in this order:
219
+ 1. a branch name
220
+ 2. a snapshot id (13 or more digits)
221
+ 3. a date or timestamp (UTC if it has no timezone), which selects the last snapshot at or before that time
222
+ 4. a named version created with `sunstone data publish --version`
223
+
224
+ Snapshot ids, timestamps and named versions resolve against `main`. Graph tags (`sunstone data tag create`) are not valid refs.
225
+ - **Deduplication**: Imports are checksum-based (SHA-256). Re-importing the same file is a no-op.
219
226
  - **Schema evolution**: Safe type promotions (e.g. int32 to int64) are allowed automatically. Unsafe changes require `--force-schema`.
220
- - **Provenance**: Every import captures source metadata, schema info, and W3C PROV-DM provenance as Iceberg table properties.
221
- - **Original files**: Source files are stored content-addressable in S3 alongside the Iceberg data.
227
+ - **Provenance**: Every import records source metadata and schema info as Iceberg table properties, and asserts W3C PROV-DM provenance into the dataset's graph.
228
+ - **Original files**: Source files are stored content-addressed in object storage next to the Iceberg data.
222
229
 
223
230
  ## sunstone: URLs
224
231
 
@@ -230,7 +237,7 @@ import sunstone
230
237
  df = sunstone.read("sunstone:namespace/table@v1.0")
231
238
  ```
232
239
 
233
- Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
240
+ The `@ref` part follows the same rules as table references above. Here `v1.0` is a named version.
234
241
 
235
242
  ## Project Structure
236
243
 
@@ -238,24 +245,26 @@ Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
238
245
  src/sunstone_data/
239
246
  ├── plugin.py # sunstone-py plugin (CLIProvider + URLHandler)
240
247
  ├── url.py # sunstone: URL parser
248
+ ├── resolve.py # Resolves a @ref to a branch or snapshot
241
249
  ├── config.py # Settings from DataEnvironment or .env
250
+ ├── discovery.py # Sign-in settings from graph-server /ui/config.json
251
+ ├── auth/ # Keycloak login and token handling
252
+ ├── graph/ # graph-server client and RDF serialization
253
+ ├── publish.py # Publishes dataset metadata to the graph
242
254
  ├── catalog.py # PyIceberg RestCatalog connection to the Iceberg REST catalog
243
255
  ├── query.py # DuckDB QuerySession with time-travel support
244
- ├── lineage.py # Lineage in Iceberg table properties
245
- ├── cli/ # Typer CLI commands + Rich formatters
256
+ ├── cli/ # Typer CLI commands
246
257
  ├── import_/ # Pipeline: fetch → parse → store → metadata
247
258
  ├── metadata/ # Frozen dataclasses, MetadataBuilder, W3C PROV-DM
248
- ├── versioning/ # Version branches/tags, refs, schema evolution, diff
259
+ ├── versioning/ # Schema evolution checks
249
260
  └── utils/ # Checksums, encoding detection, naming
250
261
  ```
251
262
 
252
263
  ## Documentation
253
264
 
254
- - [Getting Started](docs/getting-started.md) - Detailed setup instructions
255
- - [Architecture](docs/architecture.md) - System design and data flow
256
- - [CLI Reference](docs/cli-reference.md) - Complete command documentation
257
- - [Import Pipeline](docs/import-pipeline.md) - How imports work
258
- - [Development Guide](docs/development.md) - Contributing and testing
265
+ - [docs/README.md](docs/README.md) - Index of all documentation
266
+ - [docs/architecture.md](docs/architecture.md) - System overview and data flow
267
+ - [docs/specs2/](docs/specs2/) - Current design, one file per subsystem. The Python client and CLI are in [05-python-client.md](docs/specs2/05-python-client.md).
259
268
 
260
269
  ## Resetting the Stack
261
270
 
@@ -278,11 +287,11 @@ determine which environment you can use. See
278
287
  `docs/specs/2026-07-05-graph-server-auth-design.md`.
279
288
 
280
289
  No hostnames or client ids ship in this package. Each environment's
281
- `graph_url` comes from config: Sunstone-managed machines get
290
+ `base_iri` comes from config: Sunstone-managed machines get
282
291
  `/etc/sunstone/data_platform.toml` from the `fleet-config` repo, and other
283
- machines set `GRAPH_URL` or add an environment with `sunstone env add`.
292
+ machines set `BASE_IRI` or add an environment with `sunstone env add`.
284
293
  graph-server publishes the OIDC issuer and public client id at
285
- `$GRAPH_URL/ui/config.json`; the CLI reads them from there.
294
+ `{base_iri}ui/config.json`; the CLI reads them from there.
286
295
 
287
296
  ### Humans (CLI, local agents)
288
297
 
@@ -1,47 +1,43 @@
1
1
  # Sunstone Data Platform
2
2
 
3
- A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, iceberg-catalog, RustFS, and DuckDB.
3
+ A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, an Iceberg REST catalog, a versioned RDF graph in Postgres, RustFS, and DuckDB.
4
4
 
5
- Data scientists can import CSV/Excel files from local paths or URLs, query them via DuckDB with git-like version references, and track lineage through notebook sessions. All provenance is captured using W3C PROV-DM so metadata can later materialize as a knowledge graph.
5
+ Data scientists import CSV/Excel files from local paths or URLs into Iceberg tables, query them with DuckDB using git-like version references, and publish dataset metadata and W3C PROV-DM provenance into the graph. The graph is the system of record for metadata, branches and tags.
6
6
 
7
7
  ## Architecture
8
8
 
9
9
  ```
10
- ┌──────────────┐
11
- │ sunstone │
12
- │ CLI (Typer) │
13
- └──────┬───────┘
14
- │
15
- ┌────────────────┼────────────────┐
16
- │ │ │
17
- ┌────────▼───────┐ ┌─────▼──────┐ ┌───────▼────────┐
18
- │ Import Pipeline │ │ Query │ │ Versioning │
19
- │ fetch → parse → │ │ (DuckDB) │ │ branches/tags/ │
20
- │ store → metadata│ │ │ │ history/diff │
21
- └────────┬────────┘ └─────┬──────┘ └───────┬────────┘
22
- │ │ │
23
- └────────────────┼────────────────┘
24
- │
25
- ┌──────────▼──────────┐
26
- │ iceberg-catalog │
27
- │ (Iceberg REST API) │
28
- └──────────┬──────────┘
29
- │
30
- ┌──────────▼──────────┐
31
- │ RustFS (S3) │
32
- │ Iceberg tables + │
33
- │ original files │
34
- └─────────────────────┘
10
+ ┌─────────────────────────────────┐
11
+ │ sunstone_data (Python) │
12
+ │ sunstone data CLI, sunstone: │
13
+ │ URLs, import pipeline, DuckDB │
14
+ └───────┬─────────────────┬───────┘
15
+ Iceberg REST │ │ GSP, refs, SPARQL
16
+ ▼ ▼
17
+ ┌─────────────────┐ ┌─────────────────┐
18
+ │ iceberg-catalog │──▶│ graph-server │
19
+ │ :8181 │ │ :8080 │
20
+ └────────┬────────┘ └────────┬────────┘
21
+ ▼ ▼
22
+ ┌─────────────────┐ ┌─────────────────┐
23
+ │ RustFS (S3) │ │ Postgres graph │
24
+ │ Iceberg tables, │ │ (SoR), copied │
25
+ │ original files │ │ to Oxigraph │
26
+ └─────────────────┘ └─────────────────┘
35
27
  ```
36
28
 
37
- **Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | RustFS | PostgreSQL 16
29
+ On every table commit, iceberg-catalog also writes the table's descriptor into
30
+ the graph. See [docs/architecture.md](docs/architecture.md) for the
31
+ full data flow.
32
+
33
+ **Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | graph-server | Oxigraph | RustFS | PostgreSQL 16
38
34
 
39
35
  ## Endpoints
40
36
 
41
37
  - `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
42
38
  (see [the views design](docs/specs/2026-08-02-graph-server-views-design.md))
43
- - `/ui/` — the graph explorer, a 3D view of the data graph, served by
44
- graph-server (see [docs/graph-explorer.md](docs/graph-explorer.md))
39
+ - `/ui/` — the data explorer, a 3D view of the data graph, served by
40
+ graph-server (see [docs/data-explorer.md](docs/data-explorer.md))
45
41
 
46
42
  ## Local Development Setup
47
43
 
@@ -125,14 +121,16 @@ uv run scripts/smoke_test.py
125
121
 
126
122
  ### 5. Set up your environment
127
123
 
128
- The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group:
124
+ The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group. You configure one value per environment, the base IRI (`base_iri`). graph-server is served at the base, the Iceberg catalog under `{base_iri}iceberg/`, and datasets and organizations are named under it. The client discovers sign-in settings from graph-server.
129
125
 
130
126
  ```bash
131
127
  # Add your local Docker stack as an environment
132
- sunstone env add local --catalog-url http://localhost:19120 --s3-endpoint http://localhost:9000
128
+ sunstone env add local data-platform.base_iri=https://data.example.org/
133
129
  sunstone env use local
134
130
  ```
135
131
 
132
+ `base_iri` must equal the iceberg-catalog service's `BASE_IRI`. In the local Docker stack, graph-server (port 8080) and the catalog (port 8181) listen on different ports, so also set the service overrides `GRAPH_URL=http://localhost:8080` and `CATALOG_URI=http://localhost:8181` (see `.env.example`). They never affect IRIs.
133
+
136
134
  ### 6. Try it out
137
135
 
138
136
  ```bash
@@ -142,17 +140,17 @@ sunstone data import path/to/data.csv
142
140
  # Import from a URL
143
141
  sunstone data import https://example.com/dataset.csv
144
142
 
145
- # List datasets
146
- sunstone data info
143
+ # List namespaces and tables
144
+ sunstone data ls
147
145
 
148
- # View dataset metadata
149
- sunstone data info namespace.table_name
146
+ # View dataset metadata from the graph
147
+ sunstone data show namespace.table_name
150
148
 
151
- # Query history
149
+ # Snapshot history
152
150
  sunstone data history namespace.table_name
153
151
 
154
- # Compare versions
155
- sunstone data diff namespace.table_name @v1 @v2
152
+ # Publish the current snapshot as a named version
153
+ sunstone data publish namespace.table_name --version v1.0
156
154
  ```
157
155
 
158
156
  ## Running Tests
@@ -161,7 +159,7 @@ sunstone data diff namespace.table_name @v1 @v2
161
159
  # Unit tests (fast, no Docker required)
162
160
  uv run pytest tests/unit/
163
161
 
164
- # Integration tests (requires Docker stack running)
162
+ # Integration tests (run inside Docker automatically; needs Docker)
165
163
  uv run pytest tests/integration/
166
164
 
167
165
  # All tests
@@ -175,25 +173,33 @@ uv run pytest tests/unit/test_query.py -v
175
173
 
176
174
  | Command | Description |
177
175
  |---------|-------------|
178
- | `sunstone data import <source>` | Import CSV/Excel from path or URL |
179
- | `sunstone data info [table]` | View metadata for one or all datasets |
180
- | `sunstone data history <table>` | Show snapshot history (git-log style) |
181
- | `sunstone data tag <table> <tag>` | Tag a snapshot version |
182
- | `sunstone data untag <table> <tag>` | Remove a tag |
183
- | `sunstone data branches` | List branches and tags |
184
- | `sunstone data diff <table> <from> [to]` | Compare two versions |
185
- | `sunstone data lineage <table>` | Show provenance and lineage |
186
- | `sunstone data original <table>` | Download the original imported source file |
187
-
188
- All commands support `--branch` for working on version branches and `--json` for machine-readable output. Run `sunstone data <command> --help` for full options.
176
+ | `sunstone data login`, `logout`, `whoami` | Sign in to graph-server (Keycloak) |
177
+ | `sunstone data ls [namespace]` | List namespaces and tables |
178
+ | `sunstone data show <table>` | Dataset metadata from the graph (table, Turtle or JSON-LD) |
179
+ | `sunstone data info [table]` | Iceberg table properties for one or all datasets |
180
+ | `sunstone data import <source>` | Import CSV/Excel from a path or URL |
181
+ | `sunstone data publish <table>` | Publish dataset metadata to the graph; `--version` names the current snapshot |
182
+ | `sunstone data history <table>` | Iceberg snapshot history |
183
+ | `sunstone data lineage <table>` | Provenance from the published PROV facts |
184
+ | `sunstone data sparql <query>` | Run a SPARQL query through graph-server |
185
+ | `sunstone data branches`, `branch create\|delete` | List, create and delete graph branches |
186
+ | `sunstone data tags`, `tag create\|delete` | List, create and delete graph tags |
187
+
188
+ Every command takes `--json` for machine-readable output. `import`, `publish`, `show`, `history` and `tag create` take `--branch`. Run `sunstone data <command> --help` for full options.
189
189
 
190
190
  ## Key Concepts
191
191
 
192
- - **Table references** use git-like syntax: `namespace.table@branch`, `@v1.0`, `@2026-01-25`, `@snapshot_id`
193
- - **Deduplication**: Imports are checksum-based (SHA256). Re-importing the same file is a no-op.
192
+ - **Table references** use the form `namespace.table@ref`. A ref is checked in this order:
193
+ 1. a branch name
194
+ 2. a snapshot id (13 or more digits)
195
+ 3. a date or timestamp (UTC if it has no timezone), which selects the last snapshot at or before that time
196
+ 4. a named version created with `sunstone data publish --version`
197
+
198
+ Snapshot ids, timestamps and named versions resolve against `main`. Graph tags (`sunstone data tag create`) are not valid refs.
199
+ - **Deduplication**: Imports are checksum-based (SHA-256). Re-importing the same file is a no-op.
194
200
  - **Schema evolution**: Safe type promotions (e.g. int32 to int64) are allowed automatically. Unsafe changes require `--force-schema`.
195
- - **Provenance**: Every import captures source metadata, schema info, and W3C PROV-DM provenance as Iceberg table properties.
196
- - **Original files**: Source files are stored content-addressable in S3 alongside the Iceberg data.
201
+ - **Provenance**: Every import records source metadata and schema info as Iceberg table properties, and asserts W3C PROV-DM provenance into the dataset's graph.
202
+ - **Original files**: Source files are stored content-addressed in object storage next to the Iceberg data.
197
203
 
198
204
  ## sunstone: URLs
199
205
 
@@ -205,7 +211,7 @@ import sunstone
205
211
  df = sunstone.read("sunstone:namespace/table@v1.0")
206
212
  ```
207
213
 
208
- Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
214
+ The `@ref` part follows the same rules as table references above. Here `v1.0` is a named version.
209
215
 
210
216
  ## Project Structure
211
217
 
@@ -213,24 +219,26 @@ Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
213
219
  src/sunstone_data/
214
220
  ├── plugin.py # sunstone-py plugin (CLIProvider + URLHandler)
215
221
  ├── url.py # sunstone: URL parser
222
+ ├── resolve.py # Resolves a @ref to a branch or snapshot
216
223
  ├── config.py # Settings from DataEnvironment or .env
224
+ ├── discovery.py # Sign-in settings from graph-server /ui/config.json
225
+ ├── auth/ # Keycloak login and token handling
226
+ ├── graph/ # graph-server client and RDF serialization
227
+ ├── publish.py # Publishes dataset metadata to the graph
217
228
  ├── catalog.py # PyIceberg RestCatalog connection to the Iceberg REST catalog
218
229
  ├── query.py # DuckDB QuerySession with time-travel support
219
- ├── lineage.py # Lineage in Iceberg table properties
220
- ├── cli/ # Typer CLI commands + Rich formatters
230
+ ├── cli/ # Typer CLI commands
221
231
  ├── import_/ # Pipeline: fetch → parse → store → metadata
222
232
  ├── metadata/ # Frozen dataclasses, MetadataBuilder, W3C PROV-DM
223
- ├── versioning/ # Version branches/tags, refs, schema evolution, diff
233
+ ├── versioning/ # Schema evolution checks
224
234
  └── utils/ # Checksums, encoding detection, naming
225
235
  ```
226
236
 
227
237
  ## Documentation
228
238
 
229
- - [Getting Started](docs/getting-started.md) - Detailed setup instructions
230
- - [Architecture](docs/architecture.md) - System design and data flow
231
- - [CLI Reference](docs/cli-reference.md) - Complete command documentation
232
- - [Import Pipeline](docs/import-pipeline.md) - How imports work
233
- - [Development Guide](docs/development.md) - Contributing and testing
239
+ - [docs/README.md](docs/README.md) - Index of all documentation
240
+ - [docs/architecture.md](docs/architecture.md) - System overview and data flow
241
+ - [docs/specs2/](docs/specs2/) - Current design, one file per subsystem. The Python client and CLI are in [05-python-client.md](docs/specs2/05-python-client.md).
234
242
 
235
243
  ## Resetting the Stack
236
244
 
@@ -253,11 +261,11 @@ determine which environment you can use. See
253
261
  `docs/specs/2026-07-05-graph-server-auth-design.md`.
254
262
 
255
263
  No hostnames or client ids ship in this package. Each environment's
256
- `graph_url` comes from config: Sunstone-managed machines get
264
+ `base_iri` comes from config: Sunstone-managed machines get
257
265
  `/etc/sunstone/data_platform.toml` from the `fleet-config` repo, and other
258
- machines set `GRAPH_URL` or add an environment with `sunstone env add`.
266
+ machines set `BASE_IRI` or add an environment with `sunstone env add`.
259
267
  graph-server publishes the OIDC issuer and public client id at
260
- `$GRAPH_URL/ui/config.json`; the CLI reads them from there.
268
+ `{base_iri}ui/config.json`; the CLI reads them from there.
261
269
 
262
270
  ### Humans (CLI, local agents)
263
271
 
@@ -24,6 +24,7 @@ dependencies = [
24
24
  "fastexcel>=0.20.2",
25
25
  "python-slugify>=8.0.0",
26
26
  "python-dateutil>=2.9.0.post0",
27
+ "publicsuffixlist>=1.0.2.20261003",
27
28
  ]
28
29
 
29
30
  [project.entry-points."sunstone.plugins"]
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.0.0'
22
- __version_tuple__ = version_tuple = (1, 0, 0)
21
+ __version__ = version = '1.2.0'
22
+ __version_tuple__ = version_tuple = (1, 2, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -22,6 +22,7 @@ from sunstone_data.auth.client_creds import (
22
22
  refresh_access_token,
23
23
  resolve_access_token,
24
24
  )
25
+ from sunstone_data.auth.stepup import StepUpChallenge, StepUpRequired, parse_challenge, step_up
25
26
  from sunstone_data.auth.store import (
26
27
  Session,
27
28
  clear_session,
@@ -33,11 +34,15 @@ from sunstone_data.auth.store import (
33
34
  __all__ = [
34
35
  "ClientCredentialsToken",
35
36
  "Session",
37
+ "StepUpChallenge",
38
+ "StepUpRequired",
36
39
  "clear_session",
37
40
  "discover_token_endpoint",
38
41
  "read_session",
42
+ "parse_challenge",
39
43
  "refresh_access_token",
40
44
  "resolve_access_token",
41
45
  "session_path",
46
+ "step_up",
42
47
  "write_session",
43
48
  ]
@@ -72,18 +72,27 @@ def build_authorization_url(
72
72
  redirect_uri: str,
73
73
  state: str,
74
74
  code_challenge: str,
75
+ acr_values: str | None = None,
76
+ max_age: int | None = None,
75
77
  ) -> str:
76
- """Build the URL the user's browser is sent to in order to sign in."""
77
- query = urlencode(
78
- {
79
- "response_type": "code",
80
- "client_id": client_id,
81
- "redirect_uri": redirect_uri,
82
- "state": state,
83
- "code_challenge": code_challenge,
84
- "code_challenge_method": "S256",
85
- }
86
- )
78
+ """Build the URL the user's browser is sent to in order to sign in.
79
+
80
+ ``acr_values`` and ``max_age`` are the OIDC step-up parameters (RFC 9470).
81
+ They are sent only when given.
82
+ """
83
+ params = {
84
+ "response_type": "code",
85
+ "client_id": client_id,
86
+ "redirect_uri": redirect_uri,
87
+ "state": state,
88
+ "code_challenge": code_challenge,
89
+ "code_challenge_method": "S256",
90
+ }
91
+ if acr_values is not None:
92
+ params["acr_values"] = acr_values
93
+ if max_age is not None:
94
+ params["max_age"] = str(max_age)
95
+ query = urlencode(params)
87
96
  separator = "&" if urlparse(authorization_endpoint).query else "?"
88
97
  return f"{authorization_endpoint}{separator}{query}"
89
98
 
@@ -262,6 +271,8 @@ def login(
262
271
  client_id: str,
263
272
  client: httpx.Client,
264
273
  open_browser: Callable[[str], object] = webbrowser.open,
274
+ acr_values: str | None = None,
275
+ max_age: int | None = None,
265
276
  ) -> TokenResponse:
266
277
  """Run the full PKCE sign-in and return the resulting tokens.
267
278
 
@@ -276,6 +287,8 @@ def login(
276
287
  client: HTTP client for the token exchange.
277
288
  open_browser: How to show the user the authorization URL. Injected so
278
289
  tests need no browser.
290
+ acr_values: Authentication context the sign-in must satisfy (step-up).
291
+ max_age: Newest allowed age, in seconds, of the user's last sign-in.
279
292
 
280
293
  Raises:
281
294
  RuntimeError: Sign-in was refused, timed out, or the callback failed
@@ -292,6 +305,8 @@ def login(
292
305
  redirect_uri=redirect_uri,
293
306
  state=state,
294
307
  code_challenge=derive_code_challenge(verifier),
308
+ acr_values=acr_values,
309
+ max_age=max_age,
295
310
  )
296
311
  open_browser(url)
297
312
  code = verify_callback(await_callback(server, sink), state)