sunstone-data-platform 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. sunstone_data/__init__.py +36 -0
  2. sunstone_data/_version.py +24 -0
  3. sunstone_data/auth/__init__.py +43 -0
  4. sunstone_data/auth/client_creds.py +239 -0
  5. sunstone_data/auth/pkce.py +307 -0
  6. sunstone_data/auth/store.py +97 -0
  7. sunstone_data/catalog.py +175 -0
  8. sunstone_data/cli/__init__.py +5 -0
  9. sunstone_data/cli/_app.py +26 -0
  10. sunstone_data/cli/_errors.py +31 -0
  11. sunstone_data/cli/commands/__init__.py +22 -0
  12. sunstone_data/cli/commands/data_auth.py +120 -0
  13. sunstone_data/cli/commands/data_import.py +157 -0
  14. sunstone_data/cli/commands/data_info.py +166 -0
  15. sunstone_data/cli/commands/data_publish.py +69 -0
  16. sunstone_data/cli/commands/ls.py +49 -0
  17. sunstone_data/cli/commands/prov.py +58 -0
  18. sunstone_data/cli/commands/refs.py +142 -0
  19. sunstone_data/cli/commands/show.py +139 -0
  20. sunstone_data/cli/commands/snapshots.py +77 -0
  21. sunstone_data/cli/commands/sparql.py +68 -0
  22. sunstone_data/cli/main.py +30 -0
  23. sunstone_data/config.py +250 -0
  24. sunstone_data/discovery.py +317 -0
  25. sunstone_data/env_commands.py +97 -0
  26. sunstone_data/env_section.py +86 -0
  27. sunstone_data/graph/__init__.py +36 -0
  28. sunstone_data/graph/client.py +328 -0
  29. sunstone_data/graph/dataset.py +438 -0
  30. sunstone_data/graph/terms.py +34 -0
  31. sunstone_data/identifiers.py +155 -0
  32. sunstone_data/import_/__init__.py +28 -0
  33. sunstone_data/import_/fetcher.py +259 -0
  34. sunstone_data/import_/owner.py +130 -0
  35. sunstone_data/import_/parser.py +492 -0
  36. sunstone_data/import_/pipeline.py +275 -0
  37. sunstone_data/import_/storage.py +135 -0
  38. sunstone_data/metadata/__init__.py +41 -0
  39. sunstone_data/metadata/builder.py +276 -0
  40. sunstone_data/metadata/models.py +87 -0
  41. sunstone_data/metadata/provenance.py +171 -0
  42. sunstone_data/metadata/provenance_rdf.py +102 -0
  43. sunstone_data/plugin.py +62 -0
  44. sunstone_data/publish.py +134 -0
  45. sunstone_data/query.py +624 -0
  46. sunstone_data/resolve.py +178 -0
  47. sunstone_data/url.py +78 -0
  48. sunstone_data/utils/__init__.py +12 -0
  49. sunstone_data/utils/checksum.py +14 -0
  50. sunstone_data/utils/encoding.py +16 -0
  51. sunstone_data/utils/naming.py +38 -0
  52. sunstone_data/versioning/__init__.py +17 -0
  53. sunstone_data/versioning/schema_evolution.py +180 -0
  54. sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
  55. sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
  56. sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
  57. sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
  58. sunstone_pipeline/__init__.py +1 -0
  59. sunstone_pipeline/assets/__init__.py +11 -0
  60. sunstone_pipeline/assets/mirrors.py +123 -0
  61. sunstone_pipeline/assets/transforms.py +74 -0
  62. sunstone_pipeline/definitions.py +25 -0
  63. sunstone_pipeline/io_manager.py +56 -0
  64. sunstone_pipeline/resources/__init__.py +6 -0
  65. sunstone_pipeline/resources/iceberg.py +29 -0
  66. sunstone_pipeline/resources/refs.py +88 -0
@@ -0,0 +1,276 @@
1
+ """Builder for constructing ImportMetadata during pipeline execution."""
2
+
3
+ import getpass
4
+ import os
5
+ from datetime import datetime, timezone
6
+
7
+ from sunstone_data.metadata.models import (
8
+ ImportMetadata,
9
+ SchemaMetadata,
10
+ SourceMetadata,
11
+ )
12
+ from sunstone_data.metadata.provenance import (
13
+ ProvActivity,
14
+ ProvAgent,
15
+ ProvEntity,
16
+ generate_activity_id,
17
+ make_activity_uri,
18
+ make_file_uri,
19
+ make_table_uri,
20
+ make_url_uri,
21
+ )
22
+
23
+
24
+ def _get_user() -> str:
25
+ """Get current username using multiple fallback methods.
26
+
27
+ Returns:
28
+ Username string, or "unknown" if detection fails.
29
+ """
30
+ try:
31
+ return os.getlogin()
32
+ except OSError:
33
+ pass
34
+
35
+ try:
36
+ return getpass.getuser()
37
+ except Exception:
38
+ pass
39
+
40
+ return os.environ.get("USER", "unknown")
41
+
42
+
43
+ def _get_system_version() -> str:
44
+ """Get the package version for provenance records.
45
+
46
+ Looks up the installed distribution by its current package name
47
+ (``sunstone-data-platform``). The agent URN scheme below still uses
48
+ ``urn:system:sunstone-data:...`` for provenance continuity with
49
+ historical records, but the package itself was renamed.
50
+
51
+ Returns:
52
+ Version string, or "dev" if package not installed.
53
+ """
54
+ try:
55
+ from importlib.metadata import version
56
+
57
+ return version("sunstone-data-platform")
58
+ except Exception:
59
+ return "dev"
60
+
61
+
62
+ class MetadataBuilder:
63
+ """Builds ImportMetadata progressively during import pipeline.
64
+
65
+ The builder accumulates metadata as it becomes available through
66
+ the fetch -> parse -> store pipeline stages, then produces a
67
+ complete ImportMetadata instance at the end.
68
+ """
69
+
70
+ def __init__(self, source: str, import_parameters: dict | None = None):
71
+ """Initialize with original source and parameters.
72
+
73
+ Args:
74
+ source: Original source path/URL as provided by user.
75
+ import_parameters: Dict of import params (encoding_override, tags, etc).
76
+ """
77
+ self.activity_id = generate_activity_id()
78
+ self.timestamp_start = datetime.now(timezone.utc).isoformat()
79
+ self.user = _get_user()
80
+ self.system_version = _get_system_version()
81
+ self.original_source = source
82
+ self.import_parameters = import_parameters or {}
83
+
84
+ # Fields populated by set_source_info
85
+ self._source_info: SourceMetadata | None = None
86
+
87
+ # Fields populated by set_schema_info
88
+ self._schema_info: SchemaMetadata | None = None
89
+
90
+ # Fields populated by set_table_info
91
+ self._table_identifier: str | None = None
92
+ self._original_storage_path: str | None = None
93
+
94
+ # Set at finalization
95
+ self._timestamp_end: str | None = None
96
+
97
+ def set_source_info(
98
+ self,
99
+ normalized_path: str,
100
+ file_type: str,
101
+ file_size: int,
102
+ encoding: str | None,
103
+ checksum: str,
104
+ ) -> None:
105
+ """Set source metadata after fetch stage.
106
+
107
+ Args:
108
+ normalized_path: Absolute path or cleaned URL.
109
+ file_type: File type (csv, excel).
110
+ file_size: Size in bytes.
111
+ encoding: Detected encoding for CSV, None for Excel.
112
+ checksum: SHA256 checksum of the file.
113
+ """
114
+ self._source_info = SourceMetadata(
115
+ original_path=self.original_source,
116
+ normalized_path=normalized_path,
117
+ file_type=file_type,
118
+ file_size=file_size,
119
+ encoding=encoding,
120
+ checksum=checksum,
121
+ )
122
+
123
+ def set_schema_info(
124
+ self,
125
+ original_columns: list[dict],
126
+ iceberg_columns: list[dict],
127
+ row_count: int,
128
+ column_count: int,
129
+ ) -> None:
130
+ """Set schema metadata after parse stage.
131
+
132
+ Args:
133
+ original_columns: List of {name, type} from source file.
134
+ iceberg_columns: List of {name, type} after Iceberg conversion.
135
+ row_count: Number of rows in the dataset.
136
+ column_count: Number of columns in the dataset.
137
+ """
138
+ self._schema_info = SchemaMetadata(
139
+ original_columns=original_columns,
140
+ iceberg_columns=iceberg_columns,
141
+ row_count=row_count,
142
+ column_count=column_count,
143
+ )
144
+
145
+ def set_table_info(
146
+ self,
147
+ table_identifier: str,
148
+ original_storage_path: str,
149
+ ) -> None:
150
+ """Set table info after storage stage.
151
+
152
+ Args:
153
+ table_identifier: Full table identifier (namespace.table).
154
+ original_storage_path: S3 path to original file.
155
+ """
156
+ self._table_identifier = table_identifier
157
+ self._original_storage_path = original_storage_path
158
+
159
+ def finalize(self) -> ImportMetadata:
160
+ """Finalize and return completed ImportMetadata.
161
+
162
+ Sets timestamp_end and validates all required fields are present.
163
+
164
+ Returns:
165
+ Complete ImportMetadata instance.
166
+
167
+ Raises:
168
+ ValueError: If required fields are missing.
169
+ """
170
+ # Validate required fields
171
+ if self._source_info is None:
172
+ raise ValueError("Source info not set. Call set_source_info() first.")
173
+ if self._schema_info is None:
174
+ raise ValueError("Schema info not set. Call set_schema_info() first.")
175
+ if self._table_identifier is None:
176
+ raise ValueError("Table info not set. Call set_table_info() first.")
177
+ if self._original_storage_path is None:
178
+ raise ValueError("Original storage path not set. Call set_table_info() first.")
179
+
180
+ self._timestamp_end = datetime.now(timezone.utc).isoformat()
181
+
182
+ return ImportMetadata(
183
+ activity_id=self.activity_id,
184
+ timestamp_start=self.timestamp_start,
185
+ timestamp_end=self._timestamp_end,
186
+ source=self._source_info,
187
+ schema=self._schema_info,
188
+ table_identifier=self._table_identifier,
189
+ original_storage_path=self._original_storage_path,
190
+ user=self.user,
191
+ system_version=self.system_version,
192
+ import_parameters=self.import_parameters,
193
+ )
194
+
195
+ def build_provenance(self) -> tuple[ProvActivity, ProvAgent, ProvAgent]:
196
+ """Build PROV-DM graph from collected metadata.
197
+
198
+ Requires source_info, schema_info, and table_info to be set.
199
+
200
+ Returns:
201
+ Tuple of (activity, user_agent, system_agent).
202
+
203
+ Raises:
204
+ ValueError: If required fields are missing.
205
+ """
206
+ if self._source_info is None:
207
+ raise ValueError("Source info not set. Call set_source_info() first.")
208
+ if self._table_identifier is None:
209
+ raise ValueError("Table info not set. Call set_table_info() first.")
210
+
211
+ # Determine source entity URI based on whether it's a URL or file
212
+ source_path = self._source_info.original_path
213
+ if source_path.startswith(("http://", "https://")):
214
+ source_uri = make_url_uri(source_path)
215
+ else:
216
+ source_uri = make_file_uri(self._source_info.normalized_path)
217
+
218
+ # Build source entity
219
+ source_entity = ProvEntity(
220
+ uri=source_uri,
221
+ entity_type="SourceFile",
222
+ label=self._source_info.original_path,
223
+ )
224
+
225
+ # Build table entity
226
+ parts = self._table_identifier.split(".", 1)
227
+ namespace = parts[0]
228
+ table_name = parts[1] if len(parts) > 1 else parts[0]
229
+ table_entity = ProvEntity(
230
+ uri=make_table_uri(namespace, table_name),
231
+ entity_type="IcebergTable",
232
+ label=self._table_identifier,
233
+ )
234
+
235
+ # Build activity
236
+ activity = ProvActivity(
237
+ uri=make_activity_uri(self.activity_id),
238
+ activity_type="Import",
239
+ started_at=self.timestamp_start,
240
+ ended_at=self._timestamp_end,
241
+ used=[source_entity],
242
+ generated=[table_entity],
243
+ )
244
+
245
+ # Build agents
246
+ user_agent = ProvAgent(
247
+ uri=f"urn:user:{self.user}",
248
+ agent_type="User",
249
+ name=self.user,
250
+ )
251
+
252
+ system_agent = ProvAgent(
253
+ uri=f"urn:system:sunstone-data:{self.system_version}",
254
+ agent_type="System",
255
+ name="sunstone-data",
256
+ version=self.system_version,
257
+ )
258
+
259
+ return activity, user_agent, system_agent
260
+
261
+ @property
262
+ def source_info(self) -> SourceMetadata | None:
263
+ """Access collected source metadata."""
264
+ return self._source_info
265
+
266
+ @property
267
+ def schema_info(self) -> SchemaMetadata | None:
268
+ """Access collected schema metadata."""
269
+ return self._schema_info
270
+
271
+ def __repr__(self) -> str:
272
+ """Return string representation."""
273
+ return (
274
+ f"MetadataBuilder(activity_id={self.activity_id!r}, "
275
+ f"user={self.user!r}, system_version={self.system_version!r})"
276
+ )
@@ -0,0 +1,87 @@
1
+ """Metadata data models for import provenance tracking."""
2
+
3
+ from dataclasses import dataclass
4
+
5
+
6
+ @dataclass(frozen=True)
7
+ class SourceMetadata:
8
+ """Metadata about the source file.
9
+
10
+ Captures both what the user provided and the normalized/resolved form.
11
+ """
12
+
13
+ original_path: str
14
+ """Exactly what user provided (relative path, URL with params)."""
15
+
16
+ normalized_path: str
17
+ """Absolute path (resolved symlinks) or cleaned URL."""
18
+
19
+ file_type: str
20
+ """File type: 'csv' or 'excel'."""
21
+
22
+ file_size: int
23
+ """Size in bytes."""
24
+
25
+ encoding: str | None
26
+ """Detected encoding for CSV, None for Excel."""
27
+
28
+ checksum: str
29
+ """SHA256 checksum of the file."""
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class SchemaMetadata:
34
+ """Metadata about the file schema.
35
+
36
+ Captures both original file schema and final Iceberg schema after conversion.
37
+ """
38
+
39
+ original_columns: list[dict]
40
+ """List of {name: str, type: str} from source file."""
41
+
42
+ iceberg_columns: list[dict]
43
+ """List of {name: str, type: str} after Iceberg conversion."""
44
+
45
+ row_count: int
46
+ """Number of rows in the dataset."""
47
+
48
+ column_count: int
49
+ """Number of columns in the dataset."""
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class ImportMetadata:
54
+ """Complete metadata for an import activity.
55
+
56
+ Combines source info, schema info, and import context.
57
+ """
58
+
59
+ activity_id: str
60
+ """UUID for this import activity."""
61
+
62
+ timestamp_start: str
63
+ """ISO8601 timestamp when import started."""
64
+
65
+ timestamp_end: str | None
66
+ """ISO8601 timestamp when import completed."""
67
+
68
+ source: SourceMetadata
69
+ """Metadata about the source file."""
70
+
71
+ schema: SchemaMetadata
72
+ """Metadata about the schema."""
73
+
74
+ table_identifier: str
75
+ """Full table identifier (namespace.table)."""
76
+
77
+ original_storage_path: str
78
+ """S3 path to original file."""
79
+
80
+ user: str
81
+ """OS username who ran import."""
82
+
83
+ system_version: str
84
+ """Platform version (from importlib.metadata)."""
85
+
86
+ import_parameters: dict
87
+ """Parameters like encoding_override, sheet_selection, tags."""
@@ -0,0 +1,171 @@
1
+ """W3C PROV-DM provenance structures.
2
+
3
+ Provides dataclasses for Activity, Entity, and Agent following W3C PROV-DM,
4
+ plus helper functions for creating valid provenance URIs.
5
+ """
6
+
7
+ import uuid
8
+ from dataclasses import dataclass, field
9
+ from pathlib import Path
10
+ from urllib.parse import urlparse, urlunparse
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class ProvEntity:
15
+ """A PROV-DM Entity representing data artifacts.
16
+
17
+ Entities are things that can be generated or used by activities.
18
+ In our context: source files and Iceberg tables.
19
+ """
20
+
21
+ uri: str
22
+ """Entity URI (file://, https://, or iceberg://)."""
23
+
24
+ entity_type: str
25
+ """Entity type: 'SourceFile' or 'IcebergTable'."""
26
+
27
+ label: str | None = None
28
+ """Human-readable label for the entity."""
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class ProvActivity:
33
+ """A PROV-DM Activity representing data transformations.
34
+
35
+ Activities are things that occur over time and use/generate entities.
36
+ In our context: import operations.
37
+ """
38
+
39
+ uri: str
40
+ """Activity URI using UUID (urn:uuid:...)."""
41
+
42
+ activity_type: str
43
+ """Activity type: 'Import'."""
44
+
45
+ started_at: str
46
+ """ISO8601 timestamp when activity started."""
47
+
48
+ ended_at: str | None = None
49
+ """ISO8601 timestamp when activity ended."""
50
+
51
+ used: list[ProvEntity] = field(default_factory=list)
52
+ """Input entities (source files)."""
53
+
54
+ generated: list[ProvEntity] = field(default_factory=list)
55
+ """Output entities (Iceberg tables)."""
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class ProvAgent:
60
+ """A PROV-DM Agent representing actors.
61
+
62
+ Agents are responsible for activities. Can be users or systems.
63
+ """
64
+
65
+ uri: str
66
+ """Agent URI."""
67
+
68
+ agent_type: str
69
+ """Agent type: 'User' or 'System'."""
70
+
71
+ name: str
72
+ """Username or system name."""
73
+
74
+ version: str | None = None
75
+ """System version if applicable."""
76
+
77
+
78
+ def make_file_uri(path: str) -> str:
79
+ """Create a file:// URI from a local path.
80
+
81
+ Resolves the path to absolute and normalizes it.
82
+
83
+ Args:
84
+ path: Local file path (relative or absolute).
85
+
86
+ Returns:
87
+ file:// URI with absolute path.
88
+ """
89
+ resolved = Path(path).resolve()
90
+ return f"file://{resolved}"
91
+
92
+
93
+ def make_url_uri(url: str) -> str:
94
+ """Normalize a URL for use as a PROV entity URI.
95
+
96
+ Strips common tracking parameters and ensures https scheme.
97
+
98
+ Args:
99
+ url: Original URL.
100
+
101
+ Returns:
102
+ Normalized https:// URL.
103
+ """
104
+ parsed = urlparse(url)
105
+
106
+ # Upgrade http to https if applicable
107
+ scheme = "https" if parsed.scheme in ("http", "https") else parsed.scheme
108
+
109
+ # Strip common tracking parameters
110
+ tracking_params = {
111
+ "utm_source",
112
+ "utm_medium",
113
+ "utm_campaign",
114
+ "utm_term",
115
+ "utm_content",
116
+ "fbclid",
117
+ "gclid",
118
+ "ref",
119
+ }
120
+
121
+ # Parse and filter query string
122
+ if parsed.query:
123
+ params = []
124
+ for param in parsed.query.split("&"):
125
+ if "=" in param:
126
+ key = param.split("=")[0]
127
+ if key.lower() not in tracking_params:
128
+ params.append(param)
129
+ else:
130
+ params.append(param)
131
+ query = "&".join(params)
132
+ else:
133
+ query = ""
134
+
135
+ return urlunparse((scheme, parsed.netloc, parsed.path, parsed.params, query, ""))
136
+
137
+
138
+ def make_table_uri(namespace: str, table: str) -> str:
139
+ """Create an iceberg:// URI for a table.
140
+
141
+ Args:
142
+ namespace: Table namespace.
143
+ table: Table name.
144
+
145
+ Returns:
146
+ iceberg://{namespace}/{table} URI.
147
+ """
148
+ return f"iceberg://{namespace}/{table}"
149
+
150
+
151
+ def make_activity_uri(uuid_str: str) -> str:
152
+ """Create a URN for an activity UUID.
153
+
154
+ Args:
155
+ uuid_str: UUID string (with or without hyphens).
156
+
157
+ Returns:
158
+ urn:uuid:{uuid} URI.
159
+ """
160
+ # Normalize UUID format (ensure hyphens)
161
+ normalized = str(uuid.UUID(uuid_str))
162
+ return f"urn:uuid:{normalized}"
163
+
164
+
165
+ def generate_activity_id() -> str:
166
+ """Generate a new UUID for an activity.
167
+
168
+ Returns:
169
+ UUID string for identifying a provenance activity.
170
+ """
171
+ return str(uuid.uuid4())
@@ -0,0 +1,102 @@
1
+ """Map import provenance (PROV-DM dataclasses) to RDF triples (spec 180 §9 step 3).
2
+
3
+ Asserts the import's `prov:Activity` — its type, start and end times, the
4
+ agents responsible, the source entity it used with that source's checksum and
5
+ retrieval time, and `prov:wasDerivedFrom` from the produced table back to the
6
+ source.
7
+
8
+ Deliberately not asserted here: the `prov:Generation` node and
9
+ `prov:wasGeneratedBy` linking an Iceberg snapshot to this activity. That is
10
+ spec §9 step 4, already built by `graph.dataset._version_triples`. Both
11
+ mappings are merged into the same named graph over GSP `POST`, so the activity
12
+ node's IRI must be identical on both sides — see `activity_iri`.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Sequence
18
+
19
+ from sunstone_data.graph import terms
20
+ from sunstone_data.graph.dataset import IRI, RDF_TYPE, Graph, Literal, Triple, build_graph, fragment_iri
21
+ from sunstone_data.metadata.provenance import ProvActivity, ProvAgent
22
+
23
+ _PROV_ACTIVITY = IRI(terms.PROV + "Activity")
24
+ _PROV_ENTITY = IRI(terms.PROV + "Entity")
25
+ _PROV_AGENT = IRI(terms.PROV + "Agent")
26
+ _PROV_STARTED_AT_TIME = IRI(terms.PROV + "startedAtTime")
27
+ _PROV_ENDED_AT_TIME = IRI(terms.PROV + "endedAtTime")
28
+ _PROV_GENERATED_AT_TIME = IRI(terms.PROV + "generatedAtTime")
29
+ _PROV_WAS_ASSOCIATED_WITH = IRI(terms.PROV + "wasAssociatedWith")
30
+ _PROV_USED = IRI(terms.PROV + "used")
31
+ _PROV_WAS_DERIVED_FROM = IRI(terms.PROV + "wasDerivedFrom")
32
+ _DCT_IDENTIFIER = IRI(terms.DCT + "identifier")
33
+ _FOAF_NAME = IRI(terms.FOAF + "name")
34
+ _XSD_DATETIME = terms.XSD + "dateTime"
35
+
36
+
37
+ def activity_iri(dataset_iri: str, activity_id: str) -> str:
38
+ """Mint the import activity's node IRI: ``{dataset_iri}#import-{activity_id}``.
39
+
40
+ Must stay identical to the IRI `graph.dataset._version_triples` mints for
41
+ the same import, or the two separately-POSTed graphs describe two
42
+ disconnected nodes instead of one. `activity_id` is the bare UUID the
43
+ pipeline mints, without the `urn:uuid:` prefix `ProvActivity.uri` carries.
44
+ """
45
+ return fragment_iri(dataset_iri, f"import-{activity_id}")
46
+
47
+
48
+ def import_provenance_to_graph(
49
+ dataset_iri: str,
50
+ activity: ProvActivity,
51
+ agents: Sequence[ProvAgent],
52
+ checksum: str,
53
+ retrieved_at: str,
54
+ ) -> Graph:
55
+ """Build the import-provenance graph for one import (spec 180 §9 step 3).
56
+
57
+ All IRIs derive from the dataset IRI and the activity UUID the pipeline
58
+ already minted, so re-running an import that reuses the same activity id
59
+ produces the same graph — which is what makes a retried import idempotent.
60
+
61
+ Args:
62
+ dataset_iri: IRI of the dataset the activity node hangs under.
63
+ activity: The import activity, as `MetadataBuilder.build_provenance`
64
+ returns it. Its first `used` entity is the source; its first
65
+ `generated` entity, if any, is the produced table.
66
+ agents: Agents responsible for the import (the pipeline passes both a
67
+ user and a system agent).
68
+ checksum: Checksum of the fetched source, asserted as the source
69
+ entity's `dct:identifier`.
70
+ retrieved_at: ISO 8601 instant the source was fetched.
71
+
72
+ Returns:
73
+ The graph of asserted triples.
74
+
75
+ Raises:
76
+ ValueError: `activity` used no entity, so there is no source to describe.
77
+ """
78
+ if not activity.used:
79
+ raise ValueError("activity has no used entity: nothing to describe as the import source")
80
+
81
+ node = IRI(activity_iri(dataset_iri, activity.uri.removeprefix("urn:uuid:")))
82
+ source = IRI(activity.used[0].uri)
83
+
84
+ triples = [
85
+ Triple(node, RDF_TYPE, _PROV_ACTIVITY),
86
+ Triple(node, _PROV_STARTED_AT_TIME, Literal(activity.started_at, datatype=_XSD_DATETIME)),
87
+ Triple(node, _PROV_USED, source),
88
+ Triple(source, RDF_TYPE, _PROV_ENTITY),
89
+ Triple(source, _DCT_IDENTIFIER, Literal(checksum)),
90
+ Triple(source, _PROV_GENERATED_AT_TIME, Literal(retrieved_at, datatype=_XSD_DATETIME)),
91
+ ]
92
+ if activity.ended_at is not None:
93
+ triples.append(Triple(node, _PROV_ENDED_AT_TIME, Literal(activity.ended_at, datatype=_XSD_DATETIME)))
94
+ if activity.generated:
95
+ triples.append(Triple(IRI(activity.generated[0].uri), _PROV_WAS_DERIVED_FROM, source))
96
+ for agent in agents:
97
+ triples += [
98
+ Triple(node, _PROV_WAS_ASSOCIATED_WITH, IRI(agent.uri)),
99
+ Triple(IRI(agent.uri), RDF_TYPE, _PROV_AGENT),
100
+ Triple(IRI(agent.uri), _FOAF_NAME, Literal(agent.name)),
101
+ ]
102
+ return build_graph(triples)
@@ -0,0 +1,62 @@
1
+ """Sunstone data platform plugin for sunstone-py."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import BinaryIO, Callable
6
+
7
+ from sunstone_data.env_section import DataPlatformSection, DataPlatformSectionProvider
8
+
9
+
10
+ class SunstoneDataPlugin:
11
+ """Data platform plugin for sunstone-py.
12
+
13
+ Implements CLIProvider (mounts `data` subcommand group), URLHandler
14
+ (resolves sunstone: URLs to parquet data), and EnvSectionProvider
15
+ (owns the ``data-platform`` subtable in env config files).
16
+ """
17
+
18
+ def cli_groups(self):
19
+ """Return the data CLI group for mounting on the sunstone app."""
20
+ from sunstone_data.cli.main import data_app
21
+
22
+ return [("data", data_app)]
23
+
24
+ def can_handle(self, url: str) -> bool:
25
+ """Return True for sunstone: URLs.
26
+
27
+ Also returns True for the old ``sunstone://`` form so that ``open()``
28
+ (via ``parse_sunstone_url``) can raise a helpful migration error.
29
+ """
30
+ return url.startswith("sunstone:")
31
+
32
+ # --- EnvSectionProvider --------------------------------------------------
33
+ # Delegate to DataPlatformSectionProvider so the section schema can be
34
+ # used standalone (e.g. in tests) without instantiating the whole plugin.
35
+
36
+ def env_section_name(self) -> str:
37
+ return DataPlatformSectionProvider.SECTION_NAME
38
+
39
+ def env_section_model(self) -> Callable[..., DataPlatformSection]:
40
+ return DataPlatformSection
41
+
42
+ def open(self, url: str, mode: str = "rb") -> BinaryIO:
43
+ """Resolve a sunstone: URL to a parquet byte stream (spec 180 §7)."""
44
+ from sunstone_data.resolve import resolve_to_parquet
45
+ from sunstone_data.url import parse_sunstone_url
46
+
47
+ return resolve_to_parquet(parse_sunstone_url(url))
48
+
49
+
50
+ # Side-effect import: attaches `init` and `refresh` subcommands to sunstone-py's
51
+ # `env` CLI group. Registering at module load means it happens during
52
+ # entry-point discovery, before the top-level CLI dispatches the user's command.
53
+ #
54
+ # This MUST stay below SunstoneDataPlugin. Importing env_commands imports
55
+ # sunstone.cli, whose module init triggers sunstone's PluginRegistry discovery.
56
+ # That discovery loads the `data_platform` entry point (this module's
57
+ # SunstoneDataPlugin). If this import ran above the class definition, discovery
58
+ # would re-enter while the module is partially initialized — SunstoneDataPlugin
59
+ # unbound — and the registry would be cached permanently without the plugin
60
+ # (no sunstone: handler, no data-platform env section), silently breaking
61
+ # lineage capture. See tests/unit/test_plugin.py.
62
+ from sunstone_data import env_commands as _env_commands # noqa: E402, F401