sunstone-data-platform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunstone_data/__init__.py +36 -0
- sunstone_data/_version.py +24 -0
- sunstone_data/auth/__init__.py +43 -0
- sunstone_data/auth/client_creds.py +239 -0
- sunstone_data/auth/pkce.py +307 -0
- sunstone_data/auth/store.py +97 -0
- sunstone_data/catalog.py +175 -0
- sunstone_data/cli/__init__.py +5 -0
- sunstone_data/cli/_app.py +26 -0
- sunstone_data/cli/_errors.py +31 -0
- sunstone_data/cli/commands/__init__.py +22 -0
- sunstone_data/cli/commands/data_auth.py +120 -0
- sunstone_data/cli/commands/data_import.py +157 -0
- sunstone_data/cli/commands/data_info.py +166 -0
- sunstone_data/cli/commands/data_publish.py +69 -0
- sunstone_data/cli/commands/ls.py +49 -0
- sunstone_data/cli/commands/prov.py +58 -0
- sunstone_data/cli/commands/refs.py +142 -0
- sunstone_data/cli/commands/show.py +139 -0
- sunstone_data/cli/commands/snapshots.py +77 -0
- sunstone_data/cli/commands/sparql.py +68 -0
- sunstone_data/cli/main.py +30 -0
- sunstone_data/config.py +250 -0
- sunstone_data/discovery.py +317 -0
- sunstone_data/env_commands.py +97 -0
- sunstone_data/env_section.py +86 -0
- sunstone_data/graph/__init__.py +36 -0
- sunstone_data/graph/client.py +328 -0
- sunstone_data/graph/dataset.py +438 -0
- sunstone_data/graph/terms.py +34 -0
- sunstone_data/identifiers.py +155 -0
- sunstone_data/import_/__init__.py +28 -0
- sunstone_data/import_/fetcher.py +259 -0
- sunstone_data/import_/owner.py +130 -0
- sunstone_data/import_/parser.py +492 -0
- sunstone_data/import_/pipeline.py +275 -0
- sunstone_data/import_/storage.py +135 -0
- sunstone_data/metadata/__init__.py +41 -0
- sunstone_data/metadata/builder.py +276 -0
- sunstone_data/metadata/models.py +87 -0
- sunstone_data/metadata/provenance.py +171 -0
- sunstone_data/metadata/provenance_rdf.py +102 -0
- sunstone_data/plugin.py +62 -0
- sunstone_data/publish.py +134 -0
- sunstone_data/query.py +624 -0
- sunstone_data/resolve.py +178 -0
- sunstone_data/url.py +78 -0
- sunstone_data/utils/__init__.py +12 -0
- sunstone_data/utils/checksum.py +14 -0
- sunstone_data/utils/encoding.py +16 -0
- sunstone_data/utils/naming.py +38 -0
- sunstone_data/versioning/__init__.py +17 -0
- sunstone_data/versioning/schema_evolution.py +180 -0
- sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
- sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
- sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
- sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
- sunstone_pipeline/__init__.py +1 -0
- sunstone_pipeline/assets/__init__.py +11 -0
- sunstone_pipeline/assets/mirrors.py +123 -0
- sunstone_pipeline/assets/transforms.py +74 -0
- sunstone_pipeline/definitions.py +25 -0
- sunstone_pipeline/io_manager.py +56 -0
- sunstone_pipeline/resources/__init__.py +6 -0
- sunstone_pipeline/resources/iceberg.py +29 -0
- sunstone_pipeline/resources/refs.py +88 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""Builder for constructing ImportMetadata during pipeline execution."""
|
|
2
|
+
|
|
3
|
+
import getpass
|
|
4
|
+
import os
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
|
|
7
|
+
from sunstone_data.metadata.models import (
|
|
8
|
+
ImportMetadata,
|
|
9
|
+
SchemaMetadata,
|
|
10
|
+
SourceMetadata,
|
|
11
|
+
)
|
|
12
|
+
from sunstone_data.metadata.provenance import (
|
|
13
|
+
ProvActivity,
|
|
14
|
+
ProvAgent,
|
|
15
|
+
ProvEntity,
|
|
16
|
+
generate_activity_id,
|
|
17
|
+
make_activity_uri,
|
|
18
|
+
make_file_uri,
|
|
19
|
+
make_table_uri,
|
|
20
|
+
make_url_uri,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _get_user() -> str:
|
|
25
|
+
"""Get current username using multiple fallback methods.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Username string, or "unknown" if detection fails.
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
return os.getlogin()
|
|
32
|
+
except OSError:
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
return getpass.getuser()
|
|
37
|
+
except Exception:
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
return os.environ.get("USER", "unknown")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _get_system_version() -> str:
|
|
44
|
+
"""Get the package version for provenance records.
|
|
45
|
+
|
|
46
|
+
Looks up the installed distribution by its current package name
|
|
47
|
+
(``sunstone-data-platform``). The agent URN scheme below still uses
|
|
48
|
+
``urn:system:sunstone-data:...`` for provenance continuity with
|
|
49
|
+
historical records, but the package itself was renamed.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
Version string, or "dev" if package not installed.
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
from importlib.metadata import version
|
|
56
|
+
|
|
57
|
+
return version("sunstone-data-platform")
|
|
58
|
+
except Exception:
|
|
59
|
+
return "dev"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class MetadataBuilder:
|
|
63
|
+
"""Builds ImportMetadata progressively during import pipeline.
|
|
64
|
+
|
|
65
|
+
The builder accumulates metadata as it becomes available through
|
|
66
|
+
the fetch -> parse -> store pipeline stages, then produces a
|
|
67
|
+
complete ImportMetadata instance at the end.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(self, source: str, import_parameters: dict | None = None):
|
|
71
|
+
"""Initialize with original source and parameters.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
source: Original source path/URL as provided by user.
|
|
75
|
+
import_parameters: Dict of import params (encoding_override, tags, etc).
|
|
76
|
+
"""
|
|
77
|
+
self.activity_id = generate_activity_id()
|
|
78
|
+
self.timestamp_start = datetime.now(timezone.utc).isoformat()
|
|
79
|
+
self.user = _get_user()
|
|
80
|
+
self.system_version = _get_system_version()
|
|
81
|
+
self.original_source = source
|
|
82
|
+
self.import_parameters = import_parameters or {}
|
|
83
|
+
|
|
84
|
+
# Fields populated by set_source_info
|
|
85
|
+
self._source_info: SourceMetadata | None = None
|
|
86
|
+
|
|
87
|
+
# Fields populated by set_schema_info
|
|
88
|
+
self._schema_info: SchemaMetadata | None = None
|
|
89
|
+
|
|
90
|
+
# Fields populated by set_table_info
|
|
91
|
+
self._table_identifier: str | None = None
|
|
92
|
+
self._original_storage_path: str | None = None
|
|
93
|
+
|
|
94
|
+
# Set at finalization
|
|
95
|
+
self._timestamp_end: str | None = None
|
|
96
|
+
|
|
97
|
+
def set_source_info(
|
|
98
|
+
self,
|
|
99
|
+
normalized_path: str,
|
|
100
|
+
file_type: str,
|
|
101
|
+
file_size: int,
|
|
102
|
+
encoding: str | None,
|
|
103
|
+
checksum: str,
|
|
104
|
+
) -> None:
|
|
105
|
+
"""Set source metadata after fetch stage.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
normalized_path: Absolute path or cleaned URL.
|
|
109
|
+
file_type: File type (csv, excel).
|
|
110
|
+
file_size: Size in bytes.
|
|
111
|
+
encoding: Detected encoding for CSV, None for Excel.
|
|
112
|
+
checksum: SHA256 checksum of the file.
|
|
113
|
+
"""
|
|
114
|
+
self._source_info = SourceMetadata(
|
|
115
|
+
original_path=self.original_source,
|
|
116
|
+
normalized_path=normalized_path,
|
|
117
|
+
file_type=file_type,
|
|
118
|
+
file_size=file_size,
|
|
119
|
+
encoding=encoding,
|
|
120
|
+
checksum=checksum,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
def set_schema_info(
|
|
124
|
+
self,
|
|
125
|
+
original_columns: list[dict],
|
|
126
|
+
iceberg_columns: list[dict],
|
|
127
|
+
row_count: int,
|
|
128
|
+
column_count: int,
|
|
129
|
+
) -> None:
|
|
130
|
+
"""Set schema metadata after parse stage.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
original_columns: List of {name, type} from source file.
|
|
134
|
+
iceberg_columns: List of {name, type} after Iceberg conversion.
|
|
135
|
+
row_count: Number of rows in the dataset.
|
|
136
|
+
column_count: Number of columns in the dataset.
|
|
137
|
+
"""
|
|
138
|
+
self._schema_info = SchemaMetadata(
|
|
139
|
+
original_columns=original_columns,
|
|
140
|
+
iceberg_columns=iceberg_columns,
|
|
141
|
+
row_count=row_count,
|
|
142
|
+
column_count=column_count,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
def set_table_info(
|
|
146
|
+
self,
|
|
147
|
+
table_identifier: str,
|
|
148
|
+
original_storage_path: str,
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Set table info after storage stage.
|
|
151
|
+
|
|
152
|
+
Args:
|
|
153
|
+
table_identifier: Full table identifier (namespace.table).
|
|
154
|
+
original_storage_path: S3 path to original file.
|
|
155
|
+
"""
|
|
156
|
+
self._table_identifier = table_identifier
|
|
157
|
+
self._original_storage_path = original_storage_path
|
|
158
|
+
|
|
159
|
+
def finalize(self) -> ImportMetadata:
|
|
160
|
+
"""Finalize and return completed ImportMetadata.
|
|
161
|
+
|
|
162
|
+
Sets timestamp_end and validates all required fields are present.
|
|
163
|
+
|
|
164
|
+
Returns:
|
|
165
|
+
Complete ImportMetadata instance.
|
|
166
|
+
|
|
167
|
+
Raises:
|
|
168
|
+
ValueError: If required fields are missing.
|
|
169
|
+
"""
|
|
170
|
+
# Validate required fields
|
|
171
|
+
if self._source_info is None:
|
|
172
|
+
raise ValueError("Source info not set. Call set_source_info() first.")
|
|
173
|
+
if self._schema_info is None:
|
|
174
|
+
raise ValueError("Schema info not set. Call set_schema_info() first.")
|
|
175
|
+
if self._table_identifier is None:
|
|
176
|
+
raise ValueError("Table info not set. Call set_table_info() first.")
|
|
177
|
+
if self._original_storage_path is None:
|
|
178
|
+
raise ValueError("Original storage path not set. Call set_table_info() first.")
|
|
179
|
+
|
|
180
|
+
self._timestamp_end = datetime.now(timezone.utc).isoformat()
|
|
181
|
+
|
|
182
|
+
return ImportMetadata(
|
|
183
|
+
activity_id=self.activity_id,
|
|
184
|
+
timestamp_start=self.timestamp_start,
|
|
185
|
+
timestamp_end=self._timestamp_end,
|
|
186
|
+
source=self._source_info,
|
|
187
|
+
schema=self._schema_info,
|
|
188
|
+
table_identifier=self._table_identifier,
|
|
189
|
+
original_storage_path=self._original_storage_path,
|
|
190
|
+
user=self.user,
|
|
191
|
+
system_version=self.system_version,
|
|
192
|
+
import_parameters=self.import_parameters,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def build_provenance(self) -> tuple[ProvActivity, ProvAgent, ProvAgent]:
|
|
196
|
+
"""Build PROV-DM graph from collected metadata.
|
|
197
|
+
|
|
198
|
+
Requires source_info, schema_info, and table_info to be set.
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
Tuple of (activity, user_agent, system_agent).
|
|
202
|
+
|
|
203
|
+
Raises:
|
|
204
|
+
ValueError: If required fields are missing.
|
|
205
|
+
"""
|
|
206
|
+
if self._source_info is None:
|
|
207
|
+
raise ValueError("Source info not set. Call set_source_info() first.")
|
|
208
|
+
if self._table_identifier is None:
|
|
209
|
+
raise ValueError("Table info not set. Call set_table_info() first.")
|
|
210
|
+
|
|
211
|
+
# Determine source entity URI based on whether it's a URL or file
|
|
212
|
+
source_path = self._source_info.original_path
|
|
213
|
+
if source_path.startswith(("http://", "https://")):
|
|
214
|
+
source_uri = make_url_uri(source_path)
|
|
215
|
+
else:
|
|
216
|
+
source_uri = make_file_uri(self._source_info.normalized_path)
|
|
217
|
+
|
|
218
|
+
# Build source entity
|
|
219
|
+
source_entity = ProvEntity(
|
|
220
|
+
uri=source_uri,
|
|
221
|
+
entity_type="SourceFile",
|
|
222
|
+
label=self._source_info.original_path,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
# Build table entity
|
|
226
|
+
parts = self._table_identifier.split(".", 1)
|
|
227
|
+
namespace = parts[0]
|
|
228
|
+
table_name = parts[1] if len(parts) > 1 else parts[0]
|
|
229
|
+
table_entity = ProvEntity(
|
|
230
|
+
uri=make_table_uri(namespace, table_name),
|
|
231
|
+
entity_type="IcebergTable",
|
|
232
|
+
label=self._table_identifier,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
# Build activity
|
|
236
|
+
activity = ProvActivity(
|
|
237
|
+
uri=make_activity_uri(self.activity_id),
|
|
238
|
+
activity_type="Import",
|
|
239
|
+
started_at=self.timestamp_start,
|
|
240
|
+
ended_at=self._timestamp_end,
|
|
241
|
+
used=[source_entity],
|
|
242
|
+
generated=[table_entity],
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
# Build agents
|
|
246
|
+
user_agent = ProvAgent(
|
|
247
|
+
uri=f"urn:user:{self.user}",
|
|
248
|
+
agent_type="User",
|
|
249
|
+
name=self.user,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
system_agent = ProvAgent(
|
|
253
|
+
uri=f"urn:system:sunstone-data:{self.system_version}",
|
|
254
|
+
agent_type="System",
|
|
255
|
+
name="sunstone-data",
|
|
256
|
+
version=self.system_version,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
return activity, user_agent, system_agent
|
|
260
|
+
|
|
261
|
+
@property
|
|
262
|
+
def source_info(self) -> SourceMetadata | None:
|
|
263
|
+
"""Access collected source metadata."""
|
|
264
|
+
return self._source_info
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def schema_info(self) -> SchemaMetadata | None:
|
|
268
|
+
"""Access collected schema metadata."""
|
|
269
|
+
return self._schema_info
|
|
270
|
+
|
|
271
|
+
def __repr__(self) -> str:
|
|
272
|
+
"""Return string representation."""
|
|
273
|
+
return (
|
|
274
|
+
f"MetadataBuilder(activity_id={self.activity_id!r}, "
|
|
275
|
+
f"user={self.user!r}, system_version={self.system_version!r})"
|
|
276
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Metadata data models for import provenance tracking."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True)
|
|
7
|
+
class SourceMetadata:
|
|
8
|
+
"""Metadata about the source file.
|
|
9
|
+
|
|
10
|
+
Captures both what the user provided and the normalized/resolved form.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
original_path: str
|
|
14
|
+
"""Exactly what user provided (relative path, URL with params)."""
|
|
15
|
+
|
|
16
|
+
normalized_path: str
|
|
17
|
+
"""Absolute path (resolved symlinks) or cleaned URL."""
|
|
18
|
+
|
|
19
|
+
file_type: str
|
|
20
|
+
"""File type: 'csv' or 'excel'."""
|
|
21
|
+
|
|
22
|
+
file_size: int
|
|
23
|
+
"""Size in bytes."""
|
|
24
|
+
|
|
25
|
+
encoding: str | None
|
|
26
|
+
"""Detected encoding for CSV, None for Excel."""
|
|
27
|
+
|
|
28
|
+
checksum: str
|
|
29
|
+
"""SHA256 checksum of the file."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class SchemaMetadata:
|
|
34
|
+
"""Metadata about the file schema.
|
|
35
|
+
|
|
36
|
+
Captures both original file schema and final Iceberg schema after conversion.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
original_columns: list[dict]
|
|
40
|
+
"""List of {name: str, type: str} from source file."""
|
|
41
|
+
|
|
42
|
+
iceberg_columns: list[dict]
|
|
43
|
+
"""List of {name: str, type: str} after Iceberg conversion."""
|
|
44
|
+
|
|
45
|
+
row_count: int
|
|
46
|
+
"""Number of rows in the dataset."""
|
|
47
|
+
|
|
48
|
+
column_count: int
|
|
49
|
+
"""Number of columns in the dataset."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class ImportMetadata:
|
|
54
|
+
"""Complete metadata for an import activity.
|
|
55
|
+
|
|
56
|
+
Combines source info, schema info, and import context.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
activity_id: str
|
|
60
|
+
"""UUID for this import activity."""
|
|
61
|
+
|
|
62
|
+
timestamp_start: str
|
|
63
|
+
"""ISO8601 timestamp when import started."""
|
|
64
|
+
|
|
65
|
+
timestamp_end: str | None
|
|
66
|
+
"""ISO8601 timestamp when import completed."""
|
|
67
|
+
|
|
68
|
+
source: SourceMetadata
|
|
69
|
+
"""Metadata about the source file."""
|
|
70
|
+
|
|
71
|
+
schema: SchemaMetadata
|
|
72
|
+
"""Metadata about the schema."""
|
|
73
|
+
|
|
74
|
+
table_identifier: str
|
|
75
|
+
"""Full table identifier (namespace.table)."""
|
|
76
|
+
|
|
77
|
+
original_storage_path: str
|
|
78
|
+
"""S3 path to original file."""
|
|
79
|
+
|
|
80
|
+
user: str
|
|
81
|
+
"""OS username who ran import."""
|
|
82
|
+
|
|
83
|
+
system_version: str
|
|
84
|
+
"""Platform version (from importlib.metadata)."""
|
|
85
|
+
|
|
86
|
+
import_parameters: dict
|
|
87
|
+
"""Parameters like encoding_override, sheet_selection, tags."""
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""W3C PROV-DM provenance structures.
|
|
2
|
+
|
|
3
|
+
Provides dataclasses for Activity, Entity, and Agent following W3C PROV-DM,
|
|
4
|
+
plus helper functions for creating valid provenance URIs.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import uuid
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from urllib.parse import urlparse, urlunparse
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ProvEntity:
|
|
15
|
+
"""A PROV-DM Entity representing data artifacts.
|
|
16
|
+
|
|
17
|
+
Entities are things that can be generated or used by activities.
|
|
18
|
+
In our context: source files and Iceberg tables.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
uri: str
|
|
22
|
+
"""Entity URI (file://, https://, or iceberg://)."""
|
|
23
|
+
|
|
24
|
+
entity_type: str
|
|
25
|
+
"""Entity type: 'SourceFile' or 'IcebergTable'."""
|
|
26
|
+
|
|
27
|
+
label: str | None = None
|
|
28
|
+
"""Human-readable label for the entity."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class ProvActivity:
|
|
33
|
+
"""A PROV-DM Activity representing data transformations.
|
|
34
|
+
|
|
35
|
+
Activities are things that occur over time and use/generate entities.
|
|
36
|
+
In our context: import operations.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
uri: str
|
|
40
|
+
"""Activity URI using UUID (urn:uuid:...)."""
|
|
41
|
+
|
|
42
|
+
activity_type: str
|
|
43
|
+
"""Activity type: 'Import'."""
|
|
44
|
+
|
|
45
|
+
started_at: str
|
|
46
|
+
"""ISO8601 timestamp when activity started."""
|
|
47
|
+
|
|
48
|
+
ended_at: str | None = None
|
|
49
|
+
"""ISO8601 timestamp when activity ended."""
|
|
50
|
+
|
|
51
|
+
used: list[ProvEntity] = field(default_factory=list)
|
|
52
|
+
"""Input entities (source files)."""
|
|
53
|
+
|
|
54
|
+
generated: list[ProvEntity] = field(default_factory=list)
|
|
55
|
+
"""Output entities (Iceberg tables)."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class ProvAgent:
|
|
60
|
+
"""A PROV-DM Agent representing actors.
|
|
61
|
+
|
|
62
|
+
Agents are responsible for activities. Can be users or systems.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
uri: str
|
|
66
|
+
"""Agent URI."""
|
|
67
|
+
|
|
68
|
+
agent_type: str
|
|
69
|
+
"""Agent type: 'User' or 'System'."""
|
|
70
|
+
|
|
71
|
+
name: str
|
|
72
|
+
"""Username or system name."""
|
|
73
|
+
|
|
74
|
+
version: str | None = None
|
|
75
|
+
"""System version if applicable."""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def make_file_uri(path: str) -> str:
|
|
79
|
+
"""Create a file:// URI from a local path.
|
|
80
|
+
|
|
81
|
+
Resolves the path to absolute and normalizes it.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
path: Local file path (relative or absolute).
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
file:// URI with absolute path.
|
|
88
|
+
"""
|
|
89
|
+
resolved = Path(path).resolve()
|
|
90
|
+
return f"file://{resolved}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def make_url_uri(url: str) -> str:
|
|
94
|
+
"""Normalize a URL for use as a PROV entity URI.
|
|
95
|
+
|
|
96
|
+
Strips common tracking parameters and ensures https scheme.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
url: Original URL.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
Normalized https:// URL.
|
|
103
|
+
"""
|
|
104
|
+
parsed = urlparse(url)
|
|
105
|
+
|
|
106
|
+
# Upgrade http to https if applicable
|
|
107
|
+
scheme = "https" if parsed.scheme in ("http", "https") else parsed.scheme
|
|
108
|
+
|
|
109
|
+
# Strip common tracking parameters
|
|
110
|
+
tracking_params = {
|
|
111
|
+
"utm_source",
|
|
112
|
+
"utm_medium",
|
|
113
|
+
"utm_campaign",
|
|
114
|
+
"utm_term",
|
|
115
|
+
"utm_content",
|
|
116
|
+
"fbclid",
|
|
117
|
+
"gclid",
|
|
118
|
+
"ref",
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
# Parse and filter query string
|
|
122
|
+
if parsed.query:
|
|
123
|
+
params = []
|
|
124
|
+
for param in parsed.query.split("&"):
|
|
125
|
+
if "=" in param:
|
|
126
|
+
key = param.split("=")[0]
|
|
127
|
+
if key.lower() not in tracking_params:
|
|
128
|
+
params.append(param)
|
|
129
|
+
else:
|
|
130
|
+
params.append(param)
|
|
131
|
+
query = "&".join(params)
|
|
132
|
+
else:
|
|
133
|
+
query = ""
|
|
134
|
+
|
|
135
|
+
return urlunparse((scheme, parsed.netloc, parsed.path, parsed.params, query, ""))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def make_table_uri(namespace: str, table: str) -> str:
|
|
139
|
+
"""Create an iceberg:// URI for a table.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
namespace: Table namespace.
|
|
143
|
+
table: Table name.
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
iceberg://{namespace}/{table} URI.
|
|
147
|
+
"""
|
|
148
|
+
return f"iceberg://{namespace}/{table}"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def make_activity_uri(uuid_str: str) -> str:
|
|
152
|
+
"""Create a URN for an activity UUID.
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
uuid_str: UUID string (with or without hyphens).
|
|
156
|
+
|
|
157
|
+
Returns:
|
|
158
|
+
urn:uuid:{uuid} URI.
|
|
159
|
+
"""
|
|
160
|
+
# Normalize UUID format (ensure hyphens)
|
|
161
|
+
normalized = str(uuid.UUID(uuid_str))
|
|
162
|
+
return f"urn:uuid:{normalized}"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def generate_activity_id() -> str:
|
|
166
|
+
"""Generate a new UUID for an activity.
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
UUID string for identifying a provenance activity.
|
|
170
|
+
"""
|
|
171
|
+
return str(uuid.uuid4())
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Map import provenance (PROV-DM dataclasses) to RDF triples (spec 180 §9 step 3).
|
|
2
|
+
|
|
3
|
+
Asserts the import's `prov:Activity` — its type, start and end times, the
|
|
4
|
+
agents responsible, the source entity it used with that source's checksum and
|
|
5
|
+
retrieval time, and `prov:wasDerivedFrom` from the produced table back to the
|
|
6
|
+
source.
|
|
7
|
+
|
|
8
|
+
Deliberately not asserted here: the `prov:Generation` node and
|
|
9
|
+
`prov:wasGeneratedBy` linking an Iceberg snapshot to this activity. That is
|
|
10
|
+
spec §9 step 4, already built by `graph.dataset._version_triples`. Both
|
|
11
|
+
mappings are merged into the same named graph over GSP `POST`, so the activity
|
|
12
|
+
node's IRI must be identical on both sides — see `activity_iri`.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Sequence
|
|
18
|
+
|
|
19
|
+
from sunstone_data.graph import terms
|
|
20
|
+
from sunstone_data.graph.dataset import IRI, RDF_TYPE, Graph, Literal, Triple, build_graph, fragment_iri
|
|
21
|
+
from sunstone_data.metadata.provenance import ProvActivity, ProvAgent
|
|
22
|
+
|
|
23
|
+
_PROV_ACTIVITY = IRI(terms.PROV + "Activity")
|
|
24
|
+
_PROV_ENTITY = IRI(terms.PROV + "Entity")
|
|
25
|
+
_PROV_AGENT = IRI(terms.PROV + "Agent")
|
|
26
|
+
_PROV_STARTED_AT_TIME = IRI(terms.PROV + "startedAtTime")
|
|
27
|
+
_PROV_ENDED_AT_TIME = IRI(terms.PROV + "endedAtTime")
|
|
28
|
+
_PROV_GENERATED_AT_TIME = IRI(terms.PROV + "generatedAtTime")
|
|
29
|
+
_PROV_WAS_ASSOCIATED_WITH = IRI(terms.PROV + "wasAssociatedWith")
|
|
30
|
+
_PROV_USED = IRI(terms.PROV + "used")
|
|
31
|
+
_PROV_WAS_DERIVED_FROM = IRI(terms.PROV + "wasDerivedFrom")
|
|
32
|
+
_DCT_IDENTIFIER = IRI(terms.DCT + "identifier")
|
|
33
|
+
_FOAF_NAME = IRI(terms.FOAF + "name")
|
|
34
|
+
_XSD_DATETIME = terms.XSD + "dateTime"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def activity_iri(dataset_iri: str, activity_id: str) -> str:
|
|
38
|
+
"""Mint the import activity's node IRI: ``{dataset_iri}#import-{activity_id}``.
|
|
39
|
+
|
|
40
|
+
Must stay identical to the IRI `graph.dataset._version_triples` mints for
|
|
41
|
+
the same import, or the two separately-POSTed graphs describe two
|
|
42
|
+
disconnected nodes instead of one. `activity_id` is the bare UUID the
|
|
43
|
+
pipeline mints, without the `urn:uuid:` prefix `ProvActivity.uri` carries.
|
|
44
|
+
"""
|
|
45
|
+
return fragment_iri(dataset_iri, f"import-{activity_id}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def import_provenance_to_graph(
|
|
49
|
+
dataset_iri: str,
|
|
50
|
+
activity: ProvActivity,
|
|
51
|
+
agents: Sequence[ProvAgent],
|
|
52
|
+
checksum: str,
|
|
53
|
+
retrieved_at: str,
|
|
54
|
+
) -> Graph:
|
|
55
|
+
"""Build the import-provenance graph for one import (spec 180 §9 step 3).
|
|
56
|
+
|
|
57
|
+
All IRIs derive from the dataset IRI and the activity UUID the pipeline
|
|
58
|
+
already minted, so re-running an import that reuses the same activity id
|
|
59
|
+
produces the same graph — which is what makes a retried import idempotent.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
dataset_iri: IRI of the dataset the activity node hangs under.
|
|
63
|
+
activity: The import activity, as `MetadataBuilder.build_provenance`
|
|
64
|
+
returns it. Its first `used` entity is the source; its first
|
|
65
|
+
`generated` entity, if any, is the produced table.
|
|
66
|
+
agents: Agents responsible for the import (the pipeline passes both a
|
|
67
|
+
user and a system agent).
|
|
68
|
+
checksum: Checksum of the fetched source, asserted as the source
|
|
69
|
+
entity's `dct:identifier`.
|
|
70
|
+
retrieved_at: ISO 8601 instant the source was fetched.
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
The graph of asserted triples.
|
|
74
|
+
|
|
75
|
+
Raises:
|
|
76
|
+
ValueError: `activity` used no entity, so there is no source to describe.
|
|
77
|
+
"""
|
|
78
|
+
if not activity.used:
|
|
79
|
+
raise ValueError("activity has no used entity: nothing to describe as the import source")
|
|
80
|
+
|
|
81
|
+
node = IRI(activity_iri(dataset_iri, activity.uri.removeprefix("urn:uuid:")))
|
|
82
|
+
source = IRI(activity.used[0].uri)
|
|
83
|
+
|
|
84
|
+
triples = [
|
|
85
|
+
Triple(node, RDF_TYPE, _PROV_ACTIVITY),
|
|
86
|
+
Triple(node, _PROV_STARTED_AT_TIME, Literal(activity.started_at, datatype=_XSD_DATETIME)),
|
|
87
|
+
Triple(node, _PROV_USED, source),
|
|
88
|
+
Triple(source, RDF_TYPE, _PROV_ENTITY),
|
|
89
|
+
Triple(source, _DCT_IDENTIFIER, Literal(checksum)),
|
|
90
|
+
Triple(source, _PROV_GENERATED_AT_TIME, Literal(retrieved_at, datatype=_XSD_DATETIME)),
|
|
91
|
+
]
|
|
92
|
+
if activity.ended_at is not None:
|
|
93
|
+
triples.append(Triple(node, _PROV_ENDED_AT_TIME, Literal(activity.ended_at, datatype=_XSD_DATETIME)))
|
|
94
|
+
if activity.generated:
|
|
95
|
+
triples.append(Triple(IRI(activity.generated[0].uri), _PROV_WAS_DERIVED_FROM, source))
|
|
96
|
+
for agent in agents:
|
|
97
|
+
triples += [
|
|
98
|
+
Triple(node, _PROV_WAS_ASSOCIATED_WITH, IRI(agent.uri)),
|
|
99
|
+
Triple(IRI(agent.uri), RDF_TYPE, _PROV_AGENT),
|
|
100
|
+
Triple(IRI(agent.uri), _FOAF_NAME, Literal(agent.name)),
|
|
101
|
+
]
|
|
102
|
+
return build_graph(triples)
|
sunstone_data/plugin.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Sunstone data platform plugin for sunstone-py."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import BinaryIO, Callable
|
|
6
|
+
|
|
7
|
+
from sunstone_data.env_section import DataPlatformSection, DataPlatformSectionProvider
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SunstoneDataPlugin:
|
|
11
|
+
"""Data platform plugin for sunstone-py.
|
|
12
|
+
|
|
13
|
+
Implements CLIProvider (mounts `data` subcommand group), URLHandler
|
|
14
|
+
(resolves sunstone: URLs to parquet data), and EnvSectionProvider
|
|
15
|
+
(owns the ``data-platform`` subtable in env config files).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def cli_groups(self):
|
|
19
|
+
"""Return the data CLI group for mounting on the sunstone app."""
|
|
20
|
+
from sunstone_data.cli.main import data_app
|
|
21
|
+
|
|
22
|
+
return [("data", data_app)]
|
|
23
|
+
|
|
24
|
+
def can_handle(self, url: str) -> bool:
|
|
25
|
+
"""Return True for sunstone: URLs.
|
|
26
|
+
|
|
27
|
+
Also returns True for the old ``sunstone://`` form so that ``open()``
|
|
28
|
+
(via ``parse_sunstone_url``) can raise a helpful migration error.
|
|
29
|
+
"""
|
|
30
|
+
return url.startswith("sunstone:")
|
|
31
|
+
|
|
32
|
+
# --- EnvSectionProvider --------------------------------------------------
|
|
33
|
+
# Delegate to DataPlatformSectionProvider so the section schema can be
|
|
34
|
+
# used standalone (e.g. in tests) without instantiating the whole plugin.
|
|
35
|
+
|
|
36
|
+
def env_section_name(self) -> str:
|
|
37
|
+
return DataPlatformSectionProvider.SECTION_NAME
|
|
38
|
+
|
|
39
|
+
def env_section_model(self) -> Callable[..., DataPlatformSection]:
|
|
40
|
+
return DataPlatformSection
|
|
41
|
+
|
|
42
|
+
def open(self, url: str, mode: str = "rb") -> BinaryIO:
|
|
43
|
+
"""Resolve a sunstone: URL to a parquet byte stream (spec 180 §7)."""
|
|
44
|
+
from sunstone_data.resolve import resolve_to_parquet
|
|
45
|
+
from sunstone_data.url import parse_sunstone_url
|
|
46
|
+
|
|
47
|
+
return resolve_to_parquet(parse_sunstone_url(url))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# Side-effect import: attaches `init` and `refresh` subcommands to sunstone-py's
|
|
51
|
+
# `env` CLI group. Registering at module load means it happens during
|
|
52
|
+
# entry-point discovery, before the top-level CLI dispatches the user's command.
|
|
53
|
+
#
|
|
54
|
+
# This MUST stay below SunstoneDataPlugin. Importing env_commands imports
|
|
55
|
+
# sunstone.cli, whose module init triggers sunstone's PluginRegistry discovery.
|
|
56
|
+
# That discovery loads the `data_platform` entry point (this module's
|
|
57
|
+
# SunstoneDataPlugin). If this import ran above the class definition, discovery
|
|
58
|
+
# would re-enter while the module is partially initialized — SunstoneDataPlugin
|
|
59
|
+
# unbound — and the registry would be cached permanently without the plugin
|
|
60
|
+
# (no sunstone: handler, no data-platform env section), silently breaking
|
|
61
|
+
# lineage capture. See tests/unit/test_plugin.py.
|
|
62
|
+
from sunstone_data import env_commands as _env_commands # noqa: E402, F401
|