sunstone-data-platform 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. sunstone_data/__init__.py +36 -0
  2. sunstone_data/_version.py +24 -0
  3. sunstone_data/auth/__init__.py +43 -0
  4. sunstone_data/auth/client_creds.py +239 -0
  5. sunstone_data/auth/pkce.py +307 -0
  6. sunstone_data/auth/store.py +97 -0
  7. sunstone_data/catalog.py +175 -0
  8. sunstone_data/cli/__init__.py +5 -0
  9. sunstone_data/cli/_app.py +26 -0
  10. sunstone_data/cli/_errors.py +31 -0
  11. sunstone_data/cli/commands/__init__.py +22 -0
  12. sunstone_data/cli/commands/data_auth.py +120 -0
  13. sunstone_data/cli/commands/data_import.py +157 -0
  14. sunstone_data/cli/commands/data_info.py +166 -0
  15. sunstone_data/cli/commands/data_publish.py +69 -0
  16. sunstone_data/cli/commands/ls.py +49 -0
  17. sunstone_data/cli/commands/prov.py +58 -0
  18. sunstone_data/cli/commands/refs.py +142 -0
  19. sunstone_data/cli/commands/show.py +139 -0
  20. sunstone_data/cli/commands/snapshots.py +77 -0
  21. sunstone_data/cli/commands/sparql.py +68 -0
  22. sunstone_data/cli/main.py +30 -0
  23. sunstone_data/config.py +250 -0
  24. sunstone_data/discovery.py +317 -0
  25. sunstone_data/env_commands.py +97 -0
  26. sunstone_data/env_section.py +86 -0
  27. sunstone_data/graph/__init__.py +36 -0
  28. sunstone_data/graph/client.py +328 -0
  29. sunstone_data/graph/dataset.py +438 -0
  30. sunstone_data/graph/terms.py +34 -0
  31. sunstone_data/identifiers.py +155 -0
  32. sunstone_data/import_/__init__.py +28 -0
  33. sunstone_data/import_/fetcher.py +259 -0
  34. sunstone_data/import_/owner.py +130 -0
  35. sunstone_data/import_/parser.py +492 -0
  36. sunstone_data/import_/pipeline.py +275 -0
  37. sunstone_data/import_/storage.py +135 -0
  38. sunstone_data/metadata/__init__.py +41 -0
  39. sunstone_data/metadata/builder.py +276 -0
  40. sunstone_data/metadata/models.py +87 -0
  41. sunstone_data/metadata/provenance.py +171 -0
  42. sunstone_data/metadata/provenance_rdf.py +102 -0
  43. sunstone_data/plugin.py +62 -0
  44. sunstone_data/publish.py +134 -0
  45. sunstone_data/query.py +624 -0
  46. sunstone_data/resolve.py +178 -0
  47. sunstone_data/url.py +78 -0
  48. sunstone_data/utils/__init__.py +12 -0
  49. sunstone_data/utils/checksum.py +14 -0
  50. sunstone_data/utils/encoding.py +16 -0
  51. sunstone_data/utils/naming.py +38 -0
  52. sunstone_data/versioning/__init__.py +17 -0
  53. sunstone_data/versioning/schema_evolution.py +180 -0
  54. sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
  55. sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
  56. sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
  57. sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
  58. sunstone_pipeline/__init__.py +1 -0
  59. sunstone_pipeline/assets/__init__.py +11 -0
  60. sunstone_pipeline/assets/mirrors.py +123 -0
  61. sunstone_pipeline/assets/transforms.py +74 -0
  62. sunstone_pipeline/definitions.py +25 -0
  63. sunstone_pipeline/io_manager.py +56 -0
  64. sunstone_pipeline/resources/__init__.py +6 -0
  65. sunstone_pipeline/resources/iceberg.py +29 -0
  66. sunstone_pipeline/resources/refs.py +88 -0
@@ -0,0 +1,275 @@
1
+ """Import pipeline orchestration."""
2
+
3
+ from dataclasses import dataclass, field
4
+ from pathlib import Path
5
+ from typing import TYPE_CHECKING, Callable
6
+
7
+ from sunstone_data.config import Settings, get_settings
8
+ from sunstone_data.import_.fetcher import FileType, SourceInfo, fetch_source
9
+ from sunstone_data.import_.owner import (
10
+ ensure_root_prefix,
11
+ load_rules,
12
+ org_uri,
13
+ owner_root,
14
+ resolve_owner,
15
+ )
16
+ from sunstone_data.import_.storage import store_original
17
+ from sunstone_data.metadata import MetadataBuilder
18
+ from sunstone_data.utils import (
19
+ compute_checksum,
20
+ derive_namespace,
21
+ detect_encoding,
22
+ sanitize_table_name,
23
+ )
24
+
25
+ if TYPE_CHECKING:
26
+ from sunstone_data.metadata import ImportMetadata
27
+
28
+
29
+ @dataclass
30
+ class ImportResult:
31
+ """Result of import operation."""
32
+
33
+ table_name: str
34
+ namespace: str
35
+ row_count: int
36
+ column_count: int
37
+ checksum: str
38
+ original_path: str
39
+ schema_summary: str
40
+ already_existed: bool = False
41
+ #: Iceberg snapshot id the table points at after the import.
42
+ snapshot_id: int | None = None
43
+ metadata: "ImportMetadata | None" = field(default=None)
44
+
45
+
46
+ def run_import(
47
+ source: str,
48
+ name_override: str | None = None,
49
+ namespace_override: str | None = None,
50
+ owner: str | None = None,
51
+ tags: list[str] | None = None,
52
+ branch: str = "main",
53
+ force_schema: bool = False,
54
+ progress_callback: Callable[[int, int | None], None] | None = None,
55
+ settings: Settings | None = None,
56
+ ) -> ImportResult | list[ImportResult]:
57
+ """Run complete import pipeline.
58
+
59
+ Args:
60
+ source: Local file path or URL to import.
61
+ name_override: Optional table name override.
62
+ namespace_override: Optional namespace override.
63
+ owner: Owning organization path (e.g. ``un/who``). Overrides the
64
+ source-URL rules file. Required unless a rule matches the source.
65
+ tags: Optional list of tags (key:value format).
66
+ branch: Branch to import to (default: "main").
67
+ force_schema: Allow unsafe schema changes (default: False).
68
+ progress_callback: Optional progress callback(downloaded, total).
69
+ settings: Optional settings override.
70
+
71
+ Returns:
72
+ Single ImportResult for CSV, list for multi-sheet Excel.
73
+
74
+ Raises:
75
+ FileNotFoundError: If local file not found.
76
+ ValueError: If file type not supported or parsing fails.
77
+ OwnerError: If the owner cannot be resolved (no --owner and no matching
78
+ rule) or is invalid. OwnerUnresolved/OwnerInvalid both subclass
79
+ ValueError.
80
+ httpx.HTTPError: If URL download fails.
81
+ """
82
+ settings = settings or get_settings()
83
+
84
+ # Create MetadataBuilder at start
85
+ builder = MetadataBuilder(
86
+ source=source,
87
+ import_parameters={
88
+ "name_override": name_override,
89
+ "namespace_override": namespace_override,
90
+ "tags": tags,
91
+ },
92
+ )
93
+
94
+ # Step 1: Fetch source (download if URL, validate if local)
95
+ source_info = fetch_source(source, progress_callback)
96
+
97
+ try:
98
+ # Step 2: Detect encoding for CSV files
99
+ if source_info.file_type == FileType.CSV:
100
+ encoding = detect_encoding(source_info.local_path)
101
+ source_info = SourceInfo(
102
+ local_path=source_info.local_path,
103
+ filename=source_info.filename,
104
+ file_type=source_info.file_type,
105
+ encoding=encoding,
106
+ is_temp=source_info.is_temp,
107
+ source_url=source_info.source_url,
108
+ )
109
+
110
+ # Step 3: Compute checksum
111
+ checksum = compute_checksum(source_info.local_path)
112
+
113
+ # Step 4: Determine normalized path for metadata
114
+ if source_info.source_url:
115
+ # URL: use cleaned URL
116
+ normalized_path = source_info.source_url
117
+ else:
118
+ # Local file: use resolved absolute path
119
+ normalized_path = str(source_info.local_path.resolve())
120
+
121
+ # Step 5: Capture source metadata
122
+ builder.set_source_info(
123
+ normalized_path=normalized_path,
124
+ file_type=source_info.file_type.value,
125
+ file_size=source_info.local_path.stat().st_size,
126
+ encoding=source_info.encoding,
127
+ checksum=checksum,
128
+ )
129
+
130
+ # Step 6: Determine table name and namespace
131
+ base_name = name_override or sanitize_table_name(source_info.filename)
132
+ namespace = namespace_override or derive_namespace(source)
133
+
134
+ # Step 6b: Resolve the data-owner organization (strict: errors if
135
+ # neither --owner nor a matching source-URL rule resolves one). The
136
+ # owner root is authoritative for namespace level 0.
137
+ rules = load_rules(Path(settings.owner_rules_path) if settings.owner_rules_path else None)
138
+ owner_path = resolve_owner(
139
+ source_info.source_url or "",
140
+ owner,
141
+ rules=rules,
142
+ org_roots=settings.org_roots,
143
+ )
144
+ namespace = ensure_root_prefix(namespace, owner_root(owner_path))
145
+ owner_org_uri = org_uri(owner_path, settings.org_base_uri)
146
+
147
+ # Step 7: Parse file and create tables
148
+ # NOTE: Deferred import - parser module created in Plan 02-03 (wave 3)
149
+ # This import happens at runtime, not module load time, so it's safe
150
+ from sunstone_data.import_.parser import parse_and_create_tables
151
+
152
+ results = parse_and_create_tables(
153
+ source_info=source_info,
154
+ base_name=base_name,
155
+ namespace=namespace,
156
+ checksum=checksum,
157
+ source=source,
158
+ tags=tags,
159
+ branch=branch,
160
+ force_schema=force_schema,
161
+ settings=settings,
162
+ builder=builder,
163
+ owner_org=owner_org_uri,
164
+ )
165
+
166
+ # Step 8: Store original file (after successful parse)
167
+ original_path = store_original(
168
+ file_path=source_info.local_path,
169
+ checksum=checksum,
170
+ namespace=namespace,
171
+ table_name=base_name,
172
+ settings=settings,
173
+ )
174
+
175
+ # Step 9: Finalize metadata and update results
176
+ # Use the first result's table for metadata (CSV has one, Excel may have multiple)
177
+ first_result = results[0]
178
+ builder.set_table_info(
179
+ table_identifier=f"{first_result.namespace}.{first_result.table_name}",
180
+ original_storage_path=original_path,
181
+ )
182
+
183
+ # Finalize metadata
184
+ metadata = builder.finalize()
185
+
186
+ # Step 9b: Assert the import's provenance into the dataset graph and
187
+ # link the snapshot the commit produced to it (spec 180 §9 steps 3-4).
188
+ _assert_import_provenance(first_result, builder, metadata, branch)
189
+
190
+ # Update results with original path and metadata
191
+ for result in results:
192
+ result.original_path = original_path
193
+ result.metadata = metadata
194
+
195
+ return results[0] if len(results) == 1 else results
196
+
197
+ finally:
198
+ # Cleanup temp file if downloaded
199
+ if source_info.is_temp:
200
+ source_info.local_path.unlink(missing_ok=True)
201
+
202
+
203
+ def _assert_import_provenance(
204
+ result: ImportResult,
205
+ builder: MetadataBuilder,
206
+ metadata: "ImportMetadata",
207
+ branch: str,
208
+ ) -> None:
209
+ """Publish this import's provenance into the dataset's named graph.
210
+
211
+ Merges two graphs over one GSP ``POST``: the activity, agents and source
212
+ entity (spec 180 §9 step 3) and the snapshot-to-activity link (step 4).
213
+ Both name the activity node the same way, so the merged graph is one
214
+ connected node, not two.
215
+
216
+ Scoped to one table per import, matching `MetadataBuilder`: a multi-sheet
217
+ Excel import asserts provenance only for the first sheet's table.
218
+
219
+ The generation instant is read back from the committed Iceberg snapshot,
220
+ not from the builder's own clock — ADR 0002 §6 keeps those clocks separate.
221
+
222
+ Idempotent when the Iceberg write short-circuited (`already_existed`): if
223
+ the snapshot is already linked to an activity, this is a true no-op — no
224
+ second activity node for a snapshot that never changed. If the link is
225
+ missing (a crash between the table commit and this POST), it is asserted
226
+ now, reusing the original committing activity's id (DP-27).
227
+ """
228
+ from datetime import datetime, timezone
229
+
230
+ from sunstone_data.catalog import get_catalog
231
+ from sunstone_data.graph.client import N_TRIPLES, NotFound, get_graph_client
232
+ from sunstone_data.graph.dataset import (
233
+ build_graph,
234
+ parse_ntriples,
235
+ snapshot_already_has_generation,
236
+ snapshot_generation_triples,
237
+ )
238
+ from sunstone_data.metadata.provenance_rdf import import_provenance_to_graph
239
+ from sunstone_data.publish import publish_graph
240
+ from sunstone_data.query import TableRef
241
+ from sunstone_data.resolve import dataset_iri
242
+
243
+ identifier = f"{result.namespace}.{result.table_name}"
244
+ iri = dataset_iri(TableRef(namespace=result.namespace, table=result.table_name, branch=branch))
245
+
246
+ table = get_catalog(branch).load_table(identifier)
247
+ snapshot = table.current_snapshot()
248
+ generated_at = datetime.fromtimestamp(snapshot.timestamp_ms / 1000, tz=timezone.utc).isoformat()
249
+
250
+ if result.already_existed:
251
+ try:
252
+ existing_text, _ = get_graph_client().get_graph_with_etag(branch, iri, accept=N_TRIPLES)
253
+ existing_graph = parse_ntriples(existing_text)
254
+ except NotFound:
255
+ existing_graph = build_graph([])
256
+ if snapshot_already_has_generation(existing_graph, iri, str(snapshot.snapshot_id)):
257
+ return
258
+
259
+ activity, user_agent, system_agent = builder.build_provenance()
260
+ graph = import_provenance_to_graph(
261
+ dataset_iri=iri,
262
+ activity=activity,
263
+ agents=[user_agent, system_agent],
264
+ checksum=result.checksum,
265
+ retrieved_at=metadata.timestamp_start,
266
+ ) | build_graph(
267
+ snapshot_generation_triples(
268
+ dataset_iri=iri,
269
+ snapshot_id=str(snapshot.snapshot_id),
270
+ import_activity_id=builder.activity_id,
271
+ generated_at=generated_at,
272
+ metadata_location=table.metadata_location,
273
+ )
274
+ )
275
+ publish_graph(dataset_iri=iri, graph=graph, branch=branch)
@@ -0,0 +1,135 @@
1
+ """Original file storage to S3/GCS."""
2
+
3
+ import os
4
+ import re
5
+ from pathlib import Path
6
+
7
+ import s3fs
8
+
9
+ from sunstone_data.config import Settings, get_settings
10
+
11
+
12
+ def _get_filesystem(settings: Settings | None = None):
13
+ """Create filesystem connection for the configured storage backend."""
14
+ settings = settings or get_settings()
15
+
16
+ if settings.storage_backend == "gcs":
17
+ import gcsfs
18
+
19
+ kwargs = {}
20
+ if settings.gcs_endpoint:
21
+ # Tell gcsfs and underlying google-cloud-storage to use emulator
22
+ os.environ["STORAGE_EMULATOR_HOST"] = settings.gcs_endpoint
23
+ kwargs["endpoint_url"] = settings.gcs_endpoint
24
+ kwargs["token"] = "anon"
25
+ if settings.gcs_project_id:
26
+ kwargs["project"] = settings.gcs_project_id
27
+ return gcsfs.GCSFileSystem(**kwargs)
28
+ else:
29
+ return s3fs.S3FileSystem(
30
+ endpoint_url=settings.s3_endpoint,
31
+ key=settings.s3_access_key,
32
+ secret=settings.s3_secret_key,
33
+ )
34
+
35
+
36
+ def _get_bucket(settings: Settings | None = None) -> str:
37
+ """Extract bucket name from warehouse URL."""
38
+ settings = settings or get_settings()
39
+ # Handle s3://bucket/path/ and gs://bucket/path/
40
+ return re.sub(r"^[a-z0-9]+://", "", settings.warehouse).split("/")[0]
41
+
42
+
43
+ def _storage_uri(settings: Settings, path: str) -> str:
44
+ """Build a storage URI with the correct scheme for the backend."""
45
+ scheme = "gs" if settings.storage_backend == "gcs" else "s3"
46
+ return f"{scheme}://{path}"
47
+
48
+
49
+ def _strip_scheme(uri: str) -> str:
50
+ """Remove the scheme prefix (s3:// or gs://) from a URI."""
51
+ return re.sub(r"^[a-z0-9]+://", "", uri)
52
+
53
+
54
+ def store_original(
55
+ file_path: Path,
56
+ checksum: str,
57
+ namespace: str,
58
+ table_name: str,
59
+ settings: Settings | None = None,
60
+ ) -> str:
61
+ """Store original file in S3/GCS.
62
+
63
+ Uses checksum in path for content-addressable storage.
64
+
65
+ Args:
66
+ file_path: Local path to file.
67
+ checksum: SHA256 checksum of file.
68
+ namespace: Target namespace.
69
+ table_name: Target table name.
70
+ settings: Optional settings override.
71
+
72
+ Returns:
73
+ Storage URI of stored file (s3://... or gs://...).
74
+ """
75
+ settings = settings or get_settings()
76
+ fs = _get_filesystem(settings)
77
+ bucket = _get_bucket(settings)
78
+
79
+ extension = file_path.suffix
80
+ storage_path = f"originals/{namespace}/{table_name}/{checksum}{extension}"
81
+ full_path = f"{bucket}/{storage_path}"
82
+
83
+ if fs.exists(full_path):
84
+ return _storage_uri(settings, full_path)
85
+
86
+ fs.put(str(file_path), full_path)
87
+ return _storage_uri(settings, full_path)
88
+
89
+
90
+ def file_exists_by_checksum(
91
+ checksum: str,
92
+ namespace: str,
93
+ table_name: str,
94
+ settings: Settings | None = None,
95
+ ) -> str | None:
96
+ """Check if file with checksum already exists.
97
+
98
+ Args:
99
+ checksum: SHA256 checksum to look for.
100
+ namespace: Namespace to search in.
101
+ table_name: Table name to search in.
102
+ settings: Optional settings override.
103
+
104
+ Returns:
105
+ Storage URI if exists, None otherwise.
106
+ """
107
+ settings = settings or get_settings()
108
+ fs = _get_filesystem(settings)
109
+ bucket = _get_bucket(settings)
110
+
111
+ pattern = f"{bucket}/originals/{namespace}/{table_name}/{checksum}.*"
112
+ matches = fs.glob(pattern)
113
+
114
+ if matches:
115
+ return _storage_uri(settings, matches[0])
116
+ return None
117
+
118
+
119
+ def get_original(
120
+ original_path: str,
121
+ settings: Settings | None = None,
122
+ ) -> bytes:
123
+ """Retrieve original file content.
124
+
125
+ Args:
126
+ original_path: Storage URI of original file (s3://... or gs://...).
127
+ settings: Optional settings override.
128
+
129
+ Returns:
130
+ File content as bytes.
131
+ """
132
+ settings = settings or get_settings()
133
+ fs = _get_filesystem(settings)
134
+ path = _strip_scheme(original_path)
135
+ return fs.cat(path)
@@ -0,0 +1,41 @@
1
+ """Metadata module for import provenance tracking.
2
+
3
+ Provides data models for capturing extended provenance metadata at import time,
4
+ following W3C PROV-DM structures for Activity, Entity, and Agent.
5
+ """
6
+
7
+ from sunstone_data.metadata.builder import MetadataBuilder
8
+ from sunstone_data.metadata.models import (
9
+ ImportMetadata,
10
+ SchemaMetadata,
11
+ SourceMetadata,
12
+ )
13
+ from sunstone_data.metadata.provenance import (
14
+ ProvActivity,
15
+ ProvAgent,
16
+ ProvEntity,
17
+ generate_activity_id,
18
+ make_activity_uri,
19
+ make_file_uri,
20
+ make_table_uri,
21
+ make_url_uri,
22
+ )
23
+
24
+ __all__ = [
25
+ # Builder
26
+ "MetadataBuilder",
27
+ # Metadata models
28
+ "ImportMetadata",
29
+ "SchemaMetadata",
30
+ "SourceMetadata",
31
+ # PROV-DM types
32
+ "ProvActivity",
33
+ "ProvAgent",
34
+ "ProvEntity",
35
+ # URI helpers
36
+ "generate_activity_id",
37
+ "make_activity_uri",
38
+ "make_file_uri",
39
+ "make_table_uri",
40
+ "make_url_uri",
41
+ ]