sunstone-data-platform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunstone_data/__init__.py +36 -0
- sunstone_data/_version.py +24 -0
- sunstone_data/auth/__init__.py +43 -0
- sunstone_data/auth/client_creds.py +239 -0
- sunstone_data/auth/pkce.py +307 -0
- sunstone_data/auth/store.py +97 -0
- sunstone_data/catalog.py +175 -0
- sunstone_data/cli/__init__.py +5 -0
- sunstone_data/cli/_app.py +26 -0
- sunstone_data/cli/_errors.py +31 -0
- sunstone_data/cli/commands/__init__.py +22 -0
- sunstone_data/cli/commands/data_auth.py +120 -0
- sunstone_data/cli/commands/data_import.py +157 -0
- sunstone_data/cli/commands/data_info.py +166 -0
- sunstone_data/cli/commands/data_publish.py +69 -0
- sunstone_data/cli/commands/ls.py +49 -0
- sunstone_data/cli/commands/prov.py +58 -0
- sunstone_data/cli/commands/refs.py +142 -0
- sunstone_data/cli/commands/show.py +139 -0
- sunstone_data/cli/commands/snapshots.py +77 -0
- sunstone_data/cli/commands/sparql.py +68 -0
- sunstone_data/cli/main.py +30 -0
- sunstone_data/config.py +250 -0
- sunstone_data/discovery.py +317 -0
- sunstone_data/env_commands.py +97 -0
- sunstone_data/env_section.py +86 -0
- sunstone_data/graph/__init__.py +36 -0
- sunstone_data/graph/client.py +328 -0
- sunstone_data/graph/dataset.py +438 -0
- sunstone_data/graph/terms.py +34 -0
- sunstone_data/identifiers.py +155 -0
- sunstone_data/import_/__init__.py +28 -0
- sunstone_data/import_/fetcher.py +259 -0
- sunstone_data/import_/owner.py +130 -0
- sunstone_data/import_/parser.py +492 -0
- sunstone_data/import_/pipeline.py +275 -0
- sunstone_data/import_/storage.py +135 -0
- sunstone_data/metadata/__init__.py +41 -0
- sunstone_data/metadata/builder.py +276 -0
- sunstone_data/metadata/models.py +87 -0
- sunstone_data/metadata/provenance.py +171 -0
- sunstone_data/metadata/provenance_rdf.py +102 -0
- sunstone_data/plugin.py +62 -0
- sunstone_data/publish.py +134 -0
- sunstone_data/query.py +624 -0
- sunstone_data/resolve.py +178 -0
- sunstone_data/url.py +78 -0
- sunstone_data/utils/__init__.py +12 -0
- sunstone_data/utils/checksum.py +14 -0
- sunstone_data/utils/encoding.py +16 -0
- sunstone_data/utils/naming.py +38 -0
- sunstone_data/versioning/__init__.py +17 -0
- sunstone_data/versioning/schema_evolution.py +180 -0
- sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
- sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
- sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
- sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
- sunstone_pipeline/__init__.py +1 -0
- sunstone_pipeline/assets/__init__.py +11 -0
- sunstone_pipeline/assets/mirrors.py +123 -0
- sunstone_pipeline/assets/transforms.py +74 -0
- sunstone_pipeline/definitions.py +25 -0
- sunstone_pipeline/io_manager.py +56 -0
- sunstone_pipeline/resources/__init__.py +6 -0
- sunstone_pipeline/resources/iceberg.py +29 -0
- sunstone_pipeline/resources/refs.py +88 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Import pipeline orchestration."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import TYPE_CHECKING, Callable
|
|
6
|
+
|
|
7
|
+
from sunstone_data.config import Settings, get_settings
|
|
8
|
+
from sunstone_data.import_.fetcher import FileType, SourceInfo, fetch_source
|
|
9
|
+
from sunstone_data.import_.owner import (
|
|
10
|
+
ensure_root_prefix,
|
|
11
|
+
load_rules,
|
|
12
|
+
org_uri,
|
|
13
|
+
owner_root,
|
|
14
|
+
resolve_owner,
|
|
15
|
+
)
|
|
16
|
+
from sunstone_data.import_.storage import store_original
|
|
17
|
+
from sunstone_data.metadata import MetadataBuilder
|
|
18
|
+
from sunstone_data.utils import (
|
|
19
|
+
compute_checksum,
|
|
20
|
+
derive_namespace,
|
|
21
|
+
detect_encoding,
|
|
22
|
+
sanitize_table_name,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from sunstone_data.metadata import ImportMetadata
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class ImportResult:
|
|
31
|
+
"""Result of import operation."""
|
|
32
|
+
|
|
33
|
+
table_name: str
|
|
34
|
+
namespace: str
|
|
35
|
+
row_count: int
|
|
36
|
+
column_count: int
|
|
37
|
+
checksum: str
|
|
38
|
+
original_path: str
|
|
39
|
+
schema_summary: str
|
|
40
|
+
already_existed: bool = False
|
|
41
|
+
#: Iceberg snapshot id the table points at after the import.
|
|
42
|
+
snapshot_id: int | None = None
|
|
43
|
+
metadata: "ImportMetadata | None" = field(default=None)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_import(
|
|
47
|
+
source: str,
|
|
48
|
+
name_override: str | None = None,
|
|
49
|
+
namespace_override: str | None = None,
|
|
50
|
+
owner: str | None = None,
|
|
51
|
+
tags: list[str] | None = None,
|
|
52
|
+
branch: str = "main",
|
|
53
|
+
force_schema: bool = False,
|
|
54
|
+
progress_callback: Callable[[int, int | None], None] | None = None,
|
|
55
|
+
settings: Settings | None = None,
|
|
56
|
+
) -> ImportResult | list[ImportResult]:
|
|
57
|
+
"""Run complete import pipeline.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
source: Local file path or URL to import.
|
|
61
|
+
name_override: Optional table name override.
|
|
62
|
+
namespace_override: Optional namespace override.
|
|
63
|
+
owner: Owning organization path (e.g. ``un/who``). Overrides the
|
|
64
|
+
source-URL rules file. Required unless a rule matches the source.
|
|
65
|
+
tags: Optional list of tags (key:value format).
|
|
66
|
+
branch: Branch to import to (default: "main").
|
|
67
|
+
force_schema: Allow unsafe schema changes (default: False).
|
|
68
|
+
progress_callback: Optional progress callback(downloaded, total).
|
|
69
|
+
settings: Optional settings override.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
Single ImportResult for CSV, list for multi-sheet Excel.
|
|
73
|
+
|
|
74
|
+
Raises:
|
|
75
|
+
FileNotFoundError: If local file not found.
|
|
76
|
+
ValueError: If file type not supported or parsing fails.
|
|
77
|
+
OwnerError: If the owner cannot be resolved (no --owner and no matching
|
|
78
|
+
rule) or is invalid. OwnerUnresolved/OwnerInvalid both subclass
|
|
79
|
+
ValueError.
|
|
80
|
+
httpx.HTTPError: If URL download fails.
|
|
81
|
+
"""
|
|
82
|
+
settings = settings or get_settings()
|
|
83
|
+
|
|
84
|
+
# Create MetadataBuilder at start
|
|
85
|
+
builder = MetadataBuilder(
|
|
86
|
+
source=source,
|
|
87
|
+
import_parameters={
|
|
88
|
+
"name_override": name_override,
|
|
89
|
+
"namespace_override": namespace_override,
|
|
90
|
+
"tags": tags,
|
|
91
|
+
},
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Step 1: Fetch source (download if URL, validate if local)
|
|
95
|
+
source_info = fetch_source(source, progress_callback)
|
|
96
|
+
|
|
97
|
+
try:
|
|
98
|
+
# Step 2: Detect encoding for CSV files
|
|
99
|
+
if source_info.file_type == FileType.CSV:
|
|
100
|
+
encoding = detect_encoding(source_info.local_path)
|
|
101
|
+
source_info = SourceInfo(
|
|
102
|
+
local_path=source_info.local_path,
|
|
103
|
+
filename=source_info.filename,
|
|
104
|
+
file_type=source_info.file_type,
|
|
105
|
+
encoding=encoding,
|
|
106
|
+
is_temp=source_info.is_temp,
|
|
107
|
+
source_url=source_info.source_url,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Step 3: Compute checksum
|
|
111
|
+
checksum = compute_checksum(source_info.local_path)
|
|
112
|
+
|
|
113
|
+
# Step 4: Determine normalized path for metadata
|
|
114
|
+
if source_info.source_url:
|
|
115
|
+
# URL: use cleaned URL
|
|
116
|
+
normalized_path = source_info.source_url
|
|
117
|
+
else:
|
|
118
|
+
# Local file: use resolved absolute path
|
|
119
|
+
normalized_path = str(source_info.local_path.resolve())
|
|
120
|
+
|
|
121
|
+
# Step 5: Capture source metadata
|
|
122
|
+
builder.set_source_info(
|
|
123
|
+
normalized_path=normalized_path,
|
|
124
|
+
file_type=source_info.file_type.value,
|
|
125
|
+
file_size=source_info.local_path.stat().st_size,
|
|
126
|
+
encoding=source_info.encoding,
|
|
127
|
+
checksum=checksum,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
# Step 6: Determine table name and namespace
|
|
131
|
+
base_name = name_override or sanitize_table_name(source_info.filename)
|
|
132
|
+
namespace = namespace_override or derive_namespace(source)
|
|
133
|
+
|
|
134
|
+
# Step 6b: Resolve the data-owner organization (strict: errors if
|
|
135
|
+
# neither --owner nor a matching source-URL rule resolves one). The
|
|
136
|
+
# owner root is authoritative for namespace level 0.
|
|
137
|
+
rules = load_rules(Path(settings.owner_rules_path) if settings.owner_rules_path else None)
|
|
138
|
+
owner_path = resolve_owner(
|
|
139
|
+
source_info.source_url or "",
|
|
140
|
+
owner,
|
|
141
|
+
rules=rules,
|
|
142
|
+
org_roots=settings.org_roots,
|
|
143
|
+
)
|
|
144
|
+
namespace = ensure_root_prefix(namespace, owner_root(owner_path))
|
|
145
|
+
owner_org_uri = org_uri(owner_path, settings.org_base_uri)
|
|
146
|
+
|
|
147
|
+
# Step 7: Parse file and create tables
|
|
148
|
+
# NOTE: Deferred import - parser module created in Plan 02-03 (wave 3)
|
|
149
|
+
# This import happens at runtime, not module load time, so it's safe
|
|
150
|
+
from sunstone_data.import_.parser import parse_and_create_tables
|
|
151
|
+
|
|
152
|
+
results = parse_and_create_tables(
|
|
153
|
+
source_info=source_info,
|
|
154
|
+
base_name=base_name,
|
|
155
|
+
namespace=namespace,
|
|
156
|
+
checksum=checksum,
|
|
157
|
+
source=source,
|
|
158
|
+
tags=tags,
|
|
159
|
+
branch=branch,
|
|
160
|
+
force_schema=force_schema,
|
|
161
|
+
settings=settings,
|
|
162
|
+
builder=builder,
|
|
163
|
+
owner_org=owner_org_uri,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# Step 8: Store original file (after successful parse)
|
|
167
|
+
original_path = store_original(
|
|
168
|
+
file_path=source_info.local_path,
|
|
169
|
+
checksum=checksum,
|
|
170
|
+
namespace=namespace,
|
|
171
|
+
table_name=base_name,
|
|
172
|
+
settings=settings,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
# Step 9: Finalize metadata and update results
|
|
176
|
+
# Use the first result's table for metadata (CSV has one, Excel may have multiple)
|
|
177
|
+
first_result = results[0]
|
|
178
|
+
builder.set_table_info(
|
|
179
|
+
table_identifier=f"{first_result.namespace}.{first_result.table_name}",
|
|
180
|
+
original_storage_path=original_path,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# Finalize metadata
|
|
184
|
+
metadata = builder.finalize()
|
|
185
|
+
|
|
186
|
+
# Step 9b: Assert the import's provenance into the dataset graph and
|
|
187
|
+
# link the snapshot the commit produced to it (spec 180 §9 steps 3-4).
|
|
188
|
+
_assert_import_provenance(first_result, builder, metadata, branch)
|
|
189
|
+
|
|
190
|
+
# Update results with original path and metadata
|
|
191
|
+
for result in results:
|
|
192
|
+
result.original_path = original_path
|
|
193
|
+
result.metadata = metadata
|
|
194
|
+
|
|
195
|
+
return results[0] if len(results) == 1 else results
|
|
196
|
+
|
|
197
|
+
finally:
|
|
198
|
+
# Cleanup temp file if downloaded
|
|
199
|
+
if source_info.is_temp:
|
|
200
|
+
source_info.local_path.unlink(missing_ok=True)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _assert_import_provenance(
|
|
204
|
+
result: ImportResult,
|
|
205
|
+
builder: MetadataBuilder,
|
|
206
|
+
metadata: "ImportMetadata",
|
|
207
|
+
branch: str,
|
|
208
|
+
) -> None:
|
|
209
|
+
"""Publish this import's provenance into the dataset's named graph.
|
|
210
|
+
|
|
211
|
+
Merges two graphs over one GSP ``POST``: the activity, agents and source
|
|
212
|
+
entity (spec 180 §9 step 3) and the snapshot-to-activity link (step 4).
|
|
213
|
+
Both name the activity node the same way, so the merged graph is one
|
|
214
|
+
connected node, not two.
|
|
215
|
+
|
|
216
|
+
Scoped to one table per import, matching `MetadataBuilder`: a multi-sheet
|
|
217
|
+
Excel import asserts provenance only for the first sheet's table.
|
|
218
|
+
|
|
219
|
+
The generation instant is read back from the committed Iceberg snapshot,
|
|
220
|
+
not from the builder's own clock — ADR 0002 §6 keeps those clocks separate.
|
|
221
|
+
|
|
222
|
+
Idempotent when the Iceberg write short-circuited (`already_existed`): if
|
|
223
|
+
the snapshot is already linked to an activity, this is a true no-op — no
|
|
224
|
+
second activity node for a snapshot that never changed. If the link is
|
|
225
|
+
missing (a crash between the table commit and this POST), it is asserted
|
|
226
|
+
now, reusing the original committing activity's id (DP-27).
|
|
227
|
+
"""
|
|
228
|
+
from datetime import datetime, timezone
|
|
229
|
+
|
|
230
|
+
from sunstone_data.catalog import get_catalog
|
|
231
|
+
from sunstone_data.graph.client import N_TRIPLES, NotFound, get_graph_client
|
|
232
|
+
from sunstone_data.graph.dataset import (
|
|
233
|
+
build_graph,
|
|
234
|
+
parse_ntriples,
|
|
235
|
+
snapshot_already_has_generation,
|
|
236
|
+
snapshot_generation_triples,
|
|
237
|
+
)
|
|
238
|
+
from sunstone_data.metadata.provenance_rdf import import_provenance_to_graph
|
|
239
|
+
from sunstone_data.publish import publish_graph
|
|
240
|
+
from sunstone_data.query import TableRef
|
|
241
|
+
from sunstone_data.resolve import dataset_iri
|
|
242
|
+
|
|
243
|
+
identifier = f"{result.namespace}.{result.table_name}"
|
|
244
|
+
iri = dataset_iri(TableRef(namespace=result.namespace, table=result.table_name, branch=branch))
|
|
245
|
+
|
|
246
|
+
table = get_catalog(branch).load_table(identifier)
|
|
247
|
+
snapshot = table.current_snapshot()
|
|
248
|
+
generated_at = datetime.fromtimestamp(snapshot.timestamp_ms / 1000, tz=timezone.utc).isoformat()
|
|
249
|
+
|
|
250
|
+
if result.already_existed:
|
|
251
|
+
try:
|
|
252
|
+
existing_text, _ = get_graph_client().get_graph_with_etag(branch, iri, accept=N_TRIPLES)
|
|
253
|
+
existing_graph = parse_ntriples(existing_text)
|
|
254
|
+
except NotFound:
|
|
255
|
+
existing_graph = build_graph([])
|
|
256
|
+
if snapshot_already_has_generation(existing_graph, iri, str(snapshot.snapshot_id)):
|
|
257
|
+
return
|
|
258
|
+
|
|
259
|
+
activity, user_agent, system_agent = builder.build_provenance()
|
|
260
|
+
graph = import_provenance_to_graph(
|
|
261
|
+
dataset_iri=iri,
|
|
262
|
+
activity=activity,
|
|
263
|
+
agents=[user_agent, system_agent],
|
|
264
|
+
checksum=result.checksum,
|
|
265
|
+
retrieved_at=metadata.timestamp_start,
|
|
266
|
+
) | build_graph(
|
|
267
|
+
snapshot_generation_triples(
|
|
268
|
+
dataset_iri=iri,
|
|
269
|
+
snapshot_id=str(snapshot.snapshot_id),
|
|
270
|
+
import_activity_id=builder.activity_id,
|
|
271
|
+
generated_at=generated_at,
|
|
272
|
+
metadata_location=table.metadata_location,
|
|
273
|
+
)
|
|
274
|
+
)
|
|
275
|
+
publish_graph(dataset_iri=iri, graph=graph, branch=branch)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Original file storage to S3/GCS."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import s3fs
|
|
8
|
+
|
|
9
|
+
from sunstone_data.config import Settings, get_settings
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _get_filesystem(settings: Settings | None = None):
|
|
13
|
+
"""Create filesystem connection for the configured storage backend."""
|
|
14
|
+
settings = settings or get_settings()
|
|
15
|
+
|
|
16
|
+
if settings.storage_backend == "gcs":
|
|
17
|
+
import gcsfs
|
|
18
|
+
|
|
19
|
+
kwargs = {}
|
|
20
|
+
if settings.gcs_endpoint:
|
|
21
|
+
# Tell gcsfs and underlying google-cloud-storage to use emulator
|
|
22
|
+
os.environ["STORAGE_EMULATOR_HOST"] = settings.gcs_endpoint
|
|
23
|
+
kwargs["endpoint_url"] = settings.gcs_endpoint
|
|
24
|
+
kwargs["token"] = "anon"
|
|
25
|
+
if settings.gcs_project_id:
|
|
26
|
+
kwargs["project"] = settings.gcs_project_id
|
|
27
|
+
return gcsfs.GCSFileSystem(**kwargs)
|
|
28
|
+
else:
|
|
29
|
+
return s3fs.S3FileSystem(
|
|
30
|
+
endpoint_url=settings.s3_endpoint,
|
|
31
|
+
key=settings.s3_access_key,
|
|
32
|
+
secret=settings.s3_secret_key,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _get_bucket(settings: Settings | None = None) -> str:
|
|
37
|
+
"""Extract bucket name from warehouse URL."""
|
|
38
|
+
settings = settings or get_settings()
|
|
39
|
+
# Handle s3://bucket/path/ and gs://bucket/path/
|
|
40
|
+
return re.sub(r"^[a-z0-9]+://", "", settings.warehouse).split("/")[0]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _storage_uri(settings: Settings, path: str) -> str:
|
|
44
|
+
"""Build a storage URI with the correct scheme for the backend."""
|
|
45
|
+
scheme = "gs" if settings.storage_backend == "gcs" else "s3"
|
|
46
|
+
return f"{scheme}://{path}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _strip_scheme(uri: str) -> str:
|
|
50
|
+
"""Remove the scheme prefix (s3:// or gs://) from a URI."""
|
|
51
|
+
return re.sub(r"^[a-z0-9]+://", "", uri)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def store_original(
|
|
55
|
+
file_path: Path,
|
|
56
|
+
checksum: str,
|
|
57
|
+
namespace: str,
|
|
58
|
+
table_name: str,
|
|
59
|
+
settings: Settings | None = None,
|
|
60
|
+
) -> str:
|
|
61
|
+
"""Store original file in S3/GCS.
|
|
62
|
+
|
|
63
|
+
Uses checksum in path for content-addressable storage.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
file_path: Local path to file.
|
|
67
|
+
checksum: SHA256 checksum of file.
|
|
68
|
+
namespace: Target namespace.
|
|
69
|
+
table_name: Target table name.
|
|
70
|
+
settings: Optional settings override.
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
Storage URI of stored file (s3://... or gs://...).
|
|
74
|
+
"""
|
|
75
|
+
settings = settings or get_settings()
|
|
76
|
+
fs = _get_filesystem(settings)
|
|
77
|
+
bucket = _get_bucket(settings)
|
|
78
|
+
|
|
79
|
+
extension = file_path.suffix
|
|
80
|
+
storage_path = f"originals/{namespace}/{table_name}/{checksum}{extension}"
|
|
81
|
+
full_path = f"{bucket}/{storage_path}"
|
|
82
|
+
|
|
83
|
+
if fs.exists(full_path):
|
|
84
|
+
return _storage_uri(settings, full_path)
|
|
85
|
+
|
|
86
|
+
fs.put(str(file_path), full_path)
|
|
87
|
+
return _storage_uri(settings, full_path)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def file_exists_by_checksum(
|
|
91
|
+
checksum: str,
|
|
92
|
+
namespace: str,
|
|
93
|
+
table_name: str,
|
|
94
|
+
settings: Settings | None = None,
|
|
95
|
+
) -> str | None:
|
|
96
|
+
"""Check if file with checksum already exists.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
checksum: SHA256 checksum to look for.
|
|
100
|
+
namespace: Namespace to search in.
|
|
101
|
+
table_name: Table name to search in.
|
|
102
|
+
settings: Optional settings override.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
Storage URI if exists, None otherwise.
|
|
106
|
+
"""
|
|
107
|
+
settings = settings or get_settings()
|
|
108
|
+
fs = _get_filesystem(settings)
|
|
109
|
+
bucket = _get_bucket(settings)
|
|
110
|
+
|
|
111
|
+
pattern = f"{bucket}/originals/{namespace}/{table_name}/{checksum}.*"
|
|
112
|
+
matches = fs.glob(pattern)
|
|
113
|
+
|
|
114
|
+
if matches:
|
|
115
|
+
return _storage_uri(settings, matches[0])
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def get_original(
|
|
120
|
+
original_path: str,
|
|
121
|
+
settings: Settings | None = None,
|
|
122
|
+
) -> bytes:
|
|
123
|
+
"""Retrieve original file content.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
original_path: Storage URI of original file (s3://... or gs://...).
|
|
127
|
+
settings: Optional settings override.
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
File content as bytes.
|
|
131
|
+
"""
|
|
132
|
+
settings = settings or get_settings()
|
|
133
|
+
fs = _get_filesystem(settings)
|
|
134
|
+
path = _strip_scheme(original_path)
|
|
135
|
+
return fs.cat(path)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Metadata module for import provenance tracking.
|
|
2
|
+
|
|
3
|
+
Provides data models for capturing extended provenance metadata at import time,
|
|
4
|
+
following W3C PROV-DM structures for Activity, Entity, and Agent.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from sunstone_data.metadata.builder import MetadataBuilder
|
|
8
|
+
from sunstone_data.metadata.models import (
|
|
9
|
+
ImportMetadata,
|
|
10
|
+
SchemaMetadata,
|
|
11
|
+
SourceMetadata,
|
|
12
|
+
)
|
|
13
|
+
from sunstone_data.metadata.provenance import (
|
|
14
|
+
ProvActivity,
|
|
15
|
+
ProvAgent,
|
|
16
|
+
ProvEntity,
|
|
17
|
+
generate_activity_id,
|
|
18
|
+
make_activity_uri,
|
|
19
|
+
make_file_uri,
|
|
20
|
+
make_table_uri,
|
|
21
|
+
make_url_uri,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
# Builder
|
|
26
|
+
"MetadataBuilder",
|
|
27
|
+
# Metadata models
|
|
28
|
+
"ImportMetadata",
|
|
29
|
+
"SchemaMetadata",
|
|
30
|
+
"SourceMetadata",
|
|
31
|
+
# PROV-DM types
|
|
32
|
+
"ProvActivity",
|
|
33
|
+
"ProvAgent",
|
|
34
|
+
"ProvEntity",
|
|
35
|
+
# URI helpers
|
|
36
|
+
"generate_activity_id",
|
|
37
|
+
"make_activity_uri",
|
|
38
|
+
"make_file_uri",
|
|
39
|
+
"make_table_uri",
|
|
40
|
+
"make_url_uri",
|
|
41
|
+
]
|