sunstone-data-platform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunstone_data/__init__.py +36 -0
- sunstone_data/_version.py +24 -0
- sunstone_data/auth/__init__.py +43 -0
- sunstone_data/auth/client_creds.py +239 -0
- sunstone_data/auth/pkce.py +307 -0
- sunstone_data/auth/store.py +97 -0
- sunstone_data/catalog.py +175 -0
- sunstone_data/cli/__init__.py +5 -0
- sunstone_data/cli/_app.py +26 -0
- sunstone_data/cli/_errors.py +31 -0
- sunstone_data/cli/commands/__init__.py +22 -0
- sunstone_data/cli/commands/data_auth.py +120 -0
- sunstone_data/cli/commands/data_import.py +157 -0
- sunstone_data/cli/commands/data_info.py +166 -0
- sunstone_data/cli/commands/data_publish.py +69 -0
- sunstone_data/cli/commands/ls.py +49 -0
- sunstone_data/cli/commands/prov.py +58 -0
- sunstone_data/cli/commands/refs.py +142 -0
- sunstone_data/cli/commands/show.py +139 -0
- sunstone_data/cli/commands/snapshots.py +77 -0
- sunstone_data/cli/commands/sparql.py +68 -0
- sunstone_data/cli/main.py +30 -0
- sunstone_data/config.py +250 -0
- sunstone_data/discovery.py +317 -0
- sunstone_data/env_commands.py +97 -0
- sunstone_data/env_section.py +86 -0
- sunstone_data/graph/__init__.py +36 -0
- sunstone_data/graph/client.py +328 -0
- sunstone_data/graph/dataset.py +438 -0
- sunstone_data/graph/terms.py +34 -0
- sunstone_data/identifiers.py +155 -0
- sunstone_data/import_/__init__.py +28 -0
- sunstone_data/import_/fetcher.py +259 -0
- sunstone_data/import_/owner.py +130 -0
- sunstone_data/import_/parser.py +492 -0
- sunstone_data/import_/pipeline.py +275 -0
- sunstone_data/import_/storage.py +135 -0
- sunstone_data/metadata/__init__.py +41 -0
- sunstone_data/metadata/builder.py +276 -0
- sunstone_data/metadata/models.py +87 -0
- sunstone_data/metadata/provenance.py +171 -0
- sunstone_data/metadata/provenance_rdf.py +102 -0
- sunstone_data/plugin.py +62 -0
- sunstone_data/publish.py +134 -0
- sunstone_data/query.py +624 -0
- sunstone_data/resolve.py +178 -0
- sunstone_data/url.py +78 -0
- sunstone_data/utils/__init__.py +12 -0
- sunstone_data/utils/checksum.py +14 -0
- sunstone_data/utils/encoding.py +16 -0
- sunstone_data/utils/naming.py +38 -0
- sunstone_data/versioning/__init__.py +17 -0
- sunstone_data/versioning/schema_evolution.py +180 -0
- sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
- sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
- sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
- sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
- sunstone_pipeline/__init__.py +1 -0
- sunstone_pipeline/assets/__init__.py +11 -0
- sunstone_pipeline/assets/mirrors.py +123 -0
- sunstone_pipeline/assets/transforms.py +74 -0
- sunstone_pipeline/definitions.py +25 -0
- sunstone_pipeline/io_manager.py +56 -0
- sunstone_pipeline/resources/__init__.py +6 -0
- sunstone_pipeline/resources/iceberg.py +29 -0
- sunstone_pipeline/resources/refs.py +88 -0
sunstone_data/resolve.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Resolve a parsed `sunstone:` URL to Iceberg table data (spec 180 §7).
|
|
2
|
+
|
|
3
|
+
`url.parse` splits `sunstone:un/health/cases@v1.0` into a `TableRef` but does
|
|
4
|
+
not say what `v1.0` *is*. This module makes that call, then reads the data:
|
|
5
|
+
|
|
6
|
+
ref = parse_sunstone_url(url)
|
|
7
|
+
-> GET /branches (cached per Resolver)
|
|
8
|
+
-> ref.version_ref is a known branch? yes -> branch selector
|
|
9
|
+
no -> version selector
|
|
10
|
+
-> RestCatalog(".../iceberg/{branch}/").load_table("un.health.cases")
|
|
11
|
+
-> apply selector -> .to_arrow() -> parquet -> BytesIO
|
|
12
|
+
|
|
13
|
+
Branch-versus-version is decided by asking the graph server which names are
|
|
14
|
+
branches, not by pattern-matching the ref. Pattern matching cannot tell a
|
|
15
|
+
branch named `v2` from a named version called `v2`.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import io
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from datetime import datetime, timezone
|
|
23
|
+
from typing import BinaryIO, Callable
|
|
24
|
+
|
|
25
|
+
import pyarrow as pa
|
|
26
|
+
from dateutil import parser as date_parser
|
|
27
|
+
from pyiceberg.catalog.rest import RestCatalog
|
|
28
|
+
from pyiceberg.table import Table
|
|
29
|
+
|
|
30
|
+
from sunstone_data.catalog import get_catalog
|
|
31
|
+
from sunstone_data.config import get_settings
|
|
32
|
+
from sunstone_data.graph.client import N_TRIPLES, GraphClient, get_graph_client
|
|
33
|
+
from sunstone_data.graph.dataset import graph_to_description, parse_ntriples
|
|
34
|
+
from sunstone_data.query import TableRef
|
|
35
|
+
|
|
36
|
+
#: A ref made only of digits and at least this long is an Iceberg snapshot id.
|
|
37
|
+
#: Snapshot ids are epoch-millisecond-seeded 64-bit numbers, so they are always
|
|
38
|
+
#: this wide; no plausible date or version label is.
|
|
39
|
+
_SNAPSHOT_ID_MIN_DIGITS = 13
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class ResolvedRef:
|
|
44
|
+
"""The catalog branch and optional Iceberg snapshot selected by a ref."""
|
|
45
|
+
|
|
46
|
+
branch: str
|
|
47
|
+
snapshot_id: int | None = None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def dataset_iri(ref: TableRef) -> str:
|
|
51
|
+
"""The IRI of the dataset's named graph.
|
|
52
|
+
|
|
53
|
+
Mirrors the catalog's own minting rule (`iceberg-catalog`'s
|
|
54
|
+
`naming.rs::table_iri`): `{data_base_iri}/{ns levels joined by /}/{table}`.
|
|
55
|
+
"""
|
|
56
|
+
base = get_settings().data_base_iri.rstrip("/")
|
|
57
|
+
return f"{base}/{ref.namespace.replace('.', '/')}/{ref.table}"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _timestamp_ms(value: str) -> int | None:
|
|
61
|
+
"""Parse `value` as an instant in epoch milliseconds, or None if it isn't one.
|
|
62
|
+
|
|
63
|
+
A parsed value with no timezone is read as UTC. Reading it as local time
|
|
64
|
+
would make the same URL resolve to different snapshots on machines in
|
|
65
|
+
different timezones — refs travel between machines, so local time is never
|
|
66
|
+
the right default here.
|
|
67
|
+
"""
|
|
68
|
+
try:
|
|
69
|
+
parsed = date_parser.parse(value)
|
|
70
|
+
except (ValueError, TypeError, OverflowError):
|
|
71
|
+
return None
|
|
72
|
+
if parsed.tzinfo is None:
|
|
73
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
74
|
+
return int(parsed.timestamp() * 1000)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Resolver:
|
|
78
|
+
"""Turns a `TableRef`'s `version_ref` into a branch or an Iceberg snapshot.
|
|
79
|
+
|
|
80
|
+
`GET /branches` is fetched at most once per instance. Construct one per
|
|
81
|
+
logical resolve rather than sharing one across unrelated calls, so a
|
|
82
|
+
branch created meanwhile is still seen.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
graph_client: Client for the graph server. Defaults to
|
|
86
|
+
`get_graph_client()`, built lazily on first use.
|
|
87
|
+
catalog_factory: Builds a catalog for a branch. Defaults to
|
|
88
|
+
`get_catalog`.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
graph_client: GraphClient | None = None,
|
|
94
|
+
catalog_factory: Callable[[str], RestCatalog] = get_catalog,
|
|
95
|
+
):
|
|
96
|
+
self._graph_client = graph_client
|
|
97
|
+
self._catalog_factory = catalog_factory
|
|
98
|
+
self._branch_names: frozenset[str] | None = None
|
|
99
|
+
|
|
100
|
+
def resolve(self, ref: TableRef) -> pa.Table:
|
|
101
|
+
"""Read the table the ref points at.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
ValueError: The ref is not a branch, snapshot id, resolvable
|
|
105
|
+
timestamp, or named version of this table.
|
|
106
|
+
"""
|
|
107
|
+
resolved, table = self._resolve_ref(ref)
|
|
108
|
+
if table is None:
|
|
109
|
+
table = self._load(ref, resolved.branch)
|
|
110
|
+
if resolved.snapshot_id is None:
|
|
111
|
+
return table.scan().to_arrow()
|
|
112
|
+
return table.scan(snapshot_id=resolved.snapshot_id).to_arrow()
|
|
113
|
+
|
|
114
|
+
def resolve_ref(self, ref: TableRef) -> ResolvedRef:
|
|
115
|
+
"""Return the branch and optional snapshot selected by ``ref``."""
|
|
116
|
+
resolved, _ = self._resolve_ref(ref)
|
|
117
|
+
return resolved
|
|
118
|
+
|
|
119
|
+
# --- Internals ---
|
|
120
|
+
|
|
121
|
+
def _client(self) -> GraphClient:
|
|
122
|
+
if self._graph_client is None:
|
|
123
|
+
self._graph_client = get_graph_client()
|
|
124
|
+
return self._graph_client
|
|
125
|
+
|
|
126
|
+
def _resolve_ref(self, ref: TableRef) -> tuple[ResolvedRef, Table | None]:
|
|
127
|
+
if ref.version_ref is None:
|
|
128
|
+
return ResolvedRef(branch=ref.branch), None
|
|
129
|
+
if ref.version_ref in self._branches():
|
|
130
|
+
return ResolvedRef(branch=ref.version_ref), None
|
|
131
|
+
table = self._load(ref, ref.branch)
|
|
132
|
+
snapshot_id = self._snapshot_id(table, ref, ref.version_ref)
|
|
133
|
+
return ResolvedRef(branch=ref.branch, snapshot_id=snapshot_id), table
|
|
134
|
+
|
|
135
|
+
def _branches(self) -> frozenset[str]:
|
|
136
|
+
if self._branch_names is None:
|
|
137
|
+
self._branch_names = frozenset(b["name"] for b in self._client().list_branches())
|
|
138
|
+
return self._branch_names
|
|
139
|
+
|
|
140
|
+
def _load(self, ref: TableRef, branch: str) -> Table:
|
|
141
|
+
return self._catalog_factory(branch).load_table(f"{ref.namespace}.{ref.table}")
|
|
142
|
+
|
|
143
|
+
def _snapshot_id(self, table: Table, ref: TableRef, version_ref: str) -> int:
|
|
144
|
+
if version_ref.isdigit() and len(version_ref) >= _SNAPSHOT_ID_MIN_DIGITS:
|
|
145
|
+
return int(version_ref)
|
|
146
|
+
|
|
147
|
+
timestamp_ms = _timestamp_ms(version_ref)
|
|
148
|
+
if timestamp_ms is not None:
|
|
149
|
+
snapshot = table.snapshot_as_of_timestamp(timestamp_ms)
|
|
150
|
+
if snapshot is None:
|
|
151
|
+
at = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc).isoformat()
|
|
152
|
+
raise ValueError(f"{ref.namespace}.{ref.table} has no snapshot at or before {at}")
|
|
153
|
+
return snapshot.snapshot_id
|
|
154
|
+
|
|
155
|
+
return int(self._named_version(ref, version_ref))
|
|
156
|
+
|
|
157
|
+
def _named_version(self, ref: TableRef, version_ref: str) -> str:
|
|
158
|
+
"""The snapshot id a named version pins, from the dataset's graph."""
|
|
159
|
+
iri = dataset_iri(ref)
|
|
160
|
+
graph = parse_ntriples(self._client().get_graph(ref.branch, iri, accept=N_TRIPLES))
|
|
161
|
+
for version in graph_to_description(graph, iri).versions:
|
|
162
|
+
if version.version != version_ref:
|
|
163
|
+
continue
|
|
164
|
+
if version.snapshot_id is None:
|
|
165
|
+
raise ValueError(f"version {version_ref!r} of {iri} pins no Iceberg snapshot")
|
|
166
|
+
return version.snapshot_id
|
|
167
|
+
raise ValueError(f"{version_ref!r} is not a branch, snapshot id, timestamp, or version of {iri}")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def resolve_to_parquet(ref: TableRef, resolver: Resolver | None = None) -> BinaryIO:
|
|
171
|
+
"""Resolve `ref` and return its rows as a parquet byte stream."""
|
|
172
|
+
import pyarrow.parquet as pq
|
|
173
|
+
|
|
174
|
+
arrow_table = (resolver or Resolver()).resolve(ref)
|
|
175
|
+
buf = io.BytesIO()
|
|
176
|
+
pq.write_table(arrow_table, buf)
|
|
177
|
+
buf.seek(0)
|
|
178
|
+
return buf
|
sunstone_data/url.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Parse sunstone: URLs into TableRef instances."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
from sunstone_data.query import TableRef
|
|
8
|
+
|
|
9
|
+
_OLD_FORM_MESSAGE = "sunstone:// (with //) is no longer supported; use sunstone:namespace/table@ref"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_sunstone_url(url: str) -> TableRef:
|
|
13
|
+
"""Parse a sunstone: URL into a TableRef.
|
|
14
|
+
|
|
15
|
+
Format: sunstone:namespace/table[@ref], where namespace may itself have
|
|
16
|
+
multiple levels: sunstone:level1/level2/.../table[@ref]. The last path
|
|
17
|
+
segment is the table name; every segment before it is a namespace level.
|
|
18
|
+
|
|
19
|
+
There is no authority component (no ``//``): both the namespace and the
|
|
20
|
+
table live in the path. See ``docs/adr/0001-sunstone-url-scheme.md``.
|
|
21
|
+
|
|
22
|
+
Examples:
|
|
23
|
+
sunstone:kaggle/emissions_data -> latest on main
|
|
24
|
+
sunstone:kaggle/emissions_data@v1.0 -> tagged version
|
|
25
|
+
sunstone:kaggle/emissions_data@branch -> branch
|
|
26
|
+
sunstone:kaggle/emissions_data@2026-01-25 -> timestamp
|
|
27
|
+
sunstone:un/health/cases -> namespace "un.health", table "cases"
|
|
28
|
+
"""
|
|
29
|
+
# Reject the old authority-based form with a migration hint.
|
|
30
|
+
if url.startswith("sunstone://"):
|
|
31
|
+
raise ValueError(_OLD_FORM_MESSAGE)
|
|
32
|
+
|
|
33
|
+
parsed = urlparse(url)
|
|
34
|
+
|
|
35
|
+
if parsed.scheme != "sunstone":
|
|
36
|
+
raise ValueError(f"Expected sunstone: URL, got {parsed.scheme}:")
|
|
37
|
+
|
|
38
|
+
# A non-empty netloc means an authority slipped in (old form); reject it.
|
|
39
|
+
if parsed.netloc:
|
|
40
|
+
raise ValueError(_OLD_FORM_MESSAGE)
|
|
41
|
+
|
|
42
|
+
# The version ref uses @ref, not ?ref; reject query strings and fragments
|
|
43
|
+
# as a duplicate-identity hazard (see ADR 0001 §3).
|
|
44
|
+
if parsed.query or parsed.fragment:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"Query strings and fragments are not allowed in sunstone: URLs "
|
|
47
|
+
f"(the version ref uses @ref, not ?ref): got {url!r}"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
# Namespace and table both live in the path. Require at least two
|
|
51
|
+
# segments: one or more namespace levels, then the table.
|
|
52
|
+
# namespace/.../table[@ref]. The last segment is the table; everything
|
|
53
|
+
# before it is a namespace level.
|
|
54
|
+
path = parsed.path
|
|
55
|
+
segments = path.split("/")
|
|
56
|
+
if len(segments) < 2 or any(not segment for segment in segments):
|
|
57
|
+
raise ValueError(f"Expected format: sunstone:namespace/table[@ref], got {url}")
|
|
58
|
+
|
|
59
|
+
namespace = ".".join(segments[:-1])
|
|
60
|
+
table_part = segments[-1]
|
|
61
|
+
|
|
62
|
+
# Split table@ref. A trailing @ with no ref (a typo) is rejected rather
|
|
63
|
+
# than silently treated as "no ref".
|
|
64
|
+
if "@" in table_part:
|
|
65
|
+
table, ref = table_part.split("@", 1)
|
|
66
|
+
if not ref:
|
|
67
|
+
raise ValueError(f"Version ref after @ must not be empty: got {url!r}")
|
|
68
|
+
else:
|
|
69
|
+
table = table_part
|
|
70
|
+
ref = None
|
|
71
|
+
|
|
72
|
+
# Build a ref string that TableRef.parse understands
|
|
73
|
+
if ref:
|
|
74
|
+
ref_string = f"{namespace}.{table}@{ref}"
|
|
75
|
+
else:
|
|
76
|
+
ref_string = f"{namespace}.{table}"
|
|
77
|
+
|
|
78
|
+
return TableRef.parse(ref_string)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Utility modules for import pipeline."""
|
|
2
|
+
|
|
3
|
+
from sunstone_data.utils.checksum import compute_checksum
|
|
4
|
+
from sunstone_data.utils.encoding import detect_encoding
|
|
5
|
+
from sunstone_data.utils.naming import derive_namespace, sanitize_table_name
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"compute_checksum",
|
|
9
|
+
"detect_encoding",
|
|
10
|
+
"derive_namespace",
|
|
11
|
+
"sanitize_table_name",
|
|
12
|
+
]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""File checksum utilities."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def compute_checksum(file_path: Path) -> str:
|
|
8
|
+
"""Compute SHA256 checksum of file using streaming.
|
|
9
|
+
|
|
10
|
+
Uses Python 3.11+ hashlib.file_digest for efficiency.
|
|
11
|
+
"""
|
|
12
|
+
with open(file_path, "rb") as f:
|
|
13
|
+
digest = hashlib.file_digest(f, "sha256")
|
|
14
|
+
return digest.hexdigest()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Encoding detection utilities."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from charset_normalizer import from_path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def detect_encoding(file_path: Path) -> str:
|
|
9
|
+
"""Detect file encoding using charset-normalizer.
|
|
10
|
+
|
|
11
|
+
Returns detected encoding or 'utf-8' as fallback.
|
|
12
|
+
"""
|
|
13
|
+
result = from_path(file_path)
|
|
14
|
+
if result and result.best():
|
|
15
|
+
return result.best().encoding
|
|
16
|
+
return "utf-8"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Table name sanitization utilities."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from urllib.parse import urlparse
|
|
5
|
+
|
|
6
|
+
from slugify import slugify
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def sanitize_table_name(filename: str) -> str:
|
|
10
|
+
"""Convert filename to valid Iceberg table name.
|
|
11
|
+
|
|
12
|
+
Rules:
|
|
13
|
+
- Remove extension
|
|
14
|
+
- Replace spaces/special chars with underscores
|
|
15
|
+
- Lowercase
|
|
16
|
+
- Ensure starts with letter (prepend 't_' if needed)
|
|
17
|
+
- Max 128 chars (Iceberg limit)
|
|
18
|
+
"""
|
|
19
|
+
name = Path(filename).stem
|
|
20
|
+
slug = slugify(name, separator="_", lowercase=True)
|
|
21
|
+
|
|
22
|
+
if slug and not slug[0].isalpha():
|
|
23
|
+
slug = f"t_{slug}"
|
|
24
|
+
|
|
25
|
+
return slug[:128] if slug else "imported_data"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def derive_namespace(source: str) -> str:
|
|
29
|
+
"""Derive namespace from source path or URL.
|
|
30
|
+
|
|
31
|
+
Returns domain for URLs (e.g., 'kaggle' for kaggle.com),
|
|
32
|
+
'local' for local files.
|
|
33
|
+
"""
|
|
34
|
+
if source.startswith(("http://", "https://")):
|
|
35
|
+
parsed = urlparse(source)
|
|
36
|
+
domain = parsed.netloc.split(".")[-2] if "." in parsed.netloc else parsed.netloc
|
|
37
|
+
return slugify(domain, separator="_", lowercase=True) or "web"
|
|
38
|
+
return "local"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Schema evolution checks for datasets."""
|
|
2
|
+
|
|
3
|
+
from sunstone_data.versioning.schema_evolution import (
|
|
4
|
+
EvolutionType,
|
|
5
|
+
SchemaChange,
|
|
6
|
+
detect_schema_changes,
|
|
7
|
+
is_safe_type_promotion,
|
|
8
|
+
validate_schema_evolution,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"EvolutionType",
|
|
13
|
+
"SchemaChange",
|
|
14
|
+
"detect_schema_changes",
|
|
15
|
+
"is_safe_type_promotion",
|
|
16
|
+
"validate_schema_evolution",
|
|
17
|
+
]
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Schema evolution detection and safety validation.
|
|
2
|
+
|
|
3
|
+
This module provides tools for detecting schema changes between Arrow schemas
|
|
4
|
+
and validating that changes are safe (e.g., type widening is allowed, narrowing
|
|
5
|
+
requires explicit confirmation).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from enum import Enum
|
|
10
|
+
from typing import Optional
|
|
11
|
+
|
|
12
|
+
import pyarrow as pa
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class EvolutionType(Enum):
|
|
16
|
+
"""Type of schema change detected."""
|
|
17
|
+
|
|
18
|
+
ADD_COLUMN = "add_column"
|
|
19
|
+
TYPE_WIDENING = "type_widening" # Safe: int32 -> int64, float32 -> float64
|
|
20
|
+
TYPE_NARROWING = "type_narrowing" # Unsafe: requires --force-schema
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class SchemaChange:
|
|
25
|
+
"""Detected schema change between existing and new schema."""
|
|
26
|
+
|
|
27
|
+
evolution_type: EvolutionType
|
|
28
|
+
column_name: str
|
|
29
|
+
old_type: Optional[str] # None for ADD_COLUMN
|
|
30
|
+
new_type: str
|
|
31
|
+
is_safe: bool
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# Safe type promotions that don't lose data
|
|
35
|
+
# Format: (old_type_check, new_type_check)
|
|
36
|
+
SAFE_PROMOTIONS = [
|
|
37
|
+
# Integer widening
|
|
38
|
+
(pa.types.is_int8, pa.types.is_int16),
|
|
39
|
+
(pa.types.is_int8, pa.types.is_int32),
|
|
40
|
+
(pa.types.is_int8, pa.types.is_int64),
|
|
41
|
+
(pa.types.is_int16, pa.types.is_int32),
|
|
42
|
+
(pa.types.is_int16, pa.types.is_int64),
|
|
43
|
+
(pa.types.is_int32, pa.types.is_int64),
|
|
44
|
+
# Unsigned integer widening
|
|
45
|
+
(pa.types.is_uint8, pa.types.is_uint16),
|
|
46
|
+
(pa.types.is_uint8, pa.types.is_uint32),
|
|
47
|
+
(pa.types.is_uint8, pa.types.is_uint64),
|
|
48
|
+
(pa.types.is_uint16, pa.types.is_uint32),
|
|
49
|
+
(pa.types.is_uint16, pa.types.is_uint64),
|
|
50
|
+
(pa.types.is_uint32, pa.types.is_uint64),
|
|
51
|
+
# Float widening
|
|
52
|
+
(pa.types.is_float32, pa.types.is_float64),
|
|
53
|
+
# Numeric to string (always safe - can represent any number)
|
|
54
|
+
(pa.types.is_integer, pa.types.is_string),
|
|
55
|
+
(pa.types.is_floating, pa.types.is_string),
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def is_safe_type_promotion(old_type: pa.DataType, new_type: pa.DataType) -> bool:
|
|
60
|
+
"""Check if type change is a safe widening operation.
|
|
61
|
+
|
|
62
|
+
Safe promotions don't lose data or precision:
|
|
63
|
+
- int32 -> int64 (widening)
|
|
64
|
+
- float32 -> float64 (widening)
|
|
65
|
+
- int/float -> string (can represent any number as text)
|
|
66
|
+
- string <-> large_string (equivalent, just different storage)
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
old_type: Existing PyArrow data type.
|
|
70
|
+
new_type: New PyArrow data type.
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
True if the change is safe, False if it risks data loss.
|
|
74
|
+
"""
|
|
75
|
+
# Same type is always safe
|
|
76
|
+
if old_type.equals(new_type):
|
|
77
|
+
return True
|
|
78
|
+
|
|
79
|
+
# String type variations are equivalent (string, large_string)
|
|
80
|
+
# PyArrow infers different string types based on data size, but they're compatible
|
|
81
|
+
string_types = (
|
|
82
|
+
pa.types.is_string,
|
|
83
|
+
pa.types.is_large_string,
|
|
84
|
+
)
|
|
85
|
+
is_old_string = any(check(old_type) for check in string_types)
|
|
86
|
+
is_new_string = any(check(new_type) for check in string_types)
|
|
87
|
+
if is_old_string and is_new_string:
|
|
88
|
+
return True
|
|
89
|
+
|
|
90
|
+
# Check against safe promotions table
|
|
91
|
+
for old_check, new_check in SAFE_PROMOTIONS:
|
|
92
|
+
if old_check(old_type) and new_check(new_type):
|
|
93
|
+
return True
|
|
94
|
+
|
|
95
|
+
# Decimal precision increase with same scale is safe
|
|
96
|
+
if pa.types.is_decimal(old_type) and pa.types.is_decimal(new_type):
|
|
97
|
+
# PyArrow decimal types have precision and scale attributes
|
|
98
|
+
if new_type.precision >= old_type.precision and new_type.scale == old_type.scale:
|
|
99
|
+
return True
|
|
100
|
+
|
|
101
|
+
return False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def detect_schema_changes(existing: pa.Schema, new: pa.Schema) -> list[SchemaChange]:
|
|
105
|
+
"""Detect schema changes between existing and new schemas.
|
|
106
|
+
|
|
107
|
+
Identifies:
|
|
108
|
+
- New columns (ADD_COLUMN)
|
|
109
|
+
- Type changes (TYPE_WIDENING or TYPE_NARROWING)
|
|
110
|
+
|
|
111
|
+
Note: Column removal is not detected (not supported in this phase).
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
existing: Current schema in the table.
|
|
115
|
+
new: New schema from incoming data.
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
List of SchemaChange objects describing all differences.
|
|
119
|
+
"""
|
|
120
|
+
changes: list[SchemaChange] = []
|
|
121
|
+
existing_fields = {f.name: f for f in existing}
|
|
122
|
+
new_fields = {f.name: f for f in new}
|
|
123
|
+
|
|
124
|
+
# Detect new columns
|
|
125
|
+
for name, field in new_fields.items():
|
|
126
|
+
if name not in existing_fields:
|
|
127
|
+
changes.append(
|
|
128
|
+
SchemaChange(
|
|
129
|
+
evolution_type=EvolutionType.ADD_COLUMN,
|
|
130
|
+
column_name=name,
|
|
131
|
+
old_type=None,
|
|
132
|
+
new_type=str(field.type),
|
|
133
|
+
is_safe=True, # Adding columns is always safe
|
|
134
|
+
)
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# Detect type changes
|
|
138
|
+
for name, new_field in new_fields.items():
|
|
139
|
+
if name in existing_fields:
|
|
140
|
+
old_field = existing_fields[name]
|
|
141
|
+
if not old_field.type.equals(new_field.type):
|
|
142
|
+
safe = is_safe_type_promotion(old_field.type, new_field.type)
|
|
143
|
+
changes.append(
|
|
144
|
+
SchemaChange(
|
|
145
|
+
evolution_type=(EvolutionType.TYPE_WIDENING if safe else EvolutionType.TYPE_NARROWING),
|
|
146
|
+
column_name=name,
|
|
147
|
+
old_type=str(old_field.type),
|
|
148
|
+
new_type=str(new_field.type),
|
|
149
|
+
is_safe=safe,
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
return changes
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def validate_schema_evolution(changes: list[SchemaChange], force: bool = False) -> None:
|
|
157
|
+
"""Validate that schema changes are safe, or force is enabled.
|
|
158
|
+
|
|
159
|
+
Args:
|
|
160
|
+
changes: List of detected schema changes.
|
|
161
|
+
force: If True, allow unsafe changes.
|
|
162
|
+
|
|
163
|
+
Raises:
|
|
164
|
+
ValueError: If unsafe changes detected and force=False.
|
|
165
|
+
"""
|
|
166
|
+
unsafe_changes = [c for c in changes if not c.is_safe]
|
|
167
|
+
|
|
168
|
+
if unsafe_changes and not force:
|
|
169
|
+
# Build clear error message listing unsafe changes
|
|
170
|
+
change_details = []
|
|
171
|
+
for change in unsafe_changes:
|
|
172
|
+
change_details.append(f" - {change.column_name}: {change.old_type} -> {change.new_type}")
|
|
173
|
+
|
|
174
|
+
error_msg = (
|
|
175
|
+
"Unsafe schema changes detected:\n"
|
|
176
|
+
+ "\n".join(change_details)
|
|
177
|
+
+ "\n\nThese changes may cause data loss. "
|
|
178
|
+
+ "Use --force-schema to override."
|
|
179
|
+
)
|
|
180
|
+
raise ValueError(error_msg)
|