sunstone-data-platform 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. sunstone_data/__init__.py +36 -0
  2. sunstone_data/_version.py +24 -0
  3. sunstone_data/auth/__init__.py +43 -0
  4. sunstone_data/auth/client_creds.py +239 -0
  5. sunstone_data/auth/pkce.py +307 -0
  6. sunstone_data/auth/store.py +97 -0
  7. sunstone_data/catalog.py +175 -0
  8. sunstone_data/cli/__init__.py +5 -0
  9. sunstone_data/cli/_app.py +26 -0
  10. sunstone_data/cli/_errors.py +31 -0
  11. sunstone_data/cli/commands/__init__.py +22 -0
  12. sunstone_data/cli/commands/data_auth.py +120 -0
  13. sunstone_data/cli/commands/data_import.py +157 -0
  14. sunstone_data/cli/commands/data_info.py +166 -0
  15. sunstone_data/cli/commands/data_publish.py +69 -0
  16. sunstone_data/cli/commands/ls.py +49 -0
  17. sunstone_data/cli/commands/prov.py +58 -0
  18. sunstone_data/cli/commands/refs.py +142 -0
  19. sunstone_data/cli/commands/show.py +139 -0
  20. sunstone_data/cli/commands/snapshots.py +77 -0
  21. sunstone_data/cli/commands/sparql.py +68 -0
  22. sunstone_data/cli/main.py +30 -0
  23. sunstone_data/config.py +250 -0
  24. sunstone_data/discovery.py +317 -0
  25. sunstone_data/env_commands.py +97 -0
  26. sunstone_data/env_section.py +86 -0
  27. sunstone_data/graph/__init__.py +36 -0
  28. sunstone_data/graph/client.py +328 -0
  29. sunstone_data/graph/dataset.py +438 -0
  30. sunstone_data/graph/terms.py +34 -0
  31. sunstone_data/identifiers.py +155 -0
  32. sunstone_data/import_/__init__.py +28 -0
  33. sunstone_data/import_/fetcher.py +259 -0
  34. sunstone_data/import_/owner.py +130 -0
  35. sunstone_data/import_/parser.py +492 -0
  36. sunstone_data/import_/pipeline.py +275 -0
  37. sunstone_data/import_/storage.py +135 -0
  38. sunstone_data/metadata/__init__.py +41 -0
  39. sunstone_data/metadata/builder.py +276 -0
  40. sunstone_data/metadata/models.py +87 -0
  41. sunstone_data/metadata/provenance.py +171 -0
  42. sunstone_data/metadata/provenance_rdf.py +102 -0
  43. sunstone_data/plugin.py +62 -0
  44. sunstone_data/publish.py +134 -0
  45. sunstone_data/query.py +624 -0
  46. sunstone_data/resolve.py +178 -0
  47. sunstone_data/url.py +78 -0
  48. sunstone_data/utils/__init__.py +12 -0
  49. sunstone_data/utils/checksum.py +14 -0
  50. sunstone_data/utils/encoding.py +16 -0
  51. sunstone_data/utils/naming.py +38 -0
  52. sunstone_data/versioning/__init__.py +17 -0
  53. sunstone_data/versioning/schema_evolution.py +180 -0
  54. sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
  55. sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
  56. sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
  57. sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
  58. sunstone_pipeline/__init__.py +1 -0
  59. sunstone_pipeline/assets/__init__.py +11 -0
  60. sunstone_pipeline/assets/mirrors.py +123 -0
  61. sunstone_pipeline/assets/transforms.py +74 -0
  62. sunstone_pipeline/definitions.py +25 -0
  63. sunstone_pipeline/io_manager.py +56 -0
  64. sunstone_pipeline/resources/__init__.py +6 -0
  65. sunstone_pipeline/resources/iceberg.py +29 -0
  66. sunstone_pipeline/resources/refs.py +88 -0
@@ -0,0 +1,178 @@
1
+ """Resolve a parsed `sunstone:` URL to Iceberg table data (spec 180 §7).
2
+
3
+ `url.parse` splits `sunstone:un/health/cases@v1.0` into a `TableRef` but does
4
+ not say what `v1.0` *is*. This module makes that call, then reads the data:
5
+
6
+ ref = parse_sunstone_url(url)
7
+ -> GET /branches (cached per Resolver)
8
+ -> ref.version_ref is a known branch? yes -> branch selector
9
+ no -> version selector
10
+ -> RestCatalog(".../iceberg/{branch}/").load_table("un.health.cases")
11
+ -> apply selector -> .to_arrow() -> parquet -> BytesIO
12
+
13
+ Branch-versus-version is decided by asking the graph server which names are
14
+ branches, not by pattern-matching the ref. Pattern matching cannot tell a
15
+ branch named `v2` from a named version called `v2`.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import io
21
+ from dataclasses import dataclass
22
+ from datetime import datetime, timezone
23
+ from typing import BinaryIO, Callable
24
+
25
+ import pyarrow as pa
26
+ from dateutil import parser as date_parser
27
+ from pyiceberg.catalog.rest import RestCatalog
28
+ from pyiceberg.table import Table
29
+
30
+ from sunstone_data.catalog import get_catalog
31
+ from sunstone_data.config import get_settings
32
+ from sunstone_data.graph.client import N_TRIPLES, GraphClient, get_graph_client
33
+ from sunstone_data.graph.dataset import graph_to_description, parse_ntriples
34
+ from sunstone_data.query import TableRef
35
+
36
+ #: A ref made only of digits and at least this long is an Iceberg snapshot id.
37
+ #: Snapshot ids are epoch-millisecond-seeded 64-bit numbers, so they are always
38
+ #: this wide; no plausible date or version label is.
39
+ _SNAPSHOT_ID_MIN_DIGITS = 13
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class ResolvedRef:
44
+ """The catalog branch and optional Iceberg snapshot selected by a ref."""
45
+
46
+ branch: str
47
+ snapshot_id: int | None = None
48
+
49
+
50
+ def dataset_iri(ref: TableRef) -> str:
51
+ """The IRI of the dataset's named graph.
52
+
53
+ Mirrors the catalog's own minting rule (`iceberg-catalog`'s
54
+ `naming.rs::table_iri`): `{data_base_iri}/{ns levels joined by /}/{table}`.
55
+ """
56
+ base = get_settings().data_base_iri.rstrip("/")
57
+ return f"{base}/{ref.namespace.replace('.', '/')}/{ref.table}"
58
+
59
+
60
+ def _timestamp_ms(value: str) -> int | None:
61
+ """Parse `value` as an instant in epoch milliseconds, or None if it isn't one.
62
+
63
+ A parsed value with no timezone is read as UTC. Reading it as local time
64
+ would make the same URL resolve to different snapshots on machines in
65
+ different timezones — refs travel between machines, so local time is never
66
+ the right default here.
67
+ """
68
+ try:
69
+ parsed = date_parser.parse(value)
70
+ except (ValueError, TypeError, OverflowError):
71
+ return None
72
+ if parsed.tzinfo is None:
73
+ parsed = parsed.replace(tzinfo=timezone.utc)
74
+ return int(parsed.timestamp() * 1000)
75
+
76
+
77
+ class Resolver:
78
+ """Turns a `TableRef`'s `version_ref` into a branch or an Iceberg snapshot.
79
+
80
+ `GET /branches` is fetched at most once per instance. Construct one per
81
+ logical resolve rather than sharing one across unrelated calls, so a
82
+ branch created meanwhile is still seen.
83
+
84
+ Args:
85
+ graph_client: Client for the graph server. Defaults to
86
+ `get_graph_client()`, built lazily on first use.
87
+ catalog_factory: Builds a catalog for a branch. Defaults to
88
+ `get_catalog`.
89
+ """
90
+
91
+ def __init__(
92
+ self,
93
+ graph_client: GraphClient | None = None,
94
+ catalog_factory: Callable[[str], RestCatalog] = get_catalog,
95
+ ):
96
+ self._graph_client = graph_client
97
+ self._catalog_factory = catalog_factory
98
+ self._branch_names: frozenset[str] | None = None
99
+
100
+ def resolve(self, ref: TableRef) -> pa.Table:
101
+ """Read the table the ref points at.
102
+
103
+ Raises:
104
+ ValueError: The ref is not a branch, snapshot id, resolvable
105
+ timestamp, or named version of this table.
106
+ """
107
+ resolved, table = self._resolve_ref(ref)
108
+ if table is None:
109
+ table = self._load(ref, resolved.branch)
110
+ if resolved.snapshot_id is None:
111
+ return table.scan().to_arrow()
112
+ return table.scan(snapshot_id=resolved.snapshot_id).to_arrow()
113
+
114
+ def resolve_ref(self, ref: TableRef) -> ResolvedRef:
115
+ """Return the branch and optional snapshot selected by ``ref``."""
116
+ resolved, _ = self._resolve_ref(ref)
117
+ return resolved
118
+
119
+ # --- Internals ---
120
+
121
+ def _client(self) -> GraphClient:
122
+ if self._graph_client is None:
123
+ self._graph_client = get_graph_client()
124
+ return self._graph_client
125
+
126
+ def _resolve_ref(self, ref: TableRef) -> tuple[ResolvedRef, Table | None]:
127
+ if ref.version_ref is None:
128
+ return ResolvedRef(branch=ref.branch), None
129
+ if ref.version_ref in self._branches():
130
+ return ResolvedRef(branch=ref.version_ref), None
131
+ table = self._load(ref, ref.branch)
132
+ snapshot_id = self._snapshot_id(table, ref, ref.version_ref)
133
+ return ResolvedRef(branch=ref.branch, snapshot_id=snapshot_id), table
134
+
135
+ def _branches(self) -> frozenset[str]:
136
+ if self._branch_names is None:
137
+ self._branch_names = frozenset(b["name"] for b in self._client().list_branches())
138
+ return self._branch_names
139
+
140
+ def _load(self, ref: TableRef, branch: str) -> Table:
141
+ return self._catalog_factory(branch).load_table(f"{ref.namespace}.{ref.table}")
142
+
143
+ def _snapshot_id(self, table: Table, ref: TableRef, version_ref: str) -> int:
144
+ if version_ref.isdigit() and len(version_ref) >= _SNAPSHOT_ID_MIN_DIGITS:
145
+ return int(version_ref)
146
+
147
+ timestamp_ms = _timestamp_ms(version_ref)
148
+ if timestamp_ms is not None:
149
+ snapshot = table.snapshot_as_of_timestamp(timestamp_ms)
150
+ if snapshot is None:
151
+ at = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc).isoformat()
152
+ raise ValueError(f"{ref.namespace}.{ref.table} has no snapshot at or before {at}")
153
+ return snapshot.snapshot_id
154
+
155
+ return int(self._named_version(ref, version_ref))
156
+
157
+ def _named_version(self, ref: TableRef, version_ref: str) -> str:
158
+ """The snapshot id a named version pins, from the dataset's graph."""
159
+ iri = dataset_iri(ref)
160
+ graph = parse_ntriples(self._client().get_graph(ref.branch, iri, accept=N_TRIPLES))
161
+ for version in graph_to_description(graph, iri).versions:
162
+ if version.version != version_ref:
163
+ continue
164
+ if version.snapshot_id is None:
165
+ raise ValueError(f"version {version_ref!r} of {iri} pins no Iceberg snapshot")
166
+ return version.snapshot_id
167
+ raise ValueError(f"{version_ref!r} is not a branch, snapshot id, timestamp, or version of {iri}")
168
+
169
+
170
+ def resolve_to_parquet(ref: TableRef, resolver: Resolver | None = None) -> BinaryIO:
171
+ """Resolve `ref` and return its rows as a parquet byte stream."""
172
+ import pyarrow.parquet as pq
173
+
174
+ arrow_table = (resolver or Resolver()).resolve(ref)
175
+ buf = io.BytesIO()
176
+ pq.write_table(arrow_table, buf)
177
+ buf.seek(0)
178
+ return buf
sunstone_data/url.py ADDED
@@ -0,0 +1,78 @@
1
+ """Parse sunstone: URLs into TableRef instances."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from urllib.parse import urlparse
6
+
7
+ from sunstone_data.query import TableRef
8
+
9
+ _OLD_FORM_MESSAGE = "sunstone:// (with //) is no longer supported; use sunstone:namespace/table@ref"
10
+
11
+
12
+ def parse_sunstone_url(url: str) -> TableRef:
13
+ """Parse a sunstone: URL into a TableRef.
14
+
15
+ Format: sunstone:namespace/table[@ref], where namespace may itself have
16
+ multiple levels: sunstone:level1/level2/.../table[@ref]. The last path
17
+ segment is the table name; every segment before it is a namespace level.
18
+
19
+ There is no authority component (no ``//``): both the namespace and the
20
+ table live in the path. See ``docs/adr/0001-sunstone-url-scheme.md``.
21
+
22
+ Examples:
23
+ sunstone:kaggle/emissions_data -> latest on main
24
+ sunstone:kaggle/emissions_data@v1.0 -> tagged version
25
+ sunstone:kaggle/emissions_data@branch -> branch
26
+ sunstone:kaggle/emissions_data@2026-01-25 -> timestamp
27
+ sunstone:un/health/cases -> namespace "un.health", table "cases"
28
+ """
29
+ # Reject the old authority-based form with a migration hint.
30
+ if url.startswith("sunstone://"):
31
+ raise ValueError(_OLD_FORM_MESSAGE)
32
+
33
+ parsed = urlparse(url)
34
+
35
+ if parsed.scheme != "sunstone":
36
+ raise ValueError(f"Expected sunstone: URL, got {parsed.scheme}:")
37
+
38
+ # A non-empty netloc means an authority slipped in (old form); reject it.
39
+ if parsed.netloc:
40
+ raise ValueError(_OLD_FORM_MESSAGE)
41
+
42
+ # The version ref uses @ref, not ?ref; reject query strings and fragments
43
+ # as a duplicate-identity hazard (see ADR 0001 §3).
44
+ if parsed.query or parsed.fragment:
45
+ raise ValueError(
46
+ "Query strings and fragments are not allowed in sunstone: URLs "
47
+ f"(the version ref uses @ref, not ?ref): got {url!r}"
48
+ )
49
+
50
+ # Namespace and table both live in the path. Require at least two
51
+ # segments: one or more namespace levels, then the table.
52
+ # namespace/.../table[@ref]. The last segment is the table; everything
53
+ # before it is a namespace level.
54
+ path = parsed.path
55
+ segments = path.split("/")
56
+ if len(segments) < 2 or any(not segment for segment in segments):
57
+ raise ValueError(f"Expected format: sunstone:namespace/table[@ref], got {url}")
58
+
59
+ namespace = ".".join(segments[:-1])
60
+ table_part = segments[-1]
61
+
62
+ # Split table@ref. A trailing @ with no ref (a typo) is rejected rather
63
+ # than silently treated as "no ref".
64
+ if "@" in table_part:
65
+ table, ref = table_part.split("@", 1)
66
+ if not ref:
67
+ raise ValueError(f"Version ref after @ must not be empty: got {url!r}")
68
+ else:
69
+ table = table_part
70
+ ref = None
71
+
72
+ # Build a ref string that TableRef.parse understands
73
+ if ref:
74
+ ref_string = f"{namespace}.{table}@{ref}"
75
+ else:
76
+ ref_string = f"{namespace}.{table}"
77
+
78
+ return TableRef.parse(ref_string)
@@ -0,0 +1,12 @@
1
+ """Utility modules for import pipeline."""
2
+
3
+ from sunstone_data.utils.checksum import compute_checksum
4
+ from sunstone_data.utils.encoding import detect_encoding
5
+ from sunstone_data.utils.naming import derive_namespace, sanitize_table_name
6
+
7
+ __all__ = [
8
+ "compute_checksum",
9
+ "detect_encoding",
10
+ "derive_namespace",
11
+ "sanitize_table_name",
12
+ ]
@@ -0,0 +1,14 @@
1
+ """File checksum utilities."""
2
+
3
+ import hashlib
4
+ from pathlib import Path
5
+
6
+
7
+ def compute_checksum(file_path: Path) -> str:
8
+ """Compute SHA256 checksum of file using streaming.
9
+
10
+ Uses Python 3.11+ hashlib.file_digest for efficiency.
11
+ """
12
+ with open(file_path, "rb") as f:
13
+ digest = hashlib.file_digest(f, "sha256")
14
+ return digest.hexdigest()
@@ -0,0 +1,16 @@
1
+ """Encoding detection utilities."""
2
+
3
+ from pathlib import Path
4
+
5
+ from charset_normalizer import from_path
6
+
7
+
8
+ def detect_encoding(file_path: Path) -> str:
9
+ """Detect file encoding using charset-normalizer.
10
+
11
+ Returns detected encoding or 'utf-8' as fallback.
12
+ """
13
+ result = from_path(file_path)
14
+ if result and result.best():
15
+ return result.best().encoding
16
+ return "utf-8"
@@ -0,0 +1,38 @@
1
+ """Table name sanitization utilities."""
2
+
3
+ from pathlib import Path
4
+ from urllib.parse import urlparse
5
+
6
+ from slugify import slugify
7
+
8
+
9
+ def sanitize_table_name(filename: str) -> str:
10
+ """Convert filename to valid Iceberg table name.
11
+
12
+ Rules:
13
+ - Remove extension
14
+ - Replace spaces/special chars with underscores
15
+ - Lowercase
16
+ - Ensure starts with letter (prepend 't_' if needed)
17
+ - Max 128 chars (Iceberg limit)
18
+ """
19
+ name = Path(filename).stem
20
+ slug = slugify(name, separator="_", lowercase=True)
21
+
22
+ if slug and not slug[0].isalpha():
23
+ slug = f"t_{slug}"
24
+
25
+ return slug[:128] if slug else "imported_data"
26
+
27
+
28
+ def derive_namespace(source: str) -> str:
29
+ """Derive namespace from source path or URL.
30
+
31
+ Returns domain for URLs (e.g., 'kaggle' for kaggle.com),
32
+ 'local' for local files.
33
+ """
34
+ if source.startswith(("http://", "https://")):
35
+ parsed = urlparse(source)
36
+ domain = parsed.netloc.split(".")[-2] if "." in parsed.netloc else parsed.netloc
37
+ return slugify(domain, separator="_", lowercase=True) or "web"
38
+ return "local"
@@ -0,0 +1,17 @@
1
+ """Schema evolution checks for datasets."""
2
+
3
+ from sunstone_data.versioning.schema_evolution import (
4
+ EvolutionType,
5
+ SchemaChange,
6
+ detect_schema_changes,
7
+ is_safe_type_promotion,
8
+ validate_schema_evolution,
9
+ )
10
+
11
+ __all__ = [
12
+ "EvolutionType",
13
+ "SchemaChange",
14
+ "detect_schema_changes",
15
+ "is_safe_type_promotion",
16
+ "validate_schema_evolution",
17
+ ]
@@ -0,0 +1,180 @@
1
+ """Schema evolution detection and safety validation.
2
+
3
+ This module provides tools for detecting schema changes between Arrow schemas
4
+ and validating that changes are safe (e.g., type widening is allowed, narrowing
5
+ requires explicit confirmation).
6
+ """
7
+
8
+ from dataclasses import dataclass
9
+ from enum import Enum
10
+ from typing import Optional
11
+
12
+ import pyarrow as pa
13
+
14
+
15
+ class EvolutionType(Enum):
16
+ """Type of schema change detected."""
17
+
18
+ ADD_COLUMN = "add_column"
19
+ TYPE_WIDENING = "type_widening" # Safe: int32 -> int64, float32 -> float64
20
+ TYPE_NARROWING = "type_narrowing" # Unsafe: requires --force-schema
21
+
22
+
23
+ @dataclass
24
+ class SchemaChange:
25
+ """Detected schema change between existing and new schema."""
26
+
27
+ evolution_type: EvolutionType
28
+ column_name: str
29
+ old_type: Optional[str] # None for ADD_COLUMN
30
+ new_type: str
31
+ is_safe: bool
32
+
33
+
34
+ # Safe type promotions that don't lose data
35
+ # Format: (old_type_check, new_type_check)
36
+ SAFE_PROMOTIONS = [
37
+ # Integer widening
38
+ (pa.types.is_int8, pa.types.is_int16),
39
+ (pa.types.is_int8, pa.types.is_int32),
40
+ (pa.types.is_int8, pa.types.is_int64),
41
+ (pa.types.is_int16, pa.types.is_int32),
42
+ (pa.types.is_int16, pa.types.is_int64),
43
+ (pa.types.is_int32, pa.types.is_int64),
44
+ # Unsigned integer widening
45
+ (pa.types.is_uint8, pa.types.is_uint16),
46
+ (pa.types.is_uint8, pa.types.is_uint32),
47
+ (pa.types.is_uint8, pa.types.is_uint64),
48
+ (pa.types.is_uint16, pa.types.is_uint32),
49
+ (pa.types.is_uint16, pa.types.is_uint64),
50
+ (pa.types.is_uint32, pa.types.is_uint64),
51
+ # Float widening
52
+ (pa.types.is_float32, pa.types.is_float64),
53
+ # Numeric to string (always safe - can represent any number)
54
+ (pa.types.is_integer, pa.types.is_string),
55
+ (pa.types.is_floating, pa.types.is_string),
56
+ ]
57
+
58
+
59
+ def is_safe_type_promotion(old_type: pa.DataType, new_type: pa.DataType) -> bool:
60
+ """Check if type change is a safe widening operation.
61
+
62
+ Safe promotions don't lose data or precision:
63
+ - int32 -> int64 (widening)
64
+ - float32 -> float64 (widening)
65
+ - int/float -> string (can represent any number as text)
66
+ - string <-> large_string (equivalent, just different storage)
67
+
68
+ Args:
69
+ old_type: Existing PyArrow data type.
70
+ new_type: New PyArrow data type.
71
+
72
+ Returns:
73
+ True if the change is safe, False if it risks data loss.
74
+ """
75
+ # Same type is always safe
76
+ if old_type.equals(new_type):
77
+ return True
78
+
79
+ # String type variations are equivalent (string, large_string)
80
+ # PyArrow infers different string types based on data size, but they're compatible
81
+ string_types = (
82
+ pa.types.is_string,
83
+ pa.types.is_large_string,
84
+ )
85
+ is_old_string = any(check(old_type) for check in string_types)
86
+ is_new_string = any(check(new_type) for check in string_types)
87
+ if is_old_string and is_new_string:
88
+ return True
89
+
90
+ # Check against safe promotions table
91
+ for old_check, new_check in SAFE_PROMOTIONS:
92
+ if old_check(old_type) and new_check(new_type):
93
+ return True
94
+
95
+ # Decimal precision increase with same scale is safe
96
+ if pa.types.is_decimal(old_type) and pa.types.is_decimal(new_type):
97
+ # PyArrow decimal types have precision and scale attributes
98
+ if new_type.precision >= old_type.precision and new_type.scale == old_type.scale:
99
+ return True
100
+
101
+ return False
102
+
103
+
104
+ def detect_schema_changes(existing: pa.Schema, new: pa.Schema) -> list[SchemaChange]:
105
+ """Detect schema changes between existing and new schemas.
106
+
107
+ Identifies:
108
+ - New columns (ADD_COLUMN)
109
+ - Type changes (TYPE_WIDENING or TYPE_NARROWING)
110
+
111
+ Note: Column removal is not detected (not supported in this phase).
112
+
113
+ Args:
114
+ existing: Current schema in the table.
115
+ new: New schema from incoming data.
116
+
117
+ Returns:
118
+ List of SchemaChange objects describing all differences.
119
+ """
120
+ changes: list[SchemaChange] = []
121
+ existing_fields = {f.name: f for f in existing}
122
+ new_fields = {f.name: f for f in new}
123
+
124
+ # Detect new columns
125
+ for name, field in new_fields.items():
126
+ if name not in existing_fields:
127
+ changes.append(
128
+ SchemaChange(
129
+ evolution_type=EvolutionType.ADD_COLUMN,
130
+ column_name=name,
131
+ old_type=None,
132
+ new_type=str(field.type),
133
+ is_safe=True, # Adding columns is always safe
134
+ )
135
+ )
136
+
137
+ # Detect type changes
138
+ for name, new_field in new_fields.items():
139
+ if name in existing_fields:
140
+ old_field = existing_fields[name]
141
+ if not old_field.type.equals(new_field.type):
142
+ safe = is_safe_type_promotion(old_field.type, new_field.type)
143
+ changes.append(
144
+ SchemaChange(
145
+ evolution_type=(EvolutionType.TYPE_WIDENING if safe else EvolutionType.TYPE_NARROWING),
146
+ column_name=name,
147
+ old_type=str(old_field.type),
148
+ new_type=str(new_field.type),
149
+ is_safe=safe,
150
+ )
151
+ )
152
+
153
+ return changes
154
+
155
+
156
+ def validate_schema_evolution(changes: list[SchemaChange], force: bool = False) -> None:
157
+ """Validate that schema changes are safe, or force is enabled.
158
+
159
+ Args:
160
+ changes: List of detected schema changes.
161
+ force: If True, allow unsafe changes.
162
+
163
+ Raises:
164
+ ValueError: If unsafe changes detected and force=False.
165
+ """
166
+ unsafe_changes = [c for c in changes if not c.is_safe]
167
+
168
+ if unsafe_changes and not force:
169
+ # Build clear error message listing unsafe changes
170
+ change_details = []
171
+ for change in unsafe_changes:
172
+ change_details.append(f" - {change.column_name}: {change.old_type} -> {change.new_type}")
173
+
174
+ error_msg = (
175
+ "Unsafe schema changes detected:\n"
176
+ + "\n".join(change_details)
177
+ + "\n\nThese changes may cause data loss. "
178
+ + "Use --force-schema to override."
179
+ )
180
+ raise ValueError(error_msg)