spark-sql-migrations 0.0.8__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spark_sql_migrations-0.1.0/PKG-INFO +33 -0
- spark_sql_migrations-0.1.0/setup.py +59 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/spark_sql/spark_sql.py +266 -60
- spark_sql_migrations-0.1.0/spark_sql_migrations.egg-info/PKG-INFO +33 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/SOURCES.txt +0 -3
- spark_sql_migrations-0.0.8/PKG-INFO +0 -268
- spark_sql_migrations-0.0.8/setup.py +0 -303
- spark_sql_migrations-0.0.8/spark_sql_migrations/migrations_initial/first_01_intial_create_cat.sql +0 -9
- spark_sql_migrations-0.0.8/spark_sql_migrations/migrations_initial/first_02_intial_create_schema.sql +0 -4
- spark_sql_migrations-0.0.8/spark_sql_migrations/migrations_initial/first_03_intial_create_state_table.sql +0 -6
- spark_sql_migrations-0.0.8/spark_sql_migrations.egg-info/PKG-INFO +0 -268
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/LICENSE +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/MANIFEST.in +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/backend_shim.py +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/setup.cfg +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/__init__.py +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/migration_templates/default_migration_template.sql +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/py.typed +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/spark_sql/__init__.py +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/spark_utils.py +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/dependency_links.txt +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/entry_points.txt +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/namespace_packages.txt +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/requires.txt +0 -0
- {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spark_sql_migrations
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Chained, idempotent SQL migrations for Spark and Databricks.
|
|
5
|
+
Home-page: https://github.com/mcjug2015/spark_sql_migrations
|
|
6
|
+
Author: Victor Semenov
|
|
7
|
+
License: Apache-2.0
|
|
8
|
+
Project-URL: Source, https://github.com/mcjug2015/spark_sql_migrations
|
|
9
|
+
Project-URL: Issues, https://github.com/mcjug2015/spark_sql_migrations/issues
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: SQL
|
|
14
|
+
Classifier: Topic :: Database
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: Jinja2<4,>=3.1
|
|
19
|
+
Provides-Extra: local
|
|
20
|
+
Requires-Dist: pyspark[connect]<5,>=4.0; extra == "local"
|
|
21
|
+
Requires-Dist: delta-spark<5,>=4.0; extra == "local"
|
|
22
|
+
Provides-Extra: databricks
|
|
23
|
+
Requires-Dist: databricks-connect<19,>=18; extra == "databricks"
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: classifier
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: license
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: project-url
|
|
30
|
+
Dynamic: provides-extra
|
|
31
|
+
Dynamic: requires-dist
|
|
32
|
+
Dynamic: requires-python
|
|
33
|
+
Dynamic: summary
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
|
|
2
|
+
# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
|
|
3
|
+
# Target: //:dist
|
|
4
|
+
|
|
5
|
+
from setuptools import setup
|
|
6
|
+
|
|
7
|
+
setup(**{
|
|
8
|
+
'author': 'Victor Semenov',
|
|
9
|
+
'classifiers': [
|
|
10
|
+
'Development Status :: 3 - Alpha',
|
|
11
|
+
'Intended Audience :: Developers',
|
|
12
|
+
'Programming Language :: Python :: 3.12',
|
|
13
|
+
'Programming Language :: SQL',
|
|
14
|
+
'Topic :: Database',
|
|
15
|
+
'Typing :: Typed',
|
|
16
|
+
],
|
|
17
|
+
'description': 'Chained, idempotent SQL migrations for Spark and Databricks.',
|
|
18
|
+
'entry_points': {
|
|
19
|
+
'console_scripts': [
|
|
20
|
+
'spark-sql-migrations = spark_sql_migrations.spark_sql.spark_sql:cli',
|
|
21
|
+
],
|
|
22
|
+
},
|
|
23
|
+
'extras_require': {
|
|
24
|
+
'local': [
|
|
25
|
+
'pyspark[connect]>=4.0,<5',
|
|
26
|
+
'delta-spark>=4.0,<5',
|
|
27
|
+
],
|
|
28
|
+
'databricks': [
|
|
29
|
+
'databricks-connect>=18,<19',
|
|
30
|
+
],
|
|
31
|
+
},
|
|
32
|
+
'install_requires': (
|
|
33
|
+
'Jinja2<4,>=3.1',
|
|
34
|
+
),
|
|
35
|
+
'license': 'Apache-2.0',
|
|
36
|
+
'license_files': [
|
|
37
|
+
'LICENSE',
|
|
38
|
+
],
|
|
39
|
+
'name': 'spark_sql_migrations',
|
|
40
|
+
'namespace_packages': (
|
|
41
|
+
),
|
|
42
|
+
'package_data': {
|
|
43
|
+
'spark_sql_migrations': (
|
|
44
|
+
'migration_templates/default_migration_template.sql',
|
|
45
|
+
'py.typed',
|
|
46
|
+
),
|
|
47
|
+
},
|
|
48
|
+
'packages': (
|
|
49
|
+
'spark_sql_migrations',
|
|
50
|
+
'spark_sql_migrations.spark_sql',
|
|
51
|
+
),
|
|
52
|
+
'project_urls': {
|
|
53
|
+
'Source': 'https://github.com/mcjug2015/spark_sql_migrations',
|
|
54
|
+
'Issues': 'https://github.com/mcjug2015/spark_sql_migrations/issues',
|
|
55
|
+
},
|
|
56
|
+
'python_requires': '>=3.10',
|
|
57
|
+
'url': 'https://github.com/mcjug2015/spark_sql_migrations',
|
|
58
|
+
'version': '0.1.0',
|
|
59
|
+
})
|
|
@@ -27,9 +27,9 @@ import os
|
|
|
27
27
|
import re
|
|
28
28
|
import uuid
|
|
29
29
|
from dataclasses import dataclass
|
|
30
|
-
from typing import Any, Dict, List, Optional
|
|
30
|
+
from typing import Any, Dict, List, Optional, Set
|
|
31
31
|
|
|
32
|
-
from jinja2 import Environment, FileSystemLoader,
|
|
32
|
+
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
|
33
33
|
from pyspark.sql.session import SparkSession
|
|
34
34
|
from pyspark.sql.types import Row
|
|
35
35
|
|
|
@@ -38,10 +38,12 @@ from spark_sql_migrations.spark_utils import get_spark, is_dbr
|
|
|
38
38
|
logger = logging.getLogger(__name__)
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
# value returned by get_version_table_name from start of run should have been passed in to you
|
|
42
|
+
__DO_NOT_USE_VERSION_TABLE = "_spark_migrations_version"
|
|
43
|
+
# what `history` prints in place of the revision a root migration follows
|
|
44
|
+
BASE_LABEL = "<base>"
|
|
42
45
|
|
|
43
46
|
PACKAGE_NAME = "spark_sql_migrations"
|
|
44
|
-
INITIAL_MIGRATIONS_PACKAGE_PATH = "migrations_initial"
|
|
45
47
|
# where a run's rendered client migrations land, under its output folder
|
|
46
48
|
CLIENT_OUTPUT_DIRNAME = "migrations"
|
|
47
49
|
TEMPLATES_PACKAGE_PATH = "migration_templates"
|
|
@@ -58,6 +60,30 @@ class Migration:
|
|
|
58
60
|
template_name: str
|
|
59
61
|
|
|
60
62
|
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class ChainSummary:
|
|
65
|
+
"""a directory's chain as `history` found it, forks and all.
|
|
66
|
+
|
|
67
|
+
``ordered`` is oldest first and holds every migration in the directory, so a
|
|
68
|
+
listing built from it never silently drops a file. Everything else is what
|
|
69
|
+
makes the chain unapplicable: more than one ``heads`` entry is a split head,
|
|
70
|
+
a ``branch_points`` entry is where it split, more than one ``roots`` entry is
|
|
71
|
+
a second chain in the same directory, and ``orphans`` are files no walk from
|
|
72
|
+
any root reaches -- a cycle, in practice.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
ordered: List[Migration]
|
|
76
|
+
heads: List[Migration]
|
|
77
|
+
roots: List[Migration]
|
|
78
|
+
branch_points: Dict[str, List[Migration]]
|
|
79
|
+
orphans: List[Migration]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_version_table_name(spark: SparkSession) -> str:
|
|
83
|
+
# only correct ar start of run
|
|
84
|
+
return f"{spark.catalog.currentCatalog()}.{spark.catalog.currentDatabase()}._spark_migrations_version"
|
|
85
|
+
|
|
86
|
+
|
|
61
87
|
def get_default_template_path() -> str:
|
|
62
88
|
"""the template this package ships; callers may pass their own instead."""
|
|
63
89
|
return os.path.join(os.path.dirname(__file__), "..", TEMPLATES_PACKAGE_PATH, DEFAULT_TEMPLATE_NAME)
|
|
@@ -137,22 +163,32 @@ def get_ordered_migration_objs(migration_objs: List[Migration]) -> List[Migratio
|
|
|
137
163
|
return ordered
|
|
138
164
|
|
|
139
165
|
|
|
140
|
-
def
|
|
141
|
-
"""every
|
|
166
|
+
def read_migrations_dir(migrations_dir: str) -> List[Migration]:
|
|
167
|
+
"""every .sql file in one directory, parsed, in filename order and unwalked.
|
|
142
168
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
at all.
|
|
169
|
+
the headers are all this reads, so a chain that forks or points at nothing
|
|
170
|
+
comes back whole -- it is `get_ordered_migration_objs` that insists on a
|
|
171
|
+
single walkable chain, and `history` that reports one which is not.
|
|
147
172
|
"""
|
|
148
173
|
if not os.path.isdir(migrations_dir):
|
|
149
174
|
raise ValueError(f"no migrations dir at {migrations_dir};")
|
|
150
175
|
|
|
151
|
-
|
|
176
|
+
return [
|
|
152
177
|
_parse_migration(migrations_dir, template_name)
|
|
153
178
|
for template_name in sorted(os.listdir(migrations_dir))
|
|
154
179
|
if template_name.endswith(".sql")
|
|
155
180
|
]
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def get_migrations_list(migrations_dir: str, max_revision_id: Optional[str] = None) -> List[Migration]:
|
|
184
|
+
"""every migration in one directory, walked from its root to its head.
|
|
185
|
+
|
|
186
|
+
``revision_id`` cuts the walk short: the chain is returned up to and including
|
|
187
|
+
the migration carrying that revision. A revision no migration in the directory
|
|
188
|
+
carries is not an error -- the whole chain comes back, same as for no revision
|
|
189
|
+
at all.
|
|
190
|
+
"""
|
|
191
|
+
migrations = read_migrations_dir(migrations_dir)
|
|
156
192
|
if not migrations:
|
|
157
193
|
return []
|
|
158
194
|
|
|
@@ -167,19 +203,105 @@ def get_migrations_list(migrations_dir: str, max_revision_id: Optional[str] = No
|
|
|
167
203
|
return ordered
|
|
168
204
|
|
|
169
205
|
|
|
170
|
-
def
|
|
206
|
+
def get_children_by_revision(migrations: List[Migration]) -> Dict[Optional[str], List[Migration]]:
|
|
207
|
+
"""each revision's followers, keyed by the revision they follow.
|
|
208
|
+
|
|
209
|
+
the root's followers sit under None. A revision with two followers is a fork,
|
|
210
|
+
which is why this returns a list and not one migration.
|
|
211
|
+
"""
|
|
212
|
+
children: Dict[Optional[str], List[Migration]] = {}
|
|
213
|
+
for migration in sorted(migrations, key=lambda migration: migration.template_name):
|
|
214
|
+
children.setdefault(migration.prev_revision_id, []).append(migration)
|
|
215
|
+
return children
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def summarize_chain(migrations: List[Migration]) -> ChainSummary:
|
|
219
|
+
"""walk a chain for reading rather than for applying.
|
|
220
|
+
|
|
221
|
+
this raises nothing: a fork, a second root or a revision following one no file
|
|
222
|
+
carries is what the caller asked to be shown, not a reason to refuse. The walk
|
|
223
|
+
is depth first from each root, so one branch of a fork is listed to its head
|
|
224
|
+
before the next branch starts, and a revision already listed is not listed
|
|
225
|
+
again -- which is what keeps a cycle from looping here.
|
|
226
|
+
"""
|
|
227
|
+
children = get_children_by_revision(migrations)
|
|
228
|
+
known_revisions = {migration.revision_id for migration in migrations}
|
|
229
|
+
roots = [
|
|
230
|
+
migration
|
|
231
|
+
for migration in sorted(migrations, key=lambda migration: migration.template_name)
|
|
232
|
+
if not migration.prev_revision_id or migration.prev_revision_id not in known_revisions
|
|
233
|
+
]
|
|
234
|
+
|
|
235
|
+
ordered: List[Migration] = []
|
|
236
|
+
seen: Set[Migration] = set()
|
|
237
|
+
stack = list(reversed(roots))
|
|
238
|
+
while stack:
|
|
239
|
+
migration = stack.pop()
|
|
240
|
+
if migration in seen:
|
|
241
|
+
continue
|
|
242
|
+
seen.add(migration)
|
|
243
|
+
ordered.append(migration)
|
|
244
|
+
stack.extend(reversed(children.get(migration.revision_id, [])))
|
|
245
|
+
|
|
246
|
+
# orphan - somehow comes from no root
|
|
247
|
+
orphans = [migration for migration in migrations if migration not in seen]
|
|
248
|
+
ordered.extend(sorted(orphans, key=lambda migration: migration.template_name))
|
|
249
|
+
|
|
250
|
+
return ChainSummary(
|
|
251
|
+
ordered=ordered,
|
|
252
|
+
heads=[migration for migration in ordered if not children.get(migration.revision_id)],
|
|
253
|
+
roots=roots,
|
|
254
|
+
branch_points={
|
|
255
|
+
revision_id: followers for revision_id, followers in children.items() if revision_id and len(followers) > 1
|
|
256
|
+
},
|
|
257
|
+
orphans=orphans,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def format_history(summary: ChainSummary, limit: Optional[int] = None, oldest_first: bool = False) -> List[str]:
|
|
262
|
+
"""the listing, newest first like alembic's unless asked for the other end.
|
|
263
|
+
|
|
264
|
+
``limit`` cuts from the far end, so it keeps the newest with the default order
|
|
265
|
+
and the oldest with ``oldest_first`` -- either way, the end the listing starts
|
|
266
|
+
at is the end that survives. The notes are appended whole, even to a listing
|
|
267
|
+
cut short, because a split head is the thing worth seeing most.
|
|
268
|
+
"""
|
|
269
|
+
if limit is not None and limit < 1:
|
|
270
|
+
raise ValueError(f"a history limit lists at least one migration, got {limit};")
|
|
271
|
+
|
|
272
|
+
ordered = summary.ordered if oldest_first else list(reversed(summary.ordered))
|
|
273
|
+
if limit is not None:
|
|
274
|
+
ordered = ordered[:limit]
|
|
275
|
+
|
|
276
|
+
head_revisions = {migration.revision_id for migration in summary.heads}
|
|
277
|
+
branch_point_revisions = set(summary.branch_points)
|
|
278
|
+
lines = [format_history_line(migration, head_revisions, branch_point_revisions) for migration in ordered]
|
|
279
|
+
return lines + format_chain_notes(summary)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def history(migrations_dir: str, limit: Optional[int] = None, oldest_first: bool = False) -> List[str]:
|
|
283
|
+
"""the chain in one directory, as lines ready to be printed.
|
|
284
|
+
|
|
285
|
+
the files on disk are the whole of it: no catalog is read and no session is
|
|
286
|
+
built, so a chain can be read anywhere, whatever any catalog has been migrated
|
|
287
|
+
to. `get_table_version` is what says where a catalog actually stands.
|
|
288
|
+
"""
|
|
289
|
+
return format_history(summarize_chain(read_migrations_dir(migrations_dir)), limit=limit, oldest_first=oldest_first)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def get_table_version(spark: SparkSession, version_table_name: str) -> Optional[Row]:
|
|
171
293
|
"""the revision VERSION_TABLE records, or None if it holds no row yet."""
|
|
172
|
-
version_table = f"{
|
|
294
|
+
version_table = f"{version_table_name}"
|
|
173
295
|
if not spark.catalog.tableExists(version_table):
|
|
174
296
|
return None
|
|
175
297
|
return spark.read.table(version_table).select("version_num").head()
|
|
176
298
|
|
|
177
299
|
|
|
178
300
|
def get_unapplied_migrations_list(
|
|
179
|
-
spark: SparkSession,
|
|
301
|
+
spark: SparkSession, version_table_name, migrations_dir: str, max_revision_id: Optional[str] = None
|
|
180
302
|
) -> List[Migration]:
|
|
181
303
|
ordered = get_migrations_list(migrations_dir, max_revision_id)
|
|
182
|
-
current = get_table_version(spark,
|
|
304
|
+
current = get_table_version(spark, version_table_name)
|
|
183
305
|
if not current:
|
|
184
306
|
return ordered
|
|
185
307
|
|
|
@@ -187,7 +309,7 @@ def get_unapplied_migrations_list(
|
|
|
187
309
|
|
|
188
310
|
if not ordered:
|
|
189
311
|
logger.warning(
|
|
190
|
-
f"{
|
|
312
|
+
f"{version_table_name} is at {current_revision_id}, but {migrations_dir} holds no migrations; "
|
|
191
313
|
f"treating the chain as retired;"
|
|
192
314
|
)
|
|
193
315
|
return ordered
|
|
@@ -196,7 +318,9 @@ def get_unapplied_migrations_list(
|
|
|
196
318
|
if migration.revision_id == current_revision_id:
|
|
197
319
|
return ordered[index + 1 :]
|
|
198
320
|
|
|
199
|
-
raise ValueError(
|
|
321
|
+
raise ValueError(
|
|
322
|
+
f"{version_table_name} is at {current_revision_id}, which matches no migration in {migrations_dir};"
|
|
323
|
+
)
|
|
200
324
|
|
|
201
325
|
|
|
202
326
|
def gate_on_is_dbr(migration_content: str) -> str:
|
|
@@ -247,8 +371,15 @@ def create_new_migration(
|
|
|
247
371
|
migration_file.write(default_migration_content)
|
|
248
372
|
|
|
249
373
|
|
|
250
|
-
def apply_template(
|
|
251
|
-
|
|
374
|
+
def apply_template(
|
|
375
|
+
output_dir: str,
|
|
376
|
+
template,
|
|
377
|
+
template_vals: Optional[Dict[str, str]] = None,
|
|
378
|
+
) -> str:
|
|
379
|
+
prep: Dict[str, str | bool] = {}
|
|
380
|
+
prep.update(template_vals or {})
|
|
381
|
+
prep["is_dbr"] = is_dbr()
|
|
382
|
+
result_sql = template.render(**prep)
|
|
252
383
|
with open(os.path.join(output_dir, template.name.replace(".sql", "_primed.sql")), "w") as file_handle:
|
|
253
384
|
file_handle.write(result_sql)
|
|
254
385
|
return result_sql
|
|
@@ -278,43 +409,33 @@ def has_no_statements(rendered_sql: str) -> bool:
|
|
|
278
409
|
return True
|
|
279
410
|
|
|
280
411
|
|
|
281
|
-
def render_and_apply(
|
|
412
|
+
def render_and_apply(
|
|
413
|
+
spark: SparkSession,
|
|
414
|
+
output_folder: str,
|
|
415
|
+
template,
|
|
416
|
+
template_vals: Optional[Dict[str, str]] = None,
|
|
417
|
+
) -> None:
|
|
282
418
|
"""render one migration and run it, unless this engine was gated out of it."""
|
|
283
|
-
result_sql = apply_template(output_folder, template,
|
|
419
|
+
result_sql = apply_template(output_folder, template, template_vals)
|
|
284
420
|
if has_no_statements(result_sql):
|
|
285
421
|
logger.info(f"{template.name} rendered no statements; nothing to apply here;")
|
|
286
422
|
return
|
|
287
423
|
spark.sql(result_sql)
|
|
288
424
|
|
|
289
425
|
|
|
290
|
-
def
|
|
291
|
-
"""apply the package's own bootstrap chain, in filename order rather than by revision."""
|
|
292
|
-
env = Environment(
|
|
293
|
-
loader=PackageLoader(package_name=PACKAGE_NAME, package_path=INITIAL_MIGRATIONS_PACKAGE_PATH),
|
|
294
|
-
autoescape=select_autoescape(),
|
|
295
|
-
)
|
|
296
|
-
all_templates = env.list_templates(extensions=["sql"])
|
|
297
|
-
|
|
298
|
-
logger.info(f"found {len(all_templates)} migrations; first five are {all_templates[:5]};")
|
|
299
|
-
for template_name in all_templates:
|
|
300
|
-
render_and_apply(spark, output_folder, env.get_template(template_name), cat=cat, schema=schema)
|
|
301
|
-
logger.info(f"rendered {len(all_templates)} initial migrations;")
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
def record_revision(spark: SparkSession, cat: str, schema: str, revision_id: str) -> None:
|
|
305
|
-
version_table = f"{cat}.{schema}.{VERSION_TABLE}"
|
|
426
|
+
def record_revision(spark: SparkSession, version_table_name: str, revision_id: str) -> None:
|
|
306
427
|
revision_literal = revision_id.replace("'", "''")
|
|
307
|
-
spark.sql(f"delete from {
|
|
308
|
-
spark.sql(f"insert into {
|
|
428
|
+
spark.sql(f"delete from {version_table_name}")
|
|
429
|
+
spark.sql(f"insert into {version_table_name} (version_num) values ('{revision_literal}')")
|
|
309
430
|
|
|
310
431
|
|
|
311
432
|
def upgrade(
|
|
312
433
|
spark: SparkSession,
|
|
434
|
+
version_table_name: str,
|
|
313
435
|
output_folder: str,
|
|
314
436
|
migrations_dir: str,
|
|
315
|
-
cat: str,
|
|
316
|
-
schema: str,
|
|
317
437
|
max_revision_id: Optional[str] = None,
|
|
438
|
+
template_vals: Optional[Dict[str, str]] = None,
|
|
318
439
|
) -> None:
|
|
319
440
|
"""
|
|
320
441
|
TODO XXX permit applying up to a revision below the head revision
|
|
@@ -324,7 +445,7 @@ def upgrade(
|
|
|
324
445
|
logger.info(f"no chain at {migrations_dir}; skipping it;")
|
|
325
446
|
return
|
|
326
447
|
unapplied = get_unapplied_migrations_list(
|
|
327
|
-
spark,
|
|
448
|
+
spark, version_table_name, migrations_dir, max_revision_id=max_revision_id
|
|
328
449
|
)
|
|
329
450
|
|
|
330
451
|
logger.info(
|
|
@@ -334,47 +455,85 @@ def upgrade(
|
|
|
334
455
|
|
|
335
456
|
env = Environment(loader=FileSystemLoader(migrations_dir), autoescape=select_autoescape())
|
|
336
457
|
for migration in unapplied:
|
|
337
|
-
render_and_apply(spark, output_folder, env.get_template(migration.template_name),
|
|
338
|
-
record_revision(spark,
|
|
458
|
+
render_and_apply(spark, output_folder, env.get_template(migration.template_name), template_vals)
|
|
459
|
+
record_revision(spark, version_table_name, migration.revision_id)
|
|
339
460
|
logger.info(f"{migration.template_name} done; now at {migration.revision_id};")
|
|
340
461
|
|
|
341
462
|
|
|
342
463
|
def run_upgrade(
|
|
343
464
|
spark: SparkSession,
|
|
344
|
-
cat: str,
|
|
345
|
-
schema: str,
|
|
346
465
|
output_folder: str,
|
|
347
466
|
migrations_dir: str,
|
|
348
467
|
max_revision_id: Optional[str] = None,
|
|
468
|
+
template_vals: Optional[Dict[str, str]] = None,
|
|
349
469
|
) -> None:
|
|
350
|
-
initial_output_folder = os.path.join(output_folder, INITIAL_MIGRATIONS_PACKAGE_PATH)
|
|
351
470
|
client_output_folder = os.path.join(output_folder, CLIENT_OUTPUT_DIRNAME)
|
|
352
|
-
os.makedirs(initial_output_folder, exist_ok=True)
|
|
353
471
|
os.makedirs(client_output_folder, exist_ok=True)
|
|
354
|
-
|
|
355
|
-
if spark.catalog.tableExists(
|
|
356
|
-
logger.info(f"{
|
|
472
|
+
version_table_name = get_version_table_name(spark)
|
|
473
|
+
if spark.catalog.tableExists(version_table_name):
|
|
474
|
+
logger.info(f"{version_table_name} already exists;")
|
|
357
475
|
else:
|
|
358
|
-
|
|
359
|
-
upgrade(spark, client_output_folder, migrations_dir,
|
|
476
|
+
spark.sql(f"create table if not exists {version_table_name}(version_num string not null);")
|
|
477
|
+
upgrade(spark, version_table_name, client_output_folder, migrations_dir, max_revision_id, template_vals)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def format_history_line(
|
|
481
|
+
migration: Migration, head_revisions: Set[str], branch_point_revisions: Set[str]
|
|
482
|
+
) -> str: # pragma: no cover
|
|
483
|
+
"""one alembic-shaped line: `prev -> rev (tags), filename`."""
|
|
484
|
+
tags = []
|
|
485
|
+
if migration.revision_id in head_revisions:
|
|
486
|
+
tags.append("head")
|
|
487
|
+
if migration.revision_id in branch_point_revisions:
|
|
488
|
+
tags.append("branchpoint")
|
|
489
|
+
tag_text = f" ({', '.join(tags)})" if tags else ""
|
|
490
|
+
return f"{migration.prev_revision_id or BASE_LABEL} -> {migration.revision_id}{tag_text}, {migration.template_name}"
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def format_chain_notes(summary: ChainSummary) -> List[str]: # pragma: no cover
|
|
494
|
+
"""what is wrong with this chain, below the listing, or nothing if it is sound."""
|
|
495
|
+
notes = []
|
|
496
|
+
if len(summary.heads) > 1:
|
|
497
|
+
notes.append(
|
|
498
|
+
f"split heads: {', '.join(migration.revision_id for migration in summary.heads)}; "
|
|
499
|
+
f"a chain with more than one head cannot be applied until it is rejoined;"
|
|
500
|
+
)
|
|
501
|
+
for revision_id, followers in sorted(summary.branch_points.items()):
|
|
502
|
+
notes.append(
|
|
503
|
+
f"{revision_id} is the prev_revision_id of "
|
|
504
|
+
f"{', '.join(migration.template_name for migration in followers)};"
|
|
505
|
+
)
|
|
506
|
+
if len(summary.roots) > 1:
|
|
507
|
+
notes.append(
|
|
508
|
+
f"more than one root: {', '.join(migration.template_name for migration in summary.roots)}; "
|
|
509
|
+
f"a chain has exactly one migration following no other;"
|
|
510
|
+
)
|
|
511
|
+
if summary.orphans:
|
|
512
|
+
notes.append(
|
|
513
|
+
f"no root reaches {', '.join(migration.template_name for migration in summary.orphans)}; "
|
|
514
|
+
f"they are listed together at the tail end of the chain order, having none of their own;"
|
|
515
|
+
)
|
|
516
|
+
return notes
|
|
360
517
|
|
|
361
518
|
|
|
362
519
|
def main(
|
|
363
|
-
cat: str,
|
|
364
|
-
schema: str,
|
|
365
520
|
migrations_dir: str,
|
|
366
521
|
output_parent_path: Optional[str] = None,
|
|
367
522
|
max_revision_id: Optional[str] = None,
|
|
523
|
+
template_vals: Optional[Dict[str, str]] = None,
|
|
368
524
|
) -> None:
|
|
369
525
|
output_folder = get_output_folder(output_parent_path or os.path.join(os.getcwd(), "migrations_out"))
|
|
370
526
|
os.makedirs(output_folder)
|
|
371
|
-
|
|
527
|
+
spark = get_spark()
|
|
528
|
+
run_upgrade(spark, output_folder, migrations_dir, max_revision_id, template_vals)
|
|
372
529
|
|
|
373
530
|
|
|
374
531
|
def _cli_main(
|
|
375
|
-
|
|
532
|
+
migrations_dir: str,
|
|
533
|
+
max_revision_id: Optional[str] = None,
|
|
534
|
+
template_vals: Optional[Dict[str, str]] = None,
|
|
376
535
|
) -> None: # pragma: no cover
|
|
377
|
-
main(
|
|
536
|
+
main(migrations_dir, max_revision_id=max_revision_id, template_vals=template_vals)
|
|
378
537
|
|
|
379
538
|
|
|
380
539
|
def _cli_create_new_migration(
|
|
@@ -383,13 +542,35 @@ def _cli_create_new_migration(
|
|
|
383
542
|
create_new_migration(message, template_path or get_default_template_path(), output_path, add_is_dbr=add_is_dbr)
|
|
384
543
|
|
|
385
544
|
|
|
545
|
+
def _cli_history(migrations_dir: str, limit: Optional[int], oldest_first: bool) -> None: # pragma: no cover
|
|
546
|
+
"""the listing goes to stdout, so it pipes; the log records go to stderr as always."""
|
|
547
|
+
for line in history(migrations_dir, limit=limit, oldest_first=oldest_first):
|
|
548
|
+
print(line)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def parse_template_vals(text: str) -> Dict[str, str]:
|
|
552
|
+
"""`foo=bar,gee=wiz` -> {"foo": "bar", "gee": "wiz"}; the argparse `type` of --template-vals.
|
|
553
|
+
|
|
554
|
+
a key has to be usable as a jinja variable, so an identifier: one that is not would
|
|
555
|
+
render as nothing at all and the migration would go out silently missing it.
|
|
556
|
+
"""
|
|
557
|
+
template_vals: Dict[str, str] = {}
|
|
558
|
+
for pair in text.split(","):
|
|
559
|
+
if not pair.strip():
|
|
560
|
+
continue
|
|
561
|
+
key, separator, value = pair.partition("=")
|
|
562
|
+
key = key.strip()
|
|
563
|
+
if not separator or not key.isidentifier():
|
|
564
|
+
raise argparse.ArgumentTypeError(f"expected key=value with an identifier key, got {pair!r};")
|
|
565
|
+
template_vals[key] = value.strip()
|
|
566
|
+
return template_vals
|
|
567
|
+
|
|
568
|
+
|
|
386
569
|
def build_parser() -> argparse.ArgumentParser: # pragma: no cover
|
|
387
570
|
parser = argparse.ArgumentParser(description="spark_sql_migrations migration utilities")
|
|
388
571
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
389
572
|
|
|
390
573
|
p_run = subparsers.add_parser("run", help="upgrade to head/apply pending migrations")
|
|
391
|
-
p_run.add_argument("--cat", type=str, default="spark_catalog")
|
|
392
|
-
p_run.add_argument("--schema", type=str, default="default")
|
|
393
574
|
p_run.add_argument("--migrations-dir", type=str, required=True, help="the directory holding the migration chain")
|
|
394
575
|
p_run.add_argument(
|
|
395
576
|
"--max-revision-id",
|
|
@@ -397,6 +578,15 @@ def build_parser() -> argparse.ArgumentParser: # pragma: no cover
|
|
|
397
578
|
default=None,
|
|
398
579
|
help="apply the migration carrying this revision_id and stop; defaults to applying the chain to its head",
|
|
399
580
|
)
|
|
581
|
+
p_run.add_argument(
|
|
582
|
+
"--template-vals",
|
|
583
|
+
type=parse_template_vals,
|
|
584
|
+
default=None,
|
|
585
|
+
metavar="KEY=VALUE[,KEY=VALUE...]",
|
|
586
|
+
help="jinja render vars handed to every migration. is_dbr is always set from the engine and cannot be "
|
|
587
|
+
"overridden here. the whole thing is one argument, equal signs and all: "
|
|
588
|
+
"--template-vals=foo=bar,gee=wiz,pp=qq",
|
|
589
|
+
)
|
|
400
590
|
p_run.set_defaults(func=_cli_main)
|
|
401
591
|
|
|
402
592
|
p_create = subparsers.add_parser("create_new_migration", help="create a new migration from the template")
|
|
@@ -419,6 +609,22 @@ def build_parser() -> argparse.ArgumentParser: # pragma: no cover
|
|
|
419
609
|
)
|
|
420
610
|
p_create.set_defaults(func=_cli_create_new_migration)
|
|
421
611
|
|
|
612
|
+
p_history = subparsers.add_parser("history", help="list the migration chain on disk, newest first")
|
|
613
|
+
p_history.add_argument("--migrations-dir", type=str, required=True, help="the directory holding the chain")
|
|
614
|
+
p_history.add_argument(
|
|
615
|
+
"-n",
|
|
616
|
+
"--limit",
|
|
617
|
+
type=int,
|
|
618
|
+
default=None,
|
|
619
|
+
help="list at most this many migrations, counted from the end the listing starts at",
|
|
620
|
+
)
|
|
621
|
+
p_history.add_argument(
|
|
622
|
+
"--oldest-first",
|
|
623
|
+
action="store_true",
|
|
624
|
+
help="list from the root up instead of from the head down",
|
|
625
|
+
)
|
|
626
|
+
p_history.set_defaults(func=_cli_history)
|
|
627
|
+
|
|
422
628
|
return parser
|
|
423
629
|
|
|
424
630
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spark_sql_migrations
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Chained, idempotent SQL migrations for Spark and Databricks.
|
|
5
|
+
Home-page: https://github.com/mcjug2015/spark_sql_migrations
|
|
6
|
+
Author: Victor Semenov
|
|
7
|
+
License: Apache-2.0
|
|
8
|
+
Project-URL: Source, https://github.com/mcjug2015/spark_sql_migrations
|
|
9
|
+
Project-URL: Issues, https://github.com/mcjug2015/spark_sql_migrations/issues
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: SQL
|
|
14
|
+
Classifier: Topic :: Database
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: Jinja2<4,>=3.1
|
|
19
|
+
Provides-Extra: local
|
|
20
|
+
Requires-Dist: pyspark[connect]<5,>=4.0; extra == "local"
|
|
21
|
+
Requires-Dist: delta-spark<5,>=4.0; extra == "local"
|
|
22
|
+
Provides-Extra: databricks
|
|
23
|
+
Requires-Dist: databricks-connect<19,>=18; extra == "databricks"
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: classifier
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: license
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: project-url
|
|
30
|
+
Dynamic: provides-extra
|
|
31
|
+
Dynamic: requires-dist
|
|
32
|
+
Dynamic: requires-python
|
|
33
|
+
Dynamic: summary
|
{spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/SOURCES.txt
RENAMED
|
@@ -13,8 +13,5 @@ spark_sql_migrations.egg-info/namespace_packages.txt
|
|
|
13
13
|
spark_sql_migrations.egg-info/requires.txt
|
|
14
14
|
spark_sql_migrations.egg-info/top_level.txt
|
|
15
15
|
spark_sql_migrations/migration_templates/default_migration_template.sql
|
|
16
|
-
spark_sql_migrations/migrations_initial/first_01_intial_create_cat.sql
|
|
17
|
-
spark_sql_migrations/migrations_initial/first_02_intial_create_schema.sql
|
|
18
|
-
spark_sql_migrations/migrations_initial/first_03_intial_create_state_table.sql
|
|
19
16
|
spark_sql_migrations/spark_sql/__init__.py
|
|
20
17
|
spark_sql_migrations/spark_sql/spark_sql.py
|