spark-sql-migrations 0.0.8__tar.gz → 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. spark_sql_migrations-0.1.0/PKG-INFO +33 -0
  2. spark_sql_migrations-0.1.0/setup.py +59 -0
  3. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/spark_sql/spark_sql.py +266 -60
  4. spark_sql_migrations-0.1.0/spark_sql_migrations.egg-info/PKG-INFO +33 -0
  5. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/SOURCES.txt +0 -3
  6. spark_sql_migrations-0.0.8/PKG-INFO +0 -268
  7. spark_sql_migrations-0.0.8/setup.py +0 -303
  8. spark_sql_migrations-0.0.8/spark_sql_migrations/migrations_initial/first_01_intial_create_cat.sql +0 -9
  9. spark_sql_migrations-0.0.8/spark_sql_migrations/migrations_initial/first_02_intial_create_schema.sql +0 -4
  10. spark_sql_migrations-0.0.8/spark_sql_migrations/migrations_initial/first_03_intial_create_state_table.sql +0 -6
  11. spark_sql_migrations-0.0.8/spark_sql_migrations.egg-info/PKG-INFO +0 -268
  12. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/LICENSE +0 -0
  13. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/MANIFEST.in +0 -0
  14. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/backend_shim.py +0 -0
  15. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/setup.cfg +0 -0
  16. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/__init__.py +0 -0
  17. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/migration_templates/default_migration_template.sql +0 -0
  18. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/py.typed +0 -0
  19. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/spark_sql/__init__.py +0 -0
  20. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations/spark_utils.py +0 -0
  21. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/dependency_links.txt +0 -0
  22. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/entry_points.txt +0 -0
  23. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/namespace_packages.txt +0 -0
  24. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/requires.txt +0 -0
  25. {spark_sql_migrations-0.0.8 → spark_sql_migrations-0.1.0}/spark_sql_migrations.egg-info/top_level.txt +0 -0
@@ -0,0 +1,33 @@
1
+ Metadata-Version: 2.4
2
+ Name: spark_sql_migrations
3
+ Version: 0.1.0
4
+ Summary: Chained, idempotent SQL migrations for Spark and Databricks.
5
+ Home-page: https://github.com/mcjug2015/spark_sql_migrations
6
+ Author: Victor Semenov
7
+ License: Apache-2.0
8
+ Project-URL: Source, https://github.com/mcjug2015/spark_sql_migrations
9
+ Project-URL: Issues, https://github.com/mcjug2015/spark_sql_migrations/issues
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: SQL
14
+ Classifier: Topic :: Database
15
+ Classifier: Typing :: Typed
16
+ Requires-Python: >=3.10
17
+ License-File: LICENSE
18
+ Requires-Dist: Jinja2<4,>=3.1
19
+ Provides-Extra: local
20
+ Requires-Dist: pyspark[connect]<5,>=4.0; extra == "local"
21
+ Requires-Dist: delta-spark<5,>=4.0; extra == "local"
22
+ Provides-Extra: databricks
23
+ Requires-Dist: databricks-connect<19,>=18; extra == "databricks"
24
+ Dynamic: author
25
+ Dynamic: classifier
26
+ Dynamic: home-page
27
+ Dynamic: license
28
+ Dynamic: license-file
29
+ Dynamic: project-url
30
+ Dynamic: provides-extra
31
+ Dynamic: requires-dist
32
+ Dynamic: requires-python
33
+ Dynamic: summary
@@ -0,0 +1,59 @@
1
+
2
+ # DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
3
+ # Target: //:dist
4
+
5
+ from setuptools import setup
6
+
7
+ setup(**{
8
+ 'author': 'Victor Semenov',
9
+ 'classifiers': [
10
+ 'Development Status :: 3 - Alpha',
11
+ 'Intended Audience :: Developers',
12
+ 'Programming Language :: Python :: 3.12',
13
+ 'Programming Language :: SQL',
14
+ 'Topic :: Database',
15
+ 'Typing :: Typed',
16
+ ],
17
+ 'description': 'Chained, idempotent SQL migrations for Spark and Databricks.',
18
+ 'entry_points': {
19
+ 'console_scripts': [
20
+ 'spark-sql-migrations = spark_sql_migrations.spark_sql.spark_sql:cli',
21
+ ],
22
+ },
23
+ 'extras_require': {
24
+ 'local': [
25
+ 'pyspark[connect]>=4.0,<5',
26
+ 'delta-spark>=4.0,<5',
27
+ ],
28
+ 'databricks': [
29
+ 'databricks-connect>=18,<19',
30
+ ],
31
+ },
32
+ 'install_requires': (
33
+ 'Jinja2<4,>=3.1',
34
+ ),
35
+ 'license': 'Apache-2.0',
36
+ 'license_files': [
37
+ 'LICENSE',
38
+ ],
39
+ 'name': 'spark_sql_migrations',
40
+ 'namespace_packages': (
41
+ ),
42
+ 'package_data': {
43
+ 'spark_sql_migrations': (
44
+ 'migration_templates/default_migration_template.sql',
45
+ 'py.typed',
46
+ ),
47
+ },
48
+ 'packages': (
49
+ 'spark_sql_migrations',
50
+ 'spark_sql_migrations.spark_sql',
51
+ ),
52
+ 'project_urls': {
53
+ 'Source': 'https://github.com/mcjug2015/spark_sql_migrations',
54
+ 'Issues': 'https://github.com/mcjug2015/spark_sql_migrations/issues',
55
+ },
56
+ 'python_requires': '>=3.10',
57
+ 'url': 'https://github.com/mcjug2015/spark_sql_migrations',
58
+ 'version': '0.1.0',
59
+ })
@@ -27,9 +27,9 @@ import os
27
27
  import re
28
28
  import uuid
29
29
  from dataclasses import dataclass
30
- from typing import Any, Dict, List, Optional
30
+ from typing import Any, Dict, List, Optional, Set
31
31
 
32
- from jinja2 import Environment, FileSystemLoader, PackageLoader, select_autoescape
32
+ from jinja2 import Environment, FileSystemLoader, select_autoescape
33
33
  from pyspark.sql.session import SparkSession
34
34
  from pyspark.sql.types import Row
35
35
 
@@ -38,10 +38,12 @@ from spark_sql_migrations.spark_utils import get_spark, is_dbr
38
38
  logger = logging.getLogger(__name__)
39
39
 
40
40
 
41
- VERSION_TABLE = "_spark_migrations_version"
41
+ # value returned by get_version_table_name from start of run should have been passed in to you
42
+ __DO_NOT_USE_VERSION_TABLE = "_spark_migrations_version"
43
+ # what `history` prints in place of the revision a root migration follows
44
+ BASE_LABEL = "<base>"
42
45
 
43
46
  PACKAGE_NAME = "spark_sql_migrations"
44
- INITIAL_MIGRATIONS_PACKAGE_PATH = "migrations_initial"
45
47
  # where a run's rendered client migrations land, under its output folder
46
48
  CLIENT_OUTPUT_DIRNAME = "migrations"
47
49
  TEMPLATES_PACKAGE_PATH = "migration_templates"
@@ -58,6 +60,30 @@ class Migration:
58
60
  template_name: str
59
61
 
60
62
 
63
+ @dataclass(frozen=True)
64
+ class ChainSummary:
65
+ """a directory's chain as `history` found it, forks and all.
66
+
67
+ ``ordered`` is oldest first and holds every migration in the directory, so a
68
+ listing built from it never silently drops a file. Everything else is what
69
+ makes the chain unapplicable: more than one ``heads`` entry is a split head,
70
+ a ``branch_points`` entry is where it split, more than one ``roots`` entry is
71
+ a second chain in the same directory, and ``orphans`` are files no walk from
72
+ any root reaches -- a cycle, in practice.
73
+ """
74
+
75
+ ordered: List[Migration]
76
+ heads: List[Migration]
77
+ roots: List[Migration]
78
+ branch_points: Dict[str, List[Migration]]
79
+ orphans: List[Migration]
80
+
81
+
82
+ def get_version_table_name(spark: SparkSession) -> str:
83
+ # only correct ar start of run
84
+ return f"{spark.catalog.currentCatalog()}.{spark.catalog.currentDatabase()}._spark_migrations_version"
85
+
86
+
61
87
  def get_default_template_path() -> str:
62
88
  """the template this package ships; callers may pass their own instead."""
63
89
  return os.path.join(os.path.dirname(__file__), "..", TEMPLATES_PACKAGE_PATH, DEFAULT_TEMPLATE_NAME)
@@ -137,22 +163,32 @@ def get_ordered_migration_objs(migration_objs: List[Migration]) -> List[Migratio
137
163
  return ordered
138
164
 
139
165
 
140
- def get_migrations_list(migrations_dir: str, max_revision_id: Optional[str] = None) -> List[Migration]:
141
- """every migration in one directory, walked from its root to its head.
166
+ def read_migrations_dir(migrations_dir: str) -> List[Migration]:
167
+ """every .sql file in one directory, parsed, in filename order and unwalked.
142
168
 
143
- ``revision_id`` cuts the walk short: the chain is returned up to and including
144
- the migration carrying that revision. A revision no migration in the directory
145
- carries is not an error -- the whole chain comes back, same as for no revision
146
- at all.
169
+ the headers are all this reads, so a chain that forks or points at nothing
170
+ comes back whole -- it is `get_ordered_migration_objs` that insists on a
171
+ single walkable chain, and `history` that reports one which is not.
147
172
  """
148
173
  if not os.path.isdir(migrations_dir):
149
174
  raise ValueError(f"no migrations dir at {migrations_dir};")
150
175
 
151
- migrations = [
176
+ return [
152
177
  _parse_migration(migrations_dir, template_name)
153
178
  for template_name in sorted(os.listdir(migrations_dir))
154
179
  if template_name.endswith(".sql")
155
180
  ]
181
+
182
+
183
+ def get_migrations_list(migrations_dir: str, max_revision_id: Optional[str] = None) -> List[Migration]:
184
+ """every migration in one directory, walked from its root to its head.
185
+
186
+ ``revision_id`` cuts the walk short: the chain is returned up to and including
187
+ the migration carrying that revision. A revision no migration in the directory
188
+ carries is not an error -- the whole chain comes back, same as for no revision
189
+ at all.
190
+ """
191
+ migrations = read_migrations_dir(migrations_dir)
156
192
  if not migrations:
157
193
  return []
158
194
 
@@ -167,19 +203,105 @@ def get_migrations_list(migrations_dir: str, max_revision_id: Optional[str] = No
167
203
  return ordered
168
204
 
169
205
 
170
- def get_table_version(spark: SparkSession, cat: str, schema: str) -> Optional[Row]:
206
+ def get_children_by_revision(migrations: List[Migration]) -> Dict[Optional[str], List[Migration]]:
207
+ """each revision's followers, keyed by the revision they follow.
208
+
209
+ the root's followers sit under None. A revision with two followers is a fork,
210
+ which is why this returns a list and not one migration.
211
+ """
212
+ children: Dict[Optional[str], List[Migration]] = {}
213
+ for migration in sorted(migrations, key=lambda migration: migration.template_name):
214
+ children.setdefault(migration.prev_revision_id, []).append(migration)
215
+ return children
216
+
217
+
218
+ def summarize_chain(migrations: List[Migration]) -> ChainSummary:
219
+ """walk a chain for reading rather than for applying.
220
+
221
+ this raises nothing: a fork, a second root or a revision following one no file
222
+ carries is what the caller asked to be shown, not a reason to refuse. The walk
223
+ is depth first from each root, so one branch of a fork is listed to its head
224
+ before the next branch starts, and a revision already listed is not listed
225
+ again -- which is what keeps a cycle from looping here.
226
+ """
227
+ children = get_children_by_revision(migrations)
228
+ known_revisions = {migration.revision_id for migration in migrations}
229
+ roots = [
230
+ migration
231
+ for migration in sorted(migrations, key=lambda migration: migration.template_name)
232
+ if not migration.prev_revision_id or migration.prev_revision_id not in known_revisions
233
+ ]
234
+
235
+ ordered: List[Migration] = []
236
+ seen: Set[Migration] = set()
237
+ stack = list(reversed(roots))
238
+ while stack:
239
+ migration = stack.pop()
240
+ if migration in seen:
241
+ continue
242
+ seen.add(migration)
243
+ ordered.append(migration)
244
+ stack.extend(reversed(children.get(migration.revision_id, [])))
245
+
246
+ # orphan - somehow comes from no root
247
+ orphans = [migration for migration in migrations if migration not in seen]
248
+ ordered.extend(sorted(orphans, key=lambda migration: migration.template_name))
249
+
250
+ return ChainSummary(
251
+ ordered=ordered,
252
+ heads=[migration for migration in ordered if not children.get(migration.revision_id)],
253
+ roots=roots,
254
+ branch_points={
255
+ revision_id: followers for revision_id, followers in children.items() if revision_id and len(followers) > 1
256
+ },
257
+ orphans=orphans,
258
+ )
259
+
260
+
261
+ def format_history(summary: ChainSummary, limit: Optional[int] = None, oldest_first: bool = False) -> List[str]:
262
+ """the listing, newest first like alembic's unless asked for the other end.
263
+
264
+ ``limit`` cuts from the far end, so it keeps the newest with the default order
265
+ and the oldest with ``oldest_first`` -- either way, the end the listing starts
266
+ at is the end that survives. The notes are appended whole, even to a listing
267
+ cut short, because a split head is the thing worth seeing most.
268
+ """
269
+ if limit is not None and limit < 1:
270
+ raise ValueError(f"a history limit lists at least one migration, got {limit};")
271
+
272
+ ordered = summary.ordered if oldest_first else list(reversed(summary.ordered))
273
+ if limit is not None:
274
+ ordered = ordered[:limit]
275
+
276
+ head_revisions = {migration.revision_id for migration in summary.heads}
277
+ branch_point_revisions = set(summary.branch_points)
278
+ lines = [format_history_line(migration, head_revisions, branch_point_revisions) for migration in ordered]
279
+ return lines + format_chain_notes(summary)
280
+
281
+
282
+ def history(migrations_dir: str, limit: Optional[int] = None, oldest_first: bool = False) -> List[str]:
283
+ """the chain in one directory, as lines ready to be printed.
284
+
285
+ the files on disk are the whole of it: no catalog is read and no session is
286
+ built, so a chain can be read anywhere, whatever any catalog has been migrated
287
+ to. `get_table_version` is what says where a catalog actually stands.
288
+ """
289
+ return format_history(summarize_chain(read_migrations_dir(migrations_dir)), limit=limit, oldest_first=oldest_first)
290
+
291
+
292
+ def get_table_version(spark: SparkSession, version_table_name: str) -> Optional[Row]:
171
293
  """the revision VERSION_TABLE records, or None if it holds no row yet."""
172
- version_table = f"{cat}.{schema}.{VERSION_TABLE}"
294
+ version_table = f"{version_table_name}"
173
295
  if not spark.catalog.tableExists(version_table):
174
296
  return None
175
297
  return spark.read.table(version_table).select("version_num").head()
176
298
 
177
299
 
178
300
  def get_unapplied_migrations_list(
179
- spark: SparkSession, migrations_dir: str, cat: str, schema: str, max_revision_id: Optional[str] = None
301
+ spark: SparkSession, version_table_name, migrations_dir: str, max_revision_id: Optional[str] = None
180
302
  ) -> List[Migration]:
181
303
  ordered = get_migrations_list(migrations_dir, max_revision_id)
182
- current = get_table_version(spark, cat, schema)
304
+ current = get_table_version(spark, version_table_name)
183
305
  if not current:
184
306
  return ordered
185
307
 
@@ -187,7 +309,7 @@ def get_unapplied_migrations_list(
187
309
 
188
310
  if not ordered:
189
311
  logger.warning(
190
- f"{VERSION_TABLE} is at {current_revision_id}, but {migrations_dir} holds no migrations; "
312
+ f"{version_table_name} is at {current_revision_id}, but {migrations_dir} holds no migrations; "
191
313
  f"treating the chain as retired;"
192
314
  )
193
315
  return ordered
@@ -196,7 +318,9 @@ def get_unapplied_migrations_list(
196
318
  if migration.revision_id == current_revision_id:
197
319
  return ordered[index + 1 :]
198
320
 
199
- raise ValueError(f"{VERSION_TABLE} is at {current_revision_id}, which matches no migration in {migrations_dir};")
321
+ raise ValueError(
322
+ f"{version_table_name} is at {current_revision_id}, which matches no migration in {migrations_dir};"
323
+ )
200
324
 
201
325
 
202
326
  def gate_on_is_dbr(migration_content: str) -> str:
@@ -247,8 +371,15 @@ def create_new_migration(
247
371
  migration_file.write(default_migration_content)
248
372
 
249
373
 
250
- def apply_template(output_dir: str, template, cat: str, schema: str) -> str:
251
- result_sql = template.render(cat=cat, schema=schema, is_dbr=is_dbr())
374
+ def apply_template(
375
+ output_dir: str,
376
+ template,
377
+ template_vals: Optional[Dict[str, str]] = None,
378
+ ) -> str:
379
+ prep: Dict[str, str | bool] = {}
380
+ prep.update(template_vals or {})
381
+ prep["is_dbr"] = is_dbr()
382
+ result_sql = template.render(**prep)
252
383
  with open(os.path.join(output_dir, template.name.replace(".sql", "_primed.sql")), "w") as file_handle:
253
384
  file_handle.write(result_sql)
254
385
  return result_sql
@@ -278,43 +409,33 @@ def has_no_statements(rendered_sql: str) -> bool:
278
409
  return True
279
410
 
280
411
 
281
- def render_and_apply(spark: SparkSession, output_folder: str, template, cat: str, schema: str) -> None:
412
+ def render_and_apply(
413
+ spark: SparkSession,
414
+ output_folder: str,
415
+ template,
416
+ template_vals: Optional[Dict[str, str]] = None,
417
+ ) -> None:
282
418
  """render one migration and run it, unless this engine was gated out of it."""
283
- result_sql = apply_template(output_folder, template, cat=cat, schema=schema)
419
+ result_sql = apply_template(output_folder, template, template_vals)
284
420
  if has_no_statements(result_sql):
285
421
  logger.info(f"{template.name} rendered no statements; nothing to apply here;")
286
422
  return
287
423
  spark.sql(result_sql)
288
424
 
289
425
 
290
- def migrate_initial(spark: SparkSession, output_folder: str, cat: str, schema: str) -> None:
291
- """apply the package's own bootstrap chain, in filename order rather than by revision."""
292
- env = Environment(
293
- loader=PackageLoader(package_name=PACKAGE_NAME, package_path=INITIAL_MIGRATIONS_PACKAGE_PATH),
294
- autoescape=select_autoescape(),
295
- )
296
- all_templates = env.list_templates(extensions=["sql"])
297
-
298
- logger.info(f"found {len(all_templates)} migrations; first five are {all_templates[:5]};")
299
- for template_name in all_templates:
300
- render_and_apply(spark, output_folder, env.get_template(template_name), cat=cat, schema=schema)
301
- logger.info(f"rendered {len(all_templates)} initial migrations;")
302
-
303
-
304
- def record_revision(spark: SparkSession, cat: str, schema: str, revision_id: str) -> None:
305
- version_table = f"{cat}.{schema}.{VERSION_TABLE}"
426
+ def record_revision(spark: SparkSession, version_table_name: str, revision_id: str) -> None:
306
427
  revision_literal = revision_id.replace("'", "''")
307
- spark.sql(f"delete from {version_table}")
308
- spark.sql(f"insert into {version_table} (version_num) values ('{revision_literal}')")
428
+ spark.sql(f"delete from {version_table_name}")
429
+ spark.sql(f"insert into {version_table_name} (version_num) values ('{revision_literal}')")
309
430
 
310
431
 
311
432
  def upgrade(
312
433
  spark: SparkSession,
434
+ version_table_name: str,
313
435
  output_folder: str,
314
436
  migrations_dir: str,
315
- cat: str,
316
- schema: str,
317
437
  max_revision_id: Optional[str] = None,
438
+ template_vals: Optional[Dict[str, str]] = None,
318
439
  ) -> None:
319
440
  """
320
441
  TODO XXX permit applying up to a revision below the head revision
@@ -324,7 +445,7 @@ def upgrade(
324
445
  logger.info(f"no chain at {migrations_dir}; skipping it;")
325
446
  return
326
447
  unapplied = get_unapplied_migrations_list(
327
- spark, migrations_dir, cat=cat, schema=schema, max_revision_id=max_revision_id
448
+ spark, version_table_name, migrations_dir, max_revision_id=max_revision_id
328
449
  )
329
450
 
330
451
  logger.info(
@@ -334,47 +455,85 @@ def upgrade(
334
455
 
335
456
  env = Environment(loader=FileSystemLoader(migrations_dir), autoescape=select_autoescape())
336
457
  for migration in unapplied:
337
- render_and_apply(spark, output_folder, env.get_template(migration.template_name), cat=cat, schema=schema)
338
- record_revision(spark, cat, schema, migration.revision_id)
458
+ render_and_apply(spark, output_folder, env.get_template(migration.template_name), template_vals)
459
+ record_revision(spark, version_table_name, migration.revision_id)
339
460
  logger.info(f"{migration.template_name} done; now at {migration.revision_id};")
340
461
 
341
462
 
342
463
  def run_upgrade(
343
464
  spark: SparkSession,
344
- cat: str,
345
- schema: str,
346
465
  output_folder: str,
347
466
  migrations_dir: str,
348
467
  max_revision_id: Optional[str] = None,
468
+ template_vals: Optional[Dict[str, str]] = None,
349
469
  ) -> None:
350
- initial_output_folder = os.path.join(output_folder, INITIAL_MIGRATIONS_PACKAGE_PATH)
351
470
  client_output_folder = os.path.join(output_folder, CLIENT_OUTPUT_DIRNAME)
352
- os.makedirs(initial_output_folder, exist_ok=True)
353
471
  os.makedirs(client_output_folder, exist_ok=True)
354
- version_table = f"{cat}.{schema}.{VERSION_TABLE}"
355
- if spark.catalog.tableExists(version_table):
356
- logger.info(f"{version_table} already exists; skipping the initial migrations;")
472
+ version_table_name = get_version_table_name(spark)
473
+ if spark.catalog.tableExists(version_table_name):
474
+ logger.info(f"{version_table_name} already exists;")
357
475
  else:
358
- migrate_initial(spark, initial_output_folder, cat, schema)
359
- upgrade(spark, client_output_folder, migrations_dir, cat, schema, max_revision_id)
476
+ spark.sql(f"create table if not exists {version_table_name}(version_num string not null);")
477
+ upgrade(spark, version_table_name, client_output_folder, migrations_dir, max_revision_id, template_vals)
478
+
479
+
480
+ def format_history_line(
481
+ migration: Migration, head_revisions: Set[str], branch_point_revisions: Set[str]
482
+ ) -> str: # pragma: no cover
483
+ """one alembic-shaped line: `prev -> rev (tags), filename`."""
484
+ tags = []
485
+ if migration.revision_id in head_revisions:
486
+ tags.append("head")
487
+ if migration.revision_id in branch_point_revisions:
488
+ tags.append("branchpoint")
489
+ tag_text = f" ({', '.join(tags)})" if tags else ""
490
+ return f"{migration.prev_revision_id or BASE_LABEL} -> {migration.revision_id}{tag_text}, {migration.template_name}"
491
+
492
+
493
+ def format_chain_notes(summary: ChainSummary) -> List[str]: # pragma: no cover
494
+ """what is wrong with this chain, below the listing, or nothing if it is sound."""
495
+ notes = []
496
+ if len(summary.heads) > 1:
497
+ notes.append(
498
+ f"split heads: {', '.join(migration.revision_id for migration in summary.heads)}; "
499
+ f"a chain with more than one head cannot be applied until it is rejoined;"
500
+ )
501
+ for revision_id, followers in sorted(summary.branch_points.items()):
502
+ notes.append(
503
+ f"{revision_id} is the prev_revision_id of "
504
+ f"{', '.join(migration.template_name for migration in followers)};"
505
+ )
506
+ if len(summary.roots) > 1:
507
+ notes.append(
508
+ f"more than one root: {', '.join(migration.template_name for migration in summary.roots)}; "
509
+ f"a chain has exactly one migration following no other;"
510
+ )
511
+ if summary.orphans:
512
+ notes.append(
513
+ f"no root reaches {', '.join(migration.template_name for migration in summary.orphans)}; "
514
+ f"they are listed together at the tail end of the chain order, having none of their own;"
515
+ )
516
+ return notes
360
517
 
361
518
 
362
519
  def main(
363
- cat: str,
364
- schema: str,
365
520
  migrations_dir: str,
366
521
  output_parent_path: Optional[str] = None,
367
522
  max_revision_id: Optional[str] = None,
523
+ template_vals: Optional[Dict[str, str]] = None,
368
524
  ) -> None:
369
525
  output_folder = get_output_folder(output_parent_path or os.path.join(os.getcwd(), "migrations_out"))
370
526
  os.makedirs(output_folder)
371
- run_upgrade(get_spark(), cat, schema, output_folder, migrations_dir, max_revision_id)
527
+ spark = get_spark()
528
+ run_upgrade(spark, output_folder, migrations_dir, max_revision_id, template_vals)
372
529
 
373
530
 
374
531
  def _cli_main(
375
- cat: str, schema: str, migrations_dir: str, max_revision_id: Optional[str] = None
532
+ migrations_dir: str,
533
+ max_revision_id: Optional[str] = None,
534
+ template_vals: Optional[Dict[str, str]] = None,
376
535
  ) -> None: # pragma: no cover
377
- main(cat, schema, migrations_dir, max_revision_id=max_revision_id)
536
+ main(migrations_dir, max_revision_id=max_revision_id, template_vals=template_vals)
378
537
 
379
538
 
380
539
  def _cli_create_new_migration(
@@ -383,13 +542,35 @@ def _cli_create_new_migration(
383
542
  create_new_migration(message, template_path or get_default_template_path(), output_path, add_is_dbr=add_is_dbr)
384
543
 
385
544
 
545
+ def _cli_history(migrations_dir: str, limit: Optional[int], oldest_first: bool) -> None: # pragma: no cover
546
+ """the listing goes to stdout, so it pipes; the log records go to stderr as always."""
547
+ for line in history(migrations_dir, limit=limit, oldest_first=oldest_first):
548
+ print(line)
549
+
550
+
551
+ def parse_template_vals(text: str) -> Dict[str, str]:
552
+ """`foo=bar,gee=wiz` -> {"foo": "bar", "gee": "wiz"}; the argparse `type` of --template-vals.
553
+
554
+ a key has to be usable as a jinja variable, so an identifier: one that is not would
555
+ render as nothing at all and the migration would go out silently missing it.
556
+ """
557
+ template_vals: Dict[str, str] = {}
558
+ for pair in text.split(","):
559
+ if not pair.strip():
560
+ continue
561
+ key, separator, value = pair.partition("=")
562
+ key = key.strip()
563
+ if not separator or not key.isidentifier():
564
+ raise argparse.ArgumentTypeError(f"expected key=value with an identifier key, got {pair!r};")
565
+ template_vals[key] = value.strip()
566
+ return template_vals
567
+
568
+
386
569
  def build_parser() -> argparse.ArgumentParser: # pragma: no cover
387
570
  parser = argparse.ArgumentParser(description="spark_sql_migrations migration utilities")
388
571
  subparsers = parser.add_subparsers(dest="command", required=True)
389
572
 
390
573
  p_run = subparsers.add_parser("run", help="upgrade to head/apply pending migrations")
391
- p_run.add_argument("--cat", type=str, default="spark_catalog")
392
- p_run.add_argument("--schema", type=str, default="default")
393
574
  p_run.add_argument("--migrations-dir", type=str, required=True, help="the directory holding the migration chain")
394
575
  p_run.add_argument(
395
576
  "--max-revision-id",
@@ -397,6 +578,15 @@ def build_parser() -> argparse.ArgumentParser: # pragma: no cover
397
578
  default=None,
398
579
  help="apply the migration carrying this revision_id and stop; defaults to applying the chain to its head",
399
580
  )
581
+ p_run.add_argument(
582
+ "--template-vals",
583
+ type=parse_template_vals,
584
+ default=None,
585
+ metavar="KEY=VALUE[,KEY=VALUE...]",
586
+ help="jinja render vars handed to every migration. is_dbr is always set from the engine and cannot be "
587
+ "overridden here. the whole thing is one argument, equal signs and all: "
588
+ "--template-vals=foo=bar,gee=wiz,pp=qq",
589
+ )
400
590
  p_run.set_defaults(func=_cli_main)
401
591
 
402
592
  p_create = subparsers.add_parser("create_new_migration", help="create a new migration from the template")
@@ -419,6 +609,22 @@ def build_parser() -> argparse.ArgumentParser: # pragma: no cover
419
609
  )
420
610
  p_create.set_defaults(func=_cli_create_new_migration)
421
611
 
612
+ p_history = subparsers.add_parser("history", help="list the migration chain on disk, newest first")
613
+ p_history.add_argument("--migrations-dir", type=str, required=True, help="the directory holding the chain")
614
+ p_history.add_argument(
615
+ "-n",
616
+ "--limit",
617
+ type=int,
618
+ default=None,
619
+ help="list at most this many migrations, counted from the end the listing starts at",
620
+ )
621
+ p_history.add_argument(
622
+ "--oldest-first",
623
+ action="store_true",
624
+ help="list from the root up instead of from the head down",
625
+ )
626
+ p_history.set_defaults(func=_cli_history)
627
+
422
628
  return parser
423
629
 
424
630
 
@@ -0,0 +1,33 @@
1
+ Metadata-Version: 2.4
2
+ Name: spark_sql_migrations
3
+ Version: 0.1.0
4
+ Summary: Chained, idempotent SQL migrations for Spark and Databricks.
5
+ Home-page: https://github.com/mcjug2015/spark_sql_migrations
6
+ Author: Victor Semenov
7
+ License: Apache-2.0
8
+ Project-URL: Source, https://github.com/mcjug2015/spark_sql_migrations
9
+ Project-URL: Issues, https://github.com/mcjug2015/spark_sql_migrations/issues
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: SQL
14
+ Classifier: Topic :: Database
15
+ Classifier: Typing :: Typed
16
+ Requires-Python: >=3.10
17
+ License-File: LICENSE
18
+ Requires-Dist: Jinja2<4,>=3.1
19
+ Provides-Extra: local
20
+ Requires-Dist: pyspark[connect]<5,>=4.0; extra == "local"
21
+ Requires-Dist: delta-spark<5,>=4.0; extra == "local"
22
+ Provides-Extra: databricks
23
+ Requires-Dist: databricks-connect<19,>=18; extra == "databricks"
24
+ Dynamic: author
25
+ Dynamic: classifier
26
+ Dynamic: home-page
27
+ Dynamic: license
28
+ Dynamic: license-file
29
+ Dynamic: project-url
30
+ Dynamic: provides-extra
31
+ Dynamic: requires-dist
32
+ Dynamic: requires-python
33
+ Dynamic: summary
@@ -13,8 +13,5 @@ spark_sql_migrations.egg-info/namespace_packages.txt
13
13
  spark_sql_migrations.egg-info/requires.txt
14
14
  spark_sql_migrations.egg-info/top_level.txt
15
15
  spark_sql_migrations/migration_templates/default_migration_template.sql
16
- spark_sql_migrations/migrations_initial/first_01_intial_create_cat.sql
17
- spark_sql_migrations/migrations_initial/first_02_intial_create_schema.sql
18
- spark_sql_migrations/migrations_initial/first_03_intial_create_state_table.sql
19
16
  spark_sql_migrations/spark_sql/__init__.py
20
17
  spark_sql_migrations/spark_sql/spark_sql.py