spark-sql-migrations 0.0.3__tar.gz → 0.0.5__tar.gz

{spark_sql_migrations-0.0.3 → spark_sql_migrations-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spark_sql_migrations
-Version: 0.0.3
+Version: 0.0.5
 Summary: Chained, idempotent SQL migrations for Spark and Databricks.
 Home-page: https://github.com/mcjug2015/spark_sql_migrations
 Author: Victor Semenov
@@ -81,10 +81,12 @@ Migrations come from two different owners, and they behave differently:
 | **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
 | **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
 | **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
-| **when applied** | every run | once, then recorded |
+| **when applied** | until the version table exists | once, then recorded |
 
-The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
-chains are applied once each and their head revision is written to the version table.
+The bootstrap chain is applied on every run until `_spark_migrations_version` exists, and
+skipped from then on. A run that fails partway through bootstrapping replays it, so it must
+still be idempotent. Your own chains are applied once each and their head revision is
+written to the version table.
 
 ## Laying out your chains
 

{spark_sql_migrations-0.0.3 → spark_sql_migrations-0.0.5}/setup.py

@@ -82,10 +82,12 @@ Migrations come from two different owners, and they behave differently:
 | **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
 | **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
 | **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
-| **when applied** | every run | once, then recorded |
+| **when applied** | until the version table exists | once, then recorded |
 
-The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
-chains are applied once each and their head revision is written to the version table.
+The bootstrap chain is applied on every run until `_spark_migrations_version` exists, and
+skipped from then on. A run that fails partway through bootstrapping replays it, so it must
+still be idempotent. Your own chains are applied once each and their head revision is
+written to the version table.
 
 ## Laying out your chains
 
@@ -252,5 +254,5 @@ Apache License 2.0 — see [LICENSE](LICENSE).
     },
     'python_requires': '>=3.10',
     'url': 'https://github.com/mcjug2015/spark_sql_migrations',
-    'version': '0.0.3',
+    'version': '0.0.5',
 })

{spark_sql_migrations-0.0.3 → spark_sql_migrations-0.0.5}/spark_sql_migrations/spark_sql/spark_sql.py

@@ -5,7 +5,8 @@ Two kinds of migration exist, and they are owned by different parties:
 * the *initial* chain and the new-migration template ship inside this package.
   They bootstrap the catalog, the schema and the version table, are selected by
   filename suffix (``_all.sql`` / ``_dbr_only.sql``) rather than by revision,
-  and are re-applied on every run -- so they must be idempotent.
+  and are applied on every run until the version table exists -- so a run that
+  failed partway through replays them, and they must be idempotent.
 * the ``all_spark_migrations/`` and ``dbr_only_migrations/`` chains belong to
   the consuming project. The caller passes their parent directory as
   ``migrations_root``; each chain is walked from its root via
@@ -149,20 +150,20 @@ def get_migrations_list(migrations_dir):
     return get_ordered_migration_objs(migrations)
 
 
+def get_table_version(spark, migration_type, cat: str, schema: str):
+    version_table = f"{cat}.{schema}.{VERSION_TABLE}"
+    if not spark.catalog.tableExists(version_table):
+        return None
+    return spark.read.table(version_table).where(col("migration_type") == migration_type).select("version_num").head()
+
+
 def get_unapplied_migrations_list(spark, migrations_dir, migration_type, cat: str, schema: str):
     """the tail of one chain after the revision VERSION_TABLE records for it.
 
     the whole chain when the table is missing or holds no row for this type.
     """
     ordered = get_migrations_list(migrations_dir)
-
-    version_table = f"{cat}.{schema}.{VERSION_TABLE}"
-    if not spark.catalog.tableExists(version_table):
-        return ordered
-
-    current = (
-        spark.read.table(version_table).where(col("migration_type") == migration_type).select("version_num").head()
-    )
+    current = get_table_version(spark, migration_type, cat, schema)
     if not current:
         return ordered
 
@@ -179,9 +180,6 @@ def get_unapplied_migrations_list(spark, migrations_dir, migration_type, cat: st
 
 
 def create_new_migration(message: str, template_path: str, output_path: str, truncate_slug_length: int = 40):
-    """
-    TODO XXX start looking up prev revision id from db or files and including it
-    """
     rev_id = uuid.uuid4().hex[-12:]
 
     date_prefix = datetime.datetime.today().strftime("%y%m%d")
@@ -194,8 +192,15 @@ def create_new_migration(message: str, template_path: str, output_path: str, tru
 
     migration_path = os.path.join(output_path, migration_filename)
 
+    try:
+        ordered_migrations = get_migrations_list(output_path)
+        last_migration_rev_id = ordered_migrations[-1].revision_id if len(ordered_migrations) > 0 else ""
+    except ValueError:
+        last_migration_rev_id = ""
+
     default_migration_content = open(template_path).read()
     default_migration_content = default_migration_content.replace("{{revision_id}}", rev_id)
+    default_migration_content = default_migration_content.replace("{{prev_revision_id}}", last_migration_rev_id)
 
     with open(migration_path, "w") as migration_file:
         migration_file.write(default_migration_content)
@@ -288,7 +293,11 @@ def run_migrations(spark, cat, schema, output_folder, migrations_root):
     os.makedirs(initial_output_folder, exist_ok=True)
     os.makedirs(dbr_ouput_folder, exist_ok=True)
     os.makedirs(all_spark_ouput_folder, exist_ok=True)
-    migrate_initial(spark, initial_output_folder, cat, schema)
+    version_table = f"{cat}.{schema}.{VERSION_TABLE}"
+    if spark.catalog.tableExists(version_table):
+        logger.info(f"{version_table} already exists; skipping the initial migrations;")
+    else:
+        migrate_initial(spark, initial_output_folder, cat, schema)
     if is_dbr():
         migrate_w_rev(spark, dbr_ouput_folder, migrations_root, cat, schema, DBR_ONLY)
     migrate_w_rev(spark, all_spark_ouput_folder, migrations_root, cat, schema, ALL_SPARK)
@@ -322,7 +331,12 @@ def build_parser():  # pragma: no cover
 
     p_create = subparsers.add_parser("create_new_migration", help="create a new migration from the template")
     p_create.add_argument("--message", required=True)
-    p_create.add_argument("--output-path", required=True, help="the chain directory to write the new migration into")
+    p_create.add_argument(
+        "--output-path",
+        required=True,
+        help="the chain directory to write the new migration into. if sql files are in there, the last sql's rev_id"
+        "will be set as the new migrations prev_rev_id.",
+    )
     p_create.add_argument("--template-path", default=None, help="defaults to the template shipped in this package")
     p_create.set_defaults(func=_cli_create_new_migration)
 

{spark_sql_migrations-0.0.3 → spark_sql_migrations-0.0.5}/spark_sql_migrations.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spark_sql_migrations
-Version: 0.0.3
+Version: 0.0.5
 Summary: Chained, idempotent SQL migrations for Spark and Databricks.
 Home-page: https://github.com/mcjug2015/spark_sql_migrations
 Author: Victor Semenov
@@ -81,10 +81,12 @@ Migrations come from two different owners, and they behave differently:
 | **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
 | **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
 | **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
-| **when applied** | every run | once, then recorded |
+| **when applied** | until the version table exists | once, then recorded |
 
-The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
-chains are applied once each and their head revision is written to the version table.
+The bootstrap chain is applied on every run until `_spark_migrations_version` exists, and
+skipped from then on. A run that fails partway through bootstrapping replays it, so it must
+still be idempotent. Your own chains are applied once each and their head revision is
+written to the version table.
 
 ## Laying out your chains