spark-sql-migrations 0.0.2__tar.gz → 0.0.4__tar.gz

{spark_sql_migrations-0.0.2 → spark_sql_migrations-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spark_sql_migrations
-Version: 0.0.2
+Version: 0.0.4
 Summary: Chained, idempotent SQL migrations for Spark and Databricks.
 Home-page: https://github.com/mcjug2015/spark_sql_migrations
 Author: Victor Semenov
@@ -81,10 +81,12 @@ Migrations come from two different owners, and they behave differently:
 | **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
 | **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
 | **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
-| **when applied** | every run | once, then recorded |
+| **when applied** | until the version table exists | once, then recorded |
 
-The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
-chains are applied once each and their head revision is written to the version table.
+The bootstrap chain is applied on every run until `_spark_migrations_version` exists, and
+skipped from then on. A run that fails partway through bootstrapping replays it, so it must
+still be idempotent. Your own chains are applied once each and their head revision is
+written to the version table.
 
 ## Laying out your chains
 
@@ -106,9 +108,10 @@ Choose deliberately. SQL that only Databricks understands must not live in
 spark-sql-migrations run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
 ```
 
-The command configures logging via `custom_logging.setup_logging()`, so each applied
-migration is logged to stdout and to `local_log.log` in the working directory (override
-with `SPARK_SQL_MIGRATIONS_LOG_FILE`). The command needs a Spark flavour installed; it
+The command configures logging via `logging.basicConfig()`, so each applied migration is
+logged to stderr at INFO. It leaves an already-configured root logger alone, and every
+other entry point into this library inherits the host application's logging untouched.
+The command needs a Spark flavour installed; it
 fails to import on a bare install. `python -m spark_sql_migrations.spark_sql.spark_sql`
 is equivalent.
 

{spark_sql_migrations-0.0.2 → spark_sql_migrations-0.0.4}/setup.py

@@ -82,10 +82,12 @@ Migrations come from two different owners, and they behave differently:
 | **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
 | **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
 | **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
-| **when applied** | every run | once, then recorded |
+| **when applied** | until the version table exists | once, then recorded |
 
-The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
-chains are applied once each and their head revision is written to the version table.
+The bootstrap chain is applied on every run until `_spark_migrations_version` exists, and
+skipped from then on. A run that fails partway through bootstrapping replays it, so it must
+still be idempotent. Your own chains are applied once each and their head revision is
+written to the version table.
 
 ## Laying out your chains
 
@@ -110,9 +112,10 @@ spark-sql-migrations run \
     --migrations-dir path/to/migrations
 ```
 
-The command configures logging via `custom_logging.setup_logging()`, so each applied
-migration is logged to stdout and to `local_log.log` in the working directory (override
-with `SPARK_SQL_MIGRATIONS_LOG_FILE`). The command needs a Spark flavour installed; it
+The command configures logging via `logging.basicConfig()`, so each applied migration is
+logged to stderr at INFO. It leaves an already-configured root logger alone, and every
+other entry point into this library inherits the host application's logging untouched.
+The command needs a Spark flavour installed; it
 fails to import on a bare install. `python -m spark_sql_migrations.spark_sql.spark_sql`
 is equivalent.
 
@@ -251,5 +254,5 @@ Apache License 2.0 — see [LICENSE](LICENSE).
     },
     'python_requires': '>=3.10',
     'url': 'https://github.com/mcjug2015/spark_sql_migrations',
-    'version': '0.0.2',
+    'version': '0.0.4',
 })

{spark_sql_migrations-0.0.2 → spark_sql_migrations-0.0.4}/spark_sql_migrations/__init__.py

@@ -2,7 +2,6 @@ import logging
 
 # A library must not configure logging when it is imported. A NullHandler on the
 # package root keeps our records off logging.lastResort while leaving every
-# decision about handlers, levels and formats to the host application. An
-# application that wants our opinionated setup calls
-# spark_sql_migrations.custom_logging.setup_logging() from its own entry point.
+# decision about handlers, levels and formats to the host application, which
+# configures logging from its own entry point.
 logging.getLogger(__name__).addHandler(logging.NullHandler())

{spark_sql_migrations-0.0.2 → spark_sql_migrations-0.0.4}/spark_sql_migrations/spark_sql/spark_sql.py

@@ -5,7 +5,8 @@ Two kinds of migration exist, and they are owned by different parties:
 * the *initial* chain and the new-migration template ship inside this package.
   They bootstrap the catalog, the schema and the version table, are selected by
   filename suffix (``_all.sql`` / ``_dbr_only.sql``) rather than by revision,
-  and are re-applied on every run -- so they must be idempotent.
+  and are applied on every run until the version table exists -- so a run that
+  failed partway through replays them, and they must be idempotent.
 * the ``all_spark_migrations/`` and ``dbr_only_migrations/`` chains belong to
   the consuming project. The caller passes their parent directory as
   ``migrations_root``; each chain is walked from its root via
@@ -24,7 +25,6 @@ from dataclasses import dataclass
 from jinja2 import Environment, FileSystemLoader, PackageLoader, select_autoescape
 from pyspark.sql.functions import col
 
-from spark_sql_migrations import custom_logging
 from spark_sql_migrations.spark_utils import get_spark, is_dbr
 
 logger = logging.getLogger(__name__)
@@ -261,7 +261,11 @@ def record_revision(spark, cat: str, schema: str, migration_type: str, revision_
 
 
 def migrate_w_rev(spark, output_folder, migrations_root, cat: str, schema: str, migration_type: str):
-    """apply the migrations of one client chain that VERSION_TABLE hasn't recorded yet."""
+    """
+    apply the migrations of one client chain that VERSION_TABLE hasn't recorded yet.
+    TODO XXX permit applying up to a revision below the head revision
+    TODO XXX add backwards migrations, permit migrating backwards
+    """
     migrations_dir = get_migrations_dir(migrations_root, migration_type)
     unapplied = get_unapplied_migrations_list(spark, migrations_dir, migration_type, cat=cat, schema=schema)
 
@@ -285,7 +289,11 @@ def run_migrations(spark, cat, schema, output_folder, migrations_root):
     os.makedirs(initial_output_folder, exist_ok=True)
     os.makedirs(dbr_ouput_folder, exist_ok=True)
     os.makedirs(all_spark_ouput_folder, exist_ok=True)
-    migrate_initial(spark, initial_output_folder, cat, schema)
+    version_table = f"{cat}.{schema}.{VERSION_TABLE}"
+    if spark.catalog.tableExists(version_table):
+        logger.info(f"{version_table} already exists; skipping the initial migrations;")
+    else:
+        migrate_initial(spark, initial_output_folder, cat, schema)
     if is_dbr():
         migrate_w_rev(spark, dbr_ouput_folder, migrations_root, cat, schema, DBR_ONLY)
     migrate_w_rev(spark, all_spark_ouput_folder, migrations_root, cat, schema, ALL_SPARK)
@@ -330,9 +338,15 @@ def cli(argv=None):  # pragma: no cover
     """the `spark-sql-migrations` console command.
 
     an application entry point, so unlike the rest of this package it configures
-    logging -- see custom_logging's module docstring.
+    logging. basicConfig is a no-op once the root logger has handlers, which keeps
+    this from clobbering a host application that got there first.
     """
-    custom_logging.setup_logging()
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s %(asctime)s %(name)s : %(message)s",
+        datefmt="%y/%m/%d %H:%M:%S",
+    )
+    logging.getLogger("py4j").setLevel(logging.ERROR)
     kwargs = vars(build_parser().parse_args(argv))
     func = kwargs.pop("func")
     kwargs.pop("command")

{spark_sql_migrations-0.0.2 → spark_sql_migrations-0.0.4}/spark_sql_migrations.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spark_sql_migrations
-Version: 0.0.2
+Version: 0.0.4
 Summary: Chained, idempotent SQL migrations for Spark and Databricks.
 Home-page: https://github.com/mcjug2015/spark_sql_migrations
 Author: Victor Semenov
@@ -81,10 +81,12 @@ Migrations come from two different owners, and they behave differently:
 | **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
 | **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
 | **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
-| **when applied** | every run | once, then recorded |
+| **when applied** | until the version table exists | once, then recorded |
 
-The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
-chains are applied once each and their head revision is written to the version table.
+The bootstrap chain is applied on every run until `_spark_migrations_version` exists, and
+skipped from then on. A run that fails partway through bootstrapping replays it, so it must
+still be idempotent. Your own chains are applied once each and their head revision is
+written to the version table.
 
 ## Laying out your chains
 
@@ -106,9 +108,10 @@ Choose deliberately. SQL that only Databricks understands must not live in
 spark-sql-migrations run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
 ```
 
-The command configures logging via `custom_logging.setup_logging()`, so each applied
-migration is logged to stdout and to `local_log.log` in the working directory (override
-with `SPARK_SQL_MIGRATIONS_LOG_FILE`). The command needs a Spark flavour installed; it
+The command configures logging via `logging.basicConfig()`, so each applied migration is
+logged to stderr at INFO. It leaves an already-configured root logger alone, and every
+other entry point into this library inherits the host application's logging untouched.
+The command needs a Spark flavour installed; it
 fails to import on a bare install. `python -m spark_sql_migrations.spark_sql.spark_sql`
 is equivalent.
 

{spark_sql_migrations-0.0.2 → spark_sql_migrations-0.0.4}/spark_sql_migrations.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ MANIFEST.in
 backend_shim.py
 setup.py
 spark_sql_migrations/__init__.py
-spark_sql_migrations/custom_logging.py
 spark_sql_migrations/py.typed
 spark_sql_migrations/spark_utils.py
 spark_sql_migrations.egg-info/PKG-INFO

spark_sql_migrations-0.0.2/spark_sql_migrations/custom_logging.py

@@ -1,78 +0,0 @@
-"""Opinionated logging configuration, for *applications* to opt into.
-
-Importing spark_sql_migrations never configures logging: library modules take a plain
-``logging.getLogger(__name__)`` and inherit whatever the host application set
-up. An application that wants this format calls :func:`setup_logging` from its
-own entry point -- never at import time, and never from library code, because
-configuring the root logger is the application's decision to make.
-"""
-
-import logging.config
-import os
-
-LOG_FILE_ENV_VAR = "SPARK_SQL_MIGRATIONS_LOG_FILE"
-DEFAULT_LOG_FILENAME = "local_log.log"
-
-FORMAT = "%(levelname)s %(asctime)s %(filename)s->%(funcName)s->%(lineno)d : %(message)s"
-
-
-def get_log_file_path():
-    """where the file handler writes.
-
-    resolved against the caller's cwd for the same reason as the spark warehouse
-    dir: a __file__-relative default would land inside site-packages once this
-    is installed from a wheel.
-    """
-    return os.environ.get(LOG_FILE_ENV_VAR, os.path.join(os.getcwd(), DEFAULT_LOG_FILENAME))
-
-
-def build_config():
-    """built per call, not a module constant: get_log_file_path() reads the
-    environment, and at module-import time the consumer hasn't set it yet."""
-    return {
-        "version": 1,
-        "disable_existing_loggers": False,
-        "formatters": {
-            "simple": {
-                "format": FORMAT,
-                "datefmt": "%y/%m/%d %H:%M:%S",
-            }
-        },
-        "handlers": {
-            "stdout": {
-                "class": "logging.StreamHandler",
-                "level": "INFO",
-                "formatter": "simple",
-                "stream": "ext://sys.stdout",
-            },
-            "stderr": {
-                "class": "logging.StreamHandler",
-                "level": "ERROR",
-                "formatter": "simple",
-                "stream": "ext://sys.stderr",
-            },
-            "file": {
-                "class": "logging.FileHandler",
-                "formatter": "simple",
-                "filename": get_log_file_path(),
-                "mode": "a",
-            },
-        },
-        "root": {"level": "DEBUG", "handlers": ["stderr", "stdout", "file"]},
-    }
-
-
-def is_logging_configured():
-    # Check if the root logger has any handlers explicitly assigned
-    return len(logging.getLogger().handlers) >= 3
-
-
-def setup_logging():
-    """configure the root logger with our handlers; safe to call more than once.
-
-    for application entry points only -- see the module docstring.
-    """
-    if not is_logging_configured():
-        logging.config.dictConfig(build_config())
-        logging.getLogger("py4j").setLevel(logging.ERROR)
-    return logging