spark-sql-migrations 0.0.1__tar.gz → 0.0.3__tar.gz

{spark_sql_migrations-0.0.1 → spark_sql_migrations-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spark_sql_migrations
-Version: 0.0.1
+Version: 0.0.3
 Summary: Chained, idempotent SQL migrations for Spark and Databricks.
 Home-page: https://github.com/mcjug2015/spark_sql_migrations
 Author: Victor Semenov
@@ -103,9 +103,16 @@ Choose deliberately. SQL that only Databricks understands must not live in
 ## Running migrations
 
 ```bash
-python -m spark_sql_migrations.spark_sql.spark_sql run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
+spark-sql-migrations run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
 ```
 
+The command configures logging via `logging.basicConfig()`, so each applied migration is
+logged to stderr at INFO. It leaves an already-configured root logger alone, and every
+other entry point into this library inherits the host application's logging untouched.
+The command needs a Spark flavour installed; it
+fails to import on a bare install. `python -m spark_sql_migrations.spark_sql.spark_sql`
+is equivalent.
+
 Or from Python, which is what a consuming project usually wraps:
 
 ```python
@@ -122,7 +129,8 @@ def get_migrations_dir():
 main(cat="spark_catalog", schema="default", migrations_dir=get_migrations_dir())
 ```
 
-`main` builds its own session via `get_spark()`. To supply your own — a session already
+`main` builds its own session via `get_spark()` and, being library code, leaves logging to
+you. To supply your own — a session already
 configured by your job, say — call `run_migrations(spark, cat, schema, output_folder,
 migrations_root)` directly.
 
@@ -134,7 +142,7 @@ before it is executed, so you can always read the exact SQL a run applied.
 Generate one rather than hand-rolling the header:
 
 ```bash
-python -m spark_sql_migrations.spark_sql.spark_sql create_new_migration     --message "add batch id to metrics"     --output-path path/to/migrations/all_spark_migrations
+spark-sql-migrations create_new_migration     --message "add batch id to metrics"     --output-path path/to/migrations/all_spark_migrations
 ```
 
 That writes `<yymmdd>_<slug>_<revision_id>.sql` from the template shipped in the package

{spark_sql_migrations-0.0.1 → spark_sql_migrations-0.0.3}/setup.py

@@ -15,6 +15,11 @@ setup(**{
         'Typing :: Typed',
     ],
     'description': 'Chained, idempotent SQL migrations for Spark and Databricks.',
+    'entry_points': {
+        'console_scripts': [
+            'spark-sql-migrations = spark_sql_migrations.spark_sql.spark_sql:cli',
+        ],
+    },
     'extras_require': {
         'local': [
             'pyspark[connect]>=4.0,<5',
@@ -99,12 +104,19 @@ Choose deliberately. SQL that only Databricks understands must not live in
 ## Running migrations
 
 ```bash
-python -m spark_sql_migrations.spark_sql.spark_sql run \
+spark-sql-migrations run \
     --cat spark_catalog \
     --schema default \
     --migrations-dir path/to/migrations
 ```
 
+The command configures logging via `logging.basicConfig()`, so each applied migration is
+logged to stderr at INFO. It leaves an already-configured root logger alone, and every
+other entry point into this library inherits the host application's logging untouched.
+The command needs a Spark flavour installed; it
+fails to import on a bare install. `python -m spark_sql_migrations.spark_sql.spark_sql`
+is equivalent.
+
 Or from Python, which is what a consuming project usually wraps:
 
 ```python
@@ -121,7 +133,8 @@ def get_migrations_dir():
 main(cat="spark_catalog", schema="default", migrations_dir=get_migrations_dir())
 ```
 
-`main` builds its own session via `get_spark()`. To supply your own — a session already
+`main` builds its own session via `get_spark()` and, being library code, leaves logging to
+you. To supply your own — a session already
 configured by your job, say — call `run_migrations(spark, cat, schema, output_folder,
 migrations_root)` directly.
 
@@ -133,7 +146,7 @@ before it is executed, so you can always read the exact SQL a run applied.
 Generate one rather than hand-rolling the header:
 
 ```bash
-python -m spark_sql_migrations.spark_sql.spark_sql create_new_migration \
+spark-sql-migrations create_new_migration \
     --message "add batch id to metrics" \
     --output-path path/to/migrations/all_spark_migrations
 ```
@@ -239,5 +252,5 @@ Apache License 2.0 — see [LICENSE](LICENSE).
     },
     'python_requires': '>=3.10',
     'url': 'https://github.com/mcjug2015/spark_sql_migrations',
-    'version': '0.0.1',
+    'version': '0.0.3',
 })

{spark_sql_migrations-0.0.1 → spark_sql_migrations-0.0.3}/spark_sql_migrations/__init__.py

@@ -2,7 +2,6 @@ import logging
 
 # A library must not configure logging when it is imported. A NullHandler on the
 # package root keeps our records off logging.lastResort while leaving every
-# decision about handlers, levels and formats to the host application. An
-# application that wants our opinionated setup calls
-# spark_sql_migrations.custom_logging.setup_logging() from its own entry point.
+# decision about handlers, levels and formats to the host application, which
+# configures logging from its own entry point.
 logging.getLogger(__name__).addHandler(logging.NullHandler())

{spark_sql_migrations-0.0.1 → spark_sql_migrations-0.0.3}/spark_sql_migrations/spark_sql/spark_sql.py

@@ -260,7 +260,11 @@ def record_revision(spark, cat: str, schema: str, migration_type: str, revision_
 
 
 def migrate_w_rev(spark, output_folder, migrations_root, cat: str, schema: str, migration_type: str):
-    """apply the migrations of one client chain that VERSION_TABLE hasn't recorded yet."""
+    """
+    apply the migrations of one client chain that VERSION_TABLE hasn't recorded yet.
+    TODO XXX permit applying up to a revision below the head revision
+    TODO XXX add backwards migrations, permit migrating backwards
+    """
     migrations_dir = get_migrations_dir(migrations_root, migration_type)
     unapplied = get_unapplied_migrations_list(spark, migrations_dir, migration_type, cat=cat, schema=schema)
 
@@ -325,9 +329,24 @@ def build_parser():  # pragma: no cover
     return parser
 
 
-if __name__ == "__main__":  # pragma: no cover
-    args = build_parser().parse_args()
-    kwargs = vars(args)
+def cli(argv=None):  # pragma: no cover
+    """the `spark-sql-migrations` console command.
+
+    an application entry point, so unlike the rest of this package it configures
+    logging. basicConfig is a no-op once the root logger has handlers, which keeps
+    this from clobbering a host application that got there first.
+    """
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s %(asctime)s %(name)s : %(message)s",
+        datefmt="%y/%m/%d %H:%M:%S",
+    )
+    logging.getLogger("py4j").setLevel(logging.ERROR)
+    kwargs = vars(build_parser().parse_args(argv))
     func = kwargs.pop("func")
     kwargs.pop("command")
     func(**kwargs)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

{spark_sql_migrations-0.0.1 → spark_sql_migrations-0.0.3}/spark_sql_migrations.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spark_sql_migrations
-Version: 0.0.1
+Version: 0.0.3
 Summary: Chained, idempotent SQL migrations for Spark and Databricks.
 Home-page: https://github.com/mcjug2015/spark_sql_migrations
 Author: Victor Semenov
@@ -103,9 +103,16 @@ Choose deliberately. SQL that only Databricks understands must not live in
 ## Running migrations
 
 ```bash
-python -m spark_sql_migrations.spark_sql.spark_sql run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
+spark-sql-migrations run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
 ```
 
+The command configures logging via `logging.basicConfig()`, so each applied migration is
+logged to stderr at INFO. It leaves an already-configured root logger alone, and every
+other entry point into this library inherits the host application's logging untouched.
+The command needs a Spark flavour installed; it
+fails to import on a bare install. `python -m spark_sql_migrations.spark_sql.spark_sql`
+is equivalent.
+
 Or from Python, which is what a consuming project usually wraps:
 
 ```python
@@ -122,7 +129,8 @@ def get_migrations_dir():
 main(cat="spark_catalog", schema="default", migrations_dir=get_migrations_dir())
 ```
 
-`main` builds its own session via `get_spark()`. To supply your own — a session already
+`main` builds its own session via `get_spark()` and, being library code, leaves logging to
+you. To supply your own — a session already
 configured by your job, say — call `run_migrations(spark, cat, schema, output_folder,
 migrations_root)` directly.
 
@@ -134,7 +142,7 @@ before it is executed, so you can always read the exact SQL a run applied.
 Generate one rather than hand-rolling the header:
 
 ```bash
-python -m spark_sql_migrations.spark_sql.spark_sql create_new_migration     --message "add batch id to metrics"     --output-path path/to/migrations/all_spark_migrations
+spark-sql-migrations create_new_migration     --message "add batch id to metrics"     --output-path path/to/migrations/all_spark_migrations
 ```
 
 That writes `<yymmdd>_<slug>_<revision_id>.sql` from the template shipped in the package

{spark_sql_migrations-0.0.1 → spark_sql_migrations-0.0.3}/spark_sql_migrations.egg-info/SOURCES.txt

@@ -3,12 +3,12 @@ MANIFEST.in
 backend_shim.py
 setup.py
 spark_sql_migrations/__init__.py
-spark_sql_migrations/custom_logging.py
 spark_sql_migrations/py.typed
 spark_sql_migrations/spark_utils.py
 spark_sql_migrations.egg-info/PKG-INFO
 spark_sql_migrations.egg-info/SOURCES.txt
 spark_sql_migrations.egg-info/dependency_links.txt
+spark_sql_migrations.egg-info/entry_points.txt
 spark_sql_migrations.egg-info/namespace_packages.txt
 spark_sql_migrations.egg-info/requires.txt
 spark_sql_migrations.egg-info/top_level.txt

spark_sql_migrations-0.0.3/spark_sql_migrations.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+spark-sql-migrations = spark_sql_migrations.spark_sql.spark_sql:cli

spark_sql_migrations-0.0.1/spark_sql_migrations/custom_logging.py

@@ -1,78 +0,0 @@
-"""Opinionated logging configuration, for *applications* to opt into.
-
-Importing spark_sql_migrations never configures logging: library modules take a plain
-``logging.getLogger(__name__)`` and inherit whatever the host application set
-up. An application that wants this format calls :func:`setup_logging` from its
-own entry point -- never at import time, and never from library code, because
-configuring the root logger is the application's decision to make.
-"""
-
-import logging.config
-import os
-
-LOG_FILE_ENV_VAR = "SPARK_SQL_MIGRATIONS_LOG_FILE"
-DEFAULT_LOG_FILENAME = "local_log.log"
-
-FORMAT = "%(levelname)s %(asctime)s %(filename)s->%(funcName)s->%(lineno)d : %(message)s"
-
-
-def get_log_file_path():
-    """where the file handler writes.
-
-    resolved against the caller's cwd for the same reason as the spark warehouse
-    dir: a __file__-relative default would land inside site-packages once this
-    is installed from a wheel.
-    """
-    return os.environ.get(LOG_FILE_ENV_VAR, os.path.join(os.getcwd(), DEFAULT_LOG_FILENAME))
-
-
-def build_config():
-    """built per call, not a module constant: get_log_file_path() reads the
-    environment, and at module-import time the consumer hasn't set it yet."""
-    return {
-        "version": 1,
-        "disable_existing_loggers": False,
-        "formatters": {
-            "simple": {
-                "format": FORMAT,
-                "datefmt": "%y/%m/%d %H:%M:%S",
-            }
-        },
-        "handlers": {
-            "stdout": {
-                "class": "logging.StreamHandler",
-                "level": "INFO",
-                "formatter": "simple",
-                "stream": "ext://sys.stdout",
-            },
-            "stderr": {
-                "class": "logging.StreamHandler",
-                "level": "ERROR",
-                "formatter": "simple",
-                "stream": "ext://sys.stderr",
-            },
-            "file": {
-                "class": "logging.FileHandler",
-                "formatter": "simple",
-                "filename": get_log_file_path(),
-                "mode": "a",
-            },
-        },
-        "root": {"level": "DEBUG", "handlers": ["stderr", "stdout", "file"]},
-    }
-
-
-def is_logging_configured():
-    # Check if the root logger has any handlers explicitly assigned
-    return len(logging.getLogger().handlers) >= 3
-
-
-def setup_logging():
-    """configure the root logger with our handlers; safe to call more than once.
-
-    for application entry points only -- see the module docstring.
-    """
-    if not is_logging_configured():
-        logging.config.dictConfig(build_config())
-        logging.getLogger("py4j").setLevel(logging.ERROR)
-    return logging