spark-sql-migrations 0.0.1__py3-none-any.whl

spark_sql_migrations/__init__.py

@@ -0,0 +1,8 @@
+import logging
+
+# A library must not configure logging when it is imported. A NullHandler on the
+# package root keeps our records off logging.lastResort while leaving every
+# decision about handlers, levels and formats to the host application. An
+# application that wants our opinionated setup calls
+# spark_sql_migrations.custom_logging.setup_logging() from its own entry point.
+logging.getLogger(__name__).addHandler(logging.NullHandler())

spark_sql_migrations/custom_logging.py

@@ -0,0 +1,78 @@
+"""Opinionated logging configuration, for *applications* to opt into.
+
+Importing spark_sql_migrations never configures logging: library modules take a plain
+``logging.getLogger(__name__)`` and inherit whatever the host application set
+up. An application that wants this format calls :func:`setup_logging` from its
+own entry point -- never at import time, and never from library code, because
+configuring the root logger is the application's decision to make.
+"""
+
+import logging.config
+import os
+
+LOG_FILE_ENV_VAR = "SPARK_SQL_MIGRATIONS_LOG_FILE"
+DEFAULT_LOG_FILENAME = "local_log.log"
+
+FORMAT = "%(levelname)s %(asctime)s %(filename)s->%(funcName)s->%(lineno)d : %(message)s"
+
+
+def get_log_file_path():
+    """where the file handler writes.
+
+    resolved against the caller's cwd for the same reason as the spark warehouse
+    dir: a __file__-relative default would land inside site-packages once this
+    is installed from a wheel.
+    """
+    return os.environ.get(LOG_FILE_ENV_VAR, os.path.join(os.getcwd(), DEFAULT_LOG_FILENAME))
+
+
+def build_config():
+    """built per call, not a module constant: get_log_file_path() reads the
+    environment, and at module-import time the consumer hasn't set it yet."""
+    return {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "simple": {
+                "format": FORMAT,
+                "datefmt": "%y/%m/%d %H:%M:%S",
+            }
+        },
+        "handlers": {
+            "stdout": {
+                "class": "logging.StreamHandler",
+                "level": "INFO",
+                "formatter": "simple",
+                "stream": "ext://sys.stdout",
+            },
+            "stderr": {
+                "class": "logging.StreamHandler",
+                "level": "ERROR",
+                "formatter": "simple",
+                "stream": "ext://sys.stderr",
+            },
+            "file": {
+                "class": "logging.FileHandler",
+                "formatter": "simple",
+                "filename": get_log_file_path(),
+                "mode": "a",
+            },
+        },
+        "root": {"level": "DEBUG", "handlers": ["stderr", "stdout", "file"]},
+    }
+
+
+def is_logging_configured():
+    # Check if the root logger has any handlers explicitly assigned
+    return len(logging.getLogger().handlers) >= 3
+
+
+def setup_logging():
+    """configure the root logger with our handlers; safe to call more than once.
+
+    for application entry points only -- see the module docstring.
+    """
+    if not is_logging_configured():
+        logging.config.dictConfig(build_config())
+        logging.getLogger("py4j").setLevel(logging.ERROR)
+    return logging

spark_sql_migrations/migration_templates/default_migration_template.sql

@@ -0,0 +1,12 @@
+-- revision_id:{{revision_id}};
+-- prev_revision_id:{{prev_revision_id}};
+begin
+-- replace the statements below with the idempotent statements you
+-- would like to have applied to databricks db or local spark(for testing)
+-- crutch scaffolding will use jinja to fill in the cat and schema that are sent to it
+-- this is also taken care of in the gh build.
+
+-- e.g:
+-- create schema if not exists {{cat}}.{{schema}};
+-- create table if not exists {{cat}}.{{schema}}.test_table(int_id bigint, stuff string);
+end;

spark_sql_migrations/migrations_initial/first_01_intial_create_cat_dbr_only.sql

@@ -0,0 +1,4 @@
+-- revision_id:;
+begin
+create catalog if not exists {{cat}};
+end;

spark_sql_migrations/migrations_initial/first_02_intial_create_schema_all.sql

@@ -0,0 +1,4 @@
+-- revision_id:;
+begin
+create schema if not exists {{cat}}.{{schema}};
+end;

spark_sql_migrations/migrations_initial/first_03_intial_create_state_table_all.sql

@@ -0,0 +1,7 @@
+-- revision_id:;
+begin
+create table if not exists {{cat}}.{{schema}}._spark_migrations_version (
+    migration_type string not null,
+    version_num string not null
+);
+end;

spark_sql_migrations/spark_sql/spark_sql.py

@@ -0,0 +1,333 @@
+"""Chained, idempotent SQL migrations for Spark and Databricks.
+
+Two kinds of migration exist, and they are owned by different parties:
+
+* the *initial* chain and the new-migration template ship inside this package.
+  They bootstrap the catalog, the schema and the version table, are selected by
+  filename suffix (``_all.sql`` / ``_dbr_only.sql``) rather than by revision,
+  and are re-applied on every run -- so they must be idempotent.
+* the ``all_spark_migrations/`` and ``dbr_only_migrations/`` chains belong to
+  the consuming project. The caller passes their parent directory as
+  ``migrations_root``; each chain is walked from its root via
+  ``prev_revision_id`` and only the tail not yet recorded in the version table
+  is applied.
+"""
+
+import argparse
+import datetime
+import logging
+import os
+import re
+import uuid
+from dataclasses import dataclass
+
+from jinja2 import Environment, FileSystemLoader, PackageLoader, select_autoescape
+from pyspark.sql.functions import col
+
+from spark_sql_migrations.spark_utils import get_spark, is_dbr
+
+logger = logging.getLogger(__name__)
+
+
+ALL_SPARK = "all_spark"
+DBR_ONLY = "dbr_only"
+
+VERSION_TABLE = "_spark_migrations_version"
+
+PACKAGE_NAME = "spark_sql_migrations"
+INITIAL_MIGRATIONS_PACKAGE_PATH = "migrations_initial"
+TEMPLATES_PACKAGE_PATH = "migration_templates"
+DEFAULT_TEMPLATE_NAME = "default_migration_template.sql"
+
+_HEADER_RE = re.compile(r"^--\s*(revision_id|prev_revision_id)\s*:\s*(\w*)\s*;$")
+
+
+@dataclass(frozen=True)
+class Migration:
+    revision_id: str
+    prev_revision_id: str | None
+    template_name: str
+
+
+def get_migrations_dir(migrations_root, migration_type):
+    """the client-owned directory holding one chain."""
+    return os.path.join(migrations_root, f"{migration_type}_migrations")
+
+
+def get_default_template_path():
+    """the template this package ships; callers may pass their own instead."""
+    return os.path.join(os.path.dirname(__file__), "..", TEMPLATES_PACKAGE_PATH, DEFAULT_TEMPLATE_NAME)
+
+
+def _parse_migration(migrations_dir, template_name):
+    headers = {}
+    migration_file_path = os.path.join(migrations_dir, template_name)
+    if os.path.getsize(migration_file_path) == 0:
+        raise ValueError(f"{template_name} is empty;")
+    with open(migration_file_path) as migration_file:
+        for line in migration_file:
+            line = line.strip()
+            match = _HEADER_RE.match(line)
+            if match:
+                headers[match.group(1)] = match.group(2) or None
+            elif line and not line.startswith("--"):
+                break
+
+    if not headers.get("revision_id"):
+        raise ValueError(f"{template_name} has no revision_id header;")
+
+    return Migration(
+        revision_id=headers["revision_id"],
+        prev_revision_id=headers.get("prev_revision_id"),
+        template_name=template_name,
+    )
+
+
+def get_ordered_migration_objs(migration_objs):
+    by_revision = {}
+    for migration in migration_objs:
+        clash = by_revision.get(migration.revision_id)
+        if clash:
+            raise ValueError(
+                f"revision_id {migration.revision_id} is claimed by both "
+                f"{clash.template_name} and {migration.template_name};"
+            )
+        by_revision[migration.revision_id] = migration
+
+    roots = [migration for migration in migration_objs if not migration.prev_revision_id]
+    if len(roots) != 1:
+        raise ValueError(
+            f"expected exactly one migration with an empty prev_revision_id, "
+            f"found {[m.template_name for m in roots]};"
+        )
+
+    next_by_revision = {}
+    for migration in migration_objs:
+        if not migration.prev_revision_id:
+            continue
+        if migration.prev_revision_id not in by_revision:
+            raise ValueError(
+                f"{migration.template_name} points at unknown prev_revision_id {migration.prev_revision_id};"
+            )
+        sibling = next_by_revision.get(migration.prev_revision_id)
+        if sibling:
+            raise ValueError(
+                f"revision_id {migration.prev_revision_id} is the parent of both "
+                f"{sibling.template_name} and {migration.template_name};"
+            )
+        next_by_revision[migration.prev_revision_id] = migration
+
+    ordered = []
+    current = roots[0]
+    while current:
+        ordered.append(current)
+        current = next_by_revision.get(current.revision_id)
+
+    if len(ordered) != len(migration_objs):
+        walked = {migration.template_name for migration in ordered}
+        raise ValueError(
+            f"Migrations are not reachable from the root: "
+            f"{sorted(m.template_name for m in migration_objs if m.template_name not in walked)};"
+        )
+
+    return ordered
+
+
+def get_migrations_list(migrations_dir):
+    """every migration in one directory, walked from its root to its head."""
+    if not os.path.isdir(migrations_dir):
+        raise ValueError(f"no migrations dir at {migrations_dir};")
+
+    migrations = [
+        _parse_migration(migrations_dir, template_name)
+        for template_name in sorted(os.listdir(migrations_dir))
+        if template_name.endswith(".sql")
+    ]
+    if not migrations:
+        return []
+
+    return get_ordered_migration_objs(migrations)
+
+
+def get_unapplied_migrations_list(spark, migrations_dir, migration_type, cat: str, schema: str):
+    """the tail of one chain after the revision VERSION_TABLE records for it.
+
+    the whole chain when the table is missing or holds no row for this type.
+    """
+    ordered = get_migrations_list(migrations_dir)
+
+    version_table = f"{cat}.{schema}.{VERSION_TABLE}"
+    if not spark.catalog.tableExists(version_table):
+        return ordered
+
+    current = (
+        spark.read.table(version_table).where(col("migration_type") == migration_type).select("version_num").head()
+    )
+    if not current:
+        return ordered
+
+    current_revision_id = current["version_num"]
+
+    for index, migration in enumerate(ordered):
+        if migration.revision_id == current_revision_id:
+            return ordered[index + 1 :]
+
+    raise ValueError(
+        f"{VERSION_TABLE} has {migration_type} at {current_revision_id}, "
+        f"which matches no migration in {migrations_dir};"
+    )
+
+
+def create_new_migration(message: str, template_path: str, output_path: str, truncate_slug_length: int = 40):
+    """
+    TODO XXX start looking up prev revision id from db or files and including it
+    """
+    rev_id = uuid.uuid4().hex[-12:]
+
+    date_prefix = datetime.datetime.today().strftime("%y%m%d")
+
+    slug = "_".join(re.split(r"\W+", message.lower())).strip("_")
+    if len(slug) > truncate_slug_length:
+        slug = slug[:truncate_slug_length].rsplit("_", 1)[0]
+
+    migration_filename = f"{date_prefix}_{slug}_{rev_id}.sql"
+
+    migration_path = os.path.join(output_path, migration_filename)
+
+    default_migration_content = open(template_path).read()
+    default_migration_content = default_migration_content.replace("{{revision_id}}", rev_id)
+
+    with open(migration_path, "w") as migration_file:
+        migration_file.write(default_migration_content)
+
+
+def apply_template(output_dir, template, cat: str, schema: str):
+    result_sql = template.render(cat=cat, schema=schema)
+    with open(os.path.join(output_dir, template.name.replace(".sql", "_primed.sql")), "w") as file_handle:
+        file_handle.write(result_sql)
+    return result_sql
+
+
+def get_ascending_letters_within_minute():
+    micros_since_minute = datetime.datetime.now() - datetime.datetime.now().replace(second=0, microsecond=0)
+    result = str(micros_since_minute.microseconds).translate(str.maketrans("0123456789", "ABCDEFGHIJ"))
+    return result
+
+
+def get_output_folder(output_parent_path):
+    folder_name = f"{datetime.datetime.today().strftime('%Y%m%d_%H%M')}_{get_ascending_letters_within_minute()}"
+    return os.path.join(output_parent_path, folder_name)
+
+
+def use_migration_file(fname):
+    if fname.endswith("all.sql"):
+        return True
+    elif is_dbr() and fname.endswith("dbr_only.sql"):
+        return True
+    return False
+
+
+def migrate_initial(spark, output_folder, cat: str, schema: str):
+    """apply the package's own bootstrap chain; selected by suffix, not revision."""
+    env = Environment(
+        loader=PackageLoader(package_name=PACKAGE_NAME, package_path=INITIAL_MIGRATIONS_PACKAGE_PATH),
+        autoescape=select_autoescape(),
+    )
+    all_templates = env.list_templates(filter_func=use_migration_file)
+
+    logger.info(f"found {len(all_templates)} migrations; first five are {all_templates[:5]};")
+    for template_name in all_templates:
+        result_sql = apply_template(output_folder, env.get_template(template_name), cat=cat, schema=schema)
+        spark.sql(result_sql)
+    logger.info(f"invoked spark on {len(all_templates)} migrations;")
+
+
+def record_revision(spark, cat: str, schema: str, migration_type: str, revision_id: str):
+    """point VERSION_TABLE's row for one migration type at revision_id.
+
+    delete + insert rather than merge: there is one row per type, and a replay
+    after a crash between the two statements is harmless because migrations are
+    idempotent. the values are inlined rather than passed as `args` because
+    delta's DELETE does not bind named parameters.
+    """
+    version_table = f"{cat}.{schema}.{VERSION_TABLE}"
+    type_literal = migration_type.replace("'", "''")
+    revision_literal = revision_id.replace("'", "''")
+    spark.sql(f"delete from {version_table} where migration_type = '{type_literal}'")
+    spark.sql(
+        f"insert into {version_table} (migration_type, version_num) " f"values ('{type_literal}', '{revision_literal}')"
+    )
+
+
+def migrate_w_rev(spark, output_folder, migrations_root, cat: str, schema: str, migration_type: str):
+    """apply the migrations of one client chain that VERSION_TABLE hasn't recorded yet."""
+    migrations_dir = get_migrations_dir(migrations_root, migration_type)
+    unapplied = get_unapplied_migrations_list(spark, migrations_dir, migration_type, cat=cat, schema=schema)
+
+    logger.info(
+        f"{migration_type} has {len(unapplied)} unapplied migrations; "
+        f"first five are {[migration.template_name for migration in unapplied[:5]]};"
+    )
+
+    env = Environment(loader=FileSystemLoader(migrations_dir), autoescape=select_autoescape())
+    for migration in unapplied:
+        result_sql = apply_template(output_folder, env.get_template(migration.template_name), cat=cat, schema=schema)
+        spark.sql(result_sql)
+        record_revision(spark, cat, schema, migration_type, migration.revision_id)
+        logger.info(f"applied {migration.template_name}; {migration_type} is now at {migration.revision_id};")
+
+
+def run_migrations(spark, cat, schema, output_folder, migrations_root):
+    initial_output_folder = os.path.join(output_folder, INITIAL_MIGRATIONS_PACKAGE_PATH)
+    dbr_ouput_folder = os.path.join(output_folder, f"{DBR_ONLY}_migrations")
+    all_spark_ouput_folder = os.path.join(output_folder, f"{ALL_SPARK}_migrations")
+    os.makedirs(initial_output_folder, exist_ok=True)
+    os.makedirs(dbr_ouput_folder, exist_ok=True)
+    os.makedirs(all_spark_ouput_folder, exist_ok=True)
+    migrate_initial(spark, initial_output_folder, cat, schema)
+    if is_dbr():
+        migrate_w_rev(spark, dbr_ouput_folder, migrations_root, cat, schema, DBR_ONLY)
+    migrate_w_rev(spark, all_spark_ouput_folder, migrations_root, cat, schema, ALL_SPARK)
+
+
+def main(cat, schema, migrations_dir, output_parent_path=None):
+    output_folder = get_output_folder(output_parent_path or os.path.join(os.getcwd(), "migrations_out"))
+    os.makedirs(output_folder)
+    run_migrations(get_spark(), cat, schema, output_folder, migrations_dir)
+
+
+def _cli_main(cat, schema, migrations_dir):  # pragma: no cover
+    main(cat, schema, migrations_dir)
+
+
+def _cli_create_new_migration(message, output_path, template_path):  # pragma: no cover
+    create_new_migration(message, template_path or get_default_template_path(), output_path)
+
+
+def build_parser():  # pragma: no cover
+    parser = argparse.ArgumentParser(description="spark_sql_migrations migration utilities")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    p_run = subparsers.add_parser("run", help="apply all pending migrations")
+    p_run.add_argument("--cat", default="spark_catalog")
+    p_run.add_argument("--schema", default="default")
+    p_run.add_argument(
+        "--migrations-dir", required=True, help="parent of all_spark_migrations/ and dbr_only_migrations/"
+    )
+    p_run.set_defaults(func=_cli_main)
+
+    p_create = subparsers.add_parser("create_new_migration", help="create a new migration from the template")
+    p_create.add_argument("--message", required=True)
+    p_create.add_argument("--output-path", required=True, help="the chain directory to write the new migration into")
+    p_create.add_argument("--template-path", default=None, help="defaults to the template shipped in this package")
+    p_create.set_defaults(func=_cli_create_new_migration)
+
+    return parser
+
+
+if __name__ == "__main__":  # pragma: no cover
+    args = build_parser().parse_args()
+    kwargs = vars(args)
+    func = kwargs.pop("func")
+    kwargs.pop("command")
+    func(**kwargs)

spark_sql_migrations/spark_utils.py

@@ -0,0 +1,72 @@
+import os
+import sys
+
+from delta import configure_spark_with_delta_pip  # type: ignore
+from pyspark.sql.session import SparkSession
+
+APP_NAME = "spark_sql_migrations"
+
+WAREHOUSE_DIR_ENV_VAR = "SPARK_WAREHOUSE_DIR"
+METASTORE_DIR_ENV_VAR = "SPARK_METASTORE_DIR"
+DEFAULT_WAREHOUSE_DIRNAME = "spark-warehouse"
+
+
+def is_dbr():
+    try:
+        from pyspark.dbutils import DBUtils  # type: ignore # noqa: F401
+
+        # This will only succeed if the Databricks environment is available
+        return True  # pragma: no cover
+    except Exception:
+        return False
+
+
+def get_warehouse_dir():
+    """where local (non-DBR) spark keeps its warehouse.
+
+    resolved against the caller's cwd, never against __file__: installed from a
+    wheel this module lives in site-packages, so a __file__-relative default
+    would write the consumer's tables inside their virtualenv.
+    """
+    return os.environ.get(
+        WAREHOUSE_DIR_ENV_VAR,
+        os.path.join(os.getcwd(), DEFAULT_WAREHOUSE_DIRNAME),
+    )
+
+
+def get_spark(use_dbc=False):
+    if use_dbc or is_dbr():
+        from databricks.connect.session import DatabricksSession  # type: ignore # pants: no-infer-dep
+
+        os.environ["DATABRICKS_SERVERLESS_COMPUTE_ID"] = "auto"
+        return DatabricksSession.builder.getOrCreate()
+
+    spark_remote = os.environ.get("SPARK_REMOTE")
+    if spark_remote:
+        return SparkSession.builder.remote(spark_remote).getOrCreate()
+
+    # Pin the worker interpreter to the one actually running this process. Left
+    # unset, Spark resolves "python3" independently for the worker daemon, which
+    # can land on a different Python build than the driver's when launched from
+    # an IDE (mismatched sys.executable vs. PATH resolution) -- causing worker
+    # crashes like "SRE module mismatch" that don't reproduce from the CLI.
+    os.environ.setdefault("PYSPARK_PYTHON", sys.executable)
+    os.environ.setdefault("PYSPARK_DRIVER_PYTHON", sys.executable)
+
+    builder = (
+        SparkSession.builder.appName(APP_NAME)
+        .config("spark.sql.warehouse.dir", get_warehouse_dir())
+        .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+        .config(
+            "spark.sql.catalog.spark_catalog",
+            "org.apache.spark.sql.delta.catalog.DeltaCatalog",
+        )
+        .config("spark.sql.sources.default", "delta")
+    )
+    metastore_dir = os.environ.get(METASTORE_DIR_ENV_VAR)
+    if metastore_dir:
+        builder = builder.config(
+            "javax.jdo.option.ConnectionURL",
+            f"jdbc:derby:;databaseName={metastore_dir};create=true",
+        )
+    return configure_spark_with_delta_pip(builder).getOrCreate()

spark_sql_migrations-0.0.1.dist-info/METADATA

@@ -0,0 +1,216 @@
+Metadata-Version: 2.4
+Name: spark_sql_migrations
+Version: 0.0.1
+Summary: Chained, idempotent SQL migrations for Spark and Databricks.
+Home-page: https://github.com/mcjug2015/spark_sql_migrations
+Author: Victor Semenov
+License: Apache-2.0
+Project-URL: Source, https://github.com/mcjug2015/spark_sql_migrations
+Project-URL: Issues, https://github.com/mcjug2015/spark_sql_migrations/issues
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: SQL
+Classifier: Topic :: Database
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Jinja2<4,>=3.1
+Provides-Extra: local
+Requires-Dist: pyspark[connect]<5,>=4.0; extra == "local"
+Requires-Dist: delta-spark<5,>=4.0; extra == "local"
+Provides-Extra: databricks
+Requires-Dist: databricks-connect<19,>=18; extra == "databricks"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# spark_sql_migrations
+
+[![CI](https://github.com/mcjug2015/spark_sql_migrations/actions/workflows/ci.yml/badge.svg)](https://github.com/mcjug2015/spark_sql_migrations/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+
+Chained, idempotent SQL migrations for Spark and Databricks.
+
+Alembic-shaped schema migrations for a Spark/Delta catalog: your migrations are plain
+`.sql` files linked into a chain by revision headers, the applied revision is recorded in
+a version table inside your own schema, and each run applies only the tail that has not
+been applied yet.
+
+## Why
+
+Spark and Databricks have no migration story of their own. Delta gives you `CREATE TABLE
+IF NOT EXISTS`, but nothing that tracks which DDL a given catalog has seen, and nothing
+that lets you write a change once and have it apply on a laptop and on Databricks alike.
+spark_sql_migrations is the small amount of machinery that closes that gap, and nothing
+more: no ORM, no autogeneration, no Python migration scripts. You write SQL.
+
+## Installing
+
+`spark_sql_migrations` declares **no Spark of its own** — the extras decide which one you get:
+
+```bash
+pip install "spark_sql_migrations[local]"        # pyspark + delta-spark, for laptops and CI
+pip install "spark_sql_migrations[databricks]"   # databricks-connect
+pip install spark_sql_migrations                 # no Spark at all
+```
+
+Pick exactly one flavour. `databricks-connect` ships its own top-level `pyspark/` and
+`delta/` packages, so installing it alongside `pyspark`/`delta-spark` silently clobbers
+both.
+
+Unreleased changes can be consumed as the wheel built by CI (the `dist` artifact on any
+green run) or built locally with `pants package //:dist`.
+
+## The two kinds of migration
+
+Migrations come from two different owners, and they behave differently:
+
+| | ships in the wheel | owned by you |
+|---|---|---|
+| **what** | catalog, schema, `_spark_migrations_version` table | your tables and columns |
+| **where** | `spark_sql_migrations/migrations_initial/` | `all_spark_migrations/`, `dbr_only_migrations/` |
+| **selected by** | filename suffix — `_all.sql` everywhere, `_dbr_only.sql` on Databricks | the `prev_revision_id` chain |
+| **when applied** | every run | once, then recorded |
+
+The bootstrap chain is re-applied on **every** run, so it must be idempotent. Your own
+chains are applied once each and their head revision is written to the version table.
+
+## Laying out your chains
+
+Point spark_sql_migrations at a directory holding one or both chains:
+
+```
+your_project/
+└── migrations/                 <- this is --migrations-dir
+    ├── all_spark_migrations/   <- runs everywhere
+    └── dbr_only_migrations/    <- runs only when is_dbr()
+```
+
+Choose deliberately. SQL that only Databricks understands must not live in
+`all_spark_migrations/`, or your local runs will break.
+
+## Running migrations
+
+```bash
+python -m spark_sql_migrations.spark_sql.spark_sql run     --cat spark_catalog     --schema default     --migrations-dir path/to/migrations
+```
+
+Or from Python, which is what a consuming project usually wraps:
+
+```python
+import os
+
+from spark_sql_migrations.spark_sql.spark_sql import main
+
+
+def get_migrations_dir():
+    """the parent of this project's own migration chains."""
+    return os.path.dirname(__file__)
+
+
+main(cat="spark_catalog", schema="default", migrations_dir=get_migrations_dir())
+```
+
+`main` builds its own session via `get_spark()`. To supply your own — a session already
+configured by your job, say — call `run_migrations(spark, cat, schema, output_folder,
+migrations_root)` directly.
+
+Every rendered statement is written to a timestamped folder under `migrations_out/`
+before it is executed, so you can always read the exact SQL a run applied.
+
+## Writing a migration
+
+Generate one rather than hand-rolling the header:
+
+```bash
+python -m spark_sql_migrations.spark_sql.spark_sql create_new_migration     --message "add batch id to metrics"     --output-path path/to/migrations/all_spark_migrations
+```
+
+That writes `<yymmdd>_<slug>_<revision_id>.sql` from the template shipped in the package
+(override with `--template-path`), with `revision_id` filled in:
+
+```sql
+-- revision_id:e5ce0039b32b;
+-- prev_revision_id:;
+begin
+create table if not exists {{cat}}.{{schema}}.test_table(int_id bigint, stuff string);
+end;
+```
+
+Then set `prev_revision_id` to the revision this one follows. Ordering comes from that
+chain, **not** from the filename — the date prefix is for humans. A chain must have
+exactly one root (empty `prev_revision_id`) and no forks.
+
+`{{cat}}` and `{{schema}}` are Jinja placeholders rendered at apply time; never hardcode a
+catalog or schema.
+
+### Idempotency
+
+Prefer `IF NOT EXISTS`. Where Databricks doesn't offer it — notably
+`ALTER TABLE ... ADD COLUMN` — guard with a SQLSTATE exit handler:
+
+```sql
+-- revision_id:57037b6c19b4;
+-- prev_revision_id:07990e2a101e;
+begin
+  -- idempotent add: swallow FIELD_ALREADY_EXISTS (SQLSTATE 42710) if the column
+  -- is already present. Databricks has no ADD COLUMN IF NOT EXISTS, so use a
+  -- SQL-scripting EXIT handler (CONTINUE handlers are unsupported).
+  declare exit handler for sqlstate '42710'
+  begin end;
+  alter table {{cat}}.{{schema}}.metrics add column metric_batch_id string;
+end;
+```
+
+## Local vs Databricks
+
+`get_spark()` returns a `DatabricksSession` when `is_dbr()` is true and a Delta-configured
+local `SparkSession` otherwise, so the same code runs in both places. Local behaviour is
+tuned by environment variables:
+
+| Variable | Effect |
+|---|---|
+| `SPARK_WAREHOUSE_DIR` | warehouse location (default: `./spark-warehouse`) |
+| `SPARK_METASTORE_DIR` | Derby metastore location; unset means Spark's default |
+| `SPARK_REMOTE` | connect to an existing Spark Connect server instead of starting one |
+
+## Development
+
+Built with [Pants](https://www.pantsbuild.org/). Two resolves: `python-default` for the
+library, `py-reqs-dev` for tests.
+
+```bash
+# black, isort, flake8, mypy. `::` and not `/`, or the subtrees go unchecked.
+pants fmt lint check spark_sql_migrations:: spark_sql_migrations_test:: scripts::
+
+# unit, then integration as its own invocation
+pants test --use-coverage spark_sql_migrations_test/:: -spark_sql_migrations_test/integration::
+pants test spark_sql_migrations_test/integration::
+
+# wheel + sdist
+pants package //:dist
+```
+
+`scripts/run_local.sh` runs everything CI runs, including building the distribution and
+verifying all three install flavours. Branch coverage over `spark_sql_migrations/` is
+gated at 94%.
+
+Dependencies are locked. Edit `spark_sql_migrations/requirements.txt` or
+`spark_sql_migrations_test/requirements-dev.txt`, then `pants generate-lockfiles` — never
+hand-edit a
+lockfile.
+
+## License
+
+Apache License 2.0 — see [LICENSE](LICENSE).

spark_sql_migrations-0.0.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+spark_sql_migrations/__init__.py,sha256=y-dGVkeeYXwDinWN23rRRbwScEhHUwFBicF7YSWMlzM,442
+spark_sql_migrations/custom_logging.py,sha256=Dj0BlnU4L39ylKSEzdz_SaEGgj8nm7izioBOoJDo5YQ,2671
+spark_sql_migrations/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spark_sql_migrations/spark_utils.py,sha256=ImNrHsXwcxREk6YNxR7aXry1nrTjKuBC6EkzNcX9Fx8,2664
+spark_sql_migrations/migration_templates/default_migration_template.sql,sha256=fjk9ykmwtF3t9Rl_0EuzoyW8-kBGTY3PtSHBJRZYuQU,510
+spark_sql_migrations/migrations_initial/first_01_intial_create_cat_dbr_only.sql,sha256=sGAYGHC0o26E3XsGavO9BKeYYcDr8NKKHyQZP3nce-8,65
+spark_sql_migrations/migrations_initial/first_02_intial_create_schema_all.sql,sha256=qM0sUDgOTyZR-7VmEwyJil5kaygi7fAy7lmGlyXo_Og,75
+spark_sql_migrations/migrations_initial/first_03_intial_create_state_table_all.sql,sha256=JFochIA3chVvCARtjcgU2WOeGvdtmeocpI2CDA0eIQU,172
+spark_sql_migrations/spark_sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spark_sql_migrations/spark_sql/spark_sql.py,sha256=Ici1ssXcAj7UsaQSWRpgc-1eJ4q-aXh6O_AZRYZrueQ,13208
+spark_sql_migrations-0.0.1.dist-info/licenses/LICENSE,sha256=TJdL-klfWUy-m2zkW6TSKRkJv3eLTTYWgUfM1wSzm8M,11345
+spark_sql_migrations-0.0.1.dist-info/METADATA,sha256=UEVZpKkSfZuCVvZXCK8BLhq2tpIZc3lL96CF7aUzEYc,8050
+spark_sql_migrations-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+spark_sql_migrations-0.0.1.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+spark_sql_migrations-0.0.1.dist-info/top_level.txt,sha256=eC1HgBkv7dOjkovdGlvNGpx4KyC_OHkDkJg3RuTJhT4,21
+spark_sql_migrations-0.0.1.dist-info/RECORD,,

spark_sql_migrations-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

spark_sql_migrations-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Victor Semenov
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

spark_sql_migrations-0.0.1.dist-info/namespace_packages.txt

@@ -0,0 +1 @@
+

spark_sql_migrations-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+spark_sql_migrations