tablevalidator-databricks 0.1.0__py3-none-any.whl

tablevalidator_databricks/__init__.py

@@ -0,0 +1,15 @@
+"""tablevalidator_databricks: generates a widget-driven Databricks notebook
+UI on top of the table-validator package's validate_tables() API.
+
+This package contains no validation logic of its own - it only writes a
+notebook file whose cells call into table_validator's existing engine
+(see tablevalidator_databricks/templates/widget_notebook.py.tmpl)."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("tablevalidator-databricks")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+__all__ = ["__version__"]

tablevalidator_databricks/cli/main.py

@@ -0,0 +1,99 @@
+"""CLI entry point: `tablevalidator-databricks` console script."""
+
+from pathlib import Path
+
+import typer
+
+from tablevalidator_databricks.generator.generator import VALID_MODES, generate_notebook
+
+app = typer.Typer(
+    name="tablevalidator-databricks",
+    help="Generate a widget-driven Databricks notebook UI for table-validator.",
+    no_args_is_help=True,
+)
+
+DEFAULT_OUTPUT_PATH = Path("TableValidator.py")
+
+
+@app.command()
+def init(
+    output: Path = typer.Option(
+        DEFAULT_OUTPUT_PATH,
+        "--output",
+        help="Path to write the generated notebook to (default: ./TableValidator.py).",
+    ),
+    mode: str = typer.Option(
+        "full",
+        "--mode",
+        help=(
+            "basic: only the check-group multiselect and table pickers. "
+            "full (default): also shows optional filter widgets (only/ "
+            "ignore columns, row filter, primary key). "
+            "schema: pre-selects only the Catalog & Schema check group."
+        ),
+    ),
+) -> None:
+    """Generate a Databricks notebook with dbutils widgets for source/
+    target catalog.schema.table selection and validation checks - fill in
+    the widgets and run the notebook, no code to write.
+
+    Import the generated file into a Databricks workspace via
+    Workspace -> Import, with format set to "Source" (or "File" -> upload
+    directly, depending on your workspace UI version) - Databricks
+    recognizes the "# Databricks notebook source" header and cell
+    (# COMMAND ----------) markers and opens it as a real notebook, not a
+    plain text file.
+    """
+    if mode not in VALID_MODES:
+        typer.secho(
+            f"Invalid --mode {mode!r} - must be one of {', '.join(VALID_MODES)}.",
+            fg=typer.colors.RED,
+        )
+        raise typer.Exit(code=1)
+
+    generate_notebook(output, mode)
+
+    typer.echo(f"Generated notebook: {output.resolve()}")
+    typer.echo(
+        "Import it into Databricks (Workspace -> Import, format 'Source'), "
+        "fill in the widgets, and Run All."
+    )
+
+
+@app.command()
+def info() -> None:
+    """Show what this tool does and how to use the generated notebook."""
+    typer.echo(
+        "\n"
+        "tablevalidator-databricks - Databricks notebook UI for table-validator\n"
+        "------------------------------------------------------------------------\n"
+        "Generates a Databricks notebook with dbutils widgets (dynamic "
+        "catalog/schema/table dropdowns sourced from Unity Catalog, plus a "
+        "validation-checks multiselect) as a wrapper around the "
+        "table-validator package's validate_tables() API - no Python code "
+        "to write, no separate credentials to configure (the notebook "
+        "reuses its own ambient Spark session, same as validate_tables() "
+        "itself).\n"
+        "\n"
+        "Workflow:\n"
+        "\n"
+        "  1. tablevalidator-databricks init\n"
+        "     Writes a ready-to-use notebook (./TableValidator.py by "
+        "default). Use --output to write elsewhere, --mode to pick basic/"
+        "full/schema (see 'tablevalidator-databricks init --help').\n"
+        "\n"
+        "  2. Import the generated file into Databricks\n"
+        "     Workspace -> Import, format 'Source'.\n"
+        "\n"
+        "  3. Fill in the widgets and Run All\n"
+        "     Pick Source/Target Catalog, Schema, Table, and which checks "
+        "to run, then run the notebook top to bottom. Re-run the "
+        "'Create widgets' cell after changing a Catalog/Schema selection, "
+        "to repopulate the dropdowns below it (a Databricks widget "
+        "limitation, not a bug).\n"
+        "\n"
+        "This package contains no validation logic of its own - every "
+        "generated notebook calls directly into table-validator's own "
+        "validate_tables() engine, so results are identical to calling it "
+        "yourself in a code cell.\n"
+    )

tablevalidator_databricks/generator/generator.py

@@ -0,0 +1,39 @@
+"""
+Renders the bundled widget_notebook.py.tmpl into a real Databricks
+notebook file. No Jinja2 - the template has exactly two placeholders,
+substituted via plain str.replace(), so a templating dependency isn't
+worth adding for this.
+"""
+
+from __future__ import annotations
+
+import importlib.resources
+from pathlib import Path
+
+VALID_MODES = ("basic", "full", "schema")
+
+
+def generate_notebook(output: Path, mode: str) -> None:
+    """Write a widget-driven Databricks notebook to `output`.
+
+    mode controls which optional widgets/pre-selected checks the notebook
+    starts with - see tablevalidator_databricks/cli/main.py's `init`
+    command docstring for what each mode means. The underlying comparison
+    logic is identical regardless of mode; only the widgets shown differ,
+    so all three modes render from the same template file rather than
+    three separate ones.
+    """
+    if mode not in VALID_MODES:
+        raise ValueError(f"mode must be one of {VALID_MODES}, got {mode!r}")
+
+    template = (
+        importlib.resources.files("tablevalidator_databricks.templates")
+        .joinpath("widget_notebook.py.tmpl")
+        .read_text(encoding="utf-8")
+    )
+    rendered = template.replace("{{MODE}}", mode).replace(
+        "{{SHOW_OPTIONAL_WIDGETS}}", "True" if mode == "full" else "False",
+    )
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(rendered, encoding="utf-8")

tablevalidator_databricks/templates/widget_notebook.py.tmpl

@@ -0,0 +1,178 @@
+# Databricks notebook source
+# MAGIC %md
+# MAGIC # Table Validator
+# MAGIC Generated by `tablevalidator-databricks init --mode {{MODE}}`.
+# MAGIC
+# MAGIC Fill in the widgets above (Source/Target Catalog, Schema, Table, and
+# MAGIC which checks to run), then **Run All**. No code needs to be written or
+# MAGIC edited - every cell below just reads the widgets and calls the
+# MAGIC `table_validator` package's `validate_tables()` API.
+# MAGIC
+# MAGIC **Cascading dropdowns**: Databricks widgets don't auto-refresh when an
+# MAGIC upstream selection changes. After picking a different Source/Target
+# MAGIC Catalog, re-run the "Create widgets" cell below to repopulate the Schema
+# MAGIC dropdown for that catalog; likewise after changing Schema, re-run it
+# MAGIC again to repopulate Table. This is a real Databricks widget limitation,
+# MAGIC not a bug in this notebook.
+
+# COMMAND ----------
+
+# MAGIC %pip install table-validator
+
+# COMMAND ----------
+
+dbutils.library.restartPython()
+
+# COMMAND ----------
+
+from table_validator import validate_tables
+from table_validator.config.schema import ValidationType
+
+MODE = "{{MODE}}"
+SHOW_OPTIONAL_WIDGETS = {{SHOW_OPTIONAL_WIDGETS}}
+
+
+def _list_catalogs():
+    return sorted(r["catalog"] for r in spark.sql("SHOW CATALOGS").collect())
+
+
+def _list_schemas(catalog: str):
+    if not catalog:
+        return []
+    rows = spark.sql(f"SHOW SCHEMAS IN `{catalog}`").collect()
+    return sorted(r["databaseName"] for r in rows)
+
+
+def _list_tables(catalog: str, schema: str):
+    if not catalog or not schema:
+        return []
+    rows = spark.sql(f"SHOW TABLES IN `{catalog}`.`{schema}`").collect()
+    return sorted(r["tableName"] for r in rows)
+
+
+def _current_or_first(options, current):
+    """Keep a widget's existing selection across a re-run of this cell if
+    it's still valid for the (possibly new) upstream selection, otherwise
+    fall back to the first available option (or "" if there are none)."""
+    if current in options:
+        return current
+    return options[0] if options else ""
+
+# COMMAND ----------
+
+# MAGIC %md ### Create widgets
+# MAGIC Re-run this cell after changing Source/Target Catalog or Schema, to
+# MAGIC repopulate the dropdowns below them.
+
+# COMMAND ----------
+
+def _get_widget(name: str, default: str = "") -> str:
+    try:
+        return dbutils.widgets.get(name)
+    except Exception:
+        return default
+
+
+catalogs = _list_catalogs()
+
+source_catalog_current = _current_or_first(catalogs, _get_widget("source_catalog"))
+target_catalog_current = _current_or_first(catalogs, _get_widget("target_catalog"))
+
+source_schemas = _list_schemas(source_catalog_current)
+target_schemas = _list_schemas(target_catalog_current)
+
+source_schema_current = _current_or_first(source_schemas, _get_widget("source_schema"))
+target_schema_current = _current_or_first(target_schemas, _get_widget("target_schema"))
+
+source_tables = _list_tables(source_catalog_current, source_schema_current)
+target_tables = _list_tables(target_catalog_current, target_schema_current)
+
+source_table_current = _current_or_first(source_tables, _get_widget("source_table"))
+target_table_current = _current_or_first(target_tables, _get_widget("target_table"))
+
+dbutils.widgets.dropdown("source_catalog", source_catalog_current, catalogs or [""], "Source Catalog")
+dbutils.widgets.dropdown("source_schema", source_schema_current, source_schemas or [""], "Source Schema")
+dbutils.widgets.dropdown("source_table", source_table_current, source_tables or [""], "Source Table")
+
+dbutils.widgets.dropdown("target_catalog", target_catalog_current, catalogs or [""], "Target Catalog")
+dbutils.widgets.dropdown("target_schema", target_schema_current, target_schemas or [""], "Target Schema")
+dbutils.widgets.dropdown("target_table", target_table_current, target_tables or [""], "Target Table")
+
+print(
+    "Widgets created. If you just changed Source/Target Catalog or Schema, "
+    "re-run this cell once more to repopulate the dropdowns below it."
+)
+
+# COMMAND ----------
+
+# MAGIC %md ### Validation checks
+
+# COMMAND ----------
+
+_default_checks = ["Catalog & Schema", "Column", "Row"] if MODE != "schema" else ["Catalog & Schema"]
+dbutils.widgets.multiselect(
+    "checks", ",".join(_default_checks),
+    ["Catalog & Schema", "Column", "Row"], "Validation Checks",
+)
+
+if SHOW_OPTIONAL_WIDGETS:
+    dbutils.widgets.text("only_columns", "", "Only compare these columns (comma-separated, optional)")
+    dbutils.widgets.text("ignore_columns", "", "Skip these columns entirely (comma-separated, optional)")
+    dbutils.widgets.text("row_filter", "", "Row filter - SQL WHERE-fragment (optional)")
+    dbutils.widgets.text("primary_key", "", "Primary key column(s) (comma-separated, optional)")
+
+# COMMAND ----------
+
+# MAGIC %md ### Run validation
+
+# COMMAND ----------
+
+source = f"{dbutils.widgets.get('source_catalog')}.{dbutils.widgets.get('source_schema')}.{dbutils.widgets.get('source_table')}"
+target = f"{dbutils.widgets.get('target_catalog')}.{dbutils.widgets.get('target_schema')}.{dbutils.widgets.get('target_table')}"
+
+_CHECK_MAP = {
+    "Catalog & Schema": {ValidationType.CATALOG, ValidationType.SCHEMA},
+    "Column": {ValidationType.COLUMN},
+    "Row": {ValidationType.ROW},
+}
+selected_checks = [c for c in dbutils.widgets.get("checks").split(",") if c]
+enabled_validations = set()
+for check in selected_checks:
+    enabled_validations |= _CHECK_MAP.get(check, set())
+if not enabled_validations:
+    enabled_validations = None  # fall back to validate_tables()'s own "run everything" default
+
+kwargs = {"enabled_validations": enabled_validations}
+
+if SHOW_OPTIONAL_WIDGETS:
+    only_columns = [c.strip() for c in dbutils.widgets.get("only_columns").split(",") if c.strip()]
+    ignore_columns = [c.strip() for c in dbutils.widgets.get("ignore_columns").split(",") if c.strip()]
+    row_filter = dbutils.widgets.get("row_filter").strip()
+    primary_key = [c.strip() for c in dbutils.widgets.get("primary_key").split(",") if c.strip()]
+
+    if only_columns:
+        kwargs["only_columns"] = only_columns
+    if ignore_columns:
+        kwargs["ignore_columns"] = ignore_columns
+    if row_filter:
+        kwargs["row_filter"] = row_filter
+    if primary_key:
+        kwargs["primary_key"] = primary_key
+
+result = validate_tables(source, target, **kwargs)
+
+# COMMAND ----------
+
+# MAGIC %md ### Results
+
+# COMMAND ----------
+
+print(result)
+display(result.table_validation.to_dataframe())
+
+# Other sheets, mirroring the Excel report's own tabs - uncomment as needed:
+# display(result.column_validation.to_dataframe())
+# display(result.data_mismatches.to_dataframe())
+# display(result.row_hash_mismatches.to_dataframe())
+# display(result.mismatch_categories.to_dataframe())
+# display(result.suggestions.to_dataframe())

tablevalidator_databricks-0.1.0.dist-info/METADATA

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: tablevalidator-databricks
+Version: 0.1.0
+Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: typer>=0.12
+Requires-Dist: table-validator>=0.1.19
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: black>=24.0; extra == "dev"
+
+# tablevalidator-databricks
+
+A widget-driven Databricks notebook UI for [`table-validator`](https://pypi.org/project/table-validator/)
+- generates a notebook with `dbutils.widgets` for picking source/target
+tables and validation checks, so you can run a comparison without writing
+any Python.
+
+This package contains **no validation logic of its own**. It only writes a
+notebook file whose cells call directly into `table-validator`'s own
+`validate_tables()` API - the exact same engine the CLI (`tablevalidator`)
+and the notebook-native Python API already use.
+
+## Install
+
+```bash
+pip install tablevalidator-databricks
+```
+
+Installing this package also installs `table-validator` (a dependency), so
+there's nothing else to install separately.
+
+## Usage
+
+```bash
+tablevalidator-databricks init
+```
+
+Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
+
+```bash
+tablevalidator-databricks init --output MyValidation.py --mode full
+```
+
+- `--mode basic` - only the check-group multiselect and table pickers.
+- `--mode full` (default) - also shows optional filter widgets (only/
+  ignore columns, row filter, primary key).
+- `--mode schema` - pre-selects only the "Catalog & Schema" check group,
+  for a lightweight schema-shape-only comparison.
+
+Then, in your Databricks workspace: **Workspace -> Import**, format
+"Source", and select the generated file. Databricks recognizes the
+`# Databricks notebook source` header and `# COMMAND ----------` cell
+markers and opens it as a real notebook.
+
+## What the generated notebook does
+
+1. Installs `table-validator` (`%pip install table-validator`).
+2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
+   `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
+   populate Source/Target Catalog, Schema, and Table dropdowns.
+3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
+   Row - the three independently-selectable check groups the engine
+   actually supports) plus, in `full` mode, optional text widgets for
+   column filtering, a row filter, and a primary key.
+4. On run, reads every widget and calls
+   `validate_tables(source, target, enabled_validations=..., ...)`,
+   then prints a summary and displays the Table Validation sheet.
+
+**Cascading dropdowns**: Databricks widgets don't auto-refresh when an
+upstream selection changes. After changing Source/Target Catalog or
+Schema, re-run the notebook's "Create widgets" cell once to repopulate the
+dropdowns below it before running the rest of the notebook. This is a
+Databricks widget limitation, not a bug in the generated notebook.
+
+## Why a separate package
+
+`table-validator` stays focused on the actual comparison engine (CLI +
+notebook API + report generation). This package is purely a Databricks-
+specific UI layer on top of it - keeping the two independently versioned
+and installable means CLI-only or code-only users never need Databricks-
+specific tooling pulled in, and this package can add more Databricks UX
+(e.g. richer widgets) without touching the core engine at all.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT - see the `table-validator` repository's [LICENSE](../table_validator/LICENSE).

tablevalidator_databricks-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+tablevalidator_databricks/__init__.py,sha256=qhtaaSY_pHgTu5JxvXVoSAMfVC1R2I5OcPcyzLQISQo,560
+tablevalidator_databricks/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tablevalidator_databricks/cli/main.py,sha256=p4ex05jnieabbg1ANuXOCKKJd7-zlbs-PUz5fXD7KAU,3897
+tablevalidator_databricks/generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tablevalidator_databricks/generator/generator.py,sha256=Gn9ziozDmYvK7TnctnVlqu2_Oe8R4DsdN9NQYJcAzMg,1418
+tablevalidator_databricks/templates/widget_notebook.py.tmpl,sha256=k_jGdIZN6tqHPsk1tOLK-E7Qv-_kY-YGA4c90L29m-U,6759
+tablevalidator_databricks-0.1.0.dist-info/METADATA,sha256=2eVYYB4qHAUpFquB2sQyHMFLB7Z-MhTWjuRlsLZwmzU,3713
+tablevalidator_databricks-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+tablevalidator_databricks-0.1.0.dist-info/entry_points.txt,sha256=PzigbELsTjFQ9wJIge3nI3sBQWLsptchG-jyovFFreI,85
+tablevalidator_databricks-0.1.0.dist-info/top_level.txt,sha256=LgxXkFPKeoTSwNXHPQLzWd4x4qDNL86KDxqbCCIb-b4,26
+tablevalidator_databricks-0.1.0.dist-info/RECORD,,

tablevalidator_databricks-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tablevalidator_databricks-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tablevalidator-databricks = tablevalidator_databricks.cli.main:app

tablevalidator_databricks-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tablevalidator_databricks