table-validator 0.1.0__tar.gz

table_validator-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mourya S
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

table_validator-0.1.0/PKG-INFO

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: table-validator
+Version: 0.1.0
+Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.12
+Requires-Dist: rich>=13.0
+Requires-Dist: questionary>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyodbc>=4.0
+Requires-Dist: databricks-sql-connector>=3.0
+Requires-Dist: pandas>=2.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: azure-storage-blob>=12.14
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: pyarrow>=14.0
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: python-dotenv>=1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: black>=24.0; extra == "dev"
+Dynamic: license-file
+
+# table-validator
+
+Cross-platform data migration validator. Compares a source table against a
+target Databricks table/catalog and reports whether the migration is
+correct: matching schema, row counts, column statistics, and (where a
+difference is found) the exact row/column that changed. Produces a
+multi-sheet Excel report and a pass/fail summary.
+
+Currently supported sources (target is always Databricks):
+
+- Databricks catalog → Databricks catalog
+- Azure Blob Storage (CSV / Excel / Parquet) → Databricks table
+- Azure SQL Database → Databricks catalog
+
+More source platforms can be added behind the same connector/validator
+interfaces described below.
+
+## Install
+
+```bash
+pip install table-validator
+```
+
+From a local checkout (the directory containing `pyproject.toml`):
+
+```bash
+pip install .
+```
+
+For local development (editable install, so code changes take effect
+without reinstalling):
+
+```bash
+pip install -e ".[dev]"
+```
+
+Either way, this installs the `tablevalidator` command on your PATH.
+
+## CLI usage
+
+```bash
+tablevalidator info
+```
+
+Prints what the tool does and the commands below, in the order you'd
+normally run them.
+
+```bash
+tablevalidator configure
+```
+
+Interactive wizard that walks you through:
+
+1. Azure Storage account + container (optional, skip if you don't have a Blob source) and account key
+2. Azure SQL server + database (optional, skip if you don't have a SQL source) and username/password
+3. Databricks workspace URL, SQL Warehouse HTTP path, and personal access token
+4. Source table (catalog / schema / table)
+5. Target table (catalog / schema / table)
+6. Which validations to run (catalog / schema / column / row)
+
+```bash
+tablevalidator validate
+```
+
+Runs the comparison using the saved configuration and writes
+`validation_report.xlsx` in the current directory, printing a pass/fail
+summary to the console. Exit code is `0` if the overall result is PASS,
+non-zero otherwise (useful in CI). Useful flags:
+
+```bash
+tablevalidator validate --config-path /path/to/config.yaml --output /path/to/report.xlsx
+```
+
+```bash
+tablevalidator open
+```
+
+Opens the most recently generated report in your default spreadsheet app.
+
+## Quickstart (Python API)
+
+Everything the CLI does is available as a library, built from the same
+public API exported by `table_validator/__init__.py`:
+
+```python
+from table_validator import (
+    load_config,
+    CatalogValidator,
+    CatalogValidationRequest,
+    DatabricksConnector,
+)
+from table_validator.auth.databricks_auth import get_databricks_token
+
+# Non-secret settings from ~/.table_validator/config.yaml
+# (see `tablevalidator configure`); secrets from ~/.table_validator/.env.
+config = load_config()
+token = get_databricks_token(config)
+
+# DatabricksConnector wants a bare hostname, not the full workspace URL
+host = config.databricks.workspace_url.replace("https://", "").split("/")[0]
+
+databricks = DatabricksConnector(
+    host=host,
+    token=token,
+    http_path=config.databricks.http_path,
+)
+
+validator = CatalogValidator(databricks)
+
+request = CatalogValidationRequest(
+    source_catalog="source_catalog_name",
+    target_catalog="target_catalog_name",
+    schemas=["sales"],          # optional: restrict scope
+    primary_keys={"sales.orders": ["order_id"]},  # optional: enables row-level diffing
+)
+
+result = validator.compare_catalogs(request)
+
+print(result.status)  # PASS / FAIL / ERROR / SKIPPED
+```
+
+Other public entry points exported from `table_validator`:
+
+- `AzureCsvValidator` / `AzureSqlValidator` — the Blob-CSV and Azure-SQL
+  equivalents of `CatalogValidator`, returning the same
+  `CatalogValidationResponse` shape.
+- `BlobCatalogValidator` — validates every file in an Azure Blob container
+  against like-named Databricks tables.
+- `AzureConnector` / `AzureSqlConnector` — the Azure-side connectors, for
+  building your own validation flow against a connector directly.
+- `ValidatorConfig`, `default_config`, `save_config`, `require_config`,
+  `ConfigNotFoundError` — the same config load/save layer the CLI wizard
+  uses, if you want to construct or persist configuration programmatically.
+
+## Where config and secrets are stored
+
+Everything lives outside the repo, under your home directory:
+
+- `~/.table_validator/config.yaml` — non-secret configuration (table
+  references, workspace URL, which validations are enabled). Safe to
+  inspect or version-control separately if you want.
+- `~/.table_validator/.env` — credentials (Azure Storage key, Azure SQL
+  username/password, Databricks personal access token), written in
+  plaintext and restricted to owner read/write only (`chmod 600`,
+  best-effort on Windows since NTFS doesn't map POSIX permission bits).
+  **Never commit this file or add it inside the project repo** — it isn't,
+  by construction, since it's written under your home directory rather
+  than the working directory.
+
+A future version will replace manual credential entry with Azure CLI /
+Service Principal auth and Databricks CLI / OAuth login, without changing
+the config file format or any command usage above.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

table_validator-0.1.0/README.md

@@ -0,0 +1,163 @@
+# table-validator
+
+Cross-platform data migration validator. Compares a source table against a
+target Databricks table/catalog and reports whether the migration is
+correct: matching schema, row counts, column statistics, and (where a
+difference is found) the exact row/column that changed. Produces a
+multi-sheet Excel report and a pass/fail summary.
+
+Currently supported sources (target is always Databricks):
+
+- Databricks catalog → Databricks catalog
+- Azure Blob Storage (CSV / Excel / Parquet) → Databricks table
+- Azure SQL Database → Databricks catalog
+
+More source platforms can be added behind the same connector/validator
+interfaces described below.
+
+## Install
+
+```bash
+pip install table-validator
+```
+
+From a local checkout (the directory containing `pyproject.toml`):
+
+```bash
+pip install .
+```
+
+For local development (editable install, so code changes take effect
+without reinstalling):
+
+```bash
+pip install -e ".[dev]"
+```
+
+Either way, this installs the `tablevalidator` command on your PATH.
+
+## CLI usage
+
+```bash
+tablevalidator info
+```
+
+Prints what the tool does and the commands below, in the order you'd
+normally run them.
+
+```bash
+tablevalidator configure
+```
+
+Interactive wizard that walks you through:
+
+1. Azure Storage account + container (optional, skip if you don't have a Blob source) and account key
+2. Azure SQL server + database (optional, skip if you don't have a SQL source) and username/password
+3. Databricks workspace URL, SQL Warehouse HTTP path, and personal access token
+4. Source table (catalog / schema / table)
+5. Target table (catalog / schema / table)
+6. Which validations to run (catalog / schema / column / row)
+
+```bash
+tablevalidator validate
+```
+
+Runs the comparison using the saved configuration and writes
+`validation_report.xlsx` in the current directory, printing a pass/fail
+summary to the console. Exit code is `0` if the overall result is PASS,
+non-zero otherwise (useful in CI). Useful flags:
+
+```bash
+tablevalidator validate --config-path /path/to/config.yaml --output /path/to/report.xlsx
+```
+
+```bash
+tablevalidator open
+```
+
+Opens the most recently generated report in your default spreadsheet app.
+
+## Quickstart (Python API)
+
+Everything the CLI does is available as a library, built from the same
+public API exported by `table_validator/__init__.py`:
+
+```python
+from table_validator import (
+    load_config,
+    CatalogValidator,
+    CatalogValidationRequest,
+    DatabricksConnector,
+)
+from table_validator.auth.databricks_auth import get_databricks_token
+
+# Non-secret settings from ~/.table_validator/config.yaml
+# (see `tablevalidator configure`); secrets from ~/.table_validator/.env.
+config = load_config()
+token = get_databricks_token(config)
+
+# DatabricksConnector wants a bare hostname, not the full workspace URL
+host = config.databricks.workspace_url.replace("https://", "").split("/")[0]
+
+databricks = DatabricksConnector(
+    host=host,
+    token=token,
+    http_path=config.databricks.http_path,
+)
+
+validator = CatalogValidator(databricks)
+
+request = CatalogValidationRequest(
+    source_catalog="source_catalog_name",
+    target_catalog="target_catalog_name",
+    schemas=["sales"],          # optional: restrict scope
+    primary_keys={"sales.orders": ["order_id"]},  # optional: enables row-level diffing
+)
+
+result = validator.compare_catalogs(request)
+
+print(result.status)  # PASS / FAIL / ERROR / SKIPPED
+```
+
+Other public entry points exported from `table_validator`:
+
+- `AzureCsvValidator` / `AzureSqlValidator` — the Blob-CSV and Azure-SQL
+  equivalents of `CatalogValidator`, returning the same
+  `CatalogValidationResponse` shape.
+- `BlobCatalogValidator` — validates every file in an Azure Blob container
+  against like-named Databricks tables.
+- `AzureConnector` / `AzureSqlConnector` — the Azure-side connectors, for
+  building your own validation flow against a connector directly.
+- `ValidatorConfig`, `default_config`, `save_config`, `require_config`,
+  `ConfigNotFoundError` — the same config load/save layer the CLI wizard
+  uses, if you want to construct or persist configuration programmatically.
+
+## Where config and secrets are stored
+
+Everything lives outside the repo, under your home directory:
+
+- `~/.table_validator/config.yaml` — non-secret configuration (table
+  references, workspace URL, which validations are enabled). Safe to
+  inspect or version-control separately if you want.
+- `~/.table_validator/.env` — credentials (Azure Storage key, Azure SQL
+  username/password, Databricks personal access token), written in
+  plaintext and restricted to owner read/write only (`chmod 600`,
+  best-effort on Windows since NTFS doesn't map POSIX permission bits).
+  **Never commit this file or add it inside the project repo** — it isn't,
+  by construction, since it's written under your home directory rather
+  than the working directory.
+
+A future version will replace manual credential entry with Azure CLI /
+Service Principal auth and Databricks CLI / OAuth login, without changing
+the config file format or any command usage above.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

table_validator-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=68", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "table-validator"
+dynamic = ["version"]
+description = "CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+dependencies = [
+    "typer>=0.12",
+    "rich>=13.0",
+    "questionary>=2.0",
+    "pydantic>=2.0",
+    "pyodbc>=4.0",
+    "databricks-sql-connector>=3.0",
+    "pandas>=2.0",
+    "numpy>=1.24",
+    "azure-storage-blob>=12.14",
+    "openpyxl>=3.1",
+    "pyarrow>=14.0",
+    "PyYAML>=6.0",
+    "python-dotenv>=1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.4",
+    "black>=24.0",
+]
+
+[project.scripts]
+tablevalidator = "table_validator.cli.main:app"
+
+[tool.setuptools.packages.find]
+include = ["table_validator", "table_validator.*"]
+
+[tool.setuptools_scm]
+root = ".."
+tag_regex = "^v(?P<version>\\d+\\.\\d+\\.\\d+)$"
+fallback_version = "0.0.0"

table_validator-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

table_validator-0.1.0/table_validator/__init__.py

@@ -0,0 +1,46 @@
+"""table_validator: validates data migrations between Azure and Databricks Delta Lake."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("table-validator")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+from table_validator.config.manager import (
+    ConfigNotFoundError,
+    default_config,
+    load_config,
+    require_config,
+    save_config,
+)
+from table_validator.config.schema import ValidatorConfig
+from table_validator.connectors.azure_connector import AzureConnector, AzureSqlConnector
+from table_validator.connectors.databricks_connector import DatabricksConnector
+from table_validator.models import CatalogValidationRequest, CatalogValidationResponse
+from table_validator.validators.blob_discovery import BlobCatalogValidator
+from table_validator.validators.catalog_validator import CatalogValidator
+from table_validator.validators.row_validator import AzureCsvValidator, AzureSqlValidator
+
+__all__ = [
+    "__version__",
+    # Validators
+    "CatalogValidator",
+    "AzureCsvValidator",
+    "AzureSqlValidator",
+    "BlobCatalogValidator",
+    # Connectors
+    "DatabricksConnector",
+    "AzureConnector",
+    "AzureSqlConnector",
+    # Config
+    "ValidatorConfig",
+    "ConfigNotFoundError",
+    "default_config",
+    "load_config",
+    "require_config",
+    "save_config",
+    # Core request/response models
+    "CatalogValidationRequest",
+    "CatalogValidationResponse",
+]

table_validator-0.1.0/table_validator/auth/__init__.py

@@ -0,0 +1 @@
+"""Auth package: credential abstractions for Azure and Databricks."""

table_validator-0.1.0/table_validator/auth/azure_auth.py

@@ -0,0 +1,52 @@
+"""Azure auth abstraction.
+
+Phase 1 auth: credentials are entered manually via the CLI wizard and
+stored in ~/.table_validator/.env. get_azure_credential() is the single
+place that reads them - every connector must call it instead of reading
+os.environ directly, so only this function needs to change when a later
+phase adds Azure CLI / Service Principal auth.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from dotenv import dotenv_values
+
+from table_validator.config.schema import ValidatorConfig
+
+ENV_PATH = Path.home() / ".table_validator" / ".env"
+
+
+@dataclass
+class AzureCredential:
+    """Resolved Azure credentials for the connectors this tool uses today.
+
+    storage_account_key authenticates AzureConnector (Blob Storage);
+    sql_username/sql_password authenticate AzureSqlConnector (Azure SQL
+    Database). Both are optional here since a given validation run may
+    only need one of the two.
+    """
+
+    storage_account_key: Optional[str] = None
+    sql_username: Optional[str] = None
+    sql_password: Optional[str] = None
+
+
+def get_azure_credential(config: ValidatorConfig, env_path: Path = ENV_PATH) -> AzureCredential:
+    """
+    Resolve Azure credentials for the given config.
+
+    Phase 1: reads AZURE_STORAGE_KEY / AZURE_SQL_USERNAME / AZURE_SQL_PASSWORD
+    from ~/.table_validator/.env. A later phase can swap this body for
+    Azure CLI / Service Principal auth without changing any caller.
+    """
+    values = dotenv_values(env_path) if env_path.exists() else {}
+
+    return AzureCredential(
+        storage_account_key=values.get("AZURE_STORAGE_KEY") or None,
+        sql_username=values.get("AZURE_SQL_USERNAME") or None,
+        sql_password=values.get("AZURE_SQL_PASSWORD") or None,
+    )

table_validator-0.1.0/table_validator/auth/databricks_auth.py

@@ -0,0 +1,31 @@
+"""Databricks auth abstraction.
+
+Phase 1 auth: the personal access token (PAT) is entered manually via the
+CLI wizard and stored in ~/.table_validator/.env. get_databricks_token() is
+the single place that reads it - every connector must call it instead of
+reading os.environ directly, so only this function needs to change when a
+later phase adds Databricks CLI / OAuth auth.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from dotenv import dotenv_values
+
+from table_validator.config.schema import ValidatorConfig
+
+ENV_PATH = Path.home() / ".table_validator" / ".env"
+
+
+def get_databricks_token(config: ValidatorConfig, env_path: Path = ENV_PATH) -> Optional[str]:
+    """
+    Resolve the Databricks personal access token for the given config.
+
+    Phase 1: reads DATABRICKS_TOKEN from ~/.table_validator/.env. A later
+    phase can swap this body for Databricks CLI / OAuth auth without
+    changing any caller.
+    """
+    values = dotenv_values(env_path) if env_path.exists() else {}
+    return values.get("DATABRICKS_TOKEN") or None

table_validator-0.1.0/table_validator/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI package: Typer app and interactive configuration wizard."""