table-validator 0.1.8__tar.gz → 0.1.10__tar.gz

{table_validator-0.1.8/table_validator.egg-info → table_validator-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.8
+Version: 0.1.10
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9
@@ -15,6 +15,7 @@ Requires-Dist: databricks-sql-connector>=3.0
 Requires-Dist: pandas>=2.0
 Requires-Dist: numpy>=1.24
 Requires-Dist: azure-storage-blob>=12.14
+Requires-Dist: azure-identity>=1.15
 Requires-Dist: openpyxl>=3.1
 Requires-Dist: pyarrow>=14.0
 Requires-Dist: PyYAML>=6.0
@@ -38,6 +39,7 @@ Currently supported sources (target is always Databricks):
 - Databricks catalog → Databricks catalog
 - Azure Blob Storage (CSV / Excel / Parquet) → Databricks table
 - Azure SQL Database → Databricks catalog
+- Azure Synapse SQL pool (dedicated or serverless) → Databricks catalog
 
 More source platforms can be added behind the same connector/validator
 interfaces described below.
@@ -80,10 +82,11 @@ Interactive wizard that walks you through:
 
 1. Azure Storage account + container (optional, skip if you don't have a Blob source) and account key
 2. Azure SQL server + database (optional, skip if you don't have a SQL source) and username/password
-3. Databricks workspace URL, SQL Warehouse HTTP path, and personal access token
-4. Source table (catalog / schema / table)
-5. Target table (catalog / schema / table)
-6. Which validations to run (catalog / schema / column / row)
+3. Azure Synapse SQL pool endpoint + database (optional, skip if you don't have a Synapse source), then either a SQL login (username/password) or a Microsoft Entra ID service principal (tenant ID + client ID + client secret)
+4. Databricks workspace URL, SQL Warehouse HTTP path, and personal access token
+5. Source table (catalog / schema / table)
+6. Target table (catalog / schema / table)
+7. Which validations to run (catalog / schema / column / row)
 
 ```bash
 tablevalidator validate

{table_validator-0.1.8 → table_validator-0.1.10}/README.md

@@ -11,6 +11,7 @@ Currently supported sources (target is always Databricks):
 - Databricks catalog → Databricks catalog
 - Azure Blob Storage (CSV / Excel / Parquet) → Databricks table
 - Azure SQL Database → Databricks catalog
+- Azure Synapse SQL pool (dedicated or serverless) → Databricks catalog
 
 More source platforms can be added behind the same connector/validator
 interfaces described below.
@@ -53,10 +54,11 @@ Interactive wizard that walks you through:
 
 1. Azure Storage account + container (optional, skip if you don't have a Blob source) and account key
 2. Azure SQL server + database (optional, skip if you don't have a SQL source) and username/password
-3. Databricks workspace URL, SQL Warehouse HTTP path, and personal access token
-4. Source table (catalog / schema / table)
-5. Target table (catalog / schema / table)
-6. Which validations to run (catalog / schema / column / row)
+3. Azure Synapse SQL pool endpoint + database (optional, skip if you don't have a Synapse source), then either a SQL login (username/password) or a Microsoft Entra ID service principal (tenant ID + client ID + client secret)
+4. Databricks workspace URL, SQL Warehouse HTTP path, and personal access token
+5. Source table (catalog / schema / table)
+6. Target table (catalog / schema / table)
+7. Which validations to run (catalog / schema / column / row)
 
 ```bash
 tablevalidator validate

{table_validator-0.1.8 → table_validator-0.1.10}/pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
     "pandas>=2.0",
     "numpy>=1.24",
     "azure-storage-blob>=12.14",
+    "azure-identity>=1.15",
     "openpyxl>=3.1",
     "pyarrow>=14.0",
     "PyYAML>=6.0",

{table_validator-0.1.8 → table_validator-0.1.10}/table_validator/auth/azure_auth.py

@@ -26,22 +26,36 @@ class AzureCredential:
 
     storage_account_key authenticates AzureConnector (Blob Storage);
     sql_username/sql_password authenticate AzureSqlConnector (Azure SQL
-    Database). Both are optional here since a given validation run may
-    only need one of the two.
+    Database); synapse_username/synapse_password authenticate the same
+    AzureSqlConnector class when pointed at a Synapse SQL pool instead
+    (Synapse SQL is protocol-identical T-SQL/ODBC, so no separate
+    connector class exists - only a separate credential pair, since a
+    Synapse SQL login is a distinct principal from an Azure SQL DB one).
+    synapse_client_secret is the alternative to that pair: the Entra ID
+    service-principal secret used when config.azure.synapse_auth_mode is
+    'entra_service_principal', paired with the non-secret tenant_id /
+    synapse_client_id that live in config.yaml.
+    All fields are optional here since a given validation run may only
+    need one pair.
     """
 
     storage_account_key: Optional[str] = None
     sql_username: Optional[str] = None
     sql_password: Optional[str] = None
+    synapse_username: Optional[str] = None
+    synapse_password: Optional[str] = None
+    synapse_client_secret: Optional[str] = None
 
 
 def get_azure_credential(config: ValidatorConfig, env_path: Path = ENV_PATH) -> AzureCredential:
     """
     Resolve Azure credentials for the given config.
 
-    Phase 1: reads AZURE_STORAGE_KEY / AZURE_SQL_USERNAME / AZURE_SQL_PASSWORD
-    from ~/.table_validator/.env. A later phase can swap this body for
-    Azure CLI / Service Principal auth without changing any caller.
+    Phase 1: reads AZURE_STORAGE_KEY / AZURE_SQL_USERNAME /
+    AZURE_SQL_PASSWORD / SYNAPSE_USERNAME / SYNAPSE_PASSWORD /
+    SYNAPSE_CLIENT_SECRET from ~/.table_validator/.env. A later phase can
+    swap this body for Azure CLI / managed-identity auth without changing
+    any caller.
     """
     values = dotenv_values(env_path) if env_path.exists() else {}
 
@@ -49,4 +63,7 @@ def get_azure_credential(config: ValidatorConfig, env_path: Path = ENV_PATH) ->
         storage_account_key=values.get("AZURE_STORAGE_KEY") or None,
         sql_username=values.get("AZURE_SQL_USERNAME") or None,
         sql_password=values.get("AZURE_SQL_PASSWORD") or None,
+        synapse_username=values.get("SYNAPSE_USERNAME") or None,
+        synapse_password=values.get("SYNAPSE_PASSWORD") or None,
+        synapse_client_secret=values.get("SYNAPSE_CLIENT_SECRET") or None,
     )

{table_validator-0.1.8 → table_validator-0.1.10}/table_validator/auth/databricks_auth.py

@@ -29,3 +29,18 @@ def get_databricks_token(config: ValidatorConfig, env_path: Path = ENV_PATH) ->
     """
     values = dotenv_values(env_path) if env_path.exists() else {}
     return values.get("DATABRICKS_TOKEN") or None
+
+
+def host_from_workspace_url(workspace_url: Optional[str]) -> Optional[str]:
+    """DatabricksConnector wants a bare hostname; the wizard stores a full
+    https:// workspace URL, so strip the scheme and any trailing path.
+
+    Shared by cli/main.py (building the connector for `validate`) and
+    cli/wizard.py (building a connector during `configure` for the
+    column-mapping live picker) - kept here rather than in either CLI
+    module so neither has to import from the other.
+    """
+    if not workspace_url:
+        return None
+    host = workspace_url.replace("https://", "").replace("http://", "")
+    return host.split("/")[0]

{table_validator-0.1.8 → table_validator-0.1.10}/table_validator/cli/main.py

@@ -10,7 +10,7 @@ from typing import Dict, Optional
 import typer
 
 from table_validator.auth.azure_auth import get_azure_credential
-from table_validator.auth.databricks_auth import get_databricks_token
+from table_validator.auth.databricks_auth import get_databricks_token, host_from_workspace_url
 from table_validator.cli.summary_table import (
     print_summary_table,
     summary_from_excel,
@@ -19,7 +19,12 @@ from table_validator.cli.summary_table import (
 from table_validator.cli.partition_prompt import build_partition_prompt
 from table_validator.cli.wizard import run_configure_wizard
 from table_validator.config.manager import CONFIG_PATH, ConfigNotFoundError, require_config
-from table_validator.config.schema import SourceType, ValidationType, ValidatorConfig
+from table_validator.config.schema import (
+    SourceType,
+    SynapseAuthMode,
+    ValidationType,
+    ValidatorConfig,
+)
 from table_validator.connectors.azure_connector import AzureConnector, AzureSqlConnector
 from table_validator.connectors.databricks_connector import DatabricksConnector
 from table_validator.models import (
@@ -207,6 +212,18 @@ def validate(
             "prompt on its own, but --yes makes that explicit."
         ),
     ),
+    skip_category_summary: bool = typer.Option(
+        False,
+        "--skip-category-summary",
+        help=(
+            "Omit the 'Mismatch Categories' sheet (root-cause "
+            "classification + plain-English summary of Data Mismatches) "
+            "from the Excel report. Every other sheet, including Data "
+            "Mismatches itself, is unaffected - this recovers the exact "
+            "report shape from before the mismatch-categorization "
+            "feature existed, for anyone who doesn't want it."
+        ),
+    ),
 ) -> None:
     """Run validation and produce an Excel validation report."""
 
@@ -256,7 +273,7 @@ def validate(
 
     try:
         databricks = DatabricksConnector(
-            host=_host_from_workspace_url(config.databricks.workspace_url),
+            host=host_from_workspace_url(config.databricks.workspace_url),
             token=token,
             http_path=config.databricks.http_path,
         )
@@ -281,6 +298,8 @@ def validate(
             result = _run_blob_validation(config, azure_credential, databricks)
         elif config.source_type == SourceType.AZURE_SQL:
             result = _run_sql_validation(config, azure_credential, databricks)
+        elif config.source_type == SourceType.SYNAPSE:
+            result = _run_synapse_validation(config, azure_credential, databricks)
         else:
             result = _run_databricks_validation(config, databricks, mode=mode, yes=yes)
     except ValueError as exc:
@@ -310,6 +329,7 @@ def validate(
             result, str(output),
             source_type=config.source_type.value,
             enabled_validations=report_enabled_validations,
+            skip_category_summary=skip_category_summary,
         )
     except PermissionError:
         typer.secho(
@@ -444,6 +464,17 @@ def _run_databricks_validation(
         tables_restriction = [config.source_table.table]
         if config.source_table.table.lower() != config.target_table.table.lower():
             table_map = {config.source_table.table: config.target_table.table}
+    else:
+        # Schema-wide sweep (no single table named on both sides) - apply
+        # the wizard's optional only_tables allowlist / table_map renaming
+        # for this schema, if the user set either. Both are no-ops
+        # (tables_restriction stays None, table_map stays {}) unless the
+        # user actually configured them, so a plain schema sweep behaves
+        # exactly as before this feature existed.
+        if config.only_tables:
+            tables_restriction = list(config.only_tables)
+        if config.table_map:
+            table_map = dict(config.table_map)
 
     # A configured primary key only applies to the single named table
     # (not a catalog-wide sweep) - CatalogValidator looks it up by
@@ -473,6 +504,7 @@ def _run_databricks_validation(
         only_columns=config.only_columns,
         ignore_columns=config.ignore_columns,
         ignore_datatype_columns=config.ignore_datatype_columns,
+        column_map=config.column_map,
     )
 
     partition_prompt = build_partition_prompt(yes=yes)
@@ -602,6 +634,113 @@ def _run_sql_validation(
     return validator.validate(request)
 
 
+def _run_synapse_validation(
+    config: ValidatorConfig,
+    azure_credential,
+    databricks: DatabricksConnector,
+) -> CatalogValidationResponse:
+    """source_type == synapse: Azure Synapse SQL pool (dedicated or
+    serverless) against a Databricks catalog. Synapse SQL is
+    protocol-identical T-SQL over the same ODBC driver Azure SQL Database
+    uses, so this reuses AzureSqlConnector/AzureSqlValidator UNCHANGED,
+    just constructed with the Synapse server/database/credentials instead
+    of an Azure SQL DB's - no new connector or validator class, mirrors
+    _run_sql_validation's logic exactly (see that function for the
+    schema_map/table_map/primary_keys/FULL-mode rationale, identical
+    here).
+
+    Auth is the one place Synapse genuinely diverges from the Azure SQL
+    path: config.azure.synapse_auth_mode selects between a SQL login
+    (SYNAPSE_USERNAME/SYNAPSE_PASSWORD) and a Microsoft Entra ID service
+    principal (azure.tenant_id + azure.synapse_client_id +
+    SYNAPSE_CLIENT_SECRET). Entra is what a workspace with SQL
+    authentication disabled requires - the connector fetches an access
+    token and passes it via the ODBC access-token attribute rather than
+    sending UID/PWD at all.
+
+    Not yet verified against a live Synapse SQL pool for pool-specific
+    T-SQL quirks (dedicated pools have historically had some system-view/
+    HASHBYTES-adjacent gaps vs. Azure SQL DB) - flagged here rather than
+    assumed away; report back if a specific query fails against your pool
+    and it can be special-cased in AzureSqlConnector the same way
+    decimal_as_integer/float formatting already are for Databricks
+    compatibility.
+    """
+    if not config.azure.synapse_server or not config.azure.synapse_database:
+        raise ValueError(
+            "azure.synapse_server and azure.synapse_database are required "
+            "for a Synapse source. Run 'tablevalidator configure' first."
+        )
+
+    if config.azure.synapse_auth_mode == SynapseAuthMode.ENTRA_SERVICE_PRINCIPAL:
+        missing_entra = []
+        if not config.azure.tenant_id:
+            missing_entra.append("azure.tenant_id (in config.yaml)")
+        if not config.azure.synapse_client_id:
+            missing_entra.append("azure.synapse_client_id (in config.yaml)")
+        if not azure_credential.synapse_client_secret:
+            missing_entra.append("SYNAPSE_CLIENT_SECRET (in ~/.table_validator/.env)")
+        if missing_entra:
+            raise ValueError(
+                "Synapse is configured for Entra service-principal auth but "
+                "these are missing: " + ", ".join(missing_entra)
+                + ". Run 'tablevalidator configure' first."
+            )
+
+        synapse = AzureSqlConnector(
+            server=config.azure.synapse_server,
+            database=config.azure.synapse_database,
+            tenant_id=config.azure.tenant_id,
+            client_id=config.azure.synapse_client_id,
+            client_secret=azure_credential.synapse_client_secret,
+        )
+    else:
+        if not azure_credential.synapse_username or not azure_credential.synapse_password:
+            raise ValueError(
+                "No Synapse username/password found in ~/.table_validator/.env. "
+                "Run 'tablevalidator configure' first."
+            )
+
+        synapse = AzureSqlConnector(
+            server=config.azure.synapse_server,
+            database=config.azure.synapse_database,
+            username=azure_credential.synapse_username,
+            password=azure_credential.synapse_password,
+        )
+
+    schema_map: Dict[str, str] = {}
+    schemas_restriction = None
+    if config.synapse_source.schema_name and config.target_table.schema_name:
+        schema_map = {config.synapse_source.schema_name: config.target_table.schema_name}
+        schemas_restriction = [config.synapse_source.schema_name]
+
+    table_map: Dict[str, str] = {}
+    tables_restriction = None
+    if config.synapse_source.table and config.target_table.table:
+        table_map = {config.synapse_source.table: config.target_table.table}
+        tables_restriction = [config.synapse_source.table]
+
+    primary_keys: Dict[str, list] = {}
+    if config.primary_key and config.target_table.table:
+        primary_keys[config.target_table.table] = config.primary_key
+        if config.target_table.schema_name:
+            key = f"{config.target_table.schema_name}.{config.target_table.table}"
+            primary_keys[key] = config.primary_key
+
+    request = AzureSqlValidationRequest(
+        target_catalog=config.target_table.catalog or "",
+        schemas=schemas_restriction,
+        schema_map=schema_map,
+        tables=tables_restriction,
+        table_map=table_map,
+        primary_keys=primary_keys,
+        data_compare_mode=DataCompareMode.FULL,
+    )
+
+    validator = AzureSqlValidator(synapse, databricks)
+    return validator.validate(request)
+
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -631,6 +770,11 @@ def _missing_config_fields(config: ValidatorConfig) -> list:
             missing.append("azure.sql_server")
         if not config.azure.sql_database:
             missing.append("azure.sql_database")
+    elif config.source_type == SourceType.SYNAPSE:
+        if not config.azure.synapse_server:
+            missing.append("azure.synapse_server")
+        if not config.azure.synapse_database:
+            missing.append("azure.synapse_database")
     else:
         if not config.source_table.catalog:
             missing.append("source_table.catalog")
@@ -638,15 +782,6 @@ def _missing_config_fields(config: ValidatorConfig) -> list:
     return missing
 
 
-def _host_from_workspace_url(workspace_url: Optional[str]) -> Optional[str]:
-    """DatabricksConnector wants a bare hostname; the wizard stores a full
-    https:// workspace URL, so strip the scheme and any trailing path."""
-    if not workspace_url:
-        return None
-    host = workspace_url.replace("https://", "").replace("http://", "")
-    return host.split("/")[0]
-
-
 def _open_in_default_app(path: Path) -> None:
     """
     Launch `path` in whatever application the OS has associated with its
@@ -704,6 +839,17 @@ def _describe_scope(config: ValidatorConfig) -> str:
             source_desc = f"sql:{database}.{config.sql_source.schema_name} (all tables)"
         else:
             source_desc = f"sql:{database}.{config.sql_source.schema_name}.{config.sql_source.table}"
+    elif config.source_type == SourceType.SYNAPSE:
+        database = config.azure.synapse_database or "?"
+        if not config.synapse_source.schema_name:
+            source_desc = f"synapse:{database} (all schemas)"
+        elif not config.synapse_source.table:
+            source_desc = f"synapse:{database}.{config.synapse_source.schema_name} (all tables)"
+        else:
+            source_desc = (
+                f"synapse:{database}.{config.synapse_source.schema_name}."
+                f"{config.synapse_source.table}"
+            )
     else:
         source_desc = _side(config.source_table)