table-validator 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- table_validator/__init__.py +46 -0
- table_validator/auth/__init__.py +1 -0
- table_validator/auth/azure_auth.py +52 -0
- table_validator/auth/databricks_auth.py +31 -0
- table_validator/cli/__init__.py +1 -0
- table_validator/cli/main.py +722 -0
- table_validator/cli/partition_prompt.py +78 -0
- table_validator/cli/summary_table.py +146 -0
- table_validator/cli/wizard.py +429 -0
- table_validator/config/__init__.py +1 -0
- table_validator/config/manager.py +84 -0
- table_validator/config/schema.py +179 -0
- table_validator/connectors/__init__.py +1 -0
- table_validator/connectors/azure_connector.py +809 -0
- table_validator/connectors/databricks_connector.py +1230 -0
- table_validator/engine/__init__.py +1 -0
- table_validator/engine/comparison_engine.py +645 -0
- table_validator/models.py +952 -0
- table_validator/reports/__init__.py +1 -0
- table_validator/reports/excel_report.py +953 -0
- table_validator/validators/__init__.py +1 -0
- table_validator/validators/blob_discovery.py +467 -0
- table_validator/validators/catalog_validator.py +1863 -0
- table_validator/validators/row_validator.py +1727 -0
- table_validator-0.1.0.dist-info/METADATA +190 -0
- table_validator-0.1.0.dist-info/RECORD +30 -0
- table_validator-0.1.0.dist-info/WHEEL +5 -0
- table_validator-0.1.0.dist-info/entry_points.txt +2 -0
- table_validator-0.1.0.dist-info/licenses/LICENSE +21 -0
- table_validator-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""table_validator: validates data migrations between Azure and Databricks Delta Lake."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("table-validator")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
__version__ = "0.0.0"
|
|
9
|
+
|
|
10
|
+
from table_validator.config.manager import (
|
|
11
|
+
ConfigNotFoundError,
|
|
12
|
+
default_config,
|
|
13
|
+
load_config,
|
|
14
|
+
require_config,
|
|
15
|
+
save_config,
|
|
16
|
+
)
|
|
17
|
+
from table_validator.config.schema import ValidatorConfig
|
|
18
|
+
from table_validator.connectors.azure_connector import AzureConnector, AzureSqlConnector
|
|
19
|
+
from table_validator.connectors.databricks_connector import DatabricksConnector
|
|
20
|
+
from table_validator.models import CatalogValidationRequest, CatalogValidationResponse
|
|
21
|
+
from table_validator.validators.blob_discovery import BlobCatalogValidator
|
|
22
|
+
from table_validator.validators.catalog_validator import CatalogValidator
|
|
23
|
+
from table_validator.validators.row_validator import AzureCsvValidator, AzureSqlValidator
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"__version__",
|
|
27
|
+
# Validators
|
|
28
|
+
"CatalogValidator",
|
|
29
|
+
"AzureCsvValidator",
|
|
30
|
+
"AzureSqlValidator",
|
|
31
|
+
"BlobCatalogValidator",
|
|
32
|
+
# Connectors
|
|
33
|
+
"DatabricksConnector",
|
|
34
|
+
"AzureConnector",
|
|
35
|
+
"AzureSqlConnector",
|
|
36
|
+
# Config
|
|
37
|
+
"ValidatorConfig",
|
|
38
|
+
"ConfigNotFoundError",
|
|
39
|
+
"default_config",
|
|
40
|
+
"load_config",
|
|
41
|
+
"require_config",
|
|
42
|
+
"save_config",
|
|
43
|
+
# Core request/response models
|
|
44
|
+
"CatalogValidationRequest",
|
|
45
|
+
"CatalogValidationResponse",
|
|
46
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Auth package: credential abstractions for Azure and Databricks."""
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Azure auth abstraction.
|
|
2
|
+
|
|
3
|
+
Phase 1 auth: credentials are entered manually via the CLI wizard and
|
|
4
|
+
stored in ~/.table_validator/.env. get_azure_credential() is the single
|
|
5
|
+
place that reads them - every connector must call it instead of reading
|
|
6
|
+
os.environ directly, so only this function needs to change when a later
|
|
7
|
+
phase adds Azure CLI / Service Principal auth.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from dotenv import dotenv_values
|
|
17
|
+
|
|
18
|
+
from table_validator.config.schema import ValidatorConfig
|
|
19
|
+
|
|
20
|
+
ENV_PATH = Path.home() / ".table_validator" / ".env"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class AzureCredential:
|
|
25
|
+
"""Resolved Azure credentials for the connectors this tool uses today.
|
|
26
|
+
|
|
27
|
+
storage_account_key authenticates AzureConnector (Blob Storage);
|
|
28
|
+
sql_username/sql_password authenticate AzureSqlConnector (Azure SQL
|
|
29
|
+
Database). Both are optional here since a given validation run may
|
|
30
|
+
only need one of the two.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
storage_account_key: Optional[str] = None
|
|
34
|
+
sql_username: Optional[str] = None
|
|
35
|
+
sql_password: Optional[str] = None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_azure_credential(config: ValidatorConfig, env_path: Path = ENV_PATH) -> AzureCredential:
|
|
39
|
+
"""
|
|
40
|
+
Resolve Azure credentials for the given config.
|
|
41
|
+
|
|
42
|
+
Phase 1: reads AZURE_STORAGE_KEY / AZURE_SQL_USERNAME / AZURE_SQL_PASSWORD
|
|
43
|
+
from ~/.table_validator/.env. A later phase can swap this body for
|
|
44
|
+
Azure CLI / Service Principal auth without changing any caller.
|
|
45
|
+
"""
|
|
46
|
+
values = dotenv_values(env_path) if env_path.exists() else {}
|
|
47
|
+
|
|
48
|
+
return AzureCredential(
|
|
49
|
+
storage_account_key=values.get("AZURE_STORAGE_KEY") or None,
|
|
50
|
+
sql_username=values.get("AZURE_SQL_USERNAME") or None,
|
|
51
|
+
sql_password=values.get("AZURE_SQL_PASSWORD") or None,
|
|
52
|
+
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Databricks auth abstraction.
|
|
2
|
+
|
|
3
|
+
Phase 1 auth: the personal access token (PAT) is entered manually via the
|
|
4
|
+
CLI wizard and stored in ~/.table_validator/.env. get_databricks_token() is
|
|
5
|
+
the single place that reads it - every connector must call it instead of
|
|
6
|
+
reading os.environ directly, so only this function needs to change when a
|
|
7
|
+
later phase adds Databricks CLI / OAuth auth.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Optional
|
|
14
|
+
|
|
15
|
+
from dotenv import dotenv_values
|
|
16
|
+
|
|
17
|
+
from table_validator.config.schema import ValidatorConfig
|
|
18
|
+
|
|
19
|
+
ENV_PATH = Path.home() / ".table_validator" / ".env"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_databricks_token(config: ValidatorConfig, env_path: Path = ENV_PATH) -> Optional[str]:
|
|
23
|
+
"""
|
|
24
|
+
Resolve the Databricks personal access token for the given config.
|
|
25
|
+
|
|
26
|
+
Phase 1: reads DATABRICKS_TOKEN from ~/.table_validator/.env. A later
|
|
27
|
+
phase can swap this body for Databricks CLI / OAuth auth without
|
|
28
|
+
changing any caller.
|
|
29
|
+
"""
|
|
30
|
+
values = dotenv_values(env_path) if env_path.exists() else {}
|
|
31
|
+
return values.get("DATABRICKS_TOKEN") or None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""CLI package: Typer app and interactive configuration wizard."""
|