databricks-schema 0.5.0__tar.gz → 0.6.0__tar.gz

databricks_schema-0.5.0/README.md → databricks_schema-0.6.0/PKG-INFO

@@ -1,3 +1,25 @@
+Metadata-Version: 2.4
+Name: databricks-schema
+Version: 0.6.0
+Summary: Databricks Unity Catalog schema extractor
+Project-URL: Repository, https://github.com/frizzleqq/databricks-schema
+Project-URL: Issues, https://github.com/frizzleqq/databricks-schema/issues
+Author: frizzleqq
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: databricks,schema,sql,unity-catalog
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: databricks-sdk>=0.64.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
 # databricks-schema
 
 A CLI tool and Python library that uses the Databricks SDK to extract and diff Unity Catalog schemas as YAML files. It can also generate Databricks Spark SQL to apply schema changes across catalogs.
@@ -81,7 +103,7 @@ uv sync --all-groups
 The tool uses the [Databricks SDK](https://github.com/databricks/databricks-sdk-py) for auth. Configure it via environment variables:
 
 ```bash
-export DATABRICKS_HOST=https://<workspace>.cloud.databricks.com
+export DATABRICKS_HOST=https://<databricks-url>
 export DATABRICKS_TOKEN=<your-personal-access-token>
 ```
 
@@ -213,6 +235,22 @@ Include additional metadata (`owner`, `storage_location`) in the comparison:
 databricks-schema generate-sql <catalog> ./schemas/ --include-metadata
 ```
 
+### `validate`
+
+Validate local schema files for structural integrity (no Databricks connection needed):
+
+```bash
+databricks-schema validate ./schemas/
+```
+
+### `diff-files`
+
+Compare two local directories of schema files (no Databricks connection needed):
+
+```bash
+databricks-schema diff-files ./schemas-prod/ ./schemas-test/
+```
+
 ### `list-catalogs`
 
 List all accessible catalogs:
@@ -231,6 +269,8 @@ databricks-schema list-schemas <catalog>
 
 ## Python Library Usage
 
+See also the [`examples/`](examples/) directory for standalone scripts.
+
 ```python
 from pathlib import Path
 from databricks_schema import CatalogExtractor, catalog_to_yaml, schema_from_yaml

databricks_schema-0.5.0/PKG-INFO → databricks_schema-0.6.0/README.md

@@ -1,14 +1,3 @@
-Metadata-Version: 2.4
-Name: databricks-schema
-Version: 0.5.0
-Summary: Databricks Unity Catalog schema extractor
-License-File: LICENSE
-Requires-Python: >=3.11
-Requires-Dist: databricks-sdk>=0.64.0
-Requires-Dist: pydantic>=2.0
-Requires-Dist: pyyaml>=6.0
-Description-Content-Type: text/markdown
-
 # databricks-schema
 
 A CLI tool and Python library that uses the Databricks SDK to extract and diff Unity Catalog schemas as YAML files. It can also generate Databricks Spark SQL to apply schema changes across catalogs.
@@ -92,7 +81,7 @@ uv sync --all-groups
 The tool uses the [Databricks SDK](https://github.com/databricks/databricks-sdk-py) for auth. Configure it via environment variables:
 
 ```bash
-export DATABRICKS_HOST=https://<workspace>.cloud.databricks.com
+export DATABRICKS_HOST=https://<databricks-url>
 export DATABRICKS_TOKEN=<your-personal-access-token>
 ```
 
@@ -224,6 +213,22 @@ Include additional metadata (`owner`, `storage_location`) in the comparison:
 databricks-schema generate-sql <catalog> ./schemas/ --include-metadata
 ```
 
+### `validate`
+
+Validate local schema files for structural integrity (no Databricks connection needed):
+
+```bash
+databricks-schema validate ./schemas/
+```
+
+### `diff-files`
+
+Compare two local directories of schema files (no Databricks connection needed):
+
+```bash
+databricks-schema diff-files ./schemas-prod/ ./schemas-test/
+```
+
 ### `list-catalogs`
 
 List all accessible catalogs:
@@ -242,6 +247,8 @@ databricks-schema list-schemas <catalog>
 
 ## Python Library Usage
 
+See also the [`examples/`](examples/) directory for standalone scripts.
+
 ```python
 from pathlib import Path
 from databricks_schema import CatalogExtractor, catalog_to_yaml, schema_from_yaml

{databricks_schema-0.5.0 → databricks_schema-0.6.0}/databricks_schema/__init__.py

@@ -7,11 +7,13 @@ from databricks_schema.diff import (
     SchemaDiff,
     TableDiff,
     diff_catalog_with_dir,
+    diff_schema_dirs,
     diff_schemas,
 )
 from databricks_schema.extractor import CatalogExtractor
 from databricks_schema.models import Catalog, Column, ForeignKey, PrimaryKey, Schema, Table
 from databricks_schema.sql_gen import schema_diff_to_sql
+from databricks_schema.validate import ValidationIssue, ValidationResult, validate_schemas
 from databricks_schema.yaml_io import (
     catalog_from_json,
     catalog_from_yaml,
@@ -37,16 +39,19 @@ __all__ = [
     "Table",
     "TableDiff",
     "TableType",
+    "ValidationIssue",
+    "ValidationResult",
     "catalog_from_json",
     "catalog_from_yaml",
     "catalog_to_json",
     "catalog_to_yaml",
     "diff_catalog_with_dir",
+    "diff_schema_dirs",
     "diff_schemas",
-    "schema_from_json",
     "schema_diff_to_sql",
     "schema_from_json",
     "schema_from_yaml",
     "schema_to_json",
     "schema_to_yaml",
+    "validate_schemas",
 ]

{databricks_schema-0.5.0 → databricks_schema-0.6.0}/databricks_schema/cli.py

@@ -7,11 +7,19 @@ import sys
 from pathlib import Path
 
 from databricks.sdk import WorkspaceClient
-
-from databricks_schema.diff import CatalogDiff, SchemaDiff, diff_catalog_with_dir, diff_schemas
+from databricks.sdk.errors import DatabricksError, NotFound, PermissionDenied, Unauthenticated
+
+from databricks_schema.diff import (
+    CatalogDiff,
+    SchemaDiff,
+    diff_catalog_with_dir,
+    diff_schema_dirs,
+    diff_schemas,
+)
 from databricks_schema.extractor import CatalogExtractor
 from databricks_schema.models import Schema
 from databricks_schema.sql_gen import schema_diff_to_sql
+from databricks_schema.validate import validate_schemas
 from databricks_schema.yaml_io import (
     schema_from_json,
     schema_from_yaml,
@@ -237,6 +245,83 @@ def _cmd_generate_sql(args: argparse.Namespace) -> None:
             print()
 
 
+def _detect_fmt(schema_dir: Path) -> str:
+    """Auto-detect YAML or JSON format from files in a directory. Exits 2 on mixed or empty."""
+    yaml_files = list(schema_dir.glob("*.yaml"))
+    json_files = list(schema_dir.glob("*.json"))
+    if yaml_files and json_files:
+        print(f"Error: mixed YAML and JSON files in {schema_dir}.", file=sys.stderr)
+        sys.exit(2)
+    elif not yaml_files and not json_files:
+        print(f"No YAML or JSON files found in {schema_dir}.", file=sys.stderr)
+        sys.exit(2)
+    return "json" if json_files else "yaml"
+
+
+def _cmd_validate(args: argparse.Namespace) -> None:
+    """Validate local schema files for structural integrity."""
+    schema_dir: Path = args.schema_dir
+    if not schema_dir.is_dir():
+        print(f"Error: {schema_dir} is not a directory.", file=sys.stderr)
+        sys.exit(2)
+
+    fmt = _detect_fmt(schema_dir)
+    ext = ".json" if fmt == "json" else ".yaml"
+    loader = schema_from_json if fmt == "json" else schema_from_yaml
+
+    schema_filter_set: frozenset[str] | None = frozenset(args.schema) if args.schema else None
+    schemas: dict[str, Schema] = {}
+    for schema_file in sorted(schema_dir.glob(f"*{ext}")):
+        if schema_filter_set is not None and schema_file.stem not in schema_filter_set:
+            continue
+        schema = loader(schema_file.read_text(encoding="utf-8"))
+        schemas[schema.name] = schema
+
+    if not schemas:
+        print("No schemas to validate.", file=sys.stderr)
+        sys.exit(2)
+
+    result = validate_schemas(schemas)
+    if not result.has_errors:
+        print(f"OK — {len(schemas)} schema(s) validated, no issues found.")
+        return
+
+    for issue in result.issues:
+        print(f"ERROR: {issue}")
+    print(f"\n{len(result.issues)} issue(s) found in {len(schemas)} schema(s).", file=sys.stderr)
+    sys.exit(1)
+
+
+def _cmd_diff_files(args: argparse.Namespace) -> None:
+    """Compare two local directories of schema files."""
+    dir1: Path = args.dir1
+    dir2: Path = args.dir2
+    for d in (dir1, dir2):
+        if not d.is_dir():
+            print(f"Error: {d} is not a directory.", file=sys.stderr)
+            sys.exit(2)
+
+    fmt1 = _detect_fmt(dir1)
+    fmt2 = _detect_fmt(dir2)
+
+    print(f"Comparing {dir1} against {dir2}...", file=sys.stderr)
+    result = diff_schema_dirs(
+        dir1,
+        dir2,
+        fmt1=fmt1,
+        fmt2=fmt2,
+        schema_names=frozenset(args.schema) if args.schema else None,
+        include_metadata=args.include_metadata,
+    )
+
+    if not result.has_changes:
+        print("No differences found.")
+        return
+
+    _print_diff(result)
+    sys.exit(1)
+
+
 def _cmd_list_catalogs(args: argparse.Namespace) -> None:
     """List all accessible catalogs."""
     client = _make_client(args.host, args.token)
@@ -396,6 +481,45 @@ def _build_parser() -> argparse.ArgumentParser:
     _add_connection_args(gen_sql_p)
     gen_sql_p.set_defaults(func=_cmd_generate_sql)
 
+    # validate
+    validate_p = subparsers.add_parser(
+        "validate",
+        help="Validate local schema files for structural integrity (no Databricks connection).",
+    )
+    validate_p.add_argument(
+        "schema_dir", type=Path, help="Directory containing per-schema YAML or JSON files"
+    )
+    validate_p.add_argument(
+        "--schema",
+        "-s",
+        action="append",
+        metavar="SCHEMA",
+        help="Schema filter (repeatable)",
+    )
+    validate_p.set_defaults(func=_cmd_validate)
+
+    # diff-files
+    diff_files_p = subparsers.add_parser(
+        "diff-files",
+        help="Compare two local directories of schema files (no Databricks connection needed).",
+    )
+    diff_files_p.add_argument("dir1", type=Path, help="Baseline directory (old/stored state)")
+    diff_files_p.add_argument("dir2", type=Path, help="Current directory (new/live state)")
+    diff_files_p.add_argument(
+        "--schema",
+        "-s",
+        action="append",
+        metavar="SCHEMA",
+        help="Schema filter (repeatable)",
+    )
+    diff_files_p.add_argument(
+        "--include-metadata",
+        action="store_true",
+        dest="include_metadata",
+        help="Include owner in comparison",
+    )
+    diff_files_p.set_defaults(func=_cmd_diff_files)
+
     # list-catalogs
     list_catalogs_p = subparsers.add_parser("list-catalogs", help="List all accessible catalogs.")
     _add_connection_args(list_catalogs_p)
@@ -416,4 +540,12 @@ def main() -> None:
         parser.print_help()
         sys.exit(0)
     args = parser.parse_args()
-    args.func(args)
+    try:
+        args.func(args)
+    except NotFound as e:
+        print(f"Error: {type(e).__name__}: {e}", file=sys.stderr)
+        print("Check that the catalog and schema names are correct.", file=sys.stderr)
+        sys.exit(2)
+    except (Unauthenticated, PermissionDenied, DatabricksError) as e:
+        print(f"Error: {type(e).__name__}: {e}", file=sys.stderr)
+        sys.exit(2)

{databricks_schema-0.5.0 → databricks_schema-0.6.0}/databricks_schema/diff.py

@@ -128,6 +128,54 @@ def diff_schemas(live: Schema, stored: Schema, include_metadata: bool = False) -
     return SchemaDiff(name=live.name, status=status, changes=changes, tables=table_diffs)
 
 
+def diff_schema_dirs(
+    dir1: Path,
+    dir2: Path,
+    fmt1: Literal["yaml", "json"] = "yaml",
+    fmt2: Literal["yaml", "json"] = "yaml",
+    schema_names: frozenset[str] | None = None,
+    include_metadata: bool = False,
+) -> CatalogDiff:
+    """Compare two local directories of schema files.
+
+    dir1 is treated as the stored/baseline state (old).
+    dir2 is treated as the live/current state (new).
+
+    Schemas present in dir1 but not dir2 are reported as "removed".
+    Schemas present in dir2 but not dir1 are reported as "added".
+    Schemas present in both are compared with diff_schemas.
+
+    schema_names: if set, only files whose stem is in this set are loaded from either directory.
+    fmt1 / fmt2: file format for dir1 and dir2 respectively ("yaml" or "json").
+    """
+
+    def _load_dir(path: Path, fmt: Literal["yaml", "json"]) -> dict[str, Schema]:
+        ext = ".json" if fmt == "json" else ".yaml"
+        loader = schema_from_json if fmt == "json" else schema_from_yaml
+        schemas: dict[str, Schema] = {}
+        for f in sorted(path.glob(f"*{ext}")):
+            if schema_names is not None and f.stem not in schema_names:
+                continue
+            schema = loader(f.read_text(encoding="utf-8"))
+            schemas[schema.name] = schema
+        return schemas
+
+    stored = _load_dir(dir1, fmt1)
+    live = _load_dir(dir2, fmt2)
+
+    schema_diffs: list[SchemaDiff] = []
+    for name, stored_schema in stored.items():
+        if name not in live:
+            schema_diffs.append(SchemaDiff(name=name, status="removed"))
+        else:
+            schema_diffs.append(diff_schemas(live[name], stored_schema, include_metadata))
+    for name in live:
+        if name not in stored:
+            schema_diffs.append(SchemaDiff(name=name, status="added"))
+
+    return CatalogDiff(schemas=schema_diffs)
+
+
 def diff_catalog_with_dir(
     catalog: Catalog,
     schema_dir: Path,

databricks_schema-0.6.0/databricks_schema/validate.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from databricks_schema.models import Schema, Table
+
+
+@dataclass
+class ValidationIssue:
+    schema: str
+    table: str | None
+    message: str
+
+    def __str__(self) -> str:
+        location = f"{self.schema}.{self.table}" if self.table else self.schema
+        return f"{location}: {self.message}"
+
+
+@dataclass
+class ValidationResult:
+    issues: list[ValidationIssue] = field(default_factory=list)
+
+    @property
+    def has_errors(self) -> bool:
+        return bool(self.issues)
+
+
+def _validate_table(
+    schema_name: str, table: Table, all_schemas: dict[str, Schema]
+) -> list[ValidationIssue]:
+    issues: list[ValidationIssue] = []
+    col_names = [c.name for c in table.columns]
+    col_set = set(col_names)
+
+    # Duplicate column names
+    seen: set[str] = set()
+    for name in col_names:
+        if name in seen:
+            issues.append(
+                ValidationIssue(
+                    schema=schema_name,
+                    table=table.name,
+                    message=f"duplicate column name: '{name}'",
+                )
+            )
+        seen.add(name)
+
+    # PK columns exist in table
+    if table.primary_key:
+        for col in table.primary_key.columns:
+            if col not in col_set:
+                issues.append(
+                    ValidationIssue(
+                        schema=schema_name,
+                        table=table.name,
+                        message=f"primary key references unknown column: '{col}'",
+                    )
+                )
+
+    # FK checks
+    for fk in table.foreign_keys:
+        # FK source columns exist in this table
+        for col in fk.columns:
+            if col not in col_set:
+                issues.append(
+                    ValidationIssue(
+                        schema=schema_name,
+                        table=table.name,
+                        message=f"foreign key references unknown source column: '{col}'",
+                    )
+                )
+
+        # FK ref_schema + ref_table exist in loaded schemas
+        ref_schema = all_schemas.get(fk.ref_schema)
+        if ref_schema is None:
+            issues.append(
+                ValidationIssue(
+                    schema=schema_name,
+                    table=table.name,
+                    message=f"foreign key references unknown schema: '{fk.ref_schema}'",
+                )
+            )
+        else:
+            ref_table_map = {t.name: t for t in ref_schema.tables}
+            ref_table = ref_table_map.get(fk.ref_table)
+            if ref_table is None:
+                issues.append(
+                    ValidationIssue(
+                        schema=schema_name,
+                        table=table.name,
+                        message=(
+                            f"foreign key references unknown table:"
+                            f" '{fk.ref_schema}.{fk.ref_table}'"
+                        ),
+                    )
+                )
+            else:
+                # FK ref_columns exist in referenced table
+                ref_col_set = {c.name for c in ref_table.columns}
+                for col in fk.ref_columns:
+                    if col not in ref_col_set:
+                        issues.append(
+                            ValidationIssue(
+                                schema=schema_name,
+                                table=table.name,
+                                message=(
+                                    f"foreign key references unknown column '{col}'"
+                                    f" in '{fk.ref_schema}.{fk.ref_table}'"
+                                ),
+                            )
+                        )
+
+    return issues
+
+
+def validate_schemas(schemas: dict[str, Schema]) -> ValidationResult:
+    """Validate a collection of schemas for structural integrity.
+
+    Checks performed (no SDK or I/O required):
+    - Duplicate column names within a table
+    - PK columns that reference columns not present in the table
+    - FK source columns that reference columns not present in the table
+    - FK ref_schema + ref_table combinations not found in the loaded schemas
+    - FK ref_columns not found in the referenced table
+
+    Args:
+        schemas: mapping of schema name → Schema, as loaded from a directory.
+
+    Returns:
+        ValidationResult with a list of ValidationIssue for every problem found.
+    """
+    issues: list[ValidationIssue] = []
+    for schema_name, schema in schemas.items():
+        for table in schema.tables:
+            issues.extend(_validate_table(schema_name, table, schemas))
+    return ValidationResult(issues=issues)

databricks_schema-0.6.0/examples/01_extract_schema.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from databricks_schema import CatalogExtractor
+
+# Extract a single schema and inspect its contents.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMA = ["gold", "silver"]
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=SCHEMA)
+
+schema = catalog.schemas[0]
+print(f"Schema: {schema.name} ({len(schema.tables)} tables)")
+
+for table in schema.tables:
+    col_names = [c.name for c in table.columns]
+    print(f"  {table.name} [{table.table_type}]: {', '.join(col_names)}")

databricks_schema-0.6.0/examples/02_save_load_yaml.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from databricks_schema import CatalogExtractor, schema_from_yaml, schema_to_yaml
+
+# Extract schemas, save each to a YAML file, then reload and inspect the result.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMAS = ["gold", "silver"]
+OUTPUT_DIR = Path(".test_output/02_save_load_yaml")
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=SCHEMAS)
+
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+for schema in catalog.schemas:
+    path = OUTPUT_DIR / f"{schema.name}.yaml"
+    path.write_text(schema_to_yaml(schema), encoding="utf-8")
+    print(f"Saved {path}")
+
+# Reload and inspect
+for schema in catalog.schemas:
+    path = OUTPUT_DIR / f"{schema.name}.yaml"
+    loaded = schema_from_yaml(path.read_text(encoding="utf-8"))
+    print(f"Reloaded '{loaded.name}': {len(loaded.tables)} tables")
+    for table in loaded.tables:
+        print(f"  {table.name} ({len(table.columns)} columns)")

databricks_schema-0.6.0/examples/03_diff_schemas.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from databricks_schema import CatalogExtractor, diff_catalog_with_dir, schema_to_yaml
+
+# Diff live catalog schemas against local YAML files.
+# Saves a baseline on first run, then diffs live state against it.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMAS = ["gold", "silver"]
+SCHEMA_DIR = Path(".test_output/03_diff_schemas")
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=SCHEMAS)
+
+# Save baseline if not already present
+SCHEMA_DIR.mkdir(parents=True, exist_ok=True)
+for schema in catalog.schemas:
+    path = SCHEMA_DIR / f"{schema.name}.yaml"
+    if not path.exists():
+        path.write_text(schema_to_yaml(schema), encoding="utf-8")
+        print(f"Saved baseline {path}")
+
+result = diff_catalog_with_dir(catalog, SCHEMA_DIR, schema_names=frozenset(SCHEMAS))
+
+if not result.has_changes:
+    print("No changes — live catalog matches local files.")
+else:
+    for schema_diff in result.schemas:
+        if not schema_diff.has_changes:
+            continue
+        print(f"\nSchema '{schema_diff.name}': {schema_diff.status}")
+        for fc in schema_diff.changes:
+            print(f"  field '{fc.field}': {fc.old!r} -> {fc.new!r}")
+        for table_diff in schema_diff.tables:
+            print(f"  table '{table_diff.name}': {table_diff.status}")
+            for fc in table_diff.changes:
+                print(f"    field '{fc.field}': {fc.old!r} -> {fc.new!r}")
+            for col_diff in table_diff.columns:
+                print(f"    column '{col_diff.name}': {col_diff.status}")
+                for fc in col_diff.changes:
+                    print(f"      {fc.field}: {fc.old!r} -> {fc.new!r}")

databricks_schema-0.6.0/examples/04_generate_sql.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from databricks_schema import (
+    CatalogExtractor,
+    diff_schemas,
+    schema_diff_to_sql,
+    schema_from_yaml,
+    schema_to_yaml,
+)
+
+# Generate SQL to reconcile a live schema with a stored YAML baseline.
+# Simulates drift by modifying the stored YAML, then generates SQL to fix it.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMA = "gold"
+OUTPUT_DIR = Path(".test_output/04_generate_sql")
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=[SCHEMA])
+live_schema = catalog.schemas[0]
+
+# Save current state as the baseline
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+baseline_path = OUTPUT_DIR / f"{SCHEMA}.yaml"
+baseline_path.write_text(schema_to_yaml(live_schema), encoding="utf-8")
+print(f"Saved baseline {baseline_path}")
+
+# Load stored schema (this is your "desired state")
+stored_schema = schema_from_yaml(baseline_path.read_text(encoding="utf-8"))
+
+# Diff live vs stored
+schema_diff = diff_schemas(live=live_schema, stored=stored_schema)
+
+if not schema_diff.has_changes:
+    print("No drift detected — nothing to generate.")
+else:
+    sql = schema_diff_to_sql(CATALOG, schema_diff, stored_schema, allow_drop=False)
+    print(sql)

{databricks_schema-0.5.0 → databricks_schema-0.6.0}/pyproject.toml

@@ -7,10 +7,21 @@ packages = ["databricks_schema"]
 
 [project]
 name = "databricks-schema"
-version = "0.5.0"
+version = "0.6.0"
 description = "Databricks Unity Catalog schema extractor"
 readme = "README.md"
 requires-python = ">=3.11"
+license = "Apache-2.0"
+authors = [{ name = "frizzleqq" }]
+keywords = ["databricks", "unity-catalog", "schema", "sql"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
 dependencies = [
     # >=0.64.0 as we use 'entity_tag_assignments'
     "databricks-sdk>=0.64.0",
@@ -18,6 +29,10 @@ dependencies = [
     "pyyaml>=6.0",
 ]
 
+[project.urls]
+Repository = "https://github.com/frizzleqq/databricks-schema"
+Issues = "https://github.com/frizzleqq/databricks-schema/issues"
+
 [project.scripts]
 databricks-schema = "databricks_schema.cli:main"
 

{databricks_schema-0.5.0 → databricks_schema-0.6.0}/tests/test_diff.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 from databricks_schema.diff import (
     diff_catalog_with_dir,
+    diff_schema_dirs,
     diff_schemas,
 )
 from databricks_schema.models import Catalog, Column, ForeignKey, PrimaryKey, Schema, Table
@@ -246,3 +247,84 @@ class TestDiffCatalogWithDir:
         result = diff_catalog_with_dir(catalog, tmp_path, fmt="json")
         assert not result.has_changes
         assert result.schemas[0].status == "unchanged"
+
+
+class TestDiffSchemaDirs:
+    def test_no_changes(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main", tables=[_table("users", columns=[_col("id")])])
+        (d1 / "main.yaml").write_text(schema_to_yaml(schema))
+        (d2 / "main.yaml").write_text(schema_to_yaml(schema))
+        result = diff_schema_dirs(d1, d2)
+        assert not result.has_changes
+        assert result.schemas[0].status == "unchanged"
+
+    def test_schema_added_in_dir2(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main")
+        (d1 / "main.yaml").write_text(schema_to_yaml(schema))
+        (d2 / "main.yaml").write_text(schema_to_yaml(schema))
+        extra = _schema("extra")
+        (d2 / "extra.yaml").write_text(schema_to_yaml(extra))
+        result = diff_schema_dirs(d1, d2)
+        added = next(s for s in result.schemas if s.name == "extra")
+        assert added.status == "added"
+
+    def test_schema_removed_from_dir2(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main")
+        (d1 / "main.yaml").write_text(schema_to_yaml(schema))
+        result = diff_schema_dirs(d1, d2)
+        assert result.schemas[0].name == "main"
+        assert result.schemas[0].status == "removed"
+
+    def test_field_modified(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        old = _schema("main", comment="old")
+        new = _schema("main", comment="new")
+        (d1 / "main.yaml").write_text(schema_to_yaml(old))
+        (d2 / "main.yaml").write_text(schema_to_yaml(new))
+        result = diff_schema_dirs(d1, d2)
+        s = result.schemas[0]
+        assert s.status == "modified"
+        assert s.changes[0].field == "comment"
+        assert s.changes[0].old == "old"
+        assert s.changes[0].new == "new"
+
+    def test_json_format(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main", tables=[_table("t", columns=[_col("id")])])
+        (d1 / "main.json").write_text(schema_to_json(schema))
+        (d2 / "main.json").write_text(schema_to_json(schema))
+        result = diff_schema_dirs(d1, d2, fmt1="json", fmt2="json")
+        assert not result.has_changes
+
+    def test_schema_names_filter(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        main = _schema("main")
+        raw_old = _schema("raw", comment="old")
+        raw_new = _schema("raw", comment="new")
+        (d1 / "main.yaml").write_text(schema_to_yaml(main))
+        (d1 / "raw.yaml").write_text(schema_to_yaml(raw_old))
+        (d2 / "main.yaml").write_text(schema_to_yaml(main))
+        (d2 / "raw.yaml").write_text(schema_to_yaml(raw_new))
+        result = diff_schema_dirs(d1, d2, schema_names=frozenset({"main"}))
+        assert not result.has_changes

databricks_schema-0.6.0/tests/test_validate.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from databricks_schema.models import Column, ForeignKey, PrimaryKey, Schema, Table
+from databricks_schema.validate import validate_schemas
+
+
+def _col(name: str, data_type: str = "STRING") -> Column:
+    return Column(name=name, data_type=data_type)
+
+
+def _table(name: str, columns: list[Column] | None = None, **kwargs) -> Table:
+    return Table(name=name, columns=columns or [], **kwargs)
+
+
+def _schema(name: str, tables: list[Table] | None = None) -> Schema:
+    return Schema(name=name, tables=tables or [])
+
+
+def _schemas(*schemas: Schema) -> dict[str, Schema]:
+    return {s.name: s for s in schemas}
+
+
+class TestValidateClean:
+    def test_empty_schemas(self):
+        result = validate_schemas({})
+        assert not result.has_errors
+        assert result.issues == []
+
+    def test_valid_schema_no_constraints(self):
+        schema = _schema("main", [_table("users", [_col("id"), _col("email")])])
+        result = validate_schemas(_schemas(schema))
+        assert not result.has_errors
+
+    def test_valid_pk(self):
+        table = _table("users", [_col("id")], primary_key=PrimaryKey(columns=["id"]))
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert not result.has_errors
+
+    def test_valid_fk_cross_schema(self):
+        orgs = _schema("orgs", [_table("org", [_col("id")])])
+        fk = ForeignKey(columns=["org_id"], ref_schema="orgs", ref_table="org", ref_columns=["id"])
+        users = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(orgs, users))
+        assert not result.has_errors
+
+
+class TestDuplicateColumns:
+    def test_duplicate_column_name(self):
+        table = _table("t", [_col("id"), _col("id")])
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        assert len(result.issues) == 1
+        assert "duplicate column name" in result.issues[0].message
+        assert "'id'" in result.issues[0].message
+
+    def test_duplicate_reported_once_per_name(self):
+        # three columns named "x" — only one issue (detected on second occurrence)
+        table = _table("t", [_col("x"), _col("x"), _col("x")])
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        # second and third "x" both trigger, so two issues
+        assert len(result.issues) == 2
+
+
+class TestPrimaryKeyValidation:
+    def test_pk_column_missing(self):
+        table = _table("t", [_col("id")], primary_key=PrimaryKey(columns=["nonexistent"]))
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        assert any(
+            "primary key" in i.message and "'nonexistent'" in i.message for i in result.issues
+        )
+
+    def test_pk_multiple_columns_one_missing(self):
+        table = _table("t", [_col("a"), _col("b")], primary_key=PrimaryKey(columns=["a", "c"]))
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        assert len(result.issues) == 1
+        assert "'c'" in result.issues[0].message
+
+
+class TestForeignKeyValidation:
+    def test_fk_source_column_missing(self):
+        fk = ForeignKey(columns=["bad_col"], ref_schema="other", ref_table="t", ref_columns=["id"])
+        other = _schema("other", [_table("t", [_col("id")])])
+        main = _schema("main", [_table("users", [_col("id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(other, main))
+        assert result.has_errors
+        assert any(
+            "unknown source column" in i.message and "'bad_col'" in i.message for i in result.issues
+        )
+
+    def test_fk_ref_schema_missing(self):
+        fk = ForeignKey(columns=["org_id"], ref_schema="ghost", ref_table="org", ref_columns=["id"])
+        main = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(main))
+        assert result.has_errors
+        assert any("unknown schema" in i.message and "'ghost'" in i.message for i in result.issues)
+
+    def test_fk_ref_table_missing(self):
+        fk = ForeignKey(
+            columns=["org_id"], ref_schema="other", ref_table="ghost", ref_columns=["id"]
+        )
+        other = _schema("other", [_table("real_table", [_col("id")])])
+        main = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(other, main))
+        assert result.has_errors
+        assert any(
+            "unknown table" in i.message and "other.ghost" in i.message for i in result.issues
+        )
+
+    def test_fk_ref_column_missing(self):
+        fk = ForeignKey(
+            columns=["org_id"], ref_schema="other", ref_table="org", ref_columns=["bad"]
+        )
+        other = _schema("other", [_table("org", [_col("id")])])
+        main = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(other, main))
+        assert result.has_errors
+        assert any("'bad'" in i.message and "other.org" in i.message for i in result.issues)
+
+    def test_multiple_fk_issues_reported(self):
+        fk = ForeignKey(
+            columns=["bad_src"],
+            ref_schema="ghost",
+            ref_table="t",
+            ref_columns=["id"],
+        )
+        main = _schema("main", [_table("t", [_col("id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(main))
+        # bad_src missing + ghost schema missing = 2 issues
+        assert len(result.issues) == 2
+
+
+class TestIssueStr:
+    def test_str_with_table(self):
+        from databricks_schema.validate import ValidationIssue
+
+        issue = ValidationIssue(schema="main", table="users", message="some problem")
+        assert str(issue) == "main.users: some problem"
+
+    def test_str_without_table(self):
+        from databricks_schema.validate import ValidationIssue
+
+        issue = ValidationIssue(schema="main", table=None, message="schema-level problem")
+        assert str(issue) == "main: schema-level problem"

{databricks_schema-0.5.0 → databricks_schema-0.6.0}/uv.lock

@@ -233,7 +233,7 @@ wheels = [
 
 [[package]]
 name = "databricks-schema"
-version = "0.5.0"
+version = "0.6.0"
 source = { editable = "." }
 dependencies = [
     { name = "databricks-sdk" },

databricks_schema-0.5.0/examples/full_example.py

@@ -1,49 +0,0 @@
-import sys
-from pathlib import Path
-
-from databricks_schema import (
-    CatalogExtractor,
-    catalog_to_yaml,
-    diff_catalog_with_dir,
-    diff_schemas,
-    schema_from_yaml,
-    schema_to_yaml,
-)
-
-# Configuration
-CATALOG_NAME = "lake_prod"
-SCHEMA_FILTER = ["gold", "silver"]
-SINGLE_SCHEMA = "gold"
-OUTPUT_DIR = Path(".test_output/example")
-
-# Extract using configured auth
-extractor = CatalogExtractor()
-catalog = extractor.extract_catalog(CATALOG_NAME, schema_filter=SCHEMA_FILTER)
-
-# Serialise to YAML String
-yaml_text = catalog_to_yaml(catalog)
-
-# Serialise to YAML file(s)
-OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
-for s in catalog.schemas:
-    out_file = OUTPUT_DIR / f"{s.name}.yaml"
-    out_file.write_text(schema_to_yaml(s), encoding="utf-8")
-    print(f"  Wrote {out_file}", file=sys.stderr)
-
-print(f"Done — {len(catalog.schemas)} schema(s) written to {OUTPUT_DIR}", file=sys.stderr)
-
-# Deserialise from YAML
-schema = schema_from_yaml(open(OUTPUT_DIR / f"{SINGLE_SCHEMA}.yaml").read())
-print(schema.tables[0].columns)
-
-# Compare live catalog against local YAML files
-result = diff_catalog_with_dir(catalog, OUTPUT_DIR)
-if result.has_changes:
-    for schema_diff in result.schemas:
-        print(schema_diff.name, schema_diff.status)
-
-# Compare two Schema objects directly
-stored = schema_from_yaml(open(OUTPUT_DIR / f"{SINGLE_SCHEMA}.yaml").read())
-live = extractor._extract_schema(CATALOG_NAME, schema)
-diff = diff_schemas(live=live, stored=stored)