databricks-schema 0.5.1__tar.gz → 0.6.1__tar.gz

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/.gitignore

@@ -10,4 +10,7 @@ wheels/
 .venv
 
 # Tests
-.test_output/
+.test_output/
+
+# Claude Code
+.claude/

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-schema
-Version: 0.5.1
+Version: 0.6.1
 Summary: Databricks Unity Catalog schema extractor
 Project-URL: Repository, https://github.com/frizzleqq/databricks-schema
 Project-URL: Issues, https://github.com/frizzleqq/databricks-schema/issues
@@ -103,7 +103,7 @@ uv sync --all-groups
 The tool uses the [Databricks SDK](https://github.com/databricks/databricks-sdk-py) for auth. Configure it via environment variables:
 
 ```bash
-export DATABRICKS_HOST=https://<workspace>.cloud.databricks.com
+export DATABRICKS_HOST=https://<databricks-url>
 export DATABRICKS_TOKEN=<your-personal-access-token>
 ```
 
@@ -235,6 +235,22 @@ Include additional metadata (`owner`, `storage_location`) in the comparison:
 databricks-schema generate-sql <catalog> ./schemas/ --include-metadata
 ```
 
+### `validate`
+
+Validate local schema files for structural integrity (no Databricks connection needed):
+
+```bash
+databricks-schema validate ./schemas/
+```
+
+### `diff-files`
+
+Compare two local directories of schema files (no Databricks connection needed):
+
+```bash
+databricks-schema diff-files ./schemas-prod/ ./schemas-test/
+```
+
 ### `list-catalogs`
 
 List all accessible catalogs:
@@ -253,6 +269,8 @@ databricks-schema list-schemas <catalog>
 
 ## Python Library Usage
 
+See also the [`examples/`](examples/) directory for standalone scripts.
+
 ```python
 from pathlib import Path
 from databricks_schema import CatalogExtractor, catalog_to_yaml, schema_from_yaml

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/README.md

@@ -81,7 +81,7 @@ uv sync --all-groups
 The tool uses the [Databricks SDK](https://github.com/databricks/databricks-sdk-py) for auth. Configure it via environment variables:
 
 ```bash
-export DATABRICKS_HOST=https://<workspace>.cloud.databricks.com
+export DATABRICKS_HOST=https://<databricks-url>
 export DATABRICKS_TOKEN=<your-personal-access-token>
 ```
 
@@ -213,6 +213,22 @@ Include additional metadata (`owner`, `storage_location`) in the comparison:
 databricks-schema generate-sql <catalog> ./schemas/ --include-metadata
 ```
 
+### `validate`
+
+Validate local schema files for structural integrity (no Databricks connection needed):
+
+```bash
+databricks-schema validate ./schemas/
+```
+
+### `diff-files`
+
+Compare two local directories of schema files (no Databricks connection needed):
+
+```bash
+databricks-schema diff-files ./schemas-prod/ ./schemas-test/
+```
+
 ### `list-catalogs`
 
 List all accessible catalogs:
@@ -231,6 +247,8 @@ databricks-schema list-schemas <catalog>
 
 ## Python Library Usage
 
+See also the [`examples/`](examples/) directory for standalone scripts.
+
 ```python
 from pathlib import Path
 from databricks_schema import CatalogExtractor, catalog_to_yaml, schema_from_yaml

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/databricks_schema/__init__.py

@@ -7,11 +7,13 @@ from databricks_schema.diff import (
     SchemaDiff,
     TableDiff,
     diff_catalog_with_dir,
+    diff_schema_dirs,
     diff_schemas,
 )
 from databricks_schema.extractor import CatalogExtractor
 from databricks_schema.models import Catalog, Column, ForeignKey, PrimaryKey, Schema, Table
 from databricks_schema.sql_gen import schema_diff_to_sql
+from databricks_schema.validate import ValidationIssue, ValidationResult, validate_schemas
 from databricks_schema.yaml_io import (
     catalog_from_json,
     catalog_from_yaml,
@@ -37,16 +39,19 @@ __all__ = [
     "Table",
     "TableDiff",
     "TableType",
+    "ValidationIssue",
+    "ValidationResult",
     "catalog_from_json",
     "catalog_from_yaml",
     "catalog_to_json",
     "catalog_to_yaml",
     "diff_catalog_with_dir",
+    "diff_schema_dirs",
     "diff_schemas",
-    "schema_from_json",
     "schema_diff_to_sql",
     "schema_from_json",
     "schema_from_yaml",
     "schema_to_json",
     "schema_to_yaml",
+    "validate_schemas",
 ]

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/databricks_schema/cli.py

@@ -7,11 +7,19 @@ import sys
 from pathlib import Path
 
 from databricks.sdk import WorkspaceClient
-
-from databricks_schema.diff import CatalogDiff, SchemaDiff, diff_catalog_with_dir, diff_schemas
+from databricks.sdk.errors import DatabricksError, NotFound, PermissionDenied, Unauthenticated
+
+from databricks_schema.diff import (
+    CatalogDiff,
+    SchemaDiff,
+    diff_catalog_with_dir,
+    diff_schema_dirs,
+    diff_schemas,
+)
 from databricks_schema.extractor import CatalogExtractor
 from databricks_schema.models import Schema
 from databricks_schema.sql_gen import schema_diff_to_sql
+from databricks_schema.validate import validate_schemas
 from databricks_schema.yaml_io import (
     schema_from_json,
     schema_from_yaml,
@@ -237,6 +245,83 @@ def _cmd_generate_sql(args: argparse.Namespace) -> None:
             print()
 
 
+def _detect_fmt(schema_dir: Path) -> str:
+    """Auto-detect YAML or JSON format from files in a directory. Exits 2 on mixed or empty."""
+    yaml_files = list(schema_dir.glob("*.yaml"))
+    json_files = list(schema_dir.glob("*.json"))
+    if yaml_files and json_files:
+        print(f"Error: mixed YAML and JSON files in {schema_dir}.", file=sys.stderr)
+        sys.exit(2)
+    elif not yaml_files and not json_files:
+        print(f"No YAML or JSON files found in {schema_dir}.", file=sys.stderr)
+        sys.exit(2)
+    return "json" if json_files else "yaml"
+
+
+def _cmd_validate(args: argparse.Namespace) -> None:
+    """Validate local schema files for structural integrity."""
+    schema_dir: Path = args.schema_dir
+    if not schema_dir.is_dir():
+        print(f"Error: {schema_dir} is not a directory.", file=sys.stderr)
+        sys.exit(2)
+
+    fmt = _detect_fmt(schema_dir)
+    ext = ".json" if fmt == "json" else ".yaml"
+    loader = schema_from_json if fmt == "json" else schema_from_yaml
+
+    schema_filter_set: frozenset[str] | None = frozenset(args.schema) if args.schema else None
+    schemas: dict[str, Schema] = {}
+    for schema_file in sorted(schema_dir.glob(f"*{ext}")):
+        if schema_filter_set is not None and schema_file.stem not in schema_filter_set:
+            continue
+        schema = loader(schema_file.read_text(encoding="utf-8"))
+        schemas[schema.name] = schema
+
+    if not schemas:
+        print("No schemas to validate.", file=sys.stderr)
+        sys.exit(2)
+
+    result = validate_schemas(schemas)
+    if not result.has_errors:
+        print(f"OK — {len(schemas)} schema(s) validated, no issues found.")
+        return
+
+    for issue in result.issues:
+        print(f"ERROR: {issue}")
+    print(f"\n{len(result.issues)} issue(s) found in {len(schemas)} schema(s).", file=sys.stderr)
+    sys.exit(1)
+
+
+def _cmd_diff_files(args: argparse.Namespace) -> None:
+    """Compare two local directories of schema files."""
+    dir1: Path = args.dir1
+    dir2: Path = args.dir2
+    for d in (dir1, dir2):
+        if not d.is_dir():
+            print(f"Error: {d} is not a directory.", file=sys.stderr)
+            sys.exit(2)
+
+    fmt1 = _detect_fmt(dir1)
+    fmt2 = _detect_fmt(dir2)
+
+    print(f"Comparing {dir1} against {dir2}...", file=sys.stderr)
+    result = diff_schema_dirs(
+        dir1,
+        dir2,
+        fmt1=fmt1,
+        fmt2=fmt2,
+        schema_names=frozenset(args.schema) if args.schema else None,
+        include_metadata=args.include_metadata,
+    )
+
+    if not result.has_changes:
+        print("No differences found.")
+        return
+
+    _print_diff(result)
+    sys.exit(1)
+
+
 def _cmd_list_catalogs(args: argparse.Namespace) -> None:
     """List all accessible catalogs."""
     client = _make_client(args.host, args.token)
@@ -396,6 +481,45 @@ def _build_parser() -> argparse.ArgumentParser:
     _add_connection_args(gen_sql_p)
     gen_sql_p.set_defaults(func=_cmd_generate_sql)
 
+    # validate
+    validate_p = subparsers.add_parser(
+        "validate",
+        help="Validate local schema files for structural integrity (no Databricks connection).",
+    )
+    validate_p.add_argument(
+        "schema_dir", type=Path, help="Directory containing per-schema YAML or JSON files"
+    )
+    validate_p.add_argument(
+        "--schema",
+        "-s",
+        action="append",
+        metavar="SCHEMA",
+        help="Schema filter (repeatable)",
+    )
+    validate_p.set_defaults(func=_cmd_validate)
+
+    # diff-files
+    diff_files_p = subparsers.add_parser(
+        "diff-files",
+        help="Compare two local directories of schema files (no Databricks connection needed).",
+    )
+    diff_files_p.add_argument("dir1", type=Path, help="Baseline directory (old/stored state)")
+    diff_files_p.add_argument("dir2", type=Path, help="Current directory (new/live state)")
+    diff_files_p.add_argument(
+        "--schema",
+        "-s",
+        action="append",
+        metavar="SCHEMA",
+        help="Schema filter (repeatable)",
+    )
+    diff_files_p.add_argument(
+        "--include-metadata",
+        action="store_true",
+        dest="include_metadata",
+        help="Include owner in comparison",
+    )
+    diff_files_p.set_defaults(func=_cmd_diff_files)
+
     # list-catalogs
     list_catalogs_p = subparsers.add_parser("list-catalogs", help="List all accessible catalogs.")
     _add_connection_args(list_catalogs_p)
@@ -416,4 +540,12 @@ def main() -> None:
         parser.print_help()
         sys.exit(0)
     args = parser.parse_args()
-    args.func(args)
+    try:
+        args.func(args)
+    except NotFound as e:
+        print(f"Error: {type(e).__name__}: {e}", file=sys.stderr)
+        print("Check that the catalog and schema names are correct.", file=sys.stderr)
+        sys.exit(2)
+    except (Unauthenticated, PermissionDenied, DatabricksError) as e:
+        print(f"Error: {type(e).__name__}: {e}", file=sys.stderr)
+        sys.exit(2)

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/databricks_schema/diff.py

@@ -128,6 +128,54 @@ def diff_schemas(live: Schema, stored: Schema, include_metadata: bool = False) -
     return SchemaDiff(name=live.name, status=status, changes=changes, tables=table_diffs)
 
 
+def diff_schema_dirs(
+    dir1: Path,
+    dir2: Path,
+    fmt1: Literal["yaml", "json"] = "yaml",
+    fmt2: Literal["yaml", "json"] = "yaml",
+    schema_names: frozenset[str] | None = None,
+    include_metadata: bool = False,
+) -> CatalogDiff:
+    """Compare two local directories of schema files.
+
+    dir1 is treated as the stored/baseline state (old).
+    dir2 is treated as the live/current state (new).
+
+    Schemas present in dir1 but not dir2 are reported as "removed".
+    Schemas present in dir2 but not dir1 are reported as "added".
+    Schemas present in both are compared with diff_schemas.
+
+    schema_names: if set, only files whose stem is in this set are loaded from either directory.
+    fmt1 / fmt2: file format for dir1 and dir2 respectively ("yaml" or "json").
+    """
+
+    def _load_dir(path: Path, fmt: Literal["yaml", "json"]) -> dict[str, Schema]:
+        ext = ".json" if fmt == "json" else ".yaml"
+        loader = schema_from_json if fmt == "json" else schema_from_yaml
+        schemas: dict[str, Schema] = {}
+        for f in sorted(path.glob(f"*{ext}")):
+            if schema_names is not None and f.stem not in schema_names:
+                continue
+            schema = loader(f.read_text(encoding="utf-8"))
+            schemas[schema.name] = schema
+        return schemas
+
+    stored = _load_dir(dir1, fmt1)
+    live = _load_dir(dir2, fmt2)
+
+    schema_diffs: list[SchemaDiff] = []
+    for name, stored_schema in stored.items():
+        if name not in live:
+            schema_diffs.append(SchemaDiff(name=name, status="removed"))
+        else:
+            schema_diffs.append(diff_schemas(live[name], stored_schema, include_metadata))
+    for name in live:
+        if name not in stored:
+            schema_diffs.append(SchemaDiff(name=name, status="added"))
+
+    return CatalogDiff(schemas=schema_diffs)
+
+
 def diff_catalog_with_dir(
     catalog: Catalog,
     schema_dir: Path,

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/databricks_schema/extractor.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import quote
 
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.errors import NotFound
@@ -22,7 +23,10 @@ class CatalogExtractor:
     def _fetch_tags(self, entity_type: str, entity_name: str) -> dict[str, str]:
         tags: dict[str, str] = {}
         try:
-            for assignment in self.client.entity_tag_assignments.list(entity_type, entity_name):
+            # Percent-encode entity_name so special characters (e.g. slashes in column names)
+            # don't break URL path routing. Dots are safe — they're the UC name separators.
+            encoded_name = quote(entity_name, safe=".")
+            for assignment in self.client.entity_tag_assignments.list(entity_type, encoded_name):
                 key = getattr(assignment, "tag_key", None)
                 value = getattr(assignment, "tag_value", None)
                 if key is not None:

databricks_schema-0.6.1/databricks_schema/validate.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from databricks_schema.models import Schema, Table
+
+
+@dataclass
+class ValidationIssue:
+    schema: str
+    table: str | None
+    message: str
+
+    def __str__(self) -> str:
+        location = f"{self.schema}.{self.table}" if self.table else self.schema
+        return f"{location}: {self.message}"
+
+
+@dataclass
+class ValidationResult:
+    issues: list[ValidationIssue] = field(default_factory=list)
+
+    @property
+    def has_errors(self) -> bool:
+        return bool(self.issues)
+
+
+def _validate_table(
+    schema_name: str, table: Table, all_schemas: dict[str, Schema]
+) -> list[ValidationIssue]:
+    issues: list[ValidationIssue] = []
+    col_names = [c.name for c in table.columns]
+    col_set = set(col_names)
+
+    # Duplicate column names
+    seen: set[str] = set()
+    for name in col_names:
+        if name in seen:
+            issues.append(
+                ValidationIssue(
+                    schema=schema_name,
+                    table=table.name,
+                    message=f"duplicate column name: '{name}'",
+                )
+            )
+        seen.add(name)
+
+    # PK columns exist in table
+    if table.primary_key:
+        for col in table.primary_key.columns:
+            if col not in col_set:
+                issues.append(
+                    ValidationIssue(
+                        schema=schema_name,
+                        table=table.name,
+                        message=f"primary key references unknown column: '{col}'",
+                    )
+                )
+
+    # FK checks
+    for fk in table.foreign_keys:
+        # FK source columns exist in this table
+        for col in fk.columns:
+            if col not in col_set:
+                issues.append(
+                    ValidationIssue(
+                        schema=schema_name,
+                        table=table.name,
+                        message=f"foreign key references unknown source column: '{col}'",
+                    )
+                )
+
+        # FK ref_schema + ref_table exist in loaded schemas
+        ref_schema = all_schemas.get(fk.ref_schema)
+        if ref_schema is None:
+            issues.append(
+                ValidationIssue(
+                    schema=schema_name,
+                    table=table.name,
+                    message=f"foreign key references unknown schema: '{fk.ref_schema}'",
+                )
+            )
+        else:
+            ref_table_map = {t.name: t for t in ref_schema.tables}
+            ref_table = ref_table_map.get(fk.ref_table)
+            if ref_table is None:
+                issues.append(
+                    ValidationIssue(
+                        schema=schema_name,
+                        table=table.name,
+                        message=(
+                            f"foreign key references unknown table:"
+                            f" '{fk.ref_schema}.{fk.ref_table}'"
+                        ),
+                    )
+                )
+            else:
+                # FK ref_columns exist in referenced table
+                ref_col_set = {c.name for c in ref_table.columns}
+                for col in fk.ref_columns:
+                    if col not in ref_col_set:
+                        issues.append(
+                            ValidationIssue(
+                                schema=schema_name,
+                                table=table.name,
+                                message=(
+                                    f"foreign key references unknown column '{col}'"
+                                    f" in '{fk.ref_schema}.{fk.ref_table}'"
+                                ),
+                            )
+                        )
+
+    return issues
+
+
+def validate_schemas(schemas: dict[str, Schema]) -> ValidationResult:
+    """Validate a collection of schemas for structural integrity.
+
+    Checks performed (no SDK or I/O required):
+    - Duplicate column names within a table
+    - PK columns that reference columns not present in the table
+    - FK source columns that reference columns not present in the table
+    - FK ref_schema + ref_table combinations not found in the loaded schemas
+    - FK ref_columns not found in the referenced table
+
+    Args:
+        schemas: mapping of schema name → Schema, as loaded from a directory.
+
+    Returns:
+        ValidationResult with a list of ValidationIssue for every problem found.
+    """
+    issues: list[ValidationIssue] = []
+    for schema_name, schema in schemas.items():
+        for table in schema.tables:
+            issues.extend(_validate_table(schema_name, table, schemas))
+    return ValidationResult(issues=issues)

databricks_schema-0.6.1/examples/01_extract_schema.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from databricks_schema import CatalogExtractor
+
+# Extract a single schema and inspect its contents.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMA = ["gold", "silver"]
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=SCHEMA)
+
+schema = catalog.schemas[0]
+print(f"Schema: {schema.name} ({len(schema.tables)} tables)")
+
+for table in schema.tables:
+    col_names = [c.name for c in table.columns]
+    print(f"  {table.name} [{table.table_type}]: {', '.join(col_names)}")

databricks_schema-0.6.1/examples/02_save_load_yaml.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from databricks_schema import CatalogExtractor, schema_from_yaml, schema_to_yaml
+
+# Extract schemas, save each to a YAML file, then reload and inspect the result.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMAS = ["gold", "silver"]
+OUTPUT_DIR = Path(".test_output/02_save_load_yaml")
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=SCHEMAS)
+
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+for schema in catalog.schemas:
+    path = OUTPUT_DIR / f"{schema.name}.yaml"
+    path.write_text(schema_to_yaml(schema), encoding="utf-8")
+    print(f"Saved {path}")
+
+# Reload and inspect
+for schema in catalog.schemas:
+    path = OUTPUT_DIR / f"{schema.name}.yaml"
+    loaded = schema_from_yaml(path.read_text(encoding="utf-8"))
+    print(f"Reloaded '{loaded.name}': {len(loaded.tables)} tables")
+    for table in loaded.tables:
+        print(f"  {table.name} ({len(table.columns)} columns)")

databricks_schema-0.6.1/examples/03_diff_schemas.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from databricks_schema import CatalogExtractor, diff_catalog_with_dir, schema_to_yaml
+
+# Diff live catalog schemas against local YAML files.
+# Saves a baseline on first run, then diffs live state against it.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMAS = ["gold", "silver"]
+SCHEMA_DIR = Path(".test_output/03_diff_schemas")
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=SCHEMAS)
+
+# Save baseline if not already present
+SCHEMA_DIR.mkdir(parents=True, exist_ok=True)
+for schema in catalog.schemas:
+    path = SCHEMA_DIR / f"{schema.name}.yaml"
+    if not path.exists():
+        path.write_text(schema_to_yaml(schema), encoding="utf-8")
+        print(f"Saved baseline {path}")
+
+result = diff_catalog_with_dir(catalog, SCHEMA_DIR, schema_names=frozenset(SCHEMAS))
+
+if not result.has_changes:
+    print("No changes — live catalog matches local files.")
+else:
+    for schema_diff in result.schemas:
+        if not schema_diff.has_changes:
+            continue
+        print(f"\nSchema '{schema_diff.name}': {schema_diff.status}")
+        for fc in schema_diff.changes:
+            print(f"  field '{fc.field}': {fc.old!r} -> {fc.new!r}")
+        for table_diff in schema_diff.tables:
+            print(f"  table '{table_diff.name}': {table_diff.status}")
+            for fc in table_diff.changes:
+                print(f"    field '{fc.field}': {fc.old!r} -> {fc.new!r}")
+            for col_diff in table_diff.columns:
+                print(f"    column '{col_diff.name}': {col_diff.status}")
+                for fc in col_diff.changes:
+                    print(f"      {fc.field}: {fc.old!r} -> {fc.new!r}")

databricks_schema-0.6.1/examples/04_generate_sql.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from databricks_schema import (
+    CatalogExtractor,
+    diff_schemas,
+    schema_diff_to_sql,
+    schema_from_yaml,
+    schema_to_yaml,
+)
+
+# Generate SQL to reconcile a live schema with a stored YAML baseline.
+# Simulates drift by modifying the stored YAML, then generates SQL to fix it.
+# Uses default Databricks auth (env vars, ~/.databrickscfg, etc.)
+
+CATALOG = "lake_prod"
+SCHEMA = "gold"
+OUTPUT_DIR = Path(".test_output/04_generate_sql")
+
+extractor = CatalogExtractor()
+catalog = extractor.extract_catalog(CATALOG, schema_filter=[SCHEMA])
+live_schema = catalog.schemas[0]
+
+# Save current state as the baseline
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+baseline_path = OUTPUT_DIR / f"{SCHEMA}.yaml"
+baseline_path.write_text(schema_to_yaml(live_schema), encoding="utf-8")
+print(f"Saved baseline {baseline_path}")
+
+# Load stored schema (this is your "desired state")
+stored_schema = schema_from_yaml(baseline_path.read_text(encoding="utf-8"))
+
+# Diff live vs stored
+schema_diff = diff_schemas(live=live_schema, stored=stored_schema)
+
+if not schema_diff.has_changes:
+    print("No drift detected — nothing to generate.")
+else:
+    sql = schema_diff_to_sql(CATALOG, schema_diff, stored_schema, allow_drop=False)
+    print(sql)

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["databricks_schema"]
 
 [project]
 name = "databricks-schema"
-version = "0.5.1"
+version = "0.6.1"
 description = "Databricks Unity Catalog schema extractor"
 readme = "README.md"
 requires-python = ">=3.11"

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/tests/test_diff.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 from databricks_schema.diff import (
     diff_catalog_with_dir,
+    diff_schema_dirs,
     diff_schemas,
 )
 from databricks_schema.models import Catalog, Column, ForeignKey, PrimaryKey, Schema, Table
@@ -246,3 +247,84 @@ class TestDiffCatalogWithDir:
         result = diff_catalog_with_dir(catalog, tmp_path, fmt="json")
         assert not result.has_changes
         assert result.schemas[0].status == "unchanged"
+
+
+class TestDiffSchemaDirs:
+    def test_no_changes(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main", tables=[_table("users", columns=[_col("id")])])
+        (d1 / "main.yaml").write_text(schema_to_yaml(schema))
+        (d2 / "main.yaml").write_text(schema_to_yaml(schema))
+        result = diff_schema_dirs(d1, d2)
+        assert not result.has_changes
+        assert result.schemas[0].status == "unchanged"
+
+    def test_schema_added_in_dir2(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main")
+        (d1 / "main.yaml").write_text(schema_to_yaml(schema))
+        (d2 / "main.yaml").write_text(schema_to_yaml(schema))
+        extra = _schema("extra")
+        (d2 / "extra.yaml").write_text(schema_to_yaml(extra))
+        result = diff_schema_dirs(d1, d2)
+        added = next(s for s in result.schemas if s.name == "extra")
+        assert added.status == "added"
+
+    def test_schema_removed_from_dir2(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main")
+        (d1 / "main.yaml").write_text(schema_to_yaml(schema))
+        result = diff_schema_dirs(d1, d2)
+        assert result.schemas[0].name == "main"
+        assert result.schemas[0].status == "removed"
+
+    def test_field_modified(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        old = _schema("main", comment="old")
+        new = _schema("main", comment="new")
+        (d1 / "main.yaml").write_text(schema_to_yaml(old))
+        (d2 / "main.yaml").write_text(schema_to_yaml(new))
+        result = diff_schema_dirs(d1, d2)
+        s = result.schemas[0]
+        assert s.status == "modified"
+        assert s.changes[0].field == "comment"
+        assert s.changes[0].old == "old"
+        assert s.changes[0].new == "new"
+
+    def test_json_format(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        schema = _schema("main", tables=[_table("t", columns=[_col("id")])])
+        (d1 / "main.json").write_text(schema_to_json(schema))
+        (d2 / "main.json").write_text(schema_to_json(schema))
+        result = diff_schema_dirs(d1, d2, fmt1="json", fmt2="json")
+        assert not result.has_changes
+
+    def test_schema_names_filter(self, tmp_path: Path):
+        d1 = tmp_path / "d1"
+        d2 = tmp_path / "d2"
+        d1.mkdir()
+        d2.mkdir()
+        main = _schema("main")
+        raw_old = _schema("raw", comment="old")
+        raw_new = _schema("raw", comment="new")
+        (d1 / "main.yaml").write_text(schema_to_yaml(main))
+        (d1 / "raw.yaml").write_text(schema_to_yaml(raw_old))
+        (d2 / "main.yaml").write_text(schema_to_yaml(main))
+        (d2 / "raw.yaml").write_text(schema_to_yaml(raw_new))
+        result = diff_schema_dirs(d1, d2, schema_names=frozenset({"main"}))
+        assert not result.has_changes

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/tests/test_extractor.py

@@ -398,6 +398,33 @@ class TestCatalogExtractor:
 
         client.tables.get.assert_not_called()
 
+    def test_column_tags_slash_in_name(self):
+        """Column names containing slashes must be percent-encoded before tag lookup."""
+        extractor, client = self._extractor()
+        sdk_catalog = MagicMock()
+        sdk_catalog.comment = None
+        client.catalogs.get.return_value = sdk_catalog
+
+        s = MagicMock()
+        s.name = "main"
+        s.comment = None
+        s.owner = None
+        client.schemas.list.return_value = [s]
+
+        col = _make_col("/example/column", position=1, type_text="STRING")
+        sdk_table = _make_sdk_table("orders", columns=[col])
+        client.tables.list.return_value = [sdk_table]
+
+        def mock_list(entity_type, entity_name):
+            if entity_type == "columns" and entity_name == "mycat.main.orders.%2Fexample%2Fcolumn":
+                return [_make_tag("sensitivity", "high")]
+            return []
+
+        client.entity_tag_assignments.list.side_effect = mock_list
+
+        catalog = extractor.extract_catalog("mycat")
+        assert catalog.schemas[0].tables[0].columns[0].tags == {"sensitivity": "high"}
+
     def test_parallel_extraction(self):
         """With max_workers > 1, all tables are still extracted correctly."""
         extractor, client = self._extractor(max_workers=4)

databricks_schema-0.6.1/tests/test_validate.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from databricks_schema.models import Column, ForeignKey, PrimaryKey, Schema, Table
+from databricks_schema.validate import validate_schemas
+
+
+def _col(name: str, data_type: str = "STRING") -> Column:
+    return Column(name=name, data_type=data_type)
+
+
+def _table(name: str, columns: list[Column] | None = None, **kwargs) -> Table:
+    return Table(name=name, columns=columns or [], **kwargs)
+
+
+def _schema(name: str, tables: list[Table] | None = None) -> Schema:
+    return Schema(name=name, tables=tables or [])
+
+
+def _schemas(*schemas: Schema) -> dict[str, Schema]:
+    return {s.name: s for s in schemas}
+
+
+class TestValidateClean:
+    def test_empty_schemas(self):
+        result = validate_schemas({})
+        assert not result.has_errors
+        assert result.issues == []
+
+    def test_valid_schema_no_constraints(self):
+        schema = _schema("main", [_table("users", [_col("id"), _col("email")])])
+        result = validate_schemas(_schemas(schema))
+        assert not result.has_errors
+
+    def test_valid_pk(self):
+        table = _table("users", [_col("id")], primary_key=PrimaryKey(columns=["id"]))
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert not result.has_errors
+
+    def test_valid_fk_cross_schema(self):
+        orgs = _schema("orgs", [_table("org", [_col("id")])])
+        fk = ForeignKey(columns=["org_id"], ref_schema="orgs", ref_table="org", ref_columns=["id"])
+        users = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(orgs, users))
+        assert not result.has_errors
+
+
+class TestDuplicateColumns:
+    def test_duplicate_column_name(self):
+        table = _table("t", [_col("id"), _col("id")])
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        assert len(result.issues) == 1
+        assert "duplicate column name" in result.issues[0].message
+        assert "'id'" in result.issues[0].message
+
+    def test_duplicate_reported_once_per_name(self):
+        # three columns named "x" — only one issue (detected on second occurrence)
+        table = _table("t", [_col("x"), _col("x"), _col("x")])
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        # second and third "x" both trigger, so two issues
+        assert len(result.issues) == 2
+
+
+class TestPrimaryKeyValidation:
+    def test_pk_column_missing(self):
+        table = _table("t", [_col("id")], primary_key=PrimaryKey(columns=["nonexistent"]))
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        assert any(
+            "primary key" in i.message and "'nonexistent'" in i.message for i in result.issues
+        )
+
+    def test_pk_multiple_columns_one_missing(self):
+        table = _table("t", [_col("a"), _col("b")], primary_key=PrimaryKey(columns=["a", "c"]))
+        result = validate_schemas(_schemas(_schema("main", [table])))
+        assert result.has_errors
+        assert len(result.issues) == 1
+        assert "'c'" in result.issues[0].message
+
+
+class TestForeignKeyValidation:
+    def test_fk_source_column_missing(self):
+        fk = ForeignKey(columns=["bad_col"], ref_schema="other", ref_table="t", ref_columns=["id"])
+        other = _schema("other", [_table("t", [_col("id")])])
+        main = _schema("main", [_table("users", [_col("id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(other, main))
+        assert result.has_errors
+        assert any(
+            "unknown source column" in i.message and "'bad_col'" in i.message for i in result.issues
+        )
+
+    def test_fk_ref_schema_missing(self):
+        fk = ForeignKey(columns=["org_id"], ref_schema="ghost", ref_table="org", ref_columns=["id"])
+        main = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(main))
+        assert result.has_errors
+        assert any("unknown schema" in i.message and "'ghost'" in i.message for i in result.issues)
+
+    def test_fk_ref_table_missing(self):
+        fk = ForeignKey(
+            columns=["org_id"], ref_schema="other", ref_table="ghost", ref_columns=["id"]
+        )
+        other = _schema("other", [_table("real_table", [_col("id")])])
+        main = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(other, main))
+        assert result.has_errors
+        assert any(
+            "unknown table" in i.message and "other.ghost" in i.message for i in result.issues
+        )
+
+    def test_fk_ref_column_missing(self):
+        fk = ForeignKey(
+            columns=["org_id"], ref_schema="other", ref_table="org", ref_columns=["bad"]
+        )
+        other = _schema("other", [_table("org", [_col("id")])])
+        main = _schema("main", [_table("users", [_col("id"), _col("org_id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(other, main))
+        assert result.has_errors
+        assert any("'bad'" in i.message and "other.org" in i.message for i in result.issues)
+
+    def test_multiple_fk_issues_reported(self):
+        fk = ForeignKey(
+            columns=["bad_src"],
+            ref_schema="ghost",
+            ref_table="t",
+            ref_columns=["id"],
+        )
+        main = _schema("main", [_table("t", [_col("id")], foreign_keys=[fk])])
+        result = validate_schemas(_schemas(main))
+        # bad_src missing + ghost schema missing = 2 issues
+        assert len(result.issues) == 2
+
+
+class TestIssueStr:
+    def test_str_with_table(self):
+        from databricks_schema.validate import ValidationIssue
+
+        issue = ValidationIssue(schema="main", table="users", message="some problem")
+        assert str(issue) == "main.users: some problem"
+
+    def test_str_without_table(self):
+        from databricks_schema.validate import ValidationIssue
+
+        issue = ValidationIssue(schema="main", table=None, message="schema-level problem")
+        assert str(issue) == "main: schema-level problem"

{databricks_schema-0.5.1 → databricks_schema-0.6.1}/uv.lock

@@ -233,7 +233,7 @@ wheels = [
 
 [[package]]
 name = "databricks-schema"
-version = "0.5.1"
+version = "0.6.1"
 source = { editable = "." }
 dependencies = [
     { name = "databricks-sdk" },

databricks_schema-0.5.1/examples/full_example.py

@@ -1,49 +0,0 @@
-import sys
-from pathlib import Path
-
-from databricks_schema import (
-    CatalogExtractor,
-    catalog_to_yaml,
-    diff_catalog_with_dir,
-    diff_schemas,
-    schema_from_yaml,
-    schema_to_yaml,
-)
-
-# Configuration
-CATALOG_NAME = "lake_prod"
-SCHEMA_FILTER = ["gold", "silver"]
-SINGLE_SCHEMA = "gold"
-OUTPUT_DIR = Path(".test_output/example")
-
-# Extract using configured auth
-extractor = CatalogExtractor()
-catalog = extractor.extract_catalog(CATALOG_NAME, schema_filter=SCHEMA_FILTER)
-
-# Serialise to YAML String
-yaml_text = catalog_to_yaml(catalog)
-
-# Serialise to YAML file(s)
-OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
-for s in catalog.schemas:
-    out_file = OUTPUT_DIR / f"{s.name}.yaml"
-    out_file.write_text(schema_to_yaml(s), encoding="utf-8")
-    print(f"  Wrote {out_file}", file=sys.stderr)
-
-print(f"Done — {len(catalog.schemas)} schema(s) written to {OUTPUT_DIR}", file=sys.stderr)
-
-# Deserialise from YAML
-schema = schema_from_yaml(open(OUTPUT_DIR / f"{SINGLE_SCHEMA}.yaml").read())
-print(schema.tables[0].columns)
-
-# Compare live catalog against local YAML files
-result = diff_catalog_with_dir(catalog, OUTPUT_DIR)
-if result.has_changes:
-    for schema_diff in result.schemas:
-        print(schema_diff.name, schema_diff.status)
-
-# Compare two Schema objects directly
-stored = schema_from_yaml(open(OUTPUT_DIR / f"{SINGLE_SCHEMA}.yaml").read())
-live = extractor._extract_schema(CATALOG_NAME, schema)
-diff = diff_schemas(live=live, stored=stored)