datacontract-cli 0.10.14__py3-none-any.whl → 0.10.16__py3-none-any.whl

datacontract/breaking/breaking.py

@@ -1,11 +1,223 @@
 from datacontract.breaking.breaking_rules import BreakingRules
 from datacontract.model.breaking_change import BreakingChange, Location, Severity
-from datacontract.model.data_contract_specification import Field, Model, Quality
+from datacontract.model.data_contract_specification import Contact, DeprecatedQuality, Field, Info, Model, Terms
+
+
+def info_breaking_changes(
+    old_info: Info,
+    new_info: Info,
+    new_path: str,
+    include_severities: [Severity],
+) -> list[BreakingChange]:
+    results = list[BreakingChange]()
+
+    composition = ["info"]
+
+    if old_info and new_info:
+        info_definition_fields = vars(new_info) | new_info.model_extra | old_info.model_extra
+
+        for info_definition_field in info_definition_fields.keys():
+            if info_definition_field == "contact":
+                continue
+
+            old_value = getattr(old_info, info_definition_field, None)
+            new_value = getattr(new_info, info_definition_field, None)
+
+            rule_name = None
+            description = None
+
+            if old_value is None and new_value is not None:
+                rule_name = f"info_{_camel_to_snake(info_definition_field)}_added"
+                description = f"added with value: `{new_value}`"
+
+            elif old_value is not None and new_value is None:
+                rule_name = f"info_{_camel_to_snake(info_definition_field)}_removed"
+                description = "removed info property"
+
+            elif old_value != new_value:
+                rule_name = f"info_{_camel_to_snake(info_definition_field)}_updated"
+                description = f"changed from `{old_value}` to `{new_value}`"
+
+            if rule_name is not None:
+                severity = _get_rule(rule_name)
+                if severity in include_severities:
+                    results.append(
+                        BreakingChange(
+                            description=description,
+                            check_name=rule_name,
+                            severity=severity,
+                            location=Location(path=new_path, composition=composition + [info_definition_field]),
+                        )
+                    )
+
+        results.extend(
+            contact_breaking_changes(
+                old_contact=getattr(old_info, "contact", None),
+                new_contact=getattr(new_info, "contact", None),
+                composition=composition + ["contact"],
+                new_path=new_path,
+                include_severities=include_severities,
+            )
+        )
+
+    return results
+
+
+def contact_breaking_changes(
+    old_contact: Contact,
+    new_contact: Contact,
+    composition: list[str],
+    new_path: str,
+    include_severities: [Severity],
+) -> list[BreakingChange]:
+    results = list[BreakingChange]()
+
+    if not old_contact and new_contact:
+        rule_name = "contact_added"
+        severity = _get_rule(rule_name)
+        description = "added contact"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+
+    elif old_contact and not new_contact:
+        rule_name = "contact_removed"
+        severity = _get_rule(rule_name)
+        description = "removed contact"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+
+    elif old_contact and new_contact:
+        contact_definition_fields = vars(new_contact) | new_contact.model_extra | old_contact.model_extra
+
+        for contact_definition_field in contact_definition_fields.keys():
+            old_value = getattr(old_contact, contact_definition_field, None)
+            new_value = getattr(new_contact, contact_definition_field, None)
+
+            rule_name = None
+            description = None
+
+            if old_value is None and new_value is not None:
+                rule_name = f"contact_{_camel_to_snake(contact_definition_field)}_added"
+                description = f"added with value: `{new_value}`"
+
+            elif old_value is not None and new_value is None:
+                rule_name = f"contact_{_camel_to_snake(contact_definition_field)}_removed"
+                description = "removed contact property"
+
+            elif old_value != new_value:
+                rule_name = f"contact_{_camel_to_snake(contact_definition_field)}_updated"
+                description = f"changed from `{old_value}` to `{new_value}`"
+
+            if rule_name is not None:
+                severity = _get_rule(rule_name)
+                if severity in include_severities:
+                    results.append(
+                        BreakingChange(
+                            description=description,
+                            check_name=rule_name,
+                            severity=severity,
+                            location=Location(path=new_path, composition=composition + [contact_definition_field]),
+                        )
+                    )
+
+    return results
+
+
+def terms_breaking_changes(
+    old_terms: Terms,
+    new_terms: Terms,
+    new_path: str,
+    include_severities: [Severity],
+) -> list[BreakingChange]:
+    results = list[BreakingChange]()
+
+    composition = ["terms"]
+
+    if not old_terms and new_terms:
+        rule_name = "terms_added"
+        severity = _get_rule(rule_name)
+        description = "added terms"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+    elif old_terms and not new_terms:
+        rule_name = "terms_removed"
+        severity = _get_rule(rule_name)
+        description = "removed terms"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+
+    if old_terms and new_terms:
+        terms_definition_fields = vars(new_terms) | new_terms.model_extra | old_terms.model_extra
+
+        for terms_definition_field in terms_definition_fields.keys():
+            old_value = getattr(old_terms, terms_definition_field, None)
+            new_value = getattr(new_terms, terms_definition_field, None)
+
+            rule_name = None
+            description = None
+
+            if old_value is None and new_value is not None:
+                rule_name = f"terms_{_camel_to_snake(terms_definition_field)}_added"
+                description = f"added with value: `{new_value}`"
+
+            elif old_value is not None and new_value is None:
+                rule_name = f"terms_{_camel_to_snake(terms_definition_field)}_removed"
+                description = "removed info property"
+
+            elif old_value != new_value:
+                rule_name = f"terms_{_camel_to_snake(terms_definition_field)}_updated"
+                description = f"changed from `{old_value}` to `{new_value}`"
+
+            if rule_name is not None:
+                severity = _get_rule(rule_name)
+                if severity in include_severities:
+                    results.append(
+                        BreakingChange(
+                            description=description,
+                            check_name=rule_name,
+                            severity=severity,
+                            location=Location(path=new_path, composition=composition + [terms_definition_field]),
+                        )
+                    )
+
+    return results
 
 
 def quality_breaking_changes(
-    old_quality: Quality,
-    new_quality: Quality,
+    old_quality: DeprecatedQuality,
+    new_quality: DeprecatedQuality,
     new_path: str,
     include_severities: [Severity],
 ) -> list[BreakingChange]:
@@ -129,14 +341,14 @@ def model_breaking_changes(
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
-    model_definition_fields = vars(new_model)
+    model_definition_fields = vars(new_model) | new_model.model_extra | old_model.model_extra
 
     for model_definition_field in model_definition_fields.keys():
         if model_definition_field == "fields":
             continue
 
-        old_value = getattr(old_model, model_definition_field)
-        new_value = getattr(new_model, model_definition_field)
+        old_value = getattr(old_model, model_definition_field, None)
+        new_value = getattr(new_model, model_definition_field, None)
 
         rule_name = None
         description = None
@@ -237,13 +449,13 @@ def field_breaking_changes(
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
-    field_definition_fields = vars(new_field)
+    field_definition_fields = vars(new_field) | new_field.model_extra | old_field.model_extra
     for field_definition_field in field_definition_fields.keys():
         if field_definition_field == "ref_obj":
             continue
 
-        old_value = getattr(old_field, field_definition_field)
-        new_value = getattr(new_field, field_definition_field)
+        old_value = getattr(old_field, field_definition_field, None)
+        new_value = getattr(new_field, field_definition_field, None)
 
         if field_definition_field == "fields":
             results.extend(
@@ -320,9 +532,15 @@ def _get_rule(rule_name) -> Severity:
     try:
         return getattr(BreakingRules, rule_name)
     except AttributeError:
-        print(f"WARNING: Breaking Rule not found for {rule_name}!")
-        return Severity.ERROR
+        try:
+            first, *_, last = rule_name.split("_")
+            short_rule = "__".join([first, last])
+            return getattr(BreakingRules, short_rule)
+        except AttributeError:
+            print(f"WARNING: Breaking Rule not found for {rule_name}!")
+            return Severity.ERROR
 
 
 def _camel_to_snake(s):
+    s = s.replace("-", "_")
     return "".join(["_" + c.lower() if c.isupper() else c for c in s]).lstrip("_")

datacontract/breaking/breaking_rules.py

@@ -12,6 +12,10 @@ class BreakingRules:
 
     model_type_updated = Severity.ERROR
 
+    model__removed = Severity.INFO  # To support model extension keys
+    model__added = Severity.INFO
+    model__updated = Severity.INFO
+
     # field rules
     field_added = Severity.INFO
     field_removed = Severity.ERROR
@@ -34,6 +38,8 @@ class BreakingRules:
 
     field_required_updated = Severity.ERROR
 
+    field_primary_added = Severity.WARNING
+    field_primary_removed = Severity.WARNING
     field_primary_updated = Severity.WARNING
 
     field_references_added = Severity.WARNING
@@ -94,9 +100,27 @@ class BreakingRules:
     field_example_updated = Severity.INFO
     field_example_removed = Severity.INFO
 
+    field__removed = Severity.INFO  # To support field extension keys
+    field__added = Severity.INFO
+    field__updated = Severity.INFO
+
     # quality Rules
     quality_added = Severity.INFO
     quality_removed = Severity.WARNING
 
     quality_type_updated = Severity.WARNING
     quality_specification_updated = Severity.WARNING
+
+    # info rules
+    info__added = Severity.INFO  # will match `info_<somekey>_added` etc
+    info__removed = Severity.INFO
+    info__updated = Severity.INFO
+
+    contact__added = Severity.INFO
+    contact__removed = Severity.INFO
+    contact__updated = Severity.INFO
+
+    # terms rules
+    terms__added = Severity.INFO
+    terms__removed = Severity.INFO
+    terms__updated = Severity.INFO

datacontract/catalog/catalog.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from pathlib import Path
 
 import pytz
-from jinja2 import PackageLoader, Environment, select_autoescape
+from jinja2 import Environment, PackageLoader, select_autoescape
 
 from datacontract.data_contract import DataContract
 from datacontract.export.html_export import get_version

datacontract/cli.py

@@ -1,7 +1,6 @@
 from importlib import metadata
 from pathlib import Path
-from typing import Iterable, Optional
-from typing import List
+from typing import Iterable, List, Optional
 
 import typer
 import uvicorn
@@ -13,11 +12,16 @@ from typer.core import TyperGroup
 from typing_extensions import Annotated
 
 from datacontract import web
-from datacontract.catalog.catalog import create_index_html, create_data_contract_html
+from datacontract.catalog.catalog import create_data_contract_html, create_index_html
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.imports.importer import ImportFormat
-from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
-from datacontract.integration.datamesh_manager import publish_data_contract_to_datamesh_manager
+from datacontract.init.download_datacontract_file import (
+    FileExistsException,
+    download_datacontract_file,
+)
+from datacontract.integration.datamesh_manager import (
+    publish_data_contract_to_datamesh_manager,
+)
 
 DEFAULT_DATA_CONTRACT_SCHEMA_URL = "https://datacontract.com/datacontract.schema.json"
 
@@ -46,7 +50,11 @@ def version_callback(value: bool):
 def common(
     ctx: typer.Context,
     version: bool = typer.Option(
-        None, "--version", help="Prints the current version.", callback=version_callback, is_eager=True
+        None,
+        "--version",
+        help="Prints the current version.",
+        callback=version_callback,
+        is_eager=True,
     ),
 ):
     """
@@ -62,7 +70,8 @@ def common(
 @app.command()
 def init(
     location: Annotated[
-        str, typer.Argument(help="The location (url or path) of the data contract yaml to create.")
+        str,
+        typer.Argument(help="The location (url or path) of the data contract yaml to create."),
     ] = "datacontract.yaml",
     template: Annotated[
         str, typer.Option(help="URL of a template or data contract")
@@ -84,10 +93,12 @@ def init(
 @app.command()
 def lint(
     location: Annotated[
-        str, typer.Argument(help="The location (url or path) of the data contract yaml.")
+        str,
+        typer.Argument(help="The location (url or path) of the data contract yaml."),
     ] = "datacontract.yaml",
     schema: Annotated[
-        str, typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema")
+        str,
+        typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
     ] = DEFAULT_DATA_CONTRACT_SCHEMA_URL,
 ):
     """
@@ -100,10 +111,12 @@ def lint(
 @app.command()
 def test(
     location: Annotated[
-        str, typer.Argument(help="The location (url or path) of the data contract yaml.")
+        str,
+        typer.Argument(help="The location (url or path) of the data contract yaml."),
     ] = "datacontract.yaml",
     schema: Annotated[
-        str, typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema")
+        str,
+        typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
     ] = DEFAULT_DATA_CONTRACT_SCHEMA_URL,
     server: Annotated[
         str,
@@ -115,7 +128,8 @@ def test(
         ),
     ] = "all",
     examples: Annotated[
-        bool, typer.Option(help="Run the schema and quality tests on the example data within the data contract.")
+        bool,
+        typer.Option(help="Run the schema and quality tests on the example data within the data contract."),
     ] = None,
     publish: Annotated[str, typer.Option(help="The url to publish the results after the test")] = None,
     publish_to_opentelemetry: Annotated[
@@ -166,7 +180,10 @@ def export(
     # TODO: this should be a subcommand
     rdf_base: Annotated[
         Optional[str],
-        typer.Option(help="[rdf] The base URI used to generate the RDF graph.", rich_help_panel="RDF Options"),
+        typer.Option(
+            help="[rdf] The base URI used to generate the RDF graph.",
+            rich_help_panel="RDF Options",
+        ),
     ] = None,
     # TODO: this should be a subcommand
     sql_server_type: Annotated[
@@ -177,14 +194,21 @@ def export(
         ),
     ] = "auto",
     location: Annotated[
-        str, typer.Argument(help="The location (url or path) of the data contract yaml.")
+        str,
+        typer.Argument(help="The location (url or path) of the data contract yaml."),
     ] = "datacontract.yaml",
     schema: Annotated[
-        str, typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema")
+        str,
+        typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
     ] = DEFAULT_DATA_CONTRACT_SCHEMA_URL,
+    # TODO: this should be a subcommand
+    engine: Annotated[
+        Optional[str],
+        typer.Option(help="[engine] The engine used for great expection run."),
+    ] = None,
 ):
     """
-    Convert data contract to a specific format. console.prints to stdout.
+    Convert data contract to a specific format. Saves to file specified by `output` option if present, otherwise prints to stdout.
     """
     # TODO exception handling
     result = DataContract(data_contract_file=location, schema_location=schema, server=server).export(
@@ -193,10 +217,11 @@ def export(
         server=server,
         rdf_base=rdf_base,
         sql_server_type=sql_server_type,
+        engine=engine,
     )
     # Don't interpret console markup in output.
     if output is None:
-        console.print(result, markup=False)
+        console.print(result, markup=False, soft_wrap=True)
     else:
         with output.open("w") as f:
             f.write(result)
@@ -206,8 +231,15 @@ def export(
 @app.command(name="import")
 def import_(
     format: Annotated[ImportFormat, typer.Option(help="The format of the source file.")],
+    output: Annotated[
+        Path,
+        typer.Option(
+            help="Specify the file path where the Data Contract will be saved. If no path is provided, the output will be printed to stdout."
+        ),
+    ] = None,
     source: Annotated[
-        Optional[str], typer.Option(help="The path to the file or Glue Database that should be imported.")
+        Optional[str],
+        typer.Option(help="The path to the file or Glue Database that should be imported."),
     ] = None,
     glue_table: Annotated[
         Optional[List[str]],
@@ -250,7 +282,7 @@ def import_(
     ] = None,
 ):
     """
-    Create a data contract from the given source location. Prints to stdout.
+    Create a data contract from the given source location. Saves to file specified by `output` option if present, otherwise prints to stdout.
     """
     result = DataContract().import_from_source(
         format=format,
@@ -265,16 +297,23 @@ def import_(
         dbml_table=dbml_table,
         iceberg_table=iceberg_table,
     )
-    console.print(result.to_yaml())
+    if output is None:
+        console.print(result.to_yaml(), markup=False, soft_wrap=True)
+    else:
+        with output.open("w") as f:
+            f.write(result.to_yaml())
+        console.print(f"Written result to {output}")
 
 
 @app.command(name="publish")
 def publish(
     location: Annotated[
-        str, typer.Argument(help="The location (url or path) of the data contract yaml.")
+        str,
+        typer.Argument(help="The location (url or path) of the data contract yaml."),
     ] = "datacontract.yaml",
     schema: Annotated[
-        str, typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema")
+        str,
+        typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
     ] = DEFAULT_DATA_CONTRACT_SCHEMA_URL,
 ):
     """
@@ -290,11 +329,15 @@ def publish(
 @app.command(name="catalog")
 def catalog(
     files: Annotated[
-        Optional[str], typer.Option(help="Glob pattern for the data contract files to include in the catalog.")
+        Optional[str],
+        typer.Option(
+            help="Glob pattern for the data contract files to include in the catalog. Applies recursively to any subfolders."
+        ),
     ] = "*.yaml",
     output: Annotated[Optional[str], typer.Option(help="Output directory for the catalog html files.")] = "catalog/",
     schema: Annotated[
-        str, typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema")
+        str,
+        typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
     ] = DEFAULT_DATA_CONTRACT_SCHEMA_URL,
 ):
     """
@@ -305,7 +348,7 @@ def catalog(
     console.print(f"Created {output}")
 
     contracts = []
-    for file in Path().glob(files):
+    for file in Path().rglob(files):
         try:
             create_data_contract_html(contracts, file, path, schema)
         except Exception as e:
@@ -316,8 +359,14 @@ def catalog(
 
 @app.command()
 def breaking(
-    location_old: Annotated[str, typer.Argument(help="The location (url or path) of the old data contract yaml.")],
-    location_new: Annotated[str, typer.Argument(help="The location (url or path) of the new data contract yaml.")],
+    location_old: Annotated[
+        str,
+        typer.Argument(help="The location (url or path) of the old data contract yaml."),
+    ],
+    location_new: Annotated[
+        str,
+        typer.Argument(help="The location (url or path) of the new data contract yaml."),
+    ],
 ):
     """
     Identifies breaking changes between data contracts. Prints to stdout.
@@ -336,8 +385,14 @@ def breaking(
 
 @app.command()
 def changelog(
-    location_old: Annotated[str, typer.Argument(help="The location (url or path) of the old data contract yaml.")],
-    location_new: Annotated[str, typer.Argument(help="The location (url or path) of the new data contract yaml.")],
+    location_old: Annotated[
+        str,
+        typer.Argument(help="The location (url or path) of the old data contract yaml."),
+    ],
+    location_new: Annotated[
+        str,
+        typer.Argument(help="The location (url or path) of the new data contract yaml."),
+    ],
 ):
     """
     Generate a changelog between data contracts. Prints to stdout.
@@ -353,8 +408,14 @@ def changelog(
 
 @app.command()
 def diff(
-    location_old: Annotated[str, typer.Argument(help="The location (url or path) of the old data contract yaml.")],
-    location_new: Annotated[str, typer.Argument(help="The location (url or path) of the new data contract yaml.")],
+    location_old: Annotated[
+        str,
+        typer.Argument(help="The location (url or path) of the old data contract yaml."),
+    ],
+    location_new: Annotated[
+        str,
+        typer.Argument(help="The location (url or path) of the new data contract yaml."),
+    ],
 ):
     """
     PLACEHOLDER. Currently works as 'changelog' does.
@@ -391,7 +452,13 @@ def _handle_result(run):
         i = 1
         for check in run.checks:
             if check.result != "passed":
-                console.print(str(++i) + ") " + check.reason)
+                field = to_field(run, check)
+                if field:
+                    field = field + " "
+                else:
+                    field = ""
+                console.print(f"{i}) {field}{check.name}: {check.reason}")
+                i += 1
         raise typer.Exit(code=1)
 
 
@@ -401,7 +468,7 @@ def _print_table(run):
     table.add_column("Check", max_width=100)
     table.add_column("Field", max_width=32)
     table.add_column("Details", max_width=50)
-    for check in run.checks:
+    for check in sorted(run.checks, key=lambda c: (c.result or "", c.model or "", c.field or "")):
         table.add_row(with_markup(check.result), check.name, to_field(run, check), check.reason)
     console.print(table)
 

datacontract/data_contract.py

@@ -8,7 +8,12 @@ import yaml
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
-from datacontract.breaking.breaking import models_breaking_changes, quality_breaking_changes
+from datacontract.breaking.breaking import (
+    info_breaking_changes,
+    models_breaking_changes,
+    quality_breaking_changes,
+    terms_breaking_changes,
+)
 from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
     check_that_datacontract_contains_valid_server_configuration,
 )
@@ -17,7 +22,6 @@ from datacontract.engines.soda.check_soda_execute import check_soda_execute
 from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
-
 from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
 from datacontract.integration.opentelemetry import publish_test_results_to_opentelemetry
 from datacontract.lint import resolve
@@ -28,10 +32,10 @@ from datacontract.lint.linters.field_reference_linter import FieldReferenceLinte
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
 from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
-from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Severity
+from datacontract.model.breaking_change import BreakingChange, BreakingChanges, Severity
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
 
 class DataContract:
@@ -276,6 +280,24 @@ class DataContract:
 
         breaking_changes = list[BreakingChange]()
 
+        breaking_changes.extend(
+            info_breaking_changes(
+                old_info=old.info,
+                new_info=new.info,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
+
+        breaking_changes.extend(
+            terms_breaking_changes(
+                old_terms=old.terms,
+                new_terms=new.terms,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
+
         breaking_changes.extend(
             quality_breaking_changes(
                 old_quality=old.quality,