datacontract-cli 0.10.27__py3-none-any.whl → 0.10.28__py3-none-any.whl

datacontract/cli.py

@@ -11,7 +11,7 @@ from typing_extensions import Annotated
 
 from datacontract.catalog.catalog import create_data_contract_html, create_index_html
 from datacontract.data_contract import DataContract, ExportFormat
-from datacontract.imports.importer import ImportFormat
+from datacontract.imports.importer import ImportFormat, Spec
 from datacontract.init.init_template import get_init_template
 from datacontract.integration.datamesh_manager import (
     publish_data_contract_to_datamesh_manager,
@@ -126,7 +126,8 @@ def test(
             "servers (default)."
         ),
     ] = "all",
-    publish: Annotated[str, typer.Option(help="The url to publish the results after the test")] = None,
+    publish_test_results: Annotated[bool, typer.Option(help="Publish the results after the test")] = False,
+    publish: Annotated[str, typer.Option(help="DEPRECATED. The url to publish the results after the test.")] = None,
     output: Annotated[
         Path,
         typer.Option(
@@ -149,6 +150,7 @@ def test(
     run = DataContract(
         data_contract_file=location,
         schema_location=schema,
+        publish_test_results=publish_test_results,
         publish_url=publish,
         server=server,
         ssl_verification=ssl_verification,
@@ -246,6 +248,10 @@ def import_(
         Optional[str],
         typer.Option(help="The path to the file that should be imported."),
     ] = None,
+    spec: Annotated[
+        Spec,
+        typer.Option(help="The format of the data contract to import. "),
+    ] = Spec.datacontract_specification,
     dialect: Annotated[
         Optional[str],
         typer.Option(help="The SQL dialect to use when importing SQL files, e.g., postgres, tsql, bigquery."),
@@ -265,7 +271,7 @@ def import_(
         ),
     ] = None,
     unity_table_full_name: Annotated[
-        Optional[str], typer.Option(help="Full name of a table in the unity catalog")
+        Optional[List[str]], typer.Option(help="Full name of a table in the unity catalog")
     ] = None,
     dbt_model: Annotated[
         Optional[List[str]],
@@ -312,6 +318,7 @@ def import_(
     result = DataContract().import_from_source(
         format=format,
         source=source,
+        spec=spec,
         template=template,
         schema=schema,
         dialect=dialect,

datacontract/data_contract.py

@@ -1,6 +1,12 @@
 import logging
 import typing
 
+from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard
+
+from datacontract.export.odcs_v3_exporter import to_odcs_v3
+from datacontract.imports.importer import Spec
+from datacontract.imports.odcs_v3_importer import import_from_odcs
+
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
@@ -44,6 +50,7 @@ class DataContract:
         inline_definitions: bool = True,
         inline_quality: bool = True,
         ssl_verification: bool = True,
+        publish_test_results: bool = False,
     ):
         self._data_contract_file = data_contract_file
         self._data_contract_str = data_contract_str
@@ -51,6 +58,7 @@ class DataContract:
         self._schema_location = schema_location
         self._server = server
         self._publish_url = publish_url
+        self._publish_test_results = publish_test_results
         self._spark = spark
         self._duckdb_connection = duckdb_connection
         self._inline_definitions = inline_definitions
@@ -178,7 +186,7 @@ class DataContract:
 
         run.finish()
 
-        if self._publish_url is not None:
+        if self._publish_url is not None or self._publish_test_results:
             publish_test_results_to_datamesh_manager(run, self._publish_url, self._ssl_verification)
 
         return run
@@ -243,43 +251,128 @@ class DataContract:
         )
 
     def export(self, export_format: ExportFormat, model: str = "all", sql_server_type: str = "auto", **kwargs) -> str:
-        data_contract = resolve.resolve_data_contract(
-            self._data_contract_file,
-            self._data_contract_str,
-            self._data_contract,
-            schema_location=self._schema_location,
-            inline_definitions=self._inline_definitions,
-            inline_quality=self._inline_quality,
-        )
+        if export_format == ExportFormat.html or export_format == ExportFormat.mermaid:
+            data_contract = resolve.resolve_data_contract_v2(
+                self._data_contract_file,
+                self._data_contract_str,
+                self._data_contract,
+                schema_location=self._schema_location,
+                inline_definitions=self._inline_definitions,
+                inline_quality=self._inline_quality,
+            )
 
-        return exporter_factory.create(export_format).export(
-            data_contract=data_contract,
-            model=model,
-            server=self._server,
-            sql_server_type=sql_server_type,
-            export_args=kwargs,
-        )
+            return exporter_factory.create(export_format).export(
+                data_contract=data_contract,
+                model=model,
+                server=self._server,
+                sql_server_type=sql_server_type,
+                export_args=kwargs,
+            )
+        else:
+            data_contract = resolve.resolve_data_contract(
+                self._data_contract_file,
+                self._data_contract_str,
+                self._data_contract,
+                schema_location=self._schema_location,
+                inline_definitions=self._inline_definitions,
+                inline_quality=self._inline_quality,
+            )
+
+            return exporter_factory.create(export_format).export(
+                data_contract=data_contract,
+                model=model,
+                server=self._server,
+                sql_server_type=sql_server_type,
+                export_args=kwargs,
+            )
 
+    # REFACTOR THIS
+    # could be a class method, not using anything from the instance
     def import_from_source(
         self,
         format: str,
         source: typing.Optional[str] = None,
         template: typing.Optional[str] = None,
         schema: typing.Optional[str] = None,
+        spec: Spec = Spec.datacontract_specification,
         **kwargs,
-    ) -> DataContractSpecification:
-        data_contract_specification_initial = DataContract.init(template=template, schema=schema)
+    ) -> DataContractSpecification | OpenDataContractStandard:
+        id = kwargs.get("id")
+        owner = kwargs.get("owner")
 
-        imported_data_contract_specification = importer_factory.create(format).import_source(
-            data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
-        )
+        if spec == Spec.odcs:
+            data_contract_specification_initial = DataContract.init(template=template, schema=schema)
+
+            odcs_imported = importer_factory.create(format).import_source(
+                data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
+            )
+
+            if isinstance(odcs_imported, DataContractSpecification):
+                # convert automatically
+                odcs_imported = to_odcs_v3(odcs_imported)
+
+            self._overwrite_id_in_odcs(odcs_imported, id)
+            self._overwrite_owner_in_odcs(odcs_imported, owner)
+
+            return odcs_imported
+        elif spec == Spec.datacontract_specification:
+            data_contract_specification_initial = DataContract.init(template=template, schema=schema)
+
+            data_contract_specification_imported = importer_factory.create(format).import_source(
+                data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
+            )
+
+            if isinstance(data_contract_specification_imported, OpenDataContractStandard):
+                # convert automatically
+                data_contract_specification_imported = import_from_odcs(
+                    data_contract_specification_initial, data_contract_specification_imported
+                )
+
+            self._overwrite_id_in_data_contract_specification(data_contract_specification_imported, id)
+            self._overwrite_owner_in_data_contract_specification(data_contract_specification_imported, owner)
+
+            return data_contract_specification_imported
+        else:
+            raise DataContractException(
+                type="general",
+                result=ResultEnum.error,
+                name="Import Data Contract",
+                reason=f"Unsupported data contract format: {spec}",
+                engine="datacontract",
+            )
+
+    def _overwrite_id_in_data_contract_specification(
+        self, data_contract_specification: DataContractSpecification, id: str | None
+    ):
+        if not id:
+            return
+
+        data_contract_specification.id = id
+
+    def _overwrite_owner_in_data_contract_specification(
+        self, data_contract_specification: DataContractSpecification, owner: str | None
+    ):
+        if not owner:
+            return
+
+        if data_contract_specification.info is None:
+            data_contract_specification.info = Info()
+        data_contract_specification.info.owner = owner
+
+    def _overwrite_owner_in_odcs(self, odcs: OpenDataContractStandard, owner: str | None):
+        if not owner:
+            return
+
+        if odcs.customProperties is None:
+            odcs.customProperties = []
+        for customProperty in odcs.customProperties:
+            if customProperty.name == "owner":
+                customProperty.value = owner
+                return
+        odcs.customProperties.append(CustomProperty(property="owner", value=owner))
 
-        # Set id and owner if provided
-        if kwargs.get("id"):
-            data_contract_specification_initial.id = kwargs["id"]
-        if kwargs.get("owner"):
-            if data_contract_specification_initial.info is None:
-                data_contract_specification_initial.info = Info()
-            data_contract_specification_initial.info.owner = kwargs["owner"]
+    def _overwrite_id_in_odcs(self, odcs: OpenDataContractStandard, id: str | None):
+        if not id:
+            return
 
-        return imported_data_contract_specification
+        odcs.id = id

datacontract/engines/data_contract_checks.py

@@ -502,11 +502,13 @@ def prepare_query(quality: Quality, model_name: str, field_name: str = None) ->
     query = quality.query
 
     query = query.replace("{model}", model_name)
+    query = query.replace("{schema}", model_name)
     query = query.replace("{table}", model_name)
 
     if field_name is not None:
         query = query.replace("{field}", field_name)
         query = query.replace("{column}", field_name)
+        query = query.replace("{property}", field_name)
 
     return query
 

datacontract/export/html_exporter.py

@@ -6,8 +6,10 @@ import jinja_partials
 import pytz
 import yaml
 from jinja2 import Environment, PackageLoader, select_autoescape
+from open_data_contract_standard.model import OpenDataContractStandard
 
 from datacontract.export.exporter import Exporter
+from datacontract.export.mermaid_exporter import to_mermaid
 from datacontract.model.data_contract_specification import DataContractSpecification
 
 
@@ -16,9 +18,7 @@ class HtmlExporter(Exporter):
         return to_html(data_contract)
 
 
-def to_html(data_contract_spec: DataContractSpecification) -> str:
-    from datacontract.export.mermaid_exporter import to_mermaid
-
+def to_html(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str:
     # Load templates from templates folder
     package_loader = PackageLoader("datacontract", "templates")
     env = Environment(
@@ -33,28 +33,27 @@ def to_html(data_contract_spec: DataContractSpecification) -> str:
 
     # Load the required template
     # needs to be included in /MANIFEST.in
-    template = env.get_template("datacontract.html")
-
-    if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
-        quality_specification = data_contract_spec.quality.specification
-    elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
-        if data_contract_spec.quality.type == "great-expectations":
-            quality_specification = yaml.dump(
-                data_contract_spec.quality.specification, sort_keys=False, default_style="|"
-            )
-        else:
-            quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
-    else:
-        quality_specification = None
+    template_file = "datacontract.html"
+    if isinstance(data_contract_spec, OpenDataContractStandard):
+        template_file = "datacontract_odcs.html"
+
+    template = env.get_template(template_file)
 
     style_content, _, _ = package_loader.get_source(env, "style/output.css")
 
-    datacontract_yaml = data_contract_spec.to_yaml()
+    quality_specification = None
+    if isinstance(data_contract_spec, DataContractSpecification):
+        if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
+            quality_specification = data_contract_spec.quality.specification
+        elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
+            if data_contract_spec.quality.type == "great-expectations":
+                quality_specification = yaml.dump(
+                    data_contract_spec.quality.specification, sort_keys=False, default_style="|"
+                )
+            else:
+                quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
 
-    tz = pytz.timezone("UTC")
-    now = datetime.datetime.now(tz)
-    formatted_date = now.strftime("%d %b %Y %H:%M:%S UTC")
-    datacontract_cli_version = get_version()
+    datacontract_yaml = data_contract_spec.to_yaml()
 
     # Get the mermaid diagram
     mermaid_diagram = to_mermaid(data_contract_spec)
@@ -65,14 +64,20 @@ def to_html(data_contract_spec: DataContractSpecification) -> str:
         quality_specification=quality_specification,
         style=style_content,
         datacontract_yaml=datacontract_yaml,
-        formatted_date=formatted_date,
-        datacontract_cli_version=datacontract_cli_version,
+        formatted_date=_formatted_date(),
+        datacontract_cli_version=get_version(),
         mermaid_diagram=mermaid_diagram,
     )
 
     return html_string
 
 
+def _formatted_date() -> str:
+    tz = pytz.timezone("UTC")
+    now = datetime.datetime.now(tz)
+    return now.strftime("%d %b %Y %H:%M:%S UTC")
+
+
 def get_version() -> str:
     try:
         return version("datacontract_cli")

datacontract/export/mermaid_exporter.py

@@ -1,3 +1,5 @@
+from open_data_contract_standard.model import OpenDataContractStandard
+
 from datacontract.export.exporter import Exporter
 from datacontract.model.data_contract_specification import DataContractSpecification
 
@@ -7,26 +9,89 @@ class MermaidExporter(Exporter):
         return to_mermaid(data_contract)
 
 
-def to_mermaid(data_contract_spec: DataContractSpecification) -> str | None:
-    mmd_entity = "erDiagram\n\t"
-    mmd_references = []
+def to_mermaid(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str | None:
+    if isinstance(data_contract_spec, DataContractSpecification):
+        return dcs_to_mermaid(data_contract_spec)
+    elif isinstance(data_contract_spec, OpenDataContractStandard):
+        return odcs_to_mermaid(data_contract_spec)
+    else:
+        return None
+
+
+def dcs_to_mermaid(data_contract_spec: DataContractSpecification) -> str | None:
     try:
+        if not data_contract_spec.models:
+            return None
+
+        mmd_entity = "erDiagram\n"
+        mmd_references = []
+
         for model_name, model in data_contract_spec.models.items():
             entity_block = ""
+
             for field_name, field in model.fields.items():
-                entity_block += f"\t{field_name.replace('#', 'Nb').replace(' ', '_').replace('/', 'by')}{'🔑' if field.primaryKey or (field.unique and field.required) else ''}{'⌘' if field.references else ''} {field.type}\n"
+                clean_name = _sanitize_name(field_name)
+                indicators = ""
+
+                if field.primaryKey or (field.unique and field.required):
+                    indicators += "🔑"
+                if field.references:
+                    indicators += "⌘"
+
+                field_type = field.type or "unknown"
+                entity_block += f"\t{clean_name}{indicators} {field_type}\n"
+
                 if field.references:
-                    mmd_references.append(
-                        f'"📑{field.references.split(".")[0] if "." in field.references else ""}"'
-                        + "}o--{ ||"
-                        + f'"📑{model_name}"'
-                    )
+                    referenced_model = field.references.split(".")[0] if "." in field.references else ""
+                    if referenced_model:
+                        mmd_references.append(f'"📑{referenced_model}"' + "}o--{ ||" + f'"📑{model_name}"')
+
             mmd_entity += f'\t"**{model_name}**"' + "{\n" + entity_block + "}\n"
 
-        if mmd_entity == "":
+        if mmd_references:
+            mmd_entity += "\n" + "\n".join(mmd_references)
+
+        return f"{mmd_entity}\n"
+
+    except Exception as e:
+        print(f"Error generating DCS mermaid diagram: {e}")
+        return None
+
+
+def odcs_to_mermaid(data_contract_spec: OpenDataContractStandard) -> str | None:
+    try:
+        if not data_contract_spec.schema_:
             return None
-        else:
-            return f"{mmd_entity}\n"
+
+        mmd_entity = "erDiagram\n"
+
+        for schema in data_contract_spec.schema_:
+            schema_name = schema.name or schema.physicalName
+            entity_block = ""
+
+            if schema.properties:
+                for prop in schema.properties:
+                    clean_name = _sanitize_name(prop.name)
+                    indicators = ""
+
+                    if prop.primaryKey:
+                        indicators += "🔑"
+                    if getattr(prop, "partitioned", False):
+                        indicators += "🔀"
+                    if getattr(prop, "criticalDataElement", False):
+                        indicators += "⚠️"
+
+                    prop_type = prop.logicalType or prop.physicalType or "unknown"
+                    entity_block += f"\t{clean_name}{indicators} {prop_type}\n"
+
+            mmd_entity += f'\t"**{schema_name}**"' + "{\n" + entity_block + "}\n"
+
+        return f"{mmd_entity}\n"
+
     except Exception as e:
-        print(f"error : {e}")
+        print(f"Error generating ODCS mermaid diagram: {e}")
         return None
+
+
+def _sanitize_name(name: str) -> str:
+    return name.replace("#", "Nb").replace(" ", "_").replace("/", "by")

datacontract/export/odcs_v3_exporter.py

@@ -23,6 +23,12 @@ class OdcsV3Exporter(Exporter):
 
 
 def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
+    result = to_odcs_v3(data_contract_spec)
+
+    return result.to_yaml()
+
+
+def to_odcs_v3(data_contract_spec: DataContractSpecification) -> OpenDataContractStandard:
     result = OpenDataContractStandard(
         apiVersion="v3.0.1",
         kind="DataContract",
@@ -31,7 +37,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
         version=data_contract_spec.info.version,
         status=to_status(data_contract_spec.info.status),
     )
-
     if data_contract_spec.terms is not None:
         result.description = Description(
             purpose=data_contract_spec.terms.description.strip()
@@ -42,12 +47,10 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
             if data_contract_spec.terms.limitations is not None
             else None,
         )
-
     result.schema_ = []
     for model_key, model_value in data_contract_spec.models.items():
         odcs_schema = to_odcs_schema(model_key, model_value)
         result.schema_.append(odcs_schema)
-
     if data_contract_spec.servicelevels is not None:
         slas = []
         if data_contract_spec.servicelevels.availability is not None:
@@ -65,7 +68,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
 
         if len(slas) > 0:
             result.slaProperties = slas
-
     if data_contract_spec.info.contact is not None:
         support = []
         if data_contract_spec.info.contact.email is not None:
@@ -74,7 +76,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
             support.append(Support(channel="other", url=data_contract_spec.info.contact.url))
         if len(support) > 0:
             result.support = support
-
     if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0:
         servers = []
 
@@ -126,18 +127,15 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
 
         if len(servers) > 0:
             result.servers = servers
-
     custom_properties = []
     if data_contract_spec.info.owner is not None:
         custom_properties.append(CustomProperty(property="owner", value=data_contract_spec.info.owner))
     if data_contract_spec.info.model_extra is not None:
         for key, value in data_contract_spec.info.model_extra.items():
             custom_properties.append(CustomProperty(property=key, value=value))
-
     if len(custom_properties) > 0:
         result.customProperties = custom_properties
-
-    return result.to_yaml()
+    return result
 
 
 def to_odcs_schema(model_key, model_value: Model) -> SchemaObject:

datacontract/imports/excel_importer.py

@@ -31,8 +31,11 @@ logger = logging.getLogger(__name__)
 
 class ExcelImporter(Importer):
     def import_source(
-        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
-    ) -> OpenDataContractStandard:
+        self,
+        data_contract_specification: DataContractSpecification | OpenDataContractStandard,
+        source: str,
+        import_args: dict,
+    ) -> DataContractSpecification | OpenDataContractStandard:
         return import_excel_as_odcs(source)
 
 

datacontract/imports/importer.py

@@ -12,7 +12,7 @@ class Importer(ABC):
     @abstractmethod
     def import_source(
         self,
-        data_contract_specification: DataContractSpecification,
+        data_contract_specification: DataContractSpecification | OpenDataContractStandard,
         source: str,
         import_args: dict,
     ) -> DataContractSpecification | OpenDataContractStandard:
@@ -39,3 +39,12 @@ class ImportFormat(str, Enum):
     @classmethod
     def get_supported_formats(cls):
         return list(map(lambda c: c.value, cls))
+
+
+class Spec(str, Enum):
+    datacontract_specification = "datacontract_specification"
+    odcs = "odcs"
+
+    @classmethod
+    def get_supported_types(cls):
+        return list(map(lambda c: c.value, cls))

datacontract/imports/odcs_importer.py

@@ -48,9 +48,9 @@ def import_odcs(data_contract_specification: DataContractSpecification, source:
             engine="datacontract",
         )
     elif odcs_api_version.startswith("v3."):
-        from datacontract.imports.odcs_v3_importer import import_odcs_v3
+        from datacontract.imports.odcs_v3_importer import import_odcs_v3_as_dcs
 
-        return import_odcs_v3(data_contract_specification, source)
+        return import_odcs_v3_as_dcs(data_contract_specification, source)
     else:
         raise DataContractException(
             type="schema",

datacontract/imports/odcs_v3_importer.py

@@ -29,17 +29,18 @@ class OdcsImporter(Importer):
     def import_source(
         self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
     ) -> DataContractSpecification:
-        return import_odcs_v3(data_contract_specification, source)
+        return import_odcs_v3_as_dcs(data_contract_specification, source)
 
 
-def import_odcs_v3(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+def import_odcs_v3_as_dcs(
+    data_contract_specification: DataContractSpecification, source: str
+) -> DataContractSpecification:
     source_str = read_resource(source)
-    return import_odcs_v3_from_str(data_contract_specification, source_str)
+    odcs = parse_odcs_v3_from_str(source_str)
+    return import_from_odcs(data_contract_specification, odcs)
 
 
-def import_odcs_v3_from_str(
-    data_contract_specification: DataContractSpecification, source_str: str
-) -> DataContractSpecification:
+def parse_odcs_v3_from_str(source_str):
     try:
         odcs = OpenDataContractStandard.from_string(source_str)
     except Exception as e:
@@ -50,11 +51,10 @@ def import_odcs_v3_from_str(
             engine="datacontract",
             original_exception=e,
         )
-
-    return import_from_odcs_model(data_contract_specification, odcs)
+    return odcs
 
 
-def import_from_odcs_model(data_contract_specification, odcs):
+def import_from_odcs(data_contract_specification: DataContractSpecification, odcs: OpenDataContractStandard):
     data_contract_specification.id = odcs.id
     data_contract_specification.info = import_info(odcs)
     data_contract_specification.servers = import_servers(odcs)