datacontract-cli 0.10.15__py3-none-any.whl → 0.10.18__py3-none-any.whl

datacontract/export/exporter_factory.py

@@ -48,7 +48,9 @@ def load_module_class(module_path, class_name):
 exporter_factory = ExporterFactory()
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.avro, module_path="datacontract.export.avro_converter", class_name="AvroExporter"
+    name=ExportFormat.avro,
+    module_path="datacontract.export.avro_converter",
+    class_name="AvroExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -70,15 +72,21 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.dbml, module_path="datacontract.export.dbml_converter", class_name="DbmlExporter"
+    name=ExportFormat.dbml,
+    module_path="datacontract.export.dbml_converter",
+    class_name="DbmlExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.rdf, module_path="datacontract.export.rdf_converter", class_name="RdfExporter"
+    name=ExportFormat.rdf,
+    module_path="datacontract.export.rdf_converter",
+    class_name="RdfExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.dbt, module_path="datacontract.export.dbt_converter", class_name="DbtExporter"
+    name=ExportFormat.dbt,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -100,19 +108,27 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v2, module_path="datacontract.export.odcs_v2_exporter", class_name="OdcsV2Exporter"
+    name=ExportFormat.odcs_v2,
+    module_path="datacontract.export.odcs_v2_exporter",
+    class_name="OdcsV2Exporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v3, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
+    name=ExportFormat.odcs_v3,
+    module_path="datacontract.export.odcs_v3_exporter",
+    class_name="OdcsV3Exporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
+    name=ExportFormat.odcs,
+    module_path="datacontract.export.odcs_v3_exporter",
+    class_name="OdcsV3Exporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.go, module_path="datacontract.export.go_converter", class_name="GoExporter"
+    name=ExportFormat.go,
+    module_path="datacontract.export.go_converter",
+    class_name="GoExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -122,7 +138,9 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.html, module_path="datacontract.export.html_export", class_name="HtmlExporter"
+    name=ExportFormat.html,
+    module_path="datacontract.export.html_export",
+    class_name="HtmlExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -138,15 +156,21 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.sodacl, module_path="datacontract.export.sodacl_converter", class_name="SodaExporter"
+    name=ExportFormat.sodacl,
+    module_path="datacontract.export.sodacl_converter",
+    class_name="SodaExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.sql, module_path="datacontract.export.sql_converter", class_name="SqlExporter"
+    name=ExportFormat.sql,
+    module_path="datacontract.export.sql_converter",
+    class_name="SqlExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.sql_query, module_path="datacontract.export.sql_converter", class_name="SqlQueryExporter"
+    name=ExportFormat.sql_query,
+    module_path="datacontract.export.sql_converter",
+    class_name="SqlQueryExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -156,7 +180,9 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.spark, module_path="datacontract.export.spark_converter", class_name="SparkExporter"
+    name=ExportFormat.spark,
+    module_path="datacontract.export.spark_converter",
+    class_name="SparkExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -166,5 +192,17 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.dcs, module_path="datacontract.export.dcs_exporter", class_name="DcsExporter"
+    name=ExportFormat.dcs,
+    module_path="datacontract.export.dcs_exporter",
+    class_name="DcsExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.markdown,
+    module_path="datacontract.export.markdown_converter",
+    class_name="MarkdownExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
 )

datacontract/export/iceberg_converter.py

@@ -0,0 +1,188 @@
+from pyiceberg import types
+from pyiceberg.schema import Schema, assign_fresh_schema_ids
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Field,
+    Model,
+)
+
+
+class IcebergExporter(Exporter):
+    """
+    Exporter class for exporting data contracts to Iceberg schemas.
+    """
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model,
+        server,
+        sql_server_type,
+        export_args,
+    ):
+        """
+        Export the given data contract model to an Iceberg schema.
+
+        Args:
+            data_contract (DataContractSpecification): The data contract specification.
+            model: The model to export, currently just supports one model.
+            server: Not used in this implementation.
+            sql_server_type: Not used in this implementation.
+            export_args: Additional arguments for export.
+
+        Returns:
+            str: A string representation of the Iceberg json schema.
+        """
+
+        return to_iceberg(data_contract, model)
+
+
+def to_iceberg(contract: DataContractSpecification, model: str) -> str:
+    """
+    Converts a DataContractSpecification into an Iceberg json schema string. JSON string follows https://iceberg.apache.org/spec/#appendix-c-json-serialization.
+
+    Args:
+        contract (DataContractSpecification): The data contract specification containing models.
+        model: The model to export, currently just supports one model.
+
+    Returns:
+        str: A string representation of the Iceberg json schema.
+    """
+    if model is None or model == "all":
+        if len(contract.models.items()) != 1:
+            # Iceberg doesn't have a way to combine multiple models into a single schema, an alternative would be to export json lines
+            raise Exception(f"Can only output one model at a time, found {len(contract.models.items())} models")
+        for model_name, model in contract.models.items():
+            schema = to_iceberg_schema(model)
+    else:
+        if model not in contract.models:
+            raise Exception(f"model {model} not found in contract")
+        schema = to_iceberg_schema(contract.models[model])
+
+    return schema.model_dump_json()
+
+
+def to_iceberg_schema(model: Model) -> types.StructType:
+    """
+    Convert a model to a Iceberg schema.
+
+    Args:
+        model (Model): The model to convert.
+
+    Returns:
+        types.StructType: The corresponding Iceberg schema.
+    """
+    iceberg_fields = []
+    primary_keys = []
+    for field_name, spec_field in model.fields.items():
+        iceberg_field = make_field(field_name, spec_field)
+        iceberg_fields.append(iceberg_field)
+
+        if spec_field.primaryKey:
+            primary_keys.append(iceberg_field.name)
+
+    schema = Schema(*iceberg_fields)
+
+    # apply non-0 field IDs so we can set the identifier fields for the schema
+    schema = assign_fresh_schema_ids(schema)
+    for field in schema.fields:
+        if field.name in primary_keys:
+            schema.identifier_field_ids.append(field.field_id)
+
+    return schema
+
+
+def make_field(field_name, field):
+    field_type = get_field_type(field)
+
+    # Note: might want to re-populate field_id from config['icebergFieldId'] if it exists, however, it gets
+    # complicated since field_ids impact the list and map element_ids, and the importer is not keeping track of those.
+    # Even if IDs are re-constituted, it seems like the SDK code would still reset them before any operation against a catalog,
+    # so it's likely not worth it.
+
+    # Note 2: field_id defaults to 0 to signify that the exporter is not attempting to populate meaningful values.
+    # also, the Iceberg sdk catalog code will re-set the fieldIDs prior to executing any table operations on the schema
+    # ref: https://github.com/apache/iceberg-python/pull/1072
+    return types.NestedField(field_id=0, name=field_name, field_type=field_type, required=field.required is True)
+
+
+def make_list(item):
+    field_type = get_field_type(item)
+
+    # element_id defaults to 0 to signify that the exporter is not attempting to populate meaningful values (see #make_field)
+    return types.ListType(element_id=0, element_type=field_type, element_required=item.required is True)
+
+
+def make_map(field):
+    key_type = get_field_type(field.keys)
+    value_type = get_field_type(field.values)
+
+    # key_id and value_id defaults to 0 to signify that the exporter is not attempting to populate meaningful values (see #make_field)
+    return types.MapType(
+        key_id=0, key_type=key_type, value_id=0, value_type=value_type, value_required=field.values.required is True
+    )
+
+
+def to_struct_type(fields: dict[str, Field]) -> types.StructType:
+    """
+    Convert a dictionary of fields to a Iceberg StructType.
+
+    Args:
+        fields (dict[str, Field]): The fields to convert.
+
+    Returns:
+        types.StructType: The corresponding Iceberg StructType.
+    """
+    struct_fields = []
+    for field_name, field in fields.items():
+        struct_field = make_field(field_name, field)
+        struct_fields.append(struct_field)
+    return types.StructType(*struct_fields)
+
+
+def get_field_type(field: Field) -> types.IcebergType:
+    """
+    Convert a field to a Iceberg IcebergType.
+
+    Args:
+        field (Field): The field to convert.
+
+    Returns:
+        types.IcebergType: The corresponding Iceberg IcebergType.
+    """
+    field_type = field.type
+    if field_type is None or field_type in ["null"]:
+        return types.NullType()
+    if field_type == "array":
+        return make_list(field.items)
+    if field_type == "map":
+        return make_map(field)
+    if field_type in ["object", "record", "struct"]:
+        return to_struct_type(field.fields)
+    if field_type in ["string", "varchar", "text"]:
+        return types.StringType()
+    if field_type in ["number", "decimal", "numeric"]:
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return types.DecimalType(precision=precision, scale=scale)
+    if field_type in ["integer", "int"]:
+        return types.IntegerType()
+    if field_type in ["bigint", "long"]:
+        return types.LongType()
+    if field_type == "float":
+        return types.FloatType()
+    if field_type == "double":
+        return types.DoubleType()
+    if field_type == "boolean":
+        return types.BooleanType()
+    if field_type in ["timestamp", "timestamp_tz"]:
+        return types.TimestamptzType()
+    if field_type == "timestamp_ntz":
+        return types.TimestampType()
+    if field_type == "date":
+        return types.DateType()
+    if field_type == "bytes":
+        return types.BinaryType()
+    return types.BinaryType()

datacontract/export/markdown_converter.py

@@ -0,0 +1,208 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Definition,
+    Field,
+    Model,
+    Server,
+    ServiceLevel,
+)
+
+
+class MarkdownExporter(Exporter):
+    """Exporter implementation for converting data contracts to Markdown."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: dict,
+    ) -> str:
+        """Exports a data contract to Markdown format."""
+        return to_markdown(data_contract)
+
+
+def to_markdown(data_contract: DataContractSpecification) -> str:
+    """
+    Convert a data contract to its Markdown representation.
+
+    Args:
+        data_contract (DataContractSpecification): The data contract to convert.
+
+    Returns:
+        str: The Markdown representation of the data contract.
+    """
+    markdown_parts = [
+        f"# {data_contract.id}",
+        "## Info",
+        obj_attributes_to_markdown(data_contract.info),
+        "",
+        "## Servers",
+        servers_to_markdown(data_contract.servers),
+        "",
+        "## Terms",
+        obj_attributes_to_markdown(data_contract.terms),
+        "",
+        "## Models",
+        models_to_markdown(data_contract.models),
+        "",
+        "## Definitions",
+        definitions_to_markdown(data_contract.definitions),
+        "",
+        "## Service levels",
+        service_level_to_markdown(data_contract.servicelevels),
+    ]
+    return "\n".join(markdown_parts)
+
+
+def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_in_table_cell: bool = False) -> str:
+    if not obj:
+        return ""
+    if is_in_table_cell:
+        bullet_char = "•"
+        newline_char = "<br>"
+    else:
+        bullet_char = "-"
+        newline_char = "\n"
+    obj_model = obj.model_dump(exclude_unset=True, exclude=excluded_fields)
+    description_value = obj_model.pop("description", None)
+    attributes = [
+        (f"{bullet_char} `{attr}`" if value is True else f"{bullet_char} **{attr}:** {value}")
+        for attr, value in obj_model.items()
+        if value
+    ]
+    description = f"*{description_to_markdown(description_value)}*"
+    return newline_char.join([description] + attributes)
+
+
+def servers_to_markdown(servers: Dict[str, Server]) -> str:
+    if not servers:
+        return ""
+    markdown_parts = [
+        "| Name | Type | Attributes |",
+        "| ---- | ---- | ---------- |",
+    ]
+    for server_name, server in servers.items():
+        markdown_parts.append(
+            f"| {server_name} | {server.type or ''} | {obj_attributes_to_markdown(server, {'type'}, True)} |"
+        )
+    return "\n".join(markdown_parts)
+
+
+def models_to_markdown(models: Dict[str, Model]) -> str:
+    return "\n".join(model_to_markdown(model_name, model) for model_name, model in models.items())
+
+
+def model_to_markdown(model_name: str, model: Model) -> str:
+    """
+    Generate Markdown representation for a specific model.
+
+    Args:
+        model_name (str): The name of the model.
+        model (Model): The model object.
+
+    Returns:
+        str: The Markdown representation of the model.
+    """
+    parts = [
+        f"### {model_name}",
+        f"*{description_to_markdown(model.description)}*",
+        "",
+        "| Field | Type | Attributes |",
+        "| ----- | ---- | ---------- |",
+    ]
+
+    # Append generated field rows
+    parts.append(fields_to_markdown(model.fields))
+    return "\n".join(parts)
+
+
+def fields_to_markdown(
+    fields: Dict[str, Field],
+    level: int = 0,
+) -> str:
+    """
+    Generate Markdown table rows for all fields in a model.
+
+    Args:
+        fields (Dict[str, Field]): The fields to process.
+        level (int): The level of nesting for indentation.
+
+    Returns:
+        str: A Markdown table rows for the fields.
+    """
+
+    return "\n".join(field_to_markdown(field_name, field, level) for field_name, field in fields.items())
+
+
+def field_to_markdown(field_name: str, field: Field, level: int = 0) -> str:
+    """
+    Generate Markdown table rows for a single field, including nested structures.
+
+    Args:
+        field_name (str): The name of the field.
+        field (Field): The field object.
+        level (int): The level of nesting for indentation.
+
+    Returns:
+        str: A Markdown table rows for the field.
+    """
+    tabs = "&numsp;" * level
+    arrow = "&#x21b3;" if level > 0 else ""
+    column_name = f"{tabs}{arrow} {field_name}"
+
+    attributes = obj_attributes_to_markdown(field, {"type", "fields", "items", "keys", "values"}, True)
+
+    rows = [f"| {column_name} | {field.type} | {attributes} |"]
+
+    # Recursively handle nested fields, array, map
+    if field.fields:
+        rows.append(fields_to_markdown(field.fields, level + 1))
+    if field.items:
+        rows.append(field_to_markdown("items", field.items, level + 1))
+    if field.keys:
+        rows.append(field_to_markdown("keys", field.keys, level + 1))
+    if field.values:
+        rows.append(field_to_markdown("values", field.values, level + 1))
+
+    return "\n".join(rows)
+
+
+def definitions_to_markdown(definitions: Dict[str, Definition]) -> str:
+    if not definitions:
+        return ""
+    markdown_parts = [
+        "| Name | Type | Domain | Attributes |",
+        "| ---- | ---- | ------ | ---------- |",
+    ]
+    for definition_name, definition in definitions.items():
+        markdown_parts.append(
+            f"| {definition_name} | {definition.type or ''} | {definition.domain or ''} | {obj_attributes_to_markdown(definition, {'name', 'type', 'domain'}, True)} |",
+        )
+    return "\n".join(markdown_parts)
+
+
+def service_level_to_markdown(service_level: ServiceLevel | None) -> str:
+    if not service_level:
+        return ""
+    sections = {
+        "Availability": service_level.availability,
+        "Retention": service_level.retention,
+        "Latency": service_level.latency,
+        "Freshness": service_level.freshness,
+        "Frequency": service_level.frequency,
+        "Support": service_level.support,
+        "Backup": service_level.backup,
+    }
+    result = [f"### {name}\n{obj_attributes_to_markdown(attr)}\n" for name, attr in sections.items() if attr]
+    return "\n".join(result)
+
+
+def description_to_markdown(description: str | None) -> str:
+    return (description or "No description.").replace("\n", "<br>")

datacontract/export/odcs_v3_exporter.py

@@ -148,6 +148,10 @@ def to_odcs_schema(model_key, model_value: Model) -> dict:
     if properties:
         odcs_table["properties"] = properties
 
+    model_quality = to_odcs_quality_list(model_value.quality)
+    if len(model_quality) > 0:
+        odcs_table["quality"] = model_quality
+
     odcs_table["customProperties"] = []
     if model_value.model_extra is not None:
         for key, value in model_value.model_extra.items():
@@ -222,6 +226,12 @@ def to_property(field_name: str, field: Field) -> dict:
         property["examples"] = field.examples
     if field.example is not None:
         property["examples"] = [field.example]
+    if field.primaryKey is not None and field.primaryKey:
+        property["primaryKey"] = field.primaryKey
+        property["primaryKeyPosition"] = 1
+    if field.primary is not None and field.primary:
+        property["primaryKey"] = field.primary
+        property["primaryKeyPosition"] = 1
 
     property["customProperties"] = []
     if field.model_extra is not None:
@@ -257,38 +267,48 @@ def to_property(field_name: str, field: Field) -> dict:
         del property["logicalTypeOptions"]
 
     if field.quality is not None:
-        quality_property = []
-        for quality in field.quality:
-            quality_dict = {"type": quality.type}
-            if quality.description is not None:
-                quality_dict["description"] = quality.description
-            if quality.query is not None:
-                quality_dict["query"] = quality.query
-            # dialect is not supported in v3.0.0
-            if quality.mustBe is not None:
-                quality_dict["mustBe"] = quality.mustBe
-            if quality.mustNotBe is not None:
-                quality_dict["mustNotBe"] = quality.mustNotBe
-            if quality.mustBeGreaterThan is not None:
-                quality_dict["mustBeGreaterThan"] = quality.mustBeGreaterThan
-            if quality.mustBeGreaterThanOrEqualTo is not None:
-                quality_dict["mustBeGreaterThanOrEqualTo"] = quality.mustBeGreaterThanOrEqualTo
-            if quality.mustBeLessThan is not None:
-                quality_dict["mustBeLessThan"] = quality.mustBeLessThan
-            if quality.mustBeLessThanOrEqualTo is not None:
-                quality_dict["mustBeLessThanOrEqualTo"] = quality.mustBeLessThanOrEqualTo
-            if quality.mustBeBetween is not None:
-                quality_dict["mustBeBetween"] = quality.mustBeBetween
-            if quality.mustNotBeBetween is not None:
-                quality_dict["mustNotBeBetween"] = quality.mustNotBeBetween
-            if quality.engine is not None:
-                quality_dict["engine"] = quality.engine
-            if quality.implementation is not None:
-                quality_dict["implementation"] = quality.implementation
-            quality_property.append(quality_dict)
+        quality_list = field.quality
+        quality_property = to_odcs_quality_list(quality_list)
         if len(quality_property) > 0:
             property["quality"] = quality_property
 
     # todo enum
 
     return property
+
+
+def to_odcs_quality_list(quality_list):
+    quality_property = []
+    for quality in quality_list:
+        quality_property.append(to_odcs_quality(quality))
+    return quality_property
+
+
+def to_odcs_quality(quality):
+    quality_dict = {"type": quality.type}
+    if quality.description is not None:
+        quality_dict["description"] = quality.description
+    if quality.query is not None:
+        quality_dict["query"] = quality.query
+    # dialect is not supported in v3.0.0
+    if quality.mustBe is not None:
+        quality_dict["mustBe"] = quality.mustBe
+    if quality.mustNotBe is not None:
+        quality_dict["mustNotBe"] = quality.mustNotBe
+    if quality.mustBeGreaterThan is not None:
+        quality_dict["mustBeGreaterThan"] = quality.mustBeGreaterThan
+    if quality.mustBeGreaterThanOrEqualTo is not None:
+        quality_dict["mustBeGreaterThanOrEqualTo"] = quality.mustBeGreaterThanOrEqualTo
+    if quality.mustBeLessThan is not None:
+        quality_dict["mustBeLessThan"] = quality.mustBeLessThan
+    if quality.mustBeLessThanOrEqualTo is not None:
+        quality_dict["mustBeLessThanOrEqualTo"] = quality.mustBeLessThanOrEqualTo
+    if quality.mustBeBetween is not None:
+        quality_dict["mustBeBetween"] = quality.mustBeBetween
+    if quality.mustNotBeBetween is not None:
+        quality_dict["mustNotBeBetween"] = quality.mustNotBeBetween
+    if quality.engine is not None:
+        quality_dict["engine"] = quality.engine
+    if quality.implementation is not None:
+        quality_dict["implementation"] = quality.implementation
+    return quality_dict

datacontract/export/sodacl_converter.py

@@ -32,7 +32,7 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
     checks = []
     fields = model_value.fields
 
-    quote_field_name = server_type in ["postgres"]
+    quote_field_name = server_type in ["postgres", "sqlserver"]
 
     for field_name, field in fields.items():
         checks.append(check_field_is_present(field_name))
@@ -200,9 +200,9 @@ def check_quality_list(model_name, field_name, quality_list: List[Quality]):
     for quality in quality_list:
         if quality.type == "sql":
             if field_name is None:
-                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
-            else:
                 metric_name = f"{model_name}_quality_sql_{count}"
+            else:
+                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
             threshold = to_sodacl_threshold(quality)
             query = prepare_query(quality, model_name, field_name)
             if query is None:
@@ -265,6 +265,7 @@ def to_sodacl_threshold(quality: Quality) -> str | None:
     return None
 
 
+# These are deprecated root-level quality specifications, use the model-level and field-level quality fields instead
 def add_quality_checks(sodacl, data_contract_spec):
     if data_contract_spec.quality is None:
         return

datacontract/export/sql_converter.py

@@ -113,7 +113,7 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
         result += f"  {field_name} {type}"
         if field.required:
             result += " not null"
-        if field.primary:
+        if field.primaryKey or field.primary:
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'