datacontract-cli 0.10.16__py3-none-any.whl → 0.10.19__py3-none-any.whl

datacontract/export/exporter_factory.py

@@ -48,7 +48,9 @@ def load_module_class(module_path, class_name):
 exporter_factory = ExporterFactory()
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.avro, module_path="datacontract.export.avro_converter", class_name="AvroExporter"
+    name=ExportFormat.avro,
+    module_path="datacontract.export.avro_converter",
+    class_name="AvroExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -70,15 +72,21 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.dbml, module_path="datacontract.export.dbml_converter", class_name="DbmlExporter"
+    name=ExportFormat.dbml,
+    module_path="datacontract.export.dbml_converter",
+    class_name="DbmlExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.rdf, module_path="datacontract.export.rdf_converter", class_name="RdfExporter"
+    name=ExportFormat.rdf,
+    module_path="datacontract.export.rdf_converter",
+    class_name="RdfExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.dbt, module_path="datacontract.export.dbt_converter", class_name="DbtExporter"
+    name=ExportFormat.dbt,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -100,19 +108,27 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v2, module_path="datacontract.export.odcs_v2_exporter", class_name="OdcsV2Exporter"
+    name=ExportFormat.odcs_v2,
+    module_path="datacontract.export.odcs_v2_exporter",
+    class_name="OdcsV2Exporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v3, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
+    name=ExportFormat.odcs_v3,
+    module_path="datacontract.export.odcs_v3_exporter",
+    class_name="OdcsV3Exporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
+    name=ExportFormat.odcs,
+    module_path="datacontract.export.odcs_v3_exporter",
+    class_name="OdcsV3Exporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.go, module_path="datacontract.export.go_converter", class_name="GoExporter"
+    name=ExportFormat.go,
+    module_path="datacontract.export.go_converter",
+    class_name="GoExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -122,7 +138,9 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.html, module_path="datacontract.export.html_export", class_name="HtmlExporter"
+    name=ExportFormat.html,
+    module_path="datacontract.export.html_export",
+    class_name="HtmlExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -138,15 +156,21 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.sodacl, module_path="datacontract.export.sodacl_converter", class_name="SodaExporter"
+    name=ExportFormat.sodacl,
+    module_path="datacontract.export.sodacl_converter",
+    class_name="SodaExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.sql, module_path="datacontract.export.sql_converter", class_name="SqlExporter"
+    name=ExportFormat.sql,
+    module_path="datacontract.export.sql_converter",
+    class_name="SqlExporter",
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.sql_query, module_path="datacontract.export.sql_converter", class_name="SqlQueryExporter"
+    name=ExportFormat.sql_query,
+    module_path="datacontract.export.sql_converter",
+    class_name="SqlQueryExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -156,7 +180,9 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.spark, module_path="datacontract.export.spark_converter", class_name="SparkExporter"
+    name=ExportFormat.spark,
+    module_path="datacontract.export.spark_converter",
+    class_name="SparkExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -166,7 +192,15 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.dcs, module_path="datacontract.export.dcs_exporter", class_name="DcsExporter"
+    name=ExportFormat.dcs,
+    module_path="datacontract.export.dcs_exporter",
+    class_name="DcsExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.markdown,
+    module_path="datacontract.export.markdown_converter",
+    class_name="MarkdownExporter",
 )
 
 exporter_factory.register_lazy_exporter(

datacontract/export/iceberg_converter.py

@@ -105,14 +105,14 @@ def make_field(field_name, field):
     # Note 2: field_id defaults to 0 to signify that the exporter is not attempting to populate meaningful values.
     # also, the Iceberg sdk catalog code will re-set the fieldIDs prior to executing any table operations on the schema
     # ref: https://github.com/apache/iceberg-python/pull/1072
-    return types.NestedField(field_id=0, name=field_name, field_type=field_type, required=field.required)
+    return types.NestedField(field_id=0, name=field_name, field_type=field_type, required=field.required is True)
 
 
 def make_list(item):
     field_type = get_field_type(item)
 
     # element_id defaults to 0 to signify that the exporter is not attempting to populate meaningful values (see #make_field)
-    return types.ListType(element_id=0, element_type=field_type, element_required=item.required)
+    return types.ListType(element_id=0, element_type=field_type, element_required=item.required is True)
 
 
 def make_map(field):
@@ -121,7 +121,7 @@ def make_map(field):
 
     # key_id and value_id defaults to 0 to signify that the exporter is not attempting to populate meaningful values (see #make_field)
     return types.MapType(
-        key_id=0, key_type=key_type, value_id=0, value_type=value_type, value_required=field.values.required
+        key_id=0, key_type=key_type, value_id=0, value_type=value_type, value_required=field.values.required is True
     )
 
 

datacontract/export/markdown_converter.py

@@ -0,0 +1,208 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Definition,
+    Field,
+    Model,
+    Server,
+    ServiceLevel,
+)
+
+
+class MarkdownExporter(Exporter):
+    """Exporter implementation for converting data contracts to Markdown."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: dict,
+    ) -> str:
+        """Exports a data contract to Markdown format."""
+        return to_markdown(data_contract)
+
+
+def to_markdown(data_contract: DataContractSpecification) -> str:
+    """
+    Convert a data contract to its Markdown representation.
+
+    Args:
+        data_contract (DataContractSpecification): The data contract to convert.
+
+    Returns:
+        str: The Markdown representation of the data contract.
+    """
+    markdown_parts = [
+        f"# {data_contract.id}",
+        "## Info",
+        obj_attributes_to_markdown(data_contract.info),
+        "",
+        "## Servers",
+        servers_to_markdown(data_contract.servers),
+        "",
+        "## Terms",
+        obj_attributes_to_markdown(data_contract.terms),
+        "",
+        "## Models",
+        models_to_markdown(data_contract.models),
+        "",
+        "## Definitions",
+        definitions_to_markdown(data_contract.definitions),
+        "",
+        "## Service levels",
+        service_level_to_markdown(data_contract.servicelevels),
+    ]
+    return "\n".join(markdown_parts)
+
+
+def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_in_table_cell: bool = False) -> str:
+    if not obj:
+        return ""
+    if is_in_table_cell:
+        bullet_char = "•"
+        newline_char = "<br>"
+    else:
+        bullet_char = "-"
+        newline_char = "\n"
+    obj_model = obj.model_dump(exclude_unset=True, exclude=excluded_fields)
+    description_value = obj_model.pop("description", None)
+    attributes = [
+        (f"{bullet_char} `{attr}`" if value is True else f"{bullet_char} **{attr}:** {value}")
+        for attr, value in obj_model.items()
+        if value
+    ]
+    description = f"*{description_to_markdown(description_value)}*"
+    return newline_char.join([description] + attributes)
+
+
+def servers_to_markdown(servers: Dict[str, Server]) -> str:
+    if not servers:
+        return ""
+    markdown_parts = [
+        "| Name | Type | Attributes |",
+        "| ---- | ---- | ---------- |",
+    ]
+    for server_name, server in servers.items():
+        markdown_parts.append(
+            f"| {server_name} | {server.type or ''} | {obj_attributes_to_markdown(server, {'type'}, True)} |"
+        )
+    return "\n".join(markdown_parts)
+
+
+def models_to_markdown(models: Dict[str, Model]) -> str:
+    return "\n".join(model_to_markdown(model_name, model) for model_name, model in models.items())
+
+
+def model_to_markdown(model_name: str, model: Model) -> str:
+    """
+    Generate Markdown representation for a specific model.
+
+    Args:
+        model_name (str): The name of the model.
+        model (Model): The model object.
+
+    Returns:
+        str: The Markdown representation of the model.
+    """
+    parts = [
+        f"### {model_name}",
+        f"*{description_to_markdown(model.description)}*",
+        "",
+        "| Field | Type | Attributes |",
+        "| ----- | ---- | ---------- |",
+    ]
+
+    # Append generated field rows
+    parts.append(fields_to_markdown(model.fields))
+    return "\n".join(parts)
+
+
+def fields_to_markdown(
+    fields: Dict[str, Field],
+    level: int = 0,
+) -> str:
+    """
+    Generate Markdown table rows for all fields in a model.
+
+    Args:
+        fields (Dict[str, Field]): The fields to process.
+        level (int): The level of nesting for indentation.
+
+    Returns:
+        str: A Markdown table rows for the fields.
+    """
+
+    return "\n".join(field_to_markdown(field_name, field, level) for field_name, field in fields.items())
+
+
+def field_to_markdown(field_name: str, field: Field, level: int = 0) -> str:
+    """
+    Generate Markdown table rows for a single field, including nested structures.
+
+    Args:
+        field_name (str): The name of the field.
+        field (Field): The field object.
+        level (int): The level of nesting for indentation.
+
+    Returns:
+        str: A Markdown table rows for the field.
+    """
+    tabs = "&numsp;" * level
+    arrow = "&#x21b3;" if level > 0 else ""
+    column_name = f"{tabs}{arrow} {field_name}"
+
+    attributes = obj_attributes_to_markdown(field, {"type", "fields", "items", "keys", "values"}, True)
+
+    rows = [f"| {column_name} | {field.type} | {attributes} |"]
+
+    # Recursively handle nested fields, array, map
+    if field.fields:
+        rows.append(fields_to_markdown(field.fields, level + 1))
+    if field.items:
+        rows.append(field_to_markdown("items", field.items, level + 1))
+    if field.keys:
+        rows.append(field_to_markdown("keys", field.keys, level + 1))
+    if field.values:
+        rows.append(field_to_markdown("values", field.values, level + 1))
+
+    return "\n".join(rows)
+
+
+def definitions_to_markdown(definitions: Dict[str, Definition]) -> str:
+    if not definitions:
+        return ""
+    markdown_parts = [
+        "| Name | Type | Domain | Attributes |",
+        "| ---- | ---- | ------ | ---------- |",
+    ]
+    for definition_name, definition in definitions.items():
+        markdown_parts.append(
+            f"| {definition_name} | {definition.type or ''} | {definition.domain or ''} | {obj_attributes_to_markdown(definition, {'name', 'type', 'domain'}, True)} |",
+        )
+    return "\n".join(markdown_parts)
+
+
+def service_level_to_markdown(service_level: ServiceLevel | None) -> str:
+    if not service_level:
+        return ""
+    sections = {
+        "Availability": service_level.availability,
+        "Retention": service_level.retention,
+        "Latency": service_level.latency,
+        "Freshness": service_level.freshness,
+        "Frequency": service_level.frequency,
+        "Support": service_level.support,
+        "Backup": service_level.backup,
+    }
+    result = [f"### {name}\n{obj_attributes_to_markdown(attr)}\n" for name, attr in sections.items() if attr]
+    return "\n".join(result)
+
+
+def description_to_markdown(description: str | None) -> str:
+    return (description or "No description.").replace("\n", "<br>")

datacontract/export/odcs_v3_exporter.py

@@ -226,6 +226,12 @@ def to_property(field_name: str, field: Field) -> dict:
         property["examples"] = field.examples
     if field.example is not None:
         property["examples"] = [field.example]
+    if field.primaryKey is not None and field.primaryKey:
+        property["primaryKey"] = field.primaryKey
+        property["primaryKeyPosition"] = 1
+    if field.primary is not None and field.primary:
+        property["primaryKey"] = field.primary
+        property["primaryKeyPosition"] = 1
 
     property["customProperties"] = []
     if field.model_extra is not None:

datacontract/export/sodacl_converter.py

@@ -30,9 +30,10 @@ def to_sodacl_yaml(
 
 def to_checks(model_key, model_value, server_type: str, check_types: bool):
     checks = []
+    model_name = to_model_name(model_key, model_value, server_type)
     fields = model_value.fields
 
-    quote_field_name = server_type in ["postgres"]
+    quote_field_name = server_type in ["postgres", "sqlserver"]
 
     for field_name, field in fields.items():
         checks.append(check_field_is_present(field_name))
@@ -62,25 +63,41 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
         if field.enum is not None and len(field.enum) > 0:
             checks.append(check_field_enum(field_name, field.enum, quote_field_name))
         if field.quality is not None and len(field.quality) > 0:
-            quality_list = check_quality_list(model_key, field_name, field.quality)
+            quality_list = check_quality_list(model_name, field_name, field.quality)
             if (quality_list is not None) and len(quality_list) > 0:
                 checks.append(quality_list)
         # TODO references: str = None
         # TODO format
 
     if model_value.quality is not None and len(model_value.quality) > 0:
-        quality_list = check_quality_list(model_key, None, model_value.quality)
+        quality_list = check_quality_list(model_name, None, model_value.quality)
         if (quality_list is not None) and len(quality_list) > 0:
             checks.append(quality_list)
 
-    checks_for_model_key = f"checks for {model_key}"
+    checks_for_model_key = f"checks for {model_name}"
 
     if quote_field_name:
-        checks_for_model_key = f'checks for "{model_key}"'
+        checks_for_model_key = f'checks for "{model_name}"'
 
     return checks_for_model_key, checks
 
 
+def to_model_name(model_key, model_value, server_type):
+    if server_type == "databricks":
+        if model_value.config is not None and "databricksTable" in model_value.config:
+            return model_value.config["databricksTable"]
+    if server_type == "snowflake":
+        if model_value.config is not None and "snowflakeTable" in model_value.config:
+            return model_value.config["snowflakeTable"]
+    if server_type == "sqlserver":
+        if model_value.config is not None and "sqlserverTable" in model_value.config:
+            return model_value.config["sqlserverTable"]
+    if server_type == "postgres" or server_type == "postgresql":
+        if model_value.config is not None and "postgresTable" in model_value.config:
+            return model_value.config["postgresTable"]
+    return model_key
+
+
 def check_field_is_present(field_name):
     return {
         "schema": {

datacontract/export/sql_converter.py

@@ -113,7 +113,7 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
         result += f"  {field_name} {type}"
         if field.required:
             result += " not null"
-        if field.primary:
+        if field.primaryKey or field.primary:
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'

datacontract/export/sql_type_converter.py

@@ -182,11 +182,16 @@ def convert_to_databricks(field: Field) -> None | str:
     if type.lower() in ["boolean"]:
         return "BOOLEAN"
     if type.lower() in ["object", "record", "struct"]:
-        return "STRUCT"
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_to_databricks(nested_field)
+            nested_fields.append(f"{nested_field_name} {nested_field_type}")
+        return f"STRUCT<{', '.join(nested_fields)}>"
     if type.lower() in ["bytes"]:
         return "BINARY"
     if type.lower() in ["array"]:
-        return "ARRAY"
+        item_type = convert_to_databricks(field.items)
+        return f"ARRAY<{item_type}>"
     return None
 
 
@@ -311,6 +316,27 @@ def convert_type_to_sqlserver(field: Field) -> None | str:
 
 def convert_type_to_bigquery(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent bigquery types"""
+
+    # BigQuery exporter cannot be used for complex types, as the exporter has different syntax than SodaCL
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    if field.config and "bigqueryType" in field.config:
+        return field.config["bigqueryType"]
+
+    if field_type.lower() in ["array"]:
+        item_type = convert_type_to_bigquery(field.items)
+        return f"ARRAY<{item_type}>"
+
+    if field_type.lower() in ["object", "record", "struct"]:
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_type_to_bigquery(nested_field)
+            nested_fields.append(f"{nested_field_name} {nested_field_type}")
+        return f"STRUCT<{', '.join(nested_fields)}>"
+
     return map_type_to_bigquery(field)
 
 

datacontract/export/sqlalchemy_converter.py

@@ -114,7 +114,9 @@ def constant_field_value(field_name: str, field: spec.Field) -> tuple[ast.Call,
     if new_type is None:
         raise RuntimeError(f"Unsupported field type {field.type}.")
 
-    return Column(new_type, nullable=not field.required, comment=field.description, primary_key=field.primary), None
+    return Column(
+        new_type, nullable=not field.required, comment=field.description, primary_key=field.primaryKey or field.primary
+    ), None
 
 
 def column_assignment(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:

datacontract/imports/csv_importer.py

@@ -0,0 +1,89 @@
+import os
+
+import clevercsv
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Example, Field, Model, Server
+
+
+class CsvImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_csv(data_contract_specification, self.import_format, source)
+
+
+def import_csv(data_contract_specification: DataContractSpecification, format: str, source: str):
+    include_example = False
+
+    # detect encoding and dialect
+    encoding = clevercsv.encoding.get_encoding(source)
+    with open(source, "r", newline="") as fp:
+        dialect = clevercsv.Sniffer().sniff(fp.read(10000))
+
+    # using auto detecting of the format and encoding
+    df = clevercsv.read_dataframe(source)
+
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    # use the file name as table name
+    table_name = os.path.splitext(os.path.basename(source))[0]
+
+    if data_contract_specification.servers is None:
+        data_contract_specification.servers = {}
+
+    data_contract_specification.servers["production"] = Server(
+        type="local", path=source, format="csv", delimiter=dialect.delimiter
+    )
+
+    fields = {}
+    for column, dtype in df.dtypes.items():
+        field = Field()
+        field.type = map_type_from_pandas(dtype.name)
+        fields[column] = field
+
+    data_contract_specification.models[table_name] = Model(
+        type="table",
+        description=f"Csv file with encoding {encoding}",
+        fields=fields,
+    )
+
+    # multiline data is not correctly handled by yaml dump
+    if include_example:
+        if data_contract_specification.examples is None:
+            data_contract_specification.examples = []
+
+        # read first 10 lines with the detected encoding
+        with open(source, "r", encoding=encoding) as csvfile:
+            lines = csvfile.readlines()[:10]
+
+        data_contract_specification.examples.append(Example(type="csv", model=table_name, data="".join(lines)))
+
+    return data_contract_specification
+
+
+def map_type_from_pandas(sql_type: str):
+    if sql_type is None:
+        return None
+
+    sql_type_normed = sql_type.lower().strip()
+
+    if sql_type_normed == "object":
+        return "string"
+    elif sql_type_normed.startswith("str"):
+        return "string"
+    elif sql_type_normed.startswith("int"):
+        return "integer"
+    elif sql_type_normed.startswith("float"):
+        return "float"
+    elif sql_type_normed.startswith("bool"):
+        return "boolean"
+    elif sql_type_normed.startswith("timestamp"):
+        return "timestamp"
+    elif sql_type_normed == "datetime64":
+        return "date"
+    elif sql_type_normed == "timedelta[ns]":
+        return "timestamp_ntz"
+    else:
+        return "variant"

datacontract/imports/dbml_importer.py

@@ -84,7 +84,7 @@ def import_table_fields(table, references) -> dict[str, Field]:
         imported_fields[field_name] = Field()
         imported_fields[field_name].required = field.not_null
         imported_fields[field_name].description = field.note.text
-        imported_fields[field_name].primary = field.pk
+        imported_fields[field_name].primaryKey = field.pk
         imported_fields[field_name].unique = field.unique
         # This is an assumption, that these might be valid SQL Types, since
         # DBML doesn't really enforce anything other than 'no spaces' in column types