datacontract-cli 0.9.8__py3-none-any.whl → 0.10.0__py3-none-any.whl

datacontract/cli.py

@@ -16,6 +16,7 @@ from datacontract.init.download_datacontract_file import \
 
 console = Console()
 
+
 class OrderedCommands(TyperGroup):
     def list_commands(self, ctx: Context) -> Iterable[str]:
         return self.commands.keys()
@@ -153,6 +154,7 @@ class ExportFormat(str, Enum):
     avro_idl = "avro-idl"
     sql = "sql"
     sql_query = "sql-query"
+    html = "html"
 
 
 @app.command()

datacontract/data_contract.py

@@ -4,6 +4,7 @@ import tempfile
 import typing
 
 import yaml
+from pyspark.sql import SparkSession
 
 from datacontract.breaking.breaking import models_breaking_changes, \
     quality_breaking_changes
@@ -19,6 +20,7 @@ from datacontract.export.dbt_converter import to_dbt_models_yaml, \
     to_dbt_sources_yaml, to_dbt_staging_sql
 from datacontract.export.great_expectations_converter import \
     to_great_expectations
+from datacontract.export.html_export import to_html
 from datacontract.export.jsonschema_converter import to_jsonschema_json
 from datacontract.export.odcs_converter import to_odcs_yaml
 from datacontract.export.protobuf_converter import to_protobuf
@@ -39,8 +41,6 @@ from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
 from datacontract.lint.linters.field_reference_linter import \
     FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
-from datacontract.lint.linters.primary_field_linter import \
-    PrimaryFieldUniqueRequired
 from datacontract.lint.linters.quality_schema_linter import \
     QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import \
@@ -53,25 +53,6 @@ from datacontract.model.exceptions import DataContractException
 from datacontract.model.run import Run, Check
 
 
-def _determine_sql_server_type(data_contract, sql_server_type):
-    if sql_server_type == "auto":
-        if data_contract.servers is None or len(data_contract.servers) == 0:
-            raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
-
-        server_types = set([server.type for server in data_contract.servers.values()])
-        if "snowflake" in server_types:
-            return "snowflake"
-        elif "postgres" in server_types:
-            return "postgres"
-        elif "databricks" in server_types:
-            return "databricks"
-        else:
-            # default to snowflake dialect
-            return "snowflake"
-    else:
-        return sql_server_type
-
-
 class DataContract:
     def __init__(
         self,
@@ -83,7 +64,7 @@ class DataContract:
         examples: bool = False,
         publish_url: str = None,
         publish_to_opentelemetry: bool = False,
-        spark: str = None,
+        spark: SparkSession = None,
         inline_definitions: bool = False,
     ):
         self._data_contract_file = data_contract_file
@@ -102,7 +83,6 @@ class DataContract:
             FieldPatternLinter(),
             FieldReferenceLinter(),
             NoticePeriodLinter(),
-            PrimaryFieldUniqueRequired(),
             ValidFieldConstraintsLinter(),
             DescriptionLinter(),
         }
@@ -246,12 +226,12 @@ class DataContract:
             try:
                 publish_datamesh_manager(run, self._publish_url)
             except Exception:
-                logging.error("Failed to publish to datamesh manager")
+                run.log_error("Failed to publish to datamesh manager")
         if self._publish_to_opentelemetry:
             try:
                 publish_opentelemetry(run)
             except Exception:
-                logging.error("Failed to publish to opentelemetry")
+                run.log_error("Failed to publish to opentelemetry")
 
         return run
 
@@ -385,13 +365,13 @@ class DataContract:
         if export_format == "terraform":
             return to_terraform(data_contract)
         if export_format == "sql":
-            server_type = _determine_sql_server_type(data_contract, sql_server_type)
+            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
             return to_sql_ddl(data_contract, server_type=server_type)
         if export_format == "sql-query":
             if data_contract.models is None:
                 raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
 
-            server_type = _determine_sql_server_type(data_contract, sql_server_type)
+            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
 
             model_names = list(data_contract.models.keys())
 
@@ -439,10 +419,30 @@ class DataContract:
                 return to_great_expectations(data_contract, model_name)
         if export_format == "pydantic-model":
             return to_pydantic_model_str(data_contract)
+        if export_format == "html":
+            return to_html(data_contract)
         else:
             print(f"Export format {export_format} not supported.")
             return ""
 
+    def _determine_sql_server_type(self, data_contract: DataContractSpecification, sql_server_type: str):
+        if sql_server_type == "auto":
+            if data_contract.servers is None or len(data_contract.servers) == 0:
+                raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
+
+            server_types = set([server.type for server in data_contract.servers.values()])
+            if "snowflake" in server_types:
+                return "snowflake"
+            elif "postgres" in server_types:
+                return "postgres"
+            elif "databricks" in server_types:
+                return "databricks"
+            else:
+                # default to snowflake dialect
+                return "snowflake"
+        else:
+            return sql_server_type
+
     def _get_examples_server(self, data_contract, run, tmp_dir):
         run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
         format = "json"

datacontract/engines/soda/check_soda_execute.py

@@ -1,5 +1,6 @@
 import logging
 
+from pyspark.sql import SparkSession
 from soda.scan import Scan
 
 from datacontract.engines.soda.connections.bigquery import \
@@ -19,7 +20,9 @@ from datacontract.model.data_contract_specification import \
 from datacontract.model.run import Run, Check, Log
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
+def check_soda_execute(
+    run: Run, data_contract: DataContractSpecification, server: Server, spark: SparkSession, tmp_dir
+):
     if data_contract is None:
         run.log_warn("Cannot run engine soda-core, as data contract is invalid")
         return
@@ -91,6 +94,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
     # Don't check types for csv format, as they are hard to detect
     server_type = server.type
     check_types = server.format != "json" and server.format != "csv" and server.format != "avro"
+
     sodacl_yaml_str = to_sodacl_yaml(data_contract, server_type, check_types)
     # print("sodacl_yaml_str:\n" + sodacl_yaml_str)
     scan.add_sodacl_yaml_str(sodacl_yaml_str)
@@ -106,16 +110,13 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
     for c in scan_results.get("checks"):
         check = Check(
             type="schema",
-            result="passed"
-            if c.get("outcome") == "pass"
-            else "failed"
-            if c.get("outcome") == "fail"
-            else c.get("outcome"),
+            result=to_result(c),
             reason=", ".join(c.get("outcomeReasons")),
             name=c.get("name"),
             model=c.get("table"),
             field=c.get("column"),
             engine="soda-core",
+            diagnostics=c.get("diagnostics"),
         )
         update_reason(check, c)
         run.checks.append(check)
@@ -143,6 +144,16 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         return
 
 
+def to_result(c) -> str:
+    soda_outcome = c.get("outcome")
+    if soda_outcome == "pass":
+        return "passed"
+    elif soda_outcome == "fail":
+        return "failed"
+    else:
+        return soda_outcome
+
+
 def update_reason(check, c):
     """Try to find a reason in diagnostics"""
     if check.result == "passed":

datacontract/engines/soda/connections/duckdb.py

@@ -2,6 +2,7 @@ import logging
 import os
 
 import duckdb
+from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
 
 
 def get_duckdb_connection(data_contract, server):
@@ -12,7 +13,7 @@ def get_duckdb_connection(data_contract, server):
     if server.type == "s3":
         path = server.location
     setup_s3_connection(con, server)
-    for model_name in data_contract.models:
+    for model_name, model in data_contract.models.items():
         model_path = path
         if "{model}" in model_path:
             model_path = model_path.format(model=model_name)
@@ -32,12 +33,28 @@ def get_duckdb_connection(data_contract, server):
                         CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
                         """)
         elif server.format == "csv":
-            con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_csv_auto('{model_path}', hive_partitioning=1);
-                        """)
+            columns = to_csv_types(model)
+            if columns is None:
+                con.sql(
+                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1);"""
+                )
+            else:
+                con.sql(
+                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1, columns={columns});"""
+                )
     return con
 
 
+def to_csv_types(model) -> dict:
+    if model is None:
+        return None
+    columns = {}
+    # ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
+    for field_name, field in model.fields.items():
+        columns[field_name] = convert_to_duckdb_csv_type(field)
+    return columns
+
+
 def setup_s3_connection(con, server):
     s3_region = os.getenv("DATACONTRACT_S3_REGION")
     s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")

datacontract/export/avro_converter.py

@@ -4,7 +4,7 @@ from datacontract.model.data_contract_specification import Field
 
 
 def to_avro_schema(model_name, model) -> dict:
-    return to_avro_record(model_name, model.fields, model.description)
+    return to_avro_record(model_name, model.fields, model.description, model.namespace)
 
 
 def to_avro_schema_json(model_name, model) -> str:
@@ -12,10 +12,12 @@ def to_avro_schema_json(model_name, model) -> str:
     return json.dumps(schema, indent=2, sort_keys=False)
 
 
-def to_avro_record(name, fields, description) -> dict:
+def to_avro_record(name, fields, description, namespace) -> dict:
     schema = {"type": "record", "name": name}
     if description is not None:
         schema["doc"] = description
+    if namespace is not None:
+        schema["namespace"] = namespace
     schema["fields"] = to_avro_fields(fields)
     return schema
 
@@ -35,7 +37,7 @@ def to_avro_field(field, field_name):
     return avro_field
 
 
-def to_avro_type(field: Field, field_name: str):
+def to_avro_type(field: Field, field_name: str) -> str | dict:
     if field.type is None:
         return "null"
     if field.type in ["string", "varchar", "text"]:
@@ -60,7 +62,7 @@ def to_avro_type(field: Field, field_name: str):
     elif field.type in ["time"]:
         return "long"
     elif field.type in ["object", "record", "struct"]:
-        return to_avro_record(field_name, field.fields, field.description)
+        return to_avro_record(field_name, field.fields, field.description, None)
     elif field.type in ["binary"]:
         return "bytes"
     elif field.type in ["array"]:

datacontract/export/csv_type_converter.py

@@ -0,0 +1,36 @@
+# https://duckdb.org/docs/data/csv/overview.html
+# ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
+def convert_to_duckdb_csv_type(field) -> None | str:
+    type = field.type
+    if type is None:
+        return "VARCHAR"
+    if type.lower() in ["string", "varchar", "text"]:
+        return "VARCHAR"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "TIME"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "VARCHAR"
+    if type.lower() in ["float", "double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int", "long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        # not supported in CSV
+        return "VARCHAR"
+    if type.lower() in ["bytes"]:
+        # not supported in CSV
+        return "VARCHAR"
+    if type.lower() in ["array"]:
+        return "VARCHAR"
+    if type.lower() in ["null"]:
+        return "SQLNULL"
+    return "VARCHAR"

datacontract/export/great_expectations_converter.py

@@ -58,7 +58,7 @@ def model_to_expectations(fields: Dict[str, Field]) -> List[Dict[str, Any]]:
 def add_field_expectations(field_name, field: Field, expectations: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     if field.type is not None:
         expectations.append(to_column_types_exp(field_name, field.type))
-    if field.unique is not None:
+    if field.unique:
         expectations.append(to_column_unique_exp(field_name))
     if field.maxLength is not None or field.minLength is not None:
         expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength))

datacontract/export/html_export.py

@@ -0,0 +1,66 @@
+import datetime
+import logging
+from importlib.metadata import version
+
+import pytz
+import yaml
+from jinja2 import Environment, PackageLoader, select_autoescape
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification
+
+
+def to_html(data_contract_spec: DataContractSpecification) -> str:
+    # Load templates from templates folder
+    package_loader = PackageLoader("datacontract", "templates")
+    env = Environment(
+        loader=package_loader,
+        autoescape=select_autoescape(
+            enabled_extensions="html",
+            default_for_string=True,
+        ),
+    )
+
+    # Load the required template
+    template = env.get_template("datacontract.html")
+
+    if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
+        quality_specification = data_contract_spec.quality.specification
+    elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
+        if data_contract_spec.quality.type == "great-expectations":
+            quality_specification = yaml.dump(
+                data_contract_spec.quality.specification, sort_keys=False, default_style="|"
+            )
+        else:
+            quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
+    else:
+        quality_specification = None
+
+    style_content, _, _ = package_loader.get_source(env, "style/output.css")
+
+    datacontract_yaml = data_contract_spec.to_yaml()
+
+    tz = pytz.timezone('UTC')
+    now = datetime.datetime.now(tz)
+    formatted_date = now.strftime('%d %b %Y %H:%M:%S UTC')
+    datacontract_cli_version = get_version()
+
+    # Render the template with necessary data
+    html_string = template.render(
+        datacontract=data_contract_spec,
+        quality_specification=quality_specification,
+        style=style_content,
+        datacontract_yaml=datacontract_yaml,
+        formatted_date=formatted_date,
+        datacontract_cli_version=datacontract_cli_version,
+    )
+
+    return html_string
+
+
+def get_version() -> str:
+    try:
+        return version("datacontract_cli")
+    except Exception as e:
+        logging.debug("Ignoring exception", e)
+        return ""

datacontract/export/pydantic_converter.py

@@ -1,51 +1,56 @@
-import datacontract.model.data_contract_specification as spec
-import typing
 import ast
+import typing
+
+import datacontract.model.data_contract_specification as spec
+
 
 def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:
     classdefs = [generate_model_class(model_name, model) for (model_name, model) in contract.models.items()]
-    documentation = [ast.Expr(ast.Constant(contract.info.description))] if (
-        contract.info and contract.info.description) else []
-    result = ast.Module(body=[
-        ast.Import(
-            names=[ast.Name("datetime", ctx=ast.Load()),
-                   ast.Name("typing", ctx=ast.Load()),
-                   ast.Name("pydantic", ctx=ast.Load())]),
-        *documentation,
-        *classdefs],
-        type_ignores=[])
+    documentation = (
+        [ast.Expr(ast.Constant(contract.info.description))] if (contract.info and contract.info.description) else []
+    )
+    result = ast.Module(
+        body=[
+            ast.Import(
+                names=[
+                    ast.Name("datetime", ctx=ast.Load()),
+                    ast.Name("typing", ctx=ast.Load()),
+                    ast.Name("pydantic", ctx=ast.Load()),
+                ]
+            ),
+            *documentation,
+            *classdefs,
+        ],
+        type_ignores=[],
+    )
     return ast.unparse(result)
 
+
 def optional_of(node) -> ast.Subscript:
     return ast.Subscript(
-        value=ast.Attribute(
-            ast.Name(id="typing", ctx=ast.Load()),
-            attr="Optional",
-            ctx=ast.Load()),
-        slice=node)
+        value=ast.Attribute(ast.Name(id="typing", ctx=ast.Load()), attr="Optional", ctx=ast.Load()), slice=node
+    )
+
 
 def list_of(node) -> ast.Subscript:
-    return ast.Subscript(
-        value=ast.Name(id="list", ctx=ast.Load()),
-        slice=node)
+    return ast.Subscript(value=ast.Name(id="list", ctx=ast.Load()), slice=node)
+
 
 def product_of(nodes: list[typing.Any]) -> ast.Subscript:
     return ast.Subscript(
-        value=ast.Attribute(
-            value=ast.Name(id="typing", ctx=ast.Load()),
-            attr="Product",
-            ctx=ast.Load()),
-        slice=ast.Tuple(nodes, ctx=ast.Load())
+        value=ast.Attribute(value=ast.Name(id="typing", ctx=ast.Load()), attr="Product", ctx=ast.Load()),
+        slice=ast.Tuple(nodes, ctx=ast.Load()),
     )
 
 
 type_annotation_type = typing.Union[ast.Name, ast.Attribute, ast.Constant, ast.Subscript]
 
-def constant_field_annotation(field_name: str, field: spec.Field)\
-        -> tuple[type_annotation_type,
-                 typing.Optional[ast.ClassDef]]:
+
+def constant_field_annotation(
+    field_name: str, field: spec.Field
+) -> tuple[type_annotation_type, typing.Optional[ast.ClassDef]]:
     match field.type:
-        case "string"|"text"|"varchar":
+        case "string" | "text" | "varchar":
             return (ast.Name("str", ctx=ast.Load()), None)
         case "number", "decimal", "numeric":
             # Either integer or float in specification,
@@ -58,13 +63,9 @@ def constant_field_annotation(field_name: str, field: spec.Field)\
         case "boolean":
             return (ast.Name("bool", ctx=ast.Load()), None)
         case "timestamp" | "timestamp_tz" | "timestamp_ntz":
-            return (ast.Attribute(
-                value=ast.Name(id="datetime", ctx=ast.Load()),
-                attr="datetime"), None)
+            return (ast.Attribute(value=ast.Name(id="datetime", ctx=ast.Load()), attr="datetime"), None)
         case "date":
-            return (ast.Attribute(
-                value=ast.Name(id="datetime", ctx=ast.Load()),
-                attr="date"), None)
+            return (ast.Attribute(value=ast.Name(id="datetime", ctx=ast.Load()), attr="date"), None)
         case "bytes":
             return (ast.Name("bytes", ctx=ast.Load()), None)
         case "null":
@@ -86,44 +87,35 @@ def type_annotation(field_name: str, field: spec.Field) -> tuple[type_annotation
         (annotated_type, new_classes) = constant_field_annotation(field_name, field)
         return (optional_of(annotated_type), new_classes)
 
+
 def is_simple_field(field: spec.Field) -> bool:
     return field.type not in set(["object", "record", "struct"])
 
-def field_definitions(fields: dict[str, spec.Field]) ->\
-        tuple[list[ast.Expr],
-              list[ast.ClassDef]]:
+
+def field_definitions(fields: dict[str, spec.Field]) -> tuple[list[ast.Expr], list[ast.ClassDef]]:
     annotations = []
     classes = []
-    for (field_name, field) in fields.items():
+    for field_name, field in fields.items():
         (ann, new_class) = type_annotation(field_name, field)
-        annotations.append(
-            ast.AnnAssign(
-                target=ast.Name(id=field_name, ctx=ast.Store()),
-                annotation=ann,
-                simple=1))
+        annotations.append(ast.AnnAssign(target=ast.Name(id=field_name, ctx=ast.Store()), annotation=ann, simple=1))
         if field.description and is_simple_field(field):
-            annotations.append(
-                ast.Expr(ast.Constant(field.description)))
+            annotations.append(ast.Expr(ast.Constant(field.description)))
         if new_class:
             classes.append(new_class)
     return (annotations, classes)
 
+
 def generate_field_class(field_name: str, field: spec.Field) -> ast.ClassDef:
-    assert(field.type in set(["object", "record", "struct"]))
+    assert field.type in set(["object", "record", "struct"])
     (annotated_type, new_classes) = field_definitions(field.fields)
     documentation = [ast.Expr(ast.Constant(field.description))] if field.description else []
     return ast.ClassDef(
         name=field_name,
-        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()),
-                             attr="BaseModel",
-                             ctx=ast.Load())],
-        body=[
-            *documentation,
-            *new_classes,
-            *annotated_type
-        ],
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()), attr="BaseModel", ctx=ast.Load())],
+        body=[*documentation, *new_classes, *annotated_type],
         keywords=[],
-        decorator_list=[])
+        decorator_list=[],
+    )
 
 
 def generate_model_class(name: str, model_definition: spec.Model) -> ast.ClassDef:
@@ -131,10 +123,9 @@ def generate_model_class(name: str, model_definition: spec.Model) -> ast.ClassDe
     documentation = [ast.Expr(ast.Constant(model_definition.description))] if model_definition.description else []
     result = ast.ClassDef(
         name=name.capitalize(),
-        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()),
-                             attr="BaseModel",
-                             ctx=ast.Load())],
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()), attr="BaseModel", ctx=ast.Load())],
         body=[*documentation, *nested_classes, *field_assignments],
         keywords=[],
-        decorator_list=[])
+        decorator_list=[],
+    )
     return result