datacontract-cli 0.10.14__py3-none-any.whl → 0.10.16__py3-none-any.whl

datacontract/export/pandas_type_converter.py

@@ -0,0 +1,40 @@
+"""
+Module for converting data contract field types to corresponding pandas data types.
+"""
+
+from datacontract.model.data_contract_specification import Field
+
+
+def convert_to_pandas_type(field: Field) -> str:
+    """
+    Convert a data contract field type to the equivalent pandas data type.
+
+    Parameters:
+    ----------
+    field : Field
+        A Field object containing metadata about the data type of the field.
+
+    Returns:
+    -------
+    str
+        The corresponding pandas data type as a string.
+    """
+    field_type = field.type
+
+    if field_type in ["string", "varchar", "text"]:
+        return "str"
+    if field_type in ["integer", "int"]:
+        return "int32"
+    if field_type == "long":
+        return "int64"
+    if field_type == "float":
+        return "float32"
+    if field_type in ["number", "decimal", "numeric", "double"]:
+        return "float64"
+    if field_type == "boolean":
+        return "bool"
+    if field_type in ["timestamp", "timestamp_tz", "timestamp_ntz", "date"]:
+        return "datetime64[ns]"
+    if field_type == "bytes":
+        return "object"
+    return "object"

datacontract/export/protobuf_converter.py

@@ -1,5 +1,5 @@
-from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 class ProtoBufExporter(Exporter):

datacontract/export/rdf_converter.py

@@ -1,9 +1,8 @@
 from pydantic import BaseModel
-from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
-
-from datacontract.model.data_contract_specification import DataContractSpecification
+from rdflib import RDF, BNode, Graph, Literal, Namespace, URIRef
 
 from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 class RdfExporter(Exporter):
@@ -58,8 +57,8 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     else:
         g = Graph(base=Namespace(""))
 
-    dc = Namespace("https://datacontract.com/DataContractSpecification/0.9.2/")
-    dcx = Namespace("https://datacontract.com/DataContractSpecification/0.9.2/Extension/")
+    dc = Namespace("https://datacontract.com/DataContractSpecification/1.1.0/")
+    dcx = Namespace("https://datacontract.com/DataContractSpecification/1.1.0/Extension/")
 
     g.bind("dc", dc)
     g.bind("dcx", dcx)

datacontract/export/sodacl_converter.py

@@ -62,12 +62,16 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
         if field.enum is not None and len(field.enum) > 0:
             checks.append(check_field_enum(field_name, field.enum, quote_field_name))
         if field.quality is not None and len(field.quality) > 0:
-            checks.append(check_quality_list(model_key, field_name, field.quality))
+            quality_list = check_quality_list(model_key, field_name, field.quality)
+            if (quality_list is not None) and len(quality_list) > 0:
+                checks.append(quality_list)
         # TODO references: str = None
         # TODO format
 
     if model_value.quality is not None and len(model_value.quality) > 0:
-        checks.append(check_quality_list(model_key, None, model_value.quality))
+        quality_list = check_quality_list(model_key, None, model_value.quality)
+        if (quality_list is not None) and len(quality_list) > 0:
+            checks.append(quality_list)
 
     checks_for_model_key = f"checks for {model_key}"
 
@@ -196,9 +200,9 @@ def check_quality_list(model_name, field_name, quality_list: List[Quality]):
     for quality in quality_list:
         if quality.type == "sql":
             if field_name is None:
-                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
-            else:
                 metric_name = f"{model_name}_quality_sql_{count}"
+            else:
+                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
             threshold = to_sodacl_threshold(quality)
             query = prepare_query(quality, model_name, field_name)
             if query is None:
@@ -261,6 +265,7 @@ def to_sodacl_threshold(quality: Quality) -> str | None:
     return None
 
 
+# These are deprecated root-level quality specifications, use the model-level and field-level quality fields instead
 def add_quality_checks(sodacl, data_contract_spec):
     if data_contract_spec.quality is None:
         return

datacontract/export/spark_converter.py

@@ -1,10 +1,11 @@
 from pyspark.sql import types
+
+from datacontract.export.exporter import Exporter
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
-    Model,
     Field,
+    Model,
 )
-from datacontract.export.exporter import Exporter
 
 
 class SparkExporter(Exporter):
@@ -102,11 +103,11 @@ def to_struct_field(field: Field, field_name: str) -> types.StructField:
     Returns:
         types.StructField: The corresponding Spark StructField.
     """
-    data_type = to_data_type(field)
+    data_type = to_spark_data_type(field)
     return types.StructField(name=field_name, dataType=data_type, nullable=not field.required)
 
 
-def to_data_type(field: Field) -> types.DataType:
+def to_spark_data_type(field: Field) -> types.DataType:
     """
     Convert a field to a Spark DataType.
 
@@ -120,11 +121,11 @@ def to_data_type(field: Field) -> types.DataType:
     if field_type is None or field_type in ["null"]:
         return types.NullType()
     if field_type == "array":
-        return types.ArrayType(to_data_type(field.items))
+        return types.ArrayType(to_spark_data_type(field.items))
     if field_type in ["object", "record", "struct"]:
         return types.StructType(to_struct_type(field.fields))
     if field_type == "map":
-        return types.MapType(to_data_type(field.keys), to_data_type(field.values))
+        return types.MapType(to_spark_data_type(field.keys), to_spark_data_type(field.values))
     if field_type in ["string", "varchar", "text"]:
         return types.StringType()
     if field_type in ["number", "decimal", "numeric"]:

datacontract/export/sql_converter.py

@@ -1,8 +1,7 @@
+from datacontract.export.exporter import Exporter, _check_models_for_export, _determine_sql_server_type
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model
 
-from datacontract.export.exporter import Exporter, _check_models_for_export, _determine_sql_server_type
-
 
 class SqlExporter(Exporter):
     def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:

datacontract/export/sqlalchemy_converter.py

@@ -2,8 +2,7 @@ import ast
 import typing
 
 import datacontract.model.data_contract_specification as spec
-from datacontract.export.exporter import Exporter
-from datacontract.export.exporter import _determine_sql_server_type
+from datacontract.export.exporter import Exporter, _determine_sql_server_type
 
 
 class SQLAlchemyExporter(Exporter):

datacontract/export/terraform_converter.py

@@ -1,7 +1,7 @@
 import re
 
-from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Server
 
 
 class TerraformExporter(Exporter):

datacontract/imports/avro_importer.py

@@ -3,7 +3,7 @@ from typing import Dict, List
 import avro.schema
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/imports/bigquery_importer.py

@@ -3,7 +3,7 @@ import logging
 from typing import List
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/imports/dbml_importer.py

@@ -1,11 +1,11 @@
-from pydbml import PyDBML, Database
 from typing import List
 
+from pydbml import Database, PyDBML
 from pyparsing import ParseException
 
 from datacontract.imports.importer import Importer
 from datacontract.imports.sql_importer import map_type_from_sql
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/imports/dbt_importer.py

@@ -1,10 +1,14 @@
 import json
 from typing import TypedDict
 
-from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from dbt.artifacts.resources.v1.components import ColumnInfo
 from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.graph.nodes import GenericTestNode
+from dbt_common.contracts.constraints import ConstraintType
+
+from datacontract.imports.bigquery_importer import map_type_from_bigquery
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class DBTImportArgs(TypedDict, total=False):
@@ -33,7 +37,7 @@ class DbtManifestImporter(Importer):
         return import_dbt_manifest(
             data_contract_specification=data_contract_specification,
             manifest=manifest,
-            dbt_nodes=import_args.get("dbt_nodes", []),
+            dbt_nodes=import_args.get("dbt_model", []),
             resource_types=import_args.get("resource_types", ["model"]),
         )
 
@@ -42,7 +46,9 @@ def read_dbt_manifest(manifest_path: str) -> Manifest:
     """Read a manifest from file."""
     with open(file=manifest_path, mode="r", encoding="utf-8") as f:
         manifest_dict: dict = json.load(f)
-    return Manifest.from_dict(manifest_dict)
+    manifest = Manifest.from_dict(manifest_dict)
+    manifest.build_parent_and_child_maps()
+    return manifest
 
 
 def import_dbt_manifest(
@@ -57,7 +63,7 @@ def import_dbt_manifest(
     """
     data_contract_specification.info.title = manifest.metadata.project_name
     data_contract_specification.info.dbt_version = manifest.metadata.dbt_version
-
+    adapter_type = manifest.metadata.adapter_type
     data_contract_specification.models = data_contract_specification.models or {}
     for model_contents in manifest.nodes.values():
         # Only intressted in processing models.
@@ -72,7 +78,12 @@ def import_dbt_manifest(
         dc_model = Model(
             description=model_contents.description,
             tags=model_contents.tags,
-            fields=create_fields(columns=model_contents.columns),
+            fields=create_fields(
+                manifest,
+                model_unique_id=model_contents.unique_id,
+                columns=model_contents.columns,
+                adapter_type=adapter_type,
+            ),
         )
 
         data_contract_specification.models[model_contents.name] = dc_model
@@ -80,14 +91,68 @@ def import_dbt_manifest(
     return data_contract_specification
 
 
-def create_fields(columns: dict[str, ColumnInfo]) -> dict[str, Field]:
-    fields = {
-        column.name: Field(
-            description=column.description,
-            type=column.data_type if column.data_type else "",
-            tags=column.tags,
-        )
-        for column in columns.values()
-    }
+def convert_data_type_by_adapter_type(data_type: str, adapter_type: str) -> str:
+    if adapter_type == "bigquery":
+        return map_type_from_bigquery(data_type)
+    return data_type
 
+
+def create_fields(
+    manifest: Manifest, model_unique_id: str, columns: dict[str, ColumnInfo], adapter_type: str
+) -> dict[str, Field]:
+    fields = {column.name: create_field(manifest, model_unique_id, column, adapter_type) for column in columns.values()}
     return fields
+
+
+def get_column_tests(manifest: Manifest, model_name: str, column_name: str) -> list[dict[str, str]]:
+    column_tests = []
+    model_node = manifest.nodes.get(model_name)
+    if not model_node:
+        raise ValueError(f"Model {model_name} not found in manifest.")
+
+    model_unique_id = model_node.unique_id
+    test_ids = manifest.child_map.get(model_unique_id, [])
+
+    for test_id in test_ids:
+        test_node = manifest.nodes.get(test_id)
+        if not test_node or test_node.resource_type != "test":
+            continue
+
+        if not isinstance(test_node, GenericTestNode):
+            continue
+
+        if test_node.column_name != column_name:
+            continue
+
+        if test_node.config.where is not None:
+            continue
+
+        column_tests.append(
+            {
+                "test_name": test_node.name,
+                "test_type": test_node.test_metadata.name,
+                "column": test_node.column_name,
+            }
+        )
+    return column_tests
+
+
+def create_field(manifest: Manifest, model_unique_id: str, column: ColumnInfo, adapter_type: str) -> Field:
+    column_type = convert_data_type_by_adapter_type(column.data_type, adapter_type) if column.data_type else ""
+    field = Field(
+        description=column.description,
+        type=column_type,
+        tags=column.tags,
+    )
+
+    all_tests = get_column_tests(manifest, model_unique_id, column.name)
+
+    required = False
+    if any(constraint.type == ConstraintType.not_null for constraint in column.constraints):
+        required = True
+    if [test for test in all_tests if test["test_type"] == "not_null"]:
+        required = True
+    if required:
+        field.required = required
+
+    return field

datacontract/imports/glue_importer.py

@@ -1,11 +1,13 @@
-import boto3
-from typing import List, Dict, Generator
 import re
+from typing import Dict, Generator, List
+
+import boto3
+
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
-    Model,
     Field,
+    Model,
     Server,
 )
 

datacontract/imports/iceberg_importer.py

@@ -1,12 +1,11 @@
-from typing import Dict, Any
+from typing import Any, Dict
 
-from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
-
-from pyiceberg.schema import Schema
-from pyiceberg import types as iceberg_types
 from pydantic import ValidationError
+from pyiceberg import types as iceberg_types
+from pyiceberg.schema import Schema
 
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 
@@ -43,8 +42,19 @@ def import_iceberg(
 
     model = Model(type="table", title=table_name)
 
+    # Iceberg identifier_fields aren't technically primary keys since Iceberg doesn't support primary keys,
+    # but they are close enough that we can probably treat them as primary keys on the conversion.
+    # ref: https://iceberg.apache.org/spec/#identifier-field-ids
+    # this code WILL NOT support finding nested primary key fields.
+    identifier_fields_ids = schema.identifier_field_ids
+
     for field in schema.fields:
-        model.fields[field.name] = _field_from_nested_field(field)
+        model_field = _field_from_nested_field(field)
+
+        if field.field_id in identifier_fields_ids:
+            model_field.primaryKey = True
+
+        model.fields[field.name] = model_field
 
     data_contract_specification.models[table_name] = model
     return data_contract_specification

datacontract/imports/importer.py

@@ -30,6 +30,7 @@ class ImportFormat(str, Enum):
     unity = "unity"
     spark = "spark"
     iceberg = "iceberg"
+    parquet = "parquet"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/imports/importer_factory.py

@@ -1,6 +1,7 @@
 import importlib.util
 import sys
-from datacontract.imports.importer import ImportFormat, Importer
+
+from datacontract.imports.importer import Importer, ImportFormat
 
 
 class ImporterFactory:
@@ -98,3 +99,8 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.iceberg_importer",
     class_name="IcebergImporter",
 )
+importer_factory.register_lazy_importer(
+    name=ImportFormat.parquet,
+    module_path="datacontract.imports.parquet_importer",
+    class_name="ParquetImporter",
+)

datacontract/imports/jsonschema_importer.py

@@ -3,7 +3,7 @@ import json
 import fastjsonschema
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Definition
+from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 
@@ -111,7 +111,8 @@ def schema_to_args(property_schema, is_required: bool = None) -> dict:
     nested_properties = property_schema.get("properties")
     if nested_properties is not None:
         # recursive call for complex nested properties
-        field_kwargs["fields"] = jsonschema_to_args(nested_properties, property_schema["required"])
+        required = property_schema.get("required", [])
+        field_kwargs["fields"] = jsonschema_to_args(nested_properties, required)
 
     return field_kwargs
 

datacontract/imports/odcs_v2_importer.py

@@ -6,16 +6,16 @@ import yaml
 
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
+    DATACONTRACT_TYPES,
     Availability,
     Contact,
     DataContractSpecification,
+    Field,
     Info,
     Model,
-    Field,
     Retention,
     ServiceLevel,
     Terms,
-    DATACONTRACT_TYPES,
 )
 from datacontract.model.exceptions import DataContractException
 

datacontract/imports/odcs_v3_importer.py

@@ -8,16 +8,17 @@ import yaml
 from datacontract.imports.importer import Importer
 from datacontract.lint.resources import read_resource
 from datacontract.model.data_contract_specification import (
+    DATACONTRACT_TYPES,
     Availability,
     DataContractSpecification,
+    Field,
     Info,
     Model,
-    Field,
+    Quality,
     Retention,
     Server,
     ServiceLevel,
     Terms,
-    DATACONTRACT_TYPES,
 )
 from datacontract.model.exceptions import DataContractException
 
@@ -193,6 +194,10 @@ def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
         model.fields = import_fields(
             odcs_schema.get("properties"), custom_type_mappings, server_type=get_server_type(odcs_contract)
         )
+        if odcs_schema.get("quality") is not None:
+            # convert dict to pydantic model
+
+            model.quality = [Quality.model_validate(q) for q in odcs_schema.get("quality")]
         model.title = schema_name
         if odcs_schema.get("dataGranularityDescription") is not None:
             model.config = {"dataGranularityDescription": odcs_schema.get("dataGranularityDescription")}

datacontract/imports/parquet_importer.py

@@ -0,0 +1,81 @@
+import os.path
+
+import pyarrow
+from pyarrow import parquet
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Field,
+    Model,
+)
+from datacontract.model.exceptions import DataContractException
+
+
+class ParquetImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_parquet(data_contract_specification, source)
+
+
+def import_parquet(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    # use filename as schema name, remove .parquet suffix, avoid breaking the yaml output by replacing dots
+    schema_name = os.path.basename(source).removesuffix(".parquet").replace(".", "_")
+
+    fields: dict[str, Field] = {}
+
+    arrow_schema = parquet.read_schema(source)
+    for field_name in arrow_schema.names:
+        parquet_field = arrow_schema.field(field_name)
+
+        field = map_pyarrow_field_to_specification_field(parquet_field, "parquet")
+
+        if not parquet_field.nullable:
+            field.required = True
+
+        fields[field_name] = field
+
+    data_contract_specification.models[schema_name] = Model(fields=fields)
+
+    return data_contract_specification
+
+
+def map_pyarrow_field_to_specification_field(pyarrow_field: pyarrow.Field, file_format: str) -> Field:
+    if pyarrow.types.is_boolean(pyarrow_field.type):
+        return Field(type="boolean")
+    if pyarrow.types.is_int32(pyarrow_field.type):
+        return Field(type="int")
+    if pyarrow.types.is_int64(pyarrow_field.type):
+        return Field(type="long")
+    if pyarrow.types.is_integer(pyarrow_field.type):
+        return Field(type="number")
+    if pyarrow.types.is_float32(pyarrow_field.type):
+        return Field(type="float")
+    if pyarrow.types.is_float64(pyarrow_field.type):
+        return Field(type="double")
+    if pyarrow.types.is_decimal(pyarrow_field.type):
+        return Field(type="decimal", precision=pyarrow_field.type.precision, scale=pyarrow_field.type.scale)
+    if pyarrow.types.is_timestamp(pyarrow_field.type):
+        return Field(type="timestamp")
+    if pyarrow.types.is_date(pyarrow_field.type):
+        return Field(type="date")
+    if pyarrow.types.is_null(pyarrow_field.type):
+        return Field(type="null")
+    if pyarrow.types.is_binary(pyarrow_field.type):
+        return Field(type="bytes")
+    if pyarrow.types.is_string(pyarrow_field.type):
+        return Field(type="string")
+    if pyarrow.types.is_map(pyarrow_field.type) or pyarrow.types.is_dictionary(pyarrow_field.type):
+        return Field(type="map")
+    if pyarrow.types.is_struct(pyarrow_field.type):
+        return Field(type="struct")
+    if pyarrow.types.is_list(pyarrow_field.type):
+        return Field(type="array")
+
+    raise DataContractException(
+        type="schema",
+        name=f"Parse {file_format} schema",
+        reason=f"{pyarrow_field.type} currently not supported.",
+        engine="datacontract",
+    )

datacontract/imports/spark_importer.py

@@ -1,9 +1,10 @@
 from pyspark.sql import DataFrame, SparkSession, types
+
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
-    Model,
     Field,
+    Model,
     Server,
 )
 

datacontract/imports/sql_importer.py

@@ -1,7 +1,7 @@
 from simple_ddl_parser import parse_from_file
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class SqlImporter(Importer):

datacontract/imports/unity_importer.py

@@ -2,13 +2,13 @@ import json
 import os
 from typing import List, Optional
 
-from pyspark.sql import types
 from databricks.sdk import WorkspaceClient
-from databricks.sdk.service.catalog import TableInfo, ColumnInfo
+from databricks.sdk.service.catalog import ColumnInfo, TableInfo
+from pyspark.sql import types
 
 from datacontract.imports.importer import Importer
 from datacontract.imports.spark_importer import _field_from_struct_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/integration/opentelemetry.py

@@ -12,7 +12,6 @@ from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExpo
 
 from datacontract.model.run import Run
 
-
 # Publishes metrics of a test run.
 # Metric contains the values:
 # 0 == test run passed,

datacontract/lint/lint.py

@@ -1,9 +1,10 @@
 import abc
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Sequence, Any, cast
+from typing import Any, Sequence, cast
 
 from datacontract.model.run import Check
+
 from ..model.data_contract_specification import DataContractSpecification
 
 """This module contains linter definitions for linting a data contract.

datacontract/lint/linters/description_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult