datacontract-cli 0.10.4__py3-none-any.whl → 0.10.6__py3-none-any.whl

datacontract/export/sql_type_converter.py

@@ -1,15 +1,20 @@
+from datacontract.export.bigquery_converter import map_type_to_bigquery
 from datacontract.model.data_contract_specification import Field
 
 
 def convert_to_sql_type(field: Field, server_type: str) -> str:
     if server_type == "snowflake":
         return convert_to_snowflake(field)
-    if server_type == "postgres":
+    elif server_type == "postgres":
         return convert_type_to_postgres(field)
-    if server_type == "databricks":
+    elif server_type == "databricks":
         return convert_to_databricks(field)
-    if server_type == "local" or server_type == "s3":
+    elif server_type == "local" or server_type == "s3":
         return convert_to_duckdb(field)
+    elif server_type == "sqlserver":
+        return convert_type_to_sqlserver(field)
+    elif server_type == "bigquery":
+        return convert_type_to_bigquery(field)
     return field.type
 
 
@@ -59,6 +64,9 @@ def convert_to_snowflake(field: Field) -> None | str:
 # https://www.postgresql.org/docs/current/datatype.html
 # Using the name whenever possible
 def convert_type_to_postgres(field: Field) -> None | str:
+    if field.config and field.config["postgresType"] is not None:
+        return field.config["postgresType"]
+
     type = field.type
     if type is None:
         return None
@@ -100,7 +108,9 @@ def convert_type_to_postgres(field: Field) -> None | str:
 
 # databricks data types:
 # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
-def convert_to_databricks(field) -> None | str:
+def convert_to_databricks(field: Field) -> None | str:
+    if field.config and field.config["databricksType"] is not None:
+        return field.config["databricksType"]
     type = field.type
     if type is None:
         return None
@@ -136,20 +146,20 @@ def convert_to_databricks(field) -> None | str:
     return None
 
 
-def convert_to_duckdb(field) -> None | str:
+def convert_to_duckdb(field: Field) -> None | str:
     type = field.type
     if type is None:
         return None
     if type.lower() in ["string", "varchar", "text"]:
-        return "VARCHAR" # aliases: VARCHAR, CHAR, BPCHAR, STRING, TEXT, VARCHAR(n)	STRING(n), TEXT(n)
+        return "VARCHAR"  # aliases: VARCHAR, CHAR, BPCHAR, STRING, TEXT, VARCHAR(n)	STRING(n), TEXT(n)
     if type.lower() in ["timestamp", "timestamp_tz"]:
-        return "TIMESTAMP WITH TIME ZONE" # aliases: TIMESTAMPTZ
+        return "TIMESTAMP WITH TIME ZONE"  # aliases: TIMESTAMPTZ
     if type.lower() in ["timestamp_ntz"]:
         return "DATETIME"  # timestamp with microsecond precision (ignores time zone), aliases: TIMESTAMP
     if type.lower() in ["date"]:
         return "DATE"
     if type.lower() in ["time"]:
-        return "TIME" # TIME WITHOUT TIME ZONE
+        return "TIME"  # TIME WITHOUT TIME ZONE
     if type.lower() in ["number", "decimal", "numeric"]:
         # precision and scale not supported by data contract
         return "DECIMAL"
@@ -170,3 +180,73 @@ def convert_to_duckdb(field) -> None | str:
     if type.lower() in ["array"]:
         return "ARRAY"
     return None
+
+
+def convert_type_to_sqlserver(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent sqlserver types"""
+    field_type = field.type
+    if not field_type:
+        return None
+
+    # If provided sql-server config type, prefer it over default mapping
+    if sqlserver_type := get_type_config(field, "sqlserverType"):
+        return sqlserver_type
+
+    field_type = field_type.lower()
+    if field_type in ["string", "varchar", "text"]:
+        if field.format == "uuid":
+            return "uniqueidentifier"
+        return "varchar"
+    if field_type in ["timestamp", "timestamp_tz"]:
+        return "datetimeoffset"
+    if field_type in ["timestamp_ntz"]:
+        if field.format == "datetime":
+            return "datetime"
+        return "datetime2"
+    if field_type in ["date"]:
+        return "date"
+    if field_type in ["time"]:
+        return "time"
+    if field_type in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        if field_type == "number":
+            return "numeric"
+        return field_type
+    if field_type in ["float"]:
+        return "float"
+    if field_type in ["double"]:
+        return "double precision"
+    if field_type in ["integer", "int", "bigint"]:
+        return field_type
+    if field_type in ["long"]:
+        return "bigint"
+    if field_type in ["boolean"]:
+        return "bit"
+    if field_type in ["object", "record", "struct"]:
+        return "jsonb"
+    if field_type in ["bytes"]:
+        return "binary"
+    if field_type in ["array"]:
+        raise NotImplementedError("SQLServer does not support array types.")
+    return None
+
+
+def convert_type_to_bigquery(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent bigquery types"""
+    field_type = field.type
+    if not field_type:
+        return None
+
+    # If provided sql-server config type, prefer it over default mapping
+    if bigquery_type := get_type_config(field, "bigqueryType"):
+        return bigquery_type
+
+    field_type = field_type.lower()
+    return map_type_to_bigquery(field_type, field.title)
+
+
+def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:
+    """Retrieve type configuration if provided in datacontract."""
+    if not field.config:
+        return None
+    return field.config.get(config_attr, None)

datacontract/imports/avro_importer.py

@@ -1,7 +1,6 @@
 import avro.schema
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
 
@@ -38,33 +37,48 @@ def import_avro(data_contract_specification: DataContractSpecification, source:
     return data_contract_specification
 
 
+def handle_config_avro_custom_properties(field, imported_field):
+    if field.get_prop("logicalType") is not None:
+        if imported_field.config is None:
+            imported_field.config = {}
+        imported_field.config["avroLogicalType"] = field.get_prop("logicalType")
+
+    if field.default is not None:
+        if imported_field.config is None:
+            imported_field.config = {}
+        imported_field.config["avroDefault"] = field.default
+
+
 def import_record_fields(record_fields):
     imported_fields = {}
     for field in record_fields:
-        imported_fields[field.name] = Field()
-        imported_fields[field.name].required = True
-        imported_fields[field.name].description = field.doc
-        for prop in field.other_props:
-            imported_fields[field.name].__setattr__(prop, field.other_props[prop])
+        imported_field = Field()
+        imported_field.required = True
+        imported_field.description = field.doc
 
+        handle_config_avro_custom_properties(field, imported_field)
+
+        # Determine field type and handle nested structures
         if field.type.type == "record":
-            imported_fields[field.name].type = "object"
-            imported_fields[field.name].description = field.type.doc
-            imported_fields[field.name].fields = import_record_fields(field.type.fields)
+            imported_field.type = "object"
+            imported_field.description = field.type.doc
+            imported_field.fields = import_record_fields(field.type.fields)
         elif field.type.type == "union":
-            imported_fields[field.name].required = False
+            imported_field.required = False
             type = import_type_of_optional_field(field)
-            imported_fields[field.name].type = type
+            imported_field.type = type
             if type == "record":
-                imported_fields[field.name].fields = import_record_fields(get_record_from_union_field(field).fields)
+                imported_field.fields = import_record_fields(get_record_from_union_field(field).fields)
             elif type == "array":
-                imported_fields[field.name].type = "array"
-                imported_fields[field.name].items = import_avro_array_items(get_array_from_union_field(field))
+                imported_field.type = "array"
+                imported_field.items = import_avro_array_items(get_array_from_union_field(field))
         elif field.type.type == "array":
-            imported_fields[field.name].type = "array"
-            imported_fields[field.name].items = import_avro_array_items(field.type)
+            imported_field.type = "array"
+            imported_field.items = import_avro_array_items(field.type)
         else:  # primitive type
-            imported_fields[field.name].type = map_type_from_avro(field.type.type)
+            imported_field.type = map_type_from_avro(field.type.type)
+
+        imported_fields[field.name] = imported_field
 
     return imported_fields
 

datacontract/imports/bigquery_importer.py

@@ -1,14 +1,15 @@
 import json
-
 from typing import List
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+from google.cloud import bigquery
+
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
-from google.cloud import bigquery
 
-def import_bigquery_from_json(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+def import_bigquery_from_json(
+    data_contract_specification: DataContractSpecification, source: str
+) -> DataContractSpecification:
     try:
         with open(source, "r") as file:
             bigquery_schema = json.loads(file.read())
@@ -22,7 +23,13 @@ def import_bigquery_from_json(data_contract_specification: DataContractSpecifica
         )
     return convert_bigquery_schema(data_contract_specification, bigquery_schema)
 
-def import_bigquery_from_api(data_contract_specification: DataContractSpecification, bigquery_tables: List[str], bigquery_project: str, bigquery_dataset: str) -> DataContractSpecification:
+
+def import_bigquery_from_api(
+    data_contract_specification: DataContractSpecification,
+    bigquery_tables: List[str],
+    bigquery_project: str,
+    bigquery_dataset: str,
+) -> DataContractSpecification:
     client = bigquery.Client(project=bigquery_project)
 
     if bigquery_tables is None:
@@ -33,14 +40,14 @@ def import_bigquery_from_api(data_contract_specification: DataContractSpecificat
             api_table = client.get_table("{}.{}.{}".format(bigquery_project, bigquery_dataset, table))
 
         except ValueError as e:
-             raise DataContractException(
+            raise DataContractException(
                 type="schema",
                 result="failed",
                 name="Invalid table name for bigquery API",
                 reason=f"Tablename {table} is invalid for the bigquery API",
                 original_exception=e,
                 engine="datacontract",
-             )
+            )
 
         if api_table is None:
             raise DataContractException(
@@ -55,6 +62,7 @@ def import_bigquery_from_api(data_contract_specification: DataContractSpecificat
 
     return data_contract_specification
 
+
 def fetch_table_names(client: bigquery.Client, dataset: str) -> List[str]:
     table_names = []
     api_tables = client.list_tables(dataset)
@@ -63,7 +71,10 @@ def fetch_table_names(client: bigquery.Client, dataset: str) -> List[str]:
 
     return table_names
 
-def convert_bigquery_schema(data_contract_specification: DataContractSpecification, bigquery_schema: dict) -> DataContractSpecification:
+
+def convert_bigquery_schema(
+    data_contract_specification: DataContractSpecification, bigquery_schema: dict
+) -> DataContractSpecification:
     if data_contract_specification.models is None:
         data_contract_specification.models = {}
 
@@ -73,10 +84,7 @@ def convert_bigquery_schema(data_contract_specification: DataContractSpecificati
     # what exactly leads to friendlyName being set
     table_id = bigquery_schema.get("tableReference").get("tableId")
 
-    data_contract_specification.models[table_id] = Model(
-        fields=fields,
-        type='table'
-    )
+    data_contract_specification.models[table_id] = Model(fields=fields, type="table")
 
     # Copy the description, if it exists
     if bigquery_schema.get("description") is not None:
@@ -88,6 +96,7 @@ def convert_bigquery_schema(data_contract_specification: DataContractSpecificati
 
     return data_contract_specification
 
+
 def import_table_fields(table_fields):
     imported_fields = {}
     for field in table_fields:
@@ -95,7 +104,7 @@ def import_table_fields(table_fields):
         imported_fields[field_name] = Field()
         imported_fields[field_name].required = field.get("mode") == "REQUIRED"
         imported_fields[field_name].description = field.get("description")
-        
+
         if field.get("type") == "RECORD":
             imported_fields[field_name].type = "object"
             imported_fields[field_name].fields = import_table_fields(field.get("fields"))
@@ -106,7 +115,9 @@ def import_table_fields(table_fields):
             # This is a range of date/datetime/timestamp but multiple values
             # So we map it to an array
             imported_fields[field_name].type = "array"
-            imported_fields[field_name].items = Field(type = map_type_from_bigquery(field["rangeElementType"].get("type")))
+            imported_fields[field_name].items = Field(
+                type=map_type_from_bigquery(field["rangeElementType"].get("type"))
+            )
         else:  # primitive type
             imported_fields[field_name].type = map_type_from_bigquery(field.get("type"))
 
@@ -115,7 +126,7 @@ def import_table_fields(table_fields):
             # spec it is only valid for strings
             if field.get("maxLength") is not None:
                 imported_fields[field_name].maxLength = int(field.get("maxLength"))
-        
+
         if field.get("type") == "NUMERIC" or field.get("type") == "BIGNUMERIC":
             if field.get("precision") is not None:
                 imported_fields[field_name].precision = int(field.get("precision"))
@@ -125,6 +136,7 @@ def import_table_fields(table_fields):
 
     return imported_fields
 
+
 def map_type_from_bigquery(bigquery_type_str: str):
     if bigquery_type_str == "STRING":
         return "string"

datacontract/imports/jsonschema_importer.py

@@ -2,8 +2,7 @@ import json
 
 import fastjsonschema
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field, Definition
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Definition
 from datacontract.model.exceptions import DataContractException
 
 
@@ -11,55 +10,55 @@ def convert_json_schema_properties(properties, is_definition=False):
     fields = {}
     for field_name, field_schema in properties.items():
         field_kwargs = {}
-        field_type = field_schema.get('type')
+        field_type = field_schema.get("type")
 
         # Determine if the field is required and set the type to the non-null option if applicable
-        if isinstance(field_type, list) and 'null' in field_type:
-            field_kwargs['required'] = False
-            non_null_types = [t for t in field_type if t != 'null']
+        if isinstance(field_type, list) and "null" in field_type:
+            field_kwargs["required"] = False
+            non_null_types = [t for t in field_type if t != "null"]
             if non_null_types:
                 field_type = non_null_types[0]
             else:
                 field_type = None
         else:
-            field_kwargs['required'] = True
+            field_kwargs["required"] = True
 
         # Set the non-null type
         if field_type:
-            field_kwargs['type'] = field_type
+            field_kwargs["type"] = field_type
 
         for key, value in field_schema.items():
             match key:
-                case 'title':
-                    field_kwargs['title'] = value
-                case 'type':
+                case "title":
+                    field_kwargs["title"] = value
+                case "type":
                     pass  # type is already handled above
-                case 'format':
-                    field_kwargs['format'] = value
-                case 'description':
-                    field_kwargs['description'] = value
-                case 'pattern':
-                    field_kwargs['pattern'] = value
-                case 'minLength':
-                    field_kwargs['minLength'] = value
-                case 'maxLength':
-                    field_kwargs['maxLength'] = value
-                case 'minimum':
-                    field_kwargs['minimum'] = value
-                case 'exclusiveMinimum':
-                    field_kwargs['exclusiveMinimum'] = value
-                case 'maximum':
-                    field_kwargs['maximum'] = value
-                case 'exclusiveMaximum':
-                    field_kwargs['exclusiveMaximum'] = value
-                case 'enum':
-                    field_kwargs['enum'] = value
-                case 'tags':
-                    field_kwargs['tags'] = value
-                case 'properties':
-                    field_kwargs['fields'] = convert_json_schema_properties(value)
-                case 'items':
-                    field_kwargs['items'] = convert_json_schema_properties(value)
+                case "format":
+                    field_kwargs["format"] = value
+                case "description":
+                    field_kwargs["description"] = value
+                case "pattern":
+                    field_kwargs["pattern"] = value
+                case "minLength":
+                    field_kwargs["minLength"] = value
+                case "maxLength":
+                    field_kwargs["maxLength"] = value
+                case "minimum":
+                    field_kwargs["minimum"] = value
+                case "exclusiveMinimum":
+                    field_kwargs["exclusiveMinimum"] = value
+                case "maximum":
+                    field_kwargs["maximum"] = value
+                case "exclusiveMaximum":
+                    field_kwargs["exclusiveMaximum"] = value
+                case "enum":
+                    field_kwargs["enum"] = value
+                case "tags":
+                    field_kwargs["tags"] = value
+                case "properties":
+                    field_kwargs["fields"] = convert_json_schema_properties(value)
+                case "items":
+                    field_kwargs["items"] = convert_json_schema_properties(value)
 
         field = Field(**field_kwargs)
         fields[field_name] = field
@@ -78,53 +77,53 @@ def import_jsonschema(data_contract_specification: DataContractSpecification, so
             validator(json_schema)
 
             model = Model(
-                description=json_schema.get('description'),
-                type=json_schema.get('type'),
-                title=json_schema.get('title'),
-                fields=convert_json_schema_properties(json_schema.get('properties', {}))
+                description=json_schema.get("description"),
+                type=json_schema.get("type"),
+                title=json_schema.get("title"),
+                fields=convert_json_schema_properties(json_schema.get("properties", {})),
             )
-            data_contract_specification.models[json_schema.get('title', 'default_model')] = model
-            
-            if 'definitions' in json_schema:
-                for def_name, def_schema in json_schema['definitions'].items():
+            data_contract_specification.models[json_schema.get("title", "default_model")] = model
+
+            if "definitions" in json_schema:
+                for def_name, def_schema in json_schema["definitions"].items():
                     definition_kwargs = {}
-                    
+
                     for key, value in def_schema.items():
                         match key:
-                            case 'domain':
-                                definition_kwargs['domain'] = value
-                            case 'title':
-                                definition_kwargs['title'] = value
-                            case 'description':
-                                definition_kwargs['description'] = value
-                            case 'type':
-                                definition_kwargs['type'] = value
-                            case 'enum':
-                                definition_kwargs['enum'] = value
-                            case 'format':
-                                definition_kwargs['format'] = value
-                            case 'minLength':
-                                definition_kwargs['minLength'] = value
-                            case 'maxLength':
-                                definition_kwargs['maxLength'] = value
-                            case 'pattern':
-                                definition_kwargs['pattern'] = value
-                            case 'minimum':
-                                definition_kwargs['minimum'] = value
-                            case 'exclusiveMinimum':
-                                definition_kwargs['exclusiveMinimum'] = value
-                            case 'maximum':
-                                definition_kwargs['maximum'] = value
-                            case 'exclusiveMaximum':
-                                definition_kwargs['exclusiveMaximum'] = value
-                            case 'pii':
-                                definition_kwargs['pii'] = value
-                            case 'classification':
-                                definition_kwargs['classification'] = value
-                            case 'tags':
-                                definition_kwargs['tags'] = value
-                            case 'properties':
-                                definition_kwargs['fields'] = convert_json_schema_properties(value, is_definition=True)
+                            case "domain":
+                                definition_kwargs["domain"] = value
+                            case "title":
+                                definition_kwargs["title"] = value
+                            case "description":
+                                definition_kwargs["description"] = value
+                            case "type":
+                                definition_kwargs["type"] = value
+                            case "enum":
+                                definition_kwargs["enum"] = value
+                            case "format":
+                                definition_kwargs["format"] = value
+                            case "minLength":
+                                definition_kwargs["minLength"] = value
+                            case "maxLength":
+                                definition_kwargs["maxLength"] = value
+                            case "pattern":
+                                definition_kwargs["pattern"] = value
+                            case "minimum":
+                                definition_kwargs["minimum"] = value
+                            case "exclusiveMinimum":
+                                definition_kwargs["exclusiveMinimum"] = value
+                            case "maximum":
+                                definition_kwargs["maximum"] = value
+                            case "exclusiveMaximum":
+                                definition_kwargs["exclusiveMaximum"] = value
+                            case "pii":
+                                definition_kwargs["pii"] = value
+                            case "classification":
+                                definition_kwargs["classification"] = value
+                            case "tags":
+                                definition_kwargs["tags"] = value
+                            case "properties":
+                                definition_kwargs["fields"] = convert_json_schema_properties(value, is_definition=True)
 
                     definition = Definition(name=def_name, **definition_kwargs)
                     data_contract_specification.definitions[def_name] = definition
@@ -134,7 +133,7 @@ def import_jsonschema(data_contract_specification: DataContractSpecification, so
             type="schema",
             name="Parse json schema",
             reason=f"Failed to parse json schema from {source}: {e}",
-            engine="datacontract"
+            engine="datacontract",
         )
 
     except Exception as e:
@@ -146,5 +145,4 @@ def import_jsonschema(data_contract_specification: DataContractSpecification, so
             original_exception=e,
         )
 
-
     return data_contract_specification

datacontract/imports/sql_importer.py

@@ -45,7 +45,7 @@ def map_type_from_sql(sql_type: str):
         return None
 
     sql_type_normed = sql_type.lower().strip()
-    
+
     if sql_type_normed.startswith("varchar"):
         return "varchar"
     elif sql_type_normed.startswith("string"):
@@ -69,6 +69,6 @@ def map_type_from_sql(sql_type: str):
     elif sql_type_normed == "datetime2":
         return "timestamp_ntz"
     elif sql_type_normed == "datetimeoffset":
-        return "timestamp_tz"        
+        return "timestamp_tz"
     else:
         return "variant"

datacontract/lint/resolve.py

@@ -8,8 +8,7 @@ from fastjsonschema import JsonSchemaValueException
 from datacontract.lint.files import read_file
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Definition, Quality
+from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Quality
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/model/data_contract_specification.py

@@ -31,6 +31,7 @@ class Server(pyd.BaseModel):
     token: str = None  # Use ENV variable
     dataProductId: str = None
     outputPortId: str = None
+    driver: str = None
 
 
 class Terms(pyd.BaseModel):
@@ -119,38 +120,45 @@ class Quality(pyd.BaseModel):
     type: str = None
     specification: str | object = None
 
+
 class Availability(pyd.BaseModel):
     description: Optional[str] = None
     percentage: Optional[str] = None
 
+
 class Retention(pyd.BaseModel):
     description: Optional[str] = None
     period: Optional[str] = None
     unlimited: Optional[bool] = None
     timestampField: Optional[str] = None
 
+
 class Latency(pyd.BaseModel):
     description: Optional[str] = None
     threshold: Optional[str] = None
     sourceTimestampField: Optional[str] = None
     processedTimestampField: Optional[str] = None
 
+
 class Freshness(pyd.BaseModel):
     description: Optional[str] = None
     threshold: Optional[str] = None
     timestampField: Optional[str] = None
 
+
 class Frequency(pyd.BaseModel):
     description: Optional[str] = None
     type: Optional[str] = None
     interval: Optional[str] = None
     cron: Optional[str] = None
 
+
 class Support(pyd.BaseModel):
     description: Optional[str] = None
     time: Optional[str] = None
     responseTime: Optional[str] = None
 
+
 class Backup(pyd.BaseModel):
     description: Optional[str] = None
     interval: Optional[str] = None
@@ -158,6 +166,7 @@ class Backup(pyd.BaseModel):
     recoveryTime: Optional[str] = None
     recoveryPoint: Optional[str] = None
 
+
 class ServiceLevel(pyd.BaseModel):
     availability: Optional[Availability] = None
     retention: Optional[Retention] = None
@@ -167,6 +176,7 @@ class ServiceLevel(pyd.BaseModel):
     support: Optional[Support] = None
     backup: Optional[Backup] = None
 
+
 class DataContractSpecification(pyd.BaseModel):
     dataContractSpecification: str = None
     id: str = None