datacontract-cli 0.10.3__py3-none-any.whl → 0.10.4__py3-none-any.whl

datacontract/export/jsonschema_converter.py

@@ -18,12 +18,19 @@ def to_jsonschema_json(model_key, model_value: Model) -> str:
 
 
 def to_jsonschema(model_key, model_value: Model) -> dict:
-    return {
+
+    model =  {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "type": "object",
         "properties": to_properties(model_value.fields),
-        "required": to_required(model_value.fields),
+        "required": to_required(model_value.fields)
     }
+    if model_value.title:
+        model["title"] = model_value.title
+    if model_value.description:
+        model["description"] = model_value.description
+
+    return model
 
 
 def to_properties(fields: Dict[str, Field]) -> dict:
@@ -46,13 +53,46 @@ def to_property(field: Field) -> dict:
     if field.unique:
         property["unique"] = True
     if json_type == "object":
-        property["properties"] = to_properties(field.fields)
+        # TODO: any better idea to distinguish between properties and patternProperties?
+        if next(iter(field.fields.keys())).startswith("^"):
+            property["patternProperties"] = to_properties(field.fields)
+        else:
+            property["properties"] = to_properties(field.fields)
         property["required"] = to_required(field.fields)
-
+    if json_type == "array":
+        property["items"] = to_property(field.items)
+
+    if field.pattern:
+        property["pattern"] = field.pattern
+    if field.enum:
+        property["enum"] = field.enum
+    if field.minLength:
+        property["minLength"] = field.minLength
+    if field.maxLength:
+        property["maxLength"] = field.maxLength
+    if field.title:
+        property["title"] = field.title
+    if field.description:
+        property["description"] = field.description
+    if field.exclusiveMinimum:
+        property["exclusiveMinimum"] = field.exclusiveMinimum
+    if field.exclusiveMaximum:
+        property["exclusiveMaximum"] = field.exclusiveMaximum
+    if field.minimum:
+        property["minimum"] = field.minimum
+    if field.maximum:
+        property["maximum"] = field.maximum
+    if field.tags:
+        property["tags"] = field.tags
+    if field.pii:
+        property["pii"] = field.pii
+    if field.classification:
+        property["classification"] = field.classification
+
+    
     # TODO: all constraints
     return property
 
-
 def to_required(fields: Dict[str, Field]):
     required = []
     for field_name, field in fields.items():

datacontract/export/sql_converter.py

@@ -63,6 +63,7 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
     result = ""
     result += f"-- Data Contract: {data_contract_spec.id}\n"
     result += f"-- SQL Dialect: {server_type}\n"
+
     for model_name, model in iter(data_contract_spec.models.items()):
         result += _to_sql_table(table_prefix + model_name, model, server_type)
 

datacontract/export/sql_type_converter.py

@@ -8,12 +8,17 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
         return convert_type_to_postgres(field)
     if server_type == "databricks":
         return convert_to_databricks(field)
+    if server_type == "local" or server_type == "s3":
+        return convert_to_duckdb(field)
     return field.type
 
 
 # snowflake data types:
 # https://docs.snowflake.com/en/sql-reference/data-types.html
-def convert_to_snowflake(field) -> None | str:
+def convert_to_snowflake(field: Field) -> None | str:
+    if field.config and field.config["snowflakeType"] is not None:
+        return field.config["snowflakeType"]
+
     type = field.type
     # currently optimized for snowflake
     # LEARNING: data contract has no direct support for CHAR,CHARACTER
@@ -129,3 +134,39 @@ def convert_to_databricks(field) -> None | str:
     if type.lower() in ["array"]:
         return "ARRAY"
     return None
+
+
+def convert_to_duckdb(field) -> None | str:
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "VARCHAR" # aliases: VARCHAR, CHAR, BPCHAR, STRING, TEXT, VARCHAR(n)	STRING(n), TEXT(n)
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP WITH TIME ZONE" # aliases: TIMESTAMPTZ
+    if type.lower() in ["timestamp_ntz"]:
+        return "DATETIME"  # timestamp with microsecond precision (ignores time zone), aliases: TIMESTAMP
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "TIME" # TIME WITHOUT TIME ZONE
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "DECIMAL"
+    if type.lower() in ["float"]:
+        return "FLOAT"
+    if type.lower() in ["double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int"]:
+        return "INT"
+    if type.lower() in ["long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        return "STRUCT"
+    if type.lower() in ["bytes"]:
+        return "BLOB"
+    if type.lower() in ["array"]:
+        return "ARRAY"
+    return None

datacontract/imports/avro_importer.py

@@ -1,6 +1,7 @@
 import avro.schema
 
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
 
@@ -56,6 +57,9 @@ def import_record_fields(record_fields):
             imported_fields[field.name].type = type
             if type == "record":
                 imported_fields[field.name].fields = import_record_fields(get_record_from_union_field(field).fields)
+            elif type == "array":
+                imported_fields[field.name].type = "array"
+                imported_fields[field.name].items = import_avro_array_items(get_array_from_union_field(field))
         elif field.type.type == "array":
             imported_fields[field.name].type = "array"
             imported_fields[field.name].items = import_avro_array_items(field.type)
@@ -102,6 +106,13 @@ def get_record_from_union_field(field):
     return None
 
 
+def get_array_from_union_field(field):
+    for field_type in field.type.schemas:
+        if field_type.type == "array":
+            return field_type
+    return None
+
+
 def map_type_from_avro(avro_type_str: str):
     # TODO: ambiguous mapping in the export
     if avro_type_str == "null":
@@ -120,6 +131,8 @@ def map_type_from_avro(avro_type_str: str):
         return "boolean"
     elif avro_type_str == "record":
         return "record"
+    elif avro_type_str == "array":
+        return "array"
     else:
         raise DataContractException(
             type="schema",

datacontract/imports/bigquery_importer.py

@@ -0,0 +1,166 @@
+import json
+
+from typing import List
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Model, Field
+from datacontract.model.exceptions import DataContractException
+
+from google.cloud import bigquery
+
+def import_bigquery_from_json(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    try:
+        with open(source, "r") as file:
+            bigquery_schema = json.loads(file.read())
+    except json.JSONDecodeError as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse bigquery schema",
+            reason=f"Failed to parse bigquery schema from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+    return convert_bigquery_schema(data_contract_specification, bigquery_schema)
+
+def import_bigquery_from_api(data_contract_specification: DataContractSpecification, bigquery_tables: List[str], bigquery_project: str, bigquery_dataset: str) -> DataContractSpecification:
+    client = bigquery.Client(project=bigquery_project)
+
+    if bigquery_tables is None:
+        bigquery_tables = fetch_table_names(client, bigquery_dataset)
+
+    for table in bigquery_tables:
+        try:
+            api_table = client.get_table("{}.{}.{}".format(bigquery_project, bigquery_dataset, table))
+
+        except ValueError as e:
+             raise DataContractException(
+                type="schema",
+                result="failed",
+                name="Invalid table name for bigquery API",
+                reason=f"Tablename {table} is invalid for the bigquery API",
+                original_exception=e,
+                engine="datacontract",
+             )
+
+        if api_table is None:
+            raise DataContractException(
+                type="request",
+                result="failed",
+                name="Query bigtable Schema from API",
+                reason=f"Table {table} bnot found on bigtable schema Project {bigquery_project}, dataset {bigquery_dataset}.",
+                engine="datacontract",
+            )
+
+        convert_bigquery_schema(data_contract_specification, api_table.to_api_repr())
+
+    return data_contract_specification
+
+def fetch_table_names(client: bigquery.Client, dataset: str) -> List[str]:
+    table_names = []
+    api_tables = client.list_tables(dataset)
+    for api_table in api_tables:
+        table_names.append(api_table.table_id)
+
+    return table_names
+
+def convert_bigquery_schema(data_contract_specification: DataContractSpecification, bigquery_schema: dict) -> DataContractSpecification:
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    fields = import_table_fields(bigquery_schema.get("schema").get("fields"))
+
+    # Looking at actual export data, I guess this is always set and friendlyName isn't, though I couldn't say
+    # what exactly leads to friendlyName being set
+    table_id = bigquery_schema.get("tableReference").get("tableId")
+
+    data_contract_specification.models[table_id] = Model(
+        fields=fields,
+        type='table'
+    )
+
+    # Copy the description, if it exists
+    if bigquery_schema.get("description") is not None:
+        data_contract_specification.models[table_id].description = bigquery_schema.get("description")
+
+    # Set the title from friendlyName if it exists
+    if bigquery_schema.get("friendlyName") is not None:
+        data_contract_specification.models[table_id].title = bigquery_schema.get("friendlyName")
+
+    return data_contract_specification
+
+def import_table_fields(table_fields):
+    imported_fields = {}
+    for field in table_fields:
+        field_name = field.get("name")
+        imported_fields[field_name] = Field()
+        imported_fields[field_name].required = field.get("mode") == "REQUIRED"
+        imported_fields[field_name].description = field.get("description")
+        
+        if field.get("type") == "RECORD":
+            imported_fields[field_name].type = "object"
+            imported_fields[field_name].fields = import_table_fields(field.get("fields"))
+        elif field.get("type") == "STRUCT":
+            imported_fields[field_name].type = "struct"
+            imported_fields[field_name].fields = import_table_fields(field.get("fields"))
+        elif field.get("type") == "RANGE":
+            # This is a range of date/datetime/timestamp but multiple values
+            # So we map it to an array
+            imported_fields[field_name].type = "array"
+            imported_fields[field_name].items = Field(type = map_type_from_bigquery(field["rangeElementType"].get("type")))
+        else:  # primitive type
+            imported_fields[field_name].type = map_type_from_bigquery(field.get("type"))
+
+        if field.get("type") == "STRING":
+            # in bigquery both string and bytes have maxLength but in the datacontracts
+            # spec it is only valid for strings
+            if field.get("maxLength") is not None:
+                imported_fields[field_name].maxLength = int(field.get("maxLength"))
+        
+        if field.get("type") == "NUMERIC" or field.get("type") == "BIGNUMERIC":
+            if field.get("precision") is not None:
+                imported_fields[field_name].precision = int(field.get("precision"))
+
+            if field.get("scale") is not None:
+                imported_fields[field_name].scale = int(field.get("scale"))
+
+    return imported_fields
+
+def map_type_from_bigquery(bigquery_type_str: str):
+    if bigquery_type_str == "STRING":
+        return "string"
+    elif bigquery_type_str == "BYTES":
+        return "bytes"
+    elif bigquery_type_str == "INTEGER":
+        return "int"
+    elif bigquery_type_str == "INT64":
+        return "bigint"
+    elif bigquery_type_str == "FLOAT":
+        return "float"
+    elif bigquery_type_str == "FLOAT64":
+        return "double"
+    elif bigquery_type_str == "BOOLEAN" or bigquery_type_str == "BOOL":
+        return "boolean"
+    elif bigquery_type_str == "TIMESTAMP":
+        return "timestamp"
+    elif bigquery_type_str == "DATE":
+        return "date"
+    elif bigquery_type_str == "TIME":
+        return "timestamp_ntz"
+    elif bigquery_type_str == "DATETIME":
+        return "timestamp"
+    elif bigquery_type_str == "NUMERIC":
+        return "numeric"
+    elif bigquery_type_str == "BIGNUMERIC":
+        return "double"
+    elif bigquery_type_str == "GEOGRAPHY":
+        return "object"
+    elif bigquery_type_str == "JSON":
+        return "object"
+    else:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="Map bigquery type to data contract type",
+            reason=f"Unsupported type {bigquery_type_str} in bigquery json definition.",
+            engine="datacontract",
+        )

datacontract/imports/jsonschema_importer.py

@@ -0,0 +1,150 @@
+import json
+
+import fastjsonschema
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Model, Field, Definition
+from datacontract.model.exceptions import DataContractException
+
+
+def convert_json_schema_properties(properties, is_definition=False):
+    fields = {}
+    for field_name, field_schema in properties.items():
+        field_kwargs = {}
+        field_type = field_schema.get('type')
+
+        # Determine if the field is required and set the type to the non-null option if applicable
+        if isinstance(field_type, list) and 'null' in field_type:
+            field_kwargs['required'] = False
+            non_null_types = [t for t in field_type if t != 'null']
+            if non_null_types:
+                field_type = non_null_types[0]
+            else:
+                field_type = None
+        else:
+            field_kwargs['required'] = True
+
+        # Set the non-null type
+        if field_type:
+            field_kwargs['type'] = field_type
+
+        for key, value in field_schema.items():
+            match key:
+                case 'title':
+                    field_kwargs['title'] = value
+                case 'type':
+                    pass  # type is already handled above
+                case 'format':
+                    field_kwargs['format'] = value
+                case 'description':
+                    field_kwargs['description'] = value
+                case 'pattern':
+                    field_kwargs['pattern'] = value
+                case 'minLength':
+                    field_kwargs['minLength'] = value
+                case 'maxLength':
+                    field_kwargs['maxLength'] = value
+                case 'minimum':
+                    field_kwargs['minimum'] = value
+                case 'exclusiveMinimum':
+                    field_kwargs['exclusiveMinimum'] = value
+                case 'maximum':
+                    field_kwargs['maximum'] = value
+                case 'exclusiveMaximum':
+                    field_kwargs['exclusiveMaximum'] = value
+                case 'enum':
+                    field_kwargs['enum'] = value
+                case 'tags':
+                    field_kwargs['tags'] = value
+                case 'properties':
+                    field_kwargs['fields'] = convert_json_schema_properties(value)
+                case 'items':
+                    field_kwargs['items'] = convert_json_schema_properties(value)
+
+        field = Field(**field_kwargs)
+        fields[field_name] = field
+
+    return fields
+
+
+def import_jsonschema(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    try:
+        with open(source, "r") as file:
+            json_schema = json.loads(file.read())
+            validator = fastjsonschema.compile({})
+            validator(json_schema)
+
+            model = Model(
+                description=json_schema.get('description'),
+                type=json_schema.get('type'),
+                title=json_schema.get('title'),
+                fields=convert_json_schema_properties(json_schema.get('properties', {}))
+            )
+            data_contract_specification.models[json_schema.get('title', 'default_model')] = model
+            
+            if 'definitions' in json_schema:
+                for def_name, def_schema in json_schema['definitions'].items():
+                    definition_kwargs = {}
+                    
+                    for key, value in def_schema.items():
+                        match key:
+                            case 'domain':
+                                definition_kwargs['domain'] = value
+                            case 'title':
+                                definition_kwargs['title'] = value
+                            case 'description':
+                                definition_kwargs['description'] = value
+                            case 'type':
+                                definition_kwargs['type'] = value
+                            case 'enum':
+                                definition_kwargs['enum'] = value
+                            case 'format':
+                                definition_kwargs['format'] = value
+                            case 'minLength':
+                                definition_kwargs['minLength'] = value
+                            case 'maxLength':
+                                definition_kwargs['maxLength'] = value
+                            case 'pattern':
+                                definition_kwargs['pattern'] = value
+                            case 'minimum':
+                                definition_kwargs['minimum'] = value
+                            case 'exclusiveMinimum':
+                                definition_kwargs['exclusiveMinimum'] = value
+                            case 'maximum':
+                                definition_kwargs['maximum'] = value
+                            case 'exclusiveMaximum':
+                                definition_kwargs['exclusiveMaximum'] = value
+                            case 'pii':
+                                definition_kwargs['pii'] = value
+                            case 'classification':
+                                definition_kwargs['classification'] = value
+                            case 'tags':
+                                definition_kwargs['tags'] = value
+                            case 'properties':
+                                definition_kwargs['fields'] = convert_json_schema_properties(value, is_definition=True)
+
+                    definition = Definition(name=def_name, **definition_kwargs)
+                    data_contract_specification.definitions[def_name] = definition
+
+    except fastjsonschema.JsonSchemaException as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse json schema",
+            reason=f"Failed to parse json schema from {source}: {e}",
+            engine="datacontract"
+        )
+
+    except Exception as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse json schema",
+            reason=f"Failed to parse json schema from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+
+    return data_contract_specification

datacontract/model/data_contract_specification.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Dict
+from typing import List, Dict, Optional, Any
 
 import pydantic as pyd
 import yaml
@@ -58,6 +58,7 @@ class Definition(pyd.BaseModel):
     pii: bool = None
     classification: str = None
     tags: List[str] = []
+    example: str = None
 
 
 class Field(pyd.BaseModel):
@@ -84,12 +85,17 @@ class Field(pyd.BaseModel):
     tags: List[str] = []
     fields: Dict[str, "Field"] = {}
     items: "Field" = None
+    precision: int = None
+    scale: int = None
+    example: str = None
+    config: Dict[str, Any] = None
 
 
 class Model(pyd.BaseModel):
     description: str = None
     type: str = None
     namespace: str = None
+    title: str = None
     fields: Dict[str, Field] = {}
 
 
@@ -113,6 +119,53 @@ class Quality(pyd.BaseModel):
     type: str = None
     specification: str | object = None
 
+class Availability(pyd.BaseModel):
+    description: Optional[str] = None
+    percentage: Optional[str] = None
+
+class Retention(pyd.BaseModel):
+    description: Optional[str] = None
+    period: Optional[str] = None
+    unlimited: Optional[bool] = None
+    timestampField: Optional[str] = None
+
+class Latency(pyd.BaseModel):
+    description: Optional[str] = None
+    threshold: Optional[str] = None
+    sourceTimestampField: Optional[str] = None
+    processedTimestampField: Optional[str] = None
+
+class Freshness(pyd.BaseModel):
+    description: Optional[str] = None
+    threshold: Optional[str] = None
+    timestampField: Optional[str] = None
+
+class Frequency(pyd.BaseModel):
+    description: Optional[str] = None
+    type: Optional[str] = None
+    interval: Optional[str] = None
+    cron: Optional[str] = None
+
+class Support(pyd.BaseModel):
+    description: Optional[str] = None
+    time: Optional[str] = None
+    responseTime: Optional[str] = None
+
+class Backup(pyd.BaseModel):
+    description: Optional[str] = None
+    interval: Optional[str] = None
+    cron: Optional[str] = None
+    recoveryTime: Optional[str] = None
+    recoveryPoint: Optional[str] = None
+
+class ServiceLevel(pyd.BaseModel):
+    availability: Optional[Availability] = None
+    retention: Optional[Retention] = None
+    latency: Optional[Latency] = None
+    freshness: Optional[Freshness] = None
+    frequency: Optional[Frequency] = None
+    support: Optional[Support] = None
+    backup: Optional[Backup] = None
 
 class DataContractSpecification(pyd.BaseModel):
     dataContractSpecification: str = None
@@ -125,6 +178,7 @@ class DataContractSpecification(pyd.BaseModel):
     # schema: Dict[str, str]
     examples: List[Example] = []
     quality: Quality = None
+    servicelevels: Optional[ServiceLevel] = None
 
     @classmethod
     def from_file(cls, file):

datacontract/publish/publish.py

@@ -0,0 +1,32 @@
+import os
+
+import requests
+
+from datacontract.data_contract import DataContract
+
+
+def publish_to_datamesh_manager(data_contract: DataContract):
+    try:
+        headers = {"Content-Type": "application/json", "x-api-key": _require_datamesh_manager_api_key()}
+        spec = data_contract.get_data_contract_specification()
+        id = spec.id
+        url = "https://api.datamesh-manager.com/api/datacontracts/{0}".format(id)
+        request_body = spec.model_dump_json().encode("utf-8")
+        response = requests.put(
+            url=url,
+            data=request_body,
+            headers=headers,
+        )
+        if response.status_code != 200:
+            print(f"Error publishing data contract to Data Mesh Manager: {response.text}")
+            exit(1)
+        print(f"Published data contract to {url}")
+    except Exception as e:
+        print(f"Failed publishing data contract. Error: {str(e)}")
+
+
+def _require_datamesh_manager_api_key():
+    datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+    if datamesh_manager_api_key is None:
+        raise Exception("Cannot publish data contract, as DATAMESH_MANAGER_API_KEY is not set")
+    return datamesh_manager_api_key