datacontract-cli 0.10.8__py3-none-any.whl → 0.10.10__py3-none-any.whl

datacontract/imports/glue_importer.py

@@ -1,6 +1,7 @@
 import boto3
 from typing import List
 
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
     Model,
@@ -9,7 +10,14 @@ from datacontract.model.data_contract_specification import (
 )
 
 
-def get_glue_database(datebase_name: str):
+class GlueImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        return import_glue(data_contract_specification, source, import_args.get("glue_table"))
+
+
+def get_glue_database(database_name: str):
     """Get the details Glue database.
 
     Args:
@@ -18,31 +26,32 @@ def get_glue_database(datebase_name: str):
     Returns:
         set: catalogid and locationUri
     """
-
     glue = boto3.client("glue")
     try:
-        response = glue.get_database(Name=datebase_name)
+        response = glue.get_database(Name=database_name)
     except glue.exceptions.EntityNotFoundException:
-        print(f"Database not found {datebase_name}.")
+        print(f"Database not found {database_name}.")
         return (None, None)
     except Exception as e:
         # todo catch all
         print(f"Error: {e}")
         return (None, None)
 
-    return (response["Database"]["CatalogId"], response["Database"].get("LocationUri", "None"))
+    return (
+        response["Database"]["CatalogId"],
+        response["Database"].get("LocationUri", "None"),
+    )
 
 
 def get_glue_tables(database_name: str) -> List[str]:
     """Get the list of tables in a Glue database.
 
     Args:
-        database_name (str): glue database to request.
+        database_name (str): Glue database to request.
 
     Returns:
-        List[string]: List of table names
+        List[str]: List of table names
     """
-
     glue = boto3.client("glue")
 
     # Set the paginator
@@ -107,9 +116,21 @@ def get_glue_table_schema(database_name: str, table_name: str):
     return table_schema
 
 
-def import_glue(data_contract_specification: DataContractSpecification, source: str, table_names: List[str]):
-    """Import the schema of a Glue database."""
+def import_glue(
+    data_contract_specification: DataContractSpecification,
+    source: str,
+    table_names: List[str],
+):
+    """Import the schema of a Glue database.
 
+    Args:
+        data_contract_specification (DataContractSpecification): The data contract specification to update.
+        source (str): The name of the Glue database.
+        table_names (List[str]): List of table names to import. If None, all tables in the database are imported.
+
+    Returns:
+        DataContractSpecification: The updated data contract specification.
+    """
     catalogid, location_uri = get_glue_database(source)
 
     # something went wrong
@@ -131,21 +152,18 @@ def import_glue(data_contract_specification: DataContractSpecification, source:
 
         fields = {}
         for column in table_schema:
-            field = Field()
-            field.type = map_type_from_sql(column["Type"])
+            field = create_typed_field(column["Type"])
 
-            # hive partitons are required, but are not primary keys
+            # hive partitions are required, but are not primary keys
             if column.get("Hive"):
                 field.required = True
 
             field.description = column.get("Comment")
-
             fields[column["Name"]] = field
 
             if "decimal" in column["Type"]:
                 # Extract precision and scale from the string
                 perc_scale = column["Type"][8:-1].split(",")
-                print(perc_scale)
                 field.precision = int(perc_scale[0])
                 field.scale = int(perc_scale[1])
 
@@ -157,37 +175,102 @@ def import_glue(data_contract_specification: DataContractSpecification, source:
     return data_contract_specification
 
 
-def map_type_from_sql(sql_type: str):
+def create_typed_field(dtype: str) -> Field:
+    """Create a typed field based on the given data type.
+
+    Args:
+        dtype (str): The data type of the field.
+
+    Returns:
+        Field: The created field with the appropriate type.
+    """
+    field = Field()
+    dtype = dtype.strip().lower().replace(" ", "")
+    if dtype.startswith(("array", "struct")):
+        orig_dtype: str = dtype
+        if dtype.startswith("array"):
+            field.type = "array"
+            field.items = create_typed_field(orig_dtype[6:-1])
+        elif dtype.startswith("struct"):
+            field.type = "struct"
+            for f in split_struct(orig_dtype[7:-1]):
+                field.fields[f.split(":", 1)[0].strip()] = create_typed_field(f.split(":", 1)[1])
+    else:
+        field.type = map_type_from_sql(dtype)
+    return field
+
+
+def split_fields(s: str):
+    """Split a string of fields considering nested structures.
+
+    Args:
+        s (str): The string to split.
+
+    Yields:
+        str: The next field in the string.
+    """
+    counter: int = 0
+    last: int = 0
+    for i, x in enumerate(s):
+        if x in ("<", "("):
+            counter += 1
+        elif x in (">", ")"):
+            counter -= 1
+        elif x == "," and counter == 0:
+            yield s[last:i]
+            last = i + 1
+    yield s[last:]
+
+
+def split_struct(s: str) -> List[str]:
+    """Split a struct string into individual fields.
+
+    Args:
+        s (str): The struct string to split.
+
+    Returns:
+        List[str]: List of individual fields in the struct.
+    """
+    return list(split_fields(s=s))
+
+
+def map_type_from_sql(sql_type: str) -> str:
+    """Map an SQL type to a corresponding field type.
+
+    Args:
+        sql_type (str): The SQL type to map.
+
+    Returns:
+        str: The corresponding field type.
+    """
     if sql_type is None:
         return None
 
-    if sql_type.lower().startswith("varchar"):
+    sql_type = sql_type.lower()
+    if sql_type.startswith("varchar"):
         return "varchar"
-    if sql_type.lower().startswith("string"):
+    if sql_type.startswith("string"):
         return "string"
-    if sql_type.lower().startswith("text"):
+    if sql_type.startswith("text"):
         return "text"
-    elif sql_type.lower().startswith("byte"):
+    if sql_type.startswith("byte"):
         return "byte"
-    elif sql_type.lower().startswith("short"):
+    if sql_type.startswith("short"):
         return "short"
-    elif sql_type.lower().startswith("integer"):
+    if sql_type.startswith("integer") or sql_type.startswith("int"):
         return "integer"
-    elif sql_type.lower().startswith("long"):
+    if sql_type.startswith("long") or sql_type.startswith("bigint"):
         return "long"
-    elif sql_type.lower().startswith("bigint"):
-        return "long"
-    elif sql_type.lower().startswith("float"):
+    if sql_type.startswith("float"):
         return "float"
-    elif sql_type.lower().startswith("double"):
+    if sql_type.startswith("double"):
         return "double"
-    elif sql_type.lower().startswith("boolean"):
+    if sql_type.startswith("boolean"):
         return "boolean"
-    elif sql_type.lower().startswith("timestamp"):
+    if sql_type.startswith("timestamp"):
         return "timestamp"
-    elif sql_type.lower().startswith("date"):
+    if sql_type.startswith("date"):
         return "date"
-    elif sql_type.lower().startswith("decimal"):
+    if sql_type.startswith("decimal"):
         return "decimal"
-    else:
-        return "variant"
+    return "variant"

datacontract/imports/importer.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+
+from datacontract.model.data_contract_specification import DataContractSpecification
+
+
+class Importer(ABC):
+    def __init__(self, import_format) -> None:
+        self.import_format = import_format
+
+    @abstractmethod
+    def import_source(
+        self,
+        data_contract_specification: DataContractSpecification,
+        source: str,
+        import_args: dict,
+    ) -> dict:
+        pass
+
+
+class ImportFormat(str, Enum):
+    sql = "sql"
+    avro = "avro"
+    dbt = "dbt"
+    glue = "glue"
+    jsonschema = "jsonschema"
+    bigquery = "bigquery"
+    odcs = "odcs"
+    unity = "unity"
+    spark = "spark"
+
+    @classmethod
+    def get_suported_formats(cls):
+        return list(map(lambda c: c.value, cls))

datacontract/imports/importer_factory.py

@@ -0,0 +1,90 @@
+import importlib.util
+import sys
+from datacontract.imports.importer import ImportFormat, Importer
+
+
+class ImporterFactory:
+    def __init__(self):
+        self.dict_importer = {}
+        self.dict_lazy_importer = {}
+
+    def register_importer(self, name, importer: Importer):
+        self.dict_importer.update({name: importer})
+
+    def register_lazy_importer(self, name: str, module_path: str, class_name: str):
+        self.dict_lazy_importer.update({name: (module_path, class_name)})
+
+    def create(self, name) -> Importer:
+        importers = self.dict_importer.copy()
+        importers.update(self.dict_lazy_importer.copy())
+        if name not in importers.keys():
+            raise ValueError(f"The '{name}' format is not supported.")
+        importer_class = importers[name]
+        if type(importers[name]) is tuple:
+            importer_class = load_module_class(module_path=importers[name][0], class_name=importers[name][1])
+        if not importer_class:
+            raise ValueError(f"Module {name} could not be loaded.")
+        return importer_class(name)
+
+
+def import_module(module_path):
+    if importlib.util.find_spec(module_path) is not None:
+        try:
+            module = importlib.import_module(module_path)
+        except ModuleNotFoundError:
+            return None
+        sys.modules[module_path] = module
+        return module
+
+
+def load_module_class(module_path, class_name):
+    module = import_module(module_path)
+    if not module:
+        return None
+    return getattr(module, class_name)
+
+
+importer_factory = ImporterFactory()
+importer_factory.register_lazy_importer(
+    name=ImportFormat.avro,
+    module_path="datacontract.imports.avro_importer",
+    class_name="AvroImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.bigquery,
+    module_path="datacontract.imports.bigquery_importer",
+    class_name="BigQueryImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.glue,
+    module_path="datacontract.imports.glue_importer",
+    class_name="GlueImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.jsonschema,
+    module_path="datacontract.imports.jsonschema_importer",
+    class_name="JsonSchemaImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.odcs,
+    module_path="datacontract.imports.odcs_importer",
+    class_name="OdcsImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.sql,
+    module_path="datacontract.imports.sql_importer",
+    class_name="SqlImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.unity,
+    module_path="datacontract.imports.unity_importer",
+    class_name="UnityImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.spark,
+    module_path="datacontract.imports.spark_importer",
+    class_name="SparkImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.dbt, module_path="datacontract.imports.dbt_importer", class_name="DbtManifestImporter"
+)

datacontract/imports/jsonschema_importer.py

@@ -2,10 +2,18 @@ import json
 
 import fastjsonschema
 
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Definition
 from datacontract.model.exceptions import DataContractException
 
 
+class JsonSchemaImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        return import_jsonschema(data_contract_specification, source)
+
+
 def convert_json_schema_properties(properties, is_definition=False):
     fields = {}
     for field_name, field_schema in properties.items():
@@ -56,11 +64,14 @@ def convert_json_schema_properties(properties, is_definition=False):
                 case "tags":
                     field_kwargs["tags"] = value
                 case "properties":
-                    field_kwargs["fields"] = convert_json_schema_properties(value)
+                    field_kwargs["fields"] = convert_json_schema_properties(value, is_definition=is_definition)
                 case "items":
-                    field_kwargs["items"] = convert_json_schema_properties(value)
+                    field_kwargs["items"] = convert_json_schema_properties(value, is_definition=is_definition)
 
-        field = Field(**field_kwargs)
+        if is_definition:
+            field = Definition(**field_kwargs)
+        else:
+            field = Field(**field_kwargs)
         fields[field_name] = field
 
     return fields

datacontract/imports/odcs_importer.py

@@ -2,6 +2,7 @@ import datetime
 import logging
 from typing import Any, Dict, List
 import yaml
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     Availability,
     Contact,
@@ -42,6 +43,13 @@ DATACONTRACT_TYPES = [
 ]
 
 
+class OdcsImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        return import_odcs(data_contract_specification, source)
+
+
 def import_odcs(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
     try:
         with open(source, "r") as file:

datacontract/imports/spark_importer.py

@@ -0,0 +1,134 @@
+from pyspark.sql import DataFrame, SparkSession, types
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+    Field,
+    Server,
+)
+
+
+class SparkImporter(Importer):
+    def import_source(
+        self,
+        data_contract_specification: DataContractSpecification,
+        source: str,
+        import_args: dict,
+    ) -> dict:
+        """
+        Imports data from a Spark source into the data contract specification.
+
+        Args:
+            data_contract_specification: The data contract specification object.
+            source: The source string indicating the Spark tables to read.
+            import_args: Additional arguments for the import process.
+
+        Returns:
+            dict: The updated data contract specification.
+        """
+        return import_spark(data_contract_specification, source)
+
+
+def import_spark(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    """
+    Reads Spark tables and updates the data contract specification with their schemas.
+
+    Args:
+        data_contract_specification: The data contract specification to update.
+        source: A comma-separated string of Spark temporary views to read.
+
+    Returns:
+        DataContractSpecification: The updated data contract specification.
+    """
+    spark = SparkSession.builder.getOrCreate()
+    data_contract_specification.servers["local"] = Server(type="dataframe")
+    for temp_view in source.split(","):
+        temp_view = temp_view.strip()
+        df = spark.read.table(temp_view)
+        data_contract_specification.models[temp_view] = import_from_spark_df(df)
+    return data_contract_specification
+
+
+def import_from_spark_df(df: DataFrame) -> Model:
+    """
+    Converts a Spark DataFrame into a Model.
+
+    Args:
+        df: The Spark DataFrame to convert.
+
+    Returns:
+        Model: The generated data contract model.
+    """
+    model = Model()
+    schema = df.schema
+
+    for field in schema:
+        model.fields[field.name] = _field_from_spark(field)
+
+    return model
+
+
+def _field_from_spark(spark_field: types.StructField) -> Field:
+    """
+    Converts a Spark StructField into a Field object for the data contract.
+
+    Args:
+        spark_field: The Spark StructField to convert.
+
+    Returns:
+        Field: The corresponding Field object.
+    """
+    field_type = _data_type_from_spark(spark_field.dataType)
+    field = Field()
+    field.type = field_type
+    field.required = not spark_field.nullable
+
+    if field_type == "array":
+        field.items = _field_from_spark(spark_field.dataType.elementType)
+
+    if field_type == "struct":
+        field.fields = {sf.name: _field_from_spark(sf) for sf in spark_field.dataType.fields}
+
+    return field
+
+
+def _data_type_from_spark(spark_type: types.DataType) -> str:
+    """
+    Maps Spark data types to the Data Contract type system.
+
+    Args:
+        spark_type: The Spark data type to map.
+
+    Returns:
+        str: The corresponding Data Contract type.
+    """
+    if isinstance(spark_type, types.StringType):
+        return "string"
+    elif isinstance(spark_type, types.IntegerType):
+        return "integer"
+    elif isinstance(spark_type, types.LongType):
+        return "long"
+    elif isinstance(spark_type, types.FloatType):
+        return "float"
+    elif isinstance(spark_type, types.DoubleType):
+        return "double"
+    elif isinstance(spark_type, types.StructType):
+        return "struct"
+    elif isinstance(spark_type, types.ArrayType):
+        return "array"
+    elif isinstance(spark_type, types.TimestampType):
+        return "timestamp"
+    elif isinstance(spark_type, types.TimestampNTZType):
+        return "timestamp_ntz"
+    elif isinstance(spark_type, types.DateType):
+        return "date"
+    elif isinstance(spark_type, types.BooleanType):
+        return "boolean"
+    elif isinstance(spark_type, types.BinaryType):
+        return "bytes"
+    elif isinstance(spark_type, types.DecimalType):
+        return "decimal"
+    elif isinstance(spark_type, types.NullType):
+        return "null"
+    else:
+        raise ValueError(f"Unsupported Spark type: {spark_type}")

datacontract/imports/sql_importer.py

@@ -1,8 +1,16 @@
 from simple_ddl_parser import parse_from_file
 
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 
 
+class SqlImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        return import_sql(data_contract_specification, self.import_format, source)
+
+
 def import_sql(data_contract_specification: DataContractSpecification, format: str, source: str):
     ddl = parse_from_file(source, group_by_type=True)
     tables = ddl["tables"]

datacontract/imports/unity_importer.py

@@ -3,9 +3,24 @@ import requests
 import os
 import typing
 
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
+
+class UnityImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        if source is not None:
+            data_contract_specification = import_unity_from_json(data_contract_specification, source)
+        else:
+            data_contract_specification = import_unity_from_api(
+                data_contract_specification, import_args.get("unity_table_full_name")
+            )
+        return data_contract_specification
+
+
 def import_unity_from_json(
     data_contract_specification: DataContractSpecification, source: str
 ) -> DataContractSpecification:
@@ -22,23 +37,21 @@ def import_unity_from_json(
         )
     return convert_unity_schema(data_contract_specification, unity_schema)
 
+
 def import_unity_from_api(
-    data_contract_specification: DataContractSpecification,
-    unity_table_full_name: typing.Optional[str] = None
+    data_contract_specification: DataContractSpecification, unity_table_full_name: typing.Optional[str] = None
 ) -> DataContractSpecification:
-    databricks_instance = os.getenv('DATABRICKS_IMPORT_INSTANCE')
-    access_token = os.getenv('DATABRICKS_IMPORT_ACCESS_TOKEN')
+    databricks_instance = os.getenv("DATABRICKS_IMPORT_INSTANCE")
+    access_token = os.getenv("DATABRICKS_IMPORT_ACCESS_TOKEN")
 
     if not databricks_instance or not access_token:
         print("Missing environment variables for Databricks instance or access token.")
         print("Both, $DATABRICKS_IMPORT_INSTANCE and $DATABRICKS_IMPORT_ACCESS_TOKEN must be set.")
         exit(1)  # Exit if variables are not set
 
-    api_url = f'{databricks_instance}/api/2.1/unity-catalog/tables/{unity_table_full_name}'
+    api_url = f"{databricks_instance}/api/2.1/unity-catalog/tables/{unity_table_full_name}"
 
-    headers = {
-        'Authorization': f'Bearer {access_token}'
-    }
+    headers = {"Authorization": f"Bearer {access_token}"}
     response = requests.get(api_url, headers=headers)
 
     if response.status_code != 200:
@@ -46,13 +59,14 @@ def import_unity_from_api(
             type="schema",
             name="Retrieve unity catalog schema",
             reason=f"Failed to retrieve unity catalog schema from databricks instance: {response.status_code} {response.text}",
-            engine="datacontract"
+            engine="datacontract",
         )
 
     convert_unity_schema(data_contract_specification, response.json())
 
     return data_contract_specification
 
+
 def convert_unity_schema(
     data_contract_specification: DataContractSpecification, unity_schema: dict
 ) -> DataContractSpecification:

datacontract/integration/publish_datamesh_manager.py

@@ -8,18 +8,23 @@ from datacontract.model.run import Run
 def publish_datamesh_manager(run: Run, publish_url: str):
     try:
         if publish_url is None:
-            url = "https://api.datamesh-manager.com/api/runs"
+            # this url supports Data Mesh Manager and Data Contract Manager
+            url = "https://api.datamesh-manager.com/api/test-results"
         else:
             url = publish_url
-        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        if api_key is None:
+            api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
 
         if run.dataContractId is None:
             raise Exception("Cannot publish run results, as data contract ID is unknown")
 
-        if datamesh_manager_api_key is None:
-            raise Exception("Cannot publish run results, as DATAMESH_MANAGER_API_KEY is not set")
+        if api_key is None:
+            raise Exception(
+                "Cannot publish run results, as DATAMESH_MANAGER_API_KEY nor DATACONTRACT_MANAGER_API_KEY are not set"
+            )
 
-        headers = {"Content-Type": "application/json", "x-api-key": datamesh_manager_api_key}
+        headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
         response = requests.post(url, data=request_body, headers=headers)