datacontract-cli 0.10.27__py3-none-any.whl → 0.10.29__py3-none-any.whl

datacontract/imports/importer.py

@@ -12,7 +12,7 @@ class Importer(ABC):
     @abstractmethod
     def import_source(
         self,
-        data_contract_specification: DataContractSpecification,
+        data_contract_specification: DataContractSpecification | OpenDataContractStandard,
         source: str,
         import_args: dict,
     ) -> DataContractSpecification | OpenDataContractStandard:
@@ -26,6 +26,7 @@ class ImportFormat(str, Enum):
     dbml = "dbml"
     glue = "glue"
     jsonschema = "jsonschema"
+    json = "json"
     bigquery = "bigquery"
     odcs = "odcs"
     unity = "unity"
@@ -39,3 +40,12 @@ class ImportFormat(str, Enum):
     @classmethod
     def get_supported_formats(cls):
         return list(map(lambda c: c.value, cls))
+
+
+class Spec(str, Enum):
+    datacontract_specification = "datacontract_specification"
+    odcs = "odcs"
+
+    @classmethod
+    def get_supported_types(cls):
+        return list(map(lambda c: c.value, cls))

datacontract/imports/importer_factory.py

@@ -119,3 +119,10 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.excel_importer",
     class_name="ExcelImporter",
 )
+
+
+importer_factory.register_lazy_importer(
+    name=ImportFormat.json,
+    module_path="datacontract.imports.json_importer",
+    class_name="JsonImporter",
+)

datacontract/imports/json_importer.py

@@ -0,0 +1,325 @@
+import json
+import os
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Server
+
+
+class JsonImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_json(data_contract_specification, source)
+
+
+def is_ndjson(file_path: str) -> bool:
+    """Check if a file contains newline-delimited JSON."""
+    with open(file_path, "r", encoding="utf-8") as file:
+        for _ in range(5):
+            line = file.readline().strip()
+            if not line:
+                continue
+            try:
+                json.loads(line)
+                return True
+            except json.JSONDecodeError:
+                break
+    return False
+
+
+def import_json(
+    data_contract_specification: DataContractSpecification, source: str, include_examples: bool = False
+) -> DataContractSpecification:
+    # use the file name as base model name
+    base_model_name = os.path.splitext(os.path.basename(source))[0]
+
+    # check if file is newline-delimited JSON
+    if is_ndjson(source):
+        # load NDJSON data
+        json_data = []
+        with open(source, "r", encoding="utf-8") as file:
+            for line in file:
+                line = line.strip()
+                if line:
+                    try:
+                        json_data.append(json.loads(line))
+                    except json.JSONDecodeError:
+                        continue
+    else:
+        # load regular JSON data
+        with open(source, "r", encoding="utf-8") as file:
+            json_data = json.load(file)
+
+    if data_contract_specification.servers is None:
+        data_contract_specification.servers = {}
+
+    data_contract_specification.servers["production"] = Server(type="local", path=source, format="json")
+
+    # initialisation
+    models = {}
+
+    if isinstance(json_data, list) and json_data:
+        # Array of items
+        if all(isinstance(item, dict) for item in json_data[:5]):
+            # Array of objects, as table
+            fields = {}
+            for item in json_data[:20]:
+                for key, value in item.items():
+                    field_def = generate_field_definition(value, key, base_model_name, models)
+                    if key in fields:
+                        fields[key] = merge_field_definitions(fields[key], field_def)
+                    else:
+                        fields[key] = field_def
+
+            models[base_model_name] = {
+                "type": "table",
+                "description": f"Generated from JSON array in {source}",
+                "fields": fields,
+                "examples": json_data[:3] if include_examples else None,
+            }
+        else:
+            # Simple array
+            item_type, item_format = infer_array_type(json_data[:20])
+            models[base_model_name] = {
+                "type": "array",
+                "description": f"Generated from JSON array in {source}",
+                "items": {"type": item_type, "format": item_format} if item_format else {"type": item_type},
+                "examples": [json_data[:5]] if include_examples else None,
+            }
+    elif isinstance(json_data, dict):
+        # Single object
+        fields = {}
+        for key, value in json_data.items():
+            fields[key] = generate_field_definition(value, key, base_model_name, models)
+
+        models[base_model_name] = {
+            "type": "object",
+            "description": f"Generated from JSON object in {source}",
+            "fields": fields,
+            "examples": [json_data] if include_examples else None,
+        }
+    else:
+        # Primitive value
+        field_type, field_format = determine_type_and_format(json_data)
+        models[base_model_name] = {
+            "type": field_type,
+            "description": f"Generated from JSON primitive in {source}",
+            "format": field_format,
+            "examples": [json_data] if include_examples and field_type != "boolean" else None,
+        }
+
+    for model_name, model_def in models.items():
+        model_type = model_def.pop("type")
+        data_contract_specification.models[model_name] = Model(type=model_type, **model_def)
+
+    return data_contract_specification
+
+
+def generate_field_definition(
+    value: Any, field_name: str, parent_model: str, models: Dict[str, Dict[str, Any]]
+) -> Dict[str, Any]:
+    """Generate a field definition for a JSON value, creating nested models."""
+
+    if isinstance(value, dict):
+        # Handle object fields
+        fields = {}
+        for key, nested_value in value.items():
+            fields[key] = generate_field_definition(nested_value, key, parent_model, models)
+
+        return {"type": "object", "fields": fields}
+
+    elif isinstance(value, list):
+        # Handle array fields
+        if not value:
+            return {"type": "array", "items": {"type": "string"}}
+
+        if all(isinstance(item, dict) for item in value):
+            # Array of objects
+            fields = {}
+            for item in value:
+                for key, nested_value in item.items():
+                    field_def = generate_field_definition(nested_value, key, parent_model, models)
+                    if key in fields:
+                        fields[key] = merge_field_definitions(fields[key], field_def)
+                    else:
+                        fields[key] = field_def
+
+            return {"type": "array", "items": {"type": "object", "fields": fields}}
+
+        elif all(isinstance(item, list) for item in value):
+            # Array of arrays
+            inner_type, inner_format = infer_array_type(value[0])
+            return {
+                "type": "array",
+                "items": {
+                    "type": "array",
+                    "items": {"type": inner_type, "format": inner_format} if inner_format else {"type": inner_type},
+                },
+                "examples": value[:5],  # Include examples for nested arrays
+            }
+
+        else:
+            # Array of simple or mixed types
+            item_type, item_format = infer_array_type(value)
+            items_def = {"type": item_type}
+            if item_format:
+                items_def["format"] = item_format
+
+            field_def = {"type": "array", "items": items_def}
+
+            # Add examples if appropriate
+            sample_values = [item for item in value[:5] if item is not None]
+            if sample_values:
+                field_def["examples"] = sample_values
+
+            return field_def
+
+    else:
+        # Handle primitive types
+        field_type, field_format = determine_type_and_format(value)
+        field_def = {"type": field_type}
+        if field_format:
+            field_def["format"] = field_format
+
+        # Add examples
+        if value is not None and field_type != "boolean":
+            field_def["examples"] = [value]
+
+        return field_def
+
+
+def infer_array_type(array: List) -> Tuple[str, Optional[str]]:
+    """Infer the common type of items in an array."""
+    if not array:
+        return "string", None
+
+    # if all items are dictionaries with the same structure
+    if all(isinstance(item, dict) for item in array):
+        return "object", None
+
+    # if all items are of the same primitive type
+    non_null_items = [item for item in array if item is not None]
+    if not non_null_items:
+        return "null", None
+
+    types_and_formats = [determine_type_and_format(item) for item in non_null_items]
+    types = {t for t, _ in types_and_formats}
+    formats = {f for _, f in types_and_formats if f is not None}
+
+    # simplify type combinations
+    if types == {"integer", "number"}:
+        return "number", None
+    if len(types) == 1:
+        type_name = next(iter(types))
+        format_name = next(iter(formats)) if len(formats) == 1 else None
+        return type_name, format_name
+    if all(t in {"string", "integer", "number", "boolean", "null"} for t in types):
+        # If all string values have the same format, keep it
+        if len(formats) == 1 and "string" in types:
+            return "string", next(iter(formats))
+        return "string", None
+
+    # Mixed types
+    return "string", None
+
+
+def determine_type_and_format(value: Any) -> Tuple[str, Optional[str]]:
+    """determine the datacontract type and format for a JSON value."""
+    if value is None:
+        return "null", None
+    elif isinstance(value, bool):
+        return "boolean", None
+    elif isinstance(value, int):
+        return "integer", None
+    elif isinstance(value, float):
+        return "number", None
+    elif isinstance(value, str):
+        try:
+            if re.match(r"^\d{4}-\d{2}-\d{2}$", value):
+                return "string", "date"
+            elif re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?$", value):
+                return "string", "date-time"
+            elif re.match(r"^[\w\.-]+@([\w-]+\.)+[\w-]{2,4}$", value):
+                return "string", "email"
+            elif re.match(r"^[a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12}$", value.lower()):
+                return "string", "uuid"
+            else:
+                return "string", None
+        except re.error:
+            return "string", None
+    elif isinstance(value, dict):
+        return "object", None
+    elif isinstance(value, list):
+        return "array", None
+    else:
+        return "string", None
+
+
+def merge_field_definitions(field1: Dict[str, Any], field2: Dict[str, Any]) -> Dict[str, Any]:
+    """Merge two field definitions."""
+    result = field1.copy()
+    if field1.get("type") == "object" and field2.get("type") != "object":
+        return field1
+    if field2.get("type") == "object" and field1.get("type") != "object":
+        return field2
+    # Handle type differences
+    if field1.get("type") != field2.get("type"):
+        type1, _ = field1.get("type", "string"), field1.get("format")
+        type2, _ = field2.get("type", "string"), field2.get("format")
+
+        if type1 == "integer" and type2 == "number" or type1 == "number" and type2 == "integer":
+            common_type = "number"
+            common_format = None
+        elif "string" in [type1, type2]:
+            common_type = "string"
+            common_format = None
+        elif all(t in ["string", "integer", "number", "boolean", "null"] for t in [type1, type2]):
+            common_type = "string"
+            common_format = None
+        elif type1 == "array" and type2 == "array":
+            # Handle mixed array types
+            items1 = field1.get("items", {})
+            items2 = field2.get("items", {})
+            if items1.get("type") == "object" or items2.get("type") == "object":
+                if items1.get("type") == "object" and items2.get("type") == "object":
+                    merged_items = merge_field_definitions(items1, items2)
+                else:
+                    merged_items = items1 if items1.get("type") == "object" else items2
+                return {"type": "array", "items": merged_items}
+            else:
+                merged_items = merge_field_definitions(items1, items2)
+                return {"type": "array", "items": merged_items}
+        else:
+            common_type = "array" if "array" in [type1, type2] else "object"
+            common_format = None
+
+        result["type"] = common_type
+        if common_format:
+            result["format"] = common_format
+        elif "format" in result:
+            del result["format"]
+
+    # Merge examples
+    if "examples" in field2:
+        if "examples" in result:
+            combined = result["examples"] + [ex for ex in field2["examples"] if ex not in result["examples"]]
+            result["examples"] = combined[:5]  # Limit to 5 examples
+        else:
+            result["examples"] = field2["examples"]
+
+    # Handle nested structures
+    if result.get("type") == "array" and "items" in field1 and "items" in field2:
+        result["items"] = merge_field_definitions(field1["items"], field2["items"])
+    elif result.get("type") == "object" and "fields" in field1 and "fields" in field2:
+        # Merge fields from both objects
+        merged_fields = field1["fields"].copy()
+        for key, field_def in field2["fields"].items():
+            if key in merged_fields:
+                merged_fields[key] = merge_field_definitions(merged_fields[key], field_def)
+            else:
+                merged_fields[key] = field_def
+        result["fields"] = merged_fields
+
+    return result

datacontract/imports/odcs_importer.py

@@ -48,9 +48,9 @@ def import_odcs(data_contract_specification: DataContractSpecification, source:
             engine="datacontract",
         )
     elif odcs_api_version.startswith("v3."):
-        from datacontract.imports.odcs_v3_importer import import_odcs_v3
+        from datacontract.imports.odcs_v3_importer import import_odcs_v3_as_dcs
 
-        return import_odcs_v3(data_contract_specification, source)
+        return import_odcs_v3_as_dcs(data_contract_specification, source)
     else:
         raise DataContractException(
             type="schema",

datacontract/imports/odcs_v3_importer.py

@@ -29,17 +29,18 @@ class OdcsImporter(Importer):
     def import_source(
         self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
     ) -> DataContractSpecification:
-        return import_odcs_v3(data_contract_specification, source)
+        return import_odcs_v3_as_dcs(data_contract_specification, source)
 
 
-def import_odcs_v3(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+def import_odcs_v3_as_dcs(
+    data_contract_specification: DataContractSpecification, source: str
+) -> DataContractSpecification:
     source_str = read_resource(source)
-    return import_odcs_v3_from_str(data_contract_specification, source_str)
+    odcs = parse_odcs_v3_from_str(source_str)
+    return import_from_odcs(data_contract_specification, odcs)
 
 
-def import_odcs_v3_from_str(
-    data_contract_specification: DataContractSpecification, source_str: str
-) -> DataContractSpecification:
+def parse_odcs_v3_from_str(source_str):
     try:
         odcs = OpenDataContractStandard.from_string(source_str)
     except Exception as e:
@@ -50,11 +51,10 @@ def import_odcs_v3_from_str(
             engine="datacontract",
             original_exception=e,
         )
-
-    return import_from_odcs_model(data_contract_specification, odcs)
+    return odcs
 
 
-def import_from_odcs_model(data_contract_specification, odcs):
+def import_from_odcs(data_contract_specification: DataContractSpecification, odcs: OpenDataContractStandard):
     data_contract_specification.id = odcs.id
     data_contract_specification.info = import_info(odcs)
     data_contract_specification.servers = import_servers(odcs)

datacontract/imports/spark_importer.py

@@ -28,34 +28,52 @@ class SparkImporter(Importer):
             data_contract_specification: The data contract specification object.
             source: The source string indicating the Spark tables to read.
             import_args: Additional arguments for the import process.
-
         Returns:
             dict: The updated data contract specification.
         """
-        return import_spark(data_contract_specification, source)
+        dataframe = import_args.get("dataframe", None)
+        description = import_args.get("description", None)
+        return import_spark(data_contract_specification, source, dataframe, description)
 
 
-def import_spark(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+def import_spark(
+    data_contract_specification: DataContractSpecification,
+    source: str,
+    dataframe: DataFrame | None = None,
+    description: str | None = None,
+) -> DataContractSpecification:
     """
-    Reads Spark tables and updates the data contract specification with their schemas.
+    Imports schema(s) from Spark into a Data Contract Specification.
 
     Args:
-        data_contract_specification: The data contract specification to update.
-        source: A comma-separated string of Spark temporary views to read.
+        data_contract_specification (DataContractSpecification): The contract spec to update.
+        source (str): Comma-separated Spark table/view names.
+        dataframe (DataFrame | None): Optional Spark DataFrame to import.
+        description (str | None): Optional table-level description.
 
     Returns:
-        DataContractSpecification: The updated data contract specification.
+        DataContractSpecification: The updated contract spec with imported models.
     """
     spark = SparkSession.builder.getOrCreate()
     data_contract_specification.servers["local"] = Server(type="dataframe")
-    for temp_view in source.split(","):
-        temp_view = temp_view.strip()
-        df = spark.read.table(temp_view)
-        data_contract_specification.models[temp_view] = import_from_spark_df(spark, source, df)
+
+    if dataframe is not None:
+        if not isinstance(dataframe, DataFrame):
+            raise TypeError("Expected 'dataframe' to be a pyspark.sql.DataFrame")
+        data_contract_specification.models[source] = import_from_spark_df(spark, source, dataframe, description)
+        return data_contract_specification
+
+    if not source:
+        raise ValueError("Either 'dataframe' or a valid 'source' must be provided")
+
+    for table_name in map(str.strip, source.split(",")):
+        df = spark.read.table(table_name)
+        data_contract_specification.models[table_name] = import_from_spark_df(spark, table_name, df, description)
+
     return data_contract_specification
 
 
-def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Model:
+def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame, description: str) -> Model:
     """
     Converts a Spark DataFrame into a Model.
 
@@ -63,6 +81,7 @@ def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Mod
         spark: SparkSession
         source: A comma-separated string of Spark temporary views to read.
         df: The Spark DataFrame to convert.
+        description: Table level comment
 
     Returns:
         Model: The generated data contract model.
@@ -70,7 +89,10 @@ def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Mod
     model = Model()
     schema = df.schema
 
-    model.description = _table_comment_from_spark(spark, source)
+    if description is None:
+        model.description = _table_comment_from_spark(spark, source)
+    else:
+        model.description = description
 
     for field in schema:
         model.fields[field.name] = _field_from_struct_type(field)
@@ -199,7 +221,7 @@ def _table_comment_from_spark(spark: SparkSession, source: str):
         workspace_client = WorkspaceClient()
         created_table = workspace_client.tables.get(full_name=f"{source}")
         table_comment = created_table.comment
-        print(f"'{source}' table comment retrieved using 'WorkspaceClient.tables.get({source})'")
+        logger.info(f"'{source}' table comment retrieved using 'WorkspaceClient.tables.get({source})'")
         return table_comment
     except Exception:
         pass
@@ -207,7 +229,7 @@ def _table_comment_from_spark(spark: SparkSession, source: str):
     # Fallback to Spark Catalog API for Hive Metastore or Non-UC Tables
     try:
         table_comment = spark.catalog.getTable(f"{source}").description
-        print(f"'{source}' table comment retrieved using 'spark.catalog.getTable({source}).description'")
+        logger.info(f"'{source}' table comment retrieved using 'spark.catalog.getTable({source}).description'")
         return table_comment
     except Exception:
         pass
@@ -219,7 +241,7 @@ def _table_comment_from_spark(spark: SparkSession, source: str):
             if row.col_name.strip().lower() == "comment":
                 table_comment = row.data_type
                 break
-        print(f"'{source}' table comment retrieved using 'DESCRIBE TABLE EXTENDED {source}'")
+        logger.info(f"'{source}' table comment retrieved using 'DESCRIBE TABLE EXTENDED {source}'")
         return table_comment
     except Exception:
         pass

datacontract/imports/sql_importer.py

@@ -105,7 +105,7 @@ def to_dialect(import_args: dict) -> Dialects | None:
     return None
 
 
-def to_physical_type_key(dialect: Dialects | None) -> str:
+def to_physical_type_key(dialect: Dialects | str | None) -> str:
     dialect_map = {
         Dialects.TSQL: "sqlserverType",
         Dialects.POSTGRES: "postgresType",
@@ -116,6 +116,8 @@ def to_physical_type_key(dialect: Dialects | None) -> str:
         Dialects.MYSQL: "mysqlType",
         Dialects.DATABRICKS: "databricksType",
     }
+    if isinstance(dialect, str):
+        dialect = Dialects[dialect.upper()] if dialect.upper() in Dialects.__members__ else None
     return dialect_map.get(dialect, "physicalType")
 
 
@@ -198,7 +200,7 @@ def get_precision_scale(column):
     return None, None
 
 
-def map_type_from_sql(sql_type: str):
+def map_type_from_sql(sql_type: str) -> str | None:
     if sql_type is None:
         return None