datacontract-cli 0.10.9__py3-none-any.whl → 0.10.10__py3-none-any.whl

datacontract/imports/spark_importer.py

@@ -0,0 +1,134 @@
+from pyspark.sql import DataFrame, SparkSession, types
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+    Field,
+    Server,
+)
+
+
+class SparkImporter(Importer):
+    def import_source(
+        self,
+        data_contract_specification: DataContractSpecification,
+        source: str,
+        import_args: dict,
+    ) -> dict:
+        """
+        Imports data from a Spark source into the data contract specification.
+
+        Args:
+            data_contract_specification: The data contract specification object.
+            source: The source string indicating the Spark tables to read.
+            import_args: Additional arguments for the import process.
+
+        Returns:
+            dict: The updated data contract specification.
+        """
+        return import_spark(data_contract_specification, source)
+
+
+def import_spark(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    """
+    Reads Spark tables and updates the data contract specification with their schemas.
+
+    Args:
+        data_contract_specification: The data contract specification to update.
+        source: A comma-separated string of Spark temporary views to read.
+
+    Returns:
+        DataContractSpecification: The updated data contract specification.
+    """
+    spark = SparkSession.builder.getOrCreate()
+    data_contract_specification.servers["local"] = Server(type="dataframe")
+    for temp_view in source.split(","):
+        temp_view = temp_view.strip()
+        df = spark.read.table(temp_view)
+        data_contract_specification.models[temp_view] = import_from_spark_df(df)
+    return data_contract_specification
+
+
+def import_from_spark_df(df: DataFrame) -> Model:
+    """
+    Converts a Spark DataFrame into a Model.
+
+    Args:
+        df: The Spark DataFrame to convert.
+
+    Returns:
+        Model: The generated data contract model.
+    """
+    model = Model()
+    schema = df.schema
+
+    for field in schema:
+        model.fields[field.name] = _field_from_spark(field)
+
+    return model
+
+
+def _field_from_spark(spark_field: types.StructField) -> Field:
+    """
+    Converts a Spark StructField into a Field object for the data contract.
+
+    Args:
+        spark_field: The Spark StructField to convert.
+
+    Returns:
+        Field: The corresponding Field object.
+    """
+    field_type = _data_type_from_spark(spark_field.dataType)
+    field = Field()
+    field.type = field_type
+    field.required = not spark_field.nullable
+
+    if field_type == "array":
+        field.items = _field_from_spark(spark_field.dataType.elementType)
+
+    if field_type == "struct":
+        field.fields = {sf.name: _field_from_spark(sf) for sf in spark_field.dataType.fields}
+
+    return field
+
+
+def _data_type_from_spark(spark_type: types.DataType) -> str:
+    """
+    Maps Spark data types to the Data Contract type system.
+
+    Args:
+        spark_type: The Spark data type to map.
+
+    Returns:
+        str: The corresponding Data Contract type.
+    """
+    if isinstance(spark_type, types.StringType):
+        return "string"
+    elif isinstance(spark_type, types.IntegerType):
+        return "integer"
+    elif isinstance(spark_type, types.LongType):
+        return "long"
+    elif isinstance(spark_type, types.FloatType):
+        return "float"
+    elif isinstance(spark_type, types.DoubleType):
+        return "double"
+    elif isinstance(spark_type, types.StructType):
+        return "struct"
+    elif isinstance(spark_type, types.ArrayType):
+        return "array"
+    elif isinstance(spark_type, types.TimestampType):
+        return "timestamp"
+    elif isinstance(spark_type, types.TimestampNTZType):
+        return "timestamp_ntz"
+    elif isinstance(spark_type, types.DateType):
+        return "date"
+    elif isinstance(spark_type, types.BooleanType):
+        return "boolean"
+    elif isinstance(spark_type, types.BinaryType):
+        return "bytes"
+    elif isinstance(spark_type, types.DecimalType):
+        return "decimal"
+    elif isinstance(spark_type, types.NullType):
+        return "null"
+    else:
+        raise ValueError(f"Unsupported Spark type: {spark_type}")

datacontract/integration/publish_datamesh_manager.py

@@ -8,18 +8,23 @@ from datacontract.model.run import Run
 def publish_datamesh_manager(run: Run, publish_url: str):
     try:
         if publish_url is None:
-            url = "https://api.datamesh-manager.com/api/runs"
+            # this url supports Data Mesh Manager and Data Contract Manager
+            url = "https://api.datamesh-manager.com/api/test-results"
         else:
             url = publish_url
-        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        if api_key is None:
+            api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
 
         if run.dataContractId is None:
             raise Exception("Cannot publish run results, as data contract ID is unknown")
 
-        if datamesh_manager_api_key is None:
-            raise Exception("Cannot publish run results, as DATAMESH_MANAGER_API_KEY is not set")
+        if api_key is None:
+            raise Exception(
+                "Cannot publish run results, as DATAMESH_MANAGER_API_KEY nor DATACONTRACT_MANAGER_API_KEY are not set"
+            )
 
-        headers = {"Content-Type": "application/json", "x-api-key": datamesh_manager_api_key}
+        headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
         response = requests.post(url, data=request_body, headers=headers)

datacontract/lint/resolve.py

@@ -25,7 +25,7 @@ def resolve_data_contract(
             data_contract_location, schema_location, inline_definitions, inline_quality
         )
     elif data_contract_str is not None:
-        return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
+        return _resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
     elif data_contract is not None:
         return data_contract
     else:
@@ -45,7 +45,7 @@ def resolve_data_contract_from_location(
         data_contract_str = fetch_resource(location)
     else:
         data_contract_str = read_file(location)
-    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
+    return _resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
 
 
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
@@ -55,7 +55,7 @@ def inline_definitions_into_data_contract(spec: DataContractSpecification):
             if not field.ref and not field.ref_obj:
                 continue
 
-            definition = resolve_definition_ref(field.ref, spec.definitions)
+            definition = _resolve_definition_ref(field.ref, spec)
             field.ref_obj = definition
 
             for field_name in field.model_fields.keys():
@@ -67,19 +67,41 @@ def inline_definitions_into_data_contract(spec: DataContractSpecification):
                     setattr(field, extra_field_name, extra_field_value)
 
 
-def resolve_definition_ref(ref, definitions) -> Definition:
-    if ref.startswith("http://") or ref.startswith("https://"):
-        definition_str = fetch_resource(ref)
-        definition_dict = to_yaml(definition_str)
-        return Definition(**definition_dict)
-    elif ref.startswith("file://"):
-        path = ref.replace("file://", "")
-        definition_str = fetch_file(path)
-        definition_dict = to_yaml(definition_str)
-        return Definition(**definition_dict)
-    elif ref.startswith("#/definitions/"):
-        definition_name = ref.split("#/definitions/")[1]
-        return definitions[definition_name]
+def _resolve_definition_ref(ref, spec) -> Definition:
+    logging.info(f"Resolving definition ref {ref}")
+
+    if "#" in ref:
+        path, definition_path = ref.split("#")
+    else:
+        path, definition_path = ref, None
+
+    if path.startswith("http://") or path.startswith("https://"):
+        logging.info(f"Resolving definition url {path}")
+
+        definition_str = fetch_resource(path)
+        definition_dict = _to_yaml(definition_str)
+        definition = Definition(**definition_dict)
+        if definition_path is not None:
+            return _find_by_path_in_definition(definition_path, definition)
+        else:
+            return definition
+    elif path.startswith("file://"):
+        logging.info(f"Resolving definition file path {path}")
+
+        path = path.replace("file://", "")
+        definition_str = _fetch_file(path)
+        definition_dict = _to_yaml(definition_str)
+        definition = Definition(**definition_dict)
+        if definition_path is not None:
+            return _find_by_path_in_definition(definition_path, definition)
+        else:
+            return definition
+    elif ref.startswith("#"):
+        logging.info(f"Resolving definition local path {path}")
+
+        definition_path = ref[1:]
+
+        return _find_by_path_in_spec(definition_path, spec)
     else:
         raise DataContractException(
             type="lint",
@@ -90,7 +112,30 @@ def resolve_definition_ref(ref, definitions) -> Definition:
         )
 
 
-def fetch_file(path) -> str:
+def _find_by_path_in_spec(definition_path: str, spec: DataContractSpecification):
+    path_elements = definition_path.split("/")
+    definition = spec.definitions[path_elements[2]]
+    definition = _find_subfield_in_definition(definition, path_elements[3:])
+    return definition
+
+
+def _find_by_path_in_definition(definition_path: str, definition: Definition):
+    if definition_path == "" or definition_path == "/":
+        return definition
+
+    path_elements = definition_path.split("/")
+    return _find_subfield_in_definition(definition, path_elements[1:])
+
+
+def _find_subfield_in_definition(definition: Definition, path_elements):
+    while len(path_elements) > 0 and path_elements[0] == "fields":
+        definition = definition.fields[path_elements[1]]
+        path_elements = path_elements[2:]
+
+    return definition
+
+
+def _fetch_file(path) -> str:
     if not os.path.exists(path):
         raise DataContractException(
             type="export",
@@ -103,7 +148,7 @@ def fetch_file(path) -> str:
         return file.read()
 
 
-def resolve_quality_ref(quality: Quality):
+def _resolve_quality_ref(quality: Quality):
     """
     Return the content of a ref file path
     @param quality data contract quality specification
@@ -112,13 +157,13 @@ def resolve_quality_ref(quality: Quality):
         specification = quality.specification
         if quality.type == "great-expectations":
             for model, model_quality in specification.items():
-                specification[model] = get_quality_ref_file(model_quality)
+                specification[model] = _get_quality_ref_file(model_quality)
         else:
             if "$ref" in specification:
-                quality.specification = get_quality_ref_file(specification)
+                quality.specification = _get_quality_ref_file(specification)
 
 
-def get_quality_ref_file(quality_spec: str | object) -> str | object:
+def _get_quality_ref_file(quality_spec: str | object) -> str | object:
     """
     Get the file associated with a quality reference
     @param quality_spec quality specification
@@ -139,23 +184,23 @@ def get_quality_ref_file(quality_spec: str | object) -> str | object:
     return quality_spec
 
 
-def resolve_data_contract_from_str(
+def _resolve_data_contract_from_str(
     data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    data_contract_yaml_dict = to_yaml(data_contract_str)
-    validate(data_contract_yaml_dict, schema_location)
+    data_contract_yaml_dict = _to_yaml(data_contract_str)
+    _validate(data_contract_yaml_dict, schema_location)
 
     spec = DataContractSpecification(**data_contract_yaml_dict)
 
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
     if spec.quality and inline_quality:
-        resolve_quality_ref(spec.quality)
+        _resolve_quality_ref(spec.quality)
 
     return spec
 
 
-def to_yaml(data_contract_str):
+def _to_yaml(data_contract_str):
     try:
         yaml_dict = yaml.safe_load(data_contract_str)
         return yaml_dict
@@ -170,7 +215,7 @@ def to_yaml(data_contract_str):
         )
 
 
-def validate(data_contract_yaml, schema_location: str = None):
+def _validate(data_contract_yaml, schema_location: str = None):
     schema = fetch_schema(schema_location)
     try:
         fastjsonschema.validate(schema, data_contract_yaml)

datacontract/lint/schema.py

@@ -1,18 +1,37 @@
 import json
 import os
+from typing import Dict, Any
 
 import requests
 
 from datacontract.model.exceptions import DataContractException
 
 
-def fetch_schema(location: str = None):
+def fetch_schema(location: str = None) -> Dict[str, Any]:
+    """
+    Fetch and return a JSON schema from a given location.
+
+    This function retrieves a JSON schema either from a URL or a local file path.
+    If no location is provided, it defaults to the DataContract schema URL.
+
+    Args:
+        location: The URL or file path of the schema.
+
+    Returns:
+        The JSON schema as a dictionary.
+
+    Raises:
+        DataContractException: If the specified local file does not exist.
+        requests.RequestException: If there's an error fetching the schema from a URL.
+        json.JSONDecodeError: If there's an error decoding the JSON schema.
+
+    """
     if location is None:
         location = "https://datacontract.com/datacontract.schema.json"
 
     if location.startswith("http://") or location.startswith("https://"):
         response = requests.get(location)
-        return response.json()
+        schema = response.json()
     else:
         if not os.path.exists(location):
             raise DataContractException(
@@ -23,5 +42,6 @@ def fetch_schema(location: str = None):
                 result="error",
             )
         with open(location, "r") as file:
-            file_content = file.read()
-        return json.loads(file_content)
+            schema = json.load(file)
+
+    return schema

datacontract/model/data_contract_specification.py

@@ -73,6 +73,7 @@ class Definition(pyd.BaseModel):
     exclusiveMaximum: int = None
     pii: bool = None
     classification: str = None
+    fields: Dict[str, "Definition"] = {}
     tags: List[str] = []
     links: Dict[str, str] = {}
     example: str = None
@@ -107,6 +108,8 @@ class Field(pyd.BaseModel):
     links: Dict[str, str] = {}
     fields: Dict[str, "Field"] = {}
     items: "Field" = None
+    keys: "Field" = None
+    values: "Field" = None
     precision: int = None
     scale: int = None
     example: str = None

datacontract/templates/datacontract.html

@@ -250,7 +250,7 @@
       </div>
       <div class="mt-8 md:order-1 md:mt-0">
         <p class="text-center leading-5 text-gray-400">
-          Supported with ❤️ by <a href="https://datamesh-manager.com" class="text-gray-400 hover:text-gray-500">Data Mesh Manager</a>
+          Supported with ❤️ by <a href="https://datacontract-manager.com" class="text-gray-400 hover:text-gray-500">Data Contract Manager</a>
         </p>
       </div>
     </div>

datacontract/templates/index.html

@@ -190,7 +190,7 @@
       </div>
       <div class="mt-8 md:order-1 md:mt-0">
         <p class="text-center leading-5 text-gray-400">
-          Supported with ❤️ by <a href="https://datamesh-manager.com" class="text-gray-400 hover:text-gray-500">Data Mesh Manager</a>
+          Supported with ❤️ by <a href="https://datacontract-manager.com" class="text-gray-400 hover:text-gray-500">Data Contract Manager</a>
         </p>
       </div>
     </div>