datacontract-cli 0.10.14__py3-none-any.whl → 0.10.15__py3-none-any.whl

datacontract/imports/jsonschema_importer.py

@@ -3,7 +3,7 @@ import json
 import fastjsonschema
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Definition
+from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 
@@ -111,7 +111,8 @@ def schema_to_args(property_schema, is_required: bool = None) -> dict:
     nested_properties = property_schema.get("properties")
     if nested_properties is not None:
         # recursive call for complex nested properties
-        field_kwargs["fields"] = jsonschema_to_args(nested_properties, property_schema["required"])
+        required = property_schema.get("required", [])
+        field_kwargs["fields"] = jsonschema_to_args(nested_properties, required)
 
     return field_kwargs
 

datacontract/imports/odcs_v2_importer.py

@@ -6,16 +6,16 @@ import yaml
 
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
+    DATACONTRACT_TYPES,
     Availability,
     Contact,
     DataContractSpecification,
+    Field,
     Info,
     Model,
-    Field,
     Retention,
     ServiceLevel,
     Terms,
-    DATACONTRACT_TYPES,
 )
 from datacontract.model.exceptions import DataContractException
 

datacontract/imports/odcs_v3_importer.py

@@ -8,16 +8,16 @@ import yaml
 from datacontract.imports.importer import Importer
 from datacontract.lint.resources import read_resource
 from datacontract.model.data_contract_specification import (
+    DATACONTRACT_TYPES,
     Availability,
     DataContractSpecification,
+    Field,
     Info,
     Model,
-    Field,
     Retention,
     Server,
     ServiceLevel,
     Terms,
-    DATACONTRACT_TYPES,
 )
 from datacontract.model.exceptions import DataContractException
 

datacontract/imports/parquet_importer.py

@@ -0,0 +1,81 @@
+import os.path
+
+import pyarrow
+from pyarrow import parquet
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Field,
+    Model,
+)
+from datacontract.model.exceptions import DataContractException
+
+
+class ParquetImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_parquet(data_contract_specification, source)
+
+
+def import_parquet(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    # use filename as schema name, remove .parquet suffix, avoid breaking the yaml output by replacing dots
+    schema_name = os.path.basename(source).removesuffix(".parquet").replace(".", "_")
+
+    fields: dict[str, Field] = {}
+
+    arrow_schema = parquet.read_schema(source)
+    for field_name in arrow_schema.names:
+        parquet_field = arrow_schema.field(field_name)
+
+        field = map_pyarrow_field_to_specification_field(parquet_field, "parquet")
+
+        if not parquet_field.nullable:
+            field.required = True
+
+        fields[field_name] = field
+
+    data_contract_specification.models[schema_name] = Model(fields=fields)
+
+    return data_contract_specification
+
+
+def map_pyarrow_field_to_specification_field(pyarrow_field: pyarrow.Field, file_format: str) -> Field:
+    if pyarrow.types.is_boolean(pyarrow_field.type):
+        return Field(type="boolean")
+    if pyarrow.types.is_int32(pyarrow_field.type):
+        return Field(type="int")
+    if pyarrow.types.is_int64(pyarrow_field.type):
+        return Field(type="long")
+    if pyarrow.types.is_integer(pyarrow_field.type):
+        return Field(type="number")
+    if pyarrow.types.is_float32(pyarrow_field.type):
+        return Field(type="float")
+    if pyarrow.types.is_float64(pyarrow_field.type):
+        return Field(type="double")
+    if pyarrow.types.is_decimal(pyarrow_field.type):
+        return Field(type="decimal", precision=pyarrow_field.type.precision, scale=pyarrow_field.type.scale)
+    if pyarrow.types.is_timestamp(pyarrow_field.type):
+        return Field(type="timestamp")
+    if pyarrow.types.is_date(pyarrow_field.type):
+        return Field(type="date")
+    if pyarrow.types.is_null(pyarrow_field.type):
+        return Field(type="null")
+    if pyarrow.types.is_binary(pyarrow_field.type):
+        return Field(type="bytes")
+    if pyarrow.types.is_string(pyarrow_field.type):
+        return Field(type="string")
+    if pyarrow.types.is_map(pyarrow_field.type) or pyarrow.types.is_dictionary(pyarrow_field.type):
+        return Field(type="map")
+    if pyarrow.types.is_struct(pyarrow_field.type):
+        return Field(type="struct")
+    if pyarrow.types.is_list(pyarrow_field.type):
+        return Field(type="array")
+
+    raise DataContractException(
+        type="schema",
+        name=f"Parse {file_format} schema",
+        reason=f"{pyarrow_field.type} currently not supported.",
+        engine="datacontract",
+    )

datacontract/imports/spark_importer.py

@@ -1,9 +1,10 @@
 from pyspark.sql import DataFrame, SparkSession, types
+
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
-    Model,
     Field,
+    Model,
     Server,
 )
 

datacontract/imports/sql_importer.py

@@ -1,7 +1,7 @@
 from simple_ddl_parser import parse_from_file
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class SqlImporter(Importer):

datacontract/imports/unity_importer.py

@@ -2,13 +2,13 @@ import json
 import os
 from typing import List, Optional
 
-from pyspark.sql import types
 from databricks.sdk import WorkspaceClient
-from databricks.sdk.service.catalog import TableInfo, ColumnInfo
+from databricks.sdk.service.catalog import ColumnInfo, TableInfo
+from pyspark.sql import types
 
 from datacontract.imports.importer import Importer
 from datacontract.imports.spark_importer import _field_from_struct_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/integration/opentelemetry.py

@@ -12,7 +12,6 @@ from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExpo
 
 from datacontract.model.run import Run
 
-
 # Publishes metrics of a test run.
 # Metric contains the values:
 # 0 == test run passed,

datacontract/lint/lint.py

@@ -1,9 +1,10 @@
 import abc
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Sequence, Any, cast
+from typing import Any, Sequence, cast
 
 from datacontract.model.run import Check
+
 from ..model.data_contract_specification import DataContractSpecification
 
 """This module contains linter definitions for linting a data contract.

datacontract/lint/linters/description_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/example_model_linter.py

@@ -5,6 +5,7 @@ import json
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Example
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/field_pattern_linter.py

@@ -1,6 +1,7 @@
 import re
 
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/field_reference_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/notice_period_linter.py

@@ -1,6 +1,7 @@
 import re
 
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/quality_schema_linter.py

@@ -1,6 +1,7 @@
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Model
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/valid_constraints_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/resolve.py

@@ -199,7 +199,7 @@ def _resolve_data_contract_from_str(
 
     if is_open_data_contract_standard(yaml_dict):
         # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
-        data_contract_specification = DataContractSpecification(dataContractSpecification="0.9.3")
+        data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0")
         return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
 
     _validate_data_contract_specification_schema(yaml_dict, schema_location)

datacontract/lint/schema.py

@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Dict, Any
+from typing import Any, Dict
 
 import requests
 

datacontract/model/data_contract_specification.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Dict, Optional, Any
+from typing import Any, Dict, List, Optional
 
 import pydantic as pyd
 import yaml
@@ -147,6 +147,7 @@ class Field(pyd.BaseModel):
     format: str = None
     required: bool = None
     primary: bool = None
+    primaryKey: bool = None
     unique: bool | None = None
     references: str = None
     description: str | None = None
@@ -282,9 +283,14 @@ class DataContractSpecification(pyd.BaseModel):
     terms: Terms = None
     models: Dict[str, Model] = {}
     definitions: Dict[str, Definition] = {}
-    # schema: Dict[str, str]
-    examples: List[Example] = []
-    quality: Quality = None
+    examples: List[Example] = pyd.Field(
+        default_factory=list,
+        deprecated="Removed in Data Contract Specification " "v1.1.0. Use models.examples instead.",
+    )
+    quality: Quality = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use " "model-level and field-level quality instead.",
+    )
     servicelevels: Optional[ServiceLevel] = None
     links: Dict[str, str] = {}
     tags: List[str] = []
@@ -292,7 +298,7 @@ class DataContractSpecification(pyd.BaseModel):
     @classmethod
     def from_file(cls, file):
         if not os.path.exists(file):
-            raise (f"The file '{file}' does not exist.")
+            raise FileNotFoundError(f"The file '{file}' does not exist.")
         with open(file, "r") as file:
             file_content = file.read()
         return DataContractSpecification.from_string(file_content)

datacontract/model/run.py

@@ -1,15 +1,24 @@
 import logging
 from datetime import datetime, timezone
+from enum import Enum
 from typing import List, Optional
 from uuid import UUID, uuid4
 
 from pydantic import BaseModel
 
 
+class ResultEnum(str, Enum):
+    passed = "passed"
+    warning = "warning"
+    failed = "failed"
+    error = "error"
+    unknown = "unknown"
+
+
 class Check(BaseModel):
     type: str
     name: Optional[str]
-    result: str  # passed, failed, warning, unknown
+    result: ResultEnum
     engine: str
     reason: Optional[str] = None
     model: Optional[str] = None
@@ -33,29 +42,29 @@ class Run(BaseModel):
     server: Optional[str] = None
     timestampStart: datetime
     timestampEnd: datetime
-    result: str = "unknown"  # passed, warning, failed, error, unknown
+    result: ResultEnum = ResultEnum.unknown
     checks: List[Check]
     logs: List[Log]
 
     def has_passed(self):
         self.calculate_result()
-        return self.result == "passed"
+        return self.result == ResultEnum.passed
 
     def finish(self):
         self.timestampEnd = datetime.now(timezone.utc)
         self.calculate_result()
 
     def calculate_result(self):
-        if any(check.result == "error" for check in self.checks):
-            self.result = "error"
-        elif any(check.result == "failed" for check in self.checks):
-            self.result = "failed"
-        elif any(check.result == "warning" for check in self.checks):
-            self.result = "warning"
-        elif any(check.result == "passed" for check in self.checks):
-            self.result = "passed"
+        if any(check.result == ResultEnum.error for check in self.checks):
+            self.result = ResultEnum.error
+        elif any(check.result == ResultEnum.failed for check in self.checks):
+            self.result = ResultEnum.failed
+        elif any(check.result == ResultEnum.warning for check in self.checks):
+            self.result = ResultEnum.warning
+        elif any(check.result == ResultEnum.passed for check in self.checks):
+            self.result = ResultEnum.passed
         else:
-            self.result = "unknown"
+            self.result = ResultEnum.unknown
 
     def log_info(self, message: str):
         logging.info(message)

datacontract/templates/index.html

@@ -78,17 +78,17 @@
 
               <li class="col-span-1 rounded-lg bg-white shadow hover:bg-gray-50"
                   data-search="{{
-    contract.spec.info.title|lower }} {{
-    contract.spec.info.owner|lower if contract.spec.info.owner else '' }} {{
-    contract.spec.info.description|lower }} {%
+    contract.spec.info.title|lower|e }} {{
+    contract.spec.info.owner|lower|e if contract.spec.info.owner else '' }} {{
+    contract.spec.info.description|lower|e }} {%
     for model_name, model in contract.spec.models.items() %}
-      {{ model.description|lower }} {%
+      {{ model.description|lower|e }} {%
       for field_name, field in model.fields.items() %}
-        {{ field_name|lower }} {{ field.description|lower if field.description else '' }} {%
+        {{ field_name|lower|e }} {{ field.description|lower|e if field.description else '' }} {%
       endfor %}
     {% endfor %}
   ">
-                <a href="{{contract.html_link}}" >
+                <a href="{{contract.html_link|e}}" >
                   <div class="flex w-full justify-between space-x-1 p-6 pb-4">
                     <div class="flex-1 truncate">
                       <div class="flex items-center space-x-3">

datacontract/web.py

@@ -1,11 +1,10 @@
-from typing import Annotated, Union, Optional
+from typing import Annotated, Optional, Union
 
 import typer
 from fastapi import FastAPI, File
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse
 
 from datacontract.data_contract import DataContract, ExportFormat
-from fastapi.responses import PlainTextResponse
 
 app = FastAPI()