datacontract-cli 0.9.7__py3-none-any.whl → 0.9.9__py3-none-any.whl

datacontract/data_contract.py

@@ -4,62 +4,53 @@ import tempfile
 import typing
 
 import yaml
+from pyspark.sql import SparkSession
 
-from datacontract.breaking.breaking import models_breaking_changes, quality_breaking_changes
-from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import \
-    check_that_datacontract_contains_valid_server_configuration
+from datacontract.breaking.breaking import models_breaking_changes, \
+    quality_breaking_changes
+from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
+    check_that_datacontract_contains_valid_server_configuration,
+)
 from datacontract.engines.fastjsonschema.check_jsonschema import \
     check_jsonschema
 from datacontract.engines.soda.check_soda_execute import check_soda_execute
-from datacontract.export.avro_converter import to_avro_schema, to_avro_schema_json
+from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.export.avro_idl_converter import to_avro_idl
 from datacontract.export.dbt_converter import to_dbt_models_yaml, \
     to_dbt_sources_yaml, to_dbt_staging_sql
-from datacontract.export.jsonschema_converter import to_jsonschema, to_jsonschema_json
+from datacontract.export.great_expectations_converter import \
+    to_great_expectations
+from datacontract.export.html_export import to_html
+from datacontract.export.jsonschema_converter import to_jsonschema_json
 from datacontract.export.odcs_converter import to_odcs_yaml
 from datacontract.export.protobuf_converter import to_protobuf
+from datacontract.export.pydantic_converter import to_pydantic_model_str
 from datacontract.export.rdf_converter import to_rdf_n3
 from datacontract.export.sodacl_converter import to_sodacl_yaml
-from datacontract.imports.avro_importer import import_avro
 from datacontract.export.sql_converter import to_sql_ddl, to_sql_query
 from datacontract.export.terraform_converter import to_terraform
+from datacontract.imports.avro_importer import import_avro
 from datacontract.imports.sql_importer import import_sql
 from datacontract.integration.publish_datamesh_manager import \
     publish_datamesh_manager
 from datacontract.integration.publish_opentelemetry import publish_opentelemetry
 from datacontract.lint import resolve
-
-from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Severity
 from datacontract.lint.linters.description_linter import DescriptionLinter
 from datacontract.lint.linters.example_model_linter import ExampleModelLinter
-from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
 from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
-from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
+from datacontract.lint.linters.field_reference_linter import \
+    FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
-from datacontract.lint.linters.primary_field_linter import PrimaryFieldUniqueRequired
-from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
+from datacontract.lint.linters.quality_schema_linter import \
+    QualityUsesSchemaLinter
+from datacontract.lint.linters.valid_constraints_linter import \
+    ValidFieldConstraintsLinter
+from datacontract.model.breaking_change import BreakingChanges, BreakingChange, \
+    Severity
 from datacontract.model.data_contract_specification import \
     DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import \
-    Run, Check
-
-
-def _determine_sql_server_type(data_contract, sql_server_type):
-    if sql_server_type == "auto":
-        if data_contract.servers is None or len(data_contract.servers) == 0:
-            raise RuntimeError(f"Export with server_type='auto' requires servers in the data contract.")
-
-        server_types = set([server.type for server in data_contract.servers.values()])
-        if "snowflake" in server_types:
-            return "snowflake"
-        elif "postgres" in server_types:
-            return "postgres"
-        else:
-            # default to snowflake dialect
-            return "snowflake"
-    else:
-        return sql_server_type
+from datacontract.model.run import Run, Check
 
 
 class DataContract:
@@ -73,7 +64,7 @@ class DataContract:
         examples: bool = False,
         publish_url: str = None,
         publish_to_opentelemetry: bool = False,
-        spark: str = None,
+        spark: SparkSession = None,
         inline_definitions: bool = False,
     ):
         self._data_contract_file = data_contract_file
@@ -92,9 +83,8 @@ class DataContract:
             FieldPatternLinter(),
             FieldReferenceLinter(),
             NoticePeriodLinter(),
-            PrimaryFieldUniqueRequired(),
             ValidFieldConstraintsLinter(),
-            DescriptionLinter()
+            DescriptionLinter(),
         }
 
     @classmethod
@@ -104,60 +94,59 @@ class DataContract:
     def lint(self, enabled_linters: typing.Union[str, set[str]] = "all") -> Run:
         """Lint the data contract by deserializing the contract and checking the schema, as well as calling the configured linters.
 
-          enabled_linters can be either "all" or "none", or a set of linter IDs. The "schema" linter is always enabled, even with enabled_linters="none".
-          """
+        enabled_linters can be either "all" or "none", or a set of linter IDs. The "schema" linter is always enabled, even with enabled_linters="none".
+        """
         run = Run.create_run()
         try:
             run.log_info("Linting data contract")
-            data_contract = resolve.resolve_data_contract(self._data_contract_file, self._data_contract_str,
-                                                          self._data_contract, self._schema_location,
-                                                          inline_definitions=True)
-            run.checks.append(Check(
-                type="lint",
-                result="passed",
-                name="Data contract is syntactically valid",
-                engine="datacontract"
-            ))
+            data_contract = resolve.resolve_data_contract(
+                self._data_contract_file,
+                self._data_contract_str,
+                self._data_contract,
+                self._schema_location,
+                inline_definitions=True,
+            )
+            run.checks.append(
+                Check(type="lint", result="passed", name="Data contract is syntactically valid", engine="datacontract")
+            )
             if enabled_linters == "none":
                 linters_to_check = set()
             elif enabled_linters == "all":
                 linters_to_check = self.all_linters
             elif isinstance(enabled_linters, set):
-                linters_to_check = {linter for linter in self.all_linters
-                                    if linter.id in enabled_linters}
+                linters_to_check = {linter for linter in self.all_linters if linter.id in enabled_linters}
             else:
                 raise RuntimeError(f"Unknown argument enabled_linters={enabled_linters} for lint()")
             for linter in linters_to_check:
                 try:
                     run.checks.extend(linter.lint(data_contract))
                 except Exception as e:
-                    run.checks.append(Check(
-                        type="general",
-                        result="error",
-                        name=f"Linter '{linter.name}'",
-                        reason=str(e),
-                        engine="datacontract",
-                    ))
+                    run.checks.append(
+                        Check(
+                            type="general",
+                            result="error",
+                            name=f"Linter '{linter.name}'",
+                            reason=str(e),
+                            engine="datacontract",
+                        )
+                    )
             run.dataContractId = data_contract.id
             run.dataContractVersion = data_contract.info.version
         except DataContractException as e:
-            run.checks.append(Check(
-                type=e.type,
-                result=e.result,
-                name=e.name,
-                reason=e.reason,
-                engine=e.engine,
-                details=""
-            ))
+            run.checks.append(
+                Check(type=e.type, result=e.result, name=e.name, reason=e.reason, engine=e.engine, details="")
+            )
             run.log_error(str(e))
         except Exception as e:
-            run.checks.append(Check(
-                type="general",
-                result="error",
-                name="Check Data Contract",
-                reason=str(e),
-                engine="datacontract",
-            ))
+            run.checks.append(
+                Check(
+                    type="general",
+                    result="error",
+                    name="Check Data Contract",
+                    reason=str(e),
+                    engine="datacontract",
+                )
+            )
             run.log_error(str(e))
         run.finish()
         return run
@@ -165,9 +154,10 @@ class DataContract:
     def test(self) -> Run:
         run = Run.create_run()
         try:
-            run.log_info(f"Testing data contract")
-            data_contract = resolve.resolve_data_contract(self._data_contract_file, self._data_contract_str,
-                                                          self._data_contract, self._schema_location)
+            run.log_info("Testing data contract")
+            data_contract = resolve.resolve_data_contract(
+                self._data_contract_file, self._data_contract_str, self._data_contract, self._schema_location
+            )
 
             if data_contract.models is None or len(data_contract.models) == 0:
                 raise DataContractException(
@@ -213,23 +203,20 @@ class DataContract:
                 check_soda_execute(run, data_contract, server, self._spark, tmp_dir)
 
         except DataContractException as e:
-            run.checks.append(Check(
-                type=e.type,
-                result=e.result,
-                name=e.name,
-                reason=e.reason,
-                engine=e.engine,
-                details=""
-            ))
+            run.checks.append(
+                Check(type=e.type, result=e.result, name=e.name, reason=e.reason, engine=e.engine, details="")
+            )
             run.log_error(str(e))
         except Exception as e:
-            run.checks.append(Check(
-                type="general",
-                result="error",
-                name="Test Data Contract",
-                reason=str(e),
-                engine="datacontract",
-            ))
+            run.checks.append(
+                Check(
+                    type="general",
+                    result="error",
+                    name="Test Data Contract",
+                    reason=str(e),
+                    engine="datacontract",
+                )
+            )
             logging.exception("Exception occurred")
             run.log_error(str(e))
 
@@ -238,45 +225,44 @@ class DataContract:
         if self._publish_url is not None:
             try:
                 publish_datamesh_manager(run, self._publish_url)
-            except:
-                logging.error("Failed to publish to datamesh manager")
+            except Exception:
+                run.log_error("Failed to publish to datamesh manager")
         if self._publish_to_opentelemetry:
             try:
                 publish_opentelemetry(run)
-            except:
-                logging.error("Failed to publish to opentelemetry")
+            except Exception:
+                run.log_error("Failed to publish to opentelemetry")
 
         return run
 
-    def breaking(self, other: 'DataContract') -> BreakingChanges:
-        return self.changelog(
-            other,
-            include_severities=[Severity.ERROR, Severity.WARNING]
-        )
+    def breaking(self, other: "DataContract") -> BreakingChanges:
+        return self.changelog(other, include_severities=[Severity.ERROR, Severity.WARNING])
 
     def changelog(
-        self,
-        other: 'DataContract',
-        include_severities: [Severity] = (Severity.ERROR, Severity.WARNING, Severity.INFO)
+        self, other: "DataContract", include_severities: [Severity] = (Severity.ERROR, Severity.WARNING, Severity.INFO)
     ) -> BreakingChanges:
         old = self.get_data_contract_specification()
         new = other.get_data_contract_specification()
 
         breaking_changes = list[BreakingChange]()
 
-        breaking_changes.extend(quality_breaking_changes(
-            old_quality=old.quality,
-            new_quality=new.quality,
-            new_path=other._data_contract_file,
-            include_severities=include_severities,
-        ))
+        breaking_changes.extend(
+            quality_breaking_changes(
+                old_quality=old.quality,
+                new_quality=new.quality,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
 
-        breaking_changes.extend(models_breaking_changes(
-            old_models=old.models,
-            new_models=new.models,
-            new_path=other._data_contract_file,
-            include_severities=include_severities,
-        ))
+        breaking_changes.extend(
+            models_breaking_changes(
+                old_models=old.models,
+                new_models=new.models,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
 
         return BreakingChanges(breaking_changes=breaking_changes)
 
@@ -290,17 +276,20 @@ class DataContract:
         )
 
     def export(self, export_format, model: str = "all", rdf_base: str = None, sql_server_type: str = "auto") -> str:
-        data_contract = resolve.resolve_data_contract(self._data_contract_file, self._data_contract_str,
-                                                      self._data_contract, inline_definitions=True)
+        data_contract = resolve.resolve_data_contract(
+            self._data_contract_file, self._data_contract_str, self._data_contract, inline_definitions=True
+        )
         if export_format == "jsonschema":
             if data_contract.models is None:
-                raise RuntimeError( f"Export to {export_format} requires models in the data contract.")
+                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
 
             model_names = list(data_contract.models.keys())
 
             if model == "all":
                 if len(data_contract.models.items()) != 1:
-                    raise RuntimeError( f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+                    )
 
                 model_name, model_value = next(iter(data_contract.models.items()))
                 return to_jsonschema_json(model_name, model_value)
@@ -308,7 +297,9 @@ class DataContract:
                 model_name = model
                 model_value = data_contract.models.get(model_name)
                 if model_value is None:
-                    raise RuntimeError( f"Model {model_name} not found in the data contract. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
+                    )
 
                 return to_jsonschema_json(model_name, model_value)
         if export_format == "sodacl":
@@ -325,7 +316,9 @@ class DataContract:
 
             if model == "all":
                 if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+                    )
 
                 model_name, model_value = next(iter(data_contract.models.items()))
                 return to_dbt_staging_sql(data_contract, model_name, model_value)
@@ -333,7 +326,9 @@ class DataContract:
                 model_name = model
                 model_value = data_contract.models.get(model_name)
                 if model_value is None:
-                    raise RuntimeError(f"Model {model_name} not found in the data contract. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
+                    )
 
                 return to_dbt_staging_sql(data_contract, model_name, model_value)
         if export_format == "odcs":
@@ -350,7 +345,9 @@ class DataContract:
 
             if model == "all":
                 if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+                    )
 
                 model_name, model_value = next(iter(data_contract.models.items()))
                 return to_avro_schema_json(model_name, model_value)
@@ -358,7 +355,9 @@ class DataContract:
                 model_name = model
                 model_value = data_contract.models.get(model_name)
                 if model_value is None:
-                    raise RuntimeError(f"Model {model_name} not found in the data contract. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
+                    )
 
                 return to_avro_schema_json(model_name, model_value)
         if export_format == "avro-idl":
@@ -366,19 +365,21 @@ class DataContract:
         if export_format == "terraform":
             return to_terraform(data_contract)
         if export_format == "sql":
-            server_type = _determine_sql_server_type(data_contract, sql_server_type)
+            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
             return to_sql_ddl(data_contract, server_type=server_type)
         if export_format == "sql-query":
             if data_contract.models is None:
                 raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
 
-            server_type = _determine_sql_server_type(data_contract, sql_server_type)
+            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
 
             model_names = list(data_contract.models.keys())
 
             if model == "all":
                 if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+                    )
 
                 model_name, model_value = next(iter(data_contract.models.items()))
                 return to_sql_query(data_contract, model_name, model_value, server_type)
@@ -386,13 +387,62 @@ class DataContract:
                 model_name = model
                 model_value = data_contract.models.get(model_name)
                 if model_value is None:
-                    raise RuntimeError(f"Model {model_name} not found in the data contract. Available models: {model_names}")
+                    raise RuntimeError(
+                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
+                    )
 
                 return to_sql_query(data_contract, model_name, model_value, server_type)
+
+        if export_format == "great-expectations":
+            if data_contract.models is None:
+                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
+
+            model_names = list(data_contract.models.keys())
+
+            if model == "all":
+                if len(data_contract.models.items()) != 1:
+                    raise RuntimeError(
+                        f"Export to {export_format} is model specific. Specify the model via --model "
+                        f"$MODEL_NAME. Available models: {model_names}"
+                    )
+
+                model_name, model_value = next(iter(data_contract.models.items()))
+                return to_great_expectations(data_contract, model_name)
+            else:
+                model_name = model
+                model_value = data_contract.models.get(model_name)
+                if model_value is None:
+                    raise RuntimeError(
+                        f"Model {model_name} not found in the data contract. " f"Available models: {model_names}"
+                    )
+
+                return to_great_expectations(data_contract, model_name)
+        if export_format == "pydantic-model":
+            return to_pydantic_model_str(data_contract)
+        if export_format == "html":
+            return to_html(data_contract)
         else:
             print(f"Export format {export_format} not supported.")
             return ""
 
+    def _determine_sql_server_type(self, data_contract: DataContractSpecification, sql_server_type: str):
+        if sql_server_type == "auto":
+            if data_contract.servers is None or len(data_contract.servers) == 0:
+                raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
+
+            server_types = set([server.type for server in data_contract.servers.values()])
+            if "snowflake" in server_types:
+                return "snowflake"
+            elif "postgres" in server_types:
+                return "postgres"
+            elif "databricks" in server_types:
+                return "databricks"
+            else:
+                # default to snowflake dialect
+                return "snowflake"
+        else:
+            return sql_server_type
+
     def _get_examples_server(self, data_contract, run, tmp_dir):
         run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
         format = "json"
@@ -402,13 +452,13 @@ class DataContract:
             run.log_info(f"Creating example file {p}")
             with open(p, "w") as f:
                 content = ""
-                if format == "json" and type(example.data) is list:
+                if format == "json" and isinstance(example.data, list):
                     content = json.dumps(example.data)
-                elif format == "json" and type(example.data) is str:
+                elif format == "json" and isinstance(example.data, str):
                     content = example.data
-                elif format == "yaml" and type(example.data) is list:
+                elif format == "yaml" and isinstance(example.data, list):
                     content = yaml.dump(example.data, allow_unicode=True)
-                elif format == "yaml" and type(example.data) is str:
+                elif format == "yaml" and isinstance(example.data, str):
                     content = example.data
                 elif format == "csv":
                     content = example.data

datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py

@@ -3,7 +3,9 @@ from datacontract.model.exceptions import DataContractException
 from datacontract.model.run import Run
 
 
-def check_that_datacontract_contains_valid_server_configuration(run: Run, data_contract: DataContractSpecification, server_name: str):
+def check_that_datacontract_contains_valid_server_configuration(
+    run: Run, data_contract: DataContractSpecification, server_name: str
+):
     if data_contract.servers is None:
         raise DataContractException(
             type="lint",
@@ -28,4 +30,6 @@ def check_that_datacontract_contains_valid_server_configuration(run: Run, data_c
             reason=f"Cannot find server '{server_name}' in the data contract servers configuration. Skip executing tests.",
             engine="datacontract",
         )
+
+
 #     TODO check for server.type, if all required fields are present

datacontract/engines/datacontract/check_that_datacontract_file_exists.py

@@ -9,12 +9,13 @@ def check_that_datacontract_file_exists(run: Run, file_path: str):
     if file_path.startswith("http://") or file_path.startswith("https://"):
         return
     if not os.path.exists(file_path):
-        run.checks.append(Check(
-            type="lint",
-            name="Check that data contract file exists",
-            result="failed",
-            reason=f"The file '{file_path}' does not exist.",
-            engine="datacontract-cli",
-        ))
+        run.checks.append(
+            Check(
+                type="lint",
+                name="Check that data contract file exists",
+                result="failed",
+                reason=f"The file '{file_path}' does not exist.",
+                engine="datacontract-cli",
+            )
+        )
         raise Exception(f"The file '{file_path}' does not exist.")
-

datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py

@@ -14,31 +14,35 @@ def check_that_datacontract_str_is_valid(run: Run, data_contract_str: str):
     try:
         fastjsonschema.validate(schema, data_contract_yaml)
         logging.debug("YAML data is valid.")
-        run.checks.append(Check(
-            type="lint",
-            result="passed",
-            name="Check that data contract YAML is valid",
-            engine="datacontract",
-        ))
+        run.checks.append(
+            Check(
+                type="lint",
+                result="passed",
+                name="Check that data contract YAML is valid",
+                engine="datacontract",
+            )
+        )
     except JsonSchemaValueException as e:
         logging.warning("YAML data is invalid.")
         logging.warning(f"Validation error: {e.message}")
-        run.checks.append(Check(
-            type="lint",
-            result="failed",
-            name="Check that data contract YAML is valid",
-            reason=e.message,
-            engine="datacontract",
-        ))
+        run.checks.append(
+            Check(
+                type="lint",
+                result="failed",
+                name="Check that data contract YAML is valid",
+                reason=e.message,
+                engine="datacontract",
+            )
+        )
     except Exception as e:
         logging.warning("YAML data is invalid.")
         logging.warning(f"Validation error: {str(e)}")
-        run.checks.append(Check(
-            type="lint",
-            result="failed",
-            name="Check that data contract YAML is valid",
-            reason=str(e),
-            engine="datacontract",
-        ))
-
-
+        run.checks.append(
+            Check(
+                type="lint",
+                result="failed",
+                name="Check that data contract YAML is valid",
+                reason=str(e),
+                engine="datacontract",
+            )
+        )