datacontract-cli 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

datacontract/cli.py

@@ -87,14 +87,16 @@ def test(
              "Use the key of the server object in the data contract yaml file "
              "to refer to a server, e.g., `production`, or `all` for all "
              "servers (default).")] = "all",
+    examples: Annotated[bool, typer.Option(
+        help="Run the schema and quality tests on the example data within the data contract.")] = None,
     publish: Annotated[str, typer.Option(
-        help="")] = None,
+        help="The url to publish the results after the test")] = None,
 ):
     """
     Run schema and quality tests on configured servers.
     """
     print(f"Testing {location}")
-    run = DataContract(data_contract_file=location, publish_url=publish).test()
+    run = DataContract(data_contract_file=location, publish_url=publish, examples=examples).test()
     _handle_result(run)
 
 

datacontract/data_contract.py

@@ -1,5 +1,9 @@
 import json
 import logging
+import tempfile
+from typing import List
+
+import yaml
 
 from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import \
     check_that_datacontract_contains_valid_server_configuration
@@ -11,8 +15,9 @@ from datacontract.export.sodacl_converter import to_sodacl
 from datacontract.integration.publish_datamesh_manager import \
     publish_datamesh_manager
 from datacontract.lint import resolve
+from datacontract.lint.linters.example_model_linter import ExampleModelLinter
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification
+    DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
 from datacontract.model.run import \
     Run, Check
@@ -25,13 +30,17 @@ class DataContract:
         data_contract_str: str = None,
         data_contract: DataContractSpecification = None,
         server: str = None,
+        examples: bool = False,
         publish_url: str = None,
+        spark: str = None,
     ):
         self._data_contract_file = data_contract_file
         self._data_contract_str = data_contract_str
         self._data_contract = data_contract
         self._server = server
+        self._examples = examples
         self._publish_url = publish_url
+        self._spark = spark
 
     def lint(self):
         run = Run.create_run()
@@ -39,14 +48,15 @@ class DataContract:
             run.log_info("Linting data contract")
             data_contract = resolve.resolve_data_contract(self._data_contract_file, self._data_contract_str,
                                                           self._data_contract)
-            run.dataContractId = data_contract.id
-            run.dataContractVersion = data_contract.info.version
             run.checks.append(Check(
                 type="lint",
                 result="passed",
-                name="Check Data Contract",
-                engine="datacontract",
-            ))
+                name="Data contract is syntactically valid",
+                engine="datacontract"
+                ))
+            run.checks.extend(ExampleModelLinter().lint(data_contract))
+            run.dataContractId = data_contract.id
+            run.dataContractVersion = data_contract.info.version
         except DataContractException as e:
             run.checks.append(Check(
                 type=e.type,
@@ -78,20 +88,27 @@ class DataContract:
 
             check_that_datacontract_contains_valid_server_configuration(run, data_contract, self._server)
             # TODO check yaml contains models
-            server_name = list(data_contract.servers.keys())[0]
-            server = data_contract.servers.get(server_name)
-            run.log_info(f"Running tests for data contract {data_contract.id} with server {server_name}")
-            run.dataContractId = data_contract.id
-            run.dataContractVersion = data_contract.info.version
-            run.dataProductId = server.dataProductId
-            run.outputPortId = server.outputPortId
-            run.server = server_name
 
-            # 5. check server is supported type
-            # 6. check server credentials are complete
-            if server.format == "json":
-                check_jsonschema(run, data_contract, server)
-            check_soda_execute(run, data_contract, server)
+            with tempfile.TemporaryDirectory(prefix="datacontract-cli") as tmp_dir:
+                if self._examples:
+                    server_name = "examples"
+                    server = self._get_examples_server(data_contract, run, tmp_dir)
+                else:
+                    server_name = list(data_contract.servers.keys())[0]
+                    server = data_contract.servers.get(server_name)
+
+                run.log_info(f"Running tests for data contract {data_contract.id} with server {server_name}")
+                run.dataContractId = data_contract.id
+                run.dataContractVersion = data_contract.info.version
+                run.dataProductId = server.dataProductId
+                run.outputPortId = server.outputPortId
+                run.server = server_name
+
+                # 5. check server is supported type
+                # 6. check server credentials are complete
+                if server.format == "json":
+                    check_jsonschema(run, data_contract, server)
+                check_soda_execute(run, data_contract, server, self._spark)
 
         except DataContractException as e:
             run.checks.append(Check(
@@ -121,6 +138,7 @@ class DataContract:
 
         return run
 
+
     def diff(self, other):
         pass
 
@@ -136,3 +154,35 @@ class DataContract:
         else:
             print(f"Export format {export_format} not supported.")
             return ""
+
+    def _get_examples_server(self, data_contract, run, tmp_dir):
+        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
+        format = "json"
+        for example in data_contract.examples:
+            format = example.type
+            p = f"{tmp_dir}/{example.model}.{format}"
+            run.log_info(f"Creating example file {p}")
+            with open(p, "w") as f:
+                content = ""
+                if format == "json" and type(example.data) is list:
+                    content = json.dumps(example.data)
+                elif format == "json" and type(example.data) is str:
+                    content = example.data
+                elif format == "yaml" and type(example.data) is list:
+                    content = yaml.dump(example.data)
+                elif format == "yaml" and type(example.data) is str:
+                    content = example.data
+                elif format == "csv":
+                    content = example.data
+                logging.debug(f"Content of example file {p}: {content}")
+                f.write(content)
+        path = f"{tmp_dir}" + "/{model}." + format
+        delimiter = "array"
+        server = Server(
+            type="local",
+            path=path,
+            format=format,
+            delimiter=delimiter,
+        )
+        run.log_info(f"Using {server} for testing the examples")
+        return server

datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -29,29 +29,40 @@ def validate_json_stream(model_name, validate, json_stream):
         )
 
 
-def read_json_lines(file_content: str):
+def read_json_lines(file):
+    file_content = file.read()
     for line in file_content.splitlines():
         yield json.loads(line)
 
 
-def read_json_lines_from_file(file):
-    for line in file:
+def read_json_lines_content(file_content: str):
+    for line in file_content.splitlines():
         yield json.loads(line)
 
 
 def read_json_array(file):
-    data = json.loads(file)
+    data = json.load(file)
+    for item in data:
+        yield item
+
+
+def read_json_array_content(file_content: str):
+    data = json.loads(file_content)
     for item in data:
         yield item
 
 
 def read_json_file(file):
-    yield json.loads(file)
+    yield json.load(file)
+
+
+def read_json_file_content(file_content: str):
+    yield json.loads(file_content)
 
 
 def process_json_file(run, model_name, validate, file, delimiter):
     if delimiter == "new_line":
-        json_stream = read_json_lines_from_file(file)
+        json_stream = read_json_lines(file)
     elif delimiter == "array":
         json_stream = read_json_array(file)
     else:
@@ -60,18 +71,23 @@ def process_json_file(run, model_name, validate, file, delimiter):
 
 
 def process_local_file(run, server, model_name, validate):
-    if os.path.isdir(server.location):
-        return process_directory(run, server, model_name, validate)
+    path = server.path
+    if "{model}" in path:
+        path = path.format(model=model_name)
+
+    if os.path.isdir(path):
+        return process_directory(run, path, server, model_name, validate)
     else:
-        with open(server.path, 'r') as file:
+        logging.info(f"Processing file {path}")
+        with open(path, 'r') as file:
             process_json_file(run, model_name, validate, file, server.delimiter)
 
 
-def process_directory(run, server, model_name, validate):
+def process_directory(run, path, server, model_name, validate):
     success = True
-    for filename in os.listdir(server.path):
+    for filename in os.listdir(path):
         if filename.endswith('.json'):  # or make this a parameter
-            file_path = os.path.join(server.path, filename)
+            file_path = os.path.join(path, filename)
             with open(file_path, 'r') as file:
                 if not process_json_file(run, model_name, validate, file, server.delimiter):
                     success = False
@@ -82,15 +98,17 @@ def process_directory(run, server, model_name, validate):
 def process_s3_file(server, model_name, validate):
     s3_endpoint_url = server.endpointUrl
     s3_location = server.location
+    if "{model}" in s3_location:
+        s3_location = s3_location.format(model=model_name)
     json_stream = None
 
     for file_content in yield_s3_files(s3_endpoint_url, s3_location):
         if server.delimiter == "new_line":
-            json_stream = read_json_lines(file_content)
+            json_stream = read_json_lines_content(file_content)
         elif server.delimiter == "array":
-            json_stream = read_json_array(file_content)
+            json_stream = read_json_array_content(file_content)
         else:
-            json_stream = read_json_file(file_content)
+            json_stream = read_json_file_content(file_content)
 
     if json_stream is None:
         raise DataContractException(
@@ -123,39 +141,35 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
         run.log_warn("jsonschema: No models found. Skip jsonschema checks.")
         return
 
-    if len(data_contract.models) > 1:
-        run.log_warn("jsonschema: Multiple models are not supported for format 'json'")
-        return
+    for model_name, model in iter(data_contract.models.items()):
+        # Process the model
+        run.log_info(f"jsonschema: Converting model {model_name} to JSON Schema")
+        schema = to_jsonschema(model_name, model)
+        run.log_info(f"jsonschema: {schema}")
 
-    # Process the model
-    run.log_info("jsonschema: Converting model to JSON Schema")
-    model_name, model = next(iter(data_contract.models.items()))
-    schema = to_jsonschema(model_name, model)
-    run.log_info(f"jsonschema: {schema}")
+        validate = fastjsonschema.compile(schema)
 
-    validate = fastjsonschema.compile(schema)
+        # Process files based on server type
+        if server.type == "local":
+            process_local_file(run, server, model_name, validate)
+        elif server.type == "s3":
+            process_s3_file(server, model_name, validate)
+        else:
+            run.checks.append(Check(
+                type="schema",
+                name="Check that JSON has valid schema",
+                model=model_name,
+                result="warn",
+                reason=f"Server type {server.type} not supported",
+                engine="jsonschema",
+            ))
+            return
 
-    # Process files based on server type
-    if server.type == "local":
-        process_local_file(run, server, model_name, validate)
-    elif server.type == "s3":
-        process_s3_file(server, model_name, validate)
-    else:
         run.checks.append(Check(
             type="schema",
             name="Check that JSON has valid schema",
             model=model_name,
-            result="warn",
-            reason=f"Server type {server.type} not supported",
+            result="passed",
+            reason="All JSON entries are valid.",
             engine="jsonschema",
         ))
-        return
-
-    run.checks.append(Check(
-        type="schema",
-        name="Check that JSON has valid schema",
-        model=model_name,
-        result="passed",
-        reason="All JSON entries are valid.",
-        engine="jsonschema",
-    ))

datacontract/engines/fastjsonschema/s3/s3_read_files.py

@@ -9,7 +9,7 @@ def yield_s3_files(s3_endpoint_url, s3_location):
     files = fs.glob(s3_location)
     for file in files:
         with fs.open(file) as f:
-            logging.info(f"Reading file {file}")
+            logging.info(f"Downloading file {file}")
             yield f.read()
 
 

datacontract/engines/soda/check_soda_execute.py

@@ -2,7 +2,13 @@ import logging
 
 from soda.scan import Scan
 
+from datacontract.engines.soda.connections.bigquery import \
+    to_bigquery_soda_configuration
+from datacontract.engines.soda.connections.databricks import \
+    to_databricks_soda_configuration
 from datacontract.engines.soda.connections.duckdb import get_duckdb_connection
+from datacontract.engines.soda.connections.postgres import \
+    to_postgres_soda_configuration
 from datacontract.engines.soda.connections.snowflake import \
     to_snowflake_soda_configuration
 from datacontract.export.sodacl_converter import to_sodacl
@@ -12,7 +18,7 @@ from datacontract.model.run import \
     Run, Check, Log
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server):
+def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark):
     if data_contract is None:
         run.log_warn("Cannot run engine soda-core, as data contract is invalid")
         return
@@ -39,6 +45,24 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         soda_configuration_str = to_snowflake_soda_configuration(server)
         scan.add_configuration_yaml_str(soda_configuration_str)
         scan.set_data_source_name(server.type)
+    elif server.type == "bigquery":
+        soda_configuration_str = to_bigquery_soda_configuration(server)
+        scan.add_configuration_yaml_str(soda_configuration_str)
+        scan.set_data_source_name(server.type)
+    elif server.type == "postgres":
+        soda_configuration_str = to_postgres_soda_configuration(server)
+        scan.add_configuration_yaml_str(soda_configuration_str)
+        scan.set_data_source_name(server.type)
+    elif server.type == "databricks":
+        if spark is not None:
+            logging.info("Use Spark to connect to data source")
+            scan.add_spark_session(spark, data_source_name=server.type)
+            scan.set_data_source_name(server.type)
+            spark.sql(f"USE {server.catalog}.{server.schema_}")
+        else:
+            soda_configuration_str = to_databricks_soda_configuration(server)
+            scan.add_configuration_yaml_str(soda_configuration_str)
+            scan.set_data_source_name(server.type)
     else:
         run.checks.append(Check(
             type="general",

datacontract/engines/soda/connections/bigquery.py

@@ -0,0 +1,18 @@
+import os
+import yaml
+
+# https://docs.soda.io/soda/connect-bigquery.html#authentication-methods
+def to_bigquery_soda_configuration(server):
+    # with service account key, using an external json file
+    soda_configuration = {
+        f"data_source {server.type}": {
+            "type": "bigquery",
+            "account_info_json_path": os.getenv('DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH'),
+            "auth_scopes": ["https://www.googleapis.com/auth/bigquery"],
+            "project_id": server.project,
+            "dataset": server.dataset
+        }
+    }
+
+    soda_configuration_str = yaml.dump(soda_configuration)
+    return soda_configuration_str

datacontract/engines/soda/connections/databricks.py

@@ -0,0 +1,20 @@
+import os
+
+import yaml
+
+
+def to_databricks_soda_configuration(server):
+    soda_configuration = {
+        f"data_source {server.type}": {
+            "type": "spark",
+            "method": "databricks",
+            "host": server.host,
+            "catalog": server.catalog,
+            "schema": server.schema_,
+            "http_path": os.getenv('DATACONTRACT_DATABRICKS_HTTP_PATH'),
+            "token": os.getenv('DATACONTRACT_DATABRICKS_TOKEN'),
+        }
+    }
+
+    soda_configuration_str = yaml.dump(soda_configuration)
+    return soda_configuration_str

datacontract/engines/soda/connections/duckdb.py

@@ -13,7 +13,11 @@ def get_duckdb_connection(data_contract, server):
         path = server.location
     setup_s3_connection(con, server)
     for model_name in data_contract.models:
-        logging.info(f"Creating table {model_name} for {path}")
+        model_path = path
+        if "{model}" in model_path:
+            model_path = model_path.format(model = model_name)
+        logging.info(f"Creating table {model_name} for {model_path}")
+
         if server.format == "json":
             format = "auto"
             if server.delimiter == "new_line":
@@ -21,15 +25,15 @@ def get_duckdb_connection(data_contract, server):
             elif server.delimiter == "array":
                 format = "array"
             con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{path}', format='{format}', hive_partitioning=1);
+                        CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{format}', hive_partitioning=1);
                         """)
         elif server.format == "parquet":
             con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{path}', hive_partitioning=1);
+                        CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
                         """)
         elif server.format == "csv":
             con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_csv_auto('{path}', hive_partitioning=1);
+                        CREATE VIEW "{model_name}" AS SELECT * FROM read_csv_auto('{model_path}', hive_partitioning=1);
                         """)
     return con
 
@@ -38,8 +42,8 @@ def setup_s3_connection(con, server):
     s3_region = os.getenv('DATACONTRACT_S3_REGION')
     s3_access_key_id = os.getenv('DATACONTRACT_S3_ACCESS_KEY_ID')
     s3_secret_access_key = os.getenv('DATACONTRACT_S3_SECRET_ACCESS_KEY')
-    con.install_extension("httpfs")
-    con.load_extension("httpfs")
+    # con.install_extension("httpfs")
+    # con.load_extension("httpfs")
     if server.endpointUrl is not None:
         s3_endpoint = server.endpointUrl.removeprefix("http://").removeprefix("https://")
         if server.endpointUrl.startswith("http://"):

datacontract/engines/soda/connections/postgres.py

@@ -0,0 +1,21 @@
+import os
+
+import yaml
+
+
+def to_postgres_soda_configuration(server):
+    # with service account key, using an external json file
+    soda_configuration = {
+        f"data_source {server.type}": {
+            "type": "postgres",
+            "host": server.host,
+            "port": str(server.port),
+            "username": os.getenv('DATACONTRACT_POSTGRES_USERNAME'),
+            "password": os.getenv('DATACONTRACT_POSTGRES_PASSWORD'),
+            "database": server.database,
+            "schema": server.schema_,
+        }
+    }
+
+    soda_configuration_str = yaml.dump(soda_configuration)
+    return soda_configuration_str

datacontract/export/sodacl_converter.py

@@ -60,7 +60,7 @@ def check_field_type(field_name: str, type: str):
 
 def check_field_required(field_name):
     return {
-        f"missing_count(\"{field_name}\") = 0": {
+        f"missing_count({field_name}) = 0": {
             "name": f"Check that required field {field_name} has no null values"
         }
     }
@@ -68,7 +68,7 @@ def check_field_required(field_name):
 
 def check_field_unique(field_name):
     return {
-        f'duplicate_count(\"{field_name}\") = 0': {
+        f'duplicate_count({field_name}) = 0': {
             "name": f"Check that unique field {field_name} has no duplicate values"
         }
     }

datacontract/lint/lint.py

@@ -0,0 +1,126 @@
+from enum import Enum
+from dataclasses import dataclass, field
+from typing import Sequence, Any
+import abc
+
+from ..model.data_contract_specification import DataContractSpecification
+from datacontract.model.run import Check
+
+"""This module contains linter definitions for linting a data contract.
+
+Lints are quality checks that can succeed, fail, or warn. They are
+distinct from checks such as "valid yaml" or "file not found", which
+will cause the processing of the data contract to stop. Lints can be
+ignored, and are high-level requirements on the format of a data
+contract."""
+
+
+class LintSeverity(Enum):
+    """The severity of a lint message. Generally, lint messages should be
+       emitted with a severity of ERROR. WARNING should be used when the linter
+       cannot determine a lint result, for example, when an unsupported model
+       type is used.
+    """
+    ERROR = 2
+    WARNING = 1
+
+
+@dataclass
+class LinterMessage:
+    """A single linter message with attached severity and optional "model" that
+    caused the message.
+
+    Attributes:
+       outcome: The outcome of the linting, either ERROR or WARNING.
+       message: A message describing the error or warning in more detail.
+       model: The model that caused the lint to fail. Is optional.
+
+    """
+    outcome: LintSeverity
+    message: str
+    model: Any = None
+
+    @classmethod
+    def error(cls, message: str, model=None):
+        return LinterMessage(LintSeverity.ERROR, message, model)
+
+    @classmethod
+    def warning(cls, message: str, model=None):
+        return LinterMessage(LintSeverity.WARNING, message, model)
+
+
+@dataclass
+class LinterResult:
+    """Result of linting a contract. Contains multiple LinterResults from
+       the same linter or lint phase.
+
+    Attributes:
+      linter: The linter that produced these results
+      results: A list of linting results. Multiple identical linting
+          results can be present in the list. An empty list means that
+          the linter ran without producing warnings or errors.
+    """
+    results: Sequence[LinterMessage] = field(default_factory=list)
+
+    def with_warning(self, message, model=None):
+        result = LinterMessage.warning(message, model)
+        return LinterResult(self.results + [result])
+
+    def with_error(self, message, model=None):
+        result = LinterMessage.error(message, model)
+        return LinterResult(self.results + [result])
+
+    def has_errors(self) -> bool:
+        return any(map(lambda result: result.outcome == LintSeverity.ERROR,
+                       self.results))
+
+    def has_warnings(self) -> bool:
+        return any(map(lambda result: result.outcome == LintSeverity.WARNING,
+                       self.results))
+
+    def error_results(self) -> Sequence[LinterMessage]:
+        return [result for result in self.results
+                if result.outcome == LintSeverity.ERROR]
+
+    def warning_results(self) -> Sequence[LinterMessage]:
+        return [result for result in self.results
+                if result.outcome == LintSeverity.WARNING]
+
+    def no_errors_or_warnings(self) -> bool:
+        return len(self.results) == 0
+
+
+class Linter(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def name(self) -> str:
+        pass
+
+    @abc.abstractmethod
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        pass
+
+    def lint(self, contract: DataContractSpecification) -> list[Check]:
+        result = self.lint_implementation(contract)
+        checks = []
+        if not result.error_results():
+            checks.append(Check(
+                type="lint",
+                name=f"Linter '{self.name()}'",
+                result="passed",
+                engine="datacontract"
+            ))
+        else:
+            # All linter messages are treated as warnings. Severity is
+            # currently ignored, but could be used in filtering in the future
+            # Linter messages with level WARNING are currently ignored, but might
+            # be logged or printed in the future.
+            for lint_error in result.error_results():
+                checks.append(Check(
+                    type="lint",
+                    name=f"Linter '{self.name()}'",
+                    result="warning",
+                    engine="datacontract",
+                    reason=lint_error.message
+                ))
+        return checks

datacontract/lint/linters/example_model_linter.py

@@ -0,0 +1,67 @@
+import csv
+import yaml
+import json
+import io
+
+from ..lint import Linter, LinterResult
+from datacontract.model.data_contract_specification import DataContractSpecification, Example
+
+
+class ExampleModelLinter(Linter):
+    def name(self) -> str:
+        return "Example(s) match model"
+
+    @staticmethod
+    def get_example_headers(example: Example) -> list[str]:
+        match example.type:
+            case "csv":
+                dialect = csv.Sniffer().sniff(example.data)
+                data = io.StringIO(example.data)
+                reader = csv.reader(data, dialect=dialect)
+                return next(reader)
+            case "yaml":
+                data = yaml.safe_load(example.data)
+                return data.keys()
+            case "json":
+                data = json.loads(example.data)
+                return data.keys()
+
+    def lint_implementation(
+        self,
+        data_contract_yaml: DataContractSpecification
+    ) -> LinterResult:
+        """Check whether the example(s) match the model."""
+        result = LinterResult()
+        examples = data_contract_yaml.examples
+        models = data_contract_yaml.models
+        examples_with_model = []
+        for (index, example) in enumerate(examples):
+            if example.model not in models:
+                result = result.with_error(
+                    f"Example {index + 1} has non-existent model '{example.model}'")
+            else:
+                examples_with_model.append(
+                    (index, example, models.get(example.model)))
+        for (index, example, model) in examples_with_model:
+            if example.type == "custom":
+                result = result.with_warning(f"Example {index + 1} has type"
+                                             " \"custom\", cannot check model"
+                                             " conformance")
+            elif model.type == "object":
+                result = result.with_warning(
+                    f"Example {index + 1} uses a "
+                    f"model '{example.model}' with type 'object'. Linting is "
+                    "currently only supported for 'table' models")
+            else:
+                headers = self.get_example_headers(example)
+                for example_header in headers:
+                    if example_header not in model.fields:
+                        result = result.with_error(
+                            f"Example {index + 1} has field '{example_header}'"
+                            f" that's not contained in model '{example.model}'")
+                for (field_name, field_value) in model.fields.items():
+                    if field_name not in headers and field_value.required:
+                        result = result.with_error(
+                            f"Example {index + 1} is missing field '{field_name}'"
+                            f" required by model '{example.model}'")
+        return result

datacontract/model/data_contract_specification.py

@@ -23,6 +23,11 @@ class Server(BaseModel):
     account: str = None
     database: str = None
     schema_: str = pydantic.fields.Field(default=None, alias='schema')
+    host: str = None
+    port: int = None
+    catalog: str = None
+    http_path: str = None # Use ENV variable
+    token: str = None     # Use ENV variable
     dataProductId: str = None
     outputPortId: str = None
 

{datacontract_cli-0.9.2.dist-info → datacontract_cli-0.9.4.dist-info}/METADATA

@@ -1,31 +1,36 @@
 Metadata-Version: 2.1
 Name: datacontract-cli
-Version: 0.9.2
-Summary: Validate data contracts
+Version: 0.9.4
+Summary: Test data contracts
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
 Project-URL: Homepage, https://cli.datacontract.com
 Project-URL: Issues, https://github.com/datacontract/cli/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: ~=3.11
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: typer[all] ~=0.9.0
-Requires-Dist: pydantic ~=2.5.3
+Requires-Dist: pydantic <2.7.0,>=2.5.3
 Requires-Dist: pyyaml ~=6.0.1
 Requires-Dist: requests ~=2.31.0
 Requires-Dist: fastparquet ==2023.10.1
-Requires-Dist: soda-core-duckdb ~=3.1.3
-Requires-Dist: soda-core-snowflake ~=3.1.3
-Requires-Dist: snowflake-connector-python[pandas] ~=3.6.0
-Requires-Dist: duckdb ~=0.9.2
+Requires-Dist: soda-core-bigquery ~=3.1.5
+Requires-Dist: soda-core-duckdb ~=3.1.5
+Requires-Dist: soda-core-postgres ~=3.1.5
+Requires-Dist: soda-core-snowflake ~=3.1.5
+Requires-Dist: soda-core-spark[databricks] ~=3.1.5
+Requires-Dist: soda-core-spark-df ~=3.1.5
+Requires-Dist: snowflake-connector-python[pandas] <3.8,>=3.6
+Requires-Dist: duckdb ==0.10.0
 Requires-Dist: fastjsonschema ~=2.19.1
 Requires-Dist: python-dotenv ~=1.0.0
-Requires-Dist: s3fs ==2023.12.2
+Requires-Dist: s3fs ==2024.2.0
 Provides-Extra: dev
 Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: testcontainers-minio ; extra == 'dev'
+Requires-Dist: testcontainers-postgres ; extra == 'dev'
 
 # Data Contract CLI
 
@@ -44,12 +49,12 @@ It uses data contract YAML files to lint the data contract, connect to data sour
 
 ## Getting started
 
-Let's use [pip](https://pip.pypa.io/en/stable/getting-started/) to install the CLI.  
+Let's use [pip](https://pip.pypa.io/en/stable/getting-started/) to install the CLI.
 ```bash
 $ pip3 install datacontract-cli
 ```
 
-Now, let's look at this data contract:  
+Now, let's look at this data contract:
 [https://datacontract.com/examples/covid-cases/datacontract.yaml](https://datacontract.com/examples/covid-cases/datacontract.yaml)
 
 We have a _servers_ section with endpoint details to the (public) S3 bucket, _models_ for the structure of the data, and _quality_ attributes that describe the expected freshness and number of rows.
@@ -77,6 +82,9 @@ $ datacontract lint datacontract.yaml
 # execute schema and quality checks
 $ datacontract test datacontract.yaml
 
+# execute schema and quality checks on the examples within the contract
+$ datacontract test --examples datacontract.yaml
+
 # find differences between to data contracts (Coming Soon)
 $ datacontract diff datacontract-v1.yaml datacontract-v2.yaml
 
@@ -124,6 +132,7 @@ Choose the most appropriate installation method for your needs:
 
 ### pip
 Python 3.11 recommended.
+Python 3.12 available as pre-release release candidate for 0.9.3
 
 ```bash
 pip3 install datacontract-cli
@@ -135,17 +144,17 @@ pipx installs into an isolated environment.
 pipx install datacontract-cli
 ```
 
-### Homebrew (coming soon)
+### Docker
 
 ```bash
-brew install datacontract/brew/datacontract
+docker pull --platform linux/amd64 datacontract/cli
+docker run --rm --platform linux/amd64 -v ${PWD}:/home/datacontract datacontract/cli
 ```
 
-### Docker (coming soon)
+Or via an alias that automatically uses the latest version:
 
 ```bash
-docker pull datacontract/cli
-docker run --rm -v ${PWD}:/datacontract datacontract/cli
+alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" --platform linux/amd64 datacontract/cli:latest'
 ```
 
 ## Documentation
@@ -154,7 +163,7 @@ docker run --rm -v ${PWD}:/datacontract datacontract/cli
 
 Data Contract CLI can connect to data sources and run schema and quality tests to verify that the data contract is valid.
 
-```bash 
+```bash
 $ datacontract test --server production datacontract.yaml
 ```
 
@@ -168,11 +177,12 @@ The application uses different engines, based on the server `type`.
 | `s3`         | `json`     | Support for `new_line` delimited JSON files and one JSON record per file. | ✅           | fastjsonschema<br> soda-core-duckdb |
 | `s3`         | `csv`      |                                                                           | ✅           | soda-core-duckdb                    |
 | `s3`         | `delta`    |                                                                           | Coming soon | TBD                                 |
-| `postgres`   | n/a        |                                                                           | Coming soon | TBD                                 |
-| `snowflake`  | n/a        |                                                                           | ✅ | soda-core-snowflake                 |
-| `bigquery`   | n/a        |                                                                           | Coming soon | TBD                                 |
+| `postgres`   | n/a        |                                                                           | ✅           | soda-core-postgres                  |
+| `snowflake`  | n/a        |                                                                           | ✅           | soda-core-snowflake                 |
+| `bigquery`   | n/a        |                                                                           | ✅           | soda-core-bigquery                  |
 | `redshift`   | n/a        |                                                                           | Coming soon | TBD                                 |
-| `databricks` | n/a        |                                                                           | Coming soon | TBD                                 |
+| `databricks` | n/a        | Support for Databricks SQL with Unity catalog and Hive metastore.         | ✅           | soda-core-spark                     |
+| `databricks` | n/a        | Support for Spark for programmatic use in Notebooks.                      | ✅           | soda-core-spark-df                  |
 | `kafka`      | `json`     |                                                                           | Coming soon | TBD                                 |
 | `kafka`      | `avro`     |                                                                           | Coming soon | TBD                                 |
 | `kafka`      | `protobuf` |                                                                           | Coming soon | TBD                                 |
@@ -182,32 +192,178 @@ The application uses different engines, based on the server `type`.
 
 Feel free to create an issue, if you need support for an additional type.
 
-### Server Type S3
+### S3
 
-Example:
+Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
+
+#### Example
 
 datacontract.yaml
-```
+```yaml
 servers:
   production:
     type: s3
     endpointUrl: https://minio.example.com # not needed with AWS S3
     location: s3://bucket-name/path/*/*.json
-    delimiter: new_line # new_line, array, or none
     format: json
+    delimiter: new_line # new_line, array, or none
+```
+
+#### Environment Variables
+
+| Environment Variable              | Example                       | Description           |
+|-----------------------------------|-------------------------------|-----------------------|
+| `DATACONTRACT_S3_REGION`            | `eu-central-1`                  | Region of S3 bucket   |
+| `DATACONTRACT_S3_ACCESS_KEY_ID`     | `AKIAXV5Q5QABCDEFGH`            | AWS Access Key ID     |
+| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
+
+
+### Postgres
+
+Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  postgres:
+    type: postgres
+    host: localhost
+    port: 5432
+    database: postgres
+    schema: public
+models:
+  my_table_1: # corresponds to a table
+    type: table
+    fields: 
+      my_column_1: # corresponds to a column
+        type: varchar
+```
+
+#### Environment Variables
+
+| Environment Variable             | Example            | Description |
+|----------------------------------|--------------------|-------------|
+| `DATACONTRACT_POSTGRES_USERNAME` | `postgres`         | Username    |
+| `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password    |
+
+
+### BigQuery
+
+We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
+* BigQuery Job User
+* BigQuery Data Viewer
+
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: bigquery
+    project: datameshexample-product
+    dataset: datacontract_cli_test_dataset
+models:
+  datacontract_cli_test_table: # corresponds to a BigQuery table
+    type: table
+    fields: ...
+```
+
+#### Environment Variables
+
+| Environment Variable                         | Example                   | Description                                             |
+|----------------------------------------------|---------------------------|---------------------------------------------------------|
+| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
+
+
+### Databricks
+
+Works with Unity Catalog and Hive metastore.
+
+Needs a running SQL warehouse or compute cluster.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: databricks
+    host: dbc-abcdefgh-1234.cloud.databricks.com
+    catalog: acme_catalog_prod
+    schema: orders_latest
+models:
+  orders: # corresponds to a table
+    type: table
+    fields: ...
 ```
 
-Environment variables
+#### Environment Variables
+
+| Environment Variable                         | Example                              | Description                                           |
+|----------------------------------------------|--------------------------------------|-------------------------------------------------------|
+| `DATACONTRACT_DATABRICKS_TOKEN` | `dapia00000000000000000000000000000` | The personal access token to authenticate             |
+| `DATACONTRACT_DATABRICKS_HTTP_PATH` | `/sql/1.0/warehouses/b053a3ffffffff` | The HTTP path to the SQL warehouse or compute cluster |
+
+
+### Databricks (programmatic)
+
+Works with Unity Catalog and Hive metastore.
+When running in a notebook or pipeline, the provided `spark` session can be used.
+An additional authentication is not required.
+
+Requires a Databricks Runtime with Python >= 3.10.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: databricks
+    host: dbc-abcdefgh-1234.cloud.databricks.com # ignored, always use current host
+    catalog: acme_catalog_prod
+    schema: orders_latest
+models:
+  orders: # corresponds to a table
+    type: table
+    fields: ...
 ```
-export DATACONTRACT_S3_REGION=eu-central-1
-export DATACONTRACT_S3_ACCESS_KEY_ID=AKIAXV5Q5QABCDEFGH
-export DATACONTRACT_S3_SECRET_ACCESS_KEY=93S7LRrJcqLkdb2/XXXXXXXXXXXXX
+
+Notebook
+```python
+%pip install git+https://github.com/datacontract/cli.git
+dbutils.library.restartPython()
+
+from datacontract.data_contract import DataContract
+
+data_contract = DataContract(
+  data_contract_file="/Volumes/acme_catalog_prod/orders_latest/datacontract/datacontract.yaml", 
+  spark=spark)
+run = data_contract.test()
+run.result
 ```
 
 
+### Exports
+
+Available export options:
+
+| Type         | Description                                    | Status |
+|--------------|------------------------------------------------|--------|
+| `jsonschema` | Export to JSON Schema                          | ✅      | 
+| `sodacl`     | Export to SodaCL quality checks in YAML format | ✅      |
+| `dbt`        | Export to dbt model in YAML format             | TBD    |
+| `avro`       | Export to AVRO models                          | TBD    |
+| `pydantic`   | Export to pydantic models                      | TBD    |
+| `sql`        | Export to SQL DDL                              | TBD    |
+| `protobuf`   | Export to Protobuf                             | TBD    |
+
 ## Development Setup
 
-Python base interpreter should be 3.11.x
+Python base interpreter should be 3.11.x (unless working on 3.12 release candidate).
 
 ```bash
 # create venv
@@ -237,7 +393,7 @@ Docker Build
 
 ```
 docker build -t datacontract/cli .
-docker run --rm -v ${PWD}:/datacontract datacontract/cli
+docker run --rm -v ${PWD}:/home/datacontract datacontract/cli
 ```
 
 ## Contribution
@@ -251,3 +407,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
 ## Credits
 
 Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/) and [Jochen Christ](https://www.linkedin.com/in/jochenchrist/).
+
+
+
+<a href="https://github.com/datacontract/cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>

{datacontract_cli-0.9.2.dist-info → datacontract_cli-0.9.4.dist-info}/RECORD

@@ -1,31 +1,37 @@
 datacontract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datacontract/cli.py,sha256=CJFoNxU8xmvMekaYJaDvniuPn0Jx2qdi8ZCSGtWebi8,4137
-datacontract/data_contract.py,sha256=7oB1OH6Rk05wSWXKoJ8tI4Od0zcQQMX7jFnUIUBzIu8,5276
+datacontract/cli.py,sha256=CaU0B68__T6t_JzcfE4cQ54CCwKkvHnkatl_zVFIFQg,4349
+datacontract/data_contract.py,sha256=BOJ8UuT75o-nQwuE-oHxHosSn6JC1F74OHQcmjiaoCs,7371
 datacontract/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py,sha256=Tj_REcEYl2BtIR_W9k0pjdjE4CvBE-4vpFrGAvvrde4,1557
 datacontract/engines/datacontract/check_that_datacontract_file_exists.py,sha256=V_YJyt1rKkkKhghU359vaAGtC8leIGmwqR4MlrLgCJ4,620
 datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py,sha256=bRoJp8a-Hvdc2OkbcTcS0tr8M7XxNzWbJAUFrc-ceiA,1393
-datacontract/engines/fastjsonschema/check_jsonschema.py,sha256=JjNPaOmHXZDsA-f3RWNSNKlrUrK2n4bMkVChdwh20_E,5124
-datacontract/engines/fastjsonschema/s3/s3_read_files.py,sha256=5cjJf2zONs18FYUesV1io9fqg5kcDYvZ8RwhyGjNJA8,709
+datacontract/engines/fastjsonschema/check_jsonschema.py,sha256=QuvFkeA-cE2nsHL33wQLb7QuhBQg4AQDQ3pav-iw9uE,5596
+datacontract/engines/fastjsonschema/s3/s3_read_files.py,sha256=sCe028D8q04c2pYlzJuEXWmMZOQJLiaObyLXLe4UzUs,713
 datacontract/engines/soda/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datacontract/engines/soda/check_soda_execute.py,sha256=HEELb6PyDIZJbeqL1doD3VGO3eDWruSs0_ds1RZ3CEo,4305
+datacontract/engines/soda/check_soda_execute.py,sha256=WPvan3olUY7tao_75Uun7khwYLBTSRX9wtE1vCRmWJA,5572
+datacontract/engines/soda/connections/bigquery.py,sha256=_hNd7Lmo6DjLb3nqVx_pfePwSYp3_3T_hwivVlATEyI,658
 datacontract/engines/soda/connections/dask.py,sha256=iQfu4swHN_QfY9l0TdSbqAQXJvfKMIxGoZ4xiNpi4eY,1497
-datacontract/engines/soda/connections/duckdb.py,sha256=qDsphPTqYK9xuwXsr4H9qynViC3778f7VFaNGvOn3GY,2188
+datacontract/engines/soda/connections/databricks.py,sha256=tCVE2Q2BXjuxS5ZmDyH_qN6jigStBsfIikRYMQ5LKVs,561
+datacontract/engines/soda/connections/duckdb.py,sha256=_Tpfo5D1ahOUPHbnEZ1WloeCecQ2LYDUebIU3hnnBDg,2342
+datacontract/engines/soda/connections/postgres.py,sha256=ow21gzxiV2_FyOXrFYeSRefLKwRQR5_qxtOR2T1rdTI,625
 datacontract/engines/soda/connections/snowflake.py,sha256=H941nOQULZKznmarVvZcvJhseMOUwfnMsv1r_P0MMb0,719
 datacontract/export/jsonschema_converter.py,sha256=gceZ-_euhedZzPfpVG8xYI16-ro9wLwAnqfkwDNWDTE,2977
-datacontract/export/sodacl_converter.py,sha256=P7f2xeucx9oKEUg4mETyFn_WT0gMc3cfSkanmlEB_9I,2844
+datacontract/export/sodacl_converter.py,sha256=7P6be3GAsUaLchqP6GNibKvpXmpo24D6z1NvOyJCjcI,2836
 datacontract/init/download_datacontract_file.py,sha256=H_234IfZ3xezjgcZ4sb7wSCEZCDUjM1uYsUibHHj4Ow,412
 datacontract/integration/publish_datamesh_manager.py,sha256=_qn4lyUkyrO0IKFzfzWCNBqEH5Ur20M_cpieIPtgRwc,1358
 datacontract/lint/files.py,sha256=DIUetslLuBvvddgza4vEvvUBMSVeJ4I1LHFID0mmMfU,470
+datacontract/lint/lint.py,sha256=4-4vrwy-98e_Za_faPxiWM8npGMzQTQgg74_x9v7cAU,4371
 datacontract/lint/resolve.py,sha256=5VnWG7lV5YgKFdabi5c-G212caMx9g0LrGeanQ2z1r4,3022
 datacontract/lint/schema.py,sha256=9UipDhpY6jQEtC6vKZ44-NcVMbpPXBvs9HZYGQ0gsAM,174
 datacontract/lint/urls.py,sha256=LXg_yzAmG71fJPc_0QeWJ0cKEqkhtZhlZZf1hWMTFNE,1408
-datacontract/model/data_contract_specification.py,sha256=PMtXLSWsQKDVldf0HEhe-7RexQdLqbB0ARzVI7-oxS0,2011
+datacontract/lint/linters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datacontract/lint/linters/example_model_linter.py,sha256=XGXDbNc_LLzwiWstXloJ8dpabxd3xV9IMq5XO3VjKw8,2898
+datacontract/model/data_contract_specification.py,sha256=pTLQH6YoiVrL7L1CrgIh0d0cBPRFV6SUPxGy1Va1TS8,2167
 datacontract/model/exceptions.py,sha256=zhhXnKWTzEyG54N9QDVpE5F986cKuHEXN0OcR5Zy8oc,1090
 datacontract/model/run.py,sha256=AejMAlTex2oh-zQQw6fifWntPnBSLLOB-7VaexG6Ef0,2484
-datacontract_cli-0.9.2.dist-info/LICENSE,sha256=23h64qnSeIZ0DKeziWAKC-zBCt328iSbRbWBrXoYRb4,2210
-datacontract_cli-0.9.2.dist-info/METADATA,sha256=nttyQXJnpxxXVaKCxTyUCZLNTVqUAMoEX9JrdkAEWvc,10085
-datacontract_cli-0.9.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datacontract_cli-0.9.2.dist-info/entry_points.txt,sha256=D3Eqy4q_Z6bHauGd4ppIyQglwbrm1AJnLau4Ppbw9Is,54
-datacontract_cli-0.9.2.dist-info/top_level.txt,sha256=VIRjd8EIUrBYWjEXJJjtdUgc0UAJdPZjmLiOR8BRBYM,13
-datacontract_cli-0.9.2.dist-info/RECORD,,
+datacontract_cli-0.9.4.dist-info/LICENSE,sha256=23h64qnSeIZ0DKeziWAKC-zBCt328iSbRbWBrXoYRb4,2210
+datacontract_cli-0.9.4.dist-info/METADATA,sha256=Ks35GB42Js-cwZx-r2x5QfPZLM7etMTKprkc0Ui_Sso,17052
+datacontract_cli-0.9.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datacontract_cli-0.9.4.dist-info/entry_points.txt,sha256=D3Eqy4q_Z6bHauGd4ppIyQglwbrm1AJnLau4Ppbw9Is,54
+datacontract_cli-0.9.4.dist-info/top_level.txt,sha256=VIRjd8EIUrBYWjEXJJjtdUgc0UAJdPZjmLiOR8BRBYM,13
+datacontract_cli-0.9.4.dist-info/RECORD,,