datacontract-cli 0.10.24__tar.gz → 0.10.26__tar.gz

{datacontract_cli-0.10.24/datacontract_cli.egg-info → datacontract_cli-0.10.26}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 0.10.24
+Version: 0.10.26
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
+License-Expression: MIT
 Project-URL: Homepage, https://cli.datacontract.com
 Project-URL: Issues, https://github.com/datacontract/datacontract-cli/issues
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -28,17 +28,21 @@ Requires-Dist: python-dotenv<2.0.0,>=1.0.0
 Requires-Dist: boto3<2.0.0,>=1.34.41
 Requires-Dist: Jinja2<4.0.0,>=3.1.5
 Requires-Dist: jinja_partials<1.0.0,>=0.2.1
+Requires-Dist: datacontract-specification<2.0.0,>=1.1.1
+Requires-Dist: open-data-contract-standard<4.0.0,>=3.0.4
 Provides-Extra: avro
 Requires-Dist: avro==1.12.0; extra == "avro"
 Provides-Extra: bigquery
 Requires-Dist: soda-core-bigquery<3.6.0,>=3.3.20; extra == "bigquery"
 Provides-Extra: csv
 Requires-Dist: pandas>=2.0.0; extra == "csv"
+Provides-Extra: excel
+Requires-Dist: openpyxl<4.0.0,>=3.1.5; extra == "excel"
 Provides-Extra: databricks
 Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.6.0,>=3.3.20; extra == "databricks"
 Requires-Dist: databricks-sql-connector<4.1.0,>=3.7.0; extra == "databricks"
-Requires-Dist: databricks-sdk<0.50.0; extra == "databricks"
+Requires-Dist: databricks-sdk<0.51.0; extra == "databricks"
 Provides-Extra: iceberg
 Requires-Dist: pyiceberg==0.8.1; extra == "iceberg"
 Provides-Extra: kafka
@@ -66,11 +70,11 @@ Provides-Extra: rdf
 Requires-Dist: rdflib==7.0.0; extra == "rdf"
 Provides-Extra: api
 Requires-Dist: fastapi==0.115.12; extra == "api"
-Requires-Dist: uvicorn==0.34.0; extra == "api"
+Requires-Dist: uvicorn==0.34.2; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
 Provides-Extra: all
-Requires-Dist: datacontract-cli[api,bigquery,csv,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,protobuf,rdf,s3,snowflake,sqlserver,trino]; extra == "all"
+Requires-Dist: datacontract-cli[api,bigquery,csv,databricks,dbml,dbt,excel,iceberg,kafka,parquet,postgres,protobuf,rdf,s3,snowflake,sqlserver,trino]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
 Requires-Dist: httpx==0.28.1; extra == "dev"
@@ -80,7 +84,7 @@ Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<4.3.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: pymssql==2.3.2; extra == "dev"
+Requires-Dist: pymssql==2.3.4; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.2; extra == "dev"
 Requires-Dist: trino==0.333.0; extra == "dev"
@@ -900,6 +904,29 @@ models:
 | `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password    |
 
 
+#### Local
+
+Data Contract CLI can test local files in parquet, json, csv, or delta format.
+
+##### Example
+
+datacontract.yaml
+```yaml
+servers:
+  local:
+    type: local
+    path: ./*.parquet
+    format: parquet
+models:
+  my_table_1: # corresponds to a table
+    type: table
+    fields:
+      my_column_1: # corresponds to a column
+        type: varchar
+      my_column_2: # corresponds to a column
+        type: string
+```
+
 
 ### export
 ```
@@ -1370,20 +1397,21 @@ Available import options:
 
 | Type               | Description                                    | Status |
 |--------------------|------------------------------------------------|--------|
-| `sql`              | Import from SQL DDL                            | ✅      |
 | `avro`             | Import from AVRO schemas                       | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
+| `csv`              | Import from CSV File                           | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `dbt`              | Import from dbt models                         | ✅      |
+| `excel`            | Import from ODCS Excel Template                | ✅      |
+| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
+| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `spark`            | Import from Spark StructTypes                  | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
+| `parquet`          | Import from Parquet File Metadata              | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | ✅    |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `parquet`          | Import from Parquet File Metadta               | ✅      |
+| `spark`            | Import from Spark StructTypes                  | ✅      |
+| `sql`              | Import from SQL DDL                            | ✅      |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
 
@@ -1433,8 +1461,9 @@ datacontract import --format unity --source my_unity_table.json
 
 ```bash
 # Example import single table from Unity Catalog via HTTP endpoint
-export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
-export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
+export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4"
+export DATACONTRACT_DATABRICKS_TOKEN=<token>
 datacontract import --format unity --unity-table-full-name <table_full_name>
 ```
 
@@ -1455,6 +1484,17 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
 datacontract import --format dbt --source <manifest_path>
 ```
 
+### Excel
+
+Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
+
+Examples:
+
+```bash
+# Example import from ODCS Excel Template
+datacontract import --format excel --source odcs.xlsx
+```
+
 #### Glue
 
 Importing from Glue reads the necessary Data directly off of the AWS API.
@@ -1994,7 +2034,8 @@ pytest
 ```bash
 # make sure uv is installed
 uv python pin 3.11
-uv sync --all-extras
+uv pip install -e '.[dev]'
+uv run ruff check
 uv run pytest
 ```
 

{datacontract_cli-0.10.24 → datacontract_cli-0.10.26}/README.md

@@ -812,6 +812,29 @@ models:
 | `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password    |
 
 
+#### Local
+
+Data Contract CLI can test local files in parquet, json, csv, or delta format.
+
+##### Example
+
+datacontract.yaml
+```yaml
+servers:
+  local:
+    type: local
+    path: ./*.parquet
+    format: parquet
+models:
+  my_table_1: # corresponds to a table
+    type: table
+    fields:
+      my_column_1: # corresponds to a column
+        type: varchar
+      my_column_2: # corresponds to a column
+        type: string
+```
+
 
 ### export
 ```
@@ -1282,20 +1305,21 @@ Available import options:
 
 | Type               | Description                                    | Status |
 |--------------------|------------------------------------------------|--------|
-| `sql`              | Import from SQL DDL                            | ✅      |
 | `avro`             | Import from AVRO schemas                       | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
+| `csv`              | Import from CSV File                           | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `dbt`              | Import from dbt models                         | ✅      |
+| `excel`            | Import from ODCS Excel Template                | ✅      |
+| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
+| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `spark`            | Import from Spark StructTypes                  | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
+| `parquet`          | Import from Parquet File Metadata              | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | ✅    |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `parquet`          | Import from Parquet File Metadta               | ✅      |
+| `spark`            | Import from Spark StructTypes                  | ✅      |
+| `sql`              | Import from SQL DDL                            | ✅      |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
 
@@ -1345,8 +1369,9 @@ datacontract import --format unity --source my_unity_table.json
 
 ```bash
 # Example import single table from Unity Catalog via HTTP endpoint
-export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
-export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
+export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4"
+export DATACONTRACT_DATABRICKS_TOKEN=<token>
 datacontract import --format unity --unity-table-full-name <table_full_name>
 ```
 
@@ -1367,6 +1392,17 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
 datacontract import --format dbt --source <manifest_path>
 ```
 
+### Excel
+
+Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
+
+Examples:
+
+```bash
+# Example import from ODCS Excel Template
+datacontract import --format excel --source odcs.xlsx
+```
+
 #### Glue
 
 Importing from Glue reads the necessary Data directly off of the AWS API.
@@ -1906,7 +1942,8 @@ pytest
 ```bash
 # make sure uv is installed
 uv python pin 3.11
-uv sync --all-extras
+uv pip install -e '.[dev]'
+uv run ruff check
 uv run pytest
 ```
 

{datacontract_cli-0.10.24 → datacontract_cli-0.10.26}/datacontract/api.py

@@ -162,7 +162,7 @@ async def test(
     server: Annotated[
         str | None,
         Query(
-            example="production",
+            examples=["production"],
             description="The server name to test. Optional, if there is only one server.",
         ),
     ] = None,
@@ -191,7 +191,7 @@ async def lint(
     schema: Annotated[
         str | None,
         Query(
-            example="https://datacontract.com/datacontract.schema.json",
+            examples=["https://datacontract.com/datacontract.schema.json"],
             description="The schema to use for validation. This must be a URL.",
         ),
     ] = None,
@@ -220,7 +220,7 @@ def export(
     server: Annotated[
         str | None,
         Query(
-            example="production",
+            examples=["production"],
             description="The server name to export. Optional, if there is only one server.",
         ),
     ] = None,

{datacontract_cli-0.10.24 → datacontract_cli-0.10.26}/datacontract/cli.py

@@ -244,7 +244,7 @@ def import_(
     ] = None,
     source: Annotated[
         Optional[str],
-        typer.Option(help="The path to the file or Glue Database that should be imported."),
+        typer.Option(help="The path to the file that should be imported."),
     ] = None,
     dialect: Annotated[
         Optional[str],

{datacontract_cli-0.10.24 → datacontract_cli-0.10.26}/datacontract/engines/soda/connections/kafka.py

@@ -27,6 +27,7 @@ def create_spark_session():
     tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark")
     atexit.register(tmp_dir.cleanup)
 
+    pyspark_version = "3.5.5"  # MUST be the same as in the pyproject.toml
     spark = (
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
@@ -34,7 +35,7 @@ def create_spark_session():
         .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5",
+            f"org.apache.spark:spark-sql-kafka-0-10_2.12:{pyspark_version},org.apache.spark:spark-avro_2.12:{pyspark_version}",
         )
         .getOrCreate()
     )

{datacontract_cli-0.10.24 → datacontract_cli-0.10.26}/datacontract/export/great_expectations_converter.py

@@ -19,6 +19,7 @@ from datacontract.export.spark_converter import to_spark_data_type
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
+    DeprecatedQuality,
     Field,
     Quality,
 )
@@ -91,8 +92,14 @@ def to_great_expectations(
             model_key=model_key, contract_version=data_contract_spec.info.version
         )
     model_value = data_contract_spec.models.get(model_key)
-    quality_checks = get_quality_checks(data_contract_spec.quality)
+
+    # Support for Deprecated Quality
+    quality_checks = get_deprecated_quality_checks(data_contract_spec.quality)
+
+    expectations.extend(get_quality_checks(model_value.quality))
+
     expectations.extend(model_to_expectations(model_value.fields, engine, sql_server_type))
+
     expectations.extend(checks_to_expectations(quality_checks, model_key))
     model_expectation_suite = to_suite(expectations, expectation_suite_name)
 
@@ -135,6 +142,7 @@ def model_to_expectations(fields: Dict[str, Field], engine: str | None, sql_serv
     add_column_order_exp(fields, expectations)
     for field_name, field in fields.items():
         add_field_expectations(field_name, field, expectations, engine, sql_server_type)
+        expectations.extend(get_quality_checks(field.quality, field_name))
     return expectations
 
 
@@ -173,6 +181,8 @@ def add_field_expectations(
         expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength))
     if field.minimum is not None or field.maximum is not None:
         expectations.append(to_column_min_max_exp(field_name, field.minimum, field.maximum))
+    if field.enum is not None and len(field.enum) != 0:
+        expectations.append(to_column_enum_exp(field_name, field.enum))
 
     return expectations
 
@@ -266,7 +276,24 @@ def to_column_min_max_exp(field_name, minimum, maximum) -> Dict[str, Any]:
     }
 
 
-def get_quality_checks(quality: Quality) -> Dict[str, Any]:
+def to_column_enum_exp(field_name, enum_list: List[str]) -> Dict[str, Any]:
+    """Creates a expect_column_values_to_be_in_set expectation.
+
+    Args:
+        field_name (str): The name of the field.
+        enum_list (Set[str]): enum list of value.
+
+    Returns:
+        Dict[str, Any]: Column value in set expectation.
+    """
+    return {
+        "expectation_type": "expect_column_values_to_be_in_set",
+        "kwargs": {"column": field_name, "value_set": enum_list},
+        "meta": {},
+    }
+
+
+def get_deprecated_quality_checks(quality: DeprecatedQuality) -> Dict[str, Any]:
     """Retrieves quality checks defined in a data contract.
 
     Args:
@@ -288,6 +315,26 @@ def get_quality_checks(quality: Quality) -> Dict[str, Any]:
     return quality_specification
 
 
+def get_quality_checks(qualities: List[Quality], field_name: str | None = None) -> List[Dict[str, Any]]:
+    """Retrieves quality checks defined in a data contract.
+
+    Args:
+        qualities (List[Quality]): List of quality object from the model specification.
+        field_name (str | None): field name if the quality list is attached to a specific field
+
+    Returns:
+        Dict[str, Any]: Dictionary of quality checks.
+    """
+    quality_specification = []
+    for quality in qualities:
+        if quality is not None and quality.engine is not None and quality.engine.lower() == "great-expectations":
+            ge_expectation = quality.implementation
+            if field_name is not None:
+                ge_expectation["column"] = field_name
+            quality_specification.append(ge_expectation)
+    return quality_specification
+
+
 def checks_to_expectations(quality_checks: Dict[str, Any], model_key: str) -> List[Dict[str, Any]]:
     """Converts quality checks to a list of expectations.