datacontract-cli 0.10.25__py3-none-any.whl → 0.10.26__py3-none-any.whl

datacontract/engines/soda/connections/kafka.py

@@ -27,6 +27,7 @@ def create_spark_session():
     tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark")
     atexit.register(tmp_dir.cleanup)
 
+    pyspark_version = "3.5.5"  # MUST be the same as in the pyproject.toml
     spark = (
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
@@ -34,7 +35,7 @@ def create_spark_session():
         .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5",
+            f"org.apache.spark:spark-sql-kafka-0-10_2.12:{pyspark_version},org.apache.spark:spark-avro_2.12:{pyspark_version}",
         )
         .getOrCreate()
     )

datacontract/export/odcs_v3_exporter.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Any, Dict
 
 from open_data_contract_standard.model import (
     CustomProperty,
@@ -202,13 +202,31 @@ def to_logical_type(type: str) -> str | None:
         return "array"
     if type.lower() in ["array"]:
         return "array"
+    if type.lower() in ["variant"]:
+        return "variant"
     if type.lower() in ["null"]:
         return None
     return None
 
 
-def to_physical_type(type: str) -> str | None:
-    return type
+def to_physical_type(config: Dict[str, Any]) -> str | None:
+    if config is None:
+        return None
+    if "postgresType" in config:
+        return config["postgresType"]
+    elif "bigqueryType" in config:
+        return config["bigqueryType"]
+    elif "snowflakeType" in config:
+        return config["snowflakeType"]
+    elif "redshiftType" in config:
+        return config["redshiftType"]
+    elif "sqlserverType" in config:
+        return config["sqlserverType"]
+    elif "databricksType" in config:
+        return config["databricksType"]
+    elif "physicalType" in config:
+        return config["physicalType"]
+    return None
 
 
 def to_property(field_name: str, field: Field) -> SchemaProperty:
@@ -231,7 +249,7 @@ def to_property(field_name: str, field: Field) -> SchemaProperty:
 
     if field.type is not None:
         property.logicalType = to_logical_type(field.type)
-        property.physicalType = to_physical_type(field.type)
+        property.physicalType = to_physical_type(field.config)
 
     if field.description is not None:
         property.description = field.description

datacontract/export/sql_converter.py

@@ -117,6 +117,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'
+        if server_type == "snowflake" and field.description is not None:
+            result += f" COMMENT '{_escape(field.description)}'"
         if current_field_index < fields:
             result += ","
         result += "\n"
@@ -124,6 +126,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
     result += ")"
     if server_type == "databricks" and model.description is not None:
         result += f' COMMENT "{_escape(model.description)}"'
+    if server_type == "snowflake" and model.description is not None:
+        result += f" COMMENT='{_escape(model.description)}'"
     result += ";\n"
     return result
 

datacontract/export/sql_type_converter.py

@@ -197,6 +197,8 @@ def convert_to_databricks(field: Field) -> None | str:
     if type.lower() in ["array"]:
         item_type = convert_to_databricks(field.items)
         return f"ARRAY<{item_type}>"
+    if type.lower() in ["variant"]:
+        return "VARIANT"
     return None
 
 

datacontract/imports/spark_importer.py

@@ -154,5 +154,7 @@ def _data_type_from_spark(spark_type: types.DataType) -> str:
         return "null"
     elif isinstance(spark_type, types.VarcharType):
         return "varchar"
+    elif isinstance(spark_type, types.VariantType):
+        return "variant"
     else:
         raise ValueError(f"Unsupported Spark type: {spark_type}")

{datacontract_cli-0.10.25.dist-info → datacontract_cli-0.10.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 0.10.25
+Version: 0.10.26
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -70,7 +70,7 @@ Provides-Extra: rdf
 Requires-Dist: rdflib==7.0.0; extra == "rdf"
 Provides-Extra: api
 Requires-Dist: fastapi==0.115.12; extra == "api"
-Requires-Dist: uvicorn==0.34.0; extra == "api"
+Requires-Dist: uvicorn==0.34.2; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
 Provides-Extra: all
@@ -84,7 +84,7 @@ Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<4.3.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: pymssql==2.3.2; extra == "dev"
+Requires-Dist: pymssql==2.3.4; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.2; extra == "dev"
 Requires-Dist: trino==0.333.0; extra == "dev"
@@ -1397,20 +1397,21 @@ Available import options:
 
 | Type               | Description                                    | Status |
 |--------------------|------------------------------------------------|--------|
-| `sql`              | Import from SQL DDL                            | ✅      |
 | `avro`             | Import from AVRO schemas                       | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
+| `csv`              | Import from CSV File                           | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `dbt`              | Import from dbt models                         | ✅      |
+| `excel`            | Import from ODCS Excel Template                | ✅      |
+| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
+| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `spark`            | Import from Spark StructTypes                  | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
+| `parquet`          | Import from Parquet File Metadata              | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | ✅    |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `parquet`          | Import from Parquet File Metadta               | ✅      |
+| `spark`            | Import from Spark StructTypes                  | ✅      |
+| `sql`              | Import from SQL DDL                            | ✅      |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
 
@@ -1460,8 +1461,9 @@ datacontract import --format unity --source my_unity_table.json
 
 ```bash
 # Example import single table from Unity Catalog via HTTP endpoint
-export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
-export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
+export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4"
+export DATACONTRACT_DATABRICKS_TOKEN=<token>
 datacontract import --format unity --unity-table-full-name <table_full_name>
 ```
 
@@ -1482,6 +1484,17 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
 datacontract import --format dbt --source <manifest_path>
 ```
 
+### Excel
+
+Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
+
+Examples:
+
+```bash
+# Example import from ODCS Excel Template
+datacontract import --format excel --source odcs.xlsx
+```
+
 #### Glue
 
 Importing from Glue reads the necessary Data directly off of the AWS API.

{datacontract_cli-0.10.25.dist-info → datacontract_cli-0.10.26.dist-info}/RECORD

@@ -19,7 +19,7 @@ datacontract/engines/soda/check_soda_execute.py,sha256=SYJdPpkozOA62yTM7s6cfwLfg
 datacontract/engines/soda/connections/bigquery.py,sha256=C-8kxmzpYe88bJp80ObHFLMh4rpnIjnUQ7XOj0Ke7lk,903
 datacontract/engines/soda/connections/databricks.py,sha256=cMRasuO0MrSKVgHPB-9uFTGTZPFg6z9Kpk3tJ0SdR0s,943
 datacontract/engines/soda/connections/duckdb_connection.py,sha256=hy2HxktSSt1tFB7rVI7tDOe6WeTNvHSKqsEk9uIkntU,9114
-datacontract/engines/soda/connections/kafka.py,sha256=icUcsGy9lFQRTz_mXD35tahFKIIxK5aS3RCzu_YfBaQ,8625
+datacontract/engines/soda/connections/kafka.py,sha256=lnj_-3-CnJ6stetGqm6HOzN1Qatlw7xoCQU2zKBIXxU,8725
 datacontract/engines/soda/connections/postgres.py,sha256=9GTF4Es3M5vb7ocSGqAxXmslvkS5CjsPQGIuo020CFc,626
 datacontract/engines/soda/connections/snowflake.py,sha256=rfG2ysuqNM6TkvyqQKcGHFsTGJ6AROmud5VleUDRrb0,749
 datacontract/engines/soda/connections/sqlserver.py,sha256=RzGLbCUdRyfmDcqtM_AB9WZ-Xk-XYX91nkXpVNpYbvc,1440
@@ -42,15 +42,15 @@ datacontract/export/html_export.py,sha256=ojazWrb0AwSc7Vr72M_otMo-3PA8mfi8tfIy9B
 datacontract/export/iceberg_converter.py,sha256=ArcQ_Y3z_W4_kGDU_8jPRx2-pHpP3Nhx1zYoETOL3c4,6804
 datacontract/export/jsonschema_converter.py,sha256=2MT82MurcQQbrVDRj1kFsxnmFd9scNSfYI1upQSecl4,5631
 datacontract/export/markdown_converter.py,sha256=chtaZX4vXTee7JCMYmWiDQ9m55gwJjHPw6SEM3UOwpQ,6467
-datacontract/export/odcs_v3_exporter.py,sha256=4iMPsREiVwODx29KETRUMcd1RRLRYLQy4mdwzsi0qbU,13128
+datacontract/export/odcs_v3_exporter.py,sha256=ta5NMqZ-9k-w9okdkYrPbMtd4b0ldHGL0_w70EdA5ho,13769
 datacontract/export/pandas_type_converter.py,sha256=464pQ3JQKFQa1TO0HBNcEoZvQye_yUbY6jQtiBaphSc,1117
 datacontract/export/protobuf_converter.py,sha256=DHLl8BW26xqltBsd7Qhz0RhTl9YZQKCbkmjNpECgubg,7928
 datacontract/export/pydantic_converter.py,sha256=1Lt9F8i6zyQYb44MyQtsXwCWWXYxZ47SmzArr_uPqsU,5579
 datacontract/export/rdf_converter.py,sha256=4gnKus37Geth4MJ3Ruc8AbnpD_Ll9OCx8oTIEKScvh8,6435
 datacontract/export/sodacl_converter.py,sha256=lQCOcNiT7i6KGaJ1Ua4MYBYGm-EyktTGrL4FLZDi14c,1102
 datacontract/export/spark_converter.py,sha256=LCue-rLan3ki7HgzUFyBaO8YUlc6CrDNBZD-QVgUv-U,7190
-datacontract/export/sql_converter.py,sha256=BGjmOAlzB5QfzJiXP61ajV0wj4M5oJrmNZZe_4Lo1Ik,4821
-datacontract/export/sql_type_converter.py,sha256=qjm8Fdyihq3VBL4x2D7RHdWoOm6HWIJe28U4XboYCk8,13436
+datacontract/export/sql_converter.py,sha256=KA5PNmKxUWF_8QXRX8aGvMGf7pX0rZgkLe6mbK8Q7Qk,5089
+datacontract/export/sql_type_converter.py,sha256=-ZHDihXWd5Gr9XG5FyE5-NLfB5q-HTdjx6P6TdsckDA,13497
 datacontract/export/sqlalchemy_converter.py,sha256=0DMncvA811lTtd5q4ZORREQ9YH1vQm1lJeqMWsFvloE,6463
 datacontract/export/terraform_converter.py,sha256=ExFoEvErVk-gBnWJiqC38SxDUmUEydpACWc917l5RyM,2163
 datacontract/imports/avro_importer.py,sha256=ryu4iUCSPJEV1uaE3AKdxD7fUxmRJ-ta936xurbgtHc,10922
@@ -68,7 +68,7 @@ datacontract/imports/odcs_importer.py,sha256=vv2dHLGL0Cdivv1CdKn5euJwGNKmiZmXCox
 datacontract/imports/odcs_v3_importer.py,sha256=sZVBENcPMl6rt0bbT_b1lnTFs3KOe1cZ2hwWaJBQhgY,16924
 datacontract/imports/parquet_importer.py,sha256=W_0_16mX4stwDUt4GM2L7dnGmTpAySab5k13-OlTCCc,3095
 datacontract/imports/protobuf_importer.py,sha256=rlUIskv9PNi5rFQ4Hobt9zlnKpahGsb4dy5G5UJoVAw,10840
-datacontract/imports/spark_importer.py,sha256=h2na1YtdJYu9Oz07tSvwx8L4RX6aLCCDVkAv-RTKyVA,5100
+datacontract/imports/spark_importer.py,sha256=8sPI6tcH0aMlaUYdc2P_abrJfOBp7vWFgvYE5jRNwLU,5177
 datacontract/imports/sql_importer.py,sha256=ElFS2LILDOvWzW-X4emSIKltFV42i78TEoyg0bvn3II,9322
 datacontract/imports/unity_importer.py,sha256=UcPYABhLZaWNl5IkCazwAuMoVDdujsu_QteuV_Q9hgI,6737
 datacontract/init/init_template.py,sha256=BMawR-AF_vXyn_-Co-XoT8Dxj9b55V8xKk2KkJK-c1o,721
@@ -107,9 +107,9 @@ datacontract/templates/partials/model_field.html,sha256=2YBF95ypNCPFYuYKoeilRnDG
 datacontract/templates/partials/quality.html,sha256=ynEDWRn8I90Uje-xhGYgFcfwOgKI1R-CDki-EvTsauQ,1785
 datacontract/templates/partials/server.html,sha256=WkWFbz1ZvhIAUQQhH5Lkwb0HZRW907ehEnFmJSkpquQ,6235
 datacontract/templates/style/output.css,sha256=V1k6smSvlz07W2UNOkhcDFUb0HLmoas7DnNg_o8XUcA,25759
-datacontract_cli-0.10.25.dist-info/licenses/LICENSE,sha256=23h64qnSeIZ0DKeziWAKC-zBCt328iSbRbWBrXoYRb4,2210
-datacontract_cli-0.10.25.dist-info/METADATA,sha256=Kc_fZ_wuwVoDjKluxLUkH0-chM_b_lMcsCUnm9tNlyo,104455
-datacontract_cli-0.10.25.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-datacontract_cli-0.10.25.dist-info/entry_points.txt,sha256=D3Eqy4q_Z6bHauGd4ppIyQglwbrm1AJnLau4Ppbw9Is,54
-datacontract_cli-0.10.25.dist-info/top_level.txt,sha256=VIRjd8EIUrBYWjEXJJjtdUgc0UAJdPZjmLiOR8BRBYM,13
-datacontract_cli-0.10.25.dist-info/RECORD,,
+datacontract_cli-0.10.26.dist-info/licenses/LICENSE,sha256=23h64qnSeIZ0DKeziWAKC-zBCt328iSbRbWBrXoYRb4,2210
+datacontract_cli-0.10.26.dist-info/METADATA,sha256=wK_1AwnrBPOBTCRKORH63413TPMUkiVJGBSEs6GmQtQ,104876
+datacontract_cli-0.10.26.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+datacontract_cli-0.10.26.dist-info/entry_points.txt,sha256=D3Eqy4q_Z6bHauGd4ppIyQglwbrm1AJnLau4Ppbw9Is,54
+datacontract_cli-0.10.26.dist-info/top_level.txt,sha256=VIRjd8EIUrBYWjEXJJjtdUgc0UAJdPZjmLiOR8BRBYM,13
+datacontract_cli-0.10.26.dist-info/RECORD,,

{datacontract_cli-0.10.25.dist-info → datacontract_cli-0.10.26.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any