datacontract-cli 0.10.25__tar.gz → 0.10.26__tar.gz

{datacontract_cli-0.10.25/datacontract_cli.egg-info → datacontract_cli-0.10.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 0.10.25
+Version: 0.10.26
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -70,7 +70,7 @@ Provides-Extra: rdf
 Requires-Dist: rdflib==7.0.0; extra == "rdf"
 Provides-Extra: api
 Requires-Dist: fastapi==0.115.12; extra == "api"
-Requires-Dist: uvicorn==0.34.0; extra == "api"
+Requires-Dist: uvicorn==0.34.2; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
 Provides-Extra: all
@@ -84,7 +84,7 @@ Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<4.3.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: pymssql==2.3.2; extra == "dev"
+Requires-Dist: pymssql==2.3.4; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.2; extra == "dev"
 Requires-Dist: trino==0.333.0; extra == "dev"
@@ -1397,20 +1397,21 @@ Available import options:
 
 | Type               | Description                                    | Status |
 |--------------------|------------------------------------------------|--------|
-| `sql`              | Import from SQL DDL                            | ✅      |
 | `avro`             | Import from AVRO schemas                       | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
+| `csv`              | Import from CSV File                           | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `dbt`              | Import from dbt models                         | ✅      |
+| `excel`            | Import from ODCS Excel Template                | ✅      |
+| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
+| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `spark`            | Import from Spark StructTypes                  | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
+| `parquet`          | Import from Parquet File Metadata              | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | ✅    |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `parquet`          | Import from Parquet File Metadta               | ✅      |
+| `spark`            | Import from Spark StructTypes                  | ✅      |
+| `sql`              | Import from SQL DDL                            | ✅      |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
 
@@ -1460,8 +1461,9 @@ datacontract import --format unity --source my_unity_table.json
 
 ```bash
 # Example import single table from Unity Catalog via HTTP endpoint
-export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
-export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
+export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4"
+export DATACONTRACT_DATABRICKS_TOKEN=<token>
 datacontract import --format unity --unity-table-full-name <table_full_name>
 ```
 
@@ -1482,6 +1484,17 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
 datacontract import --format dbt --source <manifest_path>
 ```
 
+### Excel
+
+Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
+
+Examples:
+
+```bash
+# Example import from ODCS Excel Template
+datacontract import --format excel --source odcs.xlsx
+```
+
 #### Glue
 
 Importing from Glue reads the necessary Data directly off of the AWS API.

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/README.md

@@ -1305,20 +1305,21 @@ Available import options:
 
 | Type               | Description                                    | Status |
 |--------------------|------------------------------------------------|--------|
-| `sql`              | Import from SQL DDL                            | ✅      |
 | `avro`             | Import from AVRO schemas                       | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
+| `csv`              | Import from CSV File                           | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `dbt`              | Import from dbt models                         | ✅      |
+| `excel`            | Import from ODCS Excel Template                | ✅      |
+| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
+| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `spark`            | Import from Spark StructTypes                  | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
+| `parquet`          | Import from Parquet File Metadata              | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | ✅    |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `parquet`          | Import from Parquet File Metadta               | ✅      |
+| `spark`            | Import from Spark StructTypes                  | ✅      |
+| `sql`              | Import from SQL DDL                            | ✅      |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
 
@@ -1368,8 +1369,9 @@ datacontract import --format unity --source my_unity_table.json
 
 ```bash
 # Example import single table from Unity Catalog via HTTP endpoint
-export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
-export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
+export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4"
+export DATACONTRACT_DATABRICKS_TOKEN=<token>
 datacontract import --format unity --unity-table-full-name <table_full_name>
 ```
 
@@ -1390,6 +1392,17 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
 datacontract import --format dbt --source <manifest_path>
 ```
 
+### Excel
+
+Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
+
+Examples:
+
+```bash
+# Example import from ODCS Excel Template
+datacontract import --format excel --source odcs.xlsx
+```
+
 #### Glue
 
 Importing from Glue reads the necessary Data directly off of the AWS API.

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/datacontract/engines/soda/connections/kafka.py

@@ -27,6 +27,7 @@ def create_spark_session():
     tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark")
     atexit.register(tmp_dir.cleanup)
 
+    pyspark_version = "3.5.5"  # MUST be the same as in the pyproject.toml
     spark = (
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
@@ -34,7 +35,7 @@ def create_spark_session():
         .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5",
+            f"org.apache.spark:spark-sql-kafka-0-10_2.12:{pyspark_version},org.apache.spark:spark-avro_2.12:{pyspark_version}",
         )
         .getOrCreate()
     )

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/datacontract/export/odcs_v3_exporter.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Any, Dict
 
 from open_data_contract_standard.model import (
     CustomProperty,
@@ -202,13 +202,31 @@ def to_logical_type(type: str) -> str | None:
         return "array"
     if type.lower() in ["array"]:
         return "array"
+    if type.lower() in ["variant"]:
+        return "variant"
     if type.lower() in ["null"]:
         return None
     return None
 
 
-def to_physical_type(type: str) -> str | None:
-    return type
+def to_physical_type(config: Dict[str, Any]) -> str | None:
+    if config is None:
+        return None
+    if "postgresType" in config:
+        return config["postgresType"]
+    elif "bigqueryType" in config:
+        return config["bigqueryType"]
+    elif "snowflakeType" in config:
+        return config["snowflakeType"]
+    elif "redshiftType" in config:
+        return config["redshiftType"]
+    elif "sqlserverType" in config:
+        return config["sqlserverType"]
+    elif "databricksType" in config:
+        return config["databricksType"]
+    elif "physicalType" in config:
+        return config["physicalType"]
+    return None
 
 
 def to_property(field_name: str, field: Field) -> SchemaProperty:
@@ -231,7 +249,7 @@ def to_property(field_name: str, field: Field) -> SchemaProperty:
 
     if field.type is not None:
         property.logicalType = to_logical_type(field.type)
-        property.physicalType = to_physical_type(field.type)
+        property.physicalType = to_physical_type(field.config)
 
     if field.description is not None:
         property.description = field.description

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/datacontract/export/sql_converter.py

@@ -117,6 +117,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'
+        if server_type == "snowflake" and field.description is not None:
+            result += f" COMMENT '{_escape(field.description)}'"
         if current_field_index < fields:
             result += ","
         result += "\n"
@@ -124,6 +126,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
     result += ")"
     if server_type == "databricks" and model.description is not None:
         result += f' COMMENT "{_escape(model.description)}"'
+    if server_type == "snowflake" and model.description is not None:
+        result += f" COMMENT='{_escape(model.description)}'"
     result += ";\n"
     return result
 

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/datacontract/export/sql_type_converter.py

@@ -197,6 +197,8 @@ def convert_to_databricks(field: Field) -> None | str:
     if type.lower() in ["array"]:
         item_type = convert_to_databricks(field.items)
         return f"ARRAY<{item_type}>"
+    if type.lower() in ["variant"]:
+        return "VARIANT"
     return None
 
 

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/datacontract/imports/spark_importer.py

@@ -154,5 +154,7 @@ def _data_type_from_spark(spark_type: types.DataType) -> str:
         return "null"
     elif isinstance(spark_type, types.VarcharType):
         return "varchar"
+    elif isinstance(spark_type, types.VariantType):
+        return "variant"
     else:
         raise ValueError(f"Unsupported Spark type: {spark_type}")

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26/datacontract_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 0.10.25
+Version: 0.10.26
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -70,7 +70,7 @@ Provides-Extra: rdf
 Requires-Dist: rdflib==7.0.0; extra == "rdf"
 Provides-Extra: api
 Requires-Dist: fastapi==0.115.12; extra == "api"
-Requires-Dist: uvicorn==0.34.0; extra == "api"
+Requires-Dist: uvicorn==0.34.2; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
 Provides-Extra: all
@@ -84,7 +84,7 @@ Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<4.3.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: pymssql==2.3.2; extra == "dev"
+Requires-Dist: pymssql==2.3.4; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.2; extra == "dev"
 Requires-Dist: trino==0.333.0; extra == "dev"
@@ -1397,20 +1397,21 @@ Available import options:
 
 | Type               | Description                                    | Status |
 |--------------------|------------------------------------------------|--------|
-| `sql`              | Import from SQL DDL                            | ✅      |
 | `avro`             | Import from AVRO schemas                       | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
+| `csv`              | Import from CSV File                           | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `dbt`              | Import from dbt models                         | ✅      |
+| `excel`            | Import from ODCS Excel Template                | ✅      |
+| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
+| `jsonschema`       | Import from JSON Schemas                       | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `spark`            | Import from Spark StructTypes                  | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
+| `parquet`          | Import from Parquet File Metadata              | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | ✅    |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `parquet`          | Import from Parquet File Metadta               | ✅      |
+| `spark`            | Import from Spark StructTypes                  | ✅      |
+| `sql`              | Import from SQL DDL                            | ✅      |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
 
@@ -1460,8 +1461,9 @@ datacontract import --format unity --source my_unity_table.json
 
 ```bash
 # Example import single table from Unity Catalog via HTTP endpoint
-export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
-export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
+export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4"
+export DATACONTRACT_DATABRICKS_TOKEN=<token>
 datacontract import --format unity --unity-table-full-name <table_full_name>
 ```
 
@@ -1482,6 +1484,17 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
 datacontract import --format dbt --source <manifest_path>
 ```
 
+### Excel
+
+Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
+
+Examples:
+
+```bash
+# Example import from ODCS Excel Template
+datacontract import --format excel --source odcs.xlsx
+```
+
 #### Glue
 
 Importing from Glue reads the necessary Data directly off of the AWS API.

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/datacontract_cli.egg-info/requires.txt

@@ -23,7 +23,7 @@ datacontract-cli[api,bigquery,csv,databricks,dbml,dbt,excel,iceberg,kafka,parque
 
 [api]
 fastapi==0.115.12
-uvicorn==0.34.0
+uvicorn==0.34.2
 
 [avro]
 avro==1.12.0
@@ -55,7 +55,7 @@ pandas>=2.1.0
 pre-commit<4.3.0,>=3.7.1
 pytest
 pytest-xdist
-pymssql==2.3.2
+pymssql==2.3.4
 ruff
 testcontainers[kafka,minio,mssql,postgres]==4.9.2
 trino==0.333.0

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datacontract-cli"
-version = "0.10.25"
+version = "0.10.26"
 description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library."
 license = "MIT"
 readme = "README.md"
@@ -112,7 +112,7 @@ rdf = [
 
 api = [
   "fastapi==0.115.12",
-  "uvicorn==0.34.0",
+  "uvicorn==0.34.2",
 ]
 
 protobuf = [
@@ -132,7 +132,7 @@ dev = [
   "pre-commit>=3.7.1,<4.3.0",
   "pytest",
   "pytest-xdist",
-  "pymssql==2.3.2",
+  "pymssql==2.3.4",
   "ruff",
   "testcontainers[minio,postgres,kafka,mssql]==4.9.2",
   "trino==0.333.0",

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/tests/test_export_odcs_v3.py

@@ -44,7 +44,6 @@ schema:
             minLength: 8
             maxLength: 10
             pattern: ^B[0-9]+$
-        physicalType: varchar
         required: true
         unique: true
         tags:
@@ -63,7 +62,6 @@ schema:
         logicalTypeOptions:
             minimum: 0
             maximum: 1000000
-        physicalType: bigint
         required: true
         description: The order_total field
         quality:
@@ -75,7 +73,6 @@ schema:
             mustBeBetween: [1000, 49900]
       - name: order_status
         logicalType: string
-        physicalType: text
         required: true
     quality:
     - type: sql

{datacontract_cli-0.10.25 → datacontract_cli-0.10.26}/tests/test_export_sql.py

@@ -32,18 +32,18 @@ def test_to_sql_ddl_snowflake():
 -- Data Contract: urn:datacontract:checkout:snowflake_orders_pii_v2
 -- SQL Dialect: snowflake
 CREATE TABLE orders (
-  ORDER_ID TEXT not null,
-  ORDER_TIMESTAMP TIMESTAMP_TZ not null,
-  ORDER_TOTAL NUMBER not null,
-  CUSTOMER_ID TEXT,
-  CUSTOMER_EMAIL_ADDRESS TEXT not null,
-  PROCESSING_TIMESTAMP TIMESTAMP_LTZ not null
-);
+  ORDER_ID TEXT not null COMMENT 'An internal ID that identifies an order in the online shop.',
+  ORDER_TIMESTAMP TIMESTAMP_TZ not null COMMENT 'The business timestamp in UTC when the order was successfully registered in the source system and the payment was successful.',
+  ORDER_TOTAL NUMBER not null COMMENT 'Total amount the smallest monetary unit (e.g., cents).',
+  CUSTOMER_ID TEXT COMMENT 'Unique identifier for the customer.',
+  CUSTOMER_EMAIL_ADDRESS TEXT not null COMMENT 'The email address, as entered by the customer. The email address was not verified.',
+  PROCESSING_TIMESTAMP TIMESTAMP_LTZ not null COMMENT 'The processing timestamp in the current session’s time zone.'
+) COMMENT='One record per order. Includes cancelled and deleted orders.';
 CREATE TABLE line_items (
-  LINE_ITEM_ID TEXT not null,
-  ORDER_ID TEXT,
-  SKU TEXT
-);
+  LINE_ITEM_ID TEXT not null COMMENT 'Primary key of the lines_item_id table',
+  ORDER_ID TEXT COMMENT 'An internal ID that identifies an order in the online shop.',
+  SKU TEXT COMMENT 'The purchased article number'
+) COMMENT='A single article that is part of an order.';
 """.strip()
     assert actual == expected