datacontract-cli 0.11.1__tar.gz → 0.11.3__tar.gz

{datacontract_cli-0.11.1/datacontract_cli.egg-info → datacontract_cli-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 0.11.1
+Version: 0.11.3
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -16,12 +16,12 @@ Requires-Dist: pydantic<2.13.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.33,>=2.31
 Requires-Dist: fastjsonschema<2.22.0,>=2.19.1
-Requires-Dist: fastparquet<2025.0.0,>=2024.5.0
+Requires-Dist: fastparquet<2026.0.0,>=2024.5.0
 Requires-Dist: numpy<2.0.0,>=1.26.4
 Requires-Dist: python-multipart<1.0.0,>=0.0.20
 Requires-Dist: rich<15.0,>=13.7
-Requires-Dist: sqlglot<28.0.0,>=26.6.0
-Requires-Dist: duckdb<2.0.0,>=1.0.0
+Requires-Dist: sqlglot<29.0.0,>=26.6.0
+Requires-Dist: duckdb<1.4.0,>=1.0.0
 Requires-Dist: soda-core-duckdb<3.6.0,>=3.3.20
 Requires-Dist: setuptools>=60
 Requires-Dist: python-dotenv<2.0.0,>=1.0.0
@@ -29,7 +29,7 @@ Requires-Dist: boto3<2.0.0,>=1.34.41
 Requires-Dist: Jinja2<4.0.0,>=3.1.5
 Requires-Dist: jinja_partials<1.0.0,>=0.2.1
 Requires-Dist: datacontract-specification<2.0.0,>=1.2.3
-Requires-Dist: open-data-contract-standard<4.0.0,>=3.1.0
+Requires-Dist: open-data-contract-standard<4.0.0,>=3.1.2
 Provides-Extra: avro
 Requires-Dist: avro==1.12.1; extra == "avro"
 Provides-Extra: bigquery
@@ -41,22 +41,22 @@ Requires-Dist: openpyxl<4.0.0,>=3.1.5; extra == "excel"
 Provides-Extra: databricks
 Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.6.0,>=3.3.20; extra == "databricks"
-Requires-Dist: databricks-sql-connector<4.2.0,>=3.7.0; extra == "databricks"
+Requires-Dist: databricks-sql-connector<4.3.0,>=3.7.0; extra == "databricks"
 Requires-Dist: databricks-sdk<0.74.0; extra == "databricks"
-Requires-Dist: pyspark<4.0.0,>=3.5.5; extra == "databricks"
+Requires-Dist: pyspark<5.0.0,>=3.5.5; extra == "databricks"
 Provides-Extra: iceberg
 Requires-Dist: pyiceberg==0.10.0; extra == "iceberg"
 Provides-Extra: kafka
 Requires-Dist: datacontract-cli[avro]; extra == "kafka"
 Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "kafka"
-Requires-Dist: pyspark<4.0.0,>=3.5.5; extra == "kafka"
+Requires-Dist: pyspark<5.0.0,>=3.5.5; extra == "kafka"
 Provides-Extra: postgres
 Requires-Dist: soda-core-postgres<3.6.0,>=3.3.20; extra == "postgres"
 Provides-Extra: s3
 Requires-Dist: s3fs<2026.0.0,>=2025.2.0; extra == "s3"
-Requires-Dist: aiobotocore<2.26.0,>=2.17.0; extra == "s3"
+Requires-Dist: aiobotocore<3.2.0,>=2.17.0; extra == "s3"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python[pandas]<4.1,>=3.6; extra == "snowflake"
+Requires-Dist: snowflake-connector-python[pandas]<4.2,>=3.6; extra == "snowflake"
 Requires-Dist: soda-core-snowflake<3.6.0,>=3.3.20; extra == "snowflake"
 Provides-Extra: sqlserver
 Requires-Dist: soda-core-sqlserver<3.6.0,>=3.3.20; extra == "sqlserver"
@@ -73,10 +73,10 @@ Requires-Dist: pydbml>=1.1.1; extra == "dbml"
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
 Provides-Extra: rdf
-Requires-Dist: rdflib==7.0.0; extra == "rdf"
+Requires-Dist: rdflib==7.5.0; extra == "rdf"
 Provides-Extra: api
 Requires-Dist: fastapi==0.121.2; extra == "api"
-Requires-Dist: uvicorn==0.38.0; extra == "api"
+Requires-Dist: uvicorn==0.40.0; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
 Provides-Extra: all
@@ -86,9 +86,9 @@ Requires-Dist: datacontract-cli[all]; extra == "dev"
 Requires-Dist: httpx==0.28.1; extra == "dev"
 Requires-Dist: kafka-python; extra == "dev"
 Requires-Dist: minio==7.2.17; extra == "dev"
-Requires-Dist: moto==5.1.17; extra == "dev"
+Requires-Dist: moto==5.1.18; extra == "dev"
 Requires-Dist: pandas>=2.1.0; extra == "dev"
-Requires-Dist: pre-commit<4.5.0,>=3.7.1; extra == "dev"
+Requires-Dist: pre-commit<4.6.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
 Requires-Dist: pymssql==2.3.9; extra == "dev"
@@ -120,7 +120,7 @@ It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a P
 Let's look at this data contract:
 [https://datacontract.com/orders-v1.odcs.yaml](https://datacontract.com/orders-v1.odcs.yaml)
 
-We have a _servers_ section with endpoint details to a Postgres database, _schema_ for the structure and semantics of the data, _slaAttributes_ and _quality_ attributes that describe the expected freshness and number of rows.
+We have a _servers_ section with endpoint details to a Postgres database, _schema_ for the structure and semantics of the data, _service levels_ and _quality_ attributes that describe the expected freshness and number of rows.
 
 This data contract contains all information to connect to the database and check that the actual data meets the defined schema specification and quality expectations.
 We can use this information to test if the actual data product is compliant to the data contract.
@@ -342,6 +342,7 @@ A list of available extras:
 | Snowflake Integration   | `pip install datacontract-cli[snowflake]`  |
 | Microsoft SQL Server    | `pip install datacontract-cli[sqlserver]`  |
 | Trino                   | `pip install datacontract-cli[trino]`      |
+| Impala                  | `pip install datacontract-cli[impala]` 	   |
 | dbt                     | `pip install datacontract-cli[dbt]`        |
 | DBML                    | `pip install datacontract-cli[dbml]`       |
 | Parquet                 | `pip install datacontract-cli[parquet]`    |
@@ -493,6 +494,7 @@ Supported server types:
 - [kafka](#kafka)
 - [postgres](#postgres)
 - [trino](#trino)
+- [impala](#impala)
 - [api](#api)
 - [local](#local)
 
@@ -655,8 +657,7 @@ datacontract.yaml
 servers:
   production:
     type: azure
-    storageAccount: datameshdatabricksdemo
-    location: abfss://dataproducts/inventory_events/*.parquet
+    location: abfss://datameshdatabricksdemo.dfs.core.windows.net/inventory_events/*.parquet
     format: parquet
 ```
 
@@ -1023,6 +1024,53 @@ models:
 | `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password    |
 
 
+#### Impala
+
+Data Contract CLI can run Soda checks against an Apache Impala cluster.
+
+##### Example
+
+datacontract.yaml
+```yaml
+servers:
+  impala:
+    type: impala
+    host: my-impala-host
+    port: 443
+    # Optional default database used for Soda scans
+    database: my_database
+models:
+  my_table_1: # corresponds to a table
+    type: table
+    # fields as usual …
+```
+
+##### Environment Variables
+
+| Environment Variable                      | Example               | Description                                               |
+|-------------------------------            |--------------------   |-------------                                              |
+| `DATACONTRACT_IMPALA_USERNAME`            | `analytics_user`      | Username used to connect to Impala                        |
+| `DATACONTRACT_IMPALA_PASSWORD`            | `mysecretpassword`    | Password for the Impala user                              |
+| `DATACONTRACT_IMPALA_USE_SSL`             | `true`                | Whether to use SSL; defaults to true if unset             |
+| `DATACONTRACT_IMPALA_AUTH_MECHANISM`      | `LDAP`                | Authentication mechanism; defaults to LDAP                |
+| `DATACONTRACT_IMPALA_USE_HTTP_TRANSPORT`  | `true`                | Whether to use the HTTP transport; defaults to true       |
+| `DATACONTRACT_IMPALA_HTTP_PATH`           | `cliservice`          | HTTP path for the Impala service; defaults to cliservice  |
+
+### Type-mapping note (logicalType → Impala type)
+
+If `physicalType` is not specified in the schema, we recommend the following mapping from `logicalType` to Impala column types:
+
+|logicalType | Recommended Impala type |
+|------------|-------------------------|
+| `integer`  | `INT` or `BIGINT`       |
+| `number`   | `DOUBLE`/`decimal(..)`  |
+| `string`   | `STRING` or `VARCHAR`   |
+| `boolean`  | `BOOLEAN`               |
+| `date`     | `DATE`                  |
+| `datetime` | `TIMESTAMP`             |
+
+This keeps the Impala schema compatible with the expectations of the Soda checks generated by datacontract-cli.
+
 #### API
 
 Data Contract CLI can test APIs that return data in JSON format. 
@@ -1569,38 +1617,25 @@ datacontract import --format sql --source my_ddl.sql --dialect postgres --output
 
 Available import options:
 
-| Type               | Description                                    | Status |
-|--------------------|------------------------------------------------|--------|
-| `avro`             | Import from AVRO schemas                       | ✅      |
-| `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `dbt`              | Import from dbt models                         | ✅      |
-| `excel`            | Import from ODCS Excel Template                | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
-| `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `parquet`          | Import from Parquet File Metadata              | ✅      |
-| `protobuf`         | Import from Protobuf schemas                   | ✅      |
-| `spark`            | Import from Spark StructTypes, Variant         | ✅      |
-| `sql`              | Import from SQL DDL                            | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
-| `excel`            | Import from ODCS Excel Template                | ✅      |
-| Missing something? | Please create an issue on GitHub               | TBD     |
-
-
-#### ODCS
-
-Import from Open Data Contract Standard (ODCS) v2 or v3.
-The importer automatically detects the ODCS version and imports the data contract.
-
-Examples:
+| Type               | Description                                   | Status  |
+|--------------------|-----------------------------------------------|---------|
+| `avro`             | Import from AVRO schemas                      | ✅       |
+| `bigquery`         | Import from BigQuery Schemas                  | ✅       |
+| `csv`              | Import from CSV File                          | ✅       |
+| `dbml`             | Import from DBML models                       | ✅       |
+| `dbt`              | Import from dbt models                        | ✅       |
+| `excel`            | Import from ODCS Excel Template               | ✅       |
+| `glue`             | Import from AWS Glue DataCatalog              | ✅       |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition | partial |
+| `jsonschema`       | Import from JSON Schemas                      | ✅       |
+| `parquet`          | Import from Parquet File Metadata             | ✅       |
+| `protobuf`         | Import from Protobuf schemas                  | ✅       |
+| `spark`            | Import from Spark StructTypes, Variant        | ✅       |
+| `sql`              | Import from SQL DDL                           | ✅       |
+| `unity`            | Import from Databricks Unity Catalog          | partial |
+| `excel`            | Import from ODCS Excel Template               | ✅       |
+| Missing something? | Please create an issue on GitHub              | TBD     |
 
-```bash
-# Example import from ODCS
-datacontract import --format odcs --source my_data_contract.odcs.yaml
-```
 
 #### BigQuery
 
@@ -2202,8 +2237,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
 
 ## Related Tools
 
-- [Data Contract Manager](https://www.datacontract-manager.com/) is a commercial tool to manage data contracts. It contains a web UI, access management, and data governance for a full enterprise data marketplace.
-- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
+- [Entropy Data](https://www.entropy-data.com/) is a commercial tool to manage data contracts. It contains a web UI, access management, and data governance for a data product marketplace based on data contracts.
 - [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
 - [Data Contract Playground](https://data-catering.github.io/data-contract-playground/) allows you to validate and export your data contract to different formats within your browser.
 

{datacontract_cli-0.11.1 → datacontract_cli-0.11.3}/README.md

@@ -21,7 +21,7 @@ It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a P
 Let's look at this data contract:
 [https://datacontract.com/orders-v1.odcs.yaml](https://datacontract.com/orders-v1.odcs.yaml)
 
-We have a _servers_ section with endpoint details to a Postgres database, _schema_ for the structure and semantics of the data, _slaAttributes_ and _quality_ attributes that describe the expected freshness and number of rows.
+We have a _servers_ section with endpoint details to a Postgres database, _schema_ for the structure and semantics of the data, _service levels_ and _quality_ attributes that describe the expected freshness and number of rows.
 
 This data contract contains all information to connect to the database and check that the actual data meets the defined schema specification and quality expectations.
 We can use this information to test if the actual data product is compliant to the data contract.
@@ -243,6 +243,7 @@ A list of available extras:
 | Snowflake Integration   | `pip install datacontract-cli[snowflake]`  |
 | Microsoft SQL Server    | `pip install datacontract-cli[sqlserver]`  |
 | Trino                   | `pip install datacontract-cli[trino]`      |
+| Impala                  | `pip install datacontract-cli[impala]` 	   |
 | dbt                     | `pip install datacontract-cli[dbt]`        |
 | DBML                    | `pip install datacontract-cli[dbml]`       |
 | Parquet                 | `pip install datacontract-cli[parquet]`    |
@@ -394,6 +395,7 @@ Supported server types:
 - [kafka](#kafka)
 - [postgres](#postgres)
 - [trino](#trino)
+- [impala](#impala)
 - [api](#api)
 - [local](#local)
 
@@ -556,8 +558,7 @@ datacontract.yaml
 servers:
   production:
     type: azure
-    storageAccount: datameshdatabricksdemo
-    location: abfss://dataproducts/inventory_events/*.parquet
+    location: abfss://datameshdatabricksdemo.dfs.core.windows.net/inventory_events/*.parquet
     format: parquet
 ```
 
@@ -924,6 +925,53 @@ models:
 | `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password    |
 
 
+#### Impala
+
+Data Contract CLI can run Soda checks against an Apache Impala cluster.
+
+##### Example
+
+datacontract.yaml
+```yaml
+servers:
+  impala:
+    type: impala
+    host: my-impala-host
+    port: 443
+    # Optional default database used for Soda scans
+    database: my_database
+models:
+  my_table_1: # corresponds to a table
+    type: table
+    # fields as usual …
+```
+
+##### Environment Variables
+
+| Environment Variable                      | Example               | Description                                               |
+|-------------------------------            |--------------------   |-------------                                              |
+| `DATACONTRACT_IMPALA_USERNAME`            | `analytics_user`      | Username used to connect to Impala                        |
+| `DATACONTRACT_IMPALA_PASSWORD`            | `mysecretpassword`    | Password for the Impala user                              |
+| `DATACONTRACT_IMPALA_USE_SSL`             | `true`                | Whether to use SSL; defaults to true if unset             |
+| `DATACONTRACT_IMPALA_AUTH_MECHANISM`      | `LDAP`                | Authentication mechanism; defaults to LDAP                |
+| `DATACONTRACT_IMPALA_USE_HTTP_TRANSPORT`  | `true`                | Whether to use the HTTP transport; defaults to true       |
+| `DATACONTRACT_IMPALA_HTTP_PATH`           | `cliservice`          | HTTP path for the Impala service; defaults to cliservice  |
+
+### Type-mapping note (logicalType → Impala type)
+
+If `physicalType` is not specified in the schema, we recommend the following mapping from `logicalType` to Impala column types:
+
+|logicalType | Recommended Impala type |
+|------------|-------------------------|
+| `integer`  | `INT` or `BIGINT`       |
+| `number`   | `DOUBLE`/`decimal(..)`  |
+| `string`   | `STRING` or `VARCHAR`   |
+| `boolean`  | `BOOLEAN`               |
+| `date`     | `DATE`                  |
+| `datetime` | `TIMESTAMP`             |
+
+This keeps the Impala schema compatible with the expectations of the Soda checks generated by datacontract-cli.
+
 #### API
 
 Data Contract CLI can test APIs that return data in JSON format. 
@@ -1470,38 +1518,25 @@ datacontract import --format sql --source my_ddl.sql --dialect postgres --output
 
 Available import options:
 
-| Type               | Description                                    | Status |
-|--------------------|------------------------------------------------|--------|
-| `avro`             | Import from AVRO schemas                       | ✅      |
-| `bigquery`         | Import from BigQuery Schemas                   | ✅      |
-| `csv`              | Import from CSV File                           | ✅      |
-| `dbml`             | Import from DBML models                        | ✅      |
-| `dbt`              | Import from dbt models                         | ✅      |
-| `excel`            | Import from ODCS Excel Template                | ✅      |
-| `glue`             | Import from AWS Glue DataCatalog               | ✅      |
-| `iceberg`          | Import from an Iceberg JSON Schema Definition  | partial |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
-| `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
-| `parquet`          | Import from Parquet File Metadata              | ✅      |
-| `protobuf`         | Import from Protobuf schemas                   | ✅      |
-| `spark`            | Import from Spark StructTypes, Variant         | ✅      |
-| `sql`              | Import from SQL DDL                            | ✅      |
-| `unity`            | Import from Databricks Unity Catalog           | partial |
-| `excel`            | Import from ODCS Excel Template                | ✅      |
-| Missing something? | Please create an issue on GitHub               | TBD     |
-
-
-#### ODCS
-
-Import from Open Data Contract Standard (ODCS) v2 or v3.
-The importer automatically detects the ODCS version and imports the data contract.
-
-Examples:
+| Type               | Description                                   | Status  |
+|--------------------|-----------------------------------------------|---------|
+| `avro`             | Import from AVRO schemas                      | ✅       |
+| `bigquery`         | Import from BigQuery Schemas                  | ✅       |
+| `csv`              | Import from CSV File                          | ✅       |
+| `dbml`             | Import from DBML models                       | ✅       |
+| `dbt`              | Import from dbt models                        | ✅       |
+| `excel`            | Import from ODCS Excel Template               | ✅       |
+| `glue`             | Import from AWS Glue DataCatalog              | ✅       |
+| `iceberg`          | Import from an Iceberg JSON Schema Definition | partial |
+| `jsonschema`       | Import from JSON Schemas                      | ✅       |
+| `parquet`          | Import from Parquet File Metadata             | ✅       |
+| `protobuf`         | Import from Protobuf schemas                  | ✅       |
+| `spark`            | Import from Spark StructTypes, Variant        | ✅       |
+| `sql`              | Import from SQL DDL                           | ✅       |
+| `unity`            | Import from Databricks Unity Catalog          | partial |
+| `excel`            | Import from ODCS Excel Template               | ✅       |
+| Missing something? | Please create an issue on GitHub              | TBD     |
 
-```bash
-# Example import from ODCS
-datacontract import --format odcs --source my_data_contract.odcs.yaml
-```
 
 #### BigQuery
 
@@ -2103,8 +2138,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
 
 ## Related Tools
 
-- [Data Contract Manager](https://www.datacontract-manager.com/) is a commercial tool to manage data contracts. It contains a web UI, access management, and data governance for a full enterprise data marketplace.
-- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
+- [Entropy Data](https://www.entropy-data.com/) is a commercial tool to manage data contracts. It contains a web UI, access management, and data governance for a data product marketplace based on data contracts.
 - [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
 - [Data Contract Playground](https://data-catering.github.io/data-contract-playground/) allows you to validate and export your data contract to different formats within your browser.
 

{datacontract_cli-0.11.1 → datacontract_cli-0.11.3}/datacontract/catalog/catalog.py

@@ -14,7 +14,7 @@ from datacontract.export.html_exporter import get_version
 
 def _get_owner(odcs: OpenDataContractStandard) -> Optional[str]:
     """Get the owner from ODCS customProperties or team."""
-    if odcs.team and odcs.team.name:
+    if odcs.team and hasattr(odcs.team, "name") and odcs.team.name:
         return odcs.team.name
     if odcs.customProperties:
         for prop in odcs.customProperties:

{datacontract_cli-0.11.1 → datacontract_cli-0.11.3}/datacontract/cli.py

@@ -67,7 +67,7 @@ def common(
     pass
 
 
-@app.command()
+@app.command(name="init")
 def init(
     location: Annotated[
         str, typer.Argument(help="The location of the data contract file to create.")
@@ -90,7 +90,7 @@ def init(
     console.print("📄 data contract written to " + location)
 
 
-@app.command()
+@app.command(name="lint")
 def lint(
     location: Annotated[
         str,
@@ -125,7 +125,7 @@ def enable_debug_logging(debug: bool):
         )
 
 
-@app.command()
+@app.command(name="test")
 def test(
     location: Annotated[
         str,
@@ -187,7 +187,7 @@ def test(
     write_test_result(run, console, output_format, output, data_contract)
 
 
-@app.command()
+@app.command(name="export")
 def export(
     format: Annotated[ExportFormat, typer.Option(help="The export format.")],
     output: Annotated[
@@ -467,7 +467,7 @@ def _get_uvicorn_arguments(port: int, host: str, context: typer.Context) -> dict
     return default_args | dict(zip(trimmed_keys, context.args[1::2]))
 
 
-@app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
+@app.command(name="api", context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
 def api(
     ctx: Annotated[typer.Context, typer.Option(help="Extra arguments to pass to uvicorn.run().")],
     port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4242,

{datacontract_cli-0.11.1 → datacontract_cli-0.11.3}/datacontract/engines/data_contract_checks.py

@@ -161,7 +161,7 @@ def to_schema_name(schema_object: SchemaObject, server_type: str) -> str:
     # Use physicalName if set (ODCS standard way to specify actual table name)
     if schema_object.physicalName:
         return schema_object.physicalName
-    
+
     return schema_object.name
 
 

{datacontract_cli-0.11.1 → datacontract_cli-0.11.3}/datacontract/engines/soda/check_soda_execute.py

@@ -14,6 +14,7 @@ from open_data_contract_standard.model import OpenDataContractStandard, Server
 from datacontract.engines.soda.connections.bigquery import to_bigquery_soda_configuration
 from datacontract.engines.soda.connections.databricks import to_databricks_soda_configuration
 from datacontract.engines.soda.connections.duckdb_connection import get_duckdb_connection
+from datacontract.engines.soda.connections.impala import to_impala_soda_configuration
 from datacontract.engines.soda.connections.kafka import create_spark_session, read_kafka_topic
 from datacontract.engines.soda.connections.postgres import to_postgres_soda_configuration
 from datacontract.engines.soda.connections.snowflake import to_snowflake_soda_configuration
@@ -95,6 +96,18 @@ def check_soda_execute(
             logging.info("Use Spark to connect to data source")
             scan.add_spark_session(spark, data_source_name="datacontract-cli")
             scan.set_data_source_name("datacontract-cli")
+    
+    # ------------------------------------------------------------------
+    # NEW: native Impala server type
+    # ------------------------------------------------------------------
+    elif server.type == "impala":
+        run.log_info("Connecting to Impala via Soda engine")
+        soda_configuration_str = to_impala_soda_configuration(server)
+        scan.add_configuration_yaml_str(soda_configuration_str)
+        # data source name must match what we configure in to_impala_soda_configuration
+        scan.set_data_source_name("impala")
+
+    
     elif server.type == "kafka":
         if spark is None:
             spark = create_spark_session()
@@ -218,4 +231,4 @@ def update_reason(check, c):
                 # print(check.reason)
             break  # Exit the loop once the desired block is found
     if "fail" in c["diagnostics"]:
-        check.reason = f"Value: {c['diagnostics']['value']} Fail: {c['diagnostics']['fail']}"
+        check.reason = f"Value: {c['diagnostics']['value']} Fail: {c['diagnostics']['fail']}"

{datacontract_cli-0.11.1 → datacontract_cli-0.11.3}/datacontract/engines/soda/connections/duckdb_connection.py

@@ -1,10 +1,12 @@
 import os
+import re
 from typing import Any, List, Optional
 
 import duckdb
 from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, SchemaProperty, Server
 
 from datacontract.export.duckdb_type_converter import convert_to_duckdb_csv_type, convert_to_duckdb_json_type
+from datacontract.export.sql_type_converter import convert_to_duckdb
 from datacontract.model.run import Run
 
 
@@ -57,20 +59,9 @@ def get_duckdb_connection(
                     )
                     add_nested_views(con, model_name, schema_obj.properties)
             elif server.format == "parquet":
-                con.sql(f"""
-                            CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
-                            """)
+                create_view_with_schema_union(con, schema_obj, model_path, "read_parquet", to_parquet_types)
             elif server.format == "csv":
-                columns = to_csv_types(schema_obj)
-                run.log_info("Using columns: " + str(columns))
-                if columns is None:
-                    con.sql(
-                        f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1);"""
-                    )
-                else:
-                    con.sql(
-                        f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1, columns={columns});"""
-                    )
+                create_view_with_schema_union(con, schema_obj, model_path, "read_csv", to_csv_types)
             elif server.format == "delta":
                 con.sql("update extensions;")  # Make sure we have the latest delta extension
                 con.sql(f"""CREATE VIEW "{model_name}" AS SELECT * FROM delta_scan('{model_path}');""")
@@ -80,6 +71,34 @@ def get_duckdb_connection(
     return con
 
 
+def create_view_with_schema_union(con, schema_obj: SchemaObject, model_path: str, read_function: str, type_converter):
+    """Create a view by unioning empty schema table with data files using union_by_name"""
+    converted_types = type_converter(schema_obj)
+    model_name = schema_obj.name
+    if converted_types:
+        # Create empty table with contract schema
+        columns_def = [f'"{col_name}" {col_type}' for col_name, col_type in converted_types.items()]
+        create_empty_table = f"""CREATE TABLE "{model_name}" ({', '.join(columns_def)});"""
+        con.sql(create_empty_table)
+
+        # Read columns existing in both current data contract and data
+        intersecting_columns = con.sql(f"""SELECT column_name
+            FROM (DESCRIBE SELECT * FROM {read_function}('{model_path}', union_by_name=true, hive_partitioning=1))
+            INTERSECT SELECT column_name
+            FROM information_schema.columns
+            WHERE table_name = '{model_name}'""").fetchall()
+        selected_columns = ', '.join([column[0] for column in intersecting_columns])
+
+        # Insert data into table by name, but only columns existing in contract and data
+        insert_data_sql = f"""INSERT INTO {model_name} BY NAME
+            (SELECT {selected_columns} FROM {read_function}('{model_path}', union_by_name=true, hive_partitioning=1));"""
+        con.sql(insert_data_sql)
+    else:
+        # Fallback
+        con.sql(
+            f"""CREATE VIEW "{model_name}" AS SELECT * FROM {read_function}('{model_path}', union_by_name=true, hive_partitioning=1);"""
+        )
+
 def to_csv_types(schema_obj: SchemaObject) -> dict[Any, str | None] | None:
     if schema_obj is None:
         return None
@@ -89,6 +108,15 @@ def to_csv_types(schema_obj: SchemaObject) -> dict[Any, str | None] | None:
             columns[prop.name] = convert_to_duckdb_csv_type(prop)
     return columns
 
+def to_parquet_types(schema_obj: SchemaObject) -> dict[Any, str | None] | None:
+    """Get proper SQL types for Parquet (preserves decimals, etc.)"""
+    if schema_obj is None:
+        return None
+    columns = {}
+    if schema_obj.properties:
+        for prop in schema_obj.properties:
+            columns[prop.name] = convert_to_duckdb(prop)
+    return columns
 
 def to_json_types(schema_obj: SchemaObject) -> dict[Any, str | None] | None:
     if schema_obj is None:
@@ -140,7 +168,7 @@ def add_nested_views(con: duckdb.DuckDBPyConnection, model_name: str, properties
             add_nested_views(con, nested_model_name, prop.properties)
 
 
-def setup_s3_connection(con, server):
+def setup_s3_connection(con, server: Server):
     s3_region = os.getenv("DATACONTRACT_S3_REGION")
     s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
     s3_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
@@ -184,7 +212,7 @@ def setup_s3_connection(con, server):
             """)
 
 
-def setup_gcs_connection(con, server):
+def setup_gcs_connection(con, server: Server):
     key_id = os.getenv("DATACONTRACT_GCS_KEY_ID")
     secret = os.getenv("DATACONTRACT_GCS_SECRET")
 
@@ -202,11 +230,14 @@ def setup_gcs_connection(con, server):
     """)
 
 
-def setup_azure_connection(con, server):
+def setup_azure_connection(con, server: Server):
     tenant_id = os.getenv("DATACONTRACT_AZURE_TENANT_ID")
     client_id = os.getenv("DATACONTRACT_AZURE_CLIENT_ID")
     client_secret = os.getenv("DATACONTRACT_AZURE_CLIENT_SECRET")
-    storage_account = server.storageAccount
+    storage_account = (
+        to_azure_storage_account(server.location) if server.type == "azure" and "://" in server.location
+            else None
+        )
 
     if tenant_id is None:
         raise ValueError("Error: Environment variable DATACONTRACT_AZURE_TENANT_ID is not set")
@@ -239,3 +270,25 @@ def setup_azure_connection(con, server):
             CLIENT_SECRET '{client_secret}'
         );
         """)
+
+def to_azure_storage_account(location: str) -> str | None:
+    """
+    Converts a storage location string to extract the storage account name.
+    ODCS v3.0 has no explicit field for the storage account. It uses the location field, which is a URI.
+    This function parses a storage location string to identify and return the
+    storage account name. It handles two primary patterns:
+    1. Protocol://containerName@storageAccountName
+    2. Protocol://storageAccountName
+    :param location: The storage location string to parse, typically following
+                     the format protocol://containerName@storageAccountName. or
+                     protocol://storageAccountName.
+    :return: The extracted storage account name if found, otherwise None
+    """
+    # to catch protocol://containerName@storageAccountName. pattern from location
+    match = re.search(r"(?<=@)([^.]*)", location, re.IGNORECASE)
+    if match:
+        return match.group()
+    else:
+        # to catch protocol://storageAccountName. pattern from location
+        match = re.search(r"(?<=//)(?!@)([^.]*)", location, re.IGNORECASE)
+    return match.group() if match else None