datacontract-cli 1.0.17__tar.gz → 1.1.0__tar.gz

{datacontract_cli-1.0.17/datacontract_cli.egg-info → datacontract_cli-1.1.0}/PKG-INFO

@@ -1,13 +1,29 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.17
+Version: 1.1.0
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
-Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
+Author-email: Jochen Christ <jochen.christ@entropy-data.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@entropy-data.com>
 License-Expression: MIT
 Project-URL: Homepage, https://cli.datacontract.com
+Project-URL: Documentation, https://docs.datacontract.com
+Project-URL: Repository, https://github.com/datacontract/datacontract-cli
 Project-URL: Issues, https://github.com/datacontract/datacontract-cli/issues
-Classifier: Programming Language :: Python :: 3
+Project-URL: Changelog, https://docs.datacontract.com/release-notes
+Keywords: data-contract,data-contracts,odcs,open-data-contract-standard,data-quality,data-governance,data-mesh,data-product,data-engineering,schema,dbt,snowflake,databricks,bigquery
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
 Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
 Requires-Python: <3.15,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -41,24 +57,26 @@ Requires-Dist: pandas>=2.0.0; extra == "csv"
 Provides-Extra: excel
 Requires-Dist: openpyxl<4.0.0,>=3.1.5; extra == "excel"
 Provides-Extra: dataframe
-Requires-Dist: ibis-framework[pyspark]<13.0.0,>=10.0.0; extra == "dataframe"
-Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "dataframe"
+Requires-Dist: ibis-framework<13.0.0,>=10.0.0; extra == "dataframe"
+Requires-Dist: numpy<3,>=1.23.2; extra == "dataframe"
+Requires-Dist: packaging<27.0,>=21.3; extra == "dataframe"
+Requires-Dist: pandas<4,>=1.5.3; extra == "dataframe"
+Requires-Dist: pyarrow>=10.0.1; extra == "dataframe"
+Requires-Dist: pyarrow-hotfix>=0.4; extra == "dataframe"
+Requires-Dist: rich>=12.4.4; extra == "dataframe"
 Provides-Extra: databricks
-Requires-Dist: ibis-framework[databricks,pyspark]<13.0.0,>=10.0.0; extra == "databricks"
+Requires-Dist: ibis-framework[databricks]<13.0.0,>=10.0.0; extra == "databricks"
 Requires-Dist: databricks-sql-connector<4.5.0,>=3.7.0; extra == "databricks"
 Requires-Dist: databricks-sdk<0.123.0; extra == "databricks"
-Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "databricks"
+Requires-Dist: packaging<27.0,>=21.3; extra == "databricks"
 Provides-Extra: databricks-runtime
-Requires-Dist: ibis-framework[databricks]<13.0.0,>=10.0.0; extra == "databricks-runtime"
-Requires-Dist: databricks-sql-connector<4.5.0,>=3.7.0; extra == "databricks-runtime"
-Requires-Dist: databricks-sdk<0.123.0; extra == "databricks-runtime"
-Requires-Dist: packaging<27.0,>=21.3; extra == "databricks-runtime"
+Requires-Dist: datacontract-cli[databricks]; extra == "databricks-runtime"
 Provides-Extra: iceberg
 Requires-Dist: pyiceberg==0.11.1; extra == "iceberg"
 Provides-Extra: kafka
-Requires-Dist: datacontract-cli[avro]; extra == "kafka"
-Requires-Dist: ibis-framework[pyspark]<13.0.0,>=10.0.0; extra == "kafka"
-Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "kafka"
+Requires-Dist: datacontract-cli[duckdb,parquet]; extra == "kafka"
+Requires-Dist: confluent-kafka<3.0.0,>=2.6.0; extra == "kafka"
+Requires-Dist: fastavro<2.0.0,>=1.9.7; extra == "kafka"
 Provides-Extra: mysql
 Requires-Dist: datacontract-cli[duckdb]; extra == "mysql"
 Provides-Extra: postgres
@@ -109,12 +127,12 @@ Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
 Provides-Extra: rdf
 Requires-Dist: rdflib==7.6.0; extra == "rdf"
 Provides-Extra: api
-Requires-Dist: fastapi<0.141.0,>=0.115.0; extra == "api"
-Requires-Dist: uvicorn<0.52.0,>=0.44.0; extra == "api"
+Requires-Dist: fastapi<0.142.0,>=0.115.0; extra == "api"
+Requires-Dist: uvicorn<0.53.0,>=0.44.0; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: proto-schema-parser<3.0,>=1.0; extra == "protobuf"
 Provides-Extra: all
-Requires-Dist: datacontract-cli[api,athena,azure,bigquery,csv,databricks,dataframe,dbml,duckdb,excel,gcs,iceberg,impala,kafka,mysql,oracle,parquet,postgres,protobuf,rdf,redshift,s3,snowflake,sqlserver,trino]; extra == "all"
+Requires-Dist: datacontract-cli[api,athena,avro,azure,bigquery,csv,databricks,dataframe,dbml,duckdb,excel,gcs,iceberg,impala,kafka,mysql,oracle,parquet,postgres,protobuf,rdf,redshift,s3,snowflake,sqlserver,trino]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
 Requires-Dist: httpx==0.28.1; extra == "dev"
@@ -127,9 +145,9 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
 Requires-Dist: pymssql==2.3.13; extra == "dev"
 Requires-Dist: psycopg2-binary<3.0,>=2.9; extra == "dev"
-Requires-Dist: mysql-connector-python<9.7.0,>=8.0.30; extra == "dev"
-Requires-Dist: ruff==0.16.0; extra == "dev"
-Requires-Dist: testcontainers[kafka,minio,mssql,mysql,postgres]==4.14.2; extra == "dev"
+Requires-Dist: mysql-connector-python<26.8.0,>=8.0.30; extra == "dev"
+Requires-Dist: ruff==0.16.1; extra == "dev"
+Requires-Dist: testcontainers[kafka,minio,mssql,mysql,postgres]==4.15.0; extra == "dev"
 Requires-Dist: trino==0.338.0; extra == "dev"
 Dynamic: license-file
 
@@ -402,8 +420,8 @@ A list of available extras:
 | Google BigQuery                          | `pip install datacontract-cli[bigquery]`   |
 | CSV                                      | `pip install datacontract-cli[csv]`        |
 | Databricks Integration                   | `pip install datacontract-cli[databricks]` |
-| Databricks Runtime                       | `pip install datacontract-cli[databricks-runtime]` (inside Databricks, where the cluster provides PySpark) |
-| DataFrame (Spark)                        | `pip install datacontract-cli[dataframe]`  |
+| Databricks Runtime                       | `pip install datacontract-cli[databricks]` (also inside Databricks, using the cluster's own Spark session) |
+| DataFrame (Spark)                        | `pip install datacontract-cli[dataframe]` (PySpark not included — you supply the Spark session) |
 | DBML                                     | `pip install datacontract-cli[dbml]`       |
 | DuckDB (local/S3/GCS/Azure file testing) | `pip install datacontract-cli[duckdb]`     |
 | Excel                                    | `pip install datacontract-cli[excel]`      |
@@ -444,11 +462,9 @@ It covers everything in depth, including the complete command reference:
 
 - Install [uv](https://docs.astral.sh/uv/)
 - Python base interpreter should be 3.11.x.
-- A JDK (17 or 21) must be installed for the Spark-based tests (e.g. `test_test_kafka.py`, `test_test_delta.py`, `test_test_dataframe.py`, `test_import_spark.py`). Java 25 is not yet supported. On macOS and Linux you can install one with [SDKMAN](https://sdkman.io): `sdk install java 21.0.11-tem` (or any 21.x build from `sdk list java`). Verify with `java --version`.
 - Docker engine must be running to execute the tests.
 
 ```bash
-sdk use java 21.0.11-tem
 uv python pin 3.11
 uv venv
 uv pip install -e '.[dev]'
@@ -481,15 +497,6 @@ Try to sync all groups:
 uv sync --all-groups --all-extras
 ```
 
-#### Errors in tests that use PySpark (e.g. test_test_kafka.py)
-
-Ensure you have a JDK 17 or 21 installed. Java 25 causes issues.
-
-```
-java --version
-```
-
-
 ### Docker Build
 
 ```bash

{datacontract_cli-1.0.17 → datacontract_cli-1.1.0}/README.md

@@ -267,8 +267,8 @@ A list of available extras:
 | Google BigQuery                          | `pip install datacontract-cli[bigquery]`   |
 | CSV                                      | `pip install datacontract-cli[csv]`        |
 | Databricks Integration                   | `pip install datacontract-cli[databricks]` |
-| Databricks Runtime                       | `pip install datacontract-cli[databricks-runtime]` (inside Databricks, where the cluster provides PySpark) |
-| DataFrame (Spark)                        | `pip install datacontract-cli[dataframe]`  |
+| Databricks Runtime                       | `pip install datacontract-cli[databricks]` (also inside Databricks, using the cluster's own Spark session) |
+| DataFrame (Spark)                        | `pip install datacontract-cli[dataframe]` (PySpark not included — you supply the Spark session) |
 | DBML                                     | `pip install datacontract-cli[dbml]`       |
 | DuckDB (local/S3/GCS/Azure file testing) | `pip install datacontract-cli[duckdb]`     |
 | Excel                                    | `pip install datacontract-cli[excel]`      |
@@ -309,11 +309,9 @@ It covers everything in depth, including the complete command reference:
 
 - Install [uv](https://docs.astral.sh/uv/)
 - Python base interpreter should be 3.11.x.
-- A JDK (17 or 21) must be installed for the Spark-based tests (e.g. `test_test_kafka.py`, `test_test_delta.py`, `test_test_dataframe.py`, `test_import_spark.py`). Java 25 is not yet supported. On macOS and Linux you can install one with [SDKMAN](https://sdkman.io): `sdk install java 21.0.11-tem` (or any 21.x build from `sdk list java`). Verify with `java --version`.
 - Docker engine must be running to execute the tests.
 
 ```bash
-sdk use java 21.0.11-tem
 uv python pin 3.11
 uv venv
 uv pip install -e '.[dev]'
@@ -346,15 +344,6 @@ Try to sync all groups:
 uv sync --all-groups --all-extras
 ```
 
-#### Errors in tests that use PySpark (e.g. test_test_kafka.py)
-
-Ensure you have a JDK 17 or 21 installed. Java 25 causes issues.
-
-```
-java --version
-```
-
-
 ### Docker Build
 
 ```bash

{datacontract_cli-1.0.17 → datacontract_cli-1.1.0}/datacontract/config/settings.py

@@ -153,6 +153,8 @@ class Config(BaseSettings):
     kafka_schema_registry_url: str | None = None
     kafka_schema_registry_username: str | None = None
     kafka_schema_registry_password: SecretStr | None = None
+    kafka_max_messages: int | None = None
+    kafka_timeout: int | None = None
 
     # mysql
     mysql_username: str | None = None
@@ -558,6 +560,12 @@ class Config(BaseSettings):
     def get_kafka_schema_registry_password(self, required: bool = False) -> str | None:
         return self._str_option("kafka_schema_registry_password", required)
 
+    def get_kafka_max_messages(self) -> int | None:
+        return self._int_option("kafka_max_messages")
+
+    def get_kafka_timeout(self) -> int | None:
+        return self._int_option("kafka_timeout")
+
     # --- mysql ---
     def get_mysql_username(self, required: bool = False) -> str | None:
         return self._str_option("mysql_username", required)

{datacontract_cli-1.0.17 → datacontract_cli-1.1.0}/datacontract/engines/checks/create_checks.py

@@ -48,7 +48,8 @@ def is_check_types(server: Optional[Server]) -> bool:
 
 
 def to_schema_name(schema_object: SchemaObject, server_type: Optional[str]) -> str:
-    # Kafka uses the Spark SQL view name (the logical name), not the topic (physicalName).
+    # Kafka messages are loaded into a table named after the schema object (the logical
+    # name), not after the topic the physicalName holds.
     if server_type == "kafka":
         return schema_object.name
     if schema_object.physicalName:

{datacontract_cli-1.0.17 → datacontract_cli-1.1.0}/datacontract/engines/ibis/connections/connect.py

@@ -2,10 +2,11 @@
 
 Replaces the per-source soda configuration builders. Each branch maps the
 ``Server`` fields and the same ``DATACONTRACT_*`` environment variables the soda
-connections used onto an ``ibis.<backend>.connect(...)`` call. File sources
-reuse the DuckDB view builder (``duckdb_connection.py``) and Spark sources reuse
-the Spark/Kafka helpers (``kafka.py``), wrapping the resulting connection/session
-with the ibis duckdb / pyspark backend.
+connections used onto an ``ibis.<backend>.connect(...)`` call. File sources reuse
+the DuckDB view builder (``duckdb_connection.py``) and Kafka reuses the topic
+reader (``kafka.py``), which likewise loads its messages into DuckDB; both are
+wrapped with the ibis duckdb backend. Only the Spark-session server types
+(``dataframe``, ``databricks``) use the ibis pyspark backend.
 """
 
 from __future__ import annotations
@@ -73,12 +74,11 @@ def connect_ibis(
         return ibis.duckdb.from_connection(con)
 
     if server_type == "kafka":
-        from datacontract.engines.ibis.connections.kafka import create_spark_session, read_kafka_topic
+        from datacontract.engines.ibis.connections.kafka import read_kafka_topic
 
-        if spark is None:
-            spark = create_spark_session()
-        read_kafka_topic(spark, data_contract, server, config)
-        return ibis.pyspark.connect(session=spark)
+        run.log_info(f"Connecting to kafka {server.format} via duckdb")
+        con = read_kafka_topic(data_contract, server, run, config, duckdb_connection)
+        return ibis.duckdb.from_connection(con)
 
     if server_type == "dataframe":
         if spark is None:
@@ -236,7 +236,7 @@ def _connect_databricks(ibis, server: Server, run: Run, config: Config):
 
     if token:
         run.log_info("Connecting to databricks with a personal access token")
-        return ibis.databricks.connect(access_token=token, **kwargs)
+        return _databricks_connect(ibis, access_token=token, **kwargs)
 
     if client_id and client_secret:
         run.log_info("Connecting to databricks with an OAuth service principal (M2M)")
@@ -244,20 +244,44 @@ def _connect_databricks(ibis, server: Server, run: Run, config: Config):
         kwargs["credentials_provider"] = _databricks_credentials_provider(
             service_principal=True, host=sdk_host, client_id=client_id, client_secret=client_secret
         )
-        return ibis.databricks.connect(**kwargs)
+        return _databricks_connect(ibis, **kwargs)
 
     if profile:
         run.log_info(f"Connecting to databricks with config profile '{profile}'")
         kwargs["credentials_provider"] = _databricks_credentials_provider(profile=profile)
-        return ibis.databricks.connect(**kwargs)
+        return _databricks_connect(ibis, **kwargs)
 
     if auth_type:
         run.log_info(f"Connecting to databricks with auth_type '{auth_type}'")
-        return ibis.databricks.connect(auth_type=auth_type, **kwargs)
+        return _databricks_connect(ibis, auth_type=auth_type, **kwargs)
 
     # Nothing configured: fail with the same clear message as before.
     token = config.get_databricks_token(required=True)
-    return ibis.databricks.connect(access_token=token, **kwargs)
+    return _databricks_connect(ibis, access_token=token, **kwargs)
+
+
+def _databricks_connect(ibis, **kwargs):
+    """Connect to Databricks without creating ibis's memtable staging volume.
+
+    ibis's Databricks backend creates a staging volume for memtables in the
+    target schema at connect time (``CREATE VOLUME IF NOT EXISTS``), named
+    after the client host and pid and never cleaned up. Contract tests only
+    ever read, so the volume is never used: skipping its setup lets read-only
+    principals without CREATE VOLUME permission run tests and keeps repeated
+    runs from littering the schema with per-run volumes.
+
+    Workaround until https://github.com/dataqcompany/ibis/pull/2 (drops the
+    CREATE VOLUME requirement for read-only connections) is merged and
+    released; then the ``_post_connect`` patch can go.
+    """
+    from ibis.backends.databricks import Backend
+
+    original_post_connect = Backend._post_connect
+    Backend._post_connect = lambda self, *, memtable_volume=None: None
+    try:
+        return ibis.databricks.connect(**kwargs)
+    finally:
+        Backend._post_connect = original_post_connect
 
 
 def _databricks_credentials_provider(service_principal: bool = False, **config_kwargs):