datacontract-cli 1.0.7__tar.gz → 1.0.9__tar.gz

{datacontract_cli-1.0.7/datacontract_cli.egg-info → datacontract_cli-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.7
+Version: 1.0.9
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/datacontract/engines/checks/create_checks.py

@@ -548,7 +548,14 @@ def _quality_metric_check(model, field, quality: DataQuality, threshold: Thresho
         if field is None:
             logger.warning("Quality check invalidValues is only supported at field level")
             return []
-        valid_values = quality.arguments.get("validValues") if quality.arguments else None
+        args = quality.arguments or {}
+        valid_values = args.get("validValues")
+        pattern = args.get("pattern")
+        if valid_values is None and pattern is None:
+            logger.warning(
+                f"Quality check invalidValues on field {field} has no validValues or pattern argument; skipping"
+            )
+            return []
         return [
             _invalid_count_check(
                 model,
@@ -558,6 +565,7 @@ def _quality_metric_check(model, field, quality: DataQuality, threshold: Thresho
                 threshold=threshold,
                 category="quality",
                 valid_values=valid_values,
+                valid_regex=pattern,
                 threshold_is_percent=is_percent,
                 severity=severity,
             )

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/datacontract/engines/ibis/connections/connect.py

@@ -115,7 +115,13 @@ def connect_ibis(
         if password:
             kwargs["password"] = password
         # Redshift speaks the postgres wire protocol; ibis has no dedicated backend.
-        return ibis.postgres.connect(**kwargs)
+        con = ibis.postgres.connect(**kwargs)
+        # ibis's postgres schema introspection joins pg_catalog.pg_enum, which
+        # Redshift does not expose; patch it out so model lookups don't fail.
+        from datacontract.engines.ibis.connections.redshift_patch import apply_redshift_compatibility_patch
+
+        apply_redshift_compatibility_patch(con)
+        return con
 
     if server_type == "mysql":
         return _connect_mysql_via_duckdb(ibis, data_contract, server, run, schema_name)

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/datacontract/engines/ibis/connections/kafka.py

@@ -148,7 +148,15 @@ def process_avro_format(df, model_name: str, schema_obj: SchemaObject):
         )
 
     avro_schema = to_avro_schema_json(model_name, schema_obj)
-    df2 = df.withColumn("fixedValue", expr("substring(value, 6, length(value)-5)"))
+    # Messages serialized through the Confluent Schema Registry are framed with a
+    # 5-byte prefix (magic byte 0x00 + 4-byte schema id) that must be stripped
+    # before Avro decoding. Plain Avro messages carry no such prefix, so strip it
+    # only when the magic byte is present; otherwise stripping 5 bytes would
+    # corrupt every record and from_avro (PERMISSIVE) would silently null them out.
+    df2 = df.withColumn(
+        "fixedValue",
+        expr("CASE WHEN substring(value, 1, 1) = X'00' THEN substring(value, 6, length(value)-5) ELSE value END"),
+    )
     options = {"mode": "PERMISSIVE"}
     df2.select(from_avro(col("fixedValue"), avro_schema, options).alias("avro")).select(
         col("avro.*")

datacontract_cli-1.0.9/datacontract/engines/ibis/connections/redshift_patch.py

@@ -0,0 +1,84 @@
+"""Make ibis's Postgres schema introspection valid on Amazon Redshift.
+
+Redshift has no dedicated ibis backend; it rides the Postgres backend over the
+Postgres wire protocol (see ``connect.py``). ibis builds the column-metadata
+query behind ``con.table(...)`` (``ibis.backends.postgres.Backend.get_schema``)
+with an enum-detection ``CASE`` that joins ``pg_catalog.pg_enum``:
+
+    CASE WHEN EXISTS(
+      SELECT 1 FROM pg_catalog.pg_type t
+      INNER JOIN pg_catalog.pg_enum e ON e.enumtypid = t.oid ...
+    ) THEN 'enum' ELSE pg_catalog.format_type(...) END
+
+PostgreSQL exposes ``pg_catalog.pg_enum``; Redshift does not, so introspecting
+any model fails with:
+
+    relation "pg_catalog.pg_enum" does not exist
+
+Redshift has no enum types, so the ``CASE`` is pointless there. This module
+re-binds ``get_schema`` on a connected backend with the same query minus the
+enum join, reading the column type straight from ``pg_catalog.format_type``.
+``_get_schema_using_query`` (custom-SQL introspection) calls ``self.get_schema``
+internally, so it inherits the fix without a separate override. The patch can be
+removed once a fixed ibis release is available.
+"""
+
+from __future__ import annotations
+
+import logging
+import types
+
+logger = logging.getLogger(__name__)
+
+# ibis's get_schema query without the pg_enum-dependent enum CASE. Every other
+# relation (pg_attribute / pg_class / pg_namespace) and pg_catalog.format_type
+# is supported by Redshift's leader-node catalog.
+_TYPE_INFO_SQL = """\
+SELECT
+  a.attname AS column_name,
+  pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
+  NOT a.attnotnull AS nullable
+FROM pg_catalog.pg_attribute a
+INNER JOIN pg_catalog.pg_class c
+   ON a.attrelid = c.oid
+INNER JOIN pg_catalog.pg_namespace n
+   ON c.relnamespace = n.oid
+WHERE a.attnum > 0
+  AND NOT a.attisdropped
+  AND n.nspname = ANY(%(dbs)s)
+  AND c.relname = %(name)s
+ORDER BY a.attnum ASC"""
+
+
+def apply_redshift_compatibility_patch(con) -> None:
+    """Re-bind Postgres schema introspection to emit Redshift-compatible SQL.
+
+    Best-effort: if the backend internals differ from what we expect (e.g. a
+    future ibis refactor), the patch is skipped and the original method stays in
+    place rather than breaking the connection.
+    """
+    try:
+        con.get_schema = types.MethodType(_get_schema, con)
+    except Exception:  # pragma: no cover - defensive, never block a connection
+        logger.debug("Could not apply Redshift compatibility patch", exc_info=True)
+
+
+def _get_schema(self, name: str, *, catalog=None, database=None):
+    """Drop-in for ``Backend.get_schema`` without the ``pg_enum`` join."""
+    import ibis.common.exceptions as com
+    import ibis.expr.schema as sch
+
+    dbs = [database or self.current_database]
+    if database is None and (temp_table_db := self._session_temp_db) is not None:
+        dbs.append(temp_table_db)
+
+    type_mapper = self.compiler.type_mapper
+    con = self.con
+    params = {"dbs": dbs, "name": name}
+    with con.cursor() as cursor, con.transaction():
+        rows = cursor.execute(_TYPE_INFO_SQL, params, prepare=True).fetchall()
+
+    if not rows:
+        raise com.TableNotFound(name)
+
+    return sch.Schema({col: type_mapper.from_string(typestr, nullable=nullable) for col, typestr, nullable in rows})

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9/datacontract_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.7
+Version: 1.0.9
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/datacontract_cli.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ datacontract/engines/ibis/connections/connect.py
 datacontract/engines/ibis/connections/duckdb_connection.py
 datacontract/engines/ibis/connections/kafka.py
 datacontract/engines/ibis/connections/oracle_patch.py
+datacontract/engines/ibis/connections/redshift_patch.py
 datacontract/export/__init__.py
 datacontract/export/avro_exporter.py
 datacontract/export/avro_idl_exporter.py
@@ -317,6 +318,7 @@ tests/test_import_unity_file.py
 tests/test_integration_entropydata.py
 tests/test_lint.py
 tests/test_oracle_patch.py
+tests/test_redshift_patch.py
 tests/test_require_env.py
 tests/test_resolve.py
 tests/test_resolve_definitions.py
@@ -349,6 +351,7 @@ tests/test_test_postgres.py
 tests/test_test_quality.py
 tests/test_test_quality_local_fallback.py
 tests/test_test_quality_percent_severity.py
+tests/test_test_redshift.py
 tests/test_test_s3_csv.py
 tests/test_test_s3_delta.py
 tests/test_test_s3_json.py

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datacontract-cli"
-version = "1.0.7"
+version = "1.0.9"
 description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library."
 license = "MIT"
 readme = "README.md"

datacontract_cli-1.0.9/tests/test_redshift_patch.py

@@ -0,0 +1,91 @@
+"""Regression tests for the Redshift Postgres schema-introspection patch.
+
+Redshift rides ibis's Postgres backend, whose ``get_schema`` query joins
+``pg_catalog.pg_enum`` to detect enum columns. Redshift does not expose
+``pg_enum``, so the unpatched query fails with ``relation "pg_catalog.pg_enum"
+does not exist``. These tests assert the patched query never references
+``pg_enum`` and still types columns correctly, without needing a live Redshift.
+"""
+
+import contextlib
+
+import pytest
+
+sc = pytest.importorskip("ibis.backends.sql.compilers")
+
+from datacontract.engines.ibis.connections import redshift_patch  # noqa: E402
+
+
+class _StubCursor:
+    def __init__(self, rows):
+        self._rows = rows
+
+    def execute(self, sql, params=None, prepare=False):
+        self._executed_sql = sql
+        return self
+
+    def fetchall(self):
+        return self._rows
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+class _StubPostgresConnection:
+    """Minimal stand-in exposing what the patched introspection method touches."""
+
+    name = "postgres"
+    compiler = sc.postgres.compiler
+    current_database = "dev"
+    _session_temp_db = None
+
+    def __init__(self, rows):
+        self._cursor = _StubCursor(rows)
+
+    @property
+    def con(self):
+        return self
+
+    def cursor(self):
+        return self._cursor
+
+    @contextlib.contextmanager
+    def transaction(self):
+        yield
+
+
+def test_get_schema_query_omits_pg_enum_and_types_columns():
+    rows = [
+        ("cust_code", "character varying(10)", False),
+        ("cust_state", "character varying(24)", True),
+    ]
+    con = _StubPostgresConnection(rows)
+
+    schema = redshift_patch._get_schema(con, "customer", database="dwh_academic")
+
+    assert "pg_enum" not in con._cursor._executed_sql
+    assert schema["cust_code"].is_string()
+    assert schema["cust_code"].nullable is False
+    assert schema["cust_state"].nullable is True
+
+
+def test_get_schema_raises_table_not_found_when_empty():
+    import ibis.common.exceptions as com
+
+    con = _StubPostgresConnection([])
+    with pytest.raises(com.TableNotFound):
+        redshift_patch._get_schema(con, "missing", database="dwh_academic")
+
+
+def test_apply_patch_rebinds_get_schema():
+    class _Backend:
+        def get_schema(self, *a, **k):  # pragma: no cover - replaced by the patch
+            return "original"
+
+    backend = _Backend()
+    redshift_patch.apply_redshift_compatibility_patch(backend)
+
+    assert backend.get_schema.__func__ is redshift_patch._get_schema

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/tests/test_test_kafka.py

@@ -1,3 +1,5 @@
+import io
+import json
 import os
 import sys
 import time
@@ -11,11 +13,14 @@ if sys.version_info >= (3, 12, 1):
 
 
 from kafka import KafkaProducer
+from open_data_contract_standard.model import OpenDataContractStandard
 from testcontainers.kafka import KafkaContainer
 
 from datacontract.data_contract import DataContract
+from datacontract.export.avro_exporter import to_avro_schema_json
 
 datacontract = "fixtures/kafka/datacontract.yaml"
+datacontract_avro = "fixtures/kafka/datacontract_avro.yaml"
 
 # Skip when running under pytest-xdist workers - Spark's Java Kafka client
 # experiences timeouts when running in xdist subprocess environment
@@ -36,6 +41,56 @@ def test_test_kafka(monkeypatch):
     assert run.result == "passed"
 
 
+@pytest.mark.skipif(is_xdist_worker, reason="Spark Kafka tests fail under pytest-xdist workers")
+def test_test_kafka_avro_plain(monkeypatch):
+    """Plain Avro messages (no Confluent Schema Registry framing) must decode without
+    being corrupted by the 5-byte magic-byte/schema-id strip. Regression for #1344,
+    where every field was reported as null (missing_count == row count)."""
+    monkeypatch.delenv("DATACONTRACT_KAFKA_SASL_USERNAME", raising=False)
+
+    with KafkaContainer("confluentinc/cp-kafka:7.7.0").with_kraft() as kafka:
+        send_avro_messages_to_topic(kafka, "fixtures/kafka/data/messages.json", "inventory-events-avro")
+        data_contract_str = _setup_datacontract(kafka, datacontract_avro)
+        data_contract = DataContract(data_contract_str=data_contract_str)
+        run = data_contract.test()
+
+    print(run.pretty())
+    assert run.result == "passed"
+
+
+def send_avro_messages_to_topic(kafka: KafkaContainer, messages_file_path: str, topic_name: str):
+    """Serialize the JSON sample records as plain Avro (no Confluent prefix) and publish them."""
+    from avro.io import BinaryEncoder, DatumWriter
+    from avro.schema import parse as parse_avro_schema
+
+    print(f"Sending Avro messages from {messages_file_path} to Kafka topic {topic_name}")
+
+    bootstrap_server = kafka.get_bootstrap_server().replace("localhost", "127.0.0.1")
+    _ensure_topic_exists(bootstrap_server, topic_name)
+
+    with open(datacontract_avro) as data_contract_file:
+        odcs = OpenDataContractStandard.from_string(data_contract_file.read())
+    schema_obj = odcs.schema_[0]
+    avro_schema = parse_avro_schema(to_avro_schema_json(schema_obj.name, schema_obj))
+    writer = DatumWriter(avro_schema)
+
+    def encode(record: dict) -> bytes:
+        buffer = io.BytesIO()
+        writer.write(record, BinaryEncoder(buffer))
+        return buffer.getvalue()
+
+    producer = KafkaProducer(bootstrap_servers=bootstrap_server, value_serializer=encode)
+    messages_sent = 0
+    with open(messages_file_path) as messages_file:
+        for line in messages_file:
+            producer.send(topic=topic_name, value=json.loads(line))
+            messages_sent += 1
+
+    producer.flush()
+    producer.close()
+    print(f"Sent {messages_sent} Avro messages from {messages_file_path} to Kafka topic {topic_name}")
+
+
 def send_messages_to_topic(kafka: KafkaContainer, messages_file_path: str, topic_name: str):
     print(f"Sending messages from {messages_file_path} to Kafka topic {topic_name}")
 
@@ -88,8 +143,8 @@ def _ensure_topic_exists(bootstrap_server: str, topic_name: str, timeout_seconds
     raise TimeoutError(f"Topic {topic_name} not available after {timeout_seconds}s")
 
 
-def _setup_datacontract(kafka: KafkaContainer):
-    with open(datacontract) as data_contract_file:
+def _setup_datacontract(kafka: KafkaContainer, contract_path: str = datacontract):
+    with open(contract_path) as data_contract_file:
         data_contract_str = data_contract_file.read()
     host = kafka.get_bootstrap_server()
     # Replace localhost with 127.0.0.1 to avoid IPv4/IPv6 resolution issues

{datacontract_cli-1.0.7 → datacontract_cli-1.0.9}/tests/test_test_quality_percent_severity.py

@@ -190,3 +190,36 @@ def test_severity_warning_on_absolute_count_check():
     check = _quality_check(run, "field_invalid_values")
     assert check.result == ResultEnum.warning
     assert run.result == ResultEnum.warning
+
+
+def test_invalid_values_pattern_argument_runs_regex():
+    # 1 of 5 emails ("no-at-sign") lacks an @ => 20% invalid, threshold < 10% => fail.
+    contract = _odcs(
+        email_quality="""        quality:
+          - type: library
+            metric: invalidValues
+            unit: percent
+            mustBeLessThan: 10
+            arguments:
+              pattern: '@'
+"""
+    )
+    run = DataContract(data_contract_str=contract).test()
+    check = _quality_check(run, "field_invalid_values")
+    assert check.result == ResultEnum.failed
+    assert check.diagnostics["percent"] == 20.0
+    assert "20.0% (1 of 5 rows)" in check.reason
+
+
+def test_invalid_values_without_arguments_is_skipped(caplog):
+    # An invalidValues check with neither validValues nor pattern can never fail, therefore it should be dropped
+    contract = _odcs(
+        email_quality="""        quality:
+          - type: library
+            metric: invalidValues
+            mustBe: 0
+"""
+    )
+    checks = _checks(contract)
+    assert not any(c.type == "field_invalid_values" for c in checks)
+    assert "no validValues or pattern argument" in caplog.text

datacontract_cli-1.0.9/tests/test_test_redshift.py

@@ -0,0 +1,86 @@
+"""Smoke test for the Redshift test path against a Postgres testcontainer.
+
+Redshift has no dedicated ibis backend; it rides the Postgres backend over the
+Postgres wire protocol (see ``connect_ibis``), with two compatibility patches on
+top: tables are qualified with the configured ``schema`` (#1338) and the column
+introspection query is rewritten to drop the ``pg_catalog.pg_enum`` join that
+Redshift does not expose (``redshift_patch``).
+
+No real Redshift is reachable in CI, so this exercises the full ``type: redshift``
+code path — connect branch, schema-qualified introspection, the pg_enum-free
+``get_schema``, the batched aggregations, a pattern check, and custom SQL quality
+checks — against a wire-compatible Postgres. It proves the redshift branch and
+the patched introspection produce correct schemas and a passing run end-to-end;
+it cannot catch Redshift-only dialect divergences (POSIX regex, temp-view-backed
+``con.sql`` falling back to ``raw_sql``), which need a live cluster.
+"""
+
+import pytest
+from testcontainers.postgres import PostgresContainer
+
+from datacontract.data_contract import DataContract
+from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
+
+postgres = PostgresContainer("postgres:16")
+
+
+@pytest.fixture(scope="function", autouse=True)
+def postgres_container(request):
+    postgres.start()
+
+    def remove_container():
+        postgres.stop()
+
+    request.addfinalizer(remove_container)
+
+
+def test_test_redshift_path_via_postgres_wire(postgres_container, monkeypatch):
+    monkeypatch.setenv("DATACONTRACT_REDSHIFT_USERNAME", postgres.username)
+    monkeypatch.setenv("DATACONTRACT_REDSHIFT_PASSWORD", postgres.password)
+    _init_sql("fixtures/postgres/data/data.sql")
+
+    data_contract_str = _setup_datacontract("fixtures/redshift/odcs.yaml")
+    data_contract = DataContract(data_contract_str=data_contract_str)
+
+    run = data_contract.test()
+
+    print(run.pretty())
+    assert run.result == "passed"
+    assert all(check.result == ResultEnum.passed for check in run.checks)
+
+
+def _setup_datacontract(file):
+    with open(file) as data_contract_file:
+        data_contract_str = data_contract_file.read()
+    port = postgres.get_exposed_port(5432)
+    # The fixture uses the default Redshift port 5439 as a placeholder.
+    return data_contract_str.replace("5439", str(port))
+
+
+def _init_sql(file_path):
+    try:
+        import psycopg2
+    except ImportError as e:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="postgres extra missing",
+            reason="Install the extra datacontract-cli[postgres] to use postgres",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    connection = psycopg2.connect(
+        database=postgres.dbname,
+        user=postgres.username,
+        password=postgres.password,
+        host=postgres.get_container_host_ip(),
+        port=postgres.get_exposed_port(5432),
+    )
+    cursor = connection.cursor()
+    with open(file_path, "r") as sql_file:
+        cursor.execute(sql_file.read())
+    connection.commit()
+    cursor.close()
+    connection.close()