datacontract-cli 1.0.8__tar.gz → 1.0.9__tar.gz

{datacontract_cli-1.0.8/datacontract_cli.egg-info → datacontract_cli-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.8
+Version: 1.0.9
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT

{datacontract_cli-1.0.8 → datacontract_cli-1.0.9}/datacontract/engines/checks/create_checks.py

@@ -548,7 +548,14 @@ def _quality_metric_check(model, field, quality: DataQuality, threshold: Thresho
         if field is None:
             logger.warning("Quality check invalidValues is only supported at field level")
             return []
-        valid_values = quality.arguments.get("validValues") if quality.arguments else None
+        args = quality.arguments or {}
+        valid_values = args.get("validValues")
+        pattern = args.get("pattern")
+        if valid_values is None and pattern is None:
+            logger.warning(
+                f"Quality check invalidValues on field {field} has no validValues or pattern argument; skipping"
+            )
+            return []
         return [
             _invalid_count_check(
                 model,
@@ -558,6 +565,7 @@ def _quality_metric_check(model, field, quality: DataQuality, threshold: Thresho
                 threshold=threshold,
                 category="quality",
                 valid_values=valid_values,
+                valid_regex=pattern,
                 threshold_is_percent=is_percent,
                 severity=severity,
             )

{datacontract_cli-1.0.8 → datacontract_cli-1.0.9}/datacontract/engines/ibis/connections/kafka.py

@@ -148,7 +148,15 @@ def process_avro_format(df, model_name: str, schema_obj: SchemaObject):
         )
 
     avro_schema = to_avro_schema_json(model_name, schema_obj)
-    df2 = df.withColumn("fixedValue", expr("substring(value, 6, length(value)-5)"))
+    # Messages serialized through the Confluent Schema Registry are framed with a
+    # 5-byte prefix (magic byte 0x00 + 4-byte schema id) that must be stripped
+    # before Avro decoding. Plain Avro messages carry no such prefix, so strip it
+    # only when the magic byte is present; otherwise stripping 5 bytes would
+    # corrupt every record and from_avro (PERMISSIVE) would silently null them out.
+    df2 = df.withColumn(
+        "fixedValue",
+        expr("CASE WHEN substring(value, 1, 1) = X'00' THEN substring(value, 6, length(value)-5) ELSE value END"),
+    )
     options = {"mode": "PERMISSIVE"}
     df2.select(from_avro(col("fixedValue"), avro_schema, options).alias("avro")).select(
         col("avro.*")

{datacontract_cli-1.0.8 → datacontract_cli-1.0.9/datacontract_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.8
+Version: 1.0.9
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT

{datacontract_cli-1.0.8 → datacontract_cli-1.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datacontract-cli"
-version = "1.0.8"
+version = "1.0.9"
 description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library."
 license = "MIT"
 readme = "README.md"

{datacontract_cli-1.0.8 → datacontract_cli-1.0.9}/tests/test_test_kafka.py

@@ -1,3 +1,5 @@
+import io
+import json
 import os
 import sys
 import time
@@ -11,11 +13,14 @@ if sys.version_info >= (3, 12, 1):
 
 
 from kafka import KafkaProducer
+from open_data_contract_standard.model import OpenDataContractStandard
 from testcontainers.kafka import KafkaContainer
 
 from datacontract.data_contract import DataContract
+from datacontract.export.avro_exporter import to_avro_schema_json
 
 datacontract = "fixtures/kafka/datacontract.yaml"
+datacontract_avro = "fixtures/kafka/datacontract_avro.yaml"
 
 # Skip when running under pytest-xdist workers - Spark's Java Kafka client
 # experiences timeouts when running in xdist subprocess environment
@@ -36,6 +41,56 @@ def test_test_kafka(monkeypatch):
     assert run.result == "passed"
 
 
+@pytest.mark.skipif(is_xdist_worker, reason="Spark Kafka tests fail under pytest-xdist workers")
+def test_test_kafka_avro_plain(monkeypatch):
+    """Plain Avro messages (no Confluent Schema Registry framing) must decode without
+    being corrupted by the 5-byte magic-byte/schema-id strip. Regression for #1344,
+    where every field was reported as null (missing_count == row count)."""
+    monkeypatch.delenv("DATACONTRACT_KAFKA_SASL_USERNAME", raising=False)
+
+    with KafkaContainer("confluentinc/cp-kafka:7.7.0").with_kraft() as kafka:
+        send_avro_messages_to_topic(kafka, "fixtures/kafka/data/messages.json", "inventory-events-avro")
+        data_contract_str = _setup_datacontract(kafka, datacontract_avro)
+        data_contract = DataContract(data_contract_str=data_contract_str)
+        run = data_contract.test()
+
+    print(run.pretty())
+    assert run.result == "passed"
+
+
+def send_avro_messages_to_topic(kafka: KafkaContainer, messages_file_path: str, topic_name: str):
+    """Serialize the JSON sample records as plain Avro (no Confluent prefix) and publish them."""
+    from avro.io import BinaryEncoder, DatumWriter
+    from avro.schema import parse as parse_avro_schema
+
+    print(f"Sending Avro messages from {messages_file_path} to Kafka topic {topic_name}")
+
+    bootstrap_server = kafka.get_bootstrap_server().replace("localhost", "127.0.0.1")
+    _ensure_topic_exists(bootstrap_server, topic_name)
+
+    with open(datacontract_avro) as data_contract_file:
+        odcs = OpenDataContractStandard.from_string(data_contract_file.read())
+    schema_obj = odcs.schema_[0]
+    avro_schema = parse_avro_schema(to_avro_schema_json(schema_obj.name, schema_obj))
+    writer = DatumWriter(avro_schema)
+
+    def encode(record: dict) -> bytes:
+        buffer = io.BytesIO()
+        writer.write(record, BinaryEncoder(buffer))
+        return buffer.getvalue()
+
+    producer = KafkaProducer(bootstrap_servers=bootstrap_server, value_serializer=encode)
+    messages_sent = 0
+    with open(messages_file_path) as messages_file:
+        for line in messages_file:
+            producer.send(topic=topic_name, value=json.loads(line))
+            messages_sent += 1
+
+    producer.flush()
+    producer.close()
+    print(f"Sent {messages_sent} Avro messages from {messages_file_path} to Kafka topic {topic_name}")
+
+
 def send_messages_to_topic(kafka: KafkaContainer, messages_file_path: str, topic_name: str):
     print(f"Sending messages from {messages_file_path} to Kafka topic {topic_name}")
 
@@ -88,8 +143,8 @@ def _ensure_topic_exists(bootstrap_server: str, topic_name: str, timeout_seconds
     raise TimeoutError(f"Topic {topic_name} not available after {timeout_seconds}s")
 
 
-def _setup_datacontract(kafka: KafkaContainer):
-    with open(datacontract) as data_contract_file:
+def _setup_datacontract(kafka: KafkaContainer, contract_path: str = datacontract):
+    with open(contract_path) as data_contract_file:
         data_contract_str = data_contract_file.read()
     host = kafka.get_bootstrap_server()
     # Replace localhost with 127.0.0.1 to avoid IPv4/IPv6 resolution issues

{datacontract_cli-1.0.8 → datacontract_cli-1.0.9}/tests/test_test_quality_percent_severity.py

@@ -190,3 +190,36 @@ def test_severity_warning_on_absolute_count_check():
     check = _quality_check(run, "field_invalid_values")
     assert check.result == ResultEnum.warning
     assert run.result == ResultEnum.warning
+
+
+def test_invalid_values_pattern_argument_runs_regex():
+    # 1 of 5 emails ("no-at-sign") lacks an @ => 20% invalid, threshold < 10% => fail.
+    contract = _odcs(
+        email_quality="""        quality:
+          - type: library
+            metric: invalidValues
+            unit: percent
+            mustBeLessThan: 10
+            arguments:
+              pattern: '@'
+"""
+    )
+    run = DataContract(data_contract_str=contract).test()
+    check = _quality_check(run, "field_invalid_values")
+    assert check.result == ResultEnum.failed
+    assert check.diagnostics["percent"] == 20.0
+    assert "20.0% (1 of 5 rows)" in check.reason
+
+
+def test_invalid_values_without_arguments_is_skipped(caplog):
+    # An invalidValues check with neither validValues nor pattern can never fail, therefore it should be dropped
+    contract = _odcs(
+        email_quality="""        quality:
+          - type: library
+            metric: invalidValues
+            mustBe: 0
+"""
+    )
+    checks = _checks(contract)
+    assert not any(c.type == "field_invalid_values" for c in checks)
+    assert "no validValues or pattern argument" in caplog.text