datacontract-cli 0.10.15__tar.gz → 0.10.16__tar.gz

{datacontract_cli-0.10.15/datacontract_cli.egg-info → datacontract_cli-0.10.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datacontract-cli
-Version: 0.10.15
+Version: 0.10.16
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 Project-URL: Homepage, https://cli.datacontract.com
@@ -11,15 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer<0.14,>=0.12
-Requires-Dist: pydantic<2.10.0,>=2.8.2
+Requires-Dist: typer<0.15,>=0.12
+Requires-Dist: pydantic<2.11.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.33,>=2.31
-Requires-Dist: fastapi==0.115.5
+Requires-Dist: fastapi==0.115.6
 Requires-Dist: uvicorn==0.32.1
-Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
-Requires-Dist: fastparquet==2024.5.0
-Requires-Dist: python-multipart==0.0.12
+Requires-Dist: fastjsonschema<2.22.0,>=2.19.1
+Requires-Dist: fastparquet==2024.11.0
+Requires-Dist: python-multipart==0.0.19
 Requires-Dist: rich<13.10,>=13.7
 Requires-Dist: simple-ddl-parser==1.7.1
 Requires-Dist: duckdb==1.1.2
@@ -41,7 +41,7 @@ Requires-Dist: databricks-sql-connector<3.6.0,>=3.1.2; extra == "databricks"
 Requires-Dist: databricks-sdk<0.39.0,>=0.32.0; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.5.0,>=3.3.1; extra == "databricks"
 Provides-Extra: iceberg
-Requires-Dist: pyiceberg==0.7.1; extra == "iceberg"
+Requires-Dist: pyiceberg==0.8.1; extra == "iceberg"
 Provides-Extra: kafka
 Requires-Dist: datacontract-cli[avro]; extra == "kafka"
 Requires-Dist: soda-core-spark-df<3.5.0,>=3.3.1; extra == "kafka"
@@ -66,16 +66,16 @@ Provides-Extra: all
 Requires-Dist: datacontract-cli[bigquery,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
-Requires-Dist: httpx==0.27.2; extra == "dev"
+Requires-Dist: httpx==0.28.1; extra == "dev"
 Requires-Dist: kafka-python; extra == "dev"
-Requires-Dist: moto==5.0.18; extra == "dev"
+Requires-Dist: moto==5.0.22; extra == "dev"
 Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<4.1.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: pymssql==2.3.1; extra == "dev"
+Requires-Dist: pymssql==2.3.2; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
-Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.8.2; extra == "dev"
+Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.0; extra == "dev"
 Requires-Dist: trino==0.330.0; extra == "dev"
 
 # Data Contract CLI
@@ -841,7 +841,7 @@ models:
 │                    t-staging-sql|odcs|odcs_v2|odcs_v3|rdf|avro|protobuf                                                        │
 │                    |great-expectations|terraform|avro-idl|sql|sql-query                                                        │
 │                    |html|go|bigquery|dbml|spark|sqlalchemy|data-caterer                                                        │
-│                    |dcs]                                                                                                       │
+│                    |dcs|iceberg]                                                                                                       │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -902,6 +902,7 @@ Available export options:
 | `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅      |
 | `data-caterer`       | Export to Data Caterer in YAML format                   | ✅      |
 | `dcs`                | Export to Data Contract Specification in YAML format    | ✅      |
+| `iceberg`            | Export to an Iceberg JSON Schema Definition             | partial |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 
@@ -1025,6 +1026,63 @@ models:
     - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
     - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
 
+#### Iceberg
+
+Exports to an [Iceberg Table Json Schema Definition](https://iceberg.apache.org/spec/#appendix-c-json-serialization).
+
+This export only supports a single model export at a time because Iceberg's schema definition is for a single table and the exporter maps 1 model to 1 table, use the `--model` flag
+to limit your contract export to a single model.
+
+```bash
+ $ datacontract export --format iceberg --model orders https://datacontract.com/examples/orders-latest/datacontract.yaml --output /tmp/orders_iceberg.json
+ 
+ $ cat /tmp/orders_iceberg.json | jq '.'
+{
+  "type": "struct",
+  "fields": [
+    {
+      "id": 1,
+      "name": "order_id",
+      "type": "string",
+      "required": true
+    },
+    {
+      "id": 2,
+      "name": "order_timestamp",
+      "type": "timestamptz",
+      "required": true
+    },
+    {
+      "id": 3,
+      "name": "order_total",
+      "type": "long",
+      "required": true
+    },
+    {
+      "id": 4,
+      "name": "customer_id",
+      "type": "string",
+      "required": false
+    },
+    {
+      "id": 5,
+      "name": "customer_email_address",
+      "type": "string",
+      "required": true
+    },
+    {
+      "id": 6,
+      "name": "processed_timestamp",
+      "type": "timestamptz",
+      "required": true
+    }
+  ],
+  "schema-id": 0,
+  "identifier-field-ids": [
+    1
+  ]
+}
+```
 
 ### import
 

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/README.md

@@ -761,7 +761,7 @@ models:
 │                    t-staging-sql|odcs|odcs_v2|odcs_v3|rdf|avro|protobuf                                                        │
 │                    |great-expectations|terraform|avro-idl|sql|sql-query                                                        │
 │                    |html|go|bigquery|dbml|spark|sqlalchemy|data-caterer                                                        │
-│                    |dcs]                                                                                                       │
+│                    |dcs|iceberg]                                                                                                       │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -822,6 +822,7 @@ Available export options:
 | `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅      |
 | `data-caterer`       | Export to Data Caterer in YAML format                   | ✅      |
 | `dcs`                | Export to Data Contract Specification in YAML format    | ✅      |
+| `iceberg`            | Export to an Iceberg JSON Schema Definition             | partial |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 
@@ -945,6 +946,63 @@ models:
     - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
     - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
 
+#### Iceberg
+
+Exports to an [Iceberg Table Json Schema Definition](https://iceberg.apache.org/spec/#appendix-c-json-serialization).
+
+This export only supports a single model export at a time because Iceberg's schema definition is for a single table and the exporter maps 1 model to 1 table, use the `--model` flag
+to limit your contract export to a single model.
+
+```bash
+ $ datacontract export --format iceberg --model orders https://datacontract.com/examples/orders-latest/datacontract.yaml --output /tmp/orders_iceberg.json
+ 
+ $ cat /tmp/orders_iceberg.json | jq '.'
+{
+  "type": "struct",
+  "fields": [
+    {
+      "id": 1,
+      "name": "order_id",
+      "type": "string",
+      "required": true
+    },
+    {
+      "id": 2,
+      "name": "order_timestamp",
+      "type": "timestamptz",
+      "required": true
+    },
+    {
+      "id": 3,
+      "name": "order_total",
+      "type": "long",
+      "required": true
+    },
+    {
+      "id": 4,
+      "name": "customer_id",
+      "type": "string",
+      "required": false
+    },
+    {
+      "id": 5,
+      "name": "customer_email_address",
+      "type": "string",
+      "required": true
+    },
+    {
+      "id": 6,
+      "name": "processed_timestamp",
+      "type": "timestamptz",
+      "required": true
+    }
+  ],
+  "schema-id": 0,
+  "identifier-field-ids": [
+    1
+  ]
+}
+```
 
 ### import
 

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/datacontract/breaking/breaking.py

@@ -1,6 +1,6 @@
 from datacontract.breaking.breaking_rules import BreakingRules
 from datacontract.model.breaking_change import BreakingChange, Location, Severity
-from datacontract.model.data_contract_specification import Contact, Field, Info, Model, Quality, Terms
+from datacontract.model.data_contract_specification import Contact, DeprecatedQuality, Field, Info, Model, Terms
 
 
 def info_breaking_changes(
@@ -216,8 +216,8 @@ def terms_breaking_changes(
 
 
 def quality_breaking_changes(
-    old_quality: Quality,
-    new_quality: Quality,
+    old_quality: DeprecatedQuality,
+    new_quality: DeprecatedQuality,
     new_path: str,
     include_severities: [Severity],
 ) -> list[BreakingChange]:

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/datacontract/cli.py

@@ -221,7 +221,7 @@ def export(
     )
     # Don't interpret console markup in output.
     if output is None:
-        console.print(result, markup=False)
+        console.print(result, markup=False, soft_wrap=True)
     else:
         with output.open("w") as f:
             f.write(result)
@@ -298,7 +298,7 @@ def import_(
         iceberg_table=iceberg_table,
     )
     if output is None:
-        console.print(result.to_yaml())
+        console.print(result.to_yaml(), markup=False, soft_wrap=True)
     else:
         with output.open("w") as f:
             f.write(result.to_yaml())

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/datacontract/engines/soda/check_soda_execute.py

@@ -12,7 +12,7 @@ from datacontract.engines.soda.connections.sqlserver import to_sqlserver_soda_co
 from datacontract.engines.soda.connections.trino import to_trino_soda_configuration
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
-from datacontract.model.run import Check, Log, Run
+from datacontract.model.run import Check, Log, ResultEnum, Run
 
 
 def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
@@ -33,7 +33,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
                 Check(
                     type="general",
                     name="Check that format is supported",
-                    result="warning",
+                    result=ResultEnum.warning,
                     reason=f"Format {server.format} not yet supported by datacontract CLI",
                     engine="datacontract",
                 )
@@ -93,7 +93,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             Check(
                 type="general",
                 name="Check that server type is supported",
-                result="warning",
+                result=ResultEnum.warning,
                 reason=f"Server type {server.type} not yet supported by datacontract CLI",
                 engine="datacontract-cli",
             )
@@ -182,5 +182,5 @@ def update_reason(check, c):
                 check.reason = diagnostics_text_split[1].strip()
                 # print(check.reason)
             break  # Exit the loop once the desired block is found
-    if c["diagnostics"]["fail"] is not None:
+    if "fail" in c["diagnostics"]:
         check.reason = f"Got: {c['diagnostics']['value']} Expected: {c['diagnostics']['fail']}"

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/datacontract/export/dbt_converter.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Optional
 
 import yaml
 
@@ -52,14 +52,14 @@ def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name
         # TODO escape SQL reserved key words, probably dependent on server type
         columns.append(field_name)
     return f"""
-    select 
+    select
         {", ".join(columns)}
     from {{{{ source('{id}', '{model_name}') }}}}
 """
 
 
 def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: str = None):
-    source = {"name": data_contract_spec.id, "tables": []}
+    source = {"name": data_contract_spec.id}
     dbt = {
         "version": 2,
         "sources": [source],
@@ -69,24 +69,31 @@ def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: s
     if data_contract_spec.info.description is not None:
         source["description"] = data_contract_spec.info.description
     found_server = data_contract_spec.servers.get(server)
+    adapter_type = None
     if found_server is not None:
-        source["database"] = found_server.database
-        source["schema"] = found_server.schema_
+        adapter_type = found_server.type
+        if adapter_type == "bigquery":
+            source["database"] = found_server.project
+            source["schema"] = found_server.dataset
+        else:
+            source["database"] = found_server.database
+            source["schema"] = found_server.schema_
 
+    source["tables"] = []
     for model_key, model_value in data_contract_spec.models.items():
-        dbt_model = _to_dbt_source_table(model_key, model_value)
+        dbt_model = _to_dbt_source_table(model_key, model_value, adapter_type)
         source["tables"].append(dbt_model)
     return yaml.dump(dbt, indent=2, sort_keys=False, allow_unicode=True)
 
 
-def _to_dbt_source_table(model_key, model_value: Model) -> dict:
+def _to_dbt_source_table(model_key, model_value: Model, adapter_type: Optional[str]) -> dict:
     dbt_model = {
         "name": model_key,
     }
 
     if model_value.description is not None:
         dbt_model["description"] = model_value.description
-    columns = _to_columns(model_value.fields, False, False)
+    columns = _to_columns(model_value.fields, False, adapter_type)
     if columns:
         dbt_model["columns"] = columns
     return dbt_model
@@ -107,7 +114,7 @@ def _to_dbt_model(model_key, model_value: Model, data_contract_spec: DataContrac
         dbt_model["config"]["contract"] = {"enforced": True}
     if model_value.description is not None:
         dbt_model["description"] = model_value.description
-    columns = _to_columns(model_value.fields, _supports_constraints(model_type), True)
+    columns = _to_columns(model_value.fields, _supports_constraints(model_type), None)
     if columns:
         dbt_model["columns"] = columns
     return dbt_model
@@ -130,48 +137,47 @@ def _supports_constraints(model_type):
     return model_type == "table" or model_type == "incremental"
 
 
-def _to_columns(fields: Dict[str, Field], supports_constraints: bool, supports_datatype: bool) -> list:
+def _to_columns(fields: Dict[str, Field], supports_constraints: bool, adapter_type: Optional[str]) -> list:
     columns = []
     for field_name, field in fields.items():
-        column = _to_column(field, supports_constraints, supports_datatype)
-        column["name"] = field_name
+        column = _to_column(field_name, field, supports_constraints, adapter_type)
         columns.append(column)
     return columns
 
 
-def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool) -> dict:
-    column = {}
-    dbt_type = convert_to_sql_type(field, "snowflake")
+def _to_column(field_name: str, field: Field, supports_constraints: bool, adapter_type: Optional[str]) -> dict:
+    column = {"name": field_name}
+    adapter_type = adapter_type or "snowflake"
+    dbt_type = convert_to_sql_type(field, adapter_type)
+
+    column["data_tests"] = []
     if dbt_type is not None:
-        if supports_datatype:
-            column["data_type"] = dbt_type
-        else:
-            column.setdefault("tests", []).append(
-                {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {"column_type": dbt_type}}
-            )
+        column["data_type"] = dbt_type
+    else:
+        column["data_tests"].append(
+            {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {"column_type": dbt_type}}
+        )
     if field.description is not None:
         column["description"] = field.description
     if field.required:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "not_null"})
         else:
-            column.setdefault("tests", []).append("not_null")
+            column["data_tests"].append("not_null")
     if field.unique:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "unique"})
         else:
-            column.setdefault("tests", []).append("unique")
+            column["data_tests"].append("unique")
     if field.enum is not None and len(field.enum) > 0:
-        column.setdefault("tests", []).append({"accepted_values": {"values": field.enum}})
+        column["data_tests"].append({"accepted_values": {"values": field.enum}})
     if field.minLength is not None or field.maxLength is not None:
         length_test = {}
         if field.minLength is not None:
             length_test["min_value"] = field.minLength
         if field.maxLength is not None:
             length_test["max_value"] = field.maxLength
-        column.setdefault("tests", []).append(
-            {"dbt_expectations.expect_column_value_lengths_to_be_between": length_test}
-        )
+        column["data_tests"].append({"dbt_expectations.expect_column_value_lengths_to_be_between": length_test})
     if field.pii is not None:
         column.setdefault("meta", {})["pii"] = field.pii
     if field.classification is not None:
@@ -180,9 +186,7 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
         column.setdefault("tags", []).extend(field.tags)
     if field.pattern is not None:
         # Beware, the data contract pattern is a regex, not a like pattern
-        column.setdefault("tests", []).append(
-            {"dbt_expectations.expect_column_values_to_match_regex": {"regex": field.pattern}}
-        )
+        column["data_tests"].append({"dbt_expectations.expect_column_values_to_match_regex": {"regex": field.pattern}})
     if (
         field.minimum is not None
         or field.maximum is not None
@@ -194,7 +198,7 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
             range_test["min_value"] = field.minimum
         if field.maximum is not None:
             range_test["max_value"] = field.maximum
-        column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": range_test})
+        column["data_tests"].append({"dbt_expectations.expect_column_values_to_be_between": range_test})
     elif (
         field.exclusiveMinimum is not None
         or field.exclusiveMaximum is not None
@@ -207,18 +211,18 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
         if field.exclusiveMaximum is not None:
             range_test["max_value"] = field.exclusiveMaximum
         range_test["strictly"] = True
-        column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": range_test})
+        column["data_tests"].append({"dbt_expectations.expect_column_values_to_be_between": range_test})
     else:
         if field.minimum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {"dbt_expectations.expect_column_values_to_be_between": {"min_value": field.minimum}}
             )
         if field.maximum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {"dbt_expectations.expect_column_values_to_be_between": {"max_value": field.maximum}}
             )
         if field.exclusiveMinimum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {
                     "dbt_expectations.expect_column_values_to_be_between": {
                         "min_value": field.exclusiveMinimum,
@@ -227,7 +231,7 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
                 }
             )
         if field.exclusiveMaximum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {
                     "dbt_expectations.expect_column_values_to_be_between": {
                         "max_value": field.exclusiveMaximum,
@@ -236,5 +240,8 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
                 }
             )
 
+        if not column["data_tests"]:
+            column.pop("data_tests")
+
     # TODO: all constraints
     return column

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/datacontract/export/exporter.py

@@ -40,6 +40,7 @@ class ExportFormat(str, Enum):
     sqlalchemy = "sqlalchemy"
     data_caterer = "data-caterer"
     dcs = "dcs"
+    iceberg = "iceberg"
 
     @classmethod
     def get_supported_formats(cls):

{datacontract_cli-0.10.15 → datacontract_cli-0.10.16}/datacontract/export/exporter_factory.py

@@ -168,3 +168,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.dcs, module_path="datacontract.export.dcs_exporter", class_name="DcsExporter"
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
+)