datacontract-cli 0.10.12__tar.gz → 0.10.13__tar.gz

{datacontract_cli-0.10.12/datacontract_cli.egg-info → datacontract_cli-0.10.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datacontract-cli
-Version: 0.10.12
+Version: 0.10.13
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 Project-URL: Homepage, https://cli.datacontract.com
@@ -12,16 +12,16 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: typer<0.13,>=0.12
-Requires-Dist: pydantic<2.9.0,>=2.8.2
+Requires-Dist: pydantic<2.10.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.33,>=2.31
-Requires-Dist: fastapi==0.112.0
-Requires-Dist: uvicorn==0.30.5
+Requires-Dist: fastapi==0.114.2
+Requires-Dist: uvicorn==0.30.6
 Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
 Requires-Dist: fastparquet==2024.5.0
 Requires-Dist: python-multipart==0.0.9
-Requires-Dist: rich~=13.7.0
-Requires-Dist: simple-ddl-parser==1.6.0
+Requires-Dist: rich<13.9,>=13.7
+Requires-Dist: simple-ddl-parser==1.6.1
 Requires-Dist: duckdb==1.0.0
 Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
 Requires-Dist: setuptools>=60
@@ -29,7 +29,7 @@ Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: rdflib==7.0.0
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
 Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
-Requires-Dist: boto3<1.35.6,>=1.34.41
+Requires-Dist: boto3<1.35.20,>=1.34.41
 Requires-Dist: jinja_partials>=0.2.1
 Provides-Extra: avro
 Requires-Dist: avro==1.12.0; extra == "avro"
@@ -37,7 +37,8 @@ Provides-Extra: bigquery
 Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
 Provides-Extra: databricks
 Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
-Requires-Dist: databricks-sql-connector<3.4.0,>=3.1.2; extra == "databricks"
+Requires-Dist: databricks-sql-connector<3.5.0,>=3.1.2; extra == "databricks"
+Requires-Dist: databricks-sdk<0.33.0,>=0.32.0; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
 Provides-Extra: kafka
 Requires-Dist: datacontract-cli[avro]; extra == "kafka"
@@ -45,7 +46,7 @@ Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
 Provides-Extra: postgres
 Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
 Provides-Extra: s3
-Requires-Dist: s3fs==2024.6.1; extra == "s3"
+Requires-Dist: s3fs==2024.9.0; extra == "s3"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python[pandas]<3.13,>=3.6; extra == "snowflake"
 Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
@@ -63,7 +64,7 @@ Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
 Requires-Dist: httpx==0.27.2; extra == "dev"
 Requires-Dist: kafka-python; extra == "dev"
-Requires-Dist: moto==5.0.13; extra == "dev"
+Requires-Dist: moto==5.0.14; extra == "dev"
 Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<3.9.0,>=3.7.1; extra == "dev"
 Requires-Dist: pyarrow>=12.0.0; extra == "dev"
@@ -825,7 +826,7 @@ models:
 │ *  --format        [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db  The export format. [default: None] [required]         │
 │                    t-staging-sql|odcs|rdf|avro|protobuf|great-expectati                                                        │
 │                    ons|terraform|avro-idl|sql|sql-query|html|go|bigquer                                                        │
-│                    y|dbml|spark|sqlalchemy]                                                                                    │
+│                    y|dbml|spark|sqlalchemy|data-caterer]                                                                       │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -877,6 +878,7 @@ Available export options:
 | `DBML`               | Export to a DBML Diagram description                    | ✅     |
 | `spark`              | Export to a Spark StructType                            | ✅     |
 | `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅     |
+| `data-caterer`       | Export to Data Caterer in YAML format                   | ✅     |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 #### Great Expectations
@@ -940,6 +942,20 @@ To specify custom Avro properties in your data contract, you can define them wit
 
 >NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
 
+#### Data Caterer
+
+The export function converts the data contract to a data generation task in YAML format that can be 
+ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the 
+ability to generate production-like data in any environment based off your data contract.
+
+```shell
+datacontract export datacontract.yaml --format data-caterer --model orders
+```
+
+You can further customise the way data is generated via adding 
+[additional metadata in the YAML](https://data.catering/setup/generator/data-generator/) 
+to suit your needs.
+
 #### Example Configuration
 
 ```yaml

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/README.md

@@ -749,7 +749,7 @@ models:
 │ *  --format        [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db  The export format. [default: None] [required]         │
 │                    t-staging-sql|odcs|rdf|avro|protobuf|great-expectati                                                        │
 │                    ons|terraform|avro-idl|sql|sql-query|html|go|bigquer                                                        │
-│                    y|dbml|spark|sqlalchemy]                                                                                    │
+│                    y|dbml|spark|sqlalchemy|data-caterer]                                                                       │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -801,6 +801,7 @@ Available export options:
 | `DBML`               | Export to a DBML Diagram description                    | ✅     |
 | `spark`              | Export to a Spark StructType                            | ✅     |
 | `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅     |
+| `data-caterer`       | Export to Data Caterer in YAML format                   | ✅     |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 #### Great Expectations
@@ -864,6 +865,20 @@ To specify custom Avro properties in your data contract, you can define them wit
 
 >NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
 
+#### Data Caterer
+
+The export function converts the data contract to a data generation task in YAML format that can be 
+ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the 
+ability to generate production-like data in any environment based off your data contract.
+
+```shell
+datacontract export datacontract.yaml --format data-caterer --model orders
+```
+
+You can further customise the way data is generated via adding 
+[additional metadata in the YAML](https://data.catering/setup/generator/data-generator/) 
+to suit your needs.
+
 #### Example Configuration
 
 ```yaml

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/engines/soda/connections/kafka.py

@@ -25,9 +25,10 @@ def create_spark_session(tmp_dir: str):
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
         .config("spark.streaming.stopGracefullyOnShutdown", "true")
+        .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0,org.apache.spark:spark-avro_2.12:3.5.0",
+            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2,org.apache.spark:spark-avro_2.12:3.5.2",
         )
         .getOrCreate()
     )
@@ -112,7 +113,7 @@ def get_auth_options():
     kafka_sasl_username = os.getenv("DATACONTRACT_KAFKA_SASL_USERNAME")
     kafka_sasl_password = os.getenv("DATACONTRACT_KAFKA_SASL_PASSWORD")
 
-    if kafka_sasl_username is None:
+    if kafka_sasl_username is None or kafka_sasl_username == "":
         return {}
 
     return {

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/export/bigquery_converter.py

@@ -44,7 +44,7 @@ def to_fields_array(fields: Dict[str, Field]) -> List[Dict[str, Field]]:
 
 
 def to_field(field_name: str, field: Field) -> dict:
-    bq_type = map_type_to_bigquery(field.type, field_name)
+    bq_type = map_type_to_bigquery(field)
     bq_field = {
         "name": field_name,
         "type": bq_type,
@@ -59,10 +59,8 @@ def to_field(field_name: str, field: Field) -> dict:
             # in case the array type is a complex object, we want to copy all its fields
             bq_field["fields"] = to_fields_array(field.items.fields)
         else:
-            # otherwise we make up a structure that gets us a single field of the specified type
-            bq_field["fields"] = to_fields_array(
-                {f"{field_name}_1": Field(type=field.items.type, required=False, description="")}
-            )
+            bq_field["type"] = map_type_to_bigquery(field.items)
+
     # all of these can carry other fields
     elif bq_type.lower() in ["record", "struct"]:
         bq_field["fields"] = to_fields_array(field.fields)
@@ -79,37 +77,46 @@ def to_field(field_name: str, field: Field) -> dict:
     return bq_field
 
 
-def map_type_to_bigquery(type_str: str, field_name: str) -> str:
+def map_type_to_bigquery(field: Field) -> str:
     logger = logging.getLogger(__name__)
-    if type_str.lower() in ["string", "varchar", "text"]:
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    if field.config and "bigqueryType" in field.config:
+        return field.config["bigqueryType"]
+
+    if field_type.lower() in ["string", "varchar", "text"]:
         return "STRING"
-    elif type_str == "bytes":
+    elif field_type.lower() == "bytes":
         return "BYTES"
-    elif type_str.lower() in ["int", "integer"]:
+    elif field_type.lower() in ["int", "integer"]:
         return "INTEGER"
-    elif type_str.lower() in ["long", "bigint"]:
+    elif field_type.lower() in ["long", "bigint"]:
         return "INT64"
-    elif type_str == "float":
-        return "FLOAT"
-    elif type_str == "boolean":
+    elif field_type.lower() == "float":
+        return "FLOAT64"
+    elif field_type.lower() == "boolean":
         return "BOOL"
-    elif type_str.lower() in ["timestamp", "timestamp_tz"]:
+    elif field_type.lower() in ["timestamp", "timestamp_tz"]:
         return "TIMESTAMP"
-    elif type_str == "date":
+    elif field_type.lower() == "date":
         return "DATE"
-    elif type_str == "timestamp_ntz":
+    elif field_type.lower() == "timestamp_ntz":
         return "TIME"
-    elif type_str.lower() in ["number", "decimal", "numeric"]:
+    elif field_type.lower() in ["number", "decimal", "numeric"]:
         return "NUMERIC"
-    elif type_str == "double":
+    elif field_type.lower() == "double":
         return "BIGNUMERIC"
-    elif type_str.lower() in ["object", "record", "array"]:
+    elif field_type.lower() in ["object", "record", "array"]:
         return "RECORD"
-    elif type_str == "struct":
+    elif field_type.lower() == "struct":
         return "STRUCT"
-    elif type_str == "null":
+    elif field_type.lower() == "null":
         logger.info(
-            f"Can't properly map {field_name} to bigquery Schema, as 'null' is not supported as a type. Mapping it to STRING."
+            f"Can't properly map {field.title} to bigquery Schema, as 'null' \
+                 is not supported as a type. Mapping it to STRING."
         )
         return "STRING"
     else:
@@ -117,6 +124,6 @@ def map_type_to_bigquery(type_str: str, field_name: str) -> str:
             type="schema",
             result="failed",
             name="Map datacontract type to bigquery data type",
-            reason=f"Unsupported type {type_str} in data contract definition.",
+            reason=f"Unsupported type {field_type} in data contract definition.",
             engine="datacontract",
         )

datacontract_cli-0.10.13/datacontract/export/data_caterer_converter.py

@@ -0,0 +1,148 @@
+from typing import Dict
+
+import yaml
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Server
+
+
+class DataCatererExporter(Exporter):
+    """
+    Exporter class for Data Caterer.
+    Creates a YAML file, based on the data contract, for Data Caterer to generate synthetic data.
+    """
+
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_data_caterer_generate_yaml(data_contract, server)
+
+
+def to_data_caterer_generate_yaml(data_contract_spec: DataContractSpecification, server):
+    generation_task = {"name": data_contract_spec.info.title, "steps": []}
+    server_info = _get_server_info(data_contract_spec, server)
+
+    for model_key, model_value in data_contract_spec.models.items():
+        odcs_table = _to_data_caterer_generate_step(model_key, model_value, server_info)
+        generation_task["steps"].append(odcs_table)
+    return yaml.dump(generation_task, indent=2, sort_keys=False, allow_unicode=True)
+
+
+def _get_server_info(data_contract_spec: DataContractSpecification, server):
+    if server is not None and server in data_contract_spec.servers:
+        return data_contract_spec.servers.get(server)
+    elif server is not None:
+        raise Exception(f"Server name not found in servers list in data contract, server-name={server}")
+    elif len(data_contract_spec.servers.keys()) > 0:
+        return next(iter(data_contract_spec.servers.values()))
+    else:
+        return None
+
+
+def _to_data_caterer_generate_step(model_key, model_value: Model, server: Server) -> dict:
+    step = {
+        "name": model_key,
+        "type": _to_step_type(server),
+        "options": _to_data_source_options(model_key, server),
+        "schema": [],
+    }
+    fields = _to_fields(model_value.fields)
+    if fields:
+        step["schema"] = fields
+    return step
+
+
+def _to_step_type(server: Server):
+    if server is not None and server.type is not None:
+        if server.type in ["s3", "gcs", "azure", "local"]:
+            return server.format
+        else:
+            return server.type
+    else:
+        return "csv"
+
+
+def _to_data_source_options(model_key, server: Server):
+    options = {}
+    if server is not None and server.type is not None:
+        if server.type in ["s3", "gcs", "azure", "local"]:
+            if server.path is not None:
+                options["path"] = server.path
+            elif server.location is not None:
+                options["path"] = server.location
+            else:
+                options["path"] = "/tmp/data_caterer_data"
+        elif server.type == "postgres":
+            options["schema"] = server.schema_
+            options["table"] = model_key
+        elif server.type == "kafka":
+            options["topic"] = server.topic
+
+    return options
+
+
+def _to_fields(fields: Dict[str, Field]) -> list:
+    dc_fields = []
+    for field_name, field in fields.items():
+        column = _to_field(field_name, field)
+        dc_fields.append(column)
+    return dc_fields
+
+
+def _to_field(field_name: str, field: Field) -> dict:
+    dc_field = {"name": field_name}
+    dc_generator_opts = {}
+
+    if field.type is not None:
+        new_type = _to_data_type(field.type)
+        dc_field["type"] = _to_data_type(field.type)
+        if new_type == "object" or new_type == "record" or new_type == "struct":
+            # need to get nested field definitions
+            nested_fields = _to_fields(field.fields)
+            dc_field["schema"] = {"fields": nested_fields}
+
+    if field.enum is not None and len(field.enum) > 0:
+        dc_generator_opts["oneOf"] = field.enum
+    if field.unique is not None and field.unique:
+        dc_generator_opts["isUnique"] = field.unique
+    if field.minLength is not None:
+        dc_generator_opts["minLength"] = field.minLength
+    if field.maxLength is not None:
+        dc_generator_opts["maxLength"] = field.maxLength
+    if field.pattern is not None:
+        dc_generator_opts["regex"] = field.pattern
+    if field.minimum is not None:
+        dc_generator_opts["min"] = field.minimum
+    if field.maximum is not None:
+        dc_generator_opts["max"] = field.maximum
+
+    if len(dc_generator_opts.keys()) > 0:
+        dc_field["generator"] = {"options": dc_generator_opts}
+    return dc_field
+
+
+def _to_data_type(data_type):
+    if data_type == "number" or data_type == "numeric" or data_type == "double":
+        return "double"
+    elif data_type == "decimal" or data_type == "bigint":
+        return "decimal"
+    elif data_type == "int":
+        return "integer"
+    elif data_type == "long":
+        return "long"
+    elif data_type == "float":
+        return "float"
+    elif data_type == "string" or data_type == "text" or data_type == "varchar":
+        return "string"
+    if data_type == "boolean":
+        return "boolean"
+    if data_type == "timestamp" or data_type == "timestamp_tz" or data_type == "timestamp_ntz":
+        return "timestamp"
+    elif data_type == "date":
+        return "date"
+    elif data_type == "array":
+        return "array"
+    elif data_type == "map" or data_type == "object" or data_type == "record" or data_type == "struct":
+        return "struct"
+    elif data_type == "bytes":
+        return "binary"
+    else:
+        return "string"

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/export/exporter.py

@@ -36,6 +36,7 @@ class ExportFormat(str, Enum):
     dbml = "dbml"
     spark = "spark"
     sqlalchemy = "sqlalchemy"
+    data_caterer = "data-caterer"
 
     @classmethod
     def get_supported_formats(cls):

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/export/exporter_factory.py

@@ -62,6 +62,12 @@ exporter_factory.register_lazy_exporter(
     class_name="BigQueryExporter",
 )
 
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.data_caterer,
+    module_path="datacontract.export.data_caterer_converter",
+    class_name="DataCatererExporter",
+)
+
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.dbml, module_path="datacontract.export.dbml_converter", class_name="DbmlExporter"
 )

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/export/jsonschema_converter.py

@@ -36,7 +36,19 @@ def to_property(field: Field) -> dict:
     property = {}
     json_type, json_format = convert_type_format(field.type, field.format)
     if json_type is not None:
-        property["type"] = json_type
+        if not field.required:
+            """
+            From: https://json-schema.org/understanding-json-schema/reference/type
+            The type keyword may either be a string or an array:
+
+            If it's a string, it is the name of one of the basic types above.
+            If it is an array, it must be an array of strings, where each string 
+            is the name of one of the basic types, and each element is unique.
+            In this case, the JSON snippet is valid if it matches any of the given types.
+            """
+            property["type"] = [json_type, "null"]
+        else:
+            property["type"] = json_type
     if json_format is not None:
         property["format"] = json_format
     if field.unique:
@@ -50,7 +62,6 @@ def to_property(field: Field) -> dict:
         property["required"] = to_required(field.fields)
     if json_type == "array":
         property["items"] = to_property(field.items)
-
     if field.pattern:
         property["pattern"] = field.pattern
     if field.enum:

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/export/spark_converter.py

@@ -128,7 +128,7 @@ def to_data_type(field: Field) -> types.DataType:
     if field_type in ["string", "varchar", "text"]:
         return types.StringType()
     if field_type in ["number", "decimal", "numeric"]:
-        return types.DecimalType()
+        return types.DecimalType(precision=field.precision, scale=field.scale)
     if field_type in ["integer", "int"]:
         return types.IntegerType()
     if field_type == "long":

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/export/sql_type_converter.py

@@ -269,16 +269,7 @@ def convert_type_to_sqlserver(field: Field) -> None | str:
 
 def convert_type_to_bigquery(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent bigquery types"""
-    field_type = field.type
-    if not field_type:
-        return None
-
-    # If provided sql-server config type, prefer it over default mapping
-    if bigquery_type := get_type_config(field, "bigqueryType"):
-        return bigquery_type
-
-    field_type = field_type.lower()
-    return map_type_to_bigquery(field_type, field.title)
+    return map_type_to_bigquery(field)
 
 
 def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:

{datacontract_cli-0.10.12 → datacontract_cli-0.10.13}/datacontract/imports/spark_importer.py

@@ -80,6 +80,8 @@ def _field_from_struct_type(spark_field: types.StructField) -> Field:
     """
     field = Field()
     field.required = not spark_field.nullable
+    field.description = spark_field.metadata.get("comment")
+
     return _type_from_data_type(field, spark_field.dataType)
 
 
@@ -121,7 +123,7 @@ def _data_type_from_spark(spark_type: types.DataType) -> str:
     """
     if isinstance(spark_type, types.StringType):
         return "string"
-    elif isinstance(spark_type, types.IntegerType):
+    elif isinstance(spark_type, (types.IntegerType, types.ShortType)):
         return "integer"
     elif isinstance(spark_type, types.LongType):
         return "long"
@@ -149,5 +151,7 @@ def _data_type_from_spark(spark_type: types.DataType) -> str:
         return "decimal"
     elif isinstance(spark_type, types.NullType):
         return "null"
+    elif isinstance(spark_type, types.VarcharType):
+        return "varchar"
     else:
         raise ValueError(f"Unsupported Spark type: {spark_type}")