datacontract-cli 0.9.7__py3-none-any.whl → 0.9.9__py3-none-any.whl

datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -25,7 +25,7 @@ def validate_json_stream(model_name, validate, json_stream):
             model=model_name,
             reason=e.message,
             engine="jsonschema",
-            original_exception=e
+            original_exception=e,
         )
 
 
@@ -79,16 +79,16 @@ def process_local_file(run, server, model_name, validate):
         return process_directory(run, path, server, model_name, validate)
     else:
         logging.info(f"Processing file {path}")
-        with open(path, 'r') as file:
+        with open(path, "r") as file:
             process_json_file(run, model_name, validate, file, server.delimiter)
 
 
 def process_directory(run, path, server, model_name, validate):
     success = True
     for filename in os.listdir(path):
-        if filename.endswith('.json'):  # or make this a parameter
+        if filename.endswith(".json"):  # or make this a parameter
             file_path = os.path.join(path, filename)
-            with open(file_path, 'r') as file:
+            with open(file_path, "r") as file:
                 if not process_json_file(run, model_name, validate, file, server.delimiter):
                     success = False
                     break
@@ -127,13 +127,15 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
 
     # Early exit conditions
     if server.format != "json":
-        run.checks.append(Check(
-            type="schema",
-            name="Check that JSON has valid schema",
-            result="warning",
-            reason="Server format is not 'json'. Skip validating jsonschema.",
-            engine="jsonschema",
-        ))
+        run.checks.append(
+            Check(
+                type="schema",
+                name="Check that JSON has valid schema",
+                result="warning",
+                reason="Server format is not 'json'. Skip validating jsonschema.",
+                engine="jsonschema",
+            )
+        )
         run.log_warn("jsonschema: Server format is not 'json'. Skip jsonschema checks.")
         return
 
@@ -155,21 +157,25 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
         elif server.type == "s3":
             process_s3_file(server, model_name, validate)
         else:
-            run.checks.append(Check(
+            run.checks.append(
+                Check(
+                    type="schema",
+                    name="Check that JSON has valid schema",
+                    model=model_name,
+                    result="warn",
+                    reason=f"Server type {server.type} not supported",
+                    engine="jsonschema",
+                )
+            )
+            return
+
+        run.checks.append(
+            Check(
                 type="schema",
                 name="Check that JSON has valid schema",
                 model=model_name,
-                result="warn",
-                reason=f"Server type {server.type} not supported",
+                result="passed",
+                reason="All JSON entries are valid.",
                 engine="jsonschema",
-            ))
-            return
-
-        run.checks.append(Check(
-            type="schema",
-            name="Check that JSON has valid schema",
-            model=model_name,
-            result="passed",
-            reason="All JSON entries are valid.",
-            engine="jsonschema",
-        ))
+            )
+        )

datacontract/engines/fastjsonschema/s3/s3_read_files.py

@@ -14,9 +14,11 @@ def yield_s3_files(s3_endpoint_url, s3_location):
 
 
 def s3_fs(s3_endpoint_url):
-    aws_access_key_id = os.getenv('DATACONTRACT_S3_ACCESS_KEY_ID')
-    aws_secret_access_key = os.getenv('DATACONTRACT_S3_SECRET_ACCESS_KEY')
-    return s3fs.S3FileSystem(key=aws_access_key_id,
-                             secret=aws_secret_access_key,
-                             anon=aws_access_key_id is None,
-                             client_kwargs={'endpoint_url': s3_endpoint_url})
+    aws_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
+    aws_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
+    return s3fs.S3FileSystem(
+        key=aws_access_key_id,
+        secret=aws_secret_access_key,
+        anon=aws_access_key_id is None,
+        client_kwargs={"endpoint_url": s3_endpoint_url},
+    )

datacontract/engines/soda/check_soda_execute.py

@@ -1,5 +1,6 @@
 import logging
 
+from pyspark.sql import SparkSession
 from soda.scan import Scan
 
 from datacontract.engines.soda.connections.bigquery import \
@@ -16,11 +17,12 @@ from datacontract.engines.soda.connections.snowflake import \
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.model.data_contract_specification import \
     DataContractSpecification, Server
-from datacontract.model.run import \
-    Run, Check, Log
+from datacontract.model.run import Run, Check, Log
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
+def check_soda_execute(
+    run: Run, data_contract: DataContractSpecification, server: Server, spark: SparkSession, tmp_dir
+):
     if data_contract is None:
         run.log_warn("Cannot run engine soda-core, as data contract is invalid")
         return
@@ -34,13 +36,15 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             scan.add_duckdb_connection(duckdb_connection=con, data_source_name=server.type)
             scan.set_data_source_name(server.type)
         else:
-            run.checks.append(Check(
-                type="general",
-                name="Check that format is supported",
-                result="warning",
-                reason=f"Format {server.format} not yet supported by datacontract CLI",
-                engine="datacontract",
-            ))
+            run.checks.append(
+                Check(
+                    type="general",
+                    name="Check that format is supported",
+                    result="warning",
+                    reason=f"Format {server.format} not yet supported by datacontract CLI",
+                    engine="datacontract",
+                )
+            )
             run.log_warn(f"Format {server.format} not yet supported by datacontract CLI")
             return
     elif server.type == "snowflake":
@@ -73,21 +77,25 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         scan.set_data_source_name(server.type)
 
     else:
-        run.checks.append(Check(
-            type="general",
-            name="Check that server type is supported",
-            result="warning",
-            reason=f"Server type {server.type} not yet supported by datacontract CLI",
-            engine="datacontract-cli",
-        ))
+        run.checks.append(
+            Check(
+                type="general",
+                name="Check that server type is supported",
+                result="warning",
+                reason=f"Server type {server.type} not yet supported by datacontract CLI",
+                engine="datacontract-cli",
+            )
+        )
         run.log_warn(f"Server type {server.type} not yet supported by datacontract CLI")
         return
 
     # Don't check types for json format, as they are checked with json schema
     # Don't check types for avro format, as they are checked with avro schema
     # Don't check types for csv format, as they are hard to detect
+    server_type = server.type
     check_types = server.format != "json" and server.format != "csv" and server.format != "avro"
-    sodacl_yaml_str = to_sodacl_yaml(data_contract, check_types)
+
+    sodacl_yaml_str = to_sodacl_yaml(data_contract, server_type, check_types)
     # print("sodacl_yaml_str:\n" + sodacl_yaml_str)
     scan.add_sodacl_yaml_str(sodacl_yaml_str)
 
@@ -102,46 +110,60 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
     for c in scan_results.get("checks"):
         check = Check(
             type="schema",
-            result="passed" if c.get("outcome") == "pass" else "failed" if c.get("outcome") == "fail" else c.get(
-                "outcome"),
-            reason=', '.join(c.get("outcomeReasons")),
+            result=to_result(c),
+            reason=", ".join(c.get("outcomeReasons")),
             name=c.get("name"),
             model=c.get("table"),
             field=c.get("column"),
             engine="soda-core",
+            diagnostics=c.get("diagnostics"),
         )
         update_reason(check, c)
         run.checks.append(check)
 
     for log in scan_results.get("logs"):
-        run.logs.append(Log(
-            timestamp=log.get("timestamp"),
-            level=log.get("level"),
-            message=log.get("message"),
-        ))
+        run.logs.append(
+            Log(
+                timestamp=log.get("timestamp"),
+                level=log.get("level"),
+                message=log.get("message"),
+            )
+        )
 
     if scan.has_error_logs():
         run.log_warn("Engine soda-core has errors. See the logs for details.")
-        run.checks.append(Check(
-            type="general",
-            name="Execute quality checks",
-            result="warning",
-            reason=f"Engine soda-core has errors. See the logs for details.",
-            engine="soda-core",
-        ))
+        run.checks.append(
+            Check(
+                type="general",
+                name="Execute quality checks",
+                result="warning",
+                reason="Engine soda-core has errors. See the logs for details.",
+                engine="soda-core",
+            )
+        )
         return
 
 
+def to_result(c) -> str:
+    soda_outcome = c.get("outcome")
+    if soda_outcome == "pass":
+        return "passed"
+    elif soda_outcome == "fail":
+        return "failed"
+    else:
+        return soda_outcome
+
+
 def update_reason(check, c):
     """Try to find a reason in diagnostics"""
     if check.result == "passed":
         return
     if check.reason is not None and check.reason != "":
         return
-    for block in c['diagnostics']['blocks']:
-        if block['title'] == 'Diagnostics':
+    for block in c["diagnostics"]["blocks"]:
+        if block["title"] == "Diagnostics":
             # Extract and print the 'text' value
-            diagnostics_text = block['text']
+            diagnostics_text = block["text"]
             print(diagnostics_text)
             diagnostics_text_split = diagnostics_text.split(":icon-fail: ")
             if len(diagnostics_text_split) > 1:

datacontract/engines/soda/connections/bigquery.py

@@ -1,18 +1,20 @@
 import os
+
 import yaml
 
+
 # https://docs.soda.io/soda/connect-bigquery.html#authentication-methods
 def to_bigquery_soda_configuration(server):
     # with service account key, using an external json file
     soda_configuration = {
         f"data_source {server.type}": {
             "type": "bigquery",
-            "account_info_json_path": os.getenv('DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH'),
+            "account_info_json_path": os.getenv("DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH"),
             "auth_scopes": ["https://www.googleapis.com/auth/bigquery"],
             "project_id": server.project,
-            "dataset": server.dataset
+            "dataset": server.dataset,
         }
     }
 
     soda_configuration_str = yaml.dump(soda_configuration)
-    return soda_configuration_str
+    return soda_configuration_str

datacontract/engines/soda/connections/dask.py

@@ -1,4 +1,3 @@
-
 # def add_s3_connection_dask_json(data_contract, scan, server):
 #     s3_access_key_id = os.getenv('DATACONTRACT_S3_ACCESS_KEY_ID')
 #     s3_secret_access_key = os.getenv('DATACONTRACT_S3_SECRET_ACCESS_KEY')

datacontract/engines/soda/connections/databricks.py

@@ -11,8 +11,8 @@ def to_databricks_soda_configuration(server):
             "host": server.host,
             "catalog": server.catalog,
             "schema": server.schema_,
-            "http_path": os.getenv('DATACONTRACT_DATABRICKS_HTTP_PATH'),
-            "token": os.getenv('DATACONTRACT_DATABRICKS_TOKEN'),
+            "http_path": os.getenv("DATACONTRACT_DATABRICKS_HTTP_PATH"),
+            "token": os.getenv("DATACONTRACT_DATABRICKS_TOKEN"),
         }
     }
 

datacontract/engines/soda/connections/duckdb.py

@@ -2,6 +2,7 @@ import logging
 import os
 
 import duckdb
+from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
 
 
 def get_duckdb_connection(data_contract, server):
@@ -12,10 +13,10 @@ def get_duckdb_connection(data_contract, server):
     if server.type == "s3":
         path = server.location
     setup_s3_connection(con, server)
-    for model_name in data_contract.models:
+    for model_name, model in data_contract.models.items():
         model_path = path
         if "{model}" in model_path:
-            model_path = model_path.format(model = model_name)
+            model_path = model_path.format(model=model_name)
         logging.info(f"Creating table {model_name} for {model_path}")
 
         if server.format == "json":
@@ -32,16 +33,32 @@ def get_duckdb_connection(data_contract, server):
                         CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
                         """)
         elif server.format == "csv":
-            con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_csv_auto('{model_path}', hive_partitioning=1);
-                        """)
+            columns = to_csv_types(model)
+            if columns is None:
+                con.sql(
+                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1);"""
+                )
+            else:
+                con.sql(
+                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1, columns={columns});"""
+                )
     return con
 
 
+def to_csv_types(model) -> dict:
+    if model is None:
+        return None
+    columns = {}
+    # ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
+    for field_name, field in model.fields.items():
+        columns[field_name] = convert_to_duckdb_csv_type(field)
+    return columns
+
+
 def setup_s3_connection(con, server):
-    s3_region = os.getenv('DATACONTRACT_S3_REGION')
-    s3_access_key_id = os.getenv('DATACONTRACT_S3_ACCESS_KEY_ID')
-    s3_secret_access_key = os.getenv('DATACONTRACT_S3_SECRET_ACCESS_KEY')
+    s3_region = os.getenv("DATACONTRACT_S3_REGION")
+    s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
+    s3_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
     # con.install_extension("httpfs")
     # con.load_extension("httpfs")
     if server.endpointUrl is not None:

datacontract/engines/soda/connections/kafka.py

@@ -4,7 +4,23 @@ import pyspark.sql.functions as fn
 from pyspark.sql import SparkSession
 from pyspark.sql.avro.functions import from_avro
 from pyspark.sql.functions import from_json, col
-from pyspark.sql.types import *
+from pyspark.sql.types import (
+    StructType,
+    DataType,
+    NullType,
+    ArrayType,
+    BinaryType,
+    DateType,
+    TimestampNTZType,
+    TimestampType,
+    BooleanType,
+    LongType,
+    IntegerType,
+    DoubleType,
+    DecimalType,
+    StringType,
+    StructField,
+)
 
 from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.model.data_contract_specification import \
@@ -15,14 +31,18 @@ from datacontract.model.exceptions import DataContractException
 def create_spark_session(tmp_dir) -> SparkSession:
     # TODO: Update dependency versions when updating pyspark
     # TODO: add protobuf library
-    spark = SparkSession.builder.appName("datacontract") \
-        .config("spark.sql.warehouse.dir", tmp_dir + "/spark-warehouse") \
-        .config("spark.streaming.stopGracefullyOnShutdown", True) \
-        .config('spark.jars.packages',
-                'org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0,org.apache.spark:spark-avro_2.12:3.5.0') \
+    spark = (
+        SparkSession.builder.appName("datacontract")
+        .config("spark.sql.warehouse.dir", tmp_dir + "/spark-warehouse")
+        .config("spark.streaming.stopGracefullyOnShutdown", True)
+        .config(
+            "spark.jars.packages",
+            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0,org.apache.spark:spark-avro_2.12:3.5.0",
+        )
         .getOrCreate()
+    )
     spark.sparkContext.setLogLevel("WARN")
-    print(f'Using PySpark version {spark.version}')
+    print(f"Using PySpark version {spark.version}")
     return spark
 
 
@@ -32,14 +52,14 @@ def read_kafka_topic(spark: SparkSession, data_contract: DataContractSpecificati
     auth_options = get_auth_options()
 
     # read full kafka topic
-    df = spark \
-        .read \
-        .format("kafka") \
-        .options(**auth_options) \
-        .option("kafka.bootstrap.servers", host) \
-        .option("subscribe", topic) \
-        .option("startingOffsets", "earliest") \
+    df = (
+        spark.read.format("kafka")
+        .options(**auth_options)
+        .option("kafka.bootstrap.servers", host)
+        .option("subscribe", topic)
+        .option("startingOffsets", "earliest")
         .load()
+    )
     # TODO a warning if none or multiple models
     model_name, model = next(iter(data_contract.models.items()))
     if server.format == "avro":
@@ -73,8 +93,8 @@ def read_kafka_topic(spark: SparkSession, data_contract: DataContractSpecificati
 
 
 def get_auth_options():
-    kafka_sasl_username = os.getenv('DATACONTRACT_KAFKA_SASL_USERNAME')
-    kafka_sasl_password = os.getenv('DATACONTRACT_KAFKA_SASL_PASSWORD')
+    kafka_sasl_username = os.getenv("DATACONTRACT_KAFKA_SASL_USERNAME")
+    kafka_sasl_password = os.getenv("DATACONTRACT_KAFKA_SASL_PASSWORD")
     if kafka_sasl_username is None:
         auth_options = {}
     else:
@@ -130,4 +150,3 @@ def to_struct_field(field_name: str, field: Field) -> StructField:
         data_type = DataType()
 
     return StructField(field_name, data_type, nullable=not field.required)
-

datacontract/engines/soda/connections/postgres.py

@@ -10,12 +10,12 @@ def to_postgres_soda_configuration(server):
             "type": "postgres",
             "host": server.host,
             "port": str(server.port),
-            "username": os.getenv('DATACONTRACT_POSTGRES_USERNAME'),
-            "password": os.getenv('DATACONTRACT_POSTGRES_PASSWORD'),
+            "username": os.getenv("DATACONTRACT_POSTGRES_USERNAME"),
+            "password": os.getenv("DATACONTRACT_POSTGRES_PASSWORD"),
             "database": server.database,
             "schema": server.schema_,
         }
     }
 
     soda_configuration_str = yaml.dump(soda_configuration)
-    return soda_configuration_str
+    return soda_configuration_str

datacontract/engines/soda/connections/snowflake.py

@@ -7,13 +7,13 @@ def to_snowflake_soda_configuration(server):
     soda_configuration = {
         f"data_source {server.type}": {
             "type": "snowflake",
-            "username": os.getenv('DATACONTRACT_SNOWFLAKE_USERNAME'),
-            "password": os.getenv('DATACONTRACT_SNOWFLAKE_PASSWORD'),
-            "role": os.getenv('DATACONTRACT_SNOWFLAKE_ROLE'),
+            "username": os.getenv("DATACONTRACT_SNOWFLAKE_USERNAME"),
+            "password": os.getenv("DATACONTRACT_SNOWFLAKE_PASSWORD"),
+            "role": os.getenv("DATACONTRACT_SNOWFLAKE_ROLE"),
             "account": server.account,
             "database": server.database,
             "schema": server.schema_,
-            "warehouse": os.getenv('DATACONTRACT_SNOWFLAKE_WAREHOUSE'),
+            "warehouse": os.getenv("DATACONTRACT_SNOWFLAKE_WAREHOUSE"),
             "connection_timeout": 5,  # minutes
         }
     }

datacontract/export/avro_converter.py

@@ -4,20 +4,20 @@ from datacontract.model.data_contract_specification import Field
 
 
 def to_avro_schema(model_name, model) -> dict:
-    return to_avro_record(model_name, model.fields, model.description)
+    return to_avro_record(model_name, model.fields, model.description, model.namespace)
+
 
 def to_avro_schema_json(model_name, model) -> str:
     schema = to_avro_schema(model_name, model)
     return json.dumps(schema, indent=2, sort_keys=False)
 
 
-def to_avro_record(name, fields, description) -> dict:
-    schema = {
-        "type": "record",
-        "name": name
-    }
+def to_avro_record(name, fields, description, namespace) -> dict:
+    schema = {"type": "record", "name": name}
     if description is not None:
         schema["doc"] = description
+    if namespace is not None:
+        schema["namespace"] = namespace
     schema["fields"] = to_avro_fields(fields)
     return schema
 
@@ -30,16 +30,14 @@ def to_avro_fields(fields):
 
 
 def to_avro_field(field, field_name):
-    avro_field = {
-        "name": field_name
-    }
+    avro_field = {"name": field_name}
     if field.description is not None:
         avro_field["doc"] = field.description
     avro_field["type"] = to_avro_type(field, field_name)
     return avro_field
 
 
-def to_avro_type(field: Field, field_name: str):
+def to_avro_type(field: Field, field_name: str) -> str | dict:
     if field.type is None:
         return "null"
     if field.type in ["string", "varchar", "text"]:
@@ -64,7 +62,7 @@ def to_avro_type(field: Field, field_name: str):
     elif field.type in ["time"]:
         return "long"
     elif field.type in ["object", "record", "struct"]:
-        return to_avro_record(field_name, field.fields, field.description)
+        return to_avro_record(field_name, field.fields, field.description, None)
     elif field.type in ["binary"]:
         return "bytes"
     elif field.type in ["array"]: