datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract_cli-0.10.37.dist-info/RECORD

@@ -0,0 +1,119 @@
+datacontract/__init__.py,sha256=ThDdxDJsd7qNErLoh628nK5M7RzhJNYCmN-C6BAJFoo,405
+datacontract/api.py,sha256=nFmrJOhC5AygY9YS1VXsbvKNtW92B8AF-lXdhuCvcPE,8578
+datacontract/cli.py,sha256=ix1iwdo8bGfUW_tf1qiSoLhGBKCLcIgpVgn6diV9MB0,20742
+datacontract/data_contract.py,sha256=24QE2ym5dfwTP6vJ0OmW37GNfGCCV6y4x4-J5Ouvfjk,13248
+datacontract/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datacontract/breaking/breaking.py,sha256=DnqgxUjD-EAZcg5RBizOP9a2WxsFTaQBik0AB_m3K00,20431
+datacontract/breaking/breaking_change.py,sha256=BIDEUo1U2CQLVT2-I5PyFttxAj6zQPI1UUkEoOOQXMY,2249
+datacontract/breaking/breaking_rules.py,sha256=M9IdzVJSA7oOr1fvLQl0y9MoBKeItPz42Db2U2cjH2Y,4063
+datacontract/catalog/catalog.py,sha256=U5TpDyT9kcF086DoDSS3bWBE4Q8uj6HVuCSFaxN5kMw,2830
+datacontract/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datacontract/engines/data_contract_checks.py,sha256=LNI0OZPrzrf7sn6lPgTL4uAHSqMH2VBrmaBo8etuSE0,37453
+datacontract/engines/data_contract_test.py,sha256=8qg0SkwtTmayfzNL2U_0xgx5Hi_DUePaMt2q_JiCqX8,4543
+datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py,sha256=zrDn-_EJJ5kv0kZWAA-toeEPuBd3YQ0-U7Jb8euNUS8,1558
+datacontract/engines/datacontract/check_that_datacontract_file_exists.py,sha256=Vw-7U0GmQT2127tybxggZfpRFiZVgoIh6ndkTGM0FP4,665
+datacontract/engines/fastjsonschema/check_jsonschema.py,sha256=EKPkFM8iGyiWsHw8peErhQvbt9gu_zJ2S-NDQnupfeM,10921
+datacontract/engines/fastjsonschema/s3/s3_read_files.py,sha256=0sTDWvuu0AzSgn7fKWJxGaTmPww00TFYyDK-X0s5T3c,1193
+datacontract/engines/soda/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datacontract/engines/soda/check_soda_execute.py,sha256=qc56ZNKyHIoSFgoXzBRioOhnhgsFSJ6L-nyBU7d0fW8,8902
+datacontract/engines/soda/connections/athena.py,sha256=wjrJA9CHhl6FbSW0HulWcYlkT2_nY1s19Y2MFe4lbCU,3028
+datacontract/engines/soda/connections/bigquery.py,sha256=C-8kxmzpYe88bJp80ObHFLMh4rpnIjnUQ7XOj0Ke7lk,903
+datacontract/engines/soda/connections/databricks.py,sha256=cMRasuO0MrSKVgHPB-9uFTGTZPFg6z9Kpk3tJ0SdR0s,943
+datacontract/engines/soda/connections/duckdb_connection.py,sha256=wGiB6EKr-OZosEFvT2gkutFgAzAxFMKicfpjbIJUZwQ,9332
+datacontract/engines/soda/connections/kafka.py,sha256=lnj_-3-CnJ6stetGqm6HOzN1Qatlw7xoCQU2zKBIXxU,8725
+datacontract/engines/soda/connections/postgres.py,sha256=9GTF4Es3M5vb7ocSGqAxXmslvkS5CjsPQGIuo020CFc,626
+datacontract/engines/soda/connections/snowflake.py,sha256=rfG2ysuqNM6TkvyqQKcGHFsTGJ6AROmud5VleUDRrb0,749
+datacontract/engines/soda/connections/sqlserver.py,sha256=RzGLbCUdRyfmDcqtM_AB9WZ-Xk-XYX91nkXpVNpYbvc,1440
+datacontract/engines/soda/connections/trino.py,sha256=JvKUP9aFg_n095oWE0-bGmfbETSWEOURGEZdQuG8txA,718
+datacontract/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datacontract/export/avro_converter.py,sha256=MnfeW2x-Eko9dK6_fpdQYWtEzLkFWXfKABAUSJqiDpo,5381
+datacontract/export/avro_idl_converter.py,sha256=SGO7JfI9UGXLYFR5wMGNUH1qf6kt9lF6dUU9roVqnFo,9878
+datacontract/export/bigquery_converter.py,sha256=9mm-XP3klu1C5J87L9EL5ZyMCJhLBwsixo3aAw9QmRI,4738
+datacontract/export/custom_converter.py,sha256=xb8KbkRRgHmT4ewwC7XxtnKpe_ZMSJWBjYOaKjmO_KQ,1216
+datacontract/export/data_caterer_converter.py,sha256=eSEuy3TbqUIG_lHYEBOydAgp_CJNoGArXrcJvh81wcw,5984
+datacontract/export/dbml_converter.py,sha256=f_OZEFwRUyL-Kg2yn_G58I8iz1VfFrZh8Nbw3Wq0JDo,4777
+datacontract/export/dbt_converter.py,sha256=58bub8n22dfL8w6bKdFe28BLF0e4PHbrxO_H3rZ9wfk,11840
+datacontract/export/dcs_exporter.py,sha256=RALQ7bLAjak7EsoFFL2GFX2Oju7pnCDPCdRN_wo9wHM,210
+datacontract/export/dqx_converter.py,sha256=5UevFPE8RdFIeu4CgeVnXMNDfWU7DhR34DW7O1aVIFs,4105
+datacontract/export/duckdb_type_converter.py,sha256=hUAAbImhJUMJOXEG-UoOKQqYGrJM6UILpn2YjUuAUOw,2216
+datacontract/export/excel_exporter.py,sha256=qDVcwO38aWPMTYyXo2qJd1HJLkmj_Izhi1hdVOQ8o_w,38424
+datacontract/export/exporter.py,sha256=DfvMHDWmdqhJswLkQ5oMNojgYDblXDuRgFJRHuFSawM,3085
+datacontract/export/exporter_factory.py,sha256=UvP3_U7xj-GEjaifi__Jri6eYKx9SFXtmSrnkSbWuP0,6318
+datacontract/export/go_converter.py,sha256=Ttvbfu3YU-3GBwRD6nwCsFyZuc_hiIvJD-Jg2sT5WLw,3331
+datacontract/export/great_expectations_converter.py,sha256=Wx0mESRy4xAf8y7HjESsGsQaaei8k9xOVu3RbC6BlQM,12257
+datacontract/export/html_exporter.py,sha256=EyTMj25_Df3irZiYw1hxVZeLYWp6YSG6z3IuFUviP14,3066
+datacontract/export/iceberg_converter.py,sha256=ArcQ_Y3z_W4_kGDU_8jPRx2-pHpP3Nhx1zYoETOL3c4,6804
+datacontract/export/jsonschema_converter.py,sha256=2MT82MurcQQbrVDRj1kFsxnmFd9scNSfYI1upQSecl4,5631
+datacontract/export/markdown_converter.py,sha256=J6QEGuopR9AUyEhux1GpjmJaQa1iihsbNMAmGRQ63BQ,10430
+datacontract/export/mermaid_exporter.py,sha256=Hg2yc5DYDTEZ7etoIhB1LU6rob_sGlouDtkPxUtf6kQ,4008
+datacontract/export/odcs_v3_exporter.py,sha256=b__AiPAnCUuFQE5DPHsvXBrMeEl1t_mJ1vzTx84TMlI,13931
+datacontract/export/pandas_type_converter.py,sha256=464pQ3JQKFQa1TO0HBNcEoZvQye_yUbY6jQtiBaphSc,1117
+datacontract/export/protobuf_converter.py,sha256=DHLl8BW26xqltBsd7Qhz0RhTl9YZQKCbkmjNpECgubg,7928
+datacontract/export/pydantic_converter.py,sha256=1Lt9F8i6zyQYb44MyQtsXwCWWXYxZ47SmzArr_uPqsU,5579
+datacontract/export/rdf_converter.py,sha256=zY2BZrRxM0J6C2cgf5zA8c7FxDRImFjZUrJ4ksmvSTw,6435
+datacontract/export/sodacl_converter.py,sha256=75vQ2TnoLfjiDtWT2x8opumvotXVRs1YaIu1NLYz05M,1473
+datacontract/export/spark_converter.py,sha256=aol9ygEq29mjrZNiaK3Vdm8kEZhCgFFphuFiFDX-pOE,7953
+datacontract/export/sql_converter.py,sha256=vyLbDqzt_J3LRXpPv2W2HqUIyAtQx_S-jviBiSxh14A,5087
+datacontract/export/sql_type_converter.py,sha256=eWHRHJNeg6oOT2uUPjmcVjEf6H_qXZvDhvSCk-_iBAM,13890
+datacontract/export/sqlalchemy_converter.py,sha256=0DMncvA811lTtd5q4ZORREQ9YH1vQm1lJeqMWsFvloE,6463
+datacontract/export/terraform_converter.py,sha256=ExFoEvErVk-gBnWJiqC38SxDUmUEydpACWc917l5RyM,2163
+datacontract/imports/avro_importer.py,sha256=isfAnMq9bk-Yo5zSyTnqMegu7JIujn_sTGSTOYAc8-0,11847
+datacontract/imports/bigquery_importer.py,sha256=7TcP9FDsIas5LwJZ-HrOPXZ-NuR056sxLfDDh3vjo8E,8419
+datacontract/imports/csv_importer.py,sha256=mBsmyTvfB8q64Z3NYqv4zTDUOvoXG896hZvp3oLt5YM,5330
+datacontract/imports/dbml_importer.py,sha256=o0IOgvXN34lU1FICDHm_QUTv0DKsgwbHPHUDxQhIapE,3872
+datacontract/imports/dbt_importer.py,sha256=hQwqD9vbvwLLc6Yj3tQbar5ldI0pV-ynSiz7CZZ0JCc,8290
+datacontract/imports/excel_importer.py,sha256=eBLc9VS9OYVFYFcHFHq9HYOStAPBDfVHwmgnBHjxOmc,46415
+datacontract/imports/glue_importer.py,sha256=fiJPkvfwOCsaKKCGW19-JM5CCGXZ2mkNrVtUzp2iw6g,8370
+datacontract/imports/iceberg_importer.py,sha256=vadGJVqQKgG-j8swUytZALFB8QjbGRqZPCcPcCy0vco,5923
+datacontract/imports/importer.py,sha256=NRhR_9AWPWDNq2ac_DVUHGoJuvkVpwwaao8nDfJG_l0,1257
+datacontract/imports/importer_factory.py,sha256=RS7uwkkT7rIKGeMKgPmZhE3GVC9IfZxZhm8XN0ooa3U,4124
+datacontract/imports/json_importer.py,sha256=JeGbqAC_wAO0u8HeMA5H-KJBfs6gpp1oGIpxt6nxSZI,12641
+datacontract/imports/jsonschema_importer.py,sha256=67H__XLugV4vguHrIqzW02dtx27zYTWnOms4D1ma3bk,4961
+datacontract/imports/odcs_importer.py,sha256=ZP2u3kJsgULANTbbqkP3joOlU9cUneZOPy6Ak3oTMgs,2140
+datacontract/imports/odcs_v3_importer.py,sha256=8mWFn4Ntf0jk0DKqmrp_DN7pKlSCO88Ol_UBWzHkY20,20467
+datacontract/imports/parquet_importer.py,sha256=W_0_16mX4stwDUt4GM2L7dnGmTpAySab5k13-OlTCCc,3095
+datacontract/imports/protobuf_importer.py,sha256=rlUIskv9PNi5rFQ4Hobt9zlnKpahGsb4dy5G5UJoVAw,10840
+datacontract/imports/spark_importer.py,sha256=OxX9hJhi8e1o1pZGOKh5zWsK96SX13r0WV04kKDD61M,8964
+datacontract/imports/sql_importer.py,sha256=AdbBe7RrOEDMwdDt4huF5XmOV2EDpOP-k_m8kFQRlJg,10130
+datacontract/imports/unity_importer.py,sha256=ZoWVMPffYNAXxPa0E8d6gRBtx3l-KSx0fPqnQx81DX0,9067
+datacontract/init/init_template.py,sha256=sLCxvXHqoeW-Qes9W8GSVPfDmmu7pfnVOm-puI1-wsQ,721
+datacontract/integration/datamesh_manager.py,sha256=FT9eadzFz181lg54b49_c_x2caGJT7mR3drlZBSBJLo,3375
+datacontract/lint/files.py,sha256=tg0vq_w4LQsEr_8A5qr4hUJmHeGalUpsXJXC1t-OGC0,471
+datacontract/lint/resolve.py,sha256=TjCS0wX4OIkQsV1fXpgGwfnDRyBLrFwePyLsEVO8Qs0,15339
+datacontract/lint/resources.py,sha256=nfeZmORh1aP7EKpMKCmfbS04Te8pQ0nz64vJVkHOq3c,647
+datacontract/lint/schema.py,sha256=wijp3Ix7WNqA2gnIQ6_IxbjB6fe35nYvMNM16dtAEA4,2220
+datacontract/lint/urls.py,sha256=giac0eAYa6hha8exleL3KsiPtiFlOq8l53axtAmCilw,2529
+datacontract/model/exceptions.py,sha256=5BMuEH2qWuckNP4FTfpUEeEu6rjgGcLOD0GQugKRQ1U,1242
+datacontract/model/odcs.py,sha256=Ku-n2xLC0_5EX7KxxLWrQ5ei5kugQiJy7AChKMmRWTc,782
+datacontract/model/run.py,sha256=4UdEUaJl5RxEpN9S3swSu1vGJUVyNhOpRkdfbBZhh90,3146
+datacontract/model/data_contract_specification/__init__.py,sha256=lO7ywraknlDwJNUaSd2B9FWFsWhE8v5S-kob_shW_lg,47
+datacontract/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datacontract/output/junit_test_results.py,sha256=ZjevRMTxNSiR0HMr3bEvqv4olozPw2zEutbuLloInww,4822
+datacontract/output/output_format.py,sha256=_ZokDBo7-HXBs6czUv7kLLf9cYft_q5QaKzthsVnc58,212
+datacontract/output/test_results_writer.py,sha256=PWNLs3R_LQMH4xp5WDxLkQgY3xvj8Eyzw1jnfgkQxlc,2713
+datacontract/schemas/datacontract-1.1.0.init.yaml,sha256=ij_-ZEJP4A7ekeJfoqGpRbiCys7_YjkClNluVVo4C6E,1828
+datacontract/schemas/datacontract-1.1.0.schema.json,sha256=3Bu2rxEjkF6dNLcqi1GF4KoXBnEIopaJ87Qb8S4zUvg,62872
+datacontract/schemas/datacontract-1.2.0.init.yaml,sha256=ij_-ZEJP4A7ekeJfoqGpRbiCys7_YjkClNluVVo4C6E,1828
+datacontract/schemas/datacontract-1.2.0.schema.json,sha256=sk7oL06cug9-WozCrLH8v8MuR3a8MaV1Ztkm1P-7UFk,64226
+datacontract/schemas/datacontract-1.2.1.init.yaml,sha256=ij_-ZEJP4A7ekeJfoqGpRbiCys7_YjkClNluVVo4C6E,1828
+datacontract/schemas/datacontract-1.2.1.schema.json,sha256=Ha6F8i2jaL3BKOV5kjWgaxzykAiaSLqjIq-OEOojnx4,65233
+datacontract/schemas/odcs-3.0.1.schema.json,sha256=bRZsSXA0fV0EmV_8f1K68PlXu1m4K7JcuHpLnY3ESwQ,72933
+datacontract/schemas/odcs-3.0.2.schema.json,sha256=_J13Tqc9E7RzpSho645meE86AxeU0dIt2U12-MnAfHk,69968
+datacontract/templates/datacontract.html,sha256=dksPEnY3c66jaaVS5r5vWfG6LzyXPjA4nO_yLUirJWg,17394
+datacontract/templates/datacontract_odcs.html,sha256=u4bpcgQVqwGmR0QjijJqecOClV2ZhpDvnNMAMzj4Ezc,32659
+datacontract/templates/index.html,sha256=EY2LYSTFCwMMcezsDbYsJwM5A7As7oOEiqTBpK_MXe8,12683
+datacontract/templates/partials/datacontract_information.html,sha256=7ZBxgEgi2XndKBypeOpe03oCSRPOujC6NVlN7zexGNM,6221
+datacontract/templates/partials/datacontract_servicelevels.html,sha256=ed3QgB11B0Qq2h_NwaroGZ4pQMBPEhfeQaoS-qEipqY,11401
+datacontract/templates/partials/datacontract_terms.html,sha256=1cnJcOTpxwot2BCuZmkLF_SPfiVloLs3c8mj9WfE4sc,1865
+datacontract/templates/partials/definition.html,sha256=gZEmNvwNGGxA_Fnzx_0L6tXlAMk_EAPWr5ziRIThb_o,1005
+datacontract/templates/partials/example.html,sha256=F1dWbHDIXQScgfs4OVgqM1lR4uV4xX5j6suasXHNM88,1204
+datacontract/templates/partials/model_field.html,sha256=2YBF95ypNCPFYuYKoeilRnDG-H_FuW4JK1znkCaYCac,7625
+datacontract/templates/partials/quality.html,sha256=ynEDWRn8I90Uje-xhGYgFcfwOgKI1R-CDki-EvTsauQ,1785
+datacontract/templates/partials/server.html,sha256=dHFJtonMjhiUHtT69RUgTpkoRwmNdTRzkCdH0LtGg_4,6279
+datacontract/templates/style/output.css,sha256=ioIo1f96VW7LHhDifj6QI8QbRChJl-LlQ59EwM8MEmA,28692
+datacontract_cli-0.10.37.dist-info/licenses/LICENSE,sha256=0hcS8X51AL0UvEsx1ZM6WQcxiy9d0j5iOfzdPYM6ONU,2205
+datacontract_cli-0.10.37.dist-info/METADATA,sha256=xG1Rilzh_eBYa-gBTG4b9AreKjV5-lcZ41sVqkHtHoc,115131
+datacontract_cli-0.10.37.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datacontract_cli-0.10.37.dist-info/entry_points.txt,sha256=D3Eqy4q_Z6bHauGd4ppIyQglwbrm1AJnLau4Ppbw9Is,54
+datacontract_cli-0.10.37.dist-info/top_level.txt,sha256=VIRjd8EIUrBYWjEXJJjtdUgc0UAJdPZjmLiOR8BRBYM,13
+datacontract_cli-0.10.37.dist-info/RECORD,,

{datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.37.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.37.dist-info/licenses}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 innoQ Deutschland GmbH
+Copyright (c) 2025 Entropy Data GmbH
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py

@@ -1,48 +0,0 @@
-import logging
-
-import fastjsonschema
-import yaml
-from fastjsonschema import JsonSchemaValueException
-
-from datacontract.lint.schema import fetch_schema
-from datacontract.model.run import Check, Run
-
-
-def check_that_datacontract_str_is_valid(run: Run, data_contract_str: str):
-    schema = fetch_schema()
-    data_contract_yaml = yaml.safe_load(data_contract_str)
-    try:
-        fastjsonschema.validate(schema, data_contract_yaml)
-        logging.debug("YAML data is valid.")
-        run.checks.append(
-            Check(
-                type="lint",
-                result="passed",
-                name="Check that data contract YAML is valid",
-                engine="datacontract",
-            )
-        )
-    except JsonSchemaValueException as e:
-        logging.warning("YAML data is invalid.")
-        logging.warning(f"Validation error: {e.message}")
-        run.checks.append(
-            Check(
-                type="lint",
-                result="failed",
-                name="Check that data contract YAML is valid",
-                reason=e.message,
-                engine="datacontract",
-            )
-        )
-    except Exception as e:
-        logging.warning("YAML data is invalid.")
-        logging.warning(f"Validation error: {str(e)}")
-        run.checks.append(
-            Check(
-                type="lint",
-                result="failed",
-                name="Check that data contract YAML is valid",
-                reason=str(e),
-                engine="datacontract",
-            )
-        )

datacontract/engines/soda/connections/dask.py

@@ -1,28 +0,0 @@
-# def add_s3_connection_dask_json(data_contract, scan, server):
-#     s3_access_key_id = os.getenv('DATACONTRACT_S3_ACCESS_KEY_ID')
-#     s3_secret_access_key = os.getenv('DATACONTRACT_S3_SECRET_ACCESS_KEY')
-#     lines = server.delimiter == "new_line"
-#     for model_name in data_contract.models:
-#         logging.info(f"Connecting to {server.location}")
-#         df = dd.read_json(
-#             server.location,
-#             lines=lines,
-#             storage_options={'key': s3_access_key_id,
-#                              'secret': s3_secret_access_key,
-#                              'client_kwargs': {'endpoint_url': server.endpointUrl}
-#                              })
-#         scan.add_dask_dataframe(dataset_name=model_name, dask_df=df, data_source_name=server.type)
-
-# def add_s3_connection_dask_csv(data_contract, scan, server):
-#     s3_access_key_id = os.getenv('DATACONTRACT_S3_ACCESS_KEY_ID')
-#     s3_secret_access_key = os.getenv('DATACONTRACT_S3_SECRET_ACCESS_KEY')
-#     for model_name in data_contract.models:
-#         logging.info(f"Connecting to {server.location}")
-#         df = dd.read_csv(
-#             server.location,
-#             storage_options={'key': s3_access_key_id,
-#                              'secret': s3_secret_access_key,
-#                              'client_kwargs': {'endpoint_url': server.endpointUrl}
-#                              })
-#         scan.add_dask_dataframe(dataset_name=model_name, dask_df=df, data_source_name=server.type)
-

datacontract/engines/soda/connections/duckdb.py

@@ -1,76 +0,0 @@
-import logging
-import os
-
-import duckdb
-from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
-
-
-def get_duckdb_connection(data_contract, server):
-    con = duckdb.connect(database=":memory:")
-    path: str = ""
-    if server.type == "local":
-        path = server.path
-    if server.type == "s3":
-        path = server.location
-    setup_s3_connection(con, server)
-    for model_name, model in data_contract.models.items():
-        model_path = path
-        if "{model}" in model_path:
-            model_path = model_path.format(model=model_name)
-        logging.info(f"Creating table {model_name} for {model_path}")
-
-        if server.format == "json":
-            format = "auto"
-            if server.delimiter == "new_line":
-                format = "newline_delimited"
-            elif server.delimiter == "array":
-                format = "array"
-            con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{format}', hive_partitioning=1);
-                        """)
-        elif server.format == "parquet":
-            con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
-                        """)
-        elif server.format == "csv":
-            columns = to_csv_types(model)
-            if columns is None:
-                con.sql(
-                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1);"""
-                )
-            else:
-                con.sql(
-                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1, columns={columns});"""
-                )
-    return con
-
-
-def to_csv_types(model) -> dict:
-    if model is None:
-        return None
-    columns = {}
-    # ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
-    for field_name, field in model.fields.items():
-        columns[field_name] = convert_to_duckdb_csv_type(field)
-    return columns
-
-
-def setup_s3_connection(con, server):
-    s3_region = os.getenv("DATACONTRACT_S3_REGION")
-    s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
-    s3_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
-    # con.install_extension("httpfs")
-    # con.load_extension("httpfs")
-    if server.endpointUrl is not None:
-        s3_endpoint = server.endpointUrl.removeprefix("http://").removeprefix("https://")
-        if server.endpointUrl.startswith("http://"):
-            con.sql("SET s3_use_ssl = 0; SET s3_url_style = 'path';")
-        con.sql(f"""
-                SET s3_endpoint = '{s3_endpoint}';
-                """)
-    if s3_access_key_id is not None:
-        con.sql(f"""
-                    SET s3_region = '{s3_region}';
-                    SET s3_access_key_id = '{s3_access_key_id}';
-                    SET s3_secret_access_key = '{s3_secret_access_key}';
-                    """)

datacontract/export/csv_type_converter.py

@@ -1,36 +0,0 @@
-# https://duckdb.org/docs/data/csv/overview.html
-# ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
-def convert_to_duckdb_csv_type(field) -> None | str:
-    type = field.type
-    if type is None:
-        return "VARCHAR"
-    if type.lower() in ["string", "varchar", "text"]:
-        return "VARCHAR"
-    if type.lower() in ["timestamp", "timestamp_tz"]:
-        return "TIMESTAMP"
-    if type.lower() in ["timestamp_ntz"]:
-        return "TIMESTAMP"
-    if type.lower() in ["date"]:
-        return "DATE"
-    if type.lower() in ["time"]:
-        return "TIME"
-    if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "VARCHAR"
-    if type.lower() in ["float", "double"]:
-        return "DOUBLE"
-    if type.lower() in ["integer", "int", "long", "bigint"]:
-        return "BIGINT"
-    if type.lower() in ["boolean"]:
-        return "BOOLEAN"
-    if type.lower() in ["object", "record", "struct"]:
-        # not supported in CSV
-        return "VARCHAR"
-    if type.lower() in ["bytes"]:
-        # not supported in CSV
-        return "VARCHAR"
-    if type.lower() in ["array"]:
-        return "VARCHAR"
-    if type.lower() in ["null"]:
-        return "SQLNULL"
-    return "VARCHAR"

datacontract/export/html_export.py

@@ -1,66 +0,0 @@
-import datetime
-import logging
-from importlib.metadata import version
-
-import pytz
-import yaml
-from jinja2 import Environment, PackageLoader, select_autoescape
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
-
-
-def to_html(data_contract_spec: DataContractSpecification) -> str:
-    # Load templates from templates folder
-    package_loader = PackageLoader("datacontract", "templates")
-    env = Environment(
-        loader=package_loader,
-        autoescape=select_autoescape(
-            enabled_extensions="html",
-            default_for_string=True,
-        ),
-    )
-
-    # Load the required template
-    template = env.get_template("datacontract.html")
-
-    if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
-        quality_specification = data_contract_spec.quality.specification
-    elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
-        if data_contract_spec.quality.type == "great-expectations":
-            quality_specification = yaml.dump(
-                data_contract_spec.quality.specification, sort_keys=False, default_style="|"
-            )
-        else:
-            quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
-    else:
-        quality_specification = None
-
-    style_content, _, _ = package_loader.get_source(env, "style/output.css")
-
-    datacontract_yaml = data_contract_spec.to_yaml()
-
-    tz = pytz.timezone('UTC')
-    now = datetime.datetime.now(tz)
-    formatted_date = now.strftime('%d %b %Y %H:%M:%S UTC')
-    datacontract_cli_version = get_version()
-
-    # Render the template with necessary data
-    html_string = template.render(
-        datacontract=data_contract_spec,
-        quality_specification=quality_specification,
-        style=style_content,
-        datacontract_yaml=datacontract_yaml,
-        formatted_date=formatted_date,
-        datacontract_cli_version=datacontract_cli_version,
-    )
-
-    return html_string
-
-
-def get_version() -> str:
-    try:
-        return version("datacontract_cli")
-    except Exception as e:
-        logging.debug("Ignoring exception", e)
-        return ""

datacontract/export/odcs_converter.py

@@ -1,102 +0,0 @@
-from typing import Dict
-
-import yaml
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
-
-
-def to_odcs_yaml(data_contract_spec: DataContractSpecification):
-    odcs = {
-        "kind": "DataContract",
-        "apiVersion": "2.3.0",
-        "uuid": data_contract_spec.id,
-        "version": data_contract_spec.info.version,
-        "datasetDomain": data_contract_spec.info.owner,
-        "quantumName": data_contract_spec.info.title,
-        "status": "unknown",
-    }
-
-    if data_contract_spec.info.contact is not None:
-        if data_contract_spec.info.contact.email is not None:
-            odcs["productDl"] = data_contract_spec.info.contact.email
-        if data_contract_spec.info.contact.email is not None:
-            odcs["productFeedbackUrl"] = data_contract_spec.info.contact.url
-
-    if data_contract_spec.terms is not None:
-        odcs["description"] = {
-            "purpose": None,
-            "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
-            "limitations": data_contract_spec.terms.limitations.strip()
-            if data_contract_spec.terms.limitations is not None
-            else None,
-        }
-
-    odcs["type"] = "tables"  # required, TODO read from models.type?
-    odcs["dataset"] = []
-
-    for model_key, model_value in data_contract_spec.models.items():
-        odcs_table = to_odcs_table(model_key, model_value)
-        odcs["dataset"].append(odcs_table)
-    return yaml.dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
-
-
-def to_odcs_table(model_key, model_value: Model) -> dict:
-    odcs_table = {
-        "table": model_key,
-        "physicalName": model_key,
-        "columns": [],
-    }
-    if model_value.description is not None:
-        odcs_table["description"] = model_value.description
-    columns = to_columns(model_value.fields)
-    if columns:
-        odcs_table["columns"] = columns
-    return odcs_table
-
-
-def to_columns(fields: Dict[str, Field]) -> list:
-    columns = []
-    for field_name, field in fields.items():
-        column = to_column(field_name, field)
-        columns.append(column)
-    return columns
-
-
-def to_column(field_name: str, field: Field) -> dict:
-    column = {"column": field_name}
-    if field.type is not None:
-        column["logicalType"] = field.type
-        column["physicalType"] = field.type
-    if field.description is not None:
-        column["description"] = field.description
-    if field.required is not None:
-        column["isNullable"] = not field.required
-    if field.unique is not None:
-        column["isUnique"] = field.unique
-    if field.classification is not None:
-        column["classification"] = field.classification
-    column["tags"] = []
-    if field.tags is not None:
-        column["tags"].extend(field.tags)
-    if field.pii is not None:
-        column["tags"].append(f"pii:{str(field.pii).lower()}")
-    if field.minLength is not None:
-        column["tags"].append(f"minLength:{field.minLength}")
-    if field.maxLength is not None:
-        column["tags"].append(f"maxLength:{field.maxLength}")
-    if field.pattern is not None:
-        column["tags"].append(f"pattern:{field.pattern}")
-    if field.minimum is not None:
-        column["tags"].append(f"minimum:{field.minimum}")
-    if field.maximum is not None:
-        column["tags"].append(f"maximum:{field.maximum}")
-    if field.exclusiveMinimum is not None:
-        column["tags"].append(f"exclusiveMinimum:{field.exclusiveMinimum}")
-    if field.exclusiveMaximum is not None:
-        column["tags"].append(f"exclusiveMaximum:{field.exclusiveMaximum}")
-    if not column["tags"]:
-        del column["tags"]
-
-    # todo enum
-    return column

datacontract/init/download_datacontract_file.py

@@ -1,17 +0,0 @@
-import os
-
-import requests
-
-
-def download_datacontract_file(file_path: str, from_url: str, overwrite_file: bool):
-    if not overwrite_file and os.path.exists(file_path):
-        raise FileExistsException()
-
-    with requests.get(from_url) as response:
-        response.raise_for_status()
-        with open(file_path, "w") as f:
-            f.write(response.text)
-
-
-class FileExistsException(Exception):
-    pass

datacontract/integration/publish_datamesh_manager.py

@@ -1,33 +0,0 @@
-import os
-
-import requests
-
-from datacontract.model.run import Run
-
-
-def publish_datamesh_manager(run: Run, publish_url: str):
-    try:
-        if publish_url is None:
-            url = "https://api.datamesh-manager.com/api/runs"
-        else:
-            url = publish_url
-        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
-
-        if run.dataContractId is None:
-            raise Exception("Cannot publish run results, as data contract ID is unknown")
-
-        if datamesh_manager_api_key is None:
-            raise Exception("Cannot publish run results, as DATAMESH_MANAGER_API_KEY is not set")
-
-        headers = {"Content-Type": "application/json", "x-api-key": datamesh_manager_api_key}
-        request_body = run.model_dump_json()
-        # print("Request Body:", request_body)
-        response = requests.post(url, data=request_body, headers=headers)
-        # print("Status Code:", response.status_code)
-        # print("Response Body:", response.text)
-        if response.status_code != 200:
-            run.log_error(f"Error publishing test results to Data Mesh Manager: {response.text}")
-            return
-        run.log_info(f"Published test results to {url}")
-    except Exception as e:
-        run.log_error(f"Failed publishing test results. Error: {str(e)}")

datacontract/integration/publish_opentelemetry.py

@@ -1,107 +0,0 @@
-import logging
-import math
-import os
-from importlib import metadata
-
-from opentelemetry import metrics
-from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import \
-    OTLPMetricExporter as OTLPgRPCMetricExporter
-from opentelemetry.exporter.otlp.proto.http.metric_exporter import \
-    OTLPMetricExporter
-from opentelemetry.metrics import Observation
-from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, \
-    PeriodicExportingMetricReader
-
-from datacontract.model.run import Run
-
-
-# Publishes metrics of a test run.
-# Metric contains the values:
-# 0 == test run passed,
-# 1 == test run has warnings
-# 2 == test run failed
-# 3 == test run not possible due to an error
-# 4 == test status unknown
-#
-# Tested with these environment variables:
-#
-# OTEL_SERVICE_NAME=datacontract-cli
-# OTEL_EXPORTER_OTLP_ENDPOINT=https://YOUR_ID.apm.westeurope.azure.elastic-cloud.com:443
-# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20secret (Optional, when using SaaS Products)
-# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf and OTEL_EXPORTER_OTLP_PROTOCOL=grpc
-#
-# Current limitations:
-# - no gRPC support
-# - currently, only ConsoleExporter and OTLP Exporter
-# - Metrics only, no logs yet (but loosely planned)
-
-
-def publish_opentelemetry(run: Run):
-    try:
-        if run.dataContractId is None:
-            raise Exception("Cannot publish run results, as data contract ID is unknown")
-
-        endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
-        logging.info(f"Publishing test results to opentelemetry at {endpoint}")
-
-        telemetry = Telemetry()
-        provider = metrics.get_meter_provider()
-        meter = provider.get_meter("com.datacontract.cli", metadata.version("datacontract-cli"))
-        meter.create_observable_gauge(
-            name="datacontract.cli.test",
-            callbacks=[lambda x: _to_observation_callback(run)],
-            unit="result",
-            description="The overall result of the data contract test run",
-        )
-
-        telemetry.publish()
-    except Exception as e:
-        logging.error(f"Failed publishing test results. Error: {str(e)}")
-
-
-def _to_observation_callback(run):
-    yield _to_observation(run)
-
-
-def _to_observation(run):
-    attributes = {
-        "datacontract.id": run.dataContractId,
-        "datacontract.version": run.dataContractVersion,
-    }
-
-    if run.result == "passed":
-        result_value = 0  # think of exit codes
-    elif run.result == "warning":
-        result_value = 1
-    elif run.result == "failed":
-        result_value = 2
-    elif run.result == "error":
-        result_value = 3
-    else:
-        result_value = 4
-    return Observation(value=result_value, attributes=attributes)
-
-
-class Telemetry:
-    def __init__(self):
-        protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
-
-        # lower to allow grpc, GRPC and alike values.
-        if protocol and protocol.lower() == "grpc":
-            self.remote_exporter = OTLPgRPCMetricExporter()
-        else:
-            # Fallback to default OTEL http/protobuf which is used when the variable is not set.
-            # This Exporter also works for http/json.
-            self.remote_exporter = OTLPMetricExporter()
-
-        self.console_exporter = ConsoleMetricExporter()
-        # using math.inf so it does not collect periodically. we do this in collect ourselves, one-time.
-        self.reader = PeriodicExportingMetricReader(self.console_exporter, export_interval_millis=math.inf)
-        self.remote_reader = PeriodicExportingMetricReader(self.remote_exporter, export_interval_millis=math.inf)
-        provider = MeterProvider(metric_readers=[self.reader, self.remote_reader])
-        metrics.set_meter_provider(provider)
-
-    def publish(self):
-        self.reader.collect()
-        self.remote_reader.collect()