datacontract-cli 0.10.41__tar.gz → 0.11.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datacontract_cli-0.10.41/datacontract_cli.egg-info → datacontract_cli-0.11.2}/PKG-INFO +218 -287
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/README.md +211 -280
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/catalog/catalog.py +70 -6
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/cli.py +13 -122
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/data_contract.py +24 -145
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/data_contract_checks.py +299 -232
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/data_contract_test.py +28 -27
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +14 -10
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/fastjsonschema/check_jsonschema.py +8 -7
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/check_soda_execute.py +17 -4
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/duckdb_connection.py +81 -81
- datacontract_cli-0.11.2/datacontract/engines/soda/connections/impala.py +74 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/kafka.py +49 -28
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/oracle.py +2 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/sqlserver.py +10 -2
- datacontract_cli-0.11.2/datacontract/export/avro_exporter.py +253 -0
- datacontract_cli-0.10.41/datacontract/export/avro_idl_converter.py → datacontract_cli-0.11.2/datacontract/export/avro_idl_exporter.py +143 -74
- datacontract_cli-0.11.2/datacontract/export/bigquery_exporter.py +180 -0
- datacontract_cli-0.10.41/datacontract/export/custom_converter.py → datacontract_cli-0.11.2/datacontract/export/custom_exporter.py +5 -8
- datacontract_cli-0.11.2/datacontract/export/data_caterer_exporter.py +236 -0
- datacontract_cli-0.11.2/datacontract/export/dbml_exporter.py +173 -0
- datacontract_cli-0.11.2/datacontract/export/dbt_exporter.py +394 -0
- datacontract_cli-0.11.2/datacontract/export/dcs_exporter.py +294 -0
- datacontract_cli-0.11.2/datacontract/export/dqx_exporter.py +163 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/export/duckdb_type_converter.py +25 -15
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/export/excel_exporter.py +4 -10
- datacontract_cli-0.11.2/datacontract/export/exporter.py +122 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/export/exporter_factory.py +23 -29
- datacontract_cli-0.11.2/datacontract/export/go_exporter.py +142 -0
- datacontract_cli-0.10.41/datacontract/export/great_expectations_converter.py → datacontract_cli-0.11.2/datacontract/export/great_expectations_exporter.py +94 -104
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/export/html_exporter.py +8 -26
- datacontract_cli-0.11.2/datacontract/export/iceberg_exporter.py +315 -0
- datacontract_cli-0.11.2/datacontract/export/jsonschema_exporter.py +231 -0
- datacontract_cli-0.10.41/datacontract/export/markdown_converter.py → datacontract_cli-0.11.2/datacontract/export/markdown_exporter.py +125 -82
- datacontract_cli-0.11.2/datacontract/export/mermaid_exporter.py +73 -0
- datacontract_cli-0.11.2/datacontract/export/odcs_export_helper.py +319 -0
- datacontract_cli-0.11.2/datacontract/export/odcs_v3_exporter.py +26 -0
- datacontract_cli-0.11.2/datacontract/export/pandas_type_converter.py +56 -0
- datacontract_cli-0.10.41/datacontract/export/protobuf_converter.py → datacontract_cli-0.11.2/datacontract/export/protobuf_exporter.py +72 -72
- datacontract_cli-0.11.2/datacontract/export/pydantic_exporter.py +189 -0
- datacontract_cli-0.11.2/datacontract/export/rdf_exporter.py +169 -0
- datacontract_cli-0.10.41/datacontract/export/sodacl_converter.py → datacontract_cli-0.11.2/datacontract/export/sodacl_exporter.py +7 -7
- datacontract_cli-0.11.2/datacontract/export/spark_exporter.py +347 -0
- datacontract_cli-0.11.2/datacontract/export/sql_exporter.py +145 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/export/sql_type_converter.py +259 -147
- datacontract_cli-0.11.2/datacontract/export/sqlalchemy_exporter.py +216 -0
- datacontract_cli-0.11.2/datacontract/imports/avro_importer.py +341 -0
- datacontract_cli-0.11.2/datacontract/imports/bigquery_importer.py +257 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/csv_importer.py +81 -46
- datacontract_cli-0.11.2/datacontract/imports/dbml_importer.py +140 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/dbt_importer.py +110 -62
- datacontract_cli-0.11.2/datacontract/imports/dcs_importer.py +741 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/excel_importer.py +1 -5
- datacontract_cli-0.11.2/datacontract/imports/glue_importer.py +247 -0
- datacontract_cli-0.11.2/datacontract/imports/iceberg_importer.py +175 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/importer.py +5 -12
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/json_importer.py +112 -114
- datacontract_cli-0.11.2/datacontract/imports/jsonschema_importer.py +234 -0
- datacontract_cli-0.11.2/datacontract/imports/odcs_helper.py +306 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/odcs_importer.py +12 -16
- datacontract_cli-0.11.2/datacontract/imports/parquet_importer.py +96 -0
- datacontract_cli-0.11.2/datacontract/imports/protobuf_importer.py +234 -0
- datacontract_cli-0.11.2/datacontract/imports/spark_importer.py +221 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/sql_importer.py +68 -65
- datacontract_cli-0.11.2/datacontract/imports/unity_importer.py +165 -0
- datacontract_cli-0.11.2/datacontract/lint/resolve.py +201 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/output/test_results_writer.py +3 -5
- datacontract_cli-0.11.2/datacontract/schemas/odcs-3.1.0.schema.json +2928 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/datacontract_odcs.html +19 -1
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2/datacontract_cli.egg-info}/PKG-INFO +218 -287
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract_cli.egg-info/SOURCES.txt +24 -31
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract_cli.egg-info/requires.txt +6 -6
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/pyproject.toml +7 -7
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_api.py +1 -1
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_catalog.py +1 -1
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_data_contract_checks.py +11 -19
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_data_contract_specification.py +1 -2
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_duckdb_json.py +50 -47
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_avro.py +35 -26
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_avro_idl.py +40 -35
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_bigquery.py +4 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_complex_data_contract.py +3 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_custom.py +5 -3
- datacontract_cli-0.11.2/tests/test_export_custom_exporter.py +37 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_data_caterer.py +7 -7
- datacontract_cli-0.11.2/tests/test_export_dbml.py +122 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_dbt_models.py +34 -93
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_dbt_sources.py +8 -8
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_dbt_staging_sql.py +9 -7
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_dqx.py +3 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_go.py +5 -5
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_great_expectations.py +84 -152
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_html.py +2 -2
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_iceberg.py +90 -84
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_jsonschema.py +5 -5
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_markdown.py +5 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_mermaid.py +3 -3
- datacontract_cli-0.11.2/tests/test_export_odcs_v3.py +69 -0
- datacontract_cli-0.11.2/tests/test_export_protobuf.py +112 -0
- datacontract_cli-0.11.2/tests/test_export_pydantic.py +150 -0
- datacontract_cli-0.11.2/tests/test_export_rdf.py +189 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_sodacl.py +10 -55
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_spark.py +5 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_sql_query.py +1 -1
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_sqlalchemy.py +44 -19
- datacontract_cli-0.11.2/tests/test_import_avro.py +419 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_bigquery.py +10 -6
- datacontract_cli-0.11.2/tests/test_import_csv.py +97 -0
- datacontract_cli-0.11.2/tests/test_import_dbt.py +102 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_glue.py +0 -6
- datacontract_cli-0.11.2/tests/test_import_iceberg.py +158 -0
- datacontract_cli-0.11.2/tests/test_import_json.py +65 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_jsonschema.py +0 -3
- datacontract_cli-0.11.2/tests/test_import_odcs_v3.py +11 -0
- datacontract_cli-0.11.2/tests/test_import_parquet.py +86 -0
- datacontract_cli-0.11.2/tests/test_import_protobuf.py +75 -0
- datacontract_cli-0.11.2/tests/test_import_sql_oracle.py +230 -0
- datacontract_cli-0.11.2/tests/test_import_sql_postgres.py +147 -0
- datacontract_cli-0.11.2/tests/test_import_sql_sqlserver.py +154 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_unity_file.py +5 -37
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_lint.py +3 -21
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_resolve.py +32 -15
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_roundtrip_jsonschema.py +5 -10
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_api.py +3 -2
- datacontract_cli-0.11.2/tests/test_test_kafka.py +100 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_postgres.py +31 -3
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_sqlserver.py +1 -0
- datacontract_cli-0.10.41/datacontract/breaking/breaking.py +0 -546
- datacontract_cli-0.10.41/datacontract/breaking/breaking_change.py +0 -73
- datacontract_cli-0.10.41/datacontract/breaking/breaking_rules.py +0 -130
- datacontract_cli-0.10.41/datacontract/export/avro_converter.py +0 -138
- datacontract_cli-0.10.41/datacontract/export/bigquery_converter.py +0 -130
- datacontract_cli-0.10.41/datacontract/export/data_caterer_converter.py +0 -161
- datacontract_cli-0.10.41/datacontract/export/dbml_converter.py +0 -148
- datacontract_cli-0.10.41/datacontract/export/dbt_converter.py +0 -304
- datacontract_cli-0.10.41/datacontract/export/dcs_exporter.py +0 -6
- datacontract_cli-0.10.41/datacontract/export/dqx_converter.py +0 -126
- datacontract_cli-0.10.41/datacontract/export/exporter.py +0 -100
- datacontract_cli-0.10.41/datacontract/export/go_converter.py +0 -105
- datacontract_cli-0.10.41/datacontract/export/iceberg_converter.py +0 -188
- datacontract_cli-0.10.41/datacontract/export/jsonschema_converter.py +0 -157
- datacontract_cli-0.10.41/datacontract/export/mermaid_exporter.py +0 -110
- datacontract_cli-0.10.41/datacontract/export/odcs_v3_exporter.py +0 -377
- datacontract_cli-0.10.41/datacontract/export/pandas_type_converter.py +0 -40
- datacontract_cli-0.10.41/datacontract/export/pydantic_converter.py +0 -137
- datacontract_cli-0.10.41/datacontract/export/rdf_converter.py +0 -180
- datacontract_cli-0.10.41/datacontract/export/spark_converter.py +0 -245
- datacontract_cli-0.10.41/datacontract/export/sql_converter.py +0 -138
- datacontract_cli-0.10.41/datacontract/export/sqlalchemy_converter.py +0 -170
- datacontract_cli-0.10.41/datacontract/export/terraform_converter.py +0 -76
- datacontract_cli-0.10.41/datacontract/imports/avro_importer.py +0 -351
- datacontract_cli-0.10.41/datacontract/imports/bigquery_importer.py +0 -221
- datacontract_cli-0.10.41/datacontract/imports/dbml_importer.py +0 -112
- datacontract_cli-0.10.41/datacontract/imports/glue_importer.py +0 -288
- datacontract_cli-0.10.41/datacontract/imports/iceberg_importer.py +0 -172
- datacontract_cli-0.10.41/datacontract/imports/jsonschema_importer.py +0 -146
- datacontract_cli-0.10.41/datacontract/imports/odcs_v3_importer.py +0 -548
- datacontract_cli-0.10.41/datacontract/imports/parquet_importer.py +0 -81
- datacontract_cli-0.10.41/datacontract/imports/protobuf_importer.py +0 -294
- datacontract_cli-0.10.41/datacontract/imports/spark_importer.py +0 -262
- datacontract_cli-0.10.41/datacontract/imports/unity_importer.py +0 -219
- datacontract_cli-0.10.41/datacontract/lint/resolve.py +0 -407
- datacontract_cli-0.10.41/datacontract/model/data_contract_specification/__init__.py +0 -1
- datacontract_cli-0.10.41/datacontract/schemas/odcs-3.1.0.schema.json +0 -2809
- datacontract_cli-0.10.41/tests/test_breaking.py +0 -250
- datacontract_cli-0.10.41/tests/test_changelog.py +0 -1216
- datacontract_cli-0.10.41/tests/test_export_custom_exporter.py +0 -30
- datacontract_cli-0.10.41/tests/test_export_dbml.py +0 -137
- datacontract_cli-0.10.41/tests/test_export_odcs_v3.py +0 -134
- datacontract_cli-0.10.41/tests/test_export_protobuf.py +0 -61
- datacontract_cli-0.10.41/tests/test_export_pydantic.py +0 -122
- datacontract_cli-0.10.41/tests/test_export_rdf.py +0 -223
- datacontract_cli-0.10.41/tests/test_export_terraform.py +0 -33
- datacontract_cli-0.10.41/tests/test_import_avro.py +0 -354
- datacontract_cli-0.10.41/tests/test_import_csv.py +0 -80
- datacontract_cli-0.10.41/tests/test_import_dbt.py +0 -466
- datacontract_cli-0.10.41/tests/test_import_iceberg.py +0 -143
- datacontract_cli-0.10.41/tests/test_import_json.py +0 -151
- datacontract_cli-0.10.41/tests/test_import_odcs_v3.py +0 -124
- datacontract_cli-0.10.41/tests/test_import_parquet.py +0 -68
- datacontract_cli-0.10.41/tests/test_import_protobuf.py +0 -243
- datacontract_cli-0.10.41/tests/test_import_sql_oracle.py +0 -256
- datacontract_cli-0.10.41/tests/test_import_sql_postgres.py +0 -151
- datacontract_cli-0.10.41/tests/test_import_sql_sqlserver.py +0 -178
- datacontract_cli-0.10.41/tests/test_spec_fields_field.py +0 -17
- datacontract_cli-0.10.41/tests/test_spec_ref.py +0 -15
- datacontract_cli-0.10.41/tests/test_test_kafka.py +0 -53
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/LICENSE +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/MANIFEST.in +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/__init__.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/api.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/__init__.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/datacontract/check_that_datacontract_file_exists.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/fastjsonschema/s3/s3_read_files.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/__init__.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/athena.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/bigquery.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/databricks.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/postgres.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/snowflake.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/engines/soda/connections/trino.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/export/__init__.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/imports/importer_factory.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/init/init_template.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/integration/entropy_data.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/lint/files.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/lint/resources.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/lint/schema.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/lint/urls.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/model/exceptions.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/model/odcs.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/model/run.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/output/__init__.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/output/junit_test_results.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/output/output_format.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/py.typed +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/datacontract-1.1.0.init.yaml +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/datacontract-1.1.0.schema.json +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/datacontract-1.2.0.init.yaml +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/datacontract-1.2.0.schema.json +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/datacontract-1.2.1.init.yaml +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/datacontract-1.2.1.schema.json +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/odcs-3.0.1.schema.json +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/schemas/odcs-3.0.2.schema.json +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/datacontract.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/index.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/datacontract_information.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/datacontract_servicelevels.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/datacontract_terms.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/definition.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/example.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/model_field.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/quality.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/partials/server.html +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract/templates/style/output.css +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract_cli.egg-info/dependency_links.txt +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract_cli.egg-info/entry_points.txt +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/datacontract_cli.egg-info/top_level.txt +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/setup.cfg +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_cli.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_description_linter.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_download_datacontract_file.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_excel.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_export_sql.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_dbml.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_excel.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_import_spark.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_integration_entropydata.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_athena_iceberg.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_azure_remote.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_bigquery.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_databricks.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_dataframe.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_delta.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_gcs_csv_remote.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_gcs_json_remote.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_kafka_remote.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_local_json.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_local_json_nd.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_oracle.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_output_junit.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_parquet.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_quality.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_s3_csv.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_s3_delta.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_s3_json.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_s3_json_complex.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_s3_json_multiple_models.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_s3_json_remote.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_snowflake.py +0 -0
- {datacontract_cli-0.10.41 → datacontract_cli-0.11.2}/tests/test_test_trino.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.2
|
|
4
4
|
Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
|
|
5
5
|
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -20,8 +20,8 @@ Requires-Dist: fastparquet<2025.0.0,>=2024.5.0
|
|
|
20
20
|
Requires-Dist: numpy<2.0.0,>=1.26.4
|
|
21
21
|
Requires-Dist: python-multipart<1.0.0,>=0.0.20
|
|
22
22
|
Requires-Dist: rich<15.0,>=13.7
|
|
23
|
-
Requires-Dist: sqlglot<
|
|
24
|
-
Requires-Dist: duckdb<
|
|
23
|
+
Requires-Dist: sqlglot<29.0.0,>=26.6.0
|
|
24
|
+
Requires-Dist: duckdb<1.4.0,>=1.0.0
|
|
25
25
|
Requires-Dist: soda-core-duckdb<3.6.0,>=3.3.20
|
|
26
26
|
Requires-Dist: setuptools>=60
|
|
27
27
|
Requires-Dist: python-dotenv<2.0.0,>=1.0.0
|
|
@@ -41,7 +41,7 @@ Requires-Dist: openpyxl<4.0.0,>=3.1.5; extra == "excel"
|
|
|
41
41
|
Provides-Extra: databricks
|
|
42
42
|
Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "databricks"
|
|
43
43
|
Requires-Dist: soda-core-spark[databricks]<3.6.0,>=3.3.20; extra == "databricks"
|
|
44
|
-
Requires-Dist: databricks-sql-connector<4.
|
|
44
|
+
Requires-Dist: databricks-sql-connector<4.3.0,>=3.7.0; extra == "databricks"
|
|
45
45
|
Requires-Dist: databricks-sdk<0.74.0; extra == "databricks"
|
|
46
46
|
Requires-Dist: pyspark<4.0.0,>=3.5.5; extra == "databricks"
|
|
47
47
|
Provides-Extra: iceberg
|
|
@@ -56,7 +56,7 @@ Provides-Extra: s3
|
|
|
56
56
|
Requires-Dist: s3fs<2026.0.0,>=2025.2.0; extra == "s3"
|
|
57
57
|
Requires-Dist: aiobotocore<2.26.0,>=2.17.0; extra == "s3"
|
|
58
58
|
Provides-Extra: snowflake
|
|
59
|
-
Requires-Dist: snowflake-connector-python[pandas]<4.
|
|
59
|
+
Requires-Dist: snowflake-connector-python[pandas]<4.2,>=3.6; extra == "snowflake"
|
|
60
60
|
Requires-Dist: soda-core-snowflake<3.6.0,>=3.3.20; extra == "snowflake"
|
|
61
61
|
Provides-Extra: sqlserver
|
|
62
62
|
Requires-Dist: soda-core-sqlserver<3.6.0,>=3.3.20; extra == "sqlserver"
|
|
@@ -86,9 +86,9 @@ Requires-Dist: datacontract-cli[all]; extra == "dev"
|
|
|
86
86
|
Requires-Dist: httpx==0.28.1; extra == "dev"
|
|
87
87
|
Requires-Dist: kafka-python; extra == "dev"
|
|
88
88
|
Requires-Dist: minio==7.2.17; extra == "dev"
|
|
89
|
-
Requires-Dist: moto==5.1.
|
|
89
|
+
Requires-Dist: moto==5.1.18; extra == "dev"
|
|
90
90
|
Requires-Dist: pandas>=2.1.0; extra == "dev"
|
|
91
|
-
Requires-Dist: pre-commit<4.
|
|
91
|
+
Requires-Dist: pre-commit<4.6.0,>=3.7.1; extra == "dev"
|
|
92
92
|
Requires-Dist: pytest; extra == "dev"
|
|
93
93
|
Requires-Dist: pytest-xdist; extra == "dev"
|
|
94
94
|
Requires-Dist: pymssql==2.3.9; extra == "dev"
|
|
@@ -107,8 +107,10 @@ Dynamic: license-file
|
|
|
107
107
|
<a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social" style="max-width: 100%;"></a>
|
|
108
108
|
</p>
|
|
109
109
|
|
|
110
|
-
The `datacontract` CLI is
|
|
111
|
-
It
|
|
110
|
+
The `datacontract` CLI is an open-source command-line tool for working with [data contracts](https://datacontract.com).
|
|
111
|
+
It natively supports the [Open Data Contract Standard](https://bitol-io.github.io/open-data-contract-standard/latest/) to lint data contracts, connect to data sources and execute schema and quality tests, and export to different formats.
|
|
112
|
+
The tool is written in Python.
|
|
113
|
+
It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
|
|
112
114
|
|
|
113
115
|

|
|
114
116
|
|
|
@@ -116,11 +118,12 @@ It uses data contract YAML files as [Data Contract Specification](https://dataco
|
|
|
116
118
|
## Getting started
|
|
117
119
|
|
|
118
120
|
Let's look at this data contract:
|
|
119
|
-
[https://datacontract.com/
|
|
121
|
+
[https://datacontract.com/orders-v1.odcs.yaml](https://datacontract.com/orders-v1.odcs.yaml)
|
|
120
122
|
|
|
121
|
-
We have a _servers_ section with endpoint details to
|
|
123
|
+
We have a _servers_ section with endpoint details to a Postgres database, _schema_ for the structure and semantics of the data, _service levels_ and _quality_ attributes that describe the expected freshness and number of rows.
|
|
122
124
|
|
|
123
|
-
This data contract contains all information to connect to
|
|
125
|
+
This data contract contains all information to connect to the database and check that the actual data meets the defined schema specification and quality expectations.
|
|
126
|
+
We can use this information to test if the actual data product is compliant to the data contract.
|
|
124
127
|
|
|
125
128
|
Let's use [uv](https://docs.astral.sh/uv/) to install the CLI (or use the [Docker image](#docker)),
|
|
126
129
|
```bash
|
|
@@ -128,112 +131,112 @@ $ uv tool install --python python3.11 --upgrade 'datacontract-cli[all]'
|
|
|
128
131
|
```
|
|
129
132
|
|
|
130
133
|
|
|
131
|
-
|
|
134
|
+
Now, let's run the tests:
|
|
132
135
|
|
|
133
136
|
```bash
|
|
134
|
-
$
|
|
137
|
+
$ export DATACONTRACT_POSTGRES_USERNAME=datacontract_cli.egzhawjonpfweuutedfy
|
|
138
|
+
$ export DATACONTRACT_POSTGRES_PASSWORD=jio10JuQfDfl9JCCPdaCCpuZ1YO
|
|
139
|
+
$ datacontract test https://datacontract.com/orders-v1.odcs.yaml
|
|
135
140
|
|
|
136
141
|
# returns:
|
|
137
|
-
Testing https://datacontract.com/
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
│ passed │ Check that
|
|
143
|
-
│ passed │ Check that field
|
|
144
|
-
│ passed │ Check that field
|
|
145
|
-
│ passed │ Check that field
|
|
146
|
-
│ passed │ Check that field
|
|
147
|
-
│ passed │ Check that field
|
|
148
|
-
│ passed │
|
|
149
|
-
│ passed │ Check that
|
|
150
|
-
│ passed │ Check that
|
|
151
|
-
│ passed │
|
|
152
|
-
│ passed │ Check that
|
|
153
|
-
│ passed │
|
|
154
|
-
│ passed │ Check that
|
|
155
|
-
│ passed │ Check that
|
|
156
|
-
│ passed │ Check that field
|
|
157
|
-
│ passed │ Check that field order_id
|
|
158
|
-
│ passed │ Check that field
|
|
159
|
-
│ passed │
|
|
160
|
-
│ passed │
|
|
161
|
-
│ passed │ Check that
|
|
162
|
-
│ passed │ Check that
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
142
|
+
Testing https://datacontract.com/orders-v1.odcs.yaml
|
|
143
|
+
Server: production (type=postgres, host=aws-1-eu-central-2.pooler.supabase.com, port=6543, database=postgres, schema=dp_orders_v1)
|
|
144
|
+
╭────────┬──────────────────────────────────────────────────────────┬─────────────────────────┬─────────╮
|
|
145
|
+
│ Result │ Check │ Field │ Details │
|
|
146
|
+
├────────┼──────────────────────────────────────────────────────────┼─────────────────────────┼─────────┤
|
|
147
|
+
│ passed │ Check that field 'line_item_id' is present │ line_items.line_item_id │ │
|
|
148
|
+
│ passed │ Check that field line_item_id has type UUID │ line_items.line_item_id │ │
|
|
149
|
+
│ passed │ Check that field line_item_id has no missing values │ line_items.line_item_id │ │
|
|
150
|
+
│ passed │ Check that field 'order_id' is present │ line_items.order_id │ │
|
|
151
|
+
│ passed │ Check that field order_id has type UUID │ line_items.order_id │ │
|
|
152
|
+
│ passed │ Check that field 'price' is present │ line_items.price │ │
|
|
153
|
+
│ passed │ Check that field price has type INTEGER │ line_items.price │ │
|
|
154
|
+
│ passed │ Check that field price has no missing values │ line_items.price │ │
|
|
155
|
+
│ passed │ Check that field 'sku' is present │ line_items.sku │ │
|
|
156
|
+
│ passed │ Check that field sku has type TEXT │ line_items.sku │ │
|
|
157
|
+
│ passed │ Check that field sku has no missing values │ line_items.sku │ │
|
|
158
|
+
│ passed │ Check that field 'customer_id' is present │ orders.customer_id │ │
|
|
159
|
+
│ passed │ Check that field customer_id has type TEXT │ orders.customer_id │ │
|
|
160
|
+
│ passed │ Check that field customer_id has no missing values │ orders.customer_id │ │
|
|
161
|
+
│ passed │ Check that field 'order_id' is present │ orders.order_id │ │
|
|
162
|
+
│ passed │ Check that field order_id has type UUID │ orders.order_id │ │
|
|
163
|
+
│ passed │ Check that field order_id has no missing values │ orders.order_id │ │
|
|
164
|
+
│ passed │ Check that unique field order_id has no duplicate values │ orders.order_id │ │
|
|
165
|
+
│ passed │ Check that field 'order_status' is present │ orders.order_status │ │
|
|
166
|
+
│ passed │ Check that field order_status has type TEXT │ orders.order_status │ │
|
|
167
|
+
│ passed │ Check that field 'order_timestamp' is present │ orders.order_timestamp │ │
|
|
168
|
+
│ passed │ Check that field order_timestamp has type TIMESTAMPTZ │ orders.order_timestamp │ │
|
|
169
|
+
│ passed │ Check that field 'order_total' is present │ orders.order_total │ │
|
|
170
|
+
│ passed │ Check that field order_total has type INTEGER │ orders.order_total │ │
|
|
171
|
+
│ passed │ Check that field order_total has no missing values │ orders.order_total │ │
|
|
172
|
+
╰────────┴──────────────────────────────────────────────────────────┴─────────────────────────┴─────────╯
|
|
173
|
+
🟢 data contract is valid. Run 25 checks. Took 3.938887 seconds.
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Voilà, the CLI tested that the YAML itself is valid, all records comply with the schema, and all quality attributes are met.
|
|
177
|
+
|
|
178
|
+
We can also use the data contract metadata to export in many [formats](#format), e.g., to generate a SQL DDL:
|
|
170
179
|
|
|
171
180
|
```bash
|
|
172
|
-
$ datacontract export --format sql https://datacontract.com/
|
|
181
|
+
$ datacontract export --format sql https://datacontract.com/orders-v1.odcs.yaml
|
|
173
182
|
|
|
174
183
|
# returns:
|
|
175
|
-
-- Data Contract:
|
|
176
|
-
-- SQL Dialect:
|
|
184
|
+
-- Data Contract: orders
|
|
185
|
+
-- SQL Dialect: postgres
|
|
177
186
|
CREATE TABLE orders (
|
|
178
|
-
order_id
|
|
179
|
-
|
|
180
|
-
order_total
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
processed_timestamp TIMESTAMP_TZ not null
|
|
187
|
+
order_id None not null primary key,
|
|
188
|
+
customer_id text not null,
|
|
189
|
+
order_total integer not null,
|
|
190
|
+
order_timestamp None,
|
|
191
|
+
order_status text
|
|
184
192
|
);
|
|
185
193
|
CREATE TABLE line_items (
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
194
|
+
line_item_id None not null primary key,
|
|
195
|
+
sku text not null,
|
|
196
|
+
price integer not null,
|
|
197
|
+
order_id None
|
|
189
198
|
);
|
|
190
199
|
```
|
|
191
200
|
|
|
192
201
|
Or generate an HTML export:
|
|
193
202
|
|
|
194
203
|
```bash
|
|
195
|
-
$ datacontract export --format html https://datacontract.com/
|
|
204
|
+
$ datacontract export --format html --output orders-v1.odcs.html https://datacontract.com/orders-v1.odcs.yaml
|
|
196
205
|
```
|
|
197
206
|
|
|
198
|
-
which will create this [HTML export]
|
|
207
|
+
[//]: # (which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).)
|
|
199
208
|
|
|
200
209
|
|
|
201
210
|
## Usage
|
|
202
211
|
|
|
203
212
|
```bash
|
|
204
|
-
# create a new data contract from example and write it to
|
|
205
|
-
$ datacontract init
|
|
213
|
+
# create a new data contract from example and write it to odcs.yaml
|
|
214
|
+
$ datacontract init odcs.yaml
|
|
206
215
|
|
|
207
|
-
# lint the
|
|
208
|
-
$ datacontract lint
|
|
216
|
+
# lint the odcs.yaml
|
|
217
|
+
$ datacontract lint odcs.yaml
|
|
209
218
|
|
|
210
219
|
# execute schema and quality checks (define credentials as environment variables)
|
|
211
|
-
$ datacontract test
|
|
220
|
+
$ datacontract test odcs.yaml
|
|
212
221
|
|
|
213
222
|
# export data contract as html (other formats: avro, dbt, dbt-sources, dbt-staging-sql, jsonschema, odcs, rdf, sql, sodacl, terraform, ...)
|
|
214
|
-
$ datacontract export --format html datacontract.yaml --output
|
|
215
|
-
|
|
216
|
-
# export data contract to ODCS
|
|
217
|
-
$ datacontract export --format odcs datacontract.yaml --output odcs.yaml
|
|
218
|
-
|
|
219
|
-
# import ODCS to data contract
|
|
220
|
-
$ datacontract import --format odcs odcs.yaml --output datacontract.yaml
|
|
223
|
+
$ datacontract export --format html datacontract.yaml --output odcs.html
|
|
221
224
|
|
|
222
225
|
# import sql (other formats: avro, glue, bigquery, jsonschema, excel ...)
|
|
223
|
-
$ datacontract import --format sql --source my-ddl.sql --dialect postgres --output
|
|
226
|
+
$ datacontract import --format sql --source my-ddl.sql --dialect postgres --output odcs.yaml
|
|
224
227
|
|
|
225
228
|
# import from Excel template
|
|
226
|
-
$ datacontract import --format excel --source odcs.xlsx --output
|
|
229
|
+
$ datacontract import --format excel --source odcs.xlsx --output odcs.yaml
|
|
227
230
|
|
|
228
231
|
# export to Excel template
|
|
229
|
-
$ datacontract export --format excel --output odcs.xlsx
|
|
232
|
+
$ datacontract export --format excel --output odcs.xlsx odcs.yaml
|
|
230
233
|
```
|
|
231
234
|
|
|
232
235
|
## Programmatic (Python)
|
|
233
236
|
```python
|
|
234
237
|
from datacontract.data_contract import DataContract
|
|
235
238
|
|
|
236
|
-
data_contract = DataContract(data_contract_file="
|
|
239
|
+
data_contract = DataContract(data_contract_file="odcs.yaml")
|
|
237
240
|
run = data_contract.test()
|
|
238
241
|
if not run.has_passed():
|
|
239
242
|
print("Data quality validation failed.")
|
|
@@ -253,6 +256,14 @@ Choose the most appropriate installation method for your needs:
|
|
|
253
256
|
|
|
254
257
|
### uv
|
|
255
258
|
|
|
259
|
+
The preferred way to install is [uv](https://docs.astral.sh/uv/):
|
|
260
|
+
|
|
261
|
+
```
|
|
262
|
+
uv tool install --python python3.11 --upgrade 'datacontract-cli[all]'
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
### uvx
|
|
266
|
+
|
|
256
267
|
If you have [uv](https://docs.astral.sh/uv/) installed, you can run datacontract-cli directly without installing:
|
|
257
268
|
|
|
258
269
|
```
|
|
@@ -260,7 +271,7 @@ uv run --with 'datacontract-cli[all]' datacontract --version
|
|
|
260
271
|
```
|
|
261
272
|
|
|
262
273
|
### pip
|
|
263
|
-
Python 3.10, 3.11, and 3.12 are supported. We recommend
|
|
274
|
+
Python 3.10, 3.11, and 3.12 are supported. We recommend using Python 3.11.
|
|
264
275
|
|
|
265
276
|
```bash
|
|
266
277
|
python3 -m pip install 'datacontract-cli[all]'
|
|
@@ -313,7 +324,7 @@ The CLI tool defines several optional dependencies (also known as extras) that c
|
|
|
313
324
|
With _all_, all server dependencies are included.
|
|
314
325
|
|
|
315
326
|
```bash
|
|
316
|
-
|
|
327
|
+
uv tool install --python python3.11 --upgrade 'datacontract-cli[all]'
|
|
317
328
|
```
|
|
318
329
|
|
|
319
330
|
A list of available extras:
|
|
@@ -331,6 +342,7 @@ A list of available extras:
|
|
|
331
342
|
| Snowflake Integration | `pip install datacontract-cli[snowflake]` |
|
|
332
343
|
| Microsoft SQL Server | `pip install datacontract-cli[sqlserver]` |
|
|
333
344
|
| Trino | `pip install datacontract-cli[trino]` |
|
|
345
|
+
| Impala | `pip install datacontract-cli[impala]` |
|
|
334
346
|
| dbt | `pip install datacontract-cli[dbt]` |
|
|
335
347
|
| DBML | `pip install datacontract-cli[dbml]` |
|
|
336
348
|
| Parquet | `pip install datacontract-cli[parquet]` |
|
|
@@ -348,9 +360,6 @@ Commands
|
|
|
348
360
|
- [test](#test)
|
|
349
361
|
- [export](#export)
|
|
350
362
|
- [import](#import)
|
|
351
|
-
- [breaking](#breaking) (deprecated)
|
|
352
|
-
- [changelog](#changelog) (deprecated)
|
|
353
|
-
- [diff](#diff) (deprecated)
|
|
354
363
|
- [catalog](#catalog)
|
|
355
364
|
- [publish](#publish)
|
|
356
365
|
- [api](#api)
|
|
@@ -390,8 +399,7 @@ Commands
|
|
|
390
399
|
│ [default: datacontract.yaml] │
|
|
391
400
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
392
401
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
393
|
-
│ --schema TEXT The location (url or path) of the
|
|
394
|
-
│ Specification JSON Schema │
|
|
402
|
+
│ --schema TEXT The location (url or path) of the ODCS JSON Schema │
|
|
395
403
|
│ [default: None] │
|
|
396
404
|
│ --output PATH Specify the file path where the test results should be │
|
|
397
405
|
│ written to (e.g., │
|
|
@@ -420,8 +428,7 @@ Commands
|
|
|
420
428
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
421
429
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
422
430
|
│ --schema TEXT The location (url or path) of │
|
|
423
|
-
│ the
|
|
424
|
-
│ JSON Schema │
|
|
431
|
+
│ the ODCS JSON Schema │
|
|
425
432
|
│ [default: None] │
|
|
426
433
|
│ --server TEXT The server configuration to run │
|
|
427
434
|
│ the schema and quality tests. │
|
|
@@ -487,6 +494,7 @@ Supported server types:
|
|
|
487
494
|
- [kafka](#kafka)
|
|
488
495
|
- [postgres](#postgres)
|
|
489
496
|
- [trino](#trino)
|
|
497
|
+
- [impala](#impala)
|
|
490
498
|
- [api](#api)
|
|
491
499
|
- [local](#local)
|
|
492
500
|
|
|
@@ -564,7 +572,7 @@ servers:
|
|
|
564
572
|
regionName: eu-central-1
|
|
565
573
|
stagingDir: s3://my-bucket/athena-results/
|
|
566
574
|
models:
|
|
567
|
-
my_table: # corresponds to a table
|
|
575
|
+
my_table: # corresponds to a table or view name
|
|
568
576
|
type: table
|
|
569
577
|
fields:
|
|
570
578
|
my_column_1: # corresponds to a column
|
|
@@ -809,7 +817,7 @@ models:
|
|
|
809
817
|
|
|
810
818
|
##### Installing on Databricks Compute
|
|
811
819
|
|
|
812
|
-
**Important:** When using Databricks LTS ML runtimes (15.4, 16.4), installing via `%pip install` in notebooks can issues.
|
|
820
|
+
**Important:** When using Databricks LTS ML runtimes (15.4, 16.4), installing via `%pip install` in notebooks can cause issues.
|
|
813
821
|
|
|
814
822
|
**Recommended approach:** Use Databricks' native library management instead:
|
|
815
823
|
|
|
@@ -1017,6 +1025,53 @@ models:
|
|
|
1017
1025
|
| `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password |
|
|
1018
1026
|
|
|
1019
1027
|
|
|
1028
|
+
#### Impala
|
|
1029
|
+
|
|
1030
|
+
Data Contract CLI can run Soda checks against an Apache Impala cluster.
|
|
1031
|
+
|
|
1032
|
+
##### Example
|
|
1033
|
+
|
|
1034
|
+
datacontract.yaml
|
|
1035
|
+
```yaml
|
|
1036
|
+
servers:
|
|
1037
|
+
impala:
|
|
1038
|
+
type: impala
|
|
1039
|
+
host: my-impala-host
|
|
1040
|
+
port: 443
|
|
1041
|
+
# Optional default database used for Soda scans
|
|
1042
|
+
database: my_database
|
|
1043
|
+
models:
|
|
1044
|
+
my_table_1: # corresponds to a table
|
|
1045
|
+
type: table
|
|
1046
|
+
# fields as usual …
|
|
1047
|
+
```
|
|
1048
|
+
|
|
1049
|
+
##### Environment Variables
|
|
1050
|
+
|
|
1051
|
+
| Environment Variable | Example | Description |
|
|
1052
|
+
|------------------------------- |-------------------- |------------- |
|
|
1053
|
+
| `DATACONTRACT_IMPALA_USERNAME` | `analytics_user` | Username used to connect to Impala |
|
|
1054
|
+
| `DATACONTRACT_IMPALA_PASSWORD` | `mysecretpassword` | Password for the Impala user |
|
|
1055
|
+
| `DATACONTRACT_IMPALA_USE_SSL` | `true` | Whether to use SSL; defaults to true if unset |
|
|
1056
|
+
| `DATACONTRACT_IMPALA_AUTH_MECHANISM` | `LDAP` | Authentication mechanism; defaults to LDAP |
|
|
1057
|
+
| `DATACONTRACT_IMPALA_USE_HTTP_TRANSPORT` | `true` | Whether to use the HTTP transport; defaults to true |
|
|
1058
|
+
| `DATACONTRACT_IMPALA_HTTP_PATH` | `cliservice` | HTTP path for the Impala service; defaults to cliservice |
|
|
1059
|
+
|
|
1060
|
+
### Type-mapping note (logicalType → Impala type)
|
|
1061
|
+
|
|
1062
|
+
If `physicalType` is not specified in the schema, we recommend the following mapping from `logicalType` to Impala column types:
|
|
1063
|
+
|
|
1064
|
+
|logicalType | Recommended Impala type |
|
|
1065
|
+
|------------|-------------------------|
|
|
1066
|
+
| `integer` | `INT` or `BIGINT` |
|
|
1067
|
+
| `number` | `DOUBLE`/`decimal(..)` |
|
|
1068
|
+
| `string` | `STRING` or `VARCHAR` |
|
|
1069
|
+
| `boolean` | `BOOLEAN` |
|
|
1070
|
+
| `date` | `DATE` |
|
|
1071
|
+
| `datetime` | `TIMESTAMP` |
|
|
1072
|
+
|
|
1073
|
+
This keeps the Impala schema compatible with the expectations of the Soda checks generated by datacontract-cli.
|
|
1074
|
+
|
|
1020
1075
|
#### API
|
|
1021
1076
|
|
|
1022
1077
|
Data Contract CLI can test APIs that return data in JSON format.
|
|
@@ -1087,42 +1142,42 @@ models:
|
|
|
1087
1142
|
│ [default: datacontract.yaml] │
|
|
1088
1143
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1089
1144
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1090
|
-
│ * --format
|
|
1091
|
-
│
|
|
1092
|
-
│
|
|
1093
|
-
│
|
|
1094
|
-
│
|
|
1095
|
-
│
|
|
1096
|
-
│
|
|
1097
|
-
│
|
|
1098
|
-
│ --output
|
|
1099
|
-
│
|
|
1100
|
-
│
|
|
1101
|
-
│
|
|
1102
|
-
│
|
|
1103
|
-
│
|
|
1104
|
-
│
|
|
1105
|
-
│
|
|
1106
|
-
│
|
|
1107
|
-
│
|
|
1108
|
-
│
|
|
1109
|
-
│
|
|
1110
|
-
│
|
|
1111
|
-
│
|
|
1112
|
-
│
|
|
1113
|
-
│
|
|
1114
|
-
│ --engine
|
|
1115
|
-
│
|
|
1116
|
-
│
|
|
1117
|
-
│ --template
|
|
1118
|
-
│
|
|
1119
|
-
│
|
|
1120
|
-
│
|
|
1121
|
-
│
|
|
1122
|
-
│
|
|
1123
|
-
│ --debug
|
|
1124
|
-
│
|
|
1125
|
-
│ --help
|
|
1145
|
+
│ * --format [jsonschema|pydantic-model|sod The export format. │
|
|
1146
|
+
│ acl|dbt|dbt-sources|dbt-stagin [default: None] │
|
|
1147
|
+
│ g-sql|odcs|rdf|avro|protobuf|g [required] │
|
|
1148
|
+
│ reat-expectations|avro-idl|sql │
|
|
1149
|
+
│ |sql-query|mermaid|html|go|big │
|
|
1150
|
+
│ query|dbml|spark|sqlalchemy|da │
|
|
1151
|
+
│ ta-caterer|dcs|markdown|iceber │
|
|
1152
|
+
│ g|custom|excel|dqx] │
|
|
1153
|
+
│ --output PATH Specify the file path where │
|
|
1154
|
+
│ the exported data will be │
|
|
1155
|
+
│ saved. If no path is provided, │
|
|
1156
|
+
│ the output will be printed to │
|
|
1157
|
+
│ stdout. │
|
|
1158
|
+
│ [default: None] │
|
|
1159
|
+
│ --server TEXT The server name to export. │
|
|
1160
|
+
│ [default: None] │
|
|
1161
|
+
│ --schema-name TEXT The name of the schema to │
|
|
1162
|
+
│ export, e.g., `orders`, or │
|
|
1163
|
+
│ `all` for all schemas │
|
|
1164
|
+
│ (default). │
|
|
1165
|
+
│ [default: all] │
|
|
1166
|
+
│ --schema TEXT The location (url or path) of │
|
|
1167
|
+
│ the ODCS JSON Schema │
|
|
1168
|
+
│ [default: None] │
|
|
1169
|
+
│ --engine TEXT [engine] The engine used for │
|
|
1170
|
+
│ great expection run. │
|
|
1171
|
+
│ [default: None] │
|
|
1172
|
+
│ --template PATH The file path or URL of a │
|
|
1173
|
+
│ template. For Excel format: │
|
|
1174
|
+
│ path/URL to custom Excel │
|
|
1175
|
+
│ template. For custom format: │
|
|
1176
|
+
│ path to Jinja template. │
|
|
1177
|
+
│ [default: None] │
|
|
1178
|
+
│ --debug --no-debug Enable debug logging │
|
|
1179
|
+
│ [default: no-debug] │
|
|
1180
|
+
│ --help Show this message and exit. │
|
|
1126
1181
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1127
1182
|
╭─ RDF Options ────────────────────────────────────────────────────────────────────────────────────╮
|
|
1128
1183
|
│ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. [default: None] │
|
|
@@ -1191,7 +1246,7 @@ spark.conf.set(“spark.databricks.delta.schema.typeCheck.enabled”, “false
|
|
|
1191
1246
|
#### Great Expectations
|
|
1192
1247
|
|
|
1193
1248
|
The `export` function transforms a specified data contract into a comprehensive Great Expectations JSON suite.
|
|
1194
|
-
If the contract includes multiple models, you need to specify the names of the
|
|
1249
|
+
If the contract includes multiple models, you need to specify the names of the schema/models you wish to export.
|
|
1195
1250
|
|
|
1196
1251
|
```shell
|
|
1197
1252
|
datacontract export datacontract.yaml --format great-expectations --model orders
|
|
@@ -1249,7 +1304,7 @@ logical data types are exported.
|
|
|
1249
1304
|
|
|
1250
1305
|
#### DBT & DBT-SOURCES
|
|
1251
1306
|
|
|
1252
|
-
The export
|
|
1307
|
+
The export function converts the datacontract to dbt models in YAML format, with support for SQL dialects.
|
|
1253
1308
|
If a server is selected via the `--server` option (based on the `type` of that server) then the DBT column `data_types` match the expected data types of the server.
|
|
1254
1309
|
If no server is selected, then it defaults to `snowflake`.
|
|
1255
1310
|
|
|
@@ -1442,7 +1497,7 @@ FROM
|
|
|
1442
1497
|
{{ ref('orders') }}
|
|
1443
1498
|
```
|
|
1444
1499
|
|
|
1445
|
-
#### ODCS Excel
|
|
1500
|
+
#### ODCS Excel Template
|
|
1446
1501
|
|
|
1447
1502
|
The `export` function converts a data contract into an ODCS (Open Data Contract Standard) Excel template. This creates a user-friendly Excel spreadsheet that can be used for authoring, sharing, and managing data contracts using the familiar Excel interface.
|
|
1448
1503
|
|
|
@@ -1482,10 +1537,6 @@ For more information about the Excel template structure, visit the [ODCS Excel T
|
|
|
1482
1537
|
│ --source TEXT The path to the file that │
|
|
1483
1538
|
│ should be imported. │
|
|
1484
1539
|
│ [default: None] │
|
|
1485
|
-
│ --spec [datacontract_specificat The format of the data │
|
|
1486
|
-
│ ion|odcs] contract to import. │
|
|
1487
|
-
│ [default: │
|
|
1488
|
-
│ datacontract_specificatio… │
|
|
1489
1540
|
│ --dialect TEXT The SQL dialect to use │
|
|
1490
1541
|
│ when importing SQL files, │
|
|
1491
1542
|
│ e.g., postgres, tsql, │
|
|
@@ -1537,12 +1588,10 @@ For more information about the Excel template structure, visit the [ODCS Excel T
|
|
|
1537
1588
|
│ Iceberg schema. │
|
|
1538
1589
|
│ [default: None] │
|
|
1539
1590
|
│ --template TEXT The location (url or path) │
|
|
1540
|
-
│ of the
|
|
1541
|
-
│ Specification Template │
|
|
1591
|
+
│ of the ODCS template │
|
|
1542
1592
|
│ [default: None] │
|
|
1543
1593
|
│ --schema TEXT The location (url or path) │
|
|
1544
|
-
│ of the
|
|
1545
|
-
│ Specification JSON Schema │
|
|
1594
|
+
│ of the ODCS JSON Schema │
|
|
1546
1595
|
│ [default: None] │
|
|
1547
1596
|
│ --owner TEXT The owner or team │
|
|
1548
1597
|
│ responsible for managing │
|
|
@@ -1569,38 +1618,25 @@ datacontract import --format sql --source my_ddl.sql --dialect postgres --output
|
|
|
1569
1618
|
|
|
1570
1619
|
Available import options:
|
|
1571
1620
|
|
|
1572
|
-
| Type | Description
|
|
1573
|
-
|
|
1574
|
-
| `avro` | Import from AVRO schemas
|
|
1575
|
-
| `bigquery` | Import from BigQuery Schemas
|
|
1576
|
-
| `csv` | Import from CSV File
|
|
1577
|
-
| `dbml` | Import from DBML models
|
|
1578
|
-
| `dbt` | Import from dbt models
|
|
1579
|
-
| `excel` | Import from ODCS Excel Template
|
|
1580
|
-
| `glue` | Import from AWS Glue DataCatalog
|
|
1581
|
-
| `iceberg` | Import from an Iceberg JSON Schema Definition
|
|
1582
|
-
| `jsonschema` | Import from JSON Schemas
|
|
1583
|
-
| `
|
|
1584
|
-
| `
|
|
1585
|
-
| `
|
|
1586
|
-
| `
|
|
1587
|
-
| `
|
|
1588
|
-
| `
|
|
1589
|
-
|
|
|
1590
|
-
| Missing something? | Please create an issue on GitHub | TBD |
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
#### ODCS
|
|
1594
|
-
|
|
1595
|
-
Import from Open Data Contract Standard (ODCS) v2 or v3.
|
|
1596
|
-
The importer automatically detects the ODCS version and imports the data contract.
|
|
1621
|
+
| Type | Description | Status |
|
|
1622
|
+
|--------------------|-----------------------------------------------|---------|
|
|
1623
|
+
| `avro` | Import from AVRO schemas | ✅ |
|
|
1624
|
+
| `bigquery` | Import from BigQuery Schemas | ✅ |
|
|
1625
|
+
| `csv` | Import from CSV File | ✅ |
|
|
1626
|
+
| `dbml` | Import from DBML models | ✅ |
|
|
1627
|
+
| `dbt` | Import from dbt models | ✅ |
|
|
1628
|
+
| `excel` | Import from ODCS Excel Template | ✅ |
|
|
1629
|
+
| `glue` | Import from AWS Glue DataCatalog | ✅ |
|
|
1630
|
+
| `iceberg` | Import from an Iceberg JSON Schema Definition | partial |
|
|
1631
|
+
| `jsonschema` | Import from JSON Schemas | ✅ |
|
|
1632
|
+
| `parquet` | Import from Parquet File Metadata | ✅ |
|
|
1633
|
+
| `protobuf` | Import from Protobuf schemas | ✅ |
|
|
1634
|
+
| `spark` | Import from Spark StructTypes, Variant | ✅ |
|
|
1635
|
+
| `sql` | Import from SQL DDL | ✅ |
|
|
1636
|
+
| `unity` | Import from Databricks Unity Catalog | partial |
|
|
1637
|
+
| `excel` | Import from ODCS Excel Template | ✅ |
|
|
1638
|
+
| Missing something? | Please create an issue on GitHub | TBD |
|
|
1597
1639
|
|
|
1598
|
-
Examples:
|
|
1599
|
-
|
|
1600
|
-
```bash
|
|
1601
|
-
# Example import from ODCS
|
|
1602
|
-
datacontract import --format odcs --source my_data_contract.odcs.yaml
|
|
1603
|
-
```
|
|
1604
1640
|
|
|
1605
1641
|
#### BigQuery
|
|
1606
1642
|
|
|
@@ -1639,7 +1675,7 @@ export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com
|
|
|
1639
1675
|
export DATACONTRACT_DATABRICKS_TOKEN=<token>
|
|
1640
1676
|
datacontract import --format unity --unity-table-full-name <table_full_name>
|
|
1641
1677
|
```
|
|
1642
|
-
Please
|
|
1678
|
+
Please refer to [Databricks documentation](https://docs.databricks.com/aws/en/dev-tools/auth/unified-auth) on how to set up a profile
|
|
1643
1679
|
```bash
|
|
1644
1680
|
# Example import single table from Unity Catalog via HTTP endpoint using Profile
|
|
1645
1681
|
export DATACONTRACT_DATABRICKS_PROFILE="my-profile"
|
|
@@ -1663,7 +1699,7 @@ datacontract import --format dbt --source <manifest_path> --dbt-model <model_nam
|
|
|
1663
1699
|
datacontract import --format dbt --source <manifest_path>
|
|
1664
1700
|
```
|
|
1665
1701
|
|
|
1666
|
-
|
|
1702
|
+
#### Excel
|
|
1667
1703
|
|
|
1668
1704
|
Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
|
|
1669
1705
|
|
|
@@ -1780,78 +1816,6 @@ datacontract import --format protobuf --source "test.proto"
|
|
|
1780
1816
|
```
|
|
1781
1817
|
|
|
1782
1818
|
|
|
1783
|
-
### breaking
|
|
1784
|
-
|
|
1785
|
-
> **Deprecated:** This command is deprecated and will be removed in a future version.
|
|
1786
|
-
|
|
1787
|
-
```
|
|
1788
|
-
|
|
1789
|
-
Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW
|
|
1790
|
-
|
|
1791
|
-
Identifies breaking changes between data contracts. Prints to stdout.
|
|
1792
|
-
|
|
1793
|
-
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
1794
|
-
│ * location_old TEXT The location (url or path) of the old data contract yaml. │
|
|
1795
|
-
│ [default: None] │
|
|
1796
|
-
│ [required] │
|
|
1797
|
-
│ * location_new TEXT The location (url or path) of the new data contract yaml. │
|
|
1798
|
-
│ [default: None] │
|
|
1799
|
-
│ [required] │
|
|
1800
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1801
|
-
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1802
|
-
│ --help Show this message and exit. │
|
|
1803
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1804
|
-
|
|
1805
|
-
```
|
|
1806
|
-
|
|
1807
|
-
### changelog
|
|
1808
|
-
|
|
1809
|
-
> **Deprecated:** This command is deprecated and will be removed in a future version.
|
|
1810
|
-
|
|
1811
|
-
```
|
|
1812
|
-
|
|
1813
|
-
Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW
|
|
1814
|
-
|
|
1815
|
-
Generate a changelog between data contracts. Prints to stdout.
|
|
1816
|
-
|
|
1817
|
-
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
1818
|
-
│ * location_old TEXT The location (url or path) of the old data contract yaml. │
|
|
1819
|
-
│ [default: None] │
|
|
1820
|
-
│ [required] │
|
|
1821
|
-
│ * location_new TEXT The location (url or path) of the new data contract yaml. │
|
|
1822
|
-
│ [default: None] │
|
|
1823
|
-
│ [required] │
|
|
1824
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1825
|
-
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1826
|
-
│ --help Show this message and exit. │
|
|
1827
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1828
|
-
|
|
1829
|
-
```
|
|
1830
|
-
|
|
1831
|
-
### diff
|
|
1832
|
-
|
|
1833
|
-
> **Deprecated:** This command is deprecated and will be removed in a future version.
|
|
1834
|
-
|
|
1835
|
-
```
|
|
1836
|
-
|
|
1837
|
-
Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW
|
|
1838
|
-
|
|
1839
|
-
Generate a diff between data contracts. Prints to stdout.
|
|
1840
|
-
|
|
1841
|
-
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
1842
|
-
│ * location_old TEXT The location (url or path) of the old data contract yaml. │
|
|
1843
|
-
│ [default: None] │
|
|
1844
|
-
│ [required] │
|
|
1845
|
-
│ * location_new TEXT The location (url or path) of the new data contract yaml. │
|
|
1846
|
-
│ [default: None] │
|
|
1847
|
-
│ [required] │
|
|
1848
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1849
|
-
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1850
|
-
│ --help Show this message and exit. │
|
|
1851
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1852
|
-
|
|
1853
|
-
```
|
|
1854
|
-
|
|
1855
1819
|
### catalog
|
|
1856
1820
|
```
|
|
1857
1821
|
|
|
@@ -1865,8 +1829,7 @@ datacontract import --format protobuf --source "test.proto"
|
|
|
1865
1829
|
│ catalog. Applies recursively to any subfolders. │
|
|
1866
1830
|
│ [default: *.yaml] │
|
|
1867
1831
|
│ --output TEXT Output directory for the catalog html files. [default: catalog/] │
|
|
1868
|
-
│ --schema TEXT The location (url or path) of the
|
|
1869
|
-
│ JSON Schema │
|
|
1832
|
+
│ --schema TEXT The location (url or path) of the ODCS JSON Schema │
|
|
1870
1833
|
│ [default: None] │
|
|
1871
1834
|
│ --debug --no-debug Enable debug logging [default: no-debug] │
|
|
1872
1835
|
│ --help Show this message and exit. │
|
|
@@ -1889,7 +1852,7 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
1889
1852
|
|
|
1890
1853
|
Usage: datacontract publish [OPTIONS] [LOCATION]
|
|
1891
1854
|
|
|
1892
|
-
Publish the data contract to the Data
|
|
1855
|
+
Publish the data contract to the Entropy Data.
|
|
1893
1856
|
|
|
1894
1857
|
|
|
1895
1858
|
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
@@ -1897,8 +1860,8 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
1897
1860
|
│ [default: datacontract.yaml] │
|
|
1898
1861
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1899
1862
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1900
|
-
│ --schema TEXT The location (url or path) of the
|
|
1901
|
-
│
|
|
1863
|
+
│ --schema TEXT The location (url or path) of the ODCS JSON │
|
|
1864
|
+
│ Schema │
|
|
1902
1865
|
│ [default: None] │
|
|
1903
1866
|
│ --ssl-verification --no-ssl-verification SSL verification when publishing the data │
|
|
1904
1867
|
│ contract. │
|
|
@@ -1942,19 +1905,18 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
1942
1905
|
|
|
1943
1906
|
| Integration | Option | Description |
|
|
1944
1907
|
|-----------------------|------------------------------|---------------------------------------------------------------------------------------------------------------|
|
|
1945
|
-
| Data
|
|
1946
|
-
| Data Contract Manager | `--publish` | Push full results to the [Data Contract Manager API](https://api.datacontract-manager.com/swagger/index.html) |
|
|
1908
|
+
| Entropy Data | `--publish` | Push full results to the [Entropy Data API](https://api.entropy-data.com/swagger/index.html) |
|
|
1947
1909
|
|
|
1948
|
-
### Integration with Data
|
|
1910
|
+
### Integration with Entropy Data
|
|
1949
1911
|
|
|
1950
|
-
If you use [
|
|
1912
|
+
If you use [Entropy Data](https://entropy-data.com/), you can use the data contract URL to reference to the contract and append the `--publish` option to send and display the test results. Set an environment variable for your API key.
|
|
1951
1913
|
|
|
1952
1914
|
```bash
|
|
1953
|
-
# Fetch current data contract, execute tests on production, and publish result to data
|
|
1954
|
-
$ EXPORT
|
|
1955
|
-
$ datacontract test https://demo.
|
|
1915
|
+
# Fetch current data contract, execute tests on production, and publish result to entropy data
|
|
1916
|
+
$ EXPORT ENTROPY_DATA_API_KEY=xxx
|
|
1917
|
+
$ datacontract test https://demo.entropy-data.com/demo279750347121/datacontracts/4df9d6ee-e55d-4088-9598-b635b2fdcbbc/datacontract.yaml \
|
|
1956
1918
|
--server production \
|
|
1957
|
-
--publish https://api.
|
|
1919
|
+
--publish https://api.entropy-data.com/api/test-results
|
|
1958
1920
|
```
|
|
1959
1921
|
|
|
1960
1922
|
## Best Practices
|
|
@@ -2018,36 +1980,6 @@ Create a data contract based on the requirements from use cases.
|
|
|
2018
1980
|
$ datacontract test
|
|
2019
1981
|
```
|
|
2020
1982
|
|
|
2021
|
-
### Schema Evolution
|
|
2022
|
-
|
|
2023
|
-
> **Note:** The `breaking` and `changelog` commands referenced below are deprecated and will be removed in a future version.
|
|
2024
|
-
|
|
2025
|
-
#### Non-breaking Changes
|
|
2026
|
-
Examples: adding models or fields
|
|
2027
|
-
|
|
2028
|
-
- Add the models or fields in the datacontract.yaml
|
|
2029
|
-
- Increment the minor version of the datacontract.yaml on any change. Simply edit the datacontract.yaml for this.
|
|
2030
|
-
- You need a policy that these changes are non-breaking. That means that one cannot use the star expression in SQL to query a table under contract. Make the consequences known.
|
|
2031
|
-
- Fail the build in the Pull Request if a datacontract.yaml accidentally adds a breaking change even despite only a minor version change
|
|
2032
|
-
```bash
|
|
2033
|
-
$ datacontract breaking datacontract-from-pr.yaml datacontract-from-main.yaml
|
|
2034
|
-
```
|
|
2035
|
-
- Create a changelog of this minor change.
|
|
2036
|
-
```bash
|
|
2037
|
-
$ datacontract changelog datacontract-from-pr.yaml datacontract-from-main.yaml
|
|
2038
|
-
```
|
|
2039
|
-
#### Breaking Changes
|
|
2040
|
-
Examples: Removing or renaming models and fields.
|
|
2041
|
-
|
|
2042
|
-
- Remove or rename models and fields in the datacontract.yaml, and any other change that might be part of this new major version of this data contract.
|
|
2043
|
-
- Increment the major version of the datacontract.yaml for this and create a new file for the major version. The reason being, that one needs to offer an upgrade path for the data consumers from the old to the new major version.
|
|
2044
|
-
- As data consumers need to migrate, try to reduce the frequency of major versions by making multiple breaking changes together if possible.
|
|
2045
|
-
- Be aware of the notice period in the data contract as this is the minimum amount of time you have to offer both the old and the new version for a migration path.
|
|
2046
|
-
- Do not fear making breaking changes with data contracts. It's okay to do them in this controlled way. Really!
|
|
2047
|
-
- Create a changelog of this major change.
|
|
2048
|
-
```bash
|
|
2049
|
-
$ datacontract changelog datacontract-from-pr.yaml datacontract-from-main.yaml
|
|
2050
|
-
```
|
|
2051
1983
|
|
|
2052
1984
|
## Customizing Exporters and Importers
|
|
2053
1985
|
|
|
@@ -2214,7 +2146,7 @@ models:
|
|
|
2214
2146
|
## Development Setup
|
|
2215
2147
|
|
|
2216
2148
|
- Install [uv](https://docs.astral.sh/uv/)
|
|
2217
|
-
- Python base interpreter should be 3.11.x
|
|
2149
|
+
- Python base interpreter should be 3.11.x.
|
|
2218
2150
|
- Docker engine must be running to execute the tests.
|
|
2219
2151
|
|
|
2220
2152
|
```bash
|
|
@@ -2230,7 +2162,7 @@ uv run pytest
|
|
|
2230
2162
|
|
|
2231
2163
|
#### Windows: Some tests fail
|
|
2232
2164
|
|
|
2233
|
-
Run in wsl. (We need to fix the
|
|
2165
|
+
Run in wsl. (We need to fix the paths in the tests so that normal Windows will work, contributions are appreciated)
|
|
2234
2166
|
|
|
2235
2167
|
#### PyCharm does not pick up the `.venv`
|
|
2236
2168
|
|
|
@@ -2306,8 +2238,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
2306
2238
|
|
|
2307
2239
|
## Related Tools
|
|
2308
2240
|
|
|
2309
|
-
- [Data
|
|
2310
|
-
- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
|
|
2241
|
+
- [Entropy Data](https://www.entropy-data.com/) is a commercial tool to manage data contracts. It contains a web UI, access management, and data governance for a data product marketplace based on data contracts.
|
|
2311
2242
|
- [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
|
|
2312
2243
|
- [Data Contract Playground](https://data-catering.github.io/data-contract-playground/) allows you to validate and export your data contract to different formats within your browser.
|
|
2313
2244
|
|