datacontract-cli 0.11.8__tar.gz → 0.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datacontract_cli-0.11.8/datacontract_cli.egg-info → datacontract_cli-0.12.0}/PKG-INFO +387 -432
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/README.md +376 -421
- datacontract_cli-0.12.0/datacontract/cli.py +164 -0
- datacontract_cli-0.12.0/datacontract/command_api.py +66 -0
- datacontract_cli-0.12.0/datacontract/command_catalog.py +55 -0
- datacontract_cli-0.12.0/datacontract/command_changelog.py +21 -0
- datacontract_cli-0.12.0/datacontract/command_ci.py +115 -0
- datacontract_cli-0.12.0/datacontract/command_export.py +600 -0
- datacontract_cli-0.12.0/datacontract/command_import.py +425 -0
- datacontract_cli-0.12.0/datacontract/command_init.py +33 -0
- datacontract_cli-0.12.0/datacontract/command_lint.py +47 -0
- datacontract_cli-0.12.0/datacontract/command_publish.py +38 -0
- datacontract_cli-0.12.0/datacontract/command_test.py +111 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/data_contract.py +13 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/data_contract_checks.py +3 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/data_contract_test.py +24 -3
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +2 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/fastjsonschema/check_jsonschema.py +3 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/check_soda_execute.py +5 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/databricks.py +4 -6
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/duckdb_connection.py +9 -17
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/mysql.py +4 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/oracle.py +4 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/postgres.py +4 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/sqlserver.py +23 -17
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/trino.py +4 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/dbt_exporter.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/exporter.py +13 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/exporter_factory.py +2 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/great_expectations_exporter.py +1 -1
- datacontract_cli-0.12.0/datacontract/export/protobuf_exporter.py +356 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/odcs_helper.py +7 -5
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/sql_importer.py +51 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/model/exceptions.py +18 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0/datacontract_cli.egg-info}/PKG-INFO +387 -432
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract_cli.egg-info/SOURCES.txt +13 -0
- datacontract_cli-0.12.0/datacontract_cli.egg-info/entry_points.txt +2 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract_cli.egg-info/requires.txt +10 -10
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/pyproject.toml +13 -12
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_cli.py +10 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_data_contract_checks.py +92 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_avro.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_avro_idl.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_bigquery.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_complex_data_contract.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_custom.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_custom_model.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_data_caterer.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_dbml.py +2 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_dbt_models.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_dbt_sources.py +2 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_dbt_staging_sql.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_dqx.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_excel.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_go.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_great_expectations.py +6 -11
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_html.py +3 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_iceberg.py +3 -8
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_jsonschema.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_markdown.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_mermaid.py +3 -5
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_odcs_v3.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_protobuf.py +23 -3
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_rdf.py +2 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_spark.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_sql.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_sql_query.py +1 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_avro.py +5 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_bigquery.py +0 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_csv.py +0 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_dbml.py +2 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_dbt.py +2 -5
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_excel.py +0 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_glue.py +4 -6
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_iceberg.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_json.py +0 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_jsonschema.py +0 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_parquet.py +5 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_protobuf.py +0 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_spark.py +2 -4
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_sql_oracle.py +8 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_sql_postgres.py +9 -1
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_sql_snowflake.py +19 -9
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_sql_sqlserver.py +14 -6
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_unity_file.py +0 -2
- datacontract_cli-0.12.0/tests/test_require_env.py +28 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_roundtrip_jsonschema.py +1 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_sqlserver_soda_connection.py +26 -0
- datacontract_cli-0.12.0/tests/test_test_checks_filter.py +86 -0
- datacontract_cli-0.12.0/tests/test_test_server_not_found.py +25 -0
- datacontract_cli-0.11.8/datacontract/cli.py +0 -638
- datacontract_cli-0.11.8/datacontract/export/protobuf_exporter.py +0 -203
- datacontract_cli-0.11.8/datacontract_cli.egg-info/entry_points.txt +0 -2
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/LICENSE +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/MANIFEST.in +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/__init__.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/api.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/catalog/catalog.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/changelog/__init__.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/changelog/changelog.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/changelog/normalize.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/__init__.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/datacontract/check_that_datacontract_file_exists.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/fastjsonschema/s3/s3_read_files.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/__init__.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/athena.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/bigquery.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/impala.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/kafka.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/engines/soda/connections/snowflake.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/__init__.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/avro_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/avro_idl_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/bigquery_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/custom_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/data_caterer_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/dbml_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/dcs_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/dqx_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/duckdb_type_converter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/excel_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/go_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/html_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/iceberg_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/jsonschema_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/markdown_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/mermaid_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/odcs_export_helper.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/odcs_v3_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/pandas_type_converter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/pydantic_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/rdf_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/sodacl_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/spark_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/sql_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/sql_type_converter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/export/sqlalchemy_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/avro_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/bigquery_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/csv_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/dbml_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/dbt_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/dcs_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/excel_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/glue_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/iceberg_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/importer_factory.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/json_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/jsonschema_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/odcs_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/parquet_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/protobuf_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/spark_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/imports/unity_importer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/init/init_template.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/integration/entropy_data.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/lint/files.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/lint/resolve.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/lint/resources.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/lint/schema.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/lint/urls.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/model/changelog.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/model/odcs.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/model/run.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/__init__.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/ci_output.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/json_test_results.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/junit_test_results.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/output_format.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/test_results_writer.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/output/text_changelog_results.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/py.typed +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/datacontract-1.1.0.init.yaml +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/datacontract-1.1.0.schema.json +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/datacontract-1.2.0.init.yaml +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/datacontract-1.2.0.schema.json +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/datacontract-1.2.1.init.yaml +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/datacontract-1.2.1.schema.json +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/odcs-3.0.1.schema.json +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/odcs-3.0.2.schema.json +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/odcs-3.1.0.init.yaml +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/schemas/odcs-3.1.0.schema.json +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/datacontract.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/datacontract_odcs.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/index.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/datacontract_information.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/datacontract_servicelevels.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/datacontract_terms.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/definition.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/example.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/model_field.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/quality.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/partials/server.html +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract/templates/style/output.css +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract_cli.egg-info/dependency_links.txt +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/datacontract_cli.egg-info/top_level.txt +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/setup.cfg +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_api.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_bigquery_soda_connection.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_catalog.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_changelog.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_changelog_engine.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_changelog_normalize.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_changelog_output_text.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_ci_output.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_data_contract.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_data_contract_specification.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_description_linter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_download_datacontract_file.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_duckdb_json.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_custom_exporter.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_pydantic.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_sodacl.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_export_sqlalchemy.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_import_odcs_v3.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_integration_entropydata.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_lint.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_resolve.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_sql_type_converter_physicaltype.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_api.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_athena_iceberg.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_azure_remote.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_bigquery.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_databricks.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_dataframe.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_delta.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_gcs_csv_remote.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_gcs_json_remote.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_kafka.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_kafka_remote.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_local_json.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_local_json_nd.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_mysql.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_oracle.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_output_json.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_output_junit.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_parquet.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_postgres.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_quality.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_s3_csv.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_s3_delta.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_s3_json.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_s3_json_complex.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_s3_json_multiple_models.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_s3_json_remote.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_schema_evolution.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_snowflake.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_sqlserver.py +0 -0
- {datacontract_cli-0.11.8 → datacontract_cli-0.12.0}/tests/test_test_trino.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.12.0
|
|
4
4
|
Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
|
|
5
5
|
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -12,14 +12,14 @@ Requires-Python: <3.13,>=3.10
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: typer<0.25,>=0.18.0
|
|
15
|
-
Requires-Dist: pydantic<2.
|
|
15
|
+
Requires-Dist: pydantic<2.14.0,>=2.8.2
|
|
16
16
|
Requires-Dist: pyyaml~=6.0.1
|
|
17
17
|
Requires-Dist: requests<2.34,>=2.31
|
|
18
18
|
Requires-Dist: fastjsonschema<2.22.0,>=2.19.1
|
|
19
19
|
Requires-Dist: jsonschema<5.0.0,>=4.23.0
|
|
20
20
|
Requires-Dist: pytz>=2024.1
|
|
21
21
|
Requires-Dist: python-multipart<1.0.0,>=0.0.20
|
|
22
|
-
Requires-Dist: rich<
|
|
22
|
+
Requires-Dist: rich<16.0,>=13.7
|
|
23
23
|
Requires-Dist: sqlglot<31.0.0,>=26.6.0
|
|
24
24
|
Requires-Dist: setuptools>=60
|
|
25
25
|
Requires-Dist: python-dotenv<2.0.0,>=1.0.0
|
|
@@ -28,7 +28,7 @@ Requires-Dist: Jinja2<4.0.0,>=3.1.5
|
|
|
28
28
|
Requires-Dist: jinja_partials<1.0.0,>=0.2.1
|
|
29
29
|
Requires-Dist: datacontract-specification<2.0.0,>=1.2.3
|
|
30
30
|
Requires-Dist: open-data-contract-standard<4.0.0,>=3.1.2
|
|
31
|
-
Requires-Dist: deepdiff<
|
|
31
|
+
Requires-Dist: deepdiff<10.0.0,>=6.0.0
|
|
32
32
|
Provides-Extra: avro
|
|
33
33
|
Requires-Dist: avro==1.12.1; extra == "avro"
|
|
34
34
|
Provides-Extra: bigquery
|
|
@@ -41,7 +41,7 @@ Provides-Extra: databricks
|
|
|
41
41
|
Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "databricks"
|
|
42
42
|
Requires-Dist: soda-core-spark[databricks]<3.6.0,>=3.3.20; extra == "databricks"
|
|
43
43
|
Requires-Dist: databricks-sql-connector<4.3.0,>=3.7.0; extra == "databricks"
|
|
44
|
-
Requires-Dist: databricks-sdk<0.
|
|
44
|
+
Requires-Dist: databricks-sdk<0.104.0; extra == "databricks"
|
|
45
45
|
Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "databricks"
|
|
46
46
|
Provides-Extra: iceberg
|
|
47
47
|
Requires-Dist: pyiceberg==0.11.1; extra == "iceberg"
|
|
@@ -56,9 +56,9 @@ Provides-Extra: postgres
|
|
|
56
56
|
Requires-Dist: soda-core-postgres<3.6.0,>=3.3.20; extra == "postgres"
|
|
57
57
|
Provides-Extra: s3
|
|
58
58
|
Requires-Dist: s3fs<2027.0.0,>=2025.2.0; extra == "s3"
|
|
59
|
-
Requires-Dist: aiobotocore<3.
|
|
59
|
+
Requires-Dist: aiobotocore<3.5.0,>=2.17.0; extra == "s3"
|
|
60
60
|
Provides-Extra: snowflake
|
|
61
|
-
Requires-Dist: snowflake-connector-python[pandas]<4.
|
|
61
|
+
Requires-Dist: snowflake-connector-python[pandas]<4.5,>=3.6; extra == "snowflake"
|
|
62
62
|
Requires-Dist: soda-core-snowflake<3.6.0,>=3.3.20; extra == "snowflake"
|
|
63
63
|
Provides-Extra: sqlserver
|
|
64
64
|
Requires-Dist: soda-core-sqlserver<3.6.0,>=3.3.20; extra == "sqlserver"
|
|
@@ -80,7 +80,7 @@ Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
|
|
|
80
80
|
Provides-Extra: rdf
|
|
81
81
|
Requires-Dist: rdflib==7.6.0; extra == "rdf"
|
|
82
82
|
Provides-Extra: api
|
|
83
|
-
Requires-Dist: fastapi==0.
|
|
83
|
+
Requires-Dist: fastapi==0.136.0; extra == "api"
|
|
84
84
|
Requires-Dist: uvicorn==0.44.0; extra == "api"
|
|
85
85
|
Provides-Extra: protobuf
|
|
86
86
|
Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
|
|
@@ -91,14 +91,14 @@ Requires-Dist: datacontract-cli[all]; extra == "dev"
|
|
|
91
91
|
Requires-Dist: httpx==0.28.1; extra == "dev"
|
|
92
92
|
Requires-Dist: kafka-python; extra == "dev"
|
|
93
93
|
Requires-Dist: minio==7.2.20; extra == "dev"
|
|
94
|
-
Requires-Dist: moto==5.1.
|
|
94
|
+
Requires-Dist: moto==5.1.22; extra == "dev"
|
|
95
95
|
Requires-Dist: pandas>=2.1.0; extra == "dev"
|
|
96
96
|
Requires-Dist: pre-commit<4.6.0,>=3.7.1; extra == "dev"
|
|
97
97
|
Requires-Dist: pytest; extra == "dev"
|
|
98
98
|
Requires-Dist: pytest-xdist; extra == "dev"
|
|
99
99
|
Requires-Dist: pymssql==2.3.13; extra == "dev"
|
|
100
|
-
Requires-Dist: ruff==0.15.
|
|
101
|
-
Requires-Dist: testcontainers[kafka,minio,mssql,mysql,postgres]==4.14.
|
|
100
|
+
Requires-Dist: ruff==0.15.11; extra == "dev"
|
|
101
|
+
Requires-Dist: testcontainers[kafka,minio,mssql,mysql,postgres]==4.14.2; extra == "dev"
|
|
102
102
|
Requires-Dist: trino==0.337.0; extra == "dev"
|
|
103
103
|
Dynamic: license-file
|
|
104
104
|
|
|
@@ -186,7 +186,7 @@ Voilà, the CLI tested that the YAML itself is valid, all records comply with th
|
|
|
186
186
|
We can also use the data contract metadata to export in many [formats](#format), e.g., to generate a SQL DDL:
|
|
187
187
|
|
|
188
188
|
```bash
|
|
189
|
-
$ datacontract export
|
|
189
|
+
$ datacontract export sql https://datacontract.com/orders-v1.odcs.yaml
|
|
190
190
|
|
|
191
191
|
# returns:
|
|
192
192
|
-- Data Contract: orders
|
|
@@ -209,7 +209,7 @@ CREATE TABLE line_items (
|
|
|
209
209
|
Or generate an HTML export:
|
|
210
210
|
|
|
211
211
|
```bash
|
|
212
|
-
$ datacontract export
|
|
212
|
+
$ datacontract export html --output orders-v1.odcs.html https://datacontract.com/orders-v1.odcs.yaml
|
|
213
213
|
```
|
|
214
214
|
|
|
215
215
|
[//]: # (which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).)
|
|
@@ -230,17 +230,17 @@ $ datacontract changelog v1.odcs.yaml v2.odcs.yaml
|
|
|
230
230
|
# execute schema and quality checks (define credentials as environment variables)
|
|
231
231
|
$ datacontract test odcs.yaml
|
|
232
232
|
|
|
233
|
-
# export data contract as html (other formats: avro, dbt, dbt-sources, dbt-staging-sql, jsonschema, odcs, rdf, sql, sodacl, terraform, ...)
|
|
234
|
-
$ datacontract export
|
|
233
|
+
# export data contract as html (other formats: avro, dbt-models, dbt-sources, dbt-staging-sql, jsonschema, odcs, rdf, sql, sodacl, terraform, ...)
|
|
234
|
+
$ datacontract export html datacontract.yaml --output odcs.html
|
|
235
235
|
|
|
236
236
|
# import sql (other formats: avro, glue, bigquery, jsonschema, excel ...)
|
|
237
|
-
$ datacontract import
|
|
237
|
+
$ datacontract import sql --source my-ddl.sql --dialect postgres --output odcs.yaml
|
|
238
238
|
|
|
239
239
|
# import from Excel template
|
|
240
|
-
$ datacontract import
|
|
240
|
+
$ datacontract import excel --source odcs.xlsx --output odcs.yaml
|
|
241
241
|
|
|
242
242
|
# export to Excel template
|
|
243
|
-
$ datacontract export
|
|
243
|
+
$ datacontract export excel --output odcs.xlsx odcs.yaml
|
|
244
244
|
```
|
|
245
245
|
|
|
246
246
|
## Programmatic (Python)
|
|
@@ -386,51 +386,51 @@ Commands
|
|
|
386
386
|
|
|
387
387
|
Create an empty data contract.
|
|
388
388
|
|
|
389
|
-
|
|
390
389
|
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
391
390
|
│ location [LOCATION] The location of the data contract file to create. │
|
|
392
391
|
│ [default: datacontract.yaml] │
|
|
393
392
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
394
393
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
395
|
-
│ --template TEXT URL of a template or data contract
|
|
394
|
+
│ --template TEXT URL of a template or data contract │
|
|
396
395
|
│ --overwrite --no-overwrite Replace the existing datacontract.yaml │
|
|
397
396
|
│ [default: no-overwrite] │
|
|
398
|
-
│ --debug --no-debug Enable debug logging
|
|
397
|
+
│ --debug --no-debug Enable debug logging │
|
|
399
398
|
│ --help Show this message and exit. │
|
|
400
399
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
400
|
+
|
|
401
|
+
Example: datacontract init datacontract.yaml
|
|
402
|
+
|
|
401
403
|
|
|
402
404
|
```
|
|
403
405
|
|
|
404
406
|
### lint
|
|
405
407
|
```
|
|
406
|
-
|
|
407
|
-
Usage: datacontract lint [OPTIONS] [LOCATION]
|
|
408
|
-
|
|
409
|
-
Validate that the datacontract.yaml is correctly formatted.
|
|
410
|
-
|
|
411
|
-
╭─ Arguments
|
|
412
|
-
│ location [LOCATION] The location (url or path) of the data contract
|
|
413
|
-
│ yaml
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
│
|
|
418
|
-
│
|
|
419
|
-
│
|
|
420
|
-
│
|
|
421
|
-
│
|
|
422
|
-
│
|
|
423
|
-
│
|
|
424
|
-
│
|
|
425
|
-
│
|
|
426
|
-
│ --
|
|
427
|
-
│
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
│ --help Show this message and exit. │
|
|
433
|
-
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
408
|
+
|
|
409
|
+
Usage: datacontract lint [OPTIONS] [LOCATION]
|
|
410
|
+
|
|
411
|
+
Validate that the datacontract.yaml is correctly formatted.
|
|
412
|
+
|
|
413
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
414
|
+
│ location [LOCATION] The location (url or path) of the data contract yaml. │
|
|
415
|
+
│ [default: datacontract.yaml] │
|
|
416
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
417
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
418
|
+
│ --json-schema TEXT The location (url or path) of the ODCS JSON │
|
|
419
|
+
│ Schema │
|
|
420
|
+
│ --output PATH Specify the file path where the test results │
|
|
421
|
+
│ should be written to (e.g., │
|
|
422
|
+
│ './test-results/TEST-datacontract.xml'). If no │
|
|
423
|
+
│ path is provided, the output will be printed to │
|
|
424
|
+
│ stdout. │
|
|
425
|
+
│ --output-format [json|junit] The target format for the test results. │
|
|
426
|
+
│ --all-errors Report all JSON Schema validation errors instead │
|
|
427
|
+
│ of stopping after the first one. │
|
|
428
|
+
│ --debug --no-debug Enable debug logging │
|
|
429
|
+
│ --help Show this message and exit. │
|
|
430
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
431
|
+
|
|
432
|
+
Example: datacontract lint datacontract.yaml
|
|
433
|
+
|
|
434
434
|
|
|
435
435
|
```
|
|
436
436
|
|
|
@@ -446,10 +446,9 @@ Commands
|
|
|
446
446
|
│ * v2 TEXT The location (path) of the target (after) data contract YAML. [required] │
|
|
447
447
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
448
448
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
449
|
-
│ --debug
|
|
450
|
-
│ --help
|
|
449
|
+
│ --debug --no-debug Enable debug logging │
|
|
450
|
+
│ --help Show this message and exit. │
|
|
451
451
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
452
|
-
|
|
453
452
|
```
|
|
454
453
|
|
|
455
454
|
```bash
|
|
@@ -459,51 +458,61 @@ $ datacontract changelog v1.odcs.yaml v2.odcs.yaml
|
|
|
459
458
|
### test
|
|
460
459
|
```
|
|
461
460
|
|
|
462
|
-
Usage: datacontract test [OPTIONS] [LOCATION]
|
|
461
|
+
Usage: datacontract test [OPTIONS] [LOCATION]
|
|
463
462
|
|
|
464
463
|
Run schema and quality tests on configured servers.
|
|
465
464
|
|
|
466
|
-
|
|
467
465
|
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
468
466
|
│ location [LOCATION] The location (url or path) of the data contract yaml. │
|
|
469
467
|
│ [default: datacontract.yaml] │
|
|
470
468
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
471
469
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
472
|
-
│ --schema
|
|
473
|
-
│
|
|
474
|
-
│
|
|
475
|
-
│
|
|
476
|
-
│
|
|
477
|
-
│
|
|
478
|
-
│
|
|
479
|
-
│
|
|
480
|
-
│
|
|
481
|
-
│
|
|
482
|
-
│
|
|
483
|
-
│ --
|
|
484
|
-
│
|
|
485
|
-
│
|
|
486
|
-
│
|
|
487
|
-
│
|
|
488
|
-
│
|
|
489
|
-
│
|
|
490
|
-
│
|
|
491
|
-
│
|
|
492
|
-
│
|
|
493
|
-
│
|
|
494
|
-
│
|
|
495
|
-
│
|
|
496
|
-
│
|
|
497
|
-
│
|
|
498
|
-
│
|
|
499
|
-
│
|
|
500
|
-
│ --
|
|
501
|
-
│
|
|
502
|
-
│
|
|
503
|
-
│
|
|
504
|
-
│
|
|
505
|
-
│
|
|
470
|
+
│ --json-schema TEXT The location (url or path) │
|
|
471
|
+
│ of the ODCS JSON Schema │
|
|
472
|
+
│ --server TEXT The server configuration to │
|
|
473
|
+
│ run the schema and quality │
|
|
474
|
+
│ tests. Use the key of the │
|
|
475
|
+
│ server object in the data │
|
|
476
|
+
│ contract yaml file to refer │
|
|
477
|
+
│ to a server, e.g., │
|
|
478
|
+
│ `production`, or `all` for │
|
|
479
|
+
│ all servers (default). │
|
|
480
|
+
│ [default: all] │
|
|
481
|
+
│ --schema-name TEXT Which schema to test, e.g., │
|
|
482
|
+
│ `orders`, or `all` for all │
|
|
483
|
+
│ schemas (default). │
|
|
484
|
+
│ [default: all] │
|
|
485
|
+
│ --publish-test-results --no-publish-test-results Deprecated. Use publish │
|
|
486
|
+
│ parameter. Publish the │
|
|
487
|
+
│ results after the test │
|
|
488
|
+
│ [default: │
|
|
489
|
+
│ no-publish-test-results] │
|
|
490
|
+
│ --publish TEXT The url to publish the │
|
|
491
|
+
│ results after the test. │
|
|
492
|
+
│ --output PATH Specify the file path where │
|
|
493
|
+
│ the test results should be │
|
|
494
|
+
│ written to (e.g., │
|
|
495
|
+
│ './test-results/TEST-datac… │
|
|
496
|
+
│ --output-format [json|junit] The target format for the │
|
|
497
|
+
│ test results. │
|
|
498
|
+
│ --checks TEXT Comma-separated list of │
|
|
499
|
+
│ check categories to run │
|
|
500
|
+
│ (available: schema, │
|
|
501
|
+
│ quality, servicelevel, │
|
|
502
|
+
│ custom). Omit to enable │
|
|
503
|
+
│ all. │
|
|
504
|
+
│ --logs --no-logs Print logs │
|
|
505
|
+
│ [default: no-logs] │
|
|
506
|
+
│ --ssl-verification --no-ssl-verification SSL verification when │
|
|
507
|
+
│ publishing the data │
|
|
508
|
+
│ contract. │
|
|
509
|
+
│ [default: ssl-verification] │
|
|
510
|
+
│ --debug --no-debug Enable debug logging │
|
|
511
|
+
│ --help Show this message and exit. │
|
|
506
512
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
513
|
+
|
|
514
|
+
Example: datacontract test datacontract.yaml --server production
|
|
515
|
+
|
|
507
516
|
|
|
508
517
|
```
|
|
509
518
|
|
|
@@ -1214,65 +1223,54 @@ models:
|
|
|
1214
1223
|
|
|
1215
1224
|
### ci
|
|
1216
1225
|
```
|
|
1217
|
-
|
|
1218
|
-
Usage: datacontract ci [OPTIONS] [LOCATIONS]...
|
|
1219
|
-
|
|
1220
|
-
Run tests for CI/CD pipelines. Emits GitHub Actions annotations and step
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
│
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
│
|
|
1229
|
-
│
|
|
1230
|
-
│
|
|
1231
|
-
│
|
|
1232
|
-
│
|
|
1233
|
-
│
|
|
1234
|
-
│
|
|
1235
|
-
│
|
|
1236
|
-
│
|
|
1237
|
-
│
|
|
1238
|
-
│
|
|
1239
|
-
│
|
|
1240
|
-
│
|
|
1241
|
-
│
|
|
1242
|
-
│
|
|
1243
|
-
│
|
|
1244
|
-
│
|
|
1245
|
-
│
|
|
1246
|
-
│
|
|
1247
|
-
│
|
|
1248
|
-
│
|
|
1249
|
-
│
|
|
1250
|
-
│
|
|
1251
|
-
│
|
|
1252
|
-
│ --
|
|
1253
|
-
│
|
|
1254
|
-
│
|
|
1255
|
-
│
|
|
1256
|
-
│
|
|
1257
|
-
│
|
|
1258
|
-
│
|
|
1259
|
-
│
|
|
1260
|
-
│
|
|
1261
|
-
│
|
|
1262
|
-
│
|
|
1263
|
-
│
|
|
1264
|
-
|
|
1265
|
-
│ [default: error] │
|
|
1266
|
-
│ --ssl-verification --no-ssl-verific… SSL verification │
|
|
1267
|
-
│ when publishing the │
|
|
1268
|
-
│ data contract. │
|
|
1269
|
-
│ [default: │
|
|
1270
|
-
│ ssl-verification] │
|
|
1271
|
-
│ --debug --no-debug Enable debug │
|
|
1272
|
-
│ logging │
|
|
1273
|
-
│ --help Show this message │
|
|
1274
|
-
│ and exit. │
|
|
1275
|
-
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
1226
|
+
|
|
1227
|
+
Usage: datacontract ci [OPTIONS] [LOCATIONS]...
|
|
1228
|
+
|
|
1229
|
+
Run tests for CI/CD pipelines. Emits GitHub Actions annotations and step summary.
|
|
1230
|
+
|
|
1231
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
1232
|
+
│ locations [LOCATIONS]... The location(s) (url or path) of the data contract yaml │
|
|
1233
|
+
│ file(s). │
|
|
1234
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1235
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1236
|
+
│ --json-schema TEXT The location (url or path) │
|
|
1237
|
+
│ of the ODCS JSON Schema │
|
|
1238
|
+
│ --server TEXT The server configuration │
|
|
1239
|
+
│ to run the schema and │
|
|
1240
|
+
│ quality tests. Use the key │
|
|
1241
|
+
│ of the server object in │
|
|
1242
|
+
│ the data contract yaml │
|
|
1243
|
+
│ file to refer to a server, │
|
|
1244
|
+
│ e.g., `production`, or │
|
|
1245
|
+
│ `all` for all servers │
|
|
1246
|
+
│ (default). │
|
|
1247
|
+
│ [default: all] │
|
|
1248
|
+
│ --publish TEXT The url to publish the │
|
|
1249
|
+
│ results after the test. │
|
|
1250
|
+
│ --output PATH Specify the file path │
|
|
1251
|
+
│ where the test results │
|
|
1252
|
+
│ should be written to │
|
|
1253
|
+
│ (e.g., │
|
|
1254
|
+
│ './test-results/TEST-data… │
|
|
1255
|
+
│ --output-format [json|junit] The target format for the │
|
|
1256
|
+
│ test results. │
|
|
1257
|
+
│ --logs --no-logs Print logs │
|
|
1258
|
+
│ [default: no-logs] │
|
|
1259
|
+
│ --json Print test results as JSON │
|
|
1260
|
+
│ to stdout. │
|
|
1261
|
+
│ --fail-on [warning|error|never] Minimum severity that │
|
|
1262
|
+
│ causes a non-zero exit │
|
|
1263
|
+
│ code. │
|
|
1264
|
+
│ [default: error] │
|
|
1265
|
+
│ --ssl-verification --no-ssl-verification SSL verification when │
|
|
1266
|
+
│ publishing the data │
|
|
1267
|
+
│ contract. │
|
|
1268
|
+
│ [default: │
|
|
1269
|
+
│ ssl-verification] │
|
|
1270
|
+
│ --debug --no-debug Enable debug logging │
|
|
1271
|
+
│ --help Show this message and │
|
|
1272
|
+
│ exit. │
|
|
1273
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1276
1274
|
|
|
1277
1275
|
```
|
|
1278
1276
|
|
|
@@ -1357,125 +1355,90 @@ steps:
|
|
|
1357
1355
|
### export
|
|
1358
1356
|
```
|
|
1359
1357
|
|
|
1360
|
-
Usage: datacontract export [OPTIONS] [
|
|
1361
|
-
|
|
1362
|
-
Convert data contract to a specific format. Saves to file specified by `output` option if present,
|
|
1363
|
-
otherwise prints to stdout.
|
|
1358
|
+
Usage: datacontract export [OPTIONS] COMMAND [ARGS]...
|
|
1364
1359
|
|
|
1360
|
+
Convert a data contract to a target format.
|
|
1365
1361
|
|
|
1366
|
-
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
1367
|
-
│ location [LOCATION] The location (url or path) of the data contract yaml. │
|
|
1368
|
-
│ [default: datacontract.yaml] │
|
|
1369
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1370
1362
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1371
|
-
│
|
|
1372
|
-
│ acl|dbt|dbt-sources|dbt-stagin [default: None] │
|
|
1373
|
-
│ g-sql|odcs|rdf|avro|protobuf|g [required] │
|
|
1374
|
-
│ reat-expectations|avro-idl|sql │
|
|
1375
|
-
│ |sql-query|mermaid|html|go|big │
|
|
1376
|
-
│ query|dbml|spark|sqlalchemy|da │
|
|
1377
|
-
│ ta-caterer|dcs|markdown|iceber │
|
|
1378
|
-
│ g|custom|excel|dqx] │
|
|
1379
|
-
│ --output PATH Specify the file path where │
|
|
1380
|
-
│ the exported data will be │
|
|
1381
|
-
│ saved. If no path is provided, │
|
|
1382
|
-
│ the output will be printed to │
|
|
1383
|
-
│ stdout. │
|
|
1384
|
-
│ [default: None] │
|
|
1385
|
-
│ --server TEXT The server name to export. │
|
|
1386
|
-
│ [default: None] │
|
|
1387
|
-
│ --schema-name TEXT The name of the schema to │
|
|
1388
|
-
│ export, e.g., `orders`, or │
|
|
1389
|
-
│ `all` for all schemas │
|
|
1390
|
-
│ (default). │
|
|
1391
|
-
│ [default: all] │
|
|
1392
|
-
│ --schema TEXT The location (url or path) of │
|
|
1393
|
-
│ the ODCS JSON Schema │
|
|
1394
|
-
│ [default: None] │
|
|
1395
|
-
│ --engine TEXT [engine] The engine used for │
|
|
1396
|
-
│ great expection run. │
|
|
1397
|
-
│ [default: None] │
|
|
1398
|
-
│ --template PATH The file path or URL of a │
|
|
1399
|
-
│ template. For Excel format: │
|
|
1400
|
-
│ path/URL to custom Excel │
|
|
1401
|
-
│ template. For custom format: │
|
|
1402
|
-
│ path to Jinja template. │
|
|
1403
|
-
│ [default: None] │
|
|
1404
|
-
│ --debug --no-debug Enable debug logging │
|
|
1405
|
-
│ [default: no-debug] │
|
|
1406
|
-
│ --help Show this message and exit. │
|
|
1363
|
+
│ --help Show this message and exit. │
|
|
1407
1364
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1408
|
-
╭─
|
|
1409
|
-
│
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
│
|
|
1413
|
-
│
|
|
1414
|
-
│
|
|
1415
|
-
│
|
|
1365
|
+
╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────╮
|
|
1366
|
+
│ sql Export a data contract to SQL DDL. │
|
|
1367
|
+
│ sql-query Export a data contract to a SQL query. │
|
|
1368
|
+
│ dbt-models Export a data contract to dbt model schema YAML. │
|
|
1369
|
+
│ dbt-sources Export a data contract to dbt sources YAML. │
|
|
1370
|
+
│ dbt-staging-sql Export a data contract to a dbt staging SQL file. │
|
|
1371
|
+
│ avro Export a data contract to Avro schema. │
|
|
1372
|
+
│ avro-idl Export a data contract to Avro IDL. │
|
|
1373
|
+
│ jsonschema Export a data contract to JSON Schema. │
|
|
1374
|
+
│ pydantic-model Export a data contract to a Pydantic model. │
|
|
1375
|
+
│ protobuf Export a data contract to Protobuf schema. │
|
|
1376
|
+
│ odcs Export a data contract to ODCS format. │
|
|
1377
|
+
│ rdf Export a data contract to RDF. │
|
|
1378
|
+
│ html Export a data contract to HTML. │
|
|
1379
|
+
│ markdown Export a data contract to Markdown. │
|
|
1380
|
+
│ mermaid Export a data contract to Mermaid diagram. │
|
|
1381
|
+
│ bigquery Export a data contract to BigQuery schema. │
|
|
1382
|
+
│ dbml Export a data contract to DBML. │
|
|
1383
|
+
│ go Export a data contract to Go structs. │
|
|
1384
|
+
│ spark Export a data contract to Spark schema. │
|
|
1385
|
+
│ sqlalchemy Export a data contract to SQLAlchemy models. │
|
|
1386
|
+
│ iceberg Export a data contract to Iceberg schema. │
|
|
1387
|
+
│ sodacl Export a data contract to SodaCL checks. │
|
|
1388
|
+
│ great-expectations Export a data contract to Great Expectations suite. │
|
|
1389
|
+
│ data-caterer Export a data contract to Data Caterer format. │
|
|
1390
|
+
│ dcs Export a data contract to DCS format. │
|
|
1391
|
+
│ dqx Export a data contract to DQX format. │
|
|
1392
|
+
│ excel Export a data contract to Excel. │
|
|
1393
|
+
│ custom Export a data contract using a custom Jinja template. │
|
|
1416
1394
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1395
|
+
|
|
1396
|
+
Example: datacontract export html datacontract.yaml --output datacontract.html
|
|
1397
|
+
For SQL dialects (postgres, mysql, snowflake, databricks, sqlserver, trino, oracle), use
|
|
1398
|
+
`datacontract export sql --dialect <dialect>`.
|
|
1399
|
+
|
|
1417
1400
|
|
|
1418
1401
|
```
|
|
1419
1402
|
|
|
1403
|
+
Run `datacontract export <format> --help` to see the format-specific options (e.g. `datacontract export sql --help`). If you are missing a format, please [create an issue on GitHub](https://github.com/datacontract/datacontract-cli/issues).
|
|
1404
|
+
|
|
1405
|
+
#### Examples
|
|
1420
1406
|
```bash
|
|
1421
1407
|
# Example export data contract as HTML
|
|
1422
|
-
datacontract export
|
|
1423
|
-
```
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
| Type | Description | Status |
|
|
1428
|
-
|----------------------|---------------------------------------------------------|---------|
|
|
1429
|
-
| `html` | Export to HTML | ✅ |
|
|
1430
|
-
| `jsonschema` | Export to JSON Schema | ✅ |
|
|
1431
|
-
| `odcs` | Export to Open Data Contract Standard (ODCS) V3 | ✅ |
|
|
1432
|
-
| `sodacl` | Export to SodaCL quality checks in YAML format | ✅ |
|
|
1433
|
-
| `dbt` | Export to dbt models in YAML format | ✅ |
|
|
1434
|
-
| `dbt-sources` | Export to dbt sources in YAML format | ✅ |
|
|
1435
|
-
| `dbt-staging-sql` | Export to dbt staging SQL models | ✅ |
|
|
1436
|
-
| `rdf` | Export data contract to RDF representation in N3 format | ✅ |
|
|
1437
|
-
| `avro` | Export to AVRO models | ✅ |
|
|
1438
|
-
| `protobuf` | Export to Protobuf | ✅ |
|
|
1439
|
-
| `terraform` | Export to terraform resources | ✅ |
|
|
1440
|
-
| `sql` | Export to SQL DDL | ✅ |
|
|
1441
|
-
| `sql-query` | Export to SQL Query | ✅ |
|
|
1442
|
-
| `great-expectations` | Export to Great Expectations Suites in JSON Format | ✅ |
|
|
1443
|
-
| `bigquery` | Export to BigQuery Schemas | ✅ |
|
|
1444
|
-
| `go` | Export to Go types | ✅ |
|
|
1445
|
-
| `pydantic-model` | Export to pydantic models | ✅ |
|
|
1446
|
-
| `DBML` | Export to a DBML Diagram description | ✅ |
|
|
1447
|
-
| `spark` | Export to a Spark StructType | ✅ |
|
|
1448
|
-
| `sqlalchemy` | Export to SQLAlchemy Models | ✅ |
|
|
1449
|
-
| `data-caterer` | Export to Data Caterer in YAML format | ✅ |
|
|
1450
|
-
| `dcs` | Export to Data Contract Specification in YAML format | ✅ |
|
|
1451
|
-
| `markdown` | Export to Markdown | ✅ |
|
|
1452
|
-
| `iceberg` | Export to an Iceberg JSON Schema Definition | partial |
|
|
1453
|
-
| `excel` | Export to ODCS Excel Template | ✅ |
|
|
1454
|
-
| `custom` | Export to Custom format with Jinja | ✅ |
|
|
1455
|
-
| `dqx` | Export to DQX in YAML format | ✅ |
|
|
1456
|
-
| Missing something? | Please create an issue on GitHub | TBD |
|
|
1457
|
-
|
|
1458
|
-
#### SQL
|
|
1408
|
+
datacontract export html --output datacontract.html
|
|
1409
|
+
```
|
|
1410
|
+
|
|
1411
|
+
<details markdown="1">
|
|
1412
|
+
<summary><strong>SQL</strong></summary>
|
|
1459
1413
|
|
|
1460
1414
|
The `export` function converts a given data contract into a SQL data definition language (DDL).
|
|
1461
1415
|
|
|
1462
1416
|
```shell
|
|
1463
|
-
datacontract export datacontract.yaml --
|
|
1417
|
+
datacontract export sql datacontract.yaml --output output.sql
|
|
1464
1418
|
```
|
|
1465
1419
|
|
|
1466
|
-
|
|
1420
|
+
The SQL dialect is determined from the `servers` block in the data contract (e.g. `type: postgres`, `type: snowflake`). Alternatively, pass it explicitly:
|
|
1421
|
+
|
|
1422
|
+
```shell
|
|
1423
|
+
datacontract export sql datacontract.yaml --dialect postgres --output output.sql
|
|
1424
|
+
```
|
|
1425
|
+
|
|
1426
|
+
If using Databricks and an error is thrown when deploying SQL DDLs with `variant` columns, set the following property.
|
|
1467
1427
|
|
|
1468
1428
|
```shell
|
|
1469
1429
|
spark.conf.set(“spark.databricks.delta.schema.typeCheck.enabled”, “false”)
|
|
1470
1430
|
```
|
|
1471
1431
|
|
|
1472
|
-
|
|
1432
|
+
</details>
|
|
1433
|
+
|
|
1434
|
+
<details markdown="1">
|
|
1435
|
+
<summary><strong>Great Expectations</strong></summary>
|
|
1473
1436
|
|
|
1474
1437
|
The `export` function transforms a specified data contract into a comprehensive Great Expectations JSON suite.
|
|
1475
1438
|
If the contract includes multiple models, you need to specify the names of the schema/models you wish to export.
|
|
1476
1439
|
|
|
1477
1440
|
```shell
|
|
1478
|
-
datacontract export datacontract.yaml --
|
|
1441
|
+
datacontract export great-expectations datacontract.yaml --schema-name orders
|
|
1479
1442
|
```
|
|
1480
1443
|
|
|
1481
1444
|
The export creates a list of expectations by utilizing:
|
|
@@ -1498,13 +1461,16 @@ To further customize the export, the following optional arguments are available:
|
|
|
1498
1461
|
|
|
1499
1462
|
Providing `sql_server_type` ensures that the appropriate SQL dialect and connection settings are applied during the expectation validation.
|
|
1500
1463
|
|
|
1501
|
-
|
|
1464
|
+
</details>
|
|
1465
|
+
|
|
1466
|
+
<details markdown="1">
|
|
1467
|
+
<summary><strong>RDF</strong></summary>
|
|
1502
1468
|
|
|
1503
1469
|
The `export` function converts a given data contract into a RDF representation. You have the option to
|
|
1504
1470
|
add a base_url which will be used as the default prefix to resolve relative IRIs inside the document.
|
|
1505
1471
|
|
|
1506
1472
|
```shell
|
|
1507
|
-
datacontract export
|
|
1473
|
+
datacontract export rdf --base https://www.example.com/ datacontract.yaml
|
|
1508
1474
|
```
|
|
1509
1475
|
|
|
1510
1476
|
The data contract is mapped onto the following concepts of a yet to be defined Data Contract
|
|
@@ -1522,24 +1488,36 @@ Having the data contract inside an RDF Graph gives us access the following use c
|
|
|
1522
1488
|
- Apply graph algorithms on multiple data contracts (Find similar data contracts, find "gatekeeper"
|
|
1523
1489
|
data products, find the true domain owner of a field attribute)
|
|
1524
1490
|
|
|
1525
|
-
|
|
1491
|
+
</details>
|
|
1492
|
+
|
|
1493
|
+
<details markdown="1">
|
|
1494
|
+
<summary><strong>DBML</strong></summary>
|
|
1526
1495
|
|
|
1527
1496
|
The export function converts the logical data types of the datacontract into the specific ones of a concrete Database
|
|
1528
1497
|
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
1529
1498
|
logical data types are exported.
|
|
1530
1499
|
|
|
1531
|
-
|
|
1500
|
+
</details>
|
|
1501
|
+
|
|
1502
|
+
<details markdown="1">
|
|
1503
|
+
<summary><strong>DBT & DBT-SOURCES</strong></summary>
|
|
1532
1504
|
|
|
1533
1505
|
The export function converts the datacontract to dbt models in YAML format, with support for SQL dialects.
|
|
1534
1506
|
If a server is selected via the `--server` option (based on the `type` of that server) then the DBT column `data_types` match the expected data types of the server.
|
|
1535
1507
|
If no server is selected, then it defaults to `snowflake`.
|
|
1536
1508
|
|
|
1537
|
-
|
|
1509
|
+
</details>
|
|
1510
|
+
|
|
1511
|
+
<details markdown="1">
|
|
1512
|
+
<summary><strong>Spark</strong></summary>
|
|
1538
1513
|
|
|
1539
1514
|
The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
|
|
1540
1515
|
Spark DataFrame schema is defined as StructType. For more details about Spark Data Types please see [the spark documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
|
|
1541
1516
|
|
|
1542
|
-
|
|
1517
|
+
</details>
|
|
1518
|
+
|
|
1519
|
+
<details markdown="1">
|
|
1520
|
+
<summary><strong>Avro</strong></summary>
|
|
1543
1521
|
|
|
1544
1522
|
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
1545
1523
|
|
|
@@ -1551,7 +1529,7 @@ To specify custom Avro properties in your data contract, you can define them wit
|
|
|
1551
1529
|
|
|
1552
1530
|
>NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
|
|
1553
1531
|
|
|
1554
|
-
|
|
1532
|
+
##### Example Configuration
|
|
1555
1533
|
|
|
1556
1534
|
```yaml
|
|
1557
1535
|
models:
|
|
@@ -1567,7 +1545,7 @@ models:
|
|
|
1567
1545
|
avroDefault: 1672534861000000
|
|
1568
1546
|
```
|
|
1569
1547
|
|
|
1570
|
-
|
|
1548
|
+
##### Explanation
|
|
1571
1549
|
|
|
1572
1550
|
- **models**: The top-level key that contains different models (tables or objects) in your data contract.
|
|
1573
1551
|
- **orders**: A specific model name. Replace this with the name of your model.
|
|
@@ -1581,29 +1559,35 @@ models:
|
|
|
1581
1559
|
- **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
|
|
1582
1560
|
- **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
|
|
1583
1561
|
|
|
1584
|
-
|
|
1562
|
+
</details>
|
|
1563
|
+
|
|
1564
|
+
<details markdown="1">
|
|
1565
|
+
<summary><strong>Data Caterer</strong></summary>
|
|
1585
1566
|
|
|
1586
1567
|
The export function converts the data contract to a data generation task in YAML format that can be
|
|
1587
1568
|
ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the
|
|
1588
1569
|
ability to generate production-like data in any environment based off your data contract.
|
|
1589
1570
|
|
|
1590
1571
|
```shell
|
|
1591
|
-
datacontract export datacontract.yaml --
|
|
1572
|
+
datacontract export data-caterer datacontract.yaml --schema-name orders
|
|
1592
1573
|
```
|
|
1593
1574
|
|
|
1594
1575
|
You can further customise the way data is generated via adding
|
|
1595
1576
|
[additional metadata in the YAML](https://data.catering/setup/generator/data-generator/)
|
|
1596
1577
|
to suit your needs.
|
|
1597
1578
|
|
|
1598
|
-
|
|
1579
|
+
</details>
|
|
1580
|
+
|
|
1581
|
+
<details markdown="1">
|
|
1582
|
+
<summary><strong>Iceberg</strong></summary>
|
|
1599
1583
|
|
|
1600
1584
|
Exports to an [Iceberg Table Json Schema Definition](https://iceberg.apache.org/spec/#appendix-c-json-serialization).
|
|
1601
1585
|
|
|
1602
|
-
This export only supports a single model export at a time because Iceberg's schema definition is for a single table and the exporter maps 1 model to 1 table, use the `--
|
|
1586
|
+
This export only supports a single model export at a time because Iceberg's schema definition is for a single table and the exporter maps 1 model to 1 table, use the `--schema-name` flag
|
|
1603
1587
|
to limit your contract export to a single model.
|
|
1604
1588
|
|
|
1605
1589
|
```bash
|
|
1606
|
-
$ datacontract export
|
|
1590
|
+
$ datacontract export iceberg --schema-name orders https://datacontract.com/examples/orders-latest/datacontract.yaml --output /tmp/orders_iceberg.json
|
|
1607
1591
|
|
|
1608
1592
|
$ cat /tmp/orders_iceberg.json | jq '.'
|
|
1609
1593
|
{
|
|
@@ -1653,29 +1637,34 @@ to limit your contract export to a single model.
|
|
|
1653
1637
|
}
|
|
1654
1638
|
```
|
|
1655
1639
|
|
|
1656
|
-
|
|
1640
|
+
</details>
|
|
1641
|
+
|
|
1642
|
+
<details markdown="1">
|
|
1643
|
+
<summary><strong>Custom</strong></summary>
|
|
1657
1644
|
|
|
1658
1645
|
The export function converts the data contract specification into the custom format with Jinja. You can specify the path to a Jinja template with the `--template` argument, allowing you to output files in any format.
|
|
1659
1646
|
|
|
1660
1647
|
```shell
|
|
1661
|
-
datacontract export
|
|
1648
|
+
datacontract export custom --template template.txt datacontract.yaml
|
|
1662
1649
|
```
|
|
1663
1650
|
|
|
1664
1651
|
##### Jinja templates & variables
|
|
1665
1652
|
|
|
1666
|
-
You can directly use the Data Contract
|
|
1653
|
+
You can directly use the ODCS (Open Data Contract Standard) object as a template variable. `data_contract` is an `OpenDataContractStandard` instance; top-level fields include `name`, `id`, `version`, `schema_` (the list of schemas), `servers`, `team`, etc.
|
|
1667
1654
|
|
|
1668
1655
|
{% raw %}
|
|
1669
1656
|
```shell
|
|
1670
1657
|
$ cat template.txt
|
|
1671
|
-
title: {{ data_contract.
|
|
1672
|
-
|
|
1673
|
-
{%- for
|
|
1674
|
-
- name: {{
|
|
1658
|
+
title: {{ data_contract.name }}
|
|
1659
|
+
schemas:
|
|
1660
|
+
{%- for schema in data_contract.schema_ %}
|
|
1661
|
+
- name: {{ schema.name }}
|
|
1675
1662
|
{%- endfor %}
|
|
1676
1663
|
|
|
1677
|
-
$ datacontract export
|
|
1664
|
+
$ datacontract export custom --template template.txt datacontract.yaml
|
|
1678
1665
|
title: Orders Latest
|
|
1666
|
+
schemas:
|
|
1667
|
+
- name: orders
|
|
1679
1668
|
```
|
|
1680
1669
|
{% endraw %}
|
|
1681
1670
|
|
|
@@ -1705,7 +1694,7 @@ Below is an example of a dbt staging layer that converts a field of `type: times
|
|
|
1705
1694
|
{% endraw %}
|
|
1706
1695
|
- export command
|
|
1707
1696
|
```shell
|
|
1708
|
-
datacontract export datacontract.odcs.yaml --
|
|
1697
|
+
datacontract export custom datacontract.odcs.yaml --template template.sql --schema-name orders
|
|
1709
1698
|
```
|
|
1710
1699
|
- `output.sql`
|
|
1711
1700
|
{% raw %}
|
|
@@ -1721,12 +1710,15 @@ Below is an example of a dbt staging layer that converts a field of `type: times
|
|
|
1721
1710
|
```
|
|
1722
1711
|
{% endraw %}
|
|
1723
1712
|
|
|
1724
|
-
|
|
1713
|
+
</details>
|
|
1714
|
+
|
|
1715
|
+
<details markdown="1">
|
|
1716
|
+
<summary><strong>ODCS Excel Template</strong></summary>
|
|
1725
1717
|
|
|
1726
1718
|
The `export` function converts a data contract into an ODCS (Open Data Contract Standard) Excel template. This creates a user-friendly Excel spreadsheet that can be used for authoring, sharing, and managing data contracts using the familiar Excel interface.
|
|
1727
1719
|
|
|
1728
1720
|
```shell
|
|
1729
|
-
datacontract export
|
|
1721
|
+
datacontract export excel --output datacontract.xlsx datacontract.yaml
|
|
1730
1722
|
```
|
|
1731
1723
|
|
|
1732
1724
|
The Excel format enables:
|
|
@@ -1737,135 +1729,57 @@ The Excel format enables:
|
|
|
1737
1729
|
|
|
1738
1730
|
For more information about the Excel template structure, visit the [ODCS Excel Template repository](https://github.com/datacontract/open-data-contract-standard-excel-template).
|
|
1739
1731
|
|
|
1732
|
+
</details>
|
|
1733
|
+
|
|
1740
1734
|
### import
|
|
1741
1735
|
```
|
|
1742
1736
|
|
|
1743
|
-
Usage: datacontract import [OPTIONS]
|
|
1744
|
-
|
|
1745
|
-
Create a data contract from the given source location. Saves to file specified by `output` option
|
|
1746
|
-
if present, otherwise prints to stdout.
|
|
1737
|
+
Usage: datacontract import [OPTIONS] COMMAND [ARGS]...
|
|
1747
1738
|
|
|
1739
|
+
Create a data contract from a source format.
|
|
1748
1740
|
|
|
1749
1741
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
1750
|
-
│
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
│
|
|
1754
|
-
│
|
|
1755
|
-
│
|
|
1756
|
-
│
|
|
1757
|
-
│
|
|
1758
|
-
│
|
|
1759
|
-
│
|
|
1760
|
-
│
|
|
1761
|
-
│
|
|
1762
|
-
│
|
|
1763
|
-
│
|
|
1764
|
-
│
|
|
1765
|
-
│
|
|
1766
|
-
│
|
|
1767
|
-
│
|
|
1768
|
-
│
|
|
1769
|
-
│ --glue-table TEXT List of table ids to │
|
|
1770
|
-
│ import from the Glue │
|
|
1771
|
-
│ Database (repeat for │
|
|
1772
|
-
│ multiple table ids, leave │
|
|
1773
|
-
│ empty for all tables in │
|
|
1774
|
-
│ the dataset). │
|
|
1775
|
-
│ [default: None] │
|
|
1776
|
-
│ --bigquery-project TEXT The bigquery project id. │
|
|
1777
|
-
│ [default: None] │
|
|
1778
|
-
│ --bigquery-dataset TEXT The bigquery dataset id. │
|
|
1779
|
-
│ [default: None] │
|
|
1780
|
-
│ --bigquery-table TEXT List of table ids to │
|
|
1781
|
-
│ import from the bigquery │
|
|
1782
|
-
│ API (repeat for multiple │
|
|
1783
|
-
│ table ids, leave empty for │
|
|
1784
|
-
│ all tables in the │
|
|
1785
|
-
│ dataset). │
|
|
1786
|
-
│ [default: None] │
|
|
1787
|
-
│ --unity-table-full-name TEXT Full name of a table in │
|
|
1788
|
-
│ the unity catalog │
|
|
1789
|
-
│ [default: None] │
|
|
1790
|
-
│ --dbt-model TEXT List of models names to │
|
|
1791
|
-
│ import from the dbt │
|
|
1792
|
-
│ manifest file (repeat for │
|
|
1793
|
-
│ multiple models names, │
|
|
1794
|
-
│ leave empty for all models │
|
|
1795
|
-
│ in the dataset). │
|
|
1796
|
-
│ [default: None] │
|
|
1797
|
-
│ --dbml-schema TEXT List of schema names to │
|
|
1798
|
-
│ import from the DBML file │
|
|
1799
|
-
│ (repeat for multiple │
|
|
1800
|
-
│ schema names, leave empty │
|
|
1801
|
-
│ for all tables in the │
|
|
1802
|
-
│ file). │
|
|
1803
|
-
│ [default: None] │
|
|
1804
|
-
│ --dbml-table TEXT List of table names to │
|
|
1805
|
-
│ import from the DBML file │
|
|
1806
|
-
│ (repeat for multiple table │
|
|
1807
|
-
│ names, leave empty for all │
|
|
1808
|
-
│ tables in the file). │
|
|
1809
|
-
│ [default: None] │
|
|
1810
|
-
│ --iceberg-table TEXT Table name to assign to │
|
|
1811
|
-
│ the model created from the │
|
|
1812
|
-
│ Iceberg schema. │
|
|
1813
|
-
│ [default: None] │
|
|
1814
|
-
│ --template TEXT The location (url or path) │
|
|
1815
|
-
│ of the ODCS template │
|
|
1816
|
-
│ [default: None] │
|
|
1817
|
-
│ --schema TEXT The location (url or path) │
|
|
1818
|
-
│ of the ODCS JSON Schema │
|
|
1819
|
-
│ [default: None] │
|
|
1820
|
-
│ --owner TEXT The owner or team │
|
|
1821
|
-
│ responsible for managing │
|
|
1822
|
-
│ the data contract. │
|
|
1823
|
-
│ [default: None] │
|
|
1824
|
-
│ --id TEXT The identifier for the the │
|
|
1825
|
-
│ data contract. │
|
|
1826
|
-
│ [default: None] │
|
|
1827
|
-
│ --debug --no-debug Enable debug logging │
|
|
1828
|
-
│ [default: no-debug] │
|
|
1829
|
-
│ --help Show this message and │
|
|
1830
|
-
│ exit. │
|
|
1742
|
+
│ --help Show this message and exit. │
|
|
1743
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1744
|
+
╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────╮
|
|
1745
|
+
│ sql Import a data contract from a SQL DDL file. │
|
|
1746
|
+
│ avro Import a data contract from an Avro schema file. │
|
|
1747
|
+
│ dbt Import a data contract from a dbt manifest file. │
|
|
1748
|
+
│ dbml Import a data contract from a DBML file. │
|
|
1749
|
+
│ glue Import a data contract from AWS Glue. │
|
|
1750
|
+
│ bigquery Import a data contract from BigQuery. │
|
|
1751
|
+
│ unity Import a data contract from Databricks Unity Catalog. │
|
|
1752
|
+
│ jsonschema Import a data contract from a JSON Schema file. │
|
|
1753
|
+
│ json Import a data contract from a JSON file. │
|
|
1754
|
+
│ odcs Import a data contract from an ODCS file. │
|
|
1755
|
+
│ parquet Import a data contract from a Parquet file. │
|
|
1756
|
+
│ csv Import a data contract from a CSV file. │
|
|
1757
|
+
│ protobuf Import a data contract from a Protobuf schema file. │
|
|
1758
|
+
│ spark Import a data contract from a Spark schema. │
|
|
1759
|
+
│ iceberg Import a data contract from an Iceberg schema. │
|
|
1760
|
+
│ excel Import a data contract from an Excel file. │
|
|
1831
1761
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
1762
|
+
|
|
1763
|
+
Example: datacontract import sql --source ddl.sql --dialect postgres --output datacontract.yaml
|
|
1764
|
+
|
|
1832
1765
|
|
|
1833
1766
|
```
|
|
1834
1767
|
|
|
1835
|
-
|
|
1768
|
+
Run `datacontract import <format> --help` to see the format-specific options (e.g. `datacontract import sql --help`). If you are missing a format, please [create an issue on GitHub](https://github.com/datacontract/datacontract-cli/issues).
|
|
1769
|
+
|
|
1770
|
+
#### Examples
|
|
1836
1771
|
```bash
|
|
1837
1772
|
# Example import from SQL DDL
|
|
1838
|
-
datacontract import
|
|
1773
|
+
datacontract import sql --source my_ddl.sql --dialect postgres
|
|
1839
1774
|
# To save to file
|
|
1840
|
-
datacontract import
|
|
1775
|
+
datacontract import sql --source my_ddl.sql --dialect postgres --output datacontract.yaml
|
|
1841
1776
|
```
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
| Type | Description | Status |
|
|
1846
|
-
|--------------------|-----------------------------------------------|---------|
|
|
1847
|
-
| `avro` | Import from AVRO schemas | ✅ |
|
|
1848
|
-
| `bigquery` | Import from BigQuery Schemas | ✅ |
|
|
1849
|
-
| `csv` | Import from CSV File | ✅ |
|
|
1850
|
-
| `dbml` | Import from DBML models | ✅ |
|
|
1851
|
-
| `dbt` | Import from dbt models | ✅ |
|
|
1852
|
-
| `excel` | Import from ODCS Excel Template | ✅ |
|
|
1853
|
-
| `glue` | Import from AWS Glue DataCatalog | ✅ |
|
|
1854
|
-
| `iceberg` | Import from an Iceberg JSON Schema Definition | partial |
|
|
1855
|
-
| `jsonschema` | Import from JSON Schemas | ✅ |
|
|
1856
|
-
| `parquet` | Import from Parquet File Metadata | ✅ |
|
|
1857
|
-
| `protobuf` | Import from Protobuf schemas | ✅ |
|
|
1858
|
-
| `spark` | Import from Spark StructTypes, Variant | ✅ |
|
|
1859
|
-
| `sql` | Import from SQL DDL | ✅ |
|
|
1860
|
-
| `unity` | Import from Databricks Unity Catalog | partial |
|
|
1861
|
-
| Missing something? | Please create an issue on GitHub | TBD |
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
#### BigQuery
|
|
1777
|
+
<details markdown="1">
|
|
1778
|
+
<summary><strong>BigQuery</strong></summary>
|
|
1865
1779
|
|
|
1866
1780
|
BigQuery data can either be imported off of JSON Files generated from the table descriptions or directly from the Bigquery API. In case you want to use JSON Files, specify the `source` parameter with a path to the JSON File.
|
|
1867
1781
|
|
|
1868
|
-
To import from the Bigquery API, you have to _omit_ `source` and instead need to provide
|
|
1782
|
+
To import from the Bigquery API, you have to _omit_ `source` and instead need to provide `--project` and `--dataset`. Additionally you may specify `--table` to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the dataset will be imported.
|
|
1869
1783
|
|
|
1870
1784
|
For providing authentication to the Client, please see [the google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to) or the one [about authorizing client libraries](https://cloud.google.com/bigquery/docs/authentication#client-libs).
|
|
1871
1785
|
|
|
@@ -1873,56 +1787,66 @@ Examples:
|
|
|
1873
1787
|
|
|
1874
1788
|
```bash
|
|
1875
1789
|
# Example import from Bigquery JSON
|
|
1876
|
-
datacontract import
|
|
1790
|
+
datacontract import bigquery --source my_bigquery_table.json
|
|
1877
1791
|
```
|
|
1878
1792
|
|
|
1879
1793
|
```bash
|
|
1880
1794
|
# Example import from Bigquery API with specifying the tables to import
|
|
1881
|
-
datacontract import
|
|
1795
|
+
datacontract import bigquery --project <project_id> --dataset <dataset_id> --table <tableid_1> --table <tableid_2> --table <tableid_3>
|
|
1882
1796
|
```
|
|
1883
1797
|
|
|
1884
1798
|
```bash
|
|
1885
1799
|
# Example import from Bigquery API importing all tables in the dataset
|
|
1886
|
-
datacontract import
|
|
1800
|
+
datacontract import bigquery --project <project_id> --dataset <dataset_id>
|
|
1887
1801
|
```
|
|
1888
1802
|
|
|
1889
|
-
|
|
1803
|
+
</details>
|
|
1804
|
+
|
|
1805
|
+
<details markdown="1">
|
|
1806
|
+
<summary><strong>Unity Catalog</strong></summary>
|
|
1807
|
+
|
|
1890
1808
|
```bash
|
|
1891
1809
|
# Example import from a Unity Catalog JSON file
|
|
1892
|
-
datacontract import
|
|
1810
|
+
datacontract import unity --source my_unity_table.json
|
|
1893
1811
|
```
|
|
1894
1812
|
|
|
1895
1813
|
```bash
|
|
1896
1814
|
# Example import single table from Unity Catalog via HTTP endpoint using PAT
|
|
1897
1815
|
export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com"
|
|
1898
1816
|
export DATACONTRACT_DATABRICKS_TOKEN=<token>
|
|
1899
|
-
datacontract import
|
|
1817
|
+
datacontract import unity --table <table_full_name>
|
|
1900
1818
|
```
|
|
1901
1819
|
Please refer to [Databricks documentation](https://docs.databricks.com/aws/en/dev-tools/auth/unified-auth) on how to set up a profile
|
|
1902
1820
|
```bash
|
|
1903
1821
|
# Example import single table from Unity Catalog via HTTP endpoint using Profile
|
|
1904
1822
|
export DATACONTRACT_DATABRICKS_PROFILE="my-profile"
|
|
1905
|
-
datacontract import
|
|
1823
|
+
datacontract import unity --table <table_full_name>
|
|
1906
1824
|
```
|
|
1907
1825
|
|
|
1908
|
-
|
|
1826
|
+
</details>
|
|
1827
|
+
|
|
1828
|
+
<details markdown="1">
|
|
1829
|
+
<summary><strong>dbt</strong></summary>
|
|
1909
1830
|
|
|
1910
1831
|
Importing from dbt manifest file.
|
|
1911
|
-
You may give the
|
|
1832
|
+
You may give the `--model` parameter to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the database will be imported.
|
|
1912
1833
|
|
|
1913
1834
|
Examples:
|
|
1914
1835
|
|
|
1915
1836
|
```bash
|
|
1916
1837
|
# Example import from dbt manifest with specifying the tables to import
|
|
1917
|
-
datacontract import
|
|
1838
|
+
datacontract import dbt --source <manifest_path> --model <model_name_1> --model <model_name_2> --model <model_name_3>
|
|
1918
1839
|
```
|
|
1919
1840
|
|
|
1920
1841
|
```bash
|
|
1921
1842
|
# Example import from dbt manifest importing all tables in the database
|
|
1922
|
-
datacontract import
|
|
1843
|
+
datacontract import dbt --source <manifest_path>
|
|
1923
1844
|
```
|
|
1924
1845
|
|
|
1925
|
-
|
|
1846
|
+
</details>
|
|
1847
|
+
|
|
1848
|
+
<details markdown="1">
|
|
1849
|
+
<summary><strong>Excel</strong></summary>
|
|
1926
1850
|
|
|
1927
1851
|
Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template).
|
|
1928
1852
|
|
|
@@ -1930,33 +1854,39 @@ Examples:
|
|
|
1930
1854
|
|
|
1931
1855
|
```bash
|
|
1932
1856
|
# Example import from ODCS Excel Template
|
|
1933
|
-
datacontract import
|
|
1857
|
+
datacontract import excel --source odcs.xlsx
|
|
1934
1858
|
```
|
|
1935
1859
|
|
|
1936
|
-
|
|
1860
|
+
</details>
|
|
1861
|
+
|
|
1862
|
+
<details markdown="1">
|
|
1863
|
+
<summary><strong>Glue</strong></summary>
|
|
1937
1864
|
|
|
1938
1865
|
Importing from Glue reads the necessary Data directly off of the AWS API.
|
|
1939
|
-
You may give the
|
|
1866
|
+
You may give the `--table` parameter to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the database will be imported.
|
|
1940
1867
|
|
|
1941
1868
|
Examples:
|
|
1942
1869
|
|
|
1943
1870
|
```bash
|
|
1944
1871
|
# Example import from AWS Glue with specifying the tables to import
|
|
1945
|
-
datacontract import
|
|
1872
|
+
datacontract import glue --database <database_name> --table <table_name_1> --table <table_name_2> --table <table_name_3>
|
|
1946
1873
|
```
|
|
1947
1874
|
|
|
1948
1875
|
```bash
|
|
1949
1876
|
# Example import from AWS Glue importing all tables in the database
|
|
1950
|
-
datacontract import
|
|
1877
|
+
datacontract import glue --database <database_name>
|
|
1951
1878
|
```
|
|
1952
1879
|
|
|
1953
|
-
|
|
1880
|
+
</details>
|
|
1881
|
+
|
|
1882
|
+
<details markdown="1">
|
|
1883
|
+
<summary><strong>Spark</strong></summary>
|
|
1954
1884
|
|
|
1955
|
-
Importing from Spark table or view these must be created or accessible in the Spark context. Specify tables list in `
|
|
1885
|
+
Importing from Spark table or view these must be created or accessible in the Spark context. Specify tables list in the `--tables` option. If the tables are registered in Databricks and have a table-level description, it will also be added to the Data Contract Specification.
|
|
1956
1886
|
|
|
1957
1887
|
```bash
|
|
1958
1888
|
# Example: Import Spark table(s) from Spark context
|
|
1959
|
-
datacontract import
|
|
1889
|
+
datacontract import spark --tables "users,orders"
|
|
1960
1890
|
```
|
|
1961
1891
|
|
|
1962
1892
|
```bash
|
|
@@ -1977,67 +1907,81 @@ DataContract.import_from_source("spark", "users", dataframe = df_user, descripti
|
|
|
1977
1907
|
DataContract.import_from_source(format = "spark", source = "users", dataframe = df_user, description = "description")
|
|
1978
1908
|
```
|
|
1979
1909
|
|
|
1980
|
-
|
|
1910
|
+
</details>
|
|
1911
|
+
|
|
1912
|
+
<details markdown="1">
|
|
1913
|
+
<summary><strong>DBML</strong></summary>
|
|
1981
1914
|
|
|
1982
1915
|
Importing from DBML Documents.
|
|
1983
1916
|
**NOTE:** Since DBML does _not_ have strict requirements on the types of columns, this import _may_ create non-valid datacontracts, as not all types of fields can be properly mapped. In this case you will have to adapt the generated document manually.
|
|
1984
1917
|
We also assume, that the description for models and fields is stored in a Note within the DBML model.
|
|
1985
1918
|
|
|
1986
|
-
You may give the
|
|
1919
|
+
You may give the `--table` or `--schema` parameter to enumerate the tables or schemas that should be imported.
|
|
1987
1920
|
If no tables are given, _all_ available tables of the source will be imported. Likewise, if no schema is given, _all_ schemas are imported.
|
|
1988
1921
|
|
|
1989
1922
|
Examples:
|
|
1990
1923
|
|
|
1991
1924
|
```bash
|
|
1992
1925
|
# Example import from DBML file, importing everything
|
|
1993
|
-
datacontract import
|
|
1926
|
+
datacontract import dbml --source <file_path>
|
|
1994
1927
|
```
|
|
1995
1928
|
|
|
1996
1929
|
```bash
|
|
1997
1930
|
# Example import from DBML file, filtering for tables from specific schemas
|
|
1998
|
-
datacontract import
|
|
1931
|
+
datacontract import dbml --source <file_path> --schema <schema_1> --schema <schema_2>
|
|
1999
1932
|
```
|
|
2000
1933
|
|
|
2001
1934
|
```bash
|
|
2002
1935
|
# Example import from DBML file, filtering for tables with specific names
|
|
2003
|
-
datacontract import
|
|
1936
|
+
datacontract import dbml --source <file_path> --table <table_name_1> --table <table_name_2>
|
|
2004
1937
|
```
|
|
2005
1938
|
|
|
2006
1939
|
```bash
|
|
2007
1940
|
# Example import from DBML file, filtering for tables with specific names from a specific schema
|
|
2008
|
-
datacontract import
|
|
1941
|
+
datacontract import dbml --source <file_path> --table <table_name_1> --schema <schema_1>
|
|
2009
1942
|
```
|
|
2010
1943
|
|
|
2011
|
-
|
|
1944
|
+
</details>
|
|
1945
|
+
|
|
1946
|
+
<details markdown="1">
|
|
1947
|
+
<summary><strong>Iceberg</strong></summary>
|
|
2012
1948
|
|
|
2013
1949
|
Importing from an [Iceberg Table Json Schema Definition](https://iceberg.apache.org/spec/#appendix-c-json-serialization). Specify location of json files using the `source` parameter.
|
|
2014
1950
|
|
|
2015
1951
|
Examples:
|
|
2016
1952
|
|
|
2017
1953
|
```bash
|
|
2018
|
-
datacontract import
|
|
1954
|
+
datacontract import iceberg --source ./tests/fixtures/iceberg/simple_schema.json --table test-table
|
|
2019
1955
|
```
|
|
2020
1956
|
|
|
2021
|
-
|
|
1957
|
+
</details>
|
|
1958
|
+
|
|
1959
|
+
<details markdown="1">
|
|
1960
|
+
<summary><strong>CSV</strong></summary>
|
|
2022
1961
|
|
|
2023
1962
|
Importing from CSV File. Specify file in `source` parameter. It does autodetection for encoding and csv dialect
|
|
2024
1963
|
|
|
2025
1964
|
Example:
|
|
2026
1965
|
|
|
2027
1966
|
```bash
|
|
2028
|
-
datacontract import
|
|
1967
|
+
datacontract import csv --source "test.csv"
|
|
2029
1968
|
```
|
|
2030
1969
|
|
|
2031
|
-
|
|
1970
|
+
</details>
|
|
1971
|
+
|
|
1972
|
+
<details markdown="1">
|
|
1973
|
+
<summary><strong>protobuf</strong></summary>
|
|
2032
1974
|
|
|
2033
1975
|
Importing from protobuf File. Specify file in `source` parameter.
|
|
2034
1976
|
|
|
2035
1977
|
Example:
|
|
2036
1978
|
|
|
2037
1979
|
```bash
|
|
2038
|
-
datacontract import
|
|
1980
|
+
datacontract import protobuf --source "test.proto"
|
|
2039
1981
|
```
|
|
2040
1982
|
|
|
1983
|
+
</details>
|
|
1984
|
+
|
|
2041
1985
|
|
|
2042
1986
|
### catalog
|
|
2043
1987
|
```
|
|
@@ -2046,17 +1990,19 @@ datacontract import --format protobuf --source "test.proto"
|
|
|
2046
1990
|
|
|
2047
1991
|
Create a html catalog of data contracts.
|
|
2048
1992
|
|
|
2049
|
-
|
|
2050
1993
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
2051
|
-
│ --files
|
|
2052
|
-
│
|
|
2053
|
-
│
|
|
2054
|
-
│ --output
|
|
2055
|
-
│
|
|
2056
|
-
│
|
|
2057
|
-
│ --debug
|
|
2058
|
-
│ --help
|
|
1994
|
+
│ --files TEXT Glob pattern for the data contract files to include in the │
|
|
1995
|
+
│ catalog. Applies recursively to any subfolders. │
|
|
1996
|
+
│ [default: *.yaml] │
|
|
1997
|
+
│ --output TEXT Output directory for the catalog html files. │
|
|
1998
|
+
│ [default: catalog/] │
|
|
1999
|
+
│ --json-schema TEXT The location (url or path) of the ODCS JSON Schema │
|
|
2000
|
+
│ --debug --no-debug Enable debug logging │
|
|
2001
|
+
│ --help Show this message and exit. │
|
|
2059
2002
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
2003
|
+
|
|
2004
|
+
Example: datacontract catalog --files "**/*.yaml" --output catalog/
|
|
2005
|
+
|
|
2060
2006
|
|
|
2061
2007
|
```
|
|
2062
2008
|
|
|
@@ -2077,21 +2023,22 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
2077
2023
|
|
|
2078
2024
|
Publish the data contract to the Entropy Data.
|
|
2079
2025
|
|
|
2080
|
-
|
|
2081
2026
|
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
2082
2027
|
│ location [LOCATION] The location (url or path) of the data contract yaml. │
|
|
2083
2028
|
│ [default: datacontract.yaml] │
|
|
2084
2029
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
2085
2030
|
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
2086
|
-
│ --schema
|
|
2031
|
+
│ --json-schema TEXT The location (url or path) of the ODCS JSON │
|
|
2087
2032
|
│ Schema │
|
|
2088
|
-
│ [default: None] │
|
|
2089
2033
|
│ --ssl-verification --no-ssl-verification SSL verification when publishing the data │
|
|
2090
2034
|
│ contract. │
|
|
2091
2035
|
│ [default: ssl-verification] │
|
|
2092
|
-
│ --debug --no-debug Enable debug logging
|
|
2036
|
+
│ --debug --no-debug Enable debug logging │
|
|
2093
2037
|
│ --help Show this message and exit. │
|
|
2094
2038
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
2039
|
+
|
|
2040
|
+
Example: datacontract publish datacontract.yaml
|
|
2041
|
+
|
|
2095
2042
|
|
|
2096
2043
|
```
|
|
2097
2044
|
|
|
@@ -2102,14 +2049,19 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
2102
2049
|
|
|
2103
2050
|
Start the datacontract CLI as server application with REST API.
|
|
2104
2051
|
|
|
2105
|
-
The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
|
|
2106
|
-
commands directly from the Swagger UI.
|
|
2052
|
+
The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
|
|
2053
|
+
You can execute the commands directly from the Swagger UI.
|
|
2054
|
+
|
|
2107
2055
|
To protect the API, you can set the environment variable DATACONTRACT_CLI_API_KEY to a secret API
|
|
2108
|
-
key.
|
|
2109
|
-
|
|
2110
|
-
|
|
2056
|
+
key.
|
|
2057
|
+
To authenticate, requests must include the header 'x-api-key' with the correct API key.
|
|
2058
|
+
This is highly recommended, as data contract tests may be subject to SQL injections or leak
|
|
2059
|
+
sensitive information.
|
|
2060
|
+
|
|
2111
2061
|
To connect to servers (such as a Snowflake data source), set the credentials as environment
|
|
2112
|
-
variables as documented in
|
|
2062
|
+
variables as documented in
|
|
2063
|
+
https://cli.datacontract.com/#test
|
|
2064
|
+
|
|
2113
2065
|
It is possible to run the API with extra arguments for `uvicorn.run()` as keyword arguments, e.g.:
|
|
2114
2066
|
`datacontract api --port 1234 --root_path /datacontract`.
|
|
2115
2067
|
|
|
@@ -2118,9 +2070,12 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
2118
2070
|
│ --host TEXT Bind socket to this host. Hint: For running in docker, set it │
|
|
2119
2071
|
│ to 0.0.0.0 │
|
|
2120
2072
|
│ [default: 127.0.0.1] │
|
|
2121
|
-
│ --debug --no-debug Enable debug logging
|
|
2073
|
+
│ --debug --no-debug Enable debug logging │
|
|
2122
2074
|
│ --help Show this message and exit. │
|
|
2123
2075
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
2076
|
+
|
|
2077
|
+
Example: datacontract api --port 4242 --host 0.0.0.0
|
|
2078
|
+
|
|
2124
2079
|
|
|
2125
2080
|
```
|
|
2126
2081
|
|
|
@@ -2152,7 +2107,7 @@ Create a data contract based on the actual data. This is the fastest way to get
|
|
|
2152
2107
|
|
|
2153
2108
|
1. Use an existing physical schema (e.g., SQL DDL) as a starting point to define your logical data model in the contract. Double check right after the import whether the actual data meets the imported logical data model. Just to be sure.
|
|
2154
2109
|
```bash
|
|
2155
|
-
$ datacontract import
|
|
2110
|
+
$ datacontract import sql --source ddl.sql
|
|
2156
2111
|
$ datacontract test
|
|
2157
2112
|
```
|
|
2158
2113
|
|
|
@@ -2193,10 +2148,10 @@ Create a data contract based on the requirements from use cases.
|
|
|
2193
2148
|
into the consuming data products.
|
|
2194
2149
|
```bash
|
|
2195
2150
|
# data provider
|
|
2196
|
-
$ datacontract export
|
|
2151
|
+
$ datacontract export dbt-models
|
|
2197
2152
|
# data consumer
|
|
2198
|
-
$ datacontract export
|
|
2199
|
-
$ datacontract export
|
|
2153
|
+
$ datacontract export dbt-sources
|
|
2154
|
+
$ datacontract export dbt-staging-sql
|
|
2200
2155
|
```
|
|
2201
2156
|
|
|
2202
2157
|
4. Test that your data product implementation adheres to the contract.
|