datacontract-cli 0.10.7__tar.gz → 0.10.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- {datacontract_cli-0.10.7/datacontract_cli.egg-info → datacontract_cli-0.10.8}/PKG-INFO +113 -49
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/README.md +82 -33
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/cli.py +19 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/data_contract.py +60 -172
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/fastjsonschema/s3/s3_read_files.py +13 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/check_soda_execute.py +4 -2
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/bigquery.py +8 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/avro_converter.py +15 -3
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/avro_idl_converter.py +29 -22
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/bigquery_converter.py +15 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/dbml_converter.py +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/dbt_converter.py +26 -1
- datacontract_cli-0.10.8/datacontract/export/exporter.py +87 -0
- datacontract_cli-0.10.8/datacontract/export/exporter_factory.py +52 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/go_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/great_expectations_converter.py +10 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/html_export.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/jsonschema_converter.py +24 -16
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/odcs_converter.py +24 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/protobuf_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/pydantic_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/rdf_converter.py +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/sodacl_converter.py +7 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/sql_converter.py +32 -2
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/sql_type_converter.py +4 -5
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/terraform_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/imports/bigquery_importer.py +30 -4
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/imports/glue_importer.py +9 -0
- datacontract_cli-0.10.8/datacontract/imports/odcs_importer.py +192 -0
- datacontract_cli-0.10.8/datacontract/imports/unity_importer.py +138 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/model/data_contract_specification.py +2 -0
- datacontract_cli-0.10.8/datacontract/templates/partials/server.html +176 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/style/output.css +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/web.py +17 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8/datacontract_cli.egg-info}/PKG-INFO +113 -49
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract_cli.egg-info/SOURCES.txt +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract_cli.egg-info/requires.txt +40 -15
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/pyproject.toml +52 -16
- datacontract_cli-0.10.8/tests/test_description_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_avro.py +1 -1
- datacontract_cli-0.10.8/tests/test_export_custom_exporter.py +30 -0
- datacontract_cli-0.10.8/tests/test_export_sql.py +115 -0
- datacontract_cli-0.10.8/tests/test_import_bigquery.py +50 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_import_glue.py +3 -0
- datacontract_cli-0.10.8/tests/test_import_odcs.py +49 -0
- datacontract_cli-0.10.8/tests/test_import_unity_file.py +36 -0
- datacontract_cli-0.10.8/tests/test_roundtrip_jsonschema.py +49 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_schema.py +2 -2
- datacontract_cli-0.10.8/tests/test_test_azure_parquet_remote.py +41 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_dataframe.py +17 -9
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_parquet.py +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_postgres.py +13 -1
- datacontract_cli-0.10.7/datacontract/templates/partials/server.html +0 -144
- datacontract_cli-0.10.7/tests/test_export_sql.py +0 -71
- datacontract_cli-0.10.7/tests/test_import_bigquery.py +0 -85
- datacontract_cli-0.10.7/tests/test_test_azure_parquet_remote.py +0 -28
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/LICENSE +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/MANIFEST.in +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/breaking/breaking.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/breaking/breaking_rules.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/catalog/catalog.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/datacontract/check_that_datacontract_file_exists.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/fastjsonschema/check_jsonschema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/dask.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/databricks.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/duckdb.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/kafka.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/postgres.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/snowflake.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/engines/soda/connections/sqlserver.py +0 -0
- {datacontract_cli-0.10.7/datacontract/lint/linters → datacontract_cli-0.10.8/datacontract/export}/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/export/csv_type_converter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/imports/avro_importer.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/imports/jsonschema_importer.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/imports/sql_importer.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/init/download_datacontract_file.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/integration/publish_datamesh_manager.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/integration/publish_opentelemetry.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/files.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/lint.py +0 -0
- /datacontract_cli-0.10.7/datacontract/py.typed → /datacontract_cli-0.10.8/datacontract/lint/linters/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/description_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/example_model_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/field_pattern_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/field_reference_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/notice_period_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/quality_schema_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/linters/valid_constraints_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/resolve.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/schema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/lint/urls.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/model/breaking_change.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/model/exceptions.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/model/run.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/publish/publish.py +0 -0
- /datacontract_cli-0.10.7/tests/test_description_linter.py → /datacontract_cli-0.10.8/datacontract/py.typed +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/datacontract.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/index.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/partials/datacontract_information.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/partials/datacontract_servicelevels.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/partials/datacontract_terms.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/partials/definition.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/partials/example.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract/templates/partials/model_field.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract_cli.egg-info/dependency_links.txt +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract_cli.egg-info/entry_points.txt +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/datacontract_cli.egg-info/top_level.txt +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/setup.cfg +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_breaking.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_catalog.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_changelog.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_cli.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_documentation_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_download_datacontract_file.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_example_model_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_avro_idl.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_bigquery.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_dbml.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_dbt_models.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_dbt_sources.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_dbt_staging_sql.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_go.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_great_expectations.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_html.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_jsonschema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_odcs.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_protobuf.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_pydantic.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_rdf.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_sodacl.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_sql_query.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_export_terraform.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_field_constraint_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_field_pattern_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_field_reference_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_import_avro.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_import_jsonschema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_import_sql.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_integration_datameshmanager.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_integration_opentelemetry.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_lint.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_notice_period_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_quality_schema_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_bigquery.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_databricks.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_examples_csv.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_examples_formats_valid.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_examples_inline.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_examples_json.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_examples_missing.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_kafka.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_kafka_remote.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_local_json.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_s3_csv.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_s3_delta.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_s3_json.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_s3_json_complex.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_s3_json_multiple_models.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_s3_json_remote.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_snowflake.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_test_sqlserver.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.8}/tests/test_web.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.8
|
|
4
4
|
Summary: Test data contracts
|
|
5
5
|
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
|
|
6
6
|
Project-URL: Homepage, https://cli.datacontract.com
|
|
@@ -20,31 +20,46 @@ Requires-Dist: fastparquet==2024.5.0
|
|
|
20
20
|
Requires-Dist: python-multipart==0.0.9
|
|
21
21
|
Requires-Dist: rich~=13.7.0
|
|
22
22
|
Requires-Dist: simple-ddl-parser==1.5.1
|
|
23
|
-
Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1
|
|
24
23
|
Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
|
|
25
|
-
Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1
|
|
26
|
-
Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1
|
|
27
|
-
Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1
|
|
28
|
-
Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1
|
|
29
|
-
Requires-Dist: databricks-sql-connector<3.2.0,>=3.1.2
|
|
30
|
-
Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1
|
|
31
24
|
Requires-Dist: setuptools>=60
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist: fastjsonschema~=2.19.1
|
|
25
|
+
Requires-Dist: duckdb==1.0.0
|
|
26
|
+
Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
|
|
35
27
|
Requires-Dist: python-dotenv~=1.0.0
|
|
36
|
-
Requires-Dist: s3fs==2024.5.0
|
|
37
28
|
Requires-Dist: rdflib==7.0.0
|
|
38
|
-
Requires-Dist: avro==1.11.3
|
|
39
29
|
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
|
|
40
30
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist: botocore<1.34.114,>=1.34.41
|
|
31
|
+
Requires-Dist: boto3<1.34.130,>=1.34.41
|
|
32
|
+
Requires-Dist: botocore<1.34.128,>=1.34.41
|
|
44
33
|
Requires-Dist: jinja_partials>=0.2.1
|
|
34
|
+
Provides-Extra: avro
|
|
35
|
+
Requires-Dist: avro==1.11.3; extra == "avro"
|
|
36
|
+
Provides-Extra: bigquery
|
|
37
|
+
Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
|
|
38
|
+
Provides-Extra: databricks
|
|
39
|
+
Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
|
|
40
|
+
Requires-Dist: databricks-sql-connector<3.2.0,>=3.1.2; extra == "databricks"
|
|
41
|
+
Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
|
|
42
|
+
Provides-Extra: deltalake
|
|
43
|
+
Requires-Dist: deltalake<0.19,>=0.17; extra == "deltalake"
|
|
44
|
+
Provides-Extra: kafka
|
|
45
|
+
Requires-Dist: datacontract-cli[avro]; extra == "kafka"
|
|
46
|
+
Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
|
|
47
|
+
Provides-Extra: postgres
|
|
48
|
+
Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
|
|
49
|
+
Provides-Extra: s3
|
|
50
|
+
Requires-Dist: s3fs==2024.6.0; extra == "s3"
|
|
51
|
+
Provides-Extra: snowflake
|
|
52
|
+
Requires-Dist: snowflake-connector-python[pandas]<3.11,>=3.6; extra == "snowflake"
|
|
53
|
+
Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
|
|
54
|
+
Provides-Extra: sqlserver
|
|
55
|
+
Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1; extra == "sqlserver"
|
|
56
|
+
Provides-Extra: all
|
|
57
|
+
Requires-Dist: datacontract-cli[bigquery,databricks,deltalake,kafka,postgres,s3,snowflake,sqlserver]; extra == "all"
|
|
45
58
|
Provides-Extra: dev
|
|
59
|
+
Requires-Dist: datacontract-cli[all]; extra == "dev"
|
|
46
60
|
Requires-Dist: httpx==0.27.0; extra == "dev"
|
|
47
61
|
Requires-Dist: ruff; extra == "dev"
|
|
62
|
+
Requires-Dist: pre-commit~=3.7.1; extra == "dev"
|
|
48
63
|
Requires-Dist: pytest; extra == "dev"
|
|
49
64
|
Requires-Dist: pytest-xdist; extra == "dev"
|
|
50
65
|
Requires-Dist: moto; extra == "dev"
|
|
@@ -83,9 +98,10 @@ This data contract contains all information to connect to S3 and check that the
|
|
|
83
98
|
|
|
84
99
|
Let's use [pip](https://pip.pypa.io/en/stable/getting-started/) to install the CLI (or use the [Docker image](#docker), if you prefer).
|
|
85
100
|
```bash
|
|
86
|
-
$ python3 -m pip install datacontract-cli
|
|
101
|
+
$ python3 -m pip install datacontract-cli[all]
|
|
87
102
|
```
|
|
88
103
|
|
|
104
|
+
|
|
89
105
|
We run the tests:
|
|
90
106
|
|
|
91
107
|
```bash
|
|
@@ -155,6 +171,7 @@ $ datacontract export --format html https://datacontract.com/examples/orders-lat
|
|
|
155
171
|
|
|
156
172
|
which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).
|
|
157
173
|
|
|
174
|
+
|
|
158
175
|
## Usage
|
|
159
176
|
|
|
160
177
|
```bash
|
|
@@ -207,13 +224,13 @@ Python 3.11 recommended.
|
|
|
207
224
|
Python 3.12 available as pre-release release candidate for 0.9.3
|
|
208
225
|
|
|
209
226
|
```bash
|
|
210
|
-
python3 -m pip install datacontract-cli
|
|
227
|
+
python3 -m pip install datacontract-cli[all]
|
|
211
228
|
```
|
|
212
229
|
|
|
213
230
|
### pipx
|
|
214
231
|
pipx installs into an isolated environment.
|
|
215
232
|
```bash
|
|
216
|
-
pipx install datacontract-cli
|
|
233
|
+
pipx install datacontract-cli[all]
|
|
217
234
|
```
|
|
218
235
|
|
|
219
236
|
### Docker
|
|
@@ -229,6 +246,32 @@ Or via an alias that automatically uses the latest version:
|
|
|
229
246
|
alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" datacontract/cli:latest'
|
|
230
247
|
```
|
|
231
248
|
|
|
249
|
+
|
|
250
|
+
## Optional Dependencies
|
|
251
|
+
|
|
252
|
+
The CLI tool defines several optional dependencies (also known as extras) that can be installed for using with specific servers types.
|
|
253
|
+
With _all_, all server dependencies are included.
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
pip install datacontract-cli[all]
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
A list of available extras:
|
|
260
|
+
|
|
261
|
+
| Dependency | Installation Command |
|
|
262
|
+
|-------------------------|-------------------------------------------------------------|
|
|
263
|
+
| Avro Support | `pip install datacontract-cli[avro]` |
|
|
264
|
+
| Google BigQuery | `pip install datacontract-cli[bigquery]` |
|
|
265
|
+
| Databricks Integration | `pip install datacontract-cli[databricks]` |
|
|
266
|
+
| Deltalake Integration | `pip install datacontract-cli[deltalake]` |
|
|
267
|
+
| Kafka Integration | `pip install datacontract-cli[kafka]` |
|
|
268
|
+
| PostgreSQL Integration | `pip install datacontract-cli[postgres]` |
|
|
269
|
+
| S3 Integration | `pip install datacontract-cli[s3]` |
|
|
270
|
+
| Snowflake Integration | `pip install datacontract-cli[snowflake]` |
|
|
271
|
+
| Microsoft SQL Server | `pip install datacontract-cli[sqlserver]` |
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
|
|
232
275
|
## Documentation
|
|
233
276
|
|
|
234
277
|
Commands
|
|
@@ -420,7 +463,7 @@ models:
|
|
|
420
463
|
|
|
421
464
|
| Environment Variable | Example | Description |
|
|
422
465
|
|----------------------------------------------|---------------------------|---------------------------------------------------------|
|
|
423
|
-
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
|
|
466
|
+
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery. If this environment variable isn't set, the cli tries to use `GOOGLE_APPLICATION_CREDENTIALS` as a fallback, so if you have that set for using their Python library anyway, it should work seamlessly. |
|
|
424
467
|
|
|
425
468
|
|
|
426
469
|
|
|
@@ -544,7 +587,7 @@ models:
|
|
|
544
587
|
|
|
545
588
|
Notebook
|
|
546
589
|
```python
|
|
547
|
-
%pip install datacontract-cli
|
|
590
|
+
%pip install datacontract-cli[databricks]
|
|
548
591
|
dbutils.library.restartPython()
|
|
549
592
|
|
|
550
593
|
from datacontract.data_contract import DataContract
|
|
@@ -688,10 +731,10 @@ models:
|
|
|
688
731
|
|
|
689
732
|
```
|
|
690
733
|
|
|
691
|
-
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
692
|
-
|
|
693
|
-
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
694
|
-
|
|
734
|
+
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
735
|
+
|
|
736
|
+
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
737
|
+
|
|
695
738
|
╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
696
739
|
│ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
|
|
697
740
|
╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
@@ -792,13 +835,13 @@ data products, find the true domain owner of a field attribute)
|
|
|
792
835
|
#### DBML
|
|
793
836
|
|
|
794
837
|
The export function converts the logical data types of the datacontract into the specific ones of a concrete Database
|
|
795
|
-
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
838
|
+
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
796
839
|
logical data types are exported.
|
|
797
840
|
|
|
798
841
|
|
|
799
842
|
#### Avro
|
|
800
843
|
|
|
801
|
-
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
844
|
+
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
802
845
|
|
|
803
846
|
##### Custom Avro Properties
|
|
804
847
|
|
|
@@ -818,7 +861,7 @@ models:
|
|
|
818
861
|
description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
|
|
819
862
|
type: long
|
|
820
863
|
example: 1672534861000000 # Equivalent to 2023-01-01 01:01:01 in microseconds
|
|
821
|
-
config:
|
|
864
|
+
config:
|
|
822
865
|
avroLogicalType: local-timestamp-micros
|
|
823
866
|
avroDefault: 1672534861000000
|
|
824
867
|
```
|
|
@@ -842,21 +885,21 @@ models:
|
|
|
842
885
|
```
|
|
843
886
|
Usage: datacontract import [OPTIONS]
|
|
844
887
|
|
|
845
|
-
Create a data contract from the given source location. Prints to stdout.
|
|
846
|
-
|
|
888
|
+
Create a data contract from the given source location. Prints to stdout.
|
|
889
|
+
|
|
847
890
|
╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
848
|
-
│ * --format [sql|avro|glue|bigquery|jsonschema] The format of the source file. [default: None] [required] │
|
|
849
|
-
│ --source TEXT
|
|
850
|
-
│
|
|
851
|
-
│ --glue-table TEXT
|
|
852
|
-
│
|
|
853
|
-
│
|
|
854
|
-
│ --bigquery-project TEXT
|
|
855
|
-
│ --bigquery-dataset TEXT
|
|
856
|
-
│ --bigquery-table TEXT
|
|
857
|
-
│
|
|
858
|
-
│
|
|
859
|
-
│ --help
|
|
891
|
+
│ * --format [sql|avro|glue|bigquery|jsonschema|unity] The format of the source file. [default: None] [required] │
|
|
892
|
+
│ --source TEXT The path to the file or Glue Database that should be imported. │
|
|
893
|
+
│ [default: None] │
|
|
894
|
+
│ --glue-table TEXT List of table ids to import from the Glue Database (repeat for │
|
|
895
|
+
│ multiple table ids, leave empty for all tables in the dataset). │
|
|
896
|
+
│ [default: None] │
|
|
897
|
+
│ --bigquery-project TEXT The bigquery project id. [default: None] │
|
|
898
|
+
│ --bigquery-dataset TEXT The bigquery dataset id. [default: None] │
|
|
899
|
+
│ --bigquery-table TEXT List of table ids to import from the bigquery API (repeat for │
|
|
900
|
+
│ multiple table ids, leave empty for all tables in the dataset). │
|
|
901
|
+
│ [default: None] │
|
|
902
|
+
│ --help Show this message and exit. │
|
|
860
903
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
861
904
|
```
|
|
862
905
|
|
|
@@ -874,10 +917,11 @@ Available import options:
|
|
|
874
917
|
| `avro` | Import from AVRO schemas | ✅ |
|
|
875
918
|
| `glue` | Import from AWS Glue DataCatalog | ✅ |
|
|
876
919
|
| `protobuf` | Import from Protobuf schemas | TBD |
|
|
877
|
-
| `jsonschema` | Import from JSON Schemas | ✅
|
|
878
|
-
| `bigquery` | Import from BigQuery Schemas | ✅
|
|
920
|
+
| `jsonschema` | Import from JSON Schemas | ✅ |
|
|
921
|
+
| `bigquery` | Import from BigQuery Schemas | ✅ |
|
|
922
|
+
| `unity` | Import from Databricks Unity Catalog | partial |
|
|
879
923
|
| `dbt` | Import from dbt models | TBD |
|
|
880
|
-
| `odcs` | Import from Open Data Contract Standard (ODCS) |
|
|
924
|
+
| `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ |
|
|
881
925
|
| Missing something? | Please create an issue on GitHub | TBD |
|
|
882
926
|
|
|
883
927
|
|
|
@@ -889,7 +933,7 @@ To import from the Bigquery API, you have to _omit_ `source` and instead need to
|
|
|
889
933
|
|
|
890
934
|
For providing authentication to the Client, please see [the google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to) or the one [about authorizing client libraries](https://cloud.google.com/bigquery/docs/authentication#client-libs).
|
|
891
935
|
|
|
892
|
-
Examples:
|
|
936
|
+
Examples:
|
|
893
937
|
|
|
894
938
|
```bash
|
|
895
939
|
# Example import from Bigquery JSON
|
|
@@ -906,12 +950,26 @@ datacontract import --format bigquery --bigquery-project <project_id> --bigquery
|
|
|
906
950
|
datacontract import --format bigquery --bigquery-project <project_id> --bigquery-dataset <dataset_id>
|
|
907
951
|
```
|
|
908
952
|
|
|
953
|
+
#### Unity Catalog
|
|
954
|
+
|
|
955
|
+
```bash
|
|
956
|
+
# Example import from a Unity Catalog JSON file
|
|
957
|
+
datacontract import --format unity --source my_unity_table.json
|
|
958
|
+
```
|
|
959
|
+
|
|
960
|
+
```bash
|
|
961
|
+
# Example import single table from Unity Catalog via HTTP endpoint
|
|
962
|
+
export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
|
|
963
|
+
export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
|
|
964
|
+
datacontract import --format unity --unity-table-full-name <table_full_name>
|
|
965
|
+
```
|
|
966
|
+
|
|
909
967
|
### Glue
|
|
910
968
|
|
|
911
969
|
Importing from Glue reads the necessary Data directly off of the AWS API.
|
|
912
970
|
You may give the `glue-table` parameter to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the database will be imported.
|
|
913
971
|
|
|
914
|
-
Examples:
|
|
972
|
+
Examples:
|
|
915
973
|
|
|
916
974
|
```bash
|
|
917
975
|
# Example import from AWS Glue with specifying the tables to import
|
|
@@ -1168,8 +1226,8 @@ source venv/bin/activate
|
|
|
1168
1226
|
# Install Requirements
|
|
1169
1227
|
pip install --upgrade pip setuptools wheel
|
|
1170
1228
|
pip install -e '.[dev]'
|
|
1171
|
-
|
|
1172
|
-
|
|
1229
|
+
pre-commit install
|
|
1230
|
+
pre-commit run --all-files
|
|
1173
1231
|
pytest
|
|
1174
1232
|
```
|
|
1175
1233
|
|
|
@@ -1225,6 +1283,12 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
1225
1283
|
- [INNOQ](https://innoq.com)
|
|
1226
1284
|
- And many more. To add your company, please create a pull request.
|
|
1227
1285
|
|
|
1286
|
+
## Related Tools
|
|
1287
|
+
|
|
1288
|
+
- [Data Mesh Manager](https://www.datamesh-manager.com/) is a commercial tool to manage data products and data contracts. It supports the data contract specification and allows the user to import or export data contracts using this specification.
|
|
1289
|
+
- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
|
|
1290
|
+
- [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
|
|
1291
|
+
|
|
1228
1292
|
## License
|
|
1229
1293
|
|
|
1230
1294
|
[MIT License](LICENSE)
|
|
@@ -25,9 +25,10 @@ This data contract contains all information to connect to S3 and check that the
|
|
|
25
25
|
|
|
26
26
|
Let's use [pip](https://pip.pypa.io/en/stable/getting-started/) to install the CLI (or use the [Docker image](#docker), if you prefer).
|
|
27
27
|
```bash
|
|
28
|
-
$ python3 -m pip install datacontract-cli
|
|
28
|
+
$ python3 -m pip install datacontract-cli[all]
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
+
|
|
31
32
|
We run the tests:
|
|
32
33
|
|
|
33
34
|
```bash
|
|
@@ -97,6 +98,7 @@ $ datacontract export --format html https://datacontract.com/examples/orders-lat
|
|
|
97
98
|
|
|
98
99
|
which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).
|
|
99
100
|
|
|
101
|
+
|
|
100
102
|
## Usage
|
|
101
103
|
|
|
102
104
|
```bash
|
|
@@ -149,13 +151,13 @@ Python 3.11 recommended.
|
|
|
149
151
|
Python 3.12 available as pre-release release candidate for 0.9.3
|
|
150
152
|
|
|
151
153
|
```bash
|
|
152
|
-
python3 -m pip install datacontract-cli
|
|
154
|
+
python3 -m pip install datacontract-cli[all]
|
|
153
155
|
```
|
|
154
156
|
|
|
155
157
|
### pipx
|
|
156
158
|
pipx installs into an isolated environment.
|
|
157
159
|
```bash
|
|
158
|
-
pipx install datacontract-cli
|
|
160
|
+
pipx install datacontract-cli[all]
|
|
159
161
|
```
|
|
160
162
|
|
|
161
163
|
### Docker
|
|
@@ -171,6 +173,32 @@ Or via an alias that automatically uses the latest version:
|
|
|
171
173
|
alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" datacontract/cli:latest'
|
|
172
174
|
```
|
|
173
175
|
|
|
176
|
+
|
|
177
|
+
## Optional Dependencies
|
|
178
|
+
|
|
179
|
+
The CLI tool defines several optional dependencies (also known as extras) that can be installed for using with specific servers types.
|
|
180
|
+
With _all_, all server dependencies are included.
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
pip install datacontract-cli[all]
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
A list of available extras:
|
|
187
|
+
|
|
188
|
+
| Dependency | Installation Command |
|
|
189
|
+
|-------------------------|-------------------------------------------------------------|
|
|
190
|
+
| Avro Support | `pip install datacontract-cli[avro]` |
|
|
191
|
+
| Google BigQuery | `pip install datacontract-cli[bigquery]` |
|
|
192
|
+
| Databricks Integration | `pip install datacontract-cli[databricks]` |
|
|
193
|
+
| Deltalake Integration | `pip install datacontract-cli[deltalake]` |
|
|
194
|
+
| Kafka Integration | `pip install datacontract-cli[kafka]` |
|
|
195
|
+
| PostgreSQL Integration | `pip install datacontract-cli[postgres]` |
|
|
196
|
+
| S3 Integration | `pip install datacontract-cli[s3]` |
|
|
197
|
+
| Snowflake Integration | `pip install datacontract-cli[snowflake]` |
|
|
198
|
+
| Microsoft SQL Server | `pip install datacontract-cli[sqlserver]` |
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
|
|
174
202
|
## Documentation
|
|
175
203
|
|
|
176
204
|
Commands
|
|
@@ -362,7 +390,7 @@ models:
|
|
|
362
390
|
|
|
363
391
|
| Environment Variable | Example | Description |
|
|
364
392
|
|----------------------------------------------|---------------------------|---------------------------------------------------------|
|
|
365
|
-
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
|
|
393
|
+
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery. If this environment variable isn't set, the cli tries to use `GOOGLE_APPLICATION_CREDENTIALS` as a fallback, so if you have that set for using their Python library anyway, it should work seamlessly. |
|
|
366
394
|
|
|
367
395
|
|
|
368
396
|
|
|
@@ -486,7 +514,7 @@ models:
|
|
|
486
514
|
|
|
487
515
|
Notebook
|
|
488
516
|
```python
|
|
489
|
-
%pip install datacontract-cli
|
|
517
|
+
%pip install datacontract-cli[databricks]
|
|
490
518
|
dbutils.library.restartPython()
|
|
491
519
|
|
|
492
520
|
from datacontract.data_contract import DataContract
|
|
@@ -630,10 +658,10 @@ models:
|
|
|
630
658
|
|
|
631
659
|
```
|
|
632
660
|
|
|
633
|
-
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
634
|
-
|
|
635
|
-
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
636
|
-
|
|
661
|
+
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
662
|
+
|
|
663
|
+
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
664
|
+
|
|
637
665
|
╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
638
666
|
│ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
|
|
639
667
|
╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
@@ -734,13 +762,13 @@ data products, find the true domain owner of a field attribute)
|
|
|
734
762
|
#### DBML
|
|
735
763
|
|
|
736
764
|
The export function converts the logical data types of the datacontract into the specific ones of a concrete Database
|
|
737
|
-
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
765
|
+
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
738
766
|
logical data types are exported.
|
|
739
767
|
|
|
740
768
|
|
|
741
769
|
#### Avro
|
|
742
770
|
|
|
743
|
-
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
771
|
+
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
744
772
|
|
|
745
773
|
##### Custom Avro Properties
|
|
746
774
|
|
|
@@ -760,7 +788,7 @@ models:
|
|
|
760
788
|
description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
|
|
761
789
|
type: long
|
|
762
790
|
example: 1672534861000000 # Equivalent to 2023-01-01 01:01:01 in microseconds
|
|
763
|
-
config:
|
|
791
|
+
config:
|
|
764
792
|
avroLogicalType: local-timestamp-micros
|
|
765
793
|
avroDefault: 1672534861000000
|
|
766
794
|
```
|
|
@@ -784,21 +812,21 @@ models:
|
|
|
784
812
|
```
|
|
785
813
|
Usage: datacontract import [OPTIONS]
|
|
786
814
|
|
|
787
|
-
Create a data contract from the given source location. Prints to stdout.
|
|
788
|
-
|
|
815
|
+
Create a data contract from the given source location. Prints to stdout.
|
|
816
|
+
|
|
789
817
|
╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
790
|
-
│ * --format [sql|avro|glue|bigquery|jsonschema] The format of the source file. [default: None] [required] │
|
|
791
|
-
│ --source TEXT
|
|
792
|
-
│
|
|
793
|
-
│ --glue-table TEXT
|
|
794
|
-
│
|
|
795
|
-
│
|
|
796
|
-
│ --bigquery-project TEXT
|
|
797
|
-
│ --bigquery-dataset TEXT
|
|
798
|
-
│ --bigquery-table TEXT
|
|
799
|
-
│
|
|
800
|
-
│
|
|
801
|
-
│ --help
|
|
818
|
+
│ * --format [sql|avro|glue|bigquery|jsonschema|unity] The format of the source file. [default: None] [required] │
|
|
819
|
+
│ --source TEXT The path to the file or Glue Database that should be imported. │
|
|
820
|
+
│ [default: None] │
|
|
821
|
+
│ --glue-table TEXT List of table ids to import from the Glue Database (repeat for │
|
|
822
|
+
│ multiple table ids, leave empty for all tables in the dataset). │
|
|
823
|
+
│ [default: None] │
|
|
824
|
+
│ --bigquery-project TEXT The bigquery project id. [default: None] │
|
|
825
|
+
│ --bigquery-dataset TEXT The bigquery dataset id. [default: None] │
|
|
826
|
+
│ --bigquery-table TEXT List of table ids to import from the bigquery API (repeat for │
|
|
827
|
+
│ multiple table ids, leave empty for all tables in the dataset). │
|
|
828
|
+
│ [default: None] │
|
|
829
|
+
│ --help Show this message and exit. │
|
|
802
830
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
803
831
|
```
|
|
804
832
|
|
|
@@ -816,10 +844,11 @@ Available import options:
|
|
|
816
844
|
| `avro` | Import from AVRO schemas | ✅ |
|
|
817
845
|
| `glue` | Import from AWS Glue DataCatalog | ✅ |
|
|
818
846
|
| `protobuf` | Import from Protobuf schemas | TBD |
|
|
819
|
-
| `jsonschema` | Import from JSON Schemas | ✅
|
|
820
|
-
| `bigquery` | Import from BigQuery Schemas | ✅
|
|
847
|
+
| `jsonschema` | Import from JSON Schemas | ✅ |
|
|
848
|
+
| `bigquery` | Import from BigQuery Schemas | ✅ |
|
|
849
|
+
| `unity` | Import from Databricks Unity Catalog | partial |
|
|
821
850
|
| `dbt` | Import from dbt models | TBD |
|
|
822
|
-
| `odcs` | Import from Open Data Contract Standard (ODCS) |
|
|
851
|
+
| `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ |
|
|
823
852
|
| Missing something? | Please create an issue on GitHub | TBD |
|
|
824
853
|
|
|
825
854
|
|
|
@@ -831,7 +860,7 @@ To import from the Bigquery API, you have to _omit_ `source` and instead need to
|
|
|
831
860
|
|
|
832
861
|
For providing authentication to the Client, please see [the google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to) or the one [about authorizing client libraries](https://cloud.google.com/bigquery/docs/authentication#client-libs).
|
|
833
862
|
|
|
834
|
-
Examples:
|
|
863
|
+
Examples:
|
|
835
864
|
|
|
836
865
|
```bash
|
|
837
866
|
# Example import from Bigquery JSON
|
|
@@ -848,12 +877,26 @@ datacontract import --format bigquery --bigquery-project <project_id> --bigquery
|
|
|
848
877
|
datacontract import --format bigquery --bigquery-project <project_id> --bigquery-dataset <dataset_id>
|
|
849
878
|
```
|
|
850
879
|
|
|
880
|
+
#### Unity Catalog
|
|
881
|
+
|
|
882
|
+
```bash
|
|
883
|
+
# Example import from a Unity Catalog JSON file
|
|
884
|
+
datacontract import --format unity --source my_unity_table.json
|
|
885
|
+
```
|
|
886
|
+
|
|
887
|
+
```bash
|
|
888
|
+
# Example import single table from Unity Catalog via HTTP endpoint
|
|
889
|
+
export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
|
|
890
|
+
export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
|
|
891
|
+
datacontract import --format unity --unity-table-full-name <table_full_name>
|
|
892
|
+
```
|
|
893
|
+
|
|
851
894
|
### Glue
|
|
852
895
|
|
|
853
896
|
Importing from Glue reads the necessary Data directly off of the AWS API.
|
|
854
897
|
You may give the `glue-table` parameter to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the database will be imported.
|
|
855
898
|
|
|
856
|
-
Examples:
|
|
899
|
+
Examples:
|
|
857
900
|
|
|
858
901
|
```bash
|
|
859
902
|
# Example import from AWS Glue with specifying the tables to import
|
|
@@ -1110,8 +1153,8 @@ source venv/bin/activate
|
|
|
1110
1153
|
# Install Requirements
|
|
1111
1154
|
pip install --upgrade pip setuptools wheel
|
|
1112
1155
|
pip install -e '.[dev]'
|
|
1113
|
-
|
|
1114
|
-
|
|
1156
|
+
pre-commit install
|
|
1157
|
+
pre-commit run --all-files
|
|
1115
1158
|
pytest
|
|
1116
1159
|
```
|
|
1117
1160
|
|
|
@@ -1167,6 +1210,12 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
1167
1210
|
- [INNOQ](https://innoq.com)
|
|
1168
1211
|
- And many more. To add your company, please create a pull request.
|
|
1169
1212
|
|
|
1213
|
+
## Related Tools
|
|
1214
|
+
|
|
1215
|
+
- [Data Mesh Manager](https://www.datamesh-manager.com/) is a commercial tool to manage data products and data contracts. It supports the data contract specification and allows the user to import or export data contracts using this specification.
|
|
1216
|
+
- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
|
|
1217
|
+
- [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
|
|
1218
|
+
|
|
1170
1219
|
## License
|
|
1171
1220
|
|
|
1172
1221
|
[MIT License](LICENSE)
|
|
@@ -5,6 +5,7 @@ from typing import Iterable, Optional
|
|
|
5
5
|
from typing import List
|
|
6
6
|
|
|
7
7
|
import typer
|
|
8
|
+
import uvicorn
|
|
8
9
|
from click import Context
|
|
9
10
|
from rich import box
|
|
10
11
|
from rich.console import Console
|
|
@@ -12,6 +13,7 @@ from rich.table import Table
|
|
|
12
13
|
from typer.core import TyperGroup
|
|
13
14
|
from typing_extensions import Annotated
|
|
14
15
|
|
|
16
|
+
from datacontract import web
|
|
15
17
|
from datacontract.catalog.catalog import create_index_html, create_data_contract_html
|
|
16
18
|
from datacontract.data_contract import DataContract, ExportFormat
|
|
17
19
|
from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
|
|
@@ -183,6 +185,7 @@ def export(
|
|
|
183
185
|
result = DataContract(data_contract_file=location, server=server).export(
|
|
184
186
|
export_format=format,
|
|
185
187
|
model=model,
|
|
188
|
+
server=server,
|
|
186
189
|
rdf_base=rdf_base,
|
|
187
190
|
sql_server_type=sql_server_type,
|
|
188
191
|
)
|
|
@@ -201,6 +204,8 @@ class ImportFormat(str, Enum):
|
|
|
201
204
|
glue = "glue"
|
|
202
205
|
bigquery = "bigquery"
|
|
203
206
|
jsonschema = "jsonschema"
|
|
207
|
+
odcs="odcs"
|
|
208
|
+
unity = "unity"
|
|
204
209
|
|
|
205
210
|
|
|
206
211
|
@app.command(name="import")
|
|
@@ -223,11 +228,12 @@ def import_(
|
|
|
223
228
|
help="List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all tables in the dataset)."
|
|
224
229
|
),
|
|
225
230
|
] = None,
|
|
231
|
+
unity_table_full_name: Annotated[Optional[str], typer.Option(help="Full name of a table in the unity catalog")] = None,
|
|
226
232
|
):
|
|
227
233
|
"""
|
|
228
234
|
Create a data contract from the given source location. Prints to stdout.
|
|
229
235
|
"""
|
|
230
|
-
result = DataContract().import_from_source(format, source, glue_table, bigquery_table, bigquery_project, bigquery_dataset)
|
|
236
|
+
result = DataContract().import_from_source(format, source, glue_table, bigquery_table, bigquery_project, bigquery_dataset, unity_table_full_name)
|
|
231
237
|
console.print(result.to_yaml())
|
|
232
238
|
|
|
233
239
|
|
|
@@ -323,6 +329,18 @@ def diff(
|
|
|
323
329
|
console.print(result.changelog_str())
|
|
324
330
|
|
|
325
331
|
|
|
332
|
+
@app.command()
|
|
333
|
+
def serve(
|
|
334
|
+
port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4242,
|
|
335
|
+
host: Annotated[str, typer.Option(help="Bind socket to this host.")] = "127.0.0.1",
|
|
336
|
+
):
|
|
337
|
+
"""
|
|
338
|
+
Start the datacontract web server.
|
|
339
|
+
"""
|
|
340
|
+
|
|
341
|
+
uvicorn.run(web.app, port=port, host=host)
|
|
342
|
+
|
|
343
|
+
|
|
326
344
|
def _handle_result(run):
|
|
327
345
|
_print_table(run)
|
|
328
346
|
if run.result == "passed":
|