datacontract-cli 0.10.7__tar.gz → 0.10.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/PKG-INFO +298 -59
- datacontract_cli-0.10.7/datacontract_cli.egg-info/PKG-INFO → datacontract_cli-0.10.9/README.md +259 -97
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/catalog/catalog.py +4 -2
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/cli.py +44 -15
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/data_contract.py +52 -206
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/fastjsonschema/s3/s3_read_files.py +13 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/check_soda_execute.py +9 -2
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/bigquery.py +8 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/duckdb.py +28 -12
- datacontract_cli-0.10.9/datacontract/engines/soda/connections/trino.py +26 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/avro_converter.py +15 -3
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/avro_idl_converter.py +29 -22
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/bigquery_converter.py +15 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/dbml_converter.py +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/dbt_converter.py +26 -1
- datacontract_cli-0.10.9/datacontract/export/exporter.py +88 -0
- datacontract_cli-0.10.9/datacontract/export/exporter_factory.py +145 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/go_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/great_expectations_converter.py +10 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/html_export.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/jsonschema_converter.py +31 -23
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/odcs_converter.py +24 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/protobuf_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/pydantic_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/rdf_converter.py +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/sodacl_converter.py +23 -12
- datacontract_cli-0.10.9/datacontract/export/spark_converter.py +211 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/sql_converter.py +32 -2
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/sql_type_converter.py +32 -5
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/terraform_converter.py +6 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/imports/avro_importer.py +8 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/imports/bigquery_importer.py +47 -4
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/imports/glue_importer.py +122 -30
- datacontract_cli-0.10.9/datacontract/imports/importer.py +29 -0
- datacontract_cli-0.10.9/datacontract/imports/importer_factory.py +72 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/imports/jsonschema_importer.py +8 -0
- datacontract_cli-0.10.9/datacontract/imports/odcs_importer.py +200 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/imports/sql_importer.py +8 -0
- datacontract_cli-0.10.9/datacontract/imports/unity_importer.py +152 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/resolve.py +22 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/model/data_contract_specification.py +36 -4
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/datacontract.html +17 -2
- datacontract_cli-0.10.9/datacontract/templates/partials/datacontract_information.html +86 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/partials/datacontract_terms.html +7 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/partials/definition.html +9 -1
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/partials/model_field.html +23 -6
- datacontract_cli-0.10.9/datacontract/templates/partials/server.html +209 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/style/output.css +51 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/web.py +17 -0
- datacontract_cli-0.10.7/README.md → datacontract_cli-0.10.9/datacontract_cli.egg-info/PKG-INFO +336 -39
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract_cli.egg-info/SOURCES.txt +15 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract_cli.egg-info/requires.txt +46 -16
- datacontract_cli-0.10.9/pyproject.toml +125 -0
- datacontract_cli-0.10.9/tests/test_description_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_avro.py +1 -1
- datacontract_cli-0.10.9/tests/test_export_custom_exporter.py +31 -0
- datacontract_cli-0.10.9/tests/test_export_spark.py +142 -0
- datacontract_cli-0.10.9/tests/test_export_sql.py +115 -0
- datacontract_cli-0.10.9/tests/test_import_bigquery.py +50 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_import_glue.py +13 -1
- datacontract_cli-0.10.9/tests/test_import_odcs.py +49 -0
- datacontract_cli-0.10.9/tests/test_import_unity_file.py +37 -0
- datacontract_cli-0.10.9/tests/test_roundtrip_jsonschema.py +49 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_schema.py +2 -2
- datacontract_cli-0.10.9/tests/test_test_azure_parquet_remote.py +41 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_dataframe.py +17 -9
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_parquet.py +9 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_postgres.py +36 -11
- datacontract_cli-0.10.9/tests/test_test_trino.py +99 -0
- datacontract_cli-0.10.7/datacontract/templates/partials/datacontract_information.html +0 -66
- datacontract_cli-0.10.7/datacontract/templates/partials/server.html +0 -144
- datacontract_cli-0.10.7/pyproject.toml +0 -82
- datacontract_cli-0.10.7/tests/test_export_sql.py +0 -71
- datacontract_cli-0.10.7/tests/test_import_bigquery.py +0 -85
- datacontract_cli-0.10.7/tests/test_test_azure_parquet_remote.py +0 -28
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/LICENSE +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/MANIFEST.in +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/breaking/breaking.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/breaking/breaking_rules.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/datacontract/check_that_datacontract_file_exists.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/fastjsonschema/check_jsonschema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/dask.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/databricks.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/kafka.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/postgres.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/snowflake.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/engines/soda/connections/sqlserver.py +0 -0
- {datacontract_cli-0.10.7/datacontract/lint/linters → datacontract_cli-0.10.9/datacontract/export}/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/export/csv_type_converter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/init/download_datacontract_file.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/integration/publish_datamesh_manager.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/integration/publish_opentelemetry.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/files.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/lint.py +0 -0
- /datacontract_cli-0.10.7/datacontract/py.typed → /datacontract_cli-0.10.9/datacontract/lint/linters/__init__.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/description_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/example_model_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/field_pattern_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/field_reference_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/notice_period_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/quality_schema_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/linters/valid_constraints_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/schema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/lint/urls.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/model/breaking_change.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/model/exceptions.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/model/run.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/publish/publish.py +0 -0
- /datacontract_cli-0.10.7/tests/test_description_linter.py → /datacontract_cli-0.10.9/datacontract/py.typed +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/index.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/partials/datacontract_servicelevels.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract/templates/partials/example.html +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract_cli.egg-info/dependency_links.txt +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract_cli.egg-info/entry_points.txt +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/datacontract_cli.egg-info/top_level.txt +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/setup.cfg +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_breaking.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_catalog.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_changelog.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_cli.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_documentation_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_download_datacontract_file.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_example_model_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_avro_idl.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_bigquery.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_dbml.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_dbt_models.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_dbt_sources.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_dbt_staging_sql.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_go.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_great_expectations.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_html.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_jsonschema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_odcs.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_protobuf.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_pydantic.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_rdf.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_sodacl.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_sql_query.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_export_terraform.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_field_constraint_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_field_pattern_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_field_reference_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_import_avro.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_import_jsonschema.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_import_sql.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_integration_datameshmanager.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_integration_opentelemetry.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_lint.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_notice_period_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_quality_schema_linter.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_bigquery.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_databricks.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_examples_csv.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_examples_formats_valid.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_examples_inline.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_examples_json.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_examples_missing.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_kafka.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_kafka_remote.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_local_json.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_s3_csv.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_s3_delta.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_s3_json.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_s3_json_complex.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_s3_json_multiple_models.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_s3_json_remote.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_snowflake.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_test_sqlserver.py +0 -0
- {datacontract_cli-0.10.7 → datacontract_cli-0.10.9}/tests/test_web.py +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.10.
|
|
4
|
-
Summary:
|
|
5
|
-
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
|
|
3
|
+
Version: 0.10.9
|
|
4
|
+
Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
|
|
5
|
+
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
|
|
6
6
|
Project-URL: Homepage, https://cli.datacontract.com
|
|
7
|
-
Project-URL: Issues, https://github.com/datacontract/cli/issues
|
|
7
|
+
Project-URL: Issues, https://github.com/datacontract/datacontract-cli/issues
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
@@ -20,37 +20,56 @@ Requires-Dist: fastparquet==2024.5.0
|
|
|
20
20
|
Requires-Dist: python-multipart==0.0.9
|
|
21
21
|
Requires-Dist: rich~=13.7.0
|
|
22
22
|
Requires-Dist: simple-ddl-parser==1.5.1
|
|
23
|
-
Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1
|
|
24
23
|
Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
|
|
25
|
-
Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1
|
|
26
|
-
Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1
|
|
27
|
-
Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1
|
|
28
|
-
Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1
|
|
29
|
-
Requires-Dist: databricks-sql-connector<3.2.0,>=3.1.2
|
|
30
|
-
Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1
|
|
31
24
|
Requires-Dist: setuptools>=60
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist: fastjsonschema~=2.19.1
|
|
25
|
+
Requires-Dist: duckdb==1.0.0
|
|
26
|
+
Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
|
|
35
27
|
Requires-Dist: python-dotenv~=1.0.0
|
|
36
|
-
Requires-Dist: s3fs==2024.5.0
|
|
37
28
|
Requires-Dist: rdflib==7.0.0
|
|
38
|
-
Requires-Dist: avro==1.11.3
|
|
39
29
|
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
|
|
40
30
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist: botocore<1.34.114,>=1.34.41
|
|
31
|
+
Requires-Dist: boto3<1.34.137,>=1.34.41
|
|
32
|
+
Requires-Dist: botocore<1.34.137,>=1.34.41
|
|
44
33
|
Requires-Dist: jinja_partials>=0.2.1
|
|
34
|
+
Provides-Extra: avro
|
|
35
|
+
Requires-Dist: avro==1.11.3; extra == "avro"
|
|
36
|
+
Provides-Extra: bigquery
|
|
37
|
+
Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
|
|
38
|
+
Provides-Extra: databricks
|
|
39
|
+
Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
|
|
40
|
+
Requires-Dist: databricks-sql-connector<3.3.0,>=3.1.2; extra == "databricks"
|
|
41
|
+
Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
|
|
42
|
+
Provides-Extra: deltalake
|
|
43
|
+
Requires-Dist: deltalake<0.19,>=0.17; extra == "deltalake"
|
|
44
|
+
Provides-Extra: kafka
|
|
45
|
+
Requires-Dist: datacontract-cli[avro]; extra == "kafka"
|
|
46
|
+
Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
|
|
47
|
+
Provides-Extra: postgres
|
|
48
|
+
Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
|
|
49
|
+
Provides-Extra: s3
|
|
50
|
+
Requires-Dist: s3fs==2024.6.1; extra == "s3"
|
|
51
|
+
Provides-Extra: snowflake
|
|
52
|
+
Requires-Dist: snowflake-connector-python[pandas]<3.12,>=3.6; extra == "snowflake"
|
|
53
|
+
Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
|
|
54
|
+
Provides-Extra: sqlserver
|
|
55
|
+
Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1; extra == "sqlserver"
|
|
56
|
+
Provides-Extra: trino
|
|
57
|
+
Requires-Dist: soda-core-trino<3.4.0,>=3.3.1; extra == "trino"
|
|
58
|
+
Provides-Extra: all
|
|
59
|
+
Requires-Dist: datacontract-cli[bigquery,databricks,deltalake,kafka,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
|
|
45
60
|
Provides-Extra: dev
|
|
61
|
+
Requires-Dist: datacontract-cli[all]; extra == "dev"
|
|
46
62
|
Requires-Dist: httpx==0.27.0; extra == "dev"
|
|
47
63
|
Requires-Dist: ruff; extra == "dev"
|
|
64
|
+
Requires-Dist: pre-commit~=3.7.1; extra == "dev"
|
|
48
65
|
Requires-Dist: pytest; extra == "dev"
|
|
49
66
|
Requires-Dist: pytest-xdist; extra == "dev"
|
|
50
67
|
Requires-Dist: moto; extra == "dev"
|
|
51
68
|
Requires-Dist: pymssql==2.3.0; extra == "dev"
|
|
52
69
|
Requires-Dist: kafka-python; extra == "dev"
|
|
53
|
-
Requires-Dist:
|
|
70
|
+
Requires-Dist: trino==0.328.0; extra == "dev"
|
|
71
|
+
Requires-Dist: testcontainers<4.8,>=4.5; extra == "dev"
|
|
72
|
+
Requires-Dist: testcontainers[core]; extra == "dev"
|
|
54
73
|
Requires-Dist: testcontainers[minio]; extra == "dev"
|
|
55
74
|
Requires-Dist: testcontainers[postgres]; extra == "dev"
|
|
56
75
|
Requires-Dist: testcontainers[kafka]; extra == "dev"
|
|
@@ -83,9 +102,10 @@ This data contract contains all information to connect to S3 and check that the
|
|
|
83
102
|
|
|
84
103
|
Let's use [pip](https://pip.pypa.io/en/stable/getting-started/) to install the CLI (or use the [Docker image](#docker), if you prefer).
|
|
85
104
|
```bash
|
|
86
|
-
$ python3 -m pip install datacontract-cli
|
|
105
|
+
$ python3 -m pip install datacontract-cli[all]
|
|
87
106
|
```
|
|
88
107
|
|
|
108
|
+
|
|
89
109
|
We run the tests:
|
|
90
110
|
|
|
91
111
|
```bash
|
|
@@ -155,6 +175,7 @@ $ datacontract export --format html https://datacontract.com/examples/orders-lat
|
|
|
155
175
|
|
|
156
176
|
which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).
|
|
157
177
|
|
|
178
|
+
|
|
158
179
|
## Usage
|
|
159
180
|
|
|
160
181
|
```bash
|
|
@@ -207,13 +228,13 @@ Python 3.11 recommended.
|
|
|
207
228
|
Python 3.12 available as pre-release release candidate for 0.9.3
|
|
208
229
|
|
|
209
230
|
```bash
|
|
210
|
-
python3 -m pip install datacontract-cli
|
|
231
|
+
python3 -m pip install datacontract-cli[all]
|
|
211
232
|
```
|
|
212
233
|
|
|
213
234
|
### pipx
|
|
214
235
|
pipx installs into an isolated environment.
|
|
215
236
|
```bash
|
|
216
|
-
pipx install datacontract-cli
|
|
237
|
+
pipx install datacontract-cli[all]
|
|
217
238
|
```
|
|
218
239
|
|
|
219
240
|
### Docker
|
|
@@ -229,6 +250,33 @@ Or via an alias that automatically uses the latest version:
|
|
|
229
250
|
alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" datacontract/cli:latest'
|
|
230
251
|
```
|
|
231
252
|
|
|
253
|
+
|
|
254
|
+
## Optional Dependencies
|
|
255
|
+
|
|
256
|
+
The CLI tool defines several optional dependencies (also known as extras) that can be installed for using with specific servers types.
|
|
257
|
+
With _all_, all server dependencies are included.
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
pip install datacontract-cli[all]
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
A list of available extras:
|
|
264
|
+
|
|
265
|
+
| Dependency | Installation Command |
|
|
266
|
+
|------------------------|--------------------------------------------|
|
|
267
|
+
| Avro Support | `pip install datacontract-cli[avro]` |
|
|
268
|
+
| Google BigQuery | `pip install datacontract-cli[bigquery]` |
|
|
269
|
+
| Databricks Integration | `pip install datacontract-cli[databricks]` |
|
|
270
|
+
| Deltalake Integration | `pip install datacontract-cli[deltalake]` |
|
|
271
|
+
| Kafka Integration | `pip install datacontract-cli[kafka]` |
|
|
272
|
+
| PostgreSQL Integration | `pip install datacontract-cli[postgres]` |
|
|
273
|
+
| S3 Integration | `pip install datacontract-cli[s3]` |
|
|
274
|
+
| Snowflake Integration | `pip install datacontract-cli[snowflake]` |
|
|
275
|
+
| Microsoft SQL Server | `pip install datacontract-cli[sqlserver]` |
|
|
276
|
+
| Trino | `pip install datacontract-cli[trino]` |
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
|
|
232
280
|
## Documentation
|
|
233
281
|
|
|
234
282
|
Commands
|
|
@@ -341,6 +389,7 @@ Supported server types:
|
|
|
341
389
|
- [snowflake](#snowflake)
|
|
342
390
|
- [kafka](#kafka)
|
|
343
391
|
- [postgres](#postgres)
|
|
392
|
+
- [trino](#trino)
|
|
344
393
|
- [local](#local)
|
|
345
394
|
|
|
346
395
|
Supported formats:
|
|
@@ -386,11 +435,12 @@ servers:
|
|
|
386
435
|
|
|
387
436
|
#### Environment Variables
|
|
388
437
|
|
|
389
|
-
| Environment Variable
|
|
390
|
-
|
|
391
|
-
| `DATACONTRACT_S3_REGION` | `eu-central-1` | Region of S3 bucket
|
|
392
|
-
| `DATACONTRACT_S3_ACCESS_KEY_ID` | `AKIAXV5Q5QABCDEFGH` | AWS Access Key ID
|
|
393
|
-
| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key
|
|
438
|
+
| Environment Variable | Example | Description |
|
|
439
|
+
|-------------------------------------|---------------------------------|----------------------------------------|
|
|
440
|
+
| `DATACONTRACT_S3_REGION` | `eu-central-1` | Region of S3 bucket |
|
|
441
|
+
| `DATACONTRACT_S3_ACCESS_KEY_ID` | `AKIAXV5Q5QABCDEFGH` | AWS Access Key ID |
|
|
442
|
+
| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
|
|
443
|
+
| `DATACONTRACT_S3_SESSION_TOKEN` | `AQoDYXdzEJr...` | AWS temporary session token (optional) |
|
|
394
444
|
|
|
395
445
|
|
|
396
446
|
|
|
@@ -420,7 +470,7 @@ models:
|
|
|
420
470
|
|
|
421
471
|
| Environment Variable | Example | Description |
|
|
422
472
|
|----------------------------------------------|---------------------------|---------------------------------------------------------|
|
|
423
|
-
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
|
|
473
|
+
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery. If this environment variable isn't set, the cli tries to use `GOOGLE_APPLICATION_CREDENTIALS` as a fallback, so if you have that set for using their Python library anyway, it should work seamlessly. |
|
|
424
474
|
|
|
425
475
|
|
|
426
476
|
|
|
@@ -544,7 +594,7 @@ models:
|
|
|
544
594
|
|
|
545
595
|
Notebook
|
|
546
596
|
```python
|
|
547
|
-
%pip install datacontract-cli
|
|
597
|
+
%pip install datacontract-cli[databricks]
|
|
548
598
|
dbutils.library.restartPython()
|
|
549
599
|
|
|
550
600
|
from datacontract.data_contract import DataContract
|
|
@@ -681,6 +731,35 @@ models:
|
|
|
681
731
|
| `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
|
|
682
732
|
|
|
683
733
|
|
|
734
|
+
### Trino
|
|
735
|
+
|
|
736
|
+
Data Contract CLI can test data in Trino.
|
|
737
|
+
|
|
738
|
+
#### Example
|
|
739
|
+
|
|
740
|
+
datacontract.yaml
|
|
741
|
+
```yaml
|
|
742
|
+
servers:
|
|
743
|
+
trino:
|
|
744
|
+
type: trino
|
|
745
|
+
host: localhost
|
|
746
|
+
port: 8080
|
|
747
|
+
catalog: my_catalog
|
|
748
|
+
schema: my_schema
|
|
749
|
+
models:
|
|
750
|
+
my_table_1: # corresponds to a table
|
|
751
|
+
type: table
|
|
752
|
+
fields:
|
|
753
|
+
my_column_1: # corresponds to a column
|
|
754
|
+
type: varchar
|
|
755
|
+
```
|
|
756
|
+
|
|
757
|
+
#### Environment Variables
|
|
758
|
+
|
|
759
|
+
| Environment Variable | Example | Description |
|
|
760
|
+
|-------------------------------|--------------------|-------------|
|
|
761
|
+
| `DATACONTRACT_TRINO_USERNAME` | `trino` | Username |
|
|
762
|
+
| `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password |
|
|
684
763
|
|
|
685
764
|
|
|
686
765
|
|
|
@@ -688,10 +767,10 @@ models:
|
|
|
688
767
|
|
|
689
768
|
```
|
|
690
769
|
|
|
691
|
-
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
692
|
-
|
|
693
|
-
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
694
|
-
|
|
770
|
+
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
771
|
+
|
|
772
|
+
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
773
|
+
|
|
695
774
|
╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
696
775
|
│ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
|
|
697
776
|
╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
@@ -699,7 +778,7 @@ models:
|
|
|
699
778
|
│ * --format [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db The export format. [default: None] [required] │
|
|
700
779
|
│ t-staging-sql|odcs|rdf|avro|protobuf|great-expectati │
|
|
701
780
|
│ ons|terraform|avro-idl|sql|sql-query|html|go|bigquer │
|
|
702
|
-
│ y|dbml] │
|
|
781
|
+
│ y|dbml|spark] │
|
|
703
782
|
│ --output PATH Specify the file path where the exported data will be │
|
|
704
783
|
│ saved. If no path is provided, the output will be │
|
|
705
784
|
│ printed to stdout. │
|
|
@@ -749,6 +828,7 @@ Available export options:
|
|
|
749
828
|
| `go` | Export to Go types | ✅ |
|
|
750
829
|
| `pydantic-model` | Export to pydantic models | ✅ |
|
|
751
830
|
| `DBML` | Export to a DBML Diagram description | ✅ |
|
|
831
|
+
| `spark` | Export to a Spark StructType | ✅ |
|
|
752
832
|
| Missing something? | Please create an issue on GitHub | TBD |
|
|
753
833
|
|
|
754
834
|
#### Great Expectations
|
|
@@ -792,13 +872,17 @@ data products, find the true domain owner of a field attribute)
|
|
|
792
872
|
#### DBML
|
|
793
873
|
|
|
794
874
|
The export function converts the logical data types of the datacontract into the specific ones of a concrete Database
|
|
795
|
-
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
875
|
+
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
796
876
|
logical data types are exported.
|
|
797
877
|
|
|
878
|
+
#### Spark
|
|
879
|
+
|
|
880
|
+
The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
|
|
881
|
+
Spark DataFrame schema is defined as StructType. For more details about Spark Data Types please see [the spark documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
|
|
798
882
|
|
|
799
883
|
#### Avro
|
|
800
884
|
|
|
801
|
-
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
885
|
+
The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
|
|
802
886
|
|
|
803
887
|
##### Custom Avro Properties
|
|
804
888
|
|
|
@@ -818,7 +902,7 @@ models:
|
|
|
818
902
|
description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
|
|
819
903
|
type: long
|
|
820
904
|
example: 1672534861000000 # Equivalent to 2023-01-01 01:01:01 in microseconds
|
|
821
|
-
config:
|
|
905
|
+
config:
|
|
822
906
|
avroLogicalType: local-timestamp-micros
|
|
823
907
|
avroDefault: 1672534861000000
|
|
824
908
|
```
|
|
@@ -842,21 +926,21 @@ models:
|
|
|
842
926
|
```
|
|
843
927
|
Usage: datacontract import [OPTIONS]
|
|
844
928
|
|
|
845
|
-
Create a data contract from the given source location. Prints to stdout.
|
|
846
|
-
|
|
929
|
+
Create a data contract from the given source location. Prints to stdout.
|
|
930
|
+
|
|
847
931
|
╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
848
|
-
│ * --format [sql|avro|glue|bigquery|jsonschema] The format of the source file. [default: None] [required] │
|
|
849
|
-
│ --source TEXT
|
|
850
|
-
│
|
|
851
|
-
│ --glue-table TEXT
|
|
852
|
-
│
|
|
853
|
-
│
|
|
854
|
-
│ --bigquery-project TEXT
|
|
855
|
-
│ --bigquery-dataset TEXT
|
|
856
|
-
│ --bigquery-table TEXT
|
|
857
|
-
│
|
|
858
|
-
│
|
|
859
|
-
│ --help
|
|
932
|
+
│ * --format [sql|avro|glue|bigquery|jsonschema|unity] The format of the source file. [default: None] [required] │
|
|
933
|
+
│ --source TEXT The path to the file or Glue Database that should be imported. │
|
|
934
|
+
│ [default: None] │
|
|
935
|
+
│ --glue-table TEXT List of table ids to import from the Glue Database (repeat for │
|
|
936
|
+
│ multiple table ids, leave empty for all tables in the dataset). │
|
|
937
|
+
│ [default: None] │
|
|
938
|
+
│ --bigquery-project TEXT The bigquery project id. [default: None] │
|
|
939
|
+
│ --bigquery-dataset TEXT The bigquery dataset id. [default: None] │
|
|
940
|
+
│ --bigquery-table TEXT List of table ids to import from the bigquery API (repeat for │
|
|
941
|
+
│ multiple table ids, leave empty for all tables in the dataset). │
|
|
942
|
+
│ [default: None] │
|
|
943
|
+
│ --help Show this message and exit. │
|
|
860
944
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
861
945
|
```
|
|
862
946
|
|
|
@@ -874,10 +958,11 @@ Available import options:
|
|
|
874
958
|
| `avro` | Import from AVRO schemas | ✅ |
|
|
875
959
|
| `glue` | Import from AWS Glue DataCatalog | ✅ |
|
|
876
960
|
| `protobuf` | Import from Protobuf schemas | TBD |
|
|
877
|
-
| `jsonschema` | Import from JSON Schemas | ✅
|
|
878
|
-
| `bigquery` | Import from BigQuery Schemas | ✅
|
|
961
|
+
| `jsonschema` | Import from JSON Schemas | ✅ |
|
|
962
|
+
| `bigquery` | Import from BigQuery Schemas | ✅ |
|
|
963
|
+
| `unity` | Import from Databricks Unity Catalog | partial |
|
|
879
964
|
| `dbt` | Import from dbt models | TBD |
|
|
880
|
-
| `odcs` | Import from Open Data Contract Standard (ODCS) |
|
|
965
|
+
| `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ |
|
|
881
966
|
| Missing something? | Please create an issue on GitHub | TBD |
|
|
882
967
|
|
|
883
968
|
|
|
@@ -889,7 +974,7 @@ To import from the Bigquery API, you have to _omit_ `source` and instead need to
|
|
|
889
974
|
|
|
890
975
|
For providing authentication to the Client, please see [the google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to) or the one [about authorizing client libraries](https://cloud.google.com/bigquery/docs/authentication#client-libs).
|
|
891
976
|
|
|
892
|
-
Examples:
|
|
977
|
+
Examples:
|
|
893
978
|
|
|
894
979
|
```bash
|
|
895
980
|
# Example import from Bigquery JSON
|
|
@@ -906,12 +991,26 @@ datacontract import --format bigquery --bigquery-project <project_id> --bigquery
|
|
|
906
991
|
datacontract import --format bigquery --bigquery-project <project_id> --bigquery-dataset <dataset_id>
|
|
907
992
|
```
|
|
908
993
|
|
|
994
|
+
#### Unity Catalog
|
|
995
|
+
|
|
996
|
+
```bash
|
|
997
|
+
# Example import from a Unity Catalog JSON file
|
|
998
|
+
datacontract import --format unity --source my_unity_table.json
|
|
999
|
+
```
|
|
1000
|
+
|
|
1001
|
+
```bash
|
|
1002
|
+
# Example import single table from Unity Catalog via HTTP endpoint
|
|
1003
|
+
export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
|
|
1004
|
+
export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
|
|
1005
|
+
datacontract import --format unity --unity-table-full-name <table_full_name>
|
|
1006
|
+
```
|
|
1007
|
+
|
|
909
1008
|
### Glue
|
|
910
1009
|
|
|
911
1010
|
Importing from Glue reads the necessary Data directly off of the AWS API.
|
|
912
1011
|
You may give the `glue-table` parameter to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the database will be imported.
|
|
913
1012
|
|
|
914
|
-
Examples:
|
|
1013
|
+
Examples:
|
|
915
1014
|
|
|
916
1015
|
```bash
|
|
917
1016
|
# Example import from AWS Glue with specifying the tables to import
|
|
@@ -1156,6 +1255,121 @@ Examples: Removing or renaming models and fields.
|
|
|
1156
1255
|
$ datacontract changelog datacontract-from-pr.yaml datacontract-from-main.yaml
|
|
1157
1256
|
```
|
|
1158
1257
|
|
|
1258
|
+
## Customizing Exporters and Importers
|
|
1259
|
+
|
|
1260
|
+
### Custom Exporter
|
|
1261
|
+
Using the exporter factory to add a new custom exporter
|
|
1262
|
+
```python
|
|
1263
|
+
|
|
1264
|
+
from datacontract.data_contract import DataContract
|
|
1265
|
+
from datacontract.export.exporter import Exporter
|
|
1266
|
+
from datacontract.export.exporter_factory import exporter_factory
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
# Create a custom class that implements export method
|
|
1270
|
+
class CustomExporter(Exporter):
|
|
1271
|
+
def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
|
|
1272
|
+
result = {
|
|
1273
|
+
"title": data_contract.info.title,
|
|
1274
|
+
"version": data_contract.info.version,
|
|
1275
|
+
"description": data_contract.info.description,
|
|
1276
|
+
"email": data_contract.info.contact.email,
|
|
1277
|
+
"url": data_contract.info.contact.url,
|
|
1278
|
+
"model": model,
|
|
1279
|
+
"model_columns": ", ".join(list(data_contract.models.get(model).fields.keys())),
|
|
1280
|
+
"export_args": export_args,
|
|
1281
|
+
"custom_args": export_args.get("custom_arg", ""),
|
|
1282
|
+
}
|
|
1283
|
+
return result
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
# Register the new custom class into factory
|
|
1287
|
+
exporter_factory.register_exporter("custom", CustomExporter)
|
|
1288
|
+
|
|
1289
|
+
|
|
1290
|
+
if __name__ == "__main__":
|
|
1291
|
+
# Create a DataContract instance
|
|
1292
|
+
data_contract = DataContract(
|
|
1293
|
+
data_contract_file="/path/datacontract.yaml"
|
|
1294
|
+
)
|
|
1295
|
+
# call export
|
|
1296
|
+
result = data_contract.export(
|
|
1297
|
+
export_format="custom", model="orders", server="production", custom_arg="my_custom_arg"
|
|
1298
|
+
)
|
|
1299
|
+
print(result)
|
|
1300
|
+
|
|
1301
|
+
```
|
|
1302
|
+
Output
|
|
1303
|
+
```python
|
|
1304
|
+
{
|
|
1305
|
+
'title': 'Orders Unit Test',
|
|
1306
|
+
'version': '1.0.0',
|
|
1307
|
+
'description': 'The orders data contract',
|
|
1308
|
+
'email': 'team-orders@example.com',
|
|
1309
|
+
'url': 'https://wiki.example.com/teams/checkout',
|
|
1310
|
+
'model': 'orders',
|
|
1311
|
+
'model_columns': 'order_id, order_total, order_status',
|
|
1312
|
+
'export_args': {'server': 'production', 'custom_arg': 'my_custom_arg'},
|
|
1313
|
+
'custom_args': 'my_custom_arg'
|
|
1314
|
+
}
|
|
1315
|
+
```
|
|
1316
|
+
|
|
1317
|
+
### Custom Importer
|
|
1318
|
+
Using the importer factory to add a new custom importer
|
|
1319
|
+
```python
|
|
1320
|
+
|
|
1321
|
+
from datacontract.model.data_contract_specification import DataContractSpecification
|
|
1322
|
+
from datacontract.data_contract import DataContract
|
|
1323
|
+
from datacontract.imports.importer import Importer
|
|
1324
|
+
from datacontract.imports.importer_factory import importer_factory
|
|
1325
|
+
import json
|
|
1326
|
+
|
|
1327
|
+
# Create a custom class that implements import_source method
|
|
1328
|
+
class CustomImporter(Importer):
|
|
1329
|
+
def import_source(
|
|
1330
|
+
self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
|
|
1331
|
+
) -> dict:
|
|
1332
|
+
source_dict = json.loads(source)
|
|
1333
|
+
data_contract_specification.id = source_dict.get("id_custom")
|
|
1334
|
+
data_contract_specification.info.title = source_dict.get("title")
|
|
1335
|
+
data_contract_specification.info.description = source_dict.get("description_from_app")
|
|
1336
|
+
|
|
1337
|
+
return data_contract_specification
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
# Register the new custom class into factory
|
|
1341
|
+
importer_factory.register_importer("custom_company_importer", CustomImporter)
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
if __name__ == "__main__":
|
|
1345
|
+
# get a custom da
|
|
1346
|
+
json_from_custom_app = '{"id_custom":"uuid-custom","version":"0.0.2", "title":"my_custom_imported_data", "description_from_app": "Custom contract description"}'
|
|
1347
|
+
# Create a DataContract instance
|
|
1348
|
+
data_contract = DataContract()
|
|
1349
|
+
|
|
1350
|
+
# call import_from
|
|
1351
|
+
result = data_contract.import_from_source(
|
|
1352
|
+
format="custom_company_importer", data_contract_specification=DataContract.init(), source=json_from_custom_app
|
|
1353
|
+
)
|
|
1354
|
+
print(dict(result))
|
|
1355
|
+
|
|
1356
|
+
```
|
|
1357
|
+
Output
|
|
1358
|
+
|
|
1359
|
+
```python
|
|
1360
|
+
{
|
|
1361
|
+
'dataContractSpecification': '0.9.3',
|
|
1362
|
+
'id': 'uuid-custom',
|
|
1363
|
+
'info': Info(title='my_custom_imported_data', version='0.0.1', status=None, description='Custom contract description', owner=None, contact=None),
|
|
1364
|
+
'servers': {},
|
|
1365
|
+
'terms': None,
|
|
1366
|
+
'models': {},
|
|
1367
|
+
'definitions': {},
|
|
1368
|
+
'examples': [],
|
|
1369
|
+
'quality': None,
|
|
1370
|
+
'servicelevels': None
|
|
1371
|
+
}
|
|
1372
|
+
```
|
|
1159
1373
|
## Development Setup
|
|
1160
1374
|
|
|
1161
1375
|
Python base interpreter should be 3.11.x (unless working on 3.12 release candidate).
|
|
@@ -1168,8 +1382,8 @@ source venv/bin/activate
|
|
|
1168
1382
|
# Install Requirements
|
|
1169
1383
|
pip install --upgrade pip setuptools wheel
|
|
1170
1384
|
pip install -e '.[dev]'
|
|
1171
|
-
|
|
1172
|
-
|
|
1385
|
+
pre-commit install
|
|
1386
|
+
pre-commit run --all-files
|
|
1173
1387
|
pytest
|
|
1174
1388
|
```
|
|
1175
1389
|
|
|
@@ -1205,7 +1419,26 @@ docker compose run --rm datacontract --version
|
|
|
1205
1419
|
|
|
1206
1420
|
This command runs the container momentarily to check the version of the `datacontract` CLI. The `--rm` flag ensures that the container is automatically removed after the command executes, keeping your environment clean.
|
|
1207
1421
|
|
|
1422
|
+
## Use with pre-commit
|
|
1423
|
+
|
|
1424
|
+
To run `datacontract-cli` as part of a [pre-commit](https://pre-commit.com/) workflow, add something like the below to the `repos` list in the project's `.pre-commit-config.yaml`:
|
|
1425
|
+
|
|
1426
|
+
```yaml
|
|
1427
|
+
repos:
|
|
1428
|
+
- repo: https://github.com/datacontract/datacontract-cli
|
|
1429
|
+
rev: "v0.10.9"
|
|
1430
|
+
hooks:
|
|
1431
|
+
- id: datacontract-lint
|
|
1432
|
+
- id: datacontract-test
|
|
1433
|
+
args: ["--server", "production"]
|
|
1434
|
+
```
|
|
1435
|
+
|
|
1436
|
+
### Available Hook IDs
|
|
1208
1437
|
|
|
1438
|
+
| Hook ID | Description | Dependency |
|
|
1439
|
+
| ----------------- | ------------------------ | ---------- |
|
|
1440
|
+
| datacontract-lint | Runs the lint subcommand. | Python3 |
|
|
1441
|
+
| datacontract-test | Runs the test subcommand. Please look at [test](#test) section for all available arguments. | Python3 |
|
|
1209
1442
|
|
|
1210
1443
|
## Release Steps
|
|
1211
1444
|
|
|
@@ -1225,6 +1458,12 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
1225
1458
|
- [INNOQ](https://innoq.com)
|
|
1226
1459
|
- And many more. To add your company, please create a pull request.
|
|
1227
1460
|
|
|
1461
|
+
## Related Tools
|
|
1462
|
+
|
|
1463
|
+
- [Data Mesh Manager](https://www.datamesh-manager.com/) is a commercial tool to manage data products and data contracts. It supports the data contract specification and allows the user to import or export data contracts using this specification.
|
|
1464
|
+
- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
|
|
1465
|
+
- [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
|
|
1466
|
+
|
|
1228
1467
|
## License
|
|
1229
1468
|
|
|
1230
1469
|
[MIT License](LICENSE)
|