datacontract-cli 0.10.22__py3-none-any.whl → 0.10.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- datacontract/__init__.py +13 -0
- datacontract/catalog/catalog.py +2 -2
- datacontract/cli.py +20 -72
- datacontract/data_contract.py +5 -3
- datacontract/engines/data_contract_test.py +32 -7
- datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +2 -3
- datacontract/engines/fastjsonschema/s3/s3_read_files.py +3 -2
- datacontract/engines/soda/check_soda_execute.py +17 -4
- datacontract/engines/soda/connections/{duckdb.py → duckdb_connection.py} +66 -9
- datacontract/engines/soda/connections/kafka.py +3 -2
- datacontract/export/avro_converter.py +10 -3
- datacontract/export/bigquery_converter.py +1 -1
- datacontract/export/dbt_converter.py +13 -10
- datacontract/export/duckdb_type_converter.py +57 -0
- datacontract/export/odcs_v3_exporter.py +27 -7
- datacontract/export/protobuf_converter.py +163 -69
- datacontract/imports/avro_importer.py +31 -6
- datacontract/imports/csv_importer.py +111 -57
- datacontract/imports/importer.py +1 -0
- datacontract/imports/importer_factory.py +5 -0
- datacontract/imports/odcs_v3_importer.py +49 -7
- datacontract/imports/protobuf_importer.py +266 -0
- datacontract/lint/resolve.py +40 -12
- datacontract/model/data_contract_specification.py +2 -2
- datacontract/model/run.py +3 -0
- datacontract/output/__init__.py +0 -0
- datacontract/output/junit_test_results.py +135 -0
- datacontract/output/output_format.py +10 -0
- datacontract/output/test_results_writer.py +79 -0
- datacontract/templates/datacontract.html +2 -1
- datacontract/templates/index.html +2 -1
- {datacontract_cli-0.10.22.dist-info → datacontract_cli-0.10.24.dist-info}/METADATA +279 -193
- {datacontract_cli-0.10.22.dist-info → datacontract_cli-0.10.24.dist-info}/RECORD +37 -33
- {datacontract_cli-0.10.22.dist-info → datacontract_cli-0.10.24.dist-info}/WHEEL +1 -1
- datacontract/export/csv_type_converter.py +0 -36
- datacontract/lint/linters/quality_schema_linter.py +0 -52
- {datacontract_cli-0.10.22.dist-info → datacontract_cli-0.10.24.dist-info}/entry_points.txt +0 -0
- {datacontract_cli-0.10.22.dist-info → datacontract_cli-0.10.24.dist-info/licenses}/LICENSE +0 -0
- {datacontract_cli-0.10.22.dist-info → datacontract_cli-0.10.24.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.24
|
|
4
4
|
Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
|
|
5
5
|
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
|
|
6
6
|
Project-URL: Homepage, https://cli.datacontract.com
|
|
@@ -16,47 +16,46 @@ Requires-Dist: pydantic<2.11.0,>=2.8.2
|
|
|
16
16
|
Requires-Dist: pyyaml~=6.0.1
|
|
17
17
|
Requires-Dist: requests<2.33,>=2.31
|
|
18
18
|
Requires-Dist: fastjsonschema<2.22.0,>=2.19.1
|
|
19
|
-
Requires-Dist: fastparquet
|
|
19
|
+
Requires-Dist: fastparquet<2025.0.0,>=2024.5.0
|
|
20
20
|
Requires-Dist: numpy<2.0.0,>=1.26.4
|
|
21
|
-
Requires-Dist: python-multipart
|
|
22
|
-
Requires-Dist: rich<
|
|
21
|
+
Requires-Dist: python-multipart<1.0.0,>=0.0.20
|
|
22
|
+
Requires-Dist: rich<14.0,>=13.7
|
|
23
23
|
Requires-Dist: sqlglot<27.0.0,>=26.6.0
|
|
24
|
-
Requires-Dist: duckdb
|
|
25
|
-
Requires-Dist: soda-core-duckdb<3.
|
|
24
|
+
Requires-Dist: duckdb<2.0.0,>=1.0.0
|
|
25
|
+
Requires-Dist: soda-core-duckdb<3.6.0,>=3.3.20
|
|
26
26
|
Requires-Dist: setuptools>=60
|
|
27
|
-
Requires-Dist: python-dotenv
|
|
28
|
-
Requires-Dist: boto3<
|
|
29
|
-
Requires-Dist: Jinja2
|
|
30
|
-
Requires-Dist: jinja_partials
|
|
27
|
+
Requires-Dist: python-dotenv<2.0.0,>=1.0.0
|
|
28
|
+
Requires-Dist: boto3<2.0.0,>=1.34.41
|
|
29
|
+
Requires-Dist: Jinja2<4.0.0,>=3.1.5
|
|
30
|
+
Requires-Dist: jinja_partials<1.0.0,>=0.2.1
|
|
31
31
|
Provides-Extra: avro
|
|
32
32
|
Requires-Dist: avro==1.12.0; extra == "avro"
|
|
33
33
|
Provides-Extra: bigquery
|
|
34
|
-
Requires-Dist: soda-core-bigquery<3.
|
|
34
|
+
Requires-Dist: soda-core-bigquery<3.6.0,>=3.3.20; extra == "bigquery"
|
|
35
35
|
Provides-Extra: csv
|
|
36
|
-
Requires-Dist: clevercsv>=0.8.2; extra == "csv"
|
|
37
36
|
Requires-Dist: pandas>=2.0.0; extra == "csv"
|
|
38
37
|
Provides-Extra: databricks
|
|
39
|
-
Requires-Dist: soda-core-spark-df<3.
|
|
40
|
-
Requires-Dist: soda-core-spark[databricks]<3.
|
|
41
|
-
Requires-Dist: databricks-sql-connector<
|
|
42
|
-
Requires-Dist: databricks-sdk<0.
|
|
38
|
+
Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "databricks"
|
|
39
|
+
Requires-Dist: soda-core-spark[databricks]<3.6.0,>=3.3.20; extra == "databricks"
|
|
40
|
+
Requires-Dist: databricks-sql-connector<4.1.0,>=3.7.0; extra == "databricks"
|
|
41
|
+
Requires-Dist: databricks-sdk<0.50.0; extra == "databricks"
|
|
43
42
|
Provides-Extra: iceberg
|
|
44
43
|
Requires-Dist: pyiceberg==0.8.1; extra == "iceberg"
|
|
45
44
|
Provides-Extra: kafka
|
|
46
45
|
Requires-Dist: datacontract-cli[avro]; extra == "kafka"
|
|
47
|
-
Requires-Dist: soda-core-spark-df<3.
|
|
46
|
+
Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "kafka"
|
|
48
47
|
Provides-Extra: postgres
|
|
49
|
-
Requires-Dist: soda-core-postgres<3.
|
|
48
|
+
Requires-Dist: soda-core-postgres<3.6.0,>=3.3.20; extra == "postgres"
|
|
50
49
|
Provides-Extra: s3
|
|
51
|
-
Requires-Dist: s3fs==
|
|
50
|
+
Requires-Dist: s3fs==2025.2.0; extra == "s3"
|
|
52
51
|
Requires-Dist: aiobotocore<2.20.0,>=2.17.0; extra == "s3"
|
|
53
52
|
Provides-Extra: snowflake
|
|
54
|
-
Requires-Dist: snowflake-connector-python[pandas]<3.
|
|
55
|
-
Requires-Dist: soda-core-snowflake<3.
|
|
53
|
+
Requires-Dist: snowflake-connector-python[pandas]<3.15,>=3.6; extra == "snowflake"
|
|
54
|
+
Requires-Dist: soda-core-snowflake<3.6.0,>=3.3.20; extra == "snowflake"
|
|
56
55
|
Provides-Extra: sqlserver
|
|
57
|
-
Requires-Dist: soda-core-sqlserver<3.
|
|
56
|
+
Requires-Dist: soda-core-sqlserver<3.6.0,>=3.3.20; extra == "sqlserver"
|
|
58
57
|
Provides-Extra: trino
|
|
59
|
-
Requires-Dist: soda-core-trino<3.
|
|
58
|
+
Requires-Dist: soda-core-trino<3.6.0,>=3.3.20; extra == "trino"
|
|
60
59
|
Provides-Extra: dbt
|
|
61
60
|
Requires-Dist: dbt-core>=1.8.0; extra == "dbt"
|
|
62
61
|
Provides-Extra: dbml
|
|
@@ -66,23 +65,26 @@ Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
|
|
|
66
65
|
Provides-Extra: rdf
|
|
67
66
|
Requires-Dist: rdflib==7.0.0; extra == "rdf"
|
|
68
67
|
Provides-Extra: api
|
|
69
|
-
Requires-Dist: fastapi==0.115.
|
|
68
|
+
Requires-Dist: fastapi==0.115.12; extra == "api"
|
|
70
69
|
Requires-Dist: uvicorn==0.34.0; extra == "api"
|
|
70
|
+
Provides-Extra: protobuf
|
|
71
|
+
Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
|
|
71
72
|
Provides-Extra: all
|
|
72
|
-
Requires-Dist: datacontract-cli[api,bigquery,csv,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,rdf,s3,snowflake,sqlserver,trino]; extra == "all"
|
|
73
|
+
Requires-Dist: datacontract-cli[api,bigquery,csv,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,protobuf,rdf,s3,snowflake,sqlserver,trino]; extra == "all"
|
|
73
74
|
Provides-Extra: dev
|
|
74
75
|
Requires-Dist: datacontract-cli[all]; extra == "dev"
|
|
75
76
|
Requires-Dist: httpx==0.28.1; extra == "dev"
|
|
76
77
|
Requires-Dist: kafka-python; extra == "dev"
|
|
77
|
-
Requires-Dist: moto==5.
|
|
78
|
+
Requires-Dist: moto==5.1.3; extra == "dev"
|
|
78
79
|
Requires-Dist: pandas>=2.1.0; extra == "dev"
|
|
79
|
-
Requires-Dist: pre-commit<4.
|
|
80
|
+
Requires-Dist: pre-commit<4.3.0,>=3.7.1; extra == "dev"
|
|
80
81
|
Requires-Dist: pytest; extra == "dev"
|
|
81
82
|
Requires-Dist: pytest-xdist; extra == "dev"
|
|
82
83
|
Requires-Dist: pymssql==2.3.2; extra == "dev"
|
|
83
84
|
Requires-Dist: ruff; extra == "dev"
|
|
84
|
-
Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.
|
|
85
|
-
Requires-Dist: trino==0.
|
|
85
|
+
Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.2; extra == "dev"
|
|
86
|
+
Requires-Dist: trino==0.333.0; extra == "dev"
|
|
87
|
+
Dynamic: license-file
|
|
86
88
|
|
|
87
89
|
# Data Contract CLI
|
|
88
90
|
|
|
@@ -230,6 +232,12 @@ if not run.has_passed():
|
|
|
230
232
|
# Abort pipeline, alert, or take corrective actions...
|
|
231
233
|
```
|
|
232
234
|
|
|
235
|
+
## How to
|
|
236
|
+
|
|
237
|
+
- [How to integrate Data Contract CLI in your CI/CD pipeline as a GitHub Action](https://github.com/datacontract/datacontract-action/)
|
|
238
|
+
- [How to run the Data Contract CLI API to test data contracts with POST requests](https://cli.datacontract.com/API)
|
|
239
|
+
- [How to run Data Contract CLI in a Databricks pipeline](https://www.datamesh-architecture.com/howto/build-a-dataproduct-with-databricks#test-the-data-product)
|
|
240
|
+
|
|
233
241
|
|
|
234
242
|
## Installation
|
|
235
243
|
|
|
@@ -311,7 +319,7 @@ A list of available extras:
|
|
|
311
319
|
| Parquet | `pip install datacontract-cli[parquet]` |
|
|
312
320
|
| RDF | `pip install datacontract-cli[rdf]` |
|
|
313
321
|
| API (run as web server) | `pip install datacontract-cli[api]` |
|
|
314
|
-
|
|
322
|
+
| protobuf | `pip install datacontract-cli[protobuf]` |
|
|
315
323
|
|
|
316
324
|
|
|
317
325
|
## Documentation
|
|
@@ -335,17 +343,16 @@ Commands
|
|
|
335
343
|
|
|
336
344
|
Usage: datacontract init [OPTIONS] [LOCATION]
|
|
337
345
|
|
|
338
|
-
|
|
346
|
+
Create an empty data contract.
|
|
339
347
|
|
|
340
348
|
╭─ Arguments ──────────────────────────────────────────────────────────────────╮
|
|
341
|
-
│ location [LOCATION] The location
|
|
342
|
-
│
|
|
349
|
+
│ location [LOCATION] The location of the data contract file to │
|
|
350
|
+
│ create. │
|
|
343
351
|
│ [default: datacontract.yaml] │
|
|
344
352
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
345
353
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
346
354
|
│ --template TEXT URL of a template or data contract │
|
|
347
|
-
│ [default:
|
|
348
|
-
│ https://datacontract.com/datacontrac… │
|
|
355
|
+
│ [default: None] │
|
|
349
356
|
│ --overwrite --no-overwrite Replace the existing │
|
|
350
357
|
│ datacontract.yaml │
|
|
351
358
|
│ [default: no-overwrite] │
|
|
@@ -367,11 +374,18 @@ Commands
|
|
|
367
374
|
│ [default: datacontract.yaml] │
|
|
368
375
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
369
376
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
370
|
-
│ --schema
|
|
371
|
-
│
|
|
372
|
-
│
|
|
373
|
-
│
|
|
374
|
-
│
|
|
377
|
+
│ --schema TEXT The location (url or path) of the Data │
|
|
378
|
+
│ Contract Specification JSON Schema │
|
|
379
|
+
│ [default: None] │
|
|
380
|
+
│ --output PATH Specify the file path where the test results │
|
|
381
|
+
│ should be written to (e.g., │
|
|
382
|
+
│ './test-results/TEST-datacontract.xml'). If │
|
|
383
|
+
│ no path is provided, the output will be │
|
|
384
|
+
│ printed to stdout. │
|
|
385
|
+
│ [default: None] │
|
|
386
|
+
│ --output-format [junit] The target format for the test results. │
|
|
387
|
+
│ [default: None] │
|
|
388
|
+
│ --help Show this message and exit. │
|
|
375
389
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
376
390
|
|
|
377
391
|
```
|
|
@@ -389,36 +403,46 @@ Commands
|
|
|
389
403
|
│ [default: datacontract.yaml] │
|
|
390
404
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
391
405
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
392
|
-
│ --schema
|
|
393
|
-
│
|
|
394
|
-
│
|
|
395
|
-
│
|
|
396
|
-
│
|
|
397
|
-
│
|
|
398
|
-
│ --server
|
|
399
|
-
│
|
|
400
|
-
│
|
|
401
|
-
│
|
|
402
|
-
│
|
|
403
|
-
│
|
|
404
|
-
│
|
|
405
|
-
│
|
|
406
|
-
│
|
|
407
|
-
│
|
|
408
|
-
│
|
|
409
|
-
│
|
|
410
|
-
│ --publish
|
|
411
|
-
│
|
|
412
|
-
│
|
|
413
|
-
│
|
|
414
|
-
│
|
|
415
|
-
│
|
|
416
|
-
│
|
|
417
|
-
│
|
|
418
|
-
│
|
|
419
|
-
│
|
|
420
|
-
│ --
|
|
421
|
-
│
|
|
406
|
+
│ --schema TEXT The location (url or │
|
|
407
|
+
│ path) of the Data │
|
|
408
|
+
│ Contract │
|
|
409
|
+
│ Specification JSON │
|
|
410
|
+
│ Schema │
|
|
411
|
+
│ [default: None] │
|
|
412
|
+
│ --server TEXT The server │
|
|
413
|
+
│ configuration to run │
|
|
414
|
+
│ the schema and │
|
|
415
|
+
│ quality tests. Use │
|
|
416
|
+
│ the key of the server │
|
|
417
|
+
│ object in the data │
|
|
418
|
+
│ contract yaml file to │
|
|
419
|
+
│ refer to a server, │
|
|
420
|
+
│ e.g., `production`, │
|
|
421
|
+
│ or `all` for all │
|
|
422
|
+
│ servers (default). │
|
|
423
|
+
│ [default: all] │
|
|
424
|
+
│ --publish TEXT The url to publish │
|
|
425
|
+
│ the results after the │
|
|
426
|
+
│ test │
|
|
427
|
+
│ [default: None] │
|
|
428
|
+
│ --output PATH Specify the file path │
|
|
429
|
+
│ where the test │
|
|
430
|
+
│ results should be │
|
|
431
|
+
│ written to (e.g., │
|
|
432
|
+
│ './test-results/TEST… │
|
|
433
|
+
│ [default: None] │
|
|
434
|
+
│ --output-format [junit] The target format for │
|
|
435
|
+
│ the test results. │
|
|
436
|
+
│ [default: None] │
|
|
437
|
+
│ --logs --no-logs Print logs │
|
|
438
|
+
│ [default: no-logs] │
|
|
439
|
+
│ --ssl-verification --no-ssl-verificati… SSL verification when │
|
|
440
|
+
│ publishing the data │
|
|
441
|
+
│ contract. │
|
|
442
|
+
│ [default: │
|
|
443
|
+
│ ssl-verification] │
|
|
444
|
+
│ --help Show this message and │
|
|
445
|
+
│ exit. │
|
|
422
446
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
423
447
|
|
|
424
448
|
```
|
|
@@ -891,41 +915,40 @@ models:
|
|
|
891
915
|
│ [default: datacontract.yaml] │
|
|
892
916
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
893
917
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
894
|
-
│ * --format
|
|
895
|
-
│
|
|
896
|
-
│
|
|
897
|
-
│
|
|
898
|
-
│
|
|
899
|
-
│
|
|
900
|
-
│
|
|
901
|
-
│
|
|
902
|
-
│
|
|
903
|
-
│ --output
|
|
904
|
-
│
|
|
905
|
-
│
|
|
906
|
-
│
|
|
907
|
-
│
|
|
908
|
-
│
|
|
909
|
-
│ --server
|
|
910
|
-
│
|
|
911
|
-
│ --model
|
|
912
|
-
│
|
|
913
|
-
│
|
|
914
|
-
│
|
|
915
|
-
│
|
|
916
|
-
│
|
|
917
|
-
│ --schema
|
|
918
|
-
│
|
|
919
|
-
│
|
|
920
|
-
│
|
|
921
|
-
│
|
|
922
|
-
│
|
|
923
|
-
│
|
|
924
|
-
│
|
|
925
|
-
│
|
|
926
|
-
│
|
|
927
|
-
│
|
|
928
|
-
│ --help Show this message and exit. │
|
|
918
|
+
│ * --format [jsonschema|pydantic-model The export format. │
|
|
919
|
+
│ |sodacl|dbt|dbt-sources|db [default: None] │
|
|
920
|
+
│ t-staging-sql|odcs|rdf|avr [required] │
|
|
921
|
+
│ o|protobuf|great-expectati │
|
|
922
|
+
│ ons|terraform|avro-idl|sql │
|
|
923
|
+
│ |sql-query|html|go|bigquer │
|
|
924
|
+
│ y|dbml|spark|sqlalchemy|da │
|
|
925
|
+
│ ta-caterer|dcs|markdown|ic │
|
|
926
|
+
│ eberg|custom] │
|
|
927
|
+
│ --output PATH Specify the file path where │
|
|
928
|
+
│ the exported data will be │
|
|
929
|
+
│ saved. If no path is │
|
|
930
|
+
│ provided, the output will │
|
|
931
|
+
│ be printed to stdout. │
|
|
932
|
+
│ [default: None] │
|
|
933
|
+
│ --server TEXT The server name to export. │
|
|
934
|
+
│ [default: None] │
|
|
935
|
+
│ --model TEXT Use the key of the model in │
|
|
936
|
+
│ the data contract yaml file │
|
|
937
|
+
│ to refer to a model, e.g., │
|
|
938
|
+
│ `orders`, or `all` for all │
|
|
939
|
+
│ models (default). │
|
|
940
|
+
│ [default: all] │
|
|
941
|
+
│ --schema TEXT The location (url or path) │
|
|
942
|
+
│ of the Data Contract │
|
|
943
|
+
│ Specification JSON Schema │
|
|
944
|
+
│ [default: None] │
|
|
945
|
+
│ --engine TEXT [engine] The engine used │
|
|
946
|
+
│ for great expection run. │
|
|
947
|
+
│ [default: None] │
|
|
948
|
+
│ --template PATH [custom] The file path of │
|
|
949
|
+
│ Jinja template. │
|
|
950
|
+
│ [default: None] │
|
|
951
|
+
│ --help Show this message and exit. │
|
|
929
952
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
930
953
|
╭─ RDF Options ────────────────────────────────────────────────────────────────╮
|
|
931
954
|
│ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. │
|
|
@@ -1037,6 +1060,12 @@ The export function converts the logical data types of the datacontract into the
|
|
|
1037
1060
|
if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
|
|
1038
1061
|
logical data types are exported.
|
|
1039
1062
|
|
|
1063
|
+
#### DBT & DBT-SOURCES
|
|
1064
|
+
|
|
1065
|
+
The export funciton converts the datacontract to dbt models in YAML format, with support for SQL dialects.
|
|
1066
|
+
If a server is selected via the `--server` option (based on the `type` of that server) then the DBT column `data_types` match the expected data types of the server.
|
|
1067
|
+
If no server is selected, then it defaults to `snowflake`.
|
|
1068
|
+
|
|
1040
1069
|
#### Spark
|
|
1041
1070
|
|
|
1042
1071
|
The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
|
|
@@ -1228,71 +1257,104 @@ FROM
|
|
|
1228
1257
|
|
|
1229
1258
|
### import
|
|
1230
1259
|
```
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
│
|
|
1239
|
-
│
|
|
1240
|
-
│
|
|
1241
|
-
│
|
|
1242
|
-
│
|
|
1243
|
-
│
|
|
1244
|
-
│
|
|
1245
|
-
│
|
|
1246
|
-
│
|
|
1247
|
-
│
|
|
1248
|
-
│
|
|
1249
|
-
│
|
|
1250
|
-
│
|
|
1251
|
-
│
|
|
1252
|
-
│
|
|
1253
|
-
│
|
|
1254
|
-
│
|
|
1255
|
-
│
|
|
1256
|
-
│
|
|
1257
|
-
│
|
|
1258
|
-
│
|
|
1259
|
-
│
|
|
1260
|
-
│
|
|
1261
|
-
│ --
|
|
1262
|
-
│
|
|
1263
|
-
│
|
|
1264
|
-
│
|
|
1265
|
-
│
|
|
1266
|
-
│
|
|
1267
|
-
│
|
|
1268
|
-
│
|
|
1269
|
-
│ --
|
|
1270
|
-
│
|
|
1271
|
-
│
|
|
1272
|
-
│
|
|
1273
|
-
│
|
|
1274
|
-
│
|
|
1275
|
-
│
|
|
1276
|
-
│
|
|
1277
|
-
│
|
|
1278
|
-
│
|
|
1279
|
-
│
|
|
1280
|
-
│
|
|
1281
|
-
│
|
|
1282
|
-
│
|
|
1283
|
-
│
|
|
1284
|
-
│
|
|
1285
|
-
│
|
|
1286
|
-
│
|
|
1287
|
-
│
|
|
1288
|
-
│
|
|
1289
|
-
│
|
|
1290
|
-
│
|
|
1291
|
-
│
|
|
1292
|
-
│
|
|
1293
|
-
│
|
|
1294
|
-
│ --
|
|
1295
|
-
|
|
1260
|
+
|
|
1261
|
+
Usage: datacontract import [OPTIONS]
|
|
1262
|
+
|
|
1263
|
+
Create a data contract from the given source location. Saves to file specified
|
|
1264
|
+
by `output` option if present, otherwise prints to stdout.
|
|
1265
|
+
|
|
1266
|
+
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
1267
|
+
│ * --format [sql|avro|dbt|dbml|gl The format of the │
|
|
1268
|
+
│ ue|jsonschema|bigquer source file. │
|
|
1269
|
+
│ y|odcs|unity|spark|ic [default: None] │
|
|
1270
|
+
│ eberg|parquet|csv|pro [required] │
|
|
1271
|
+
│ tobuf] │
|
|
1272
|
+
│ --output PATH Specify the file path │
|
|
1273
|
+
│ where the Data │
|
|
1274
|
+
│ Contract will be │
|
|
1275
|
+
│ saved. If no path is │
|
|
1276
|
+
│ provided, the output │
|
|
1277
|
+
│ will be printed to │
|
|
1278
|
+
│ stdout. │
|
|
1279
|
+
│ [default: None] │
|
|
1280
|
+
│ --source TEXT The path to the file │
|
|
1281
|
+
│ or Glue Database that │
|
|
1282
|
+
│ should be imported. │
|
|
1283
|
+
│ [default: None] │
|
|
1284
|
+
│ --dialect TEXT The SQL dialect to │
|
|
1285
|
+
│ use when importing │
|
|
1286
|
+
│ SQL files, e.g., │
|
|
1287
|
+
│ postgres, tsql, │
|
|
1288
|
+
│ bigquery. │
|
|
1289
|
+
│ [default: None] │
|
|
1290
|
+
│ --glue-table TEXT List of table ids to │
|
|
1291
|
+
│ import from the Glue │
|
|
1292
|
+
│ Database (repeat for │
|
|
1293
|
+
│ multiple table ids, │
|
|
1294
|
+
│ leave empty for all │
|
|
1295
|
+
│ tables in the │
|
|
1296
|
+
│ dataset). │
|
|
1297
|
+
│ [default: None] │
|
|
1298
|
+
│ --bigquery-project TEXT The bigquery project │
|
|
1299
|
+
│ id. │
|
|
1300
|
+
│ [default: None] │
|
|
1301
|
+
│ --bigquery-dataset TEXT The bigquery dataset │
|
|
1302
|
+
│ id. │
|
|
1303
|
+
│ [default: None] │
|
|
1304
|
+
│ --bigquery-table TEXT List of table ids to │
|
|
1305
|
+
│ import from the │
|
|
1306
|
+
│ bigquery API (repeat │
|
|
1307
|
+
│ for multiple table │
|
|
1308
|
+
│ ids, leave empty for │
|
|
1309
|
+
│ all tables in the │
|
|
1310
|
+
│ dataset). │
|
|
1311
|
+
│ [default: None] │
|
|
1312
|
+
│ --unity-table-full-n… TEXT Full name of a table │
|
|
1313
|
+
│ in the unity catalog │
|
|
1314
|
+
│ [default: None] │
|
|
1315
|
+
│ --dbt-model TEXT List of models names │
|
|
1316
|
+
│ to import from the │
|
|
1317
|
+
│ dbt manifest file │
|
|
1318
|
+
│ (repeat for multiple │
|
|
1319
|
+
│ models names, leave │
|
|
1320
|
+
│ empty for all models │
|
|
1321
|
+
│ in the dataset). │
|
|
1322
|
+
│ [default: None] │
|
|
1323
|
+
│ --dbml-schema TEXT List of schema names │
|
|
1324
|
+
│ to import from the │
|
|
1325
|
+
│ DBML file (repeat for │
|
|
1326
|
+
│ multiple schema │
|
|
1327
|
+
│ names, leave empty │
|
|
1328
|
+
│ for all tables in the │
|
|
1329
|
+
│ file). │
|
|
1330
|
+
│ [default: None] │
|
|
1331
|
+
│ --dbml-table TEXT List of table names │
|
|
1332
|
+
│ to import from the │
|
|
1333
|
+
│ DBML file (repeat for │
|
|
1334
|
+
│ multiple table names, │
|
|
1335
|
+
│ leave empty for all │
|
|
1336
|
+
│ tables in the file). │
|
|
1337
|
+
│ [default: None] │
|
|
1338
|
+
│ --iceberg-table TEXT Table name to assign │
|
|
1339
|
+
│ to the model created │
|
|
1340
|
+
│ from the Iceberg │
|
|
1341
|
+
│ schema. │
|
|
1342
|
+
│ [default: None] │
|
|
1343
|
+
│ --template TEXT The location (url or │
|
|
1344
|
+
│ path) of the Data │
|
|
1345
|
+
│ Contract │
|
|
1346
|
+
│ Specification │
|
|
1347
|
+
│ Template │
|
|
1348
|
+
│ [default: None] │
|
|
1349
|
+
│ --schema TEXT The location (url or │
|
|
1350
|
+
│ path) of the Data │
|
|
1351
|
+
│ Contract │
|
|
1352
|
+
│ Specification JSON │
|
|
1353
|
+
│ Schema │
|
|
1354
|
+
│ [default: None] │
|
|
1355
|
+
│ --help Show this message and │
|
|
1356
|
+
│ exit. │
|
|
1357
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
1296
1358
|
|
|
1297
1359
|
```
|
|
1298
1360
|
|
|
@@ -1319,7 +1381,7 @@ Available import options:
|
|
|
1319
1381
|
| `spark` | Import from Spark StructTypes | ✅ |
|
|
1320
1382
|
| `dbml` | Import from DBML models | ✅ |
|
|
1321
1383
|
| `csv` | Import from CSV File | ✅ |
|
|
1322
|
-
| `protobuf` | Import from Protobuf schemas |
|
|
1384
|
+
| `protobuf` | Import from Protobuf schemas | ✅ |
|
|
1323
1385
|
| `iceberg` | Import from an Iceberg JSON Schema Definition | partial |
|
|
1324
1386
|
| `parquet` | Import from Parquet File Metadta | ✅ |
|
|
1325
1387
|
| Missing something? | Please create an issue on GitHub | TBD |
|
|
@@ -1471,6 +1533,16 @@ Example:
|
|
|
1471
1533
|
datacontract import --format csv --source "test.csv"
|
|
1472
1534
|
```
|
|
1473
1535
|
|
|
1536
|
+
#### protobuf
|
|
1537
|
+
|
|
1538
|
+
Importing from protobuf File. Specify file in `source` parameter.
|
|
1539
|
+
|
|
1540
|
+
Example:
|
|
1541
|
+
|
|
1542
|
+
```bash
|
|
1543
|
+
datacontract import --format protobuf --source "test.proto"
|
|
1544
|
+
```
|
|
1545
|
+
|
|
1474
1546
|
|
|
1475
1547
|
### breaking
|
|
1476
1548
|
```
|
|
@@ -1546,7 +1618,7 @@ datacontract import --format csv --source "test.csv"
|
|
|
1546
1618
|
|
|
1547
1619
|
Usage: datacontract catalog [OPTIONS]
|
|
1548
1620
|
|
|
1549
|
-
Create
|
|
1621
|
+
Create a html catalog of data contracts.
|
|
1550
1622
|
|
|
1551
1623
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
1552
1624
|
│ --files TEXT Glob pattern for the data contract files to include in │
|
|
@@ -1556,8 +1628,7 @@ datacontract import --format csv --source "test.csv"
|
|
|
1556
1628
|
│ [default: catalog/] │
|
|
1557
1629
|
│ --schema TEXT The location (url or path) of the Data Contract │
|
|
1558
1630
|
│ Specification JSON Schema │
|
|
1559
|
-
│ [default:
|
|
1560
|
-
│ https://datacontract.com/datacontract.schema.json] │
|
|
1631
|
+
│ [default: None] │
|
|
1561
1632
|
│ --help Show this message and exit. │
|
|
1562
1633
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
1563
1634
|
|
|
@@ -1590,8 +1661,7 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
1590
1661
|
│ path) of the Data │
|
|
1591
1662
|
│ Contract Specification │
|
|
1592
1663
|
│ JSON Schema │
|
|
1593
|
-
│ [default:
|
|
1594
|
-
│ https://datacontract.c… │
|
|
1664
|
+
│ [default: None] │
|
|
1595
1665
|
│ --ssl-verification --no-ssl-verification SSL verification when │
|
|
1596
1666
|
│ publishing the data │
|
|
1597
1667
|
│ contract. │
|
|
@@ -1605,21 +1675,27 @@ datacontract catalog --files "*.odcs.yaml"
|
|
|
1605
1675
|
|
|
1606
1676
|
### api
|
|
1607
1677
|
```
|
|
1608
|
-
|
|
1609
|
-
Usage: datacontract api [OPTIONS]
|
|
1610
|
-
|
|
1611
|
-
Start the datacontract CLI as server application with REST API.
|
|
1612
|
-
The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1678
|
+
|
|
1679
|
+
Usage: datacontract api [OPTIONS]
|
|
1680
|
+
|
|
1681
|
+
Start the datacontract CLI as server application with REST API.
|
|
1682
|
+
The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
|
|
1683
|
+
You can execute the commands directly from the Swagger UI.
|
|
1684
|
+
To protect the API, you can set the environment variable
|
|
1685
|
+
DATACONTRACT_CLI_API_KEY to a secret API key. To authenticate, requests must
|
|
1686
|
+
include the header 'x-api-key' with the correct API key. This is highly
|
|
1687
|
+
recommended, as data contract tests may be subject to SQL injections or leak
|
|
1688
|
+
sensitive information.
|
|
1689
|
+
To connect to servers (such as a Snowflake data source), set the credentials
|
|
1690
|
+
as environment variables as documented in https://cli.datacontract.com/#test
|
|
1691
|
+
|
|
1692
|
+
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
1693
|
+
│ --port INTEGER Bind socket to this port. [default: 4242] │
|
|
1694
|
+
│ --host TEXT Bind socket to this host. Hint: For running in │
|
|
1695
|
+
│ docker, set it to 0.0.0.0 │
|
|
1696
|
+
│ [default: 127.0.0.1] │
|
|
1697
|
+
│ --help Show this message and exit. │
|
|
1698
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
1623
1699
|
|
|
1624
1700
|
```
|
|
1625
1701
|
|
|
@@ -1902,7 +1978,7 @@ Python base interpreter should be 3.11.x (unless working on 3.12 release candida
|
|
|
1902
1978
|
|
|
1903
1979
|
```bash
|
|
1904
1980
|
# create venv
|
|
1905
|
-
python3 -m venv venv
|
|
1981
|
+
python3.11 -m venv venv
|
|
1906
1982
|
source venv/bin/activate
|
|
1907
1983
|
|
|
1908
1984
|
# Install Requirements
|
|
@@ -1913,6 +1989,15 @@ pre-commit run --all-files
|
|
|
1913
1989
|
pytest
|
|
1914
1990
|
```
|
|
1915
1991
|
|
|
1992
|
+
### Use uv (recommended)
|
|
1993
|
+
|
|
1994
|
+
```bash
|
|
1995
|
+
# make sure uv is installed
|
|
1996
|
+
uv python pin 3.11
|
|
1997
|
+
uv sync --all-extras
|
|
1998
|
+
uv run pytest
|
|
1999
|
+
```
|
|
2000
|
+
|
|
1916
2001
|
|
|
1917
2002
|
### Docker Build
|
|
1918
2003
|
|
|
@@ -1984,6 +2069,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
1984
2069
|
|
|
1985
2070
|
- [INNOQ](https://innoq.com)
|
|
1986
2071
|
- [Data Catering](https://data.catering/)
|
|
2072
|
+
- [Oliver Wyman](https://www.oliverwyman.com/)
|
|
1987
2073
|
- And many more. To add your company, please create a pull request.
|
|
1988
2074
|
|
|
1989
2075
|
## Related Tools
|