datacontract-cli 0.10.0__py3-none-any.whl → 0.10.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- datacontract/breaking/breaking_rules.py +4 -0
- datacontract/catalog/catalog.py +74 -0
- datacontract/cli.py +39 -3
- datacontract/data_contract.py +12 -1
- datacontract/engines/fastjsonschema/check_jsonschema.py +1 -2
- datacontract/engines/soda/check_soda_execute.py +9 -15
- datacontract/engines/soda/connections/duckdb.py +83 -14
- datacontract/engines/soda/connections/kafka.py +108 -105
- datacontract/export/avro_idl_converter.py +1 -2
- datacontract/export/dbt_converter.py +1 -2
- datacontract/export/great_expectations_converter.py +1 -2
- datacontract/export/html_export.py +3 -4
- datacontract/export/jsonschema_converter.py +1 -2
- datacontract/export/odcs_converter.py +1 -2
- datacontract/export/rdf_converter.py +1 -1
- datacontract/export/sodacl_converter.py +1 -2
- datacontract/export/terraform_converter.py +1 -2
- datacontract/imports/avro_importer.py +1 -2
- datacontract/imports/glue_importer.py +183 -0
- datacontract/imports/sql_importer.py +20 -9
- datacontract/integration/publish_opentelemetry.py +3 -6
- datacontract/lint/linters/example_model_linter.py +1 -2
- datacontract/lint/linters/field_pattern_linter.py +1 -2
- datacontract/lint/linters/notice_period_linter.py +1 -2
- datacontract/lint/linters/quality_schema_linter.py +1 -2
- datacontract/lint/resolve.py +9 -6
- datacontract/model/data_contract_specification.py +2 -0
- datacontract/templates/datacontract.html +76 -21
- datacontract/templates/style/output.css +113 -4
- {datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.2.dist-info}/METADATA +185 -107
- {datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.2.dist-info}/RECORD +35 -33
- {datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.2.dist-info}/LICENSE +0 -0
- {datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.2.dist-info}/WHEEL +0 -0
- {datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.2.dist-info}/entry_points.txt +0 -0
- {datacontract_cli-0.10.0.dist-info → datacontract_cli-0.10.2.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.2
|
|
4
4
|
Summary: Test data contracts
|
|
5
5
|
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
|
|
6
6
|
Project-URL: Homepage, https://cli.datacontract.com
|
|
@@ -19,7 +19,7 @@ Requires-Dist: fastapi ==0.110.1
|
|
|
19
19
|
Requires-Dist: fastparquet ==2024.2.0
|
|
20
20
|
Requires-Dist: python-multipart ==0.0.9
|
|
21
21
|
Requires-Dist: rich ~=13.7.0
|
|
22
|
-
Requires-Dist: simple-ddl-parser ==1.0
|
|
22
|
+
Requires-Dist: simple-ddl-parser ==1.1.0
|
|
23
23
|
Requires-Dist: soda-core-bigquery <3.4.0,>=3.3.1
|
|
24
24
|
Requires-Dist: soda-core-duckdb <3.4.0,>=3.3.1
|
|
25
25
|
Requires-Dist: soda-core-postgres <3.4.0,>=3.3.1
|
|
@@ -27,7 +27,7 @@ Requires-Dist: soda-core-snowflake <3.4.0,>=3.3.1
|
|
|
27
27
|
Requires-Dist: soda-core-spark[databricks] <3.4.0,>=3.3.1
|
|
28
28
|
Requires-Dist: soda-core-spark-df <3.4.0,>=3.3.1
|
|
29
29
|
Requires-Dist: snowflake-connector-python[pandas] <3.8,>=3.6
|
|
30
|
-
Requires-Dist: duckdb ==0.10.
|
|
30
|
+
Requires-Dist: duckdb ==0.10.2
|
|
31
31
|
Requires-Dist: fastjsonschema ~=2.19.1
|
|
32
32
|
Requires-Dist: python-dotenv ~=1.0.0
|
|
33
33
|
Requires-Dist: s3fs ==2024.3.1
|
|
@@ -35,12 +35,16 @@ Requires-Dist: rdflib ==7.0.0
|
|
|
35
35
|
Requires-Dist: avro ==1.11.3
|
|
36
36
|
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc ~=1.16.0
|
|
37
37
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http ~=1.16.0
|
|
38
|
+
Requires-Dist: deltalake ~=0.17.0
|
|
39
|
+
Requires-Dist: boto3 <1.34.70,>=1.34.41
|
|
40
|
+
Requires-Dist: botocore <1.34.70,>=1.34.41
|
|
38
41
|
Provides-Extra: dev
|
|
39
42
|
Requires-Dist: httpx ==0.27.0 ; extra == 'dev'
|
|
40
43
|
Requires-Dist: ruff ; extra == 'dev'
|
|
41
44
|
Requires-Dist: pytest ; extra == 'dev'
|
|
42
45
|
Requires-Dist: pytest-xdist ; extra == 'dev'
|
|
43
|
-
Requires-Dist:
|
|
46
|
+
Requires-Dist: moto ; extra == 'dev'
|
|
47
|
+
Requires-Dist: testcontainers <4.0.0 ; extra == 'dev'
|
|
44
48
|
Requires-Dist: testcontainers-minio ; extra == 'dev'
|
|
45
49
|
Requires-Dist: testcontainers-postgres ; extra == 'dev'
|
|
46
50
|
Requires-Dist: testcontainers-kafka ; extra == 'dev'
|
|
@@ -48,10 +52,10 @@ Requires-Dist: testcontainers-kafka ; extra == 'dev'
|
|
|
48
52
|
# Data Contract CLI
|
|
49
53
|
|
|
50
54
|
<p>
|
|
51
|
-
<a href="https://github.com/datacontract/cli/actions/workflows/ci.yaml?query=branch%3Amain">
|
|
52
|
-
<img alt="Test Workflow" src="https://img.shields.io/github/actions/workflow/status/datacontract/cli/ci.yaml?branch=main"></a>
|
|
53
|
-
<a href="https://github.com/datacontract/cli">
|
|
54
|
-
<img alt="Stars" src="https://img.shields.io/github/stars/datacontract/cli" /></a>
|
|
55
|
+
<a href="https://github.com/datacontract/datacontract-cli/actions/workflows/ci.yaml?query=branch%3Amain">
|
|
56
|
+
<img alt="Test Workflow" src="https://img.shields.io/github/actions/workflow/status/datacontract/datacontract-cli/ci.yaml?branch=main"></a>
|
|
57
|
+
<a href="https://github.com/datacontract/datacontract-cli">
|
|
58
|
+
<img alt="Stars" src="https://img.shields.io/github/stars/datacontract/datacontract-cli" /></a>
|
|
55
59
|
<a href="https://datacontract.com/slack" rel="nofollow"><img src="https://camo.githubusercontent.com/5ade1fd1e76a6ab860802cdd2941fe2501e2ca2cb534e5d8968dbf864c13d33d/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f736c61636b2d6a6f696e5f636861742d77686974652e7376673f6c6f676f3d736c61636b267374796c653d736f6369616c" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social" style="max-width: 100%;"></a>
|
|
56
60
|
</p>
|
|
57
61
|
|
|
@@ -136,7 +140,13 @@ CREATE TABLE line_items (
|
|
|
136
140
|
);
|
|
137
141
|
```
|
|
138
142
|
|
|
139
|
-
Or generate
|
|
143
|
+
Or generate an HTML export:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
$ datacontract export --format html https://datacontract.com/examples/orders-latest/datacontract.yaml > datacontract.html
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).
|
|
140
150
|
|
|
141
151
|
## Usage
|
|
142
152
|
|
|
@@ -224,6 +234,7 @@ Commands
|
|
|
224
234
|
- [breaking](#breaking)
|
|
225
235
|
- [changelog](#changelog)
|
|
226
236
|
- [diff](#diff)
|
|
237
|
+
- [catalog](#catalog)
|
|
227
238
|
|
|
228
239
|
### init
|
|
229
240
|
|
|
@@ -296,46 +307,49 @@ Commands
|
|
|
296
307
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
297
308
|
```
|
|
298
309
|
|
|
299
|
-
Data Contract CLI
|
|
310
|
+
Data Contract CLI connects to a data source and runs schema and quality tests to verify that the data contract is valid.
|
|
300
311
|
|
|
301
312
|
```bash
|
|
302
313
|
$ datacontract test --server production datacontract.yaml
|
|
303
314
|
```
|
|
304
315
|
|
|
305
|
-
To connect to the databases the `server` block in the datacontract.yaml is used to set up the connection.
|
|
316
|
+
To connect to the databases the `server` block in the datacontract.yaml is used to set up the connection.
|
|
317
|
+
In addition, credentials, such as username and passwords, may be defined with environment variables.
|
|
306
318
|
|
|
307
319
|
The application uses different engines, based on the server `type`.
|
|
308
|
-
Internally, it connects with DuckDB, Spark, or a native connection and executes the most tests with
|
|
309
|
-
|
|
320
|
+
Internally, it connects with DuckDB, Spark, or a native connection and executes the most tests with _soda-core_ and _fastjsonschema_.
|
|
321
|
+
|
|
322
|
+
Credentials are provided with environment variables.
|
|
310
323
|
|
|
311
324
|
Supported server types:
|
|
312
325
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
Feel free to create an issue, if you need support for an additional type.
|
|
326
|
+
- [s3](#S3)
|
|
327
|
+
- [bigquery](#bigquery)
|
|
328
|
+
- [azure](#azure)
|
|
329
|
+
- [databricks](#databricks)
|
|
330
|
+
- [databricks (programmatic)](#databricks-programmatic)
|
|
331
|
+
- [snowflake](#snowflake)
|
|
332
|
+
- [kafka](#kafka)
|
|
333
|
+
- [postgres](#postgres)
|
|
334
|
+
- [local](#local)
|
|
335
|
+
|
|
336
|
+
Supported formats:
|
|
337
|
+
|
|
338
|
+
- parquet
|
|
339
|
+
- json
|
|
340
|
+
- csv
|
|
341
|
+
- delta
|
|
342
|
+
- iceberg (coming soon)
|
|
343
|
+
|
|
344
|
+
Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/issues), if you need support for an additional type and formats.
|
|
333
345
|
|
|
334
346
|
### S3
|
|
335
347
|
|
|
336
348
|
Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
|
|
337
349
|
|
|
338
|
-
####
|
|
350
|
+
#### Examples
|
|
351
|
+
|
|
352
|
+
##### JSON
|
|
339
353
|
|
|
340
354
|
datacontract.yaml
|
|
341
355
|
```yaml
|
|
@@ -348,6 +362,18 @@ servers:
|
|
|
348
362
|
delimiter: new_line # new_line, array, or none
|
|
349
363
|
```
|
|
350
364
|
|
|
365
|
+
##### Delta Tables
|
|
366
|
+
|
|
367
|
+
datacontract.yaml
|
|
368
|
+
```yaml
|
|
369
|
+
servers:
|
|
370
|
+
production:
|
|
371
|
+
type: s3
|
|
372
|
+
endpointUrl: https://minio.example.com # not needed with AWS S3
|
|
373
|
+
location: s3://bucket-name/path/table.delta # path to the Delta table folder containing parquet data files and the _delta_log
|
|
374
|
+
format: delta
|
|
375
|
+
```
|
|
376
|
+
|
|
351
377
|
#### Environment Variables
|
|
352
378
|
|
|
353
379
|
| Environment Variable | Example | Description |
|
|
@@ -357,76 +383,40 @@ servers:
|
|
|
357
383
|
| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
|
|
358
384
|
|
|
359
385
|
|
|
360
|
-
### Postgres
|
|
361
|
-
|
|
362
|
-
Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
|
|
363
|
-
|
|
364
|
-
#### Example
|
|
365
|
-
|
|
366
|
-
datacontract.yaml
|
|
367
|
-
```yaml
|
|
368
|
-
servers:
|
|
369
|
-
postgres:
|
|
370
|
-
type: postgres
|
|
371
|
-
host: localhost
|
|
372
|
-
port: 5432
|
|
373
|
-
database: postgres
|
|
374
|
-
schema: public
|
|
375
|
-
models:
|
|
376
|
-
my_table_1: # corresponds to a table
|
|
377
|
-
type: table
|
|
378
|
-
fields:
|
|
379
|
-
my_column_1: # corresponds to a column
|
|
380
|
-
type: varchar
|
|
381
|
-
```
|
|
382
|
-
|
|
383
|
-
#### Environment Variables
|
|
384
|
-
|
|
385
|
-
| Environment Variable | Example | Description |
|
|
386
|
-
|----------------------------------|--------------------|-------------|
|
|
387
|
-
| `DATACONTRACT_POSTGRES_USERNAME` | `postgres` | Username |
|
|
388
|
-
| `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
|
|
389
386
|
|
|
387
|
+
### BigQuery
|
|
390
388
|
|
|
391
|
-
|
|
389
|
+
We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
|
|
390
|
+
* BigQuery Job User
|
|
391
|
+
* BigQuery Data Viewer
|
|
392
392
|
|
|
393
|
-
Data Contract CLI can test data in Snowflake.
|
|
394
393
|
|
|
395
394
|
#### Example
|
|
396
395
|
|
|
397
396
|
datacontract.yaml
|
|
398
397
|
```yaml
|
|
399
|
-
|
|
400
398
|
servers:
|
|
401
|
-
|
|
402
|
-
type:
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
schema: ORDERS_PII_V2
|
|
399
|
+
production:
|
|
400
|
+
type: bigquery
|
|
401
|
+
project: datameshexample-product
|
|
402
|
+
dataset: datacontract_cli_test_dataset
|
|
406
403
|
models:
|
|
407
|
-
|
|
404
|
+
datacontract_cli_test_table: # corresponds to a BigQuery table
|
|
408
405
|
type: table
|
|
409
|
-
fields:
|
|
410
|
-
my_column_1: # corresponds to a column
|
|
411
|
-
type: varchar
|
|
406
|
+
fields: ...
|
|
412
407
|
```
|
|
413
408
|
|
|
414
409
|
#### Environment Variables
|
|
415
410
|
|
|
416
|
-
| Environment Variable
|
|
417
|
-
|
|
418
|
-
| `
|
|
419
|
-
| `DATACONTRACT_SNOWFLAKE_PASSWORD` | `mysecretpassword` | Password |
|
|
420
|
-
| `DATACONTRACT_SNOWFLAKE_ROLE` | `DATAVALIDATION` | The snowflake role to use. |
|
|
421
|
-
| `DATACONTRACT_SNOWFLAKE_WAREHOUSE` | `COMPUTE_WH` | The Snowflake Warehouse to use executing the tests. |
|
|
411
|
+
| Environment Variable | Example | Description |
|
|
412
|
+
|----------------------------------------------|---------------------------|---------------------------------------------------------|
|
|
413
|
+
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
|
|
422
414
|
|
|
423
415
|
|
|
424
|
-
### BigQuery
|
|
425
416
|
|
|
426
|
-
|
|
427
|
-
* BigQuery Job User
|
|
428
|
-
* BigQuery Data Viewer
|
|
417
|
+
### Azure
|
|
429
418
|
|
|
419
|
+
Data Contract CLI can test data that is stored in Azure Blob storage or Azure Data Lake Storage (Gen2) (ADLS) in various formats.
|
|
430
420
|
|
|
431
421
|
#### Example
|
|
432
422
|
|
|
@@ -434,20 +424,21 @@ datacontract.yaml
|
|
|
434
424
|
```yaml
|
|
435
425
|
servers:
|
|
436
426
|
production:
|
|
437
|
-
type:
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
models:
|
|
441
|
-
datacontract_cli_test_table: # corresponds to a BigQuery table
|
|
442
|
-
type: table
|
|
443
|
-
fields: ...
|
|
427
|
+
type: azure
|
|
428
|
+
location: abfss://datameshdatabricksdemo.dfs.core.windows.net/dataproducts/inventory_events/*.parquet
|
|
429
|
+
format: parquet
|
|
444
430
|
```
|
|
445
431
|
|
|
446
432
|
#### Environment Variables
|
|
447
433
|
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
|
434
|
+
Authentication works with an Azure Service Principal (SPN) aka App Registration with a secret.
|
|
435
|
+
|
|
436
|
+
| Environment Variable | Example | Description |
|
|
437
|
+
|-----------------------------------|-------------------------------|------------------------------------------------------|
|
|
438
|
+
| `DATACONTRACT_AZURE_TENANT_ID` | `79f5b80f-10ff-40b9-9d1f-774b42d605fc` | The Azure Tenant ID |
|
|
439
|
+
| `DATACONTRACT_AZURE_CLIENT_ID` | `3cf7ce49-e2e9-4cbc-a922-4328d4a58622` | The ApplicationID / ClientID of the app registration |
|
|
440
|
+
| `DATACONTRACT_AZURE_CLIENT_SECRET` | `yZK8Q~GWO1MMXXXXXXXXXXXXX` | The Client Secret value |
|
|
441
|
+
|
|
451
442
|
|
|
452
443
|
|
|
453
444
|
### Databricks
|
|
@@ -518,6 +509,41 @@ run = data_contract.test()
|
|
|
518
509
|
run.result
|
|
519
510
|
```
|
|
520
511
|
|
|
512
|
+
|
|
513
|
+
### Snowflake
|
|
514
|
+
|
|
515
|
+
Data Contract CLI can test data in Snowflake.
|
|
516
|
+
|
|
517
|
+
#### Example
|
|
518
|
+
|
|
519
|
+
datacontract.yaml
|
|
520
|
+
```yaml
|
|
521
|
+
|
|
522
|
+
servers:
|
|
523
|
+
snowflake:
|
|
524
|
+
type: snowflake
|
|
525
|
+
account: abcdefg-xn12345
|
|
526
|
+
database: ORDER_DB
|
|
527
|
+
schema: ORDERS_PII_V2
|
|
528
|
+
models:
|
|
529
|
+
my_table_1: # corresponds to a table
|
|
530
|
+
type: table
|
|
531
|
+
fields:
|
|
532
|
+
my_column_1: # corresponds to a column
|
|
533
|
+
type: varchar
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
#### Environment Variables
|
|
537
|
+
|
|
538
|
+
| Environment Variable | Example | Description |
|
|
539
|
+
|------------------------------------|--------------------|-----------------------------------------------------|
|
|
540
|
+
| `DATACONTRACT_SNOWFLAKE_USERNAME` | `datacontract` | Username |
|
|
541
|
+
| `DATACONTRACT_SNOWFLAKE_PASSWORD` | `mysecretpassword` | Password |
|
|
542
|
+
| `DATACONTRACT_SNOWFLAKE_ROLE` | `DATAVALIDATION` | The snowflake role to use. |
|
|
543
|
+
| `DATACONTRACT_SNOWFLAKE_WAREHOUSE` | `COMPUTE_WH` | The Snowflake Warehouse to use executing the tests. |
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
|
|
521
547
|
### Kafka
|
|
522
548
|
|
|
523
549
|
Kafka support is currently considered experimental.
|
|
@@ -542,13 +568,44 @@ servers:
|
|
|
542
568
|
| `DATACONTRACT_KAFKA_SASL_PASSWORD` | `xxx` | The SASL password (secret). |
|
|
543
569
|
|
|
544
570
|
|
|
571
|
+
### Postgres
|
|
572
|
+
|
|
573
|
+
Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
|
|
574
|
+
|
|
575
|
+
#### Example
|
|
576
|
+
|
|
577
|
+
datacontract.yaml
|
|
578
|
+
```yaml
|
|
579
|
+
servers:
|
|
580
|
+
postgres:
|
|
581
|
+
type: postgres
|
|
582
|
+
host: localhost
|
|
583
|
+
port: 5432
|
|
584
|
+
database: postgres
|
|
585
|
+
schema: public
|
|
586
|
+
models:
|
|
587
|
+
my_table_1: # corresponds to a table
|
|
588
|
+
type: table
|
|
589
|
+
fields:
|
|
590
|
+
my_column_1: # corresponds to a column
|
|
591
|
+
type: varchar
|
|
592
|
+
```
|
|
593
|
+
|
|
594
|
+
#### Environment Variables
|
|
595
|
+
|
|
596
|
+
| Environment Variable | Example | Description |
|
|
597
|
+
|----------------------------------|--------------------|-------------|
|
|
598
|
+
| `DATACONTRACT_POSTGRES_USERNAME` | `postgres` | Username |
|
|
599
|
+
| `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
|
|
600
|
+
|
|
601
|
+
|
|
545
602
|
|
|
546
603
|
### export
|
|
547
604
|
|
|
548
605
|
```
|
|
549
606
|
Usage: datacontract export [OPTIONS] [LOCATION]
|
|
550
607
|
|
|
551
|
-
Convert data contract to a specific format.
|
|
608
|
+
Convert data contract to a specific format. Prints to stdout or to the specified output file.
|
|
552
609
|
|
|
553
610
|
╭─ Arguments ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
554
611
|
│ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
|
|
@@ -641,15 +698,15 @@ data products, find the true domain owner of a field attribute)
|
|
|
641
698
|
### import
|
|
642
699
|
|
|
643
700
|
```
|
|
644
|
-
Usage: datacontract import [OPTIONS]
|
|
645
|
-
|
|
646
|
-
Create a data contract from the given source
|
|
647
|
-
|
|
648
|
-
╭─ Options
|
|
649
|
-
│ * --format [sql|avro] The format of the source file. [default: None] [required]
|
|
650
|
-
│ * --source TEXT
|
|
651
|
-
│ --help
|
|
652
|
-
|
|
701
|
+
Usage: datacontract import [OPTIONS]
|
|
702
|
+
|
|
703
|
+
Create a data contract from the given source location. Prints to stdout.
|
|
704
|
+
|
|
705
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
706
|
+
│ * --format [sql|avro|glue] The format of the source file. [default: None] [required] │
|
|
707
|
+
│ * --source TEXT The path to the file or Glue Database that should be imported. [default: None] [required] │
|
|
708
|
+
│ --help Show this message and exit. │
|
|
709
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
653
710
|
```
|
|
654
711
|
|
|
655
712
|
Example:
|
|
@@ -664,6 +721,7 @@ Available import options:
|
|
|
664
721
|
|--------------------|------------------------------------------------|---------|
|
|
665
722
|
| `sql` | Import from SQL DDL | ✅ |
|
|
666
723
|
| `avro` | Import from AVRO schemas | ✅ |
|
|
724
|
+
| `glue` | Import from AWS Glue DataCatalog | ✅ |
|
|
667
725
|
| `protobuf` | Import from Protobuf schemas | TBD |
|
|
668
726
|
| `jsonschema` | Import from JSON Schemas | TBD |
|
|
669
727
|
| `bigquery` | Import from BigQuery Schemas | TBD |
|
|
@@ -720,6 +778,21 @@ Available import options:
|
|
|
720
778
|
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
721
779
|
```
|
|
722
780
|
|
|
781
|
+
### catalog
|
|
782
|
+
|
|
783
|
+
```
|
|
784
|
+
|
|
785
|
+
Usage: datacontract catalog [OPTIONS]
|
|
786
|
+
|
|
787
|
+
Create an html catalog of data contracts.
|
|
788
|
+
|
|
789
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
790
|
+
│ --files TEXT Glob pattern for the data contract files to include in the catalog. [default: *.yaml] │
|
|
791
|
+
│ --output TEXT Output directory for the catalog html files. [default: catalog/] │
|
|
792
|
+
│ --help Show this message and exit. │
|
|
793
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
794
|
+
```
|
|
795
|
+
|
|
723
796
|
|
|
724
797
|
## Integrations
|
|
725
798
|
|
|
@@ -887,7 +960,7 @@ source venv/bin/activate
|
|
|
887
960
|
pip install --upgrade pip setuptools wheel
|
|
888
961
|
pip install -e '.[dev]'
|
|
889
962
|
ruff check --fix
|
|
890
|
-
ruff format
|
|
963
|
+
ruff format
|
|
891
964
|
pytest
|
|
892
965
|
```
|
|
893
966
|
|
|
@@ -938,6 +1011,11 @@ This command runs the container momentarily to check the version of the `datacon
|
|
|
938
1011
|
|
|
939
1012
|
We are happy to receive your contributions. Propose your change in an issue or directly create a pull request with your improvements.
|
|
940
1013
|
|
|
1014
|
+
## Companies using this tool
|
|
1015
|
+
|
|
1016
|
+
- [INNOQ](https://innoq.com)
|
|
1017
|
+
- And many more. To add your company, please create a pull request.
|
|
1018
|
+
|
|
941
1019
|
## License
|
|
942
1020
|
|
|
943
1021
|
[MIT License](LICENSE)
|
|
@@ -948,4 +1026,4 @@ Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/)
|
|
|
948
1026
|
|
|
949
1027
|
|
|
950
1028
|
|
|
951
|
-
<a href="https://github.com/datacontract/cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
|
|
1029
|
+
<a href="https://github.com/datacontract/datacontract-cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
|
|
@@ -1,66 +1,68 @@
|
|
|
1
1
|
datacontract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
datacontract/cli.py,sha256=
|
|
3
|
-
datacontract/data_contract.py,sha256
|
|
2
|
+
datacontract/cli.py,sha256=MEeXCL4HjLTwc91KYzjKEg1s5RHgX8fm2OBNqcZxFmg,11855
|
|
3
|
+
datacontract/data_contract.py,sha256=NonmJ1YIKRtQXUanft30WJt8BKh67Si-71jOw6HIsic,22230
|
|
4
4
|
datacontract/web.py,sha256=SWglmbqy3NV5h4VDsG0OpwhASJT9uve7w6FPwkBjIYM,457
|
|
5
5
|
datacontract/breaking/breaking.py,sha256=Abw59Xd_CypS6Us-hvsGnPvCUo-a-cl8RzDCmjG4yfk,11719
|
|
6
|
-
datacontract/breaking/breaking_rules.py,sha256=
|
|
6
|
+
datacontract/breaking/breaking_rules.py,sha256=0ExDYOj5Clyq6sIRIXTpxSbcRxMHYrMqJXqZ92goKsA,3032
|
|
7
|
+
datacontract/catalog/catalog.py,sha256=O471eXvdl-ttRG2qDEHn1hb7Drr4o68d_86MFKBf7JA,2458
|
|
7
8
|
datacontract/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
9
|
datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py,sha256=TXO47ON3NjwYI4Y2eBYklMOCo7vAtYzqLPAhZhii6dg,1565
|
|
9
10
|
datacontract/engines/datacontract/check_that_datacontract_file_exists.py,sha256=XHvxX6BrTdZk2wN55zd6ER4k7ILDyGzjbCbZxQAJ2iE,665
|
|
10
11
|
datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py,sha256=CsxFGyInjpsylmoofByV1b-wpFhG1jtRWxSUpp-BXk8,1525
|
|
11
|
-
datacontract/engines/fastjsonschema/check_jsonschema.py,sha256=
|
|
12
|
+
datacontract/engines/fastjsonschema/check_jsonschema.py,sha256=rXGfGDu9-RJomj9WcVe8vEfluR25vn2rOS7BeOVQ0XA,5748
|
|
12
13
|
datacontract/engines/fastjsonschema/s3/s3_read_files.py,sha256=iupiyqBa1dzgT2BtVGna-BjC5rqe6MTLs2QRp8GTs7M,665
|
|
13
14
|
datacontract/engines/soda/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
datacontract/engines/soda/check_soda_execute.py,sha256=
|
|
15
|
+
datacontract/engines/soda/check_soda_execute.py,sha256=p1C6B3rxIroluxSGfcBmf7jH2xOih4JLHv2xMlP915E,6656
|
|
15
16
|
datacontract/engines/soda/connections/bigquery.py,sha256=Ao0KaJe4R28auU_4umxvVaLB6ZHEbKaNoYZ-RfAUmeo,662
|
|
16
17
|
datacontract/engines/soda/connections/dask.py,sha256=Yy6Et2n_vDVsdjtqyBWDSZt7mnjPzPk_MZ-92VZHfnY,1496
|
|
17
18
|
datacontract/engines/soda/connections/databricks.py,sha256=lpMju-o_TzLZeF0EEVwePPr8JahqvFnj5xRYjF15fc8,561
|
|
18
|
-
datacontract/engines/soda/connections/duckdb.py,sha256=
|
|
19
|
-
datacontract/engines/soda/connections/kafka.py,sha256=
|
|
19
|
+
datacontract/engines/soda/connections/duckdb.py,sha256=tVKP7YaAIOajM7LVeW290m_3UUfrJ7M_OWis3DmFf2Q,5586
|
|
20
|
+
datacontract/engines/soda/connections/kafka.py,sha256=TZGJIDcyqeIXKYZomHRc9gIJMoV-etD0ewVfKYQbNMU,5529
|
|
20
21
|
datacontract/engines/soda/connections/postgres.py,sha256=9GTF4Es3M5vb7ocSGqAxXmslvkS5CjsPQGIuo020CFc,626
|
|
21
22
|
datacontract/engines/soda/connections/snowflake.py,sha256=y1t2a1DWY4_tr5k-X5_nhLE6v1rfCwTahzhtHR91x9A,719
|
|
22
23
|
datacontract/export/avro_converter.py,sha256=fFrKGDjC48RBF9-Xy2w3Eh3BXxSkmflOkXPTs881Ows,2320
|
|
23
|
-
datacontract/export/avro_idl_converter.py,sha256=
|
|
24
|
+
datacontract/export/avro_idl_converter.py,sha256=_2acoImuBqNqEt97OpBSewWT_w3aDBOdNosuy0gbkSY,9576
|
|
24
25
|
datacontract/export/csv_type_converter.py,sha256=ZZuJwBgQnafZC7PPvAXsBf2IajPJq8TYZ1l8Qq0GYeI,1290
|
|
25
|
-
datacontract/export/dbt_converter.py,sha256=
|
|
26
|
-
datacontract/export/great_expectations_converter.py,sha256=
|
|
27
|
-
datacontract/export/html_export.py,sha256=
|
|
28
|
-
datacontract/export/jsonschema_converter.py,sha256=
|
|
29
|
-
datacontract/export/odcs_converter.py,sha256=
|
|
26
|
+
datacontract/export/dbt_converter.py,sha256=7ruAOEh5OgWZcmGsN4pjD5EQE6b_RVzRwOPAJRdrvyE,8543
|
|
27
|
+
datacontract/export/great_expectations_converter.py,sha256=nJwYgyQhvbApZDZ54hPFKVoVeEOWSA7Zxf6_ZIsapdI,4890
|
|
28
|
+
datacontract/export/html_export.py,sha256=n0dhxgqC36Ho6mVBmHV6cMoTMW_fQywD9nXyTwvAK9Q,2232
|
|
29
|
+
datacontract/export/jsonschema_converter.py,sha256=wN2Zx8wXOyaXsSt1azlDS_4ctFUZ9EKTgJ2RRq5Rqy4,3147
|
|
30
|
+
datacontract/export/odcs_converter.py,sha256=MHvIbvJKI-OB5JJaobCcegkvn_GxVhfTLoRDXRm133w,3726
|
|
30
31
|
datacontract/export/protobuf_converter.py,sha256=K40yEdfRLYDLUx8bn3_an-cwdHfWv8_SBgzGLJT6mX4,2974
|
|
31
32
|
datacontract/export/pydantic_converter.py,sha256=hlpi-qRNKd3ecnvO_kelCaODbHdMHiTZ0AG8BV-pxzc,5351
|
|
32
|
-
datacontract/export/rdf_converter.py,sha256=
|
|
33
|
-
datacontract/export/sodacl_converter.py,sha256=
|
|
33
|
+
datacontract/export/rdf_converter.py,sha256=Q7VVcRjoBJmXBFFM8qeORttA2VkucaDxIQopqwe_Ukg,6109
|
|
34
|
+
datacontract/export/sodacl_converter.py,sha256=BCP7FxhK_OV9lMf9mU7DjWf1k9MzTqfvtdQ6XFeXTlA,6738
|
|
34
35
|
datacontract/export/sql_converter.py,sha256=OZNBGBShmWrXip5uy6LsyDkb2UmbDKhUcZP4apNNNFI,3754
|
|
35
36
|
datacontract/export/sql_type_converter.py,sha256=kiFFX54dfSgXXqbUCtFz0ujEt0Ie5JnuuWbcvJc0kos,4627
|
|
36
|
-
datacontract/export/terraform_converter.py,sha256
|
|
37
|
-
datacontract/imports/avro_importer.py,sha256=
|
|
38
|
-
datacontract/imports/
|
|
37
|
+
datacontract/export/terraform_converter.py,sha256=-xIIspVrvCyB2AVf1vd7bVGkWI3iiMUHX1btM_o1h-g,1943
|
|
38
|
+
datacontract/imports/avro_importer.py,sha256=jILYE8EWuat1W_LLt7HuJTmcaBOmDVKvx4ZswphlVkc,4794
|
|
39
|
+
datacontract/imports/glue_importer.py,sha256=9wvX5jE6hPYNW5eoqcFIjySb41q-vbUW-j9XHYKN18o,5265
|
|
40
|
+
datacontract/imports/sql_importer.py,sha256=HD9NL0A965Q1NCklHJHM0Vowx6dKVF6MQ74VqL2J8Ig,2431
|
|
39
41
|
datacontract/init/download_datacontract_file.py,sha256=pj_4mhWKlEtfueWohDgkb1nyuG5ERDipUDszxKwpZUs,413
|
|
40
42
|
datacontract/integration/publish_datamesh_manager.py,sha256=3bWgWrdcJip0V3EzlLmuxBzAQBn6evEHbhrAKx1qEjY,1300
|
|
41
|
-
datacontract/integration/publish_opentelemetry.py,sha256=
|
|
43
|
+
datacontract/integration/publish_opentelemetry.py,sha256=PjU9XH2wgisg0dBP34bUvQazxVZjp37Zfck_xEu5sPQ,3812
|
|
42
44
|
datacontract/lint/files.py,sha256=tg0vq_w4LQsEr_8A5qr4hUJmHeGalUpsXJXC1t-OGC0,471
|
|
43
45
|
datacontract/lint/lint.py,sha256=GhgBwfDgihXfWIu1WUfYeJoxfYacVJrwq3L2KO4Z2Io,5076
|
|
44
|
-
datacontract/lint/resolve.py,sha256=
|
|
46
|
+
datacontract/lint/resolve.py,sha256=lDZoy5cNb_eIbPDruBQsrxyo5tiUCbEvsCLxGLnCfU0,6471
|
|
45
47
|
datacontract/lint/schema.py,sha256=FIzubF1C9TnhEQBeoDsLuhc6HymCMx_v435Hjbcz4U8,838
|
|
46
48
|
datacontract/lint/urls.py,sha256=NHTbwadnm1VCxsiF7clfDvsxJ8-D3Mi5AzDj029E6wQ,1325
|
|
47
49
|
datacontract/lint/linters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
48
50
|
datacontract/lint/linters/description_linter.py,sha256=Cm99NTWo75kDRIW63qz5uEa8xsug_8dFURPRC1twodw,1554
|
|
49
|
-
datacontract/lint/linters/example_model_linter.py,sha256=
|
|
50
|
-
datacontract/lint/linters/field_pattern_linter.py,sha256=
|
|
51
|
+
datacontract/lint/linters/example_model_linter.py,sha256=EljSvzI4RSswdZrGa840bX6Zjat2GI16o5yDmKJfDOg,3972
|
|
52
|
+
datacontract/lint/linters/field_pattern_linter.py,sha256=H7GTdsK8R68gZx0CLsLXWlp-p83S-OG80m6gUY1Q2FA,1083
|
|
51
53
|
datacontract/lint/linters/field_reference_linter.py,sha256=fRNI483JpWV_ewWoNqfuVkNqYwQ-7z68q_xhHQZPO7g,1550
|
|
52
|
-
datacontract/lint/linters/notice_period_linter.py,sha256=
|
|
53
|
-
datacontract/lint/linters/quality_schema_linter.py,sha256
|
|
54
|
+
datacontract/lint/linters/notice_period_linter.py,sha256=oHSOLU2uXd7BDvfEd_VjeXS77atVzYXzrmAeG-wJwBE,2129
|
|
55
|
+
datacontract/lint/linters/quality_schema_linter.py,sha256=-zaI49okETJYo-qR-J6rQqMRI9QX17UK9ASf1GxgYnk,2181
|
|
54
56
|
datacontract/lint/linters/valid_constraints_linter.py,sha256=GeeE2hojMt24EoHuSHE1lURA2v2p5Dmxc7Qq8IvZQH0,4915
|
|
55
57
|
datacontract/model/breaking_change.py,sha256=BIDEUo1U2CQLVT2-I5PyFttxAj6zQPI1UUkEoOOQXMY,2249
|
|
56
|
-
datacontract/model/data_contract_specification.py,sha256=
|
|
58
|
+
datacontract/model/data_contract_specification.py,sha256=etwFSrpa2gyF20u5AOXhyv1JTX9u6tojIeE-frAtCq0,3525
|
|
57
59
|
datacontract/model/exceptions.py,sha256=zW9NoyzwsND-c9UqgyTVuezUVGEc6KK1Uc2zl12loyo,1178
|
|
58
60
|
datacontract/model/run.py,sha256=v7QNsPwYS1otQrpjeOaFsLRnxO8OgbdK_d-Z_e3iy0s,2610
|
|
59
|
-
datacontract/templates/datacontract.html,sha256=
|
|
60
|
-
datacontract/templates/style/output.css,sha256=
|
|
61
|
-
datacontract_cli-0.10.
|
|
62
|
-
datacontract_cli-0.10.
|
|
63
|
-
datacontract_cli-0.10.
|
|
64
|
-
datacontract_cli-0.10.
|
|
65
|
-
datacontract_cli-0.10.
|
|
66
|
-
datacontract_cli-0.10.
|
|
61
|
+
datacontract/templates/datacontract.html,sha256=nXP_0YZtB8yG8mGfItO7jbujoPw1Bt3-_6Ri2mqqWg0,28099
|
|
62
|
+
datacontract/templates/style/output.css,sha256=kbDuhgaN0RYfmMKPPJPo5p_aoWmY2gUDtrxUQCkq_-g,23673
|
|
63
|
+
datacontract_cli-0.10.2.dist-info/LICENSE,sha256=23h64qnSeIZ0DKeziWAKC-zBCt328iSbRbWBrXoYRb4,2210
|
|
64
|
+
datacontract_cli-0.10.2.dist-info/METADATA,sha256=pvvtIUZP0zPDqL37FX1crtUdERKqA-euP86IE20IXRo,63241
|
|
65
|
+
datacontract_cli-0.10.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
66
|
+
datacontract_cli-0.10.2.dist-info/entry_points.txt,sha256=D3Eqy4q_Z6bHauGd4ppIyQglwbrm1AJnLau4Ppbw9Is,54
|
|
67
|
+
datacontract_cli-0.10.2.dist-info/top_level.txt,sha256=VIRjd8EIUrBYWjEXJJjtdUgc0UAJdPZjmLiOR8BRBYM,13
|
|
68
|
+
datacontract_cli-0.10.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|