datacontract-cli 0.10.18__py3-none-any.whl → 0.10.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of datacontract-cli might be problematic. Click here for more details.

Files changed (30) hide show
  1. datacontract/cli.py +22 -30
  2. datacontract/data_contract.py +7 -8
  3. datacontract/engines/soda/connections/duckdb.py +22 -9
  4. datacontract/export/data_caterer_converter.py +20 -7
  5. datacontract/export/sodacl_converter.py +21 -4
  6. datacontract/export/sql_type_converter.py +7 -2
  7. datacontract/imports/csv_importer.py +89 -0
  8. datacontract/imports/importer.py +1 -0
  9. datacontract/imports/importer_factory.py +5 -0
  10. datacontract/init/init_template.py +20 -0
  11. datacontract/integration/datamesh_manager.py +5 -10
  12. datacontract/lint/linters/field_reference_linter.py +10 -1
  13. datacontract/lint/resolve.py +22 -1
  14. datacontract/lint/schema.py +10 -3
  15. datacontract/lint/urls.py +9 -5
  16. datacontract/model/data_contract_specification.py +2 -0
  17. datacontract/schemas/datacontract-1.1.0.init.yaml +91 -0
  18. datacontract/schemas/datacontract-1.1.0.schema.json +1975 -0
  19. datacontract/schemas/odcs-3.0.1.schema.json +2634 -0
  20. datacontract/templates/datacontract.html +20 -1
  21. datacontract/templates/partials/definition.html +15 -5
  22. datacontract/templates/partials/model_field.html +9 -0
  23. datacontract/web.py +170 -36
  24. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/METADATA +448 -297
  25. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/RECORD +29 -25
  26. datacontract/init/download_datacontract_file.py +0 -17
  27. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/LICENSE +0 -0
  28. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/WHEEL +0 -0
  29. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/entry_points.txt +0 -0
  30. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/top_level.txt +0 -0
{datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.20.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: datacontract-cli
3
- Version: 0.10.18
3
+ Version: 0.10.20
4
4
  Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
5
5
  Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
6
6
  Project-URL: Homepage, https://cli.datacontract.com
@@ -15,8 +15,6 @@ Requires-Dist: typer<0.16,>=0.15.1
15
15
  Requires-Dist: pydantic<2.11.0,>=2.8.2
16
16
  Requires-Dist: pyyaml~=6.0.1
17
17
  Requires-Dist: requests<2.33,>=2.31
18
- Requires-Dist: fastapi==0.115.6
19
- Requires-Dist: uvicorn==0.34.0
20
18
  Requires-Dist: fastjsonschema<2.22.0,>=2.19.1
21
19
  Requires-Dist: fastparquet==2024.11.0
22
20
  Requires-Dist: numpy<2.0.0,>=1.26.4
@@ -34,6 +32,9 @@ Provides-Extra: avro
34
32
  Requires-Dist: avro==1.12.0; extra == "avro"
35
33
  Provides-Extra: bigquery
36
34
  Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.20; extra == "bigquery"
35
+ Provides-Extra: csv
36
+ Requires-Dist: clevercsv>=0.8.2; extra == "csv"
37
+ Requires-Dist: pandas>=2.0.0; extra == "csv"
37
38
  Provides-Extra: databricks
38
39
  Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.20; extra == "databricks"
39
40
  Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.20; extra == "databricks"
@@ -62,8 +63,11 @@ Provides-Extra: dbml
62
63
  Requires-Dist: pydbml>=1.1.1; extra == "dbml"
63
64
  Provides-Extra: parquet
64
65
  Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
66
+ Provides-Extra: web
67
+ Requires-Dist: fastapi==0.115.6; extra == "web"
68
+ Requires-Dist: uvicorn==0.34.0; extra == "web"
65
69
  Provides-Extra: all
66
- Requires-Dist: datacontract-cli[bigquery,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
70
+ Requires-Dist: datacontract-cli[bigquery,csv,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,s3,snowflake,sqlserver,trino,web]; extra == "all"
67
71
  Provides-Extra: dev
68
72
  Requires-Dist: datacontract-cli[all]; extra == "dev"
69
73
  Requires-Dist: httpx==0.28.1; extra == "dev"
@@ -76,7 +80,7 @@ Requires-Dist: pytest-xdist; extra == "dev"
76
80
  Requires-Dist: pymssql==2.3.2; extra == "dev"
77
81
  Requires-Dist: ruff; extra == "dev"
78
82
  Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.0; extra == "dev"
79
- Requires-Dist: trino==0.331.0; extra == "dev"
83
+ Requires-Dist: trino==0.332.0; extra == "dev"
80
84
 
81
85
  # Data Contract CLI
82
86
 
@@ -304,71 +308,101 @@ Commands
304
308
  - [serve](#serve)
305
309
 
306
310
  ### init
307
-
308
311
  ```
309
- Usage: datacontract init [OPTIONS] [LOCATION]
310
-
311
- Download a datacontract.yaml template and write it to file.
312
+
313
+ Usage: datacontract init [OPTIONS] [LOCATION]
314
+
315
+ Download a datacontract.yaml template and write it to file.
316
+
317
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
318
+ │ location [LOCATION] The location (url or path) of the data contract │
319
+ │ yaml to create. │
320
+ │ [default: datacontract.yaml] │
321
+ ╰──────────────────────────────────────────────────────────────────────────────╯
322
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
323
+ │ --template TEXT URL of a template or data contract │
324
+ │ [default: │
325
+ │ https://datacontract.com/datacontrac… │
326
+ │ --overwrite --no-overwrite Replace the existing │
327
+ │ datacontract.yaml │
328
+ │ [default: no-overwrite] │
329
+ │ --help Show this message and exit. │
330
+ ╰──────────────────────────────────────────────────────────────────────────────╯
312
331
 
313
- ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────╮
314
- │ location [LOCATION] The location (url or path) of the data contract yaml to create. │
315
- │ [default: datacontract.yaml] │
316
- ╰──────────────────────────────────────────────────────────────────────────────────────────────╯
317
- ╭─ Options ────────────────────────────────────────────────────────────────────────────────────╮
318
- │ --template TEXT URL of a template or data contract │
319
- │ [default: │
320
- │ https://datacontract.com/datacontract.init.yaml] │
321
- │ --overwrite --no-overwrite Replace the existing datacontract.yaml │
322
- │ [default: no-overwrite] │
323
- │ --help Show this message and exit. │
324
- ╰──────────────────────────────────────────────────────────────────────────────────────────────╯
325
332
  ```
326
333
 
327
334
  ### lint
328
-
329
335
  ```
330
- Usage: datacontract lint [OPTIONS] [LOCATION]
336
+
337
+ Usage: datacontract lint [OPTIONS] [LOCATION]
338
+
339
+ Validate that the datacontract.yaml is correctly formatted.
340
+
341
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
342
+ │ location [LOCATION] The location (url or path) of the data contract │
343
+ │ yaml. │
344
+ │ [default: datacontract.yaml] │
345
+ ╰──────────────────────────────────────────────────────────────────────────────╯
346
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
347
+ │ --schema TEXT The location (url or path) of the Data Contract │
348
+ │ Specification JSON Schema │
349
+ │ [default: │
350
+ │ https://datacontract.com/datacontract.schema.json] │
351
+ │ --help Show this message and exit. │
352
+ ╰──────────────────────────────────────────────────────────────────────────────╯
331
353
 
332
- Validate that the datacontract.yaml is correctly formatted.
333
-
334
- ╭─ Arguments ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
335
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
336
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
337
- ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
338
- │ --schema TEXT The location (url or path) of the Data Contract Specification JSON Schema │
339
- │ [default: https://datacontract.com/datacontract.schema.json] │
340
- │ --help Show this message and exit. │
341
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
342
354
  ```
343
355
 
344
356
  ### test
345
-
346
357
  ```
347
- Usage: datacontract test [OPTIONS] [LOCATION]
348
-
349
- Run schema and quality tests on configured servers.
350
-
351
- ╭─ Arguments ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
352
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
353
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
354
- ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
355
- │ --schema TEXT The location (url or path) of the Data Contract │
356
- │ Specification JSON Schema │
357
- │ [default: │
358
- │ https://datacontract.com/datacontract.schema.json] │
359
- │ --server TEXT The server configuration to run the schema and quality │
360
- │ tests. Use the key of the server object in the data │
361
- │ contract yaml file to refer to a server, e.g., │
362
- │ `production`, or `all` for all servers (default). │
363
- │ [default: all] │
364
- │ --examples --no-examples Run the schema and quality tests on the example data │
365
- │ within the data contract. │
366
- │ [default: no-examples] │
367
- │ --publish TEXT The url to publish the results after the test │
368
- │ [default: None] │
369
- │ --logs --no-logs Print logs [default: no-logs] │
370
- │ --help Show this message and exit. │
371
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
358
+
359
+ Usage: datacontract test [OPTIONS] [LOCATION]
360
+
361
+ Run schema and quality tests on configured servers.
362
+
363
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
364
+ │ location [LOCATION] The location (url or path) of the data contract │
365
+ │ yaml. │
366
+ │ [default: datacontract.yaml] │
367
+ ╰──────────────────────────────────────────────────────────────────────────────╯
368
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
369
+ │ --schema TEXT The location (url or │
370
+ │ path) of the Data │
371
+ │ Contract Specification │
372
+ │ JSON Schema │
373
+ │ [default: │
374
+ │ https://datacontract.c… │
375
+ │ --server TEXT The server │
376
+ │ configuration to run │
377
+ │ the schema and quality │
378
+ │ tests. Use the key of │
379
+ │ the server object in │
380
+ │ the data contract yaml │
381
+ │ file to refer to a │
382
+ │ server, e.g., │
383
+ │ `production`, or `all` │
384
+ │ for all servers │
385
+ │ (default). │
386
+ │ [default: all] │
387
+ │ --examples --no-examples Run the schema and │
388
+ │ quality tests on the │
389
+ │ example data within the │
390
+ │ data contract. │
391
+ │ [default: no-examples] │
392
+ │ --publish TEXT The url to publish the │
393
+ │ results after the test │
394
+ │ [default: None] │
395
+ │ --logs --no-logs Print logs │
396
+ │ [default: no-logs] │
397
+ │ --ssl-verification --no-ssl-verification SSL verification when │
398
+ │ publishing the data │
399
+ │ contract. │
400
+ │ [default: │
401
+ │ ssl-verification] │
402
+ │ --help Show this message and │
403
+ │ exit. │
404
+ ╰──────────────────────────────────────────────────────────────────────────────╯
405
+
372
406
  ```
373
407
 
374
408
  Data Contract CLI connects to a data source and runs schema and quality tests to verify that the data contract is valid.
@@ -410,7 +444,7 @@ Supported formats:
410
444
 
411
445
  Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/issues), if you need support for an additional type and formats.
412
446
 
413
- ### S3
447
+ #### S3
414
448
 
415
449
  Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
416
450
 
@@ -420,9 +454,9 @@ Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant
420
454
  - Parquet
421
455
  - Iceberg (coming soon)
422
456
 
423
- #### Examples
457
+ ##### Examples
424
458
 
425
- ##### JSON
459
+ ###### JSON
426
460
 
427
461
  datacontract.yaml
428
462
  ```yaml
@@ -435,7 +469,7 @@ servers:
435
469
  delimiter: new_line # new_line, array, or none
436
470
  ```
437
471
 
438
- ##### Delta Tables
472
+ ###### Delta Tables
439
473
 
440
474
  datacontract.yaml
441
475
  ```yaml
@@ -447,7 +481,7 @@ servers:
447
481
  format: delta
448
482
  ```
449
483
 
450
- #### Environment Variables
484
+ ##### Environment Variables
451
485
 
452
486
  | Environment Variable | Example | Description |
453
487
  |-------------------------------------|---------------------------------|----------------------------------------|
@@ -458,12 +492,12 @@ servers:
458
492
 
459
493
 
460
494
 
461
- ### Google Cloud Storage (GCS)
495
+ #### Google Cloud Storage (GCS)
462
496
 
463
497
  The [S3](#S3) integration also works with files on Google Cloud Storage through its [interoperability](https://cloud.google.com/storage/docs/interoperability).
464
498
  Use `https://storage.googleapis.com` as the endpoint URL.
465
499
 
466
- #### Example
500
+ ##### Example
467
501
 
468
502
  datacontract.yaml
469
503
  ```yaml
@@ -476,7 +510,7 @@ servers:
476
510
  delimiter: new_line # new_line, array, or none
477
511
  ```
478
512
 
479
- #### Environment Variables
513
+ ##### Environment Variables
480
514
 
481
515
  | Environment Variable | Example | Description |
482
516
  |-------------------------------------|----------------|------------------------------------------------------------------------------------------|
@@ -484,14 +518,14 @@ servers:
484
518
  | `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `PDWWpb...` | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Secret |
485
519
 
486
520
 
487
- ### BigQuery
521
+ #### BigQuery
488
522
 
489
523
  We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
490
524
  * BigQuery Job User
491
525
  * BigQuery Data Viewer
492
526
 
493
527
 
494
- #### Example
528
+ ##### Example
495
529
 
496
530
  datacontract.yaml
497
531
  ```yaml
@@ -506,29 +540,30 @@ models:
506
540
  fields: ...
507
541
  ```
508
542
 
509
- #### Environment Variables
543
+ ##### Environment Variables
510
544
 
511
545
  | Environment Variable | Example | Description |
512
546
  |----------------------------------------------|---------------------------|---------------------------------------------------------|
513
547
  | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery. If this environment variable isn't set, the cli tries to use `GOOGLE_APPLICATION_CREDENTIALS` as a fallback, so if you have that set for using their Python library anyway, it should work seamlessly. |
514
548
 
515
549
 
516
- ### Azure
550
+ #### Azure
517
551
 
518
552
  Data Contract CLI can test data that is stored in Azure Blob storage or Azure Data Lake Storage (Gen2) (ADLS) in various formats.
519
553
 
520
- #### Example
554
+ ##### Example
521
555
 
522
556
  datacontract.yaml
523
557
  ```yaml
524
558
  servers:
525
559
  production:
526
560
  type: azure
527
- location: abfss://datameshdatabricksdemo.dfs.core.windows.net/dataproducts/inventory_events/*.parquet
561
+ storageAccount: datameshdatabricksdemo
562
+ location: abfss://dataproducts/inventory_events/*.parquet
528
563
  format: parquet
529
564
  ```
530
565
 
531
- #### Environment Variables
566
+ ##### Environment Variables
532
567
 
533
568
  Authentication works with an Azure Service Principal (SPN) aka App Registration with a secret.
534
569
 
@@ -540,11 +575,11 @@ Authentication works with an Azure Service Principal (SPN) aka App Registration
540
575
 
541
576
 
542
577
 
543
- ### Sqlserver
578
+ #### Sqlserver
544
579
 
545
580
  Data Contract CLI can test data in MS SQL Server (including Azure SQL, Synapse Analytics SQL Pool).
546
581
 
547
- #### Example
582
+ ##### Example
548
583
 
549
584
  datacontract.yaml
550
585
  ```yaml
@@ -564,7 +599,7 @@ models:
564
599
  type: varchar
565
600
  ```
566
601
 
567
- #### Environment Variables
602
+ ##### Environment Variables
568
603
 
569
604
  | Environment Variable | Example| Description |
570
605
  |---------------------------------------------------|--------|----------------------------------------------|
@@ -577,13 +612,13 @@ models:
577
612
 
578
613
 
579
614
 
580
- ### Databricks
615
+ #### Databricks
581
616
 
582
617
  Works with Unity Catalog and Hive metastore.
583
618
 
584
619
  Needs a running SQL warehouse or compute cluster.
585
620
 
586
- #### Example
621
+ ##### Example
587
622
 
588
623
  datacontract.yaml
589
624
  ```yaml
@@ -598,7 +633,7 @@ models:
598
633
  fields: ...
599
634
  ```
600
635
 
601
- #### Environment Variables
636
+ ##### Environment Variables
602
637
 
603
638
  | Environment Variable | Example | Description |
604
639
  |-------------------------------------------|--------------------------------------|-----------------------------------------------------------|
@@ -607,7 +642,7 @@ models:
607
642
  | `DATACONTRACT_DATABRICKS_SERVER_HOSTNAME` | `dbc-abcdefgh-1234.cloud.databricks.com` | The host name of the SQL warehouse or compute cluster |
608
643
 
609
644
 
610
- ### Databricks (programmatic)
645
+ #### Databricks (programmatic)
611
646
 
612
647
  Works with Unity Catalog and Hive metastore.
613
648
  When running in a notebook or pipeline, the provided `spark` session can be used.
@@ -615,7 +650,7 @@ An additional authentication is not required.
615
650
 
616
651
  Requires a Databricks Runtime with Python >= 3.10.
617
652
 
618
- #### Example
653
+ ##### Example
619
654
 
620
655
  datacontract.yaml
621
656
  ```yaml
@@ -645,7 +680,7 @@ run = data_contract.test()
645
680
  run.result
646
681
  ```
647
682
 
648
- ### Dataframe (programmatic)
683
+ #### Dataframe (programmatic)
649
684
 
650
685
  Works with Spark DataFrames.
651
686
  DataFrames need to be created as named temporary views.
@@ -653,7 +688,7 @@ Multiple temporary views are supported if your data contract contains multiple m
653
688
 
654
689
  Testing DataFrames is useful to test your datasets in a pipeline before writing them to a data source.
655
690
 
656
- #### Example
691
+ ##### Example
657
692
 
658
693
  datacontract.yaml
659
694
  ```yaml
@@ -681,11 +716,11 @@ assert run.result == "passed"
681
716
  ```
682
717
 
683
718
 
684
- ### Snowflake
719
+ #### Snowflake
685
720
 
686
721
  Data Contract CLI can test data in Snowflake.
687
722
 
688
- #### Example
723
+ ##### Example
689
724
 
690
725
  datacontract.yaml
691
726
  ```yaml
@@ -704,11 +739,11 @@ models:
704
739
  type: varchar
705
740
  ```
706
741
 
707
- #### Environment Variables
708
- All [parameters supported by Soda](https://docs.soda.io/soda/connect-snowflake.html), uppercased and prepended by `DATACONTRACT_SNOWFLAKE_` prefix.
742
+ ##### Environment Variables
743
+ All [parameters supported by Soda](https://docs.soda.io/soda/connect-snowflake.html), uppercased and prepended by `DATACONTRACT_SNOWFLAKE_` prefix.
709
744
  For example:
710
745
 
711
- | Soda parameter | Environment Variable |
746
+ | Soda parameter | Environment Variable |
712
747
  |----------------------|---------------------------------------------|
713
748
  | `username` | `DATACONTRACT_SNOWFLAKE_USERNAME` |
714
749
  | `password` | `DATACONTRACT_SNOWFLAKE_PASSWORD` |
@@ -721,7 +756,7 @@ Beware, that parameters:
721
756
  * `database`
722
757
  * `schema`
723
758
 
724
- are obtained from the `servers` section of the YAML-file.
759
+ are obtained from the `servers` section of the YAML-file.
725
760
  E.g. from the example above:
726
761
  ```yaml
727
762
  servers:
@@ -732,11 +767,11 @@ servers:
732
767
  ```
733
768
 
734
769
 
735
- ### Kafka
770
+ #### Kafka
736
771
 
737
772
  Kafka support is currently considered experimental.
738
773
 
739
- #### Example
774
+ ##### Example
740
775
 
741
776
  datacontract.yaml
742
777
  ```yaml
@@ -748,7 +783,7 @@ servers:
748
783
  format: json
749
784
  ```
750
785
 
751
- #### Environment Variables
786
+ ##### Environment Variables
752
787
 
753
788
  | Environment Variable | Example | Description |
754
789
  |-------------------------------------|---------|----------------------------------------------------------------------------------|
@@ -757,11 +792,11 @@ servers:
757
792
  | `DATACONTRACT_KAFKA_SASL_MECHANISM` | `PLAIN` | Default `PLAIN`. Other supported mechanisms: `SCRAM-SHA-256` and `SCRAM-SHA-512` |
758
793
 
759
794
 
760
- ### Postgres
795
+ #### Postgres
761
796
 
762
797
  Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
763
798
 
764
- #### Example
799
+ ##### Example
765
800
 
766
801
  datacontract.yaml
767
802
  ```yaml
@@ -780,7 +815,7 @@ models:
780
815
  type: varchar
781
816
  ```
782
817
 
783
- #### Environment Variables
818
+ ##### Environment Variables
784
819
 
785
820
  | Environment Variable | Example | Description |
786
821
  |----------------------------------|--------------------|-------------|
@@ -788,11 +823,11 @@ models:
788
823
  | `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
789
824
 
790
825
 
791
- ### Trino
826
+ #### Trino
792
827
 
793
828
  Data Contract CLI can test data in Trino.
794
829
 
795
- #### Example
830
+ ##### Example
796
831
 
797
832
  datacontract.yaml
798
833
  ```yaml
@@ -815,7 +850,7 @@ models:
815
850
  trinoType: row(en_us varchar, pt_br varchar)
816
851
  ```
817
852
 
818
- #### Environment Variables
853
+ ##### Environment Variables
819
854
 
820
855
  | Environment Variable | Example | Description |
821
856
  |-------------------------------|--------------------|-------------|
@@ -825,46 +860,64 @@ models:
825
860
 
826
861
 
827
862
  ### export
828
-
829
863
  ```
830
- Usage: datacontract export [OPTIONS] [LOCATION]
831
-
832
- Convert data contract to a specific format. Saves to file specified by `output` option if present, otherwise prints to stdout.
833
-
834
- ╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
835
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
836
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
837
- ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
838
- │ * --format [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db The export format. [default: None] [required] │
839
- │ t-staging-sql|odcs|odcs_v2|odcs_v3|rdf|avro|protobuf │
840
- │ |great-expectations|terraform|avro-idl|sql|sql-query │
841
- │ |html|go|bigquery|dbml|spark|sqlalchemy|data-caterer │
842
- │ |dcs|markdown|iceberg] │
843
- │ --output PATH Specify the file path where the exported data will be │
844
- │ saved. If no path is provided, the output will be │
845
- │ printed to stdout. │
846
- │ [default: None] │
847
- │ --server TEXT The server name to export. [default: None] │
848
- │ --model TEXT Use the key of the model in the data contract yaml │
849
- │ file to refer to a model, e.g., `orders`, or `all` │
850
- │ for all models (default). │
851
- │ [default: all] │
852
- │ --schema TEXT The location (url or path) of the Data Contract │
853
- │ Specification JSON Schema │
854
- │ [default: │
855
- │ https://datacontract.com/datacontract.schema.json] │
856
- │ --engine TEXT [engine] The engine used for great expection run. │
857
- │ [default: None] │
858
- │ --help Show this message and exit. │
859
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
860
- ╭─ RDF Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
861
- │ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. [default: None] │
862
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
863
- ╭─ SQL Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
864
- │ --sql-server-type TEXT [sql] The server type to determine the sql dialect. By default, it uses 'auto' to automatically │
865
- │ detect the sql dialect via the specified servers in the data contract. │
866
- │ [default: auto] │
867
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
864
+
865
+ Usage: datacontract export [OPTIONS] [LOCATION]
866
+
867
+ Convert data contract to a specific format. Saves to file specified by
868
+ `output` option if present, otherwise prints to stdout.
869
+
870
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
871
+ │ location [LOCATION] The location (url or path) of the data contract │
872
+ │ yaml. │
873
+ │ [default: datacontract.yaml] │
874
+ ╰──────────────────────────────────────────────────────────────────────────────╯
875
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
876
+ │ * --format [jsonschema|pydantic-model| The export format. │
877
+ │ sodacl|dbt|dbt-sources|dbt- [default: None] │
878
+ │ staging-sql|odcs|odcs_v2|od [required] │
879
+ │ cs_v3|rdf|avro|protobuf|gre │
880
+ │ at-expectations|terraform|a │
881
+ │ vro-idl|sql|sql-query|html| │
882
+ │ go|bigquery|dbml|spark|sqla │
883
+ │ lchemy|data-caterer|dcs|mar │
884
+ │ kdown|iceberg] │
885
+ │ --output PATH Specify the file path where │
886
+ │ the exported data will be │
887
+ │ saved. If no path is │
888
+ │ provided, the output will be │
889
+ │ printed to stdout. │
890
+ │ [default: None] │
891
+ │ --server TEXT The server name to export. │
892
+ │ [default: None] │
893
+ │ --model TEXT Use the key of the model in │
894
+ │ the data contract yaml file │
895
+ │ to refer to a model, e.g., │
896
+ │ `orders`, or `all` for all │
897
+ │ models (default). │
898
+ │ [default: all] │
899
+ │ --schema TEXT The location (url or path) │
900
+ │ of the Data Contract │
901
+ │ Specification JSON Schema │
902
+ │ [default: │
903
+ │ https://datacontract.com/da… │
904
+ │ --engine TEXT [engine] The engine used for │
905
+ │ great expection run. │
906
+ │ [default: None] │
907
+ │ --help Show this message and exit. │
908
+ ╰──────────────────────────────────────────────────────────────────────────────╯
909
+ ╭─ RDF Options ────────────────────────────────────────────────────────────────╮
910
+ │ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. │
911
+ │ [default: None] │
912
+ ╰──────────────────────────────────────────────────────────────────────────────╯
913
+ ╭─ SQL Options ────────────────────────────────────────────────────────────────╮
914
+ │ --sql-server-type TEXT [sql] The server type to determine the sql │
915
+ │ dialect. By default, it uses 'auto' to │
916
+ │ automatically detect the sql dialect via the │
917
+ │ specified servers in the data contract. │
918
+ │ [default: auto] │
919
+ ╰──────────────────────────────────────────────────────────────────────────────╯
920
+
868
921
  ```
869
922
 
870
923
  ```bash
@@ -919,7 +972,7 @@ The export creates a list of expectations by utilizing:
919
972
  - The data from the Model definition with a fixed mapping
920
973
  - The expectations provided in the quality field for each model (find here the expectations gallery: [Great Expectations Gallery](https://greatexpectations.io/expectations/))
921
974
 
922
- ### Additional Arguments
975
+ ##### Additional Arguments
923
976
 
924
977
  To further customize the export, the following optional arguments are available:
925
978
 
@@ -966,8 +1019,8 @@ logical data types are exported.
966
1019
 
967
1020
  #### Spark
968
1021
 
969
- The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
970
- Spark DataFrame schema is defined as StructType. For more details about Spark Data Types please see [the spark documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
1022
+ The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
1023
+ Spark DataFrame schema is defined as StructType. For more details about Spark Data Types please see [the spark documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
971
1024
 
972
1025
  #### Avro
973
1026
 
@@ -981,20 +1034,6 @@ To specify custom Avro properties in your data contract, you can define them wit
981
1034
 
982
1035
  >NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
983
1036
 
984
- #### Data Caterer
985
-
986
- The export function converts the data contract to a data generation task in YAML format that can be
987
- ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the
988
- ability to generate production-like data in any environment based off your data contract.
989
-
990
- ```shell
991
- datacontract export datacontract.yaml --format data-caterer --model orders
992
- ```
993
-
994
- You can further customise the way data is generated via adding
995
- [additional metadata in the YAML](https://data.catering/setup/generator/data-generator/)
996
- to suit your needs.
997
-
998
1037
  #### Example Configuration
999
1038
 
1000
1039
  ```yaml
@@ -1025,6 +1064,20 @@ models:
1025
1064
  - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
1026
1065
  - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
1027
1066
 
1067
+ #### Data Caterer
1068
+
1069
+ The export function converts the data contract to a data generation task in YAML format that can be
1070
+ ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the
1071
+ ability to generate production-like data in any environment based off your data contract.
1072
+
1073
+ ```shell
1074
+ datacontract export datacontract.yaml --format data-caterer --model orders
1075
+ ```
1076
+
1077
+ You can further customise the way data is generated via adding
1078
+ [additional metadata in the YAML](https://data.catering/setup/generator/data-generator/)
1079
+ to suit your needs.
1080
+
1028
1081
  #### Iceberg
1029
1082
 
1030
1083
  Exports to an [Iceberg Table Json Schema Definition](https://iceberg.apache.org/spec/#appendix-c-json-serialization).
@@ -1084,55 +1137,100 @@ to limit your contract export to a single model.
1084
1137
  ```
1085
1138
 
1086
1139
  ### import
1087
-
1088
1140
  ```
1089
- Usage: datacontract import [OPTIONS]
1090
-
1091
- Create a data contract from the given source location. Saves to file specified by `output` option if present, otherwise
1092
- prints to stdout.
1093
-
1094
- ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1095
- │ * --format [sql|avro|dbt|dbml|glue|jsonschema|bigquery The format of the source file. │
1096
- │ |odcs|unity|spark|iceberg|parquet] [default: None] │
1097
- │ [required] │
1098
- │ --output PATH Specify the file path where the Data │
1099
- │ Contract will be saved. If no path is │
1100
- │ provided, the output will be printed to │
1101
- │ stdout. │
1102
- │ [default: None] │
1103
- │ --source TEXT The path to the file or Glue Database that │
1104
- │ should be imported. │
1105
- │ [default: None] │
1106
- │ --glue-table TEXT List of table ids to import from the Glue │
1107
- │ Database (repeat for multiple table ids, │
1108
- │ leave empty for all tables in the dataset). │
1109
- │ [default: None] │
1110
- │ --bigquery-project TEXT The bigquery project id. [default: None] │
1111
- │ --bigquery-dataset TEXT The bigquery dataset id. [default: None] │
1112
- │ --bigquery-table TEXT List of table ids to import from the │
1113
- │ bigquery API (repeat for multiple table ids, │
1114
- │ leave empty for all tables in the dataset). │
1115
- │ [default: None] │
1116
- │ --unity-table-full-name TEXT Full name of a table in the unity catalog │
1117
- │ [default: None] │
1118
- │ --dbt-model TEXT List of models names to import from the dbt │
1119
- │ manifest file (repeat for multiple models │
1120
- │ names, leave empty for all models in the │
1121
- │ dataset). │
1122
- │ [default: None] │
1123
- │ --dbml-schema TEXT List of schema names to import from the DBML │
1124
- │ file (repeat for multiple schema names, │
1125
- │ leave empty for all tables in the file). │
1126
- │ [default: None] │
1127
- │ --dbml-table TEXT List of table names to import from the DBML │
1128
- │ file (repeat for multiple table names, leave │
1129
- │ empty for all tables in the file). │
1130
- │ [default: None] │
1131
- │ --iceberg-table TEXT Table name to assign to the model created │
1132
- │ from the Iceberg schema. │
1133
- │ [default: None] │
1134
- │ --help Show this message and exit. │
1135
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1141
+
1142
+ Usage: datacontract import [OPTIONS]
1143
+
1144
+ Create a data contract from the given source location. Saves to file specified
1145
+ by `output` option if present, otherwise prints to stdout.
1146
+
1147
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1148
+ │ * --format [sql|avro|dbt|dbml|gl The format of the │
1149
+ │ ue|jsonschema|bigquer source file. │
1150
+ │ y|odcs|unity|spark|ic [default: None] │
1151
+ │ eberg|parquet|csv] [required] │
1152
+ │ --output PATH Specify the file path │
1153
+ │ where the Data │
1154
+ │ Contract will be │
1155
+ │ saved. If no path is │
1156
+ │ provided, the output │
1157
+ │ will be printed to │
1158
+ │ stdout. │
1159
+ │ [default: None] │
1160
+ │ --source TEXT The path to the file │
1161
+ │ or Glue Database that │
1162
+ │ should be imported. │
1163
+ │ [default: None] │
1164
+ │ --glue-table TEXT List of table ids to │
1165
+ │ import from the Glue │
1166
+ │ Database (repeat for │
1167
+ │ multiple table ids, │
1168
+ │ leave empty for all │
1169
+ │ tables in the │
1170
+ │ dataset). │
1171
+ │ [default: None] │
1172
+ │ --bigquery-project TEXT The bigquery project │
1173
+ │ id. │
1174
+ │ [default: None] │
1175
+ │ --bigquery-dataset TEXT The bigquery dataset │
1176
+ │ id. │
1177
+ │ [default: None] │
1178
+ │ --bigquery-table TEXT List of table ids to │
1179
+ │ import from the │
1180
+ │ bigquery API (repeat │
1181
+ │ for multiple table │
1182
+ │ ids, leave empty for │
1183
+ │ all tables in the │
1184
+ │ dataset). │
1185
+ │ [default: None] │
1186
+ │ --unity-table-full-n… TEXT Full name of a table │
1187
+ │ in the unity catalog │
1188
+ │ [default: None] │
1189
+ │ --dbt-model TEXT List of models names │
1190
+ │ to import from the │
1191
+ │ dbt manifest file │
1192
+ │ (repeat for multiple │
1193
+ │ models names, leave │
1194
+ │ empty for all models │
1195
+ │ in the dataset). │
1196
+ │ [default: None] │
1197
+ │ --dbml-schema TEXT List of schema names │
1198
+ │ to import from the │
1199
+ │ DBML file (repeat for │
1200
+ │ multiple schema │
1201
+ │ names, leave empty │
1202
+ │ for all tables in the │
1203
+ │ file). │
1204
+ │ [default: None] │
1205
+ │ --dbml-table TEXT List of table names │
1206
+ │ to import from the │
1207
+ │ DBML file (repeat for │
1208
+ │ multiple table names, │
1209
+ │ leave empty for all │
1210
+ │ tables in the file). │
1211
+ │ [default: None] │
1212
+ │ --iceberg-table TEXT Table name to assign │
1213
+ │ to the model created │
1214
+ │ from the Iceberg │
1215
+ │ schema. │
1216
+ │ [default: None] │
1217
+ │ --template TEXT The location (url or │
1218
+ │ path) of the Data │
1219
+ │ Contract │
1220
+ │ Specification │
1221
+ │ Template │
1222
+ │ [default: None] │
1223
+ │ --schema TEXT The location (url or │
1224
+ │ path) of the Data │
1225
+ │ Contract │
1226
+ │ Specification JSON │
1227
+ │ Schema │
1228
+ │ [default: │
1229
+ │ https://datacontract… │
1230
+ │ --help Show this message and │
1231
+ │ exit. │
1232
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1233
+
1136
1234
  ```
1137
1235
 
1138
1236
  Example:
@@ -1157,6 +1255,7 @@ Available import options:
1157
1255
  | `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ |
1158
1256
  | `spark` | Import from Spark StructTypes | ✅ |
1159
1257
  | `dbml` | Import from DBML models | ✅ |
1258
+ | `csv` | Import from CSV File | ✅ |
1160
1259
  | `protobuf` | Import from Protobuf schemas | TBD |
1161
1260
  | `iceberg` | Import from an Iceberg JSON Schema Definition | partial |
1162
1261
  | `parquet` | Import from Parquet File Metadta | ✅ |
@@ -1299,67 +1398,106 @@ Examples:
1299
1398
  datacontract import --format iceberg --source ./tests/fixtures/iceberg/simple_schema.json --iceberg-table test-table
1300
1399
  ```
1301
1400
 
1302
- ### breaking
1401
+ #### CSV
1303
1402
 
1304
- ```
1305
- Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW
1403
+ Importing from CSV File. Specify file in `source` parameter. It does autodetection for encoding and csv dialect
1306
1404
 
1307
- Identifies breaking changes between data contracts. Prints to stdout.
1405
+ Example:
1308
1406
 
1309
- ╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1310
- │ * location_old TEXT The location (url or path) of the old data contract yaml. [default: None] [required] │
1311
- │ * location_new TEXT The location (url or path) of the new data contract yaml. [default: None] [required] │
1312
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1313
- ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1314
- │ --help Show this message and exit. │
1315
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1407
+ ```bash
1408
+ datacontract import --format csv --source "test.csv"
1316
1409
  ```
1317
1410
 
1318
- ### changelog
1319
1411
 
1412
+ ### breaking
1320
1413
  ```
1321
- Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW
1414
+
1415
+ Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW
1416
+
1417
+ Identifies breaking changes between data contracts. Prints to stdout.
1418
+
1419
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1420
+ │ * location_old TEXT The location (url or path) of the old data │
1421
+ │ contract yaml. │
1422
+ │ [default: None] │
1423
+ │ [required] │
1424
+ │ * location_new TEXT The location (url or path) of the new data │
1425
+ │ contract yaml. │
1426
+ │ [default: None] │
1427
+ │ [required] │
1428
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1429
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1430
+ │ --help Show this message and exit. │
1431
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1322
1432
 
1323
- Generate a changelog between data contracts. Prints to stdout.
1324
-
1325
- ╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1326
- │ * location_old TEXT The location (url or path) of the old data contract yaml. [default: None] [required] │
1327
- │ * location_new TEXT The location (url or path) of the new data contract yaml. [default: None] [required] │
1328
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1329
- ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1330
- │ --help Show this message and exit. │
1331
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1332
1433
  ```
1333
1434
 
1334
- ### diff
1435
+ ### changelog
1436
+ ```
1437
+
1438
+ Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW
1439
+
1440
+ Generate a changelog between data contracts. Prints to stdout.
1441
+
1442
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1443
+ │ * location_old TEXT The location (url or path) of the old data │
1444
+ │ contract yaml. │
1445
+ │ [default: None] │
1446
+ │ [required] │
1447
+ │ * location_new TEXT The location (url or path) of the new data │
1448
+ │ contract yaml. │
1449
+ │ [default: None] │
1450
+ │ [required] │
1451
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1452
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1453
+ │ --help Show this message and exit. │
1454
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1335
1455
 
1336
1456
  ```
1337
- Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW
1338
1457
 
1339
- PLACEHOLDER. Currently works as 'changelog' does.
1458
+ ### diff
1459
+ ```
1460
+
1461
+ Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW
1462
+
1463
+ PLACEHOLDER. Currently works as 'changelog' does.
1464
+
1465
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1466
+ │ * location_old TEXT The location (url or path) of the old data │
1467
+ │ contract yaml. │
1468
+ │ [default: None] │
1469
+ │ [required] │
1470
+ │ * location_new TEXT The location (url or path) of the new data │
1471
+ │ contract yaml. │
1472
+ │ [default: None] │
1473
+ │ [required] │
1474
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1475
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1476
+ │ --help Show this message and exit. │
1477
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1340
1478
 
1341
- ╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1342
- │ * location_old TEXT The location (url or path) of the old data contract yaml. [default: None] [required] │
1343
- │ * location_new TEXT The location (url or path) of the new data contract yaml. [default: None] [required] │
1344
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1345
- ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1346
- │ --help Show this message and exit. │
1347
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1348
1479
  ```
1349
1480
 
1350
1481
  ### catalog
1351
-
1352
1482
  ```
1483
+
1484
+ Usage: datacontract catalog [OPTIONS]
1485
+
1486
+ Create an html catalog of data contracts.
1487
+
1488
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1489
+ │ --files TEXT Glob pattern for the data contract files to include in │
1490
+ │ the catalog. Applies recursively to any subfolders. │
1491
+ │ [default: *.yaml] │
1492
+ │ --output TEXT Output directory for the catalog html files. │
1493
+ │ [default: catalog/] │
1494
+ │ --schema TEXT The location (url or path) of the Data Contract │
1495
+ │ Specification JSON Schema │
1496
+ │ [default: │
1497
+ │ https://datacontract.com/datacontract.schema.json] │
1498
+ │ --help Show this message and exit. │
1499
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1353
1500
 
1354
- Usage: datacontract catalog [OPTIONS]
1355
-
1356
- Create an html catalog of data contracts.
1357
-
1358
- ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1359
- │ --files TEXT Glob pattern for the data contract files to include in the catalog. [default: *.yaml] │
1360
- │ --output TEXT Output directory for the catalog html files. [default: catalog/] │
1361
- │ --help Show this message and exit. │
1362
- ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1363
1501
  ```
1364
1502
 
1365
1503
  Examples:
@@ -1372,35 +1510,48 @@ datacontract catalog --output "."
1372
1510
  datacontract catalog --files "*.odcs.yaml"
1373
1511
  ```
1374
1512
 
1375
- ### Publish
1376
-
1377
- ```
1378
-
1379
- Usage: datacontract publish [OPTIONS] [LOCATION]
1380
-
1381
- Publish the data contract to the Data Mesh Manager.
1382
-
1383
- ╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1384
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
1385
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1386
- ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1387
- │ --help Show this message and exit. │
1388
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1389
- ```
1390
-
1391
- ### Serve
1392
-
1393
- ```
1394
-
1395
- Usage: datacontract serve [OPTIONS]
1396
-
1397
- Start the datacontract web server.
1398
-
1399
- ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1400
- │ --port INTEGER Bind socket to this port. [default: 4242] │
1401
- │ --host TEXT Bind socket to this host. [default: 127.0.0.1] │
1402
- │ --help Show this message and exit. │
1403
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1513
+ ### publish
1514
+ ```
1515
+
1516
+ Usage: datacontract publish [OPTIONS] [LOCATION]
1517
+
1518
+ Publish the data contract to the Data Mesh Manager.
1519
+
1520
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1521
+ │ location [LOCATION] The location (url or path) of the data contract │
1522
+ │ yaml. │
1523
+ │ [default: datacontract.yaml] │
1524
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1525
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1526
+ │ --schema TEXT The location (url or │
1527
+ │ path) of the Data │
1528
+ │ Contract Specification │
1529
+ │ JSON Schema │
1530
+ │ [default: │
1531
+ │ https://datacontract.c… │
1532
+ │ --ssl-verification --no-ssl-verification SSL verification when │
1533
+ │ publishing the data │
1534
+ │ contract. │
1535
+ │ [default: │
1536
+ │ ssl-verification] │
1537
+ │ --help Show this message and │
1538
+ │ exit. │
1539
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1540
+
1541
+ ```
1542
+
1543
+ ### serve
1544
+ ```
1545
+
1546
+ Usage: datacontract serve [OPTIONS]
1547
+
1548
+ Start the datacontract web server.
1549
+
1550
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1551
+ │ --port INTEGER Bind socket to this port. [default: 4242] │
1552
+ │ --host TEXT Bind socket to this host. [default: 127.0.0.1] │
1553
+ │ --help Show this message and exit. │
1554
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1404
1555
 
1405
1556
  ```
1406
1557
 
@@ -1418,7 +1569,7 @@ If you use [Data Mesh Manager](https://datamesh-manager.com/) or [Data Contract
1418
1569
  ```bash
1419
1570
  # Fetch current data contract, execute tests on production, and publish result to data mesh manager
1420
1571
  $ EXPORT DATAMESH_MANAGER_API_KEY=xxx
1421
- $ datacontract test https://demo.datamesh-manager.com/demo279750347121/datacontracts/4df9d6ee-e55d-4088-9598-b635b2fdcbbc/datacontract.yaml \
1572
+ $ datacontract test https://demo.datamesh-manager.com/demo279750347121/datacontracts/4df9d6ee-e55d-4088-9598-b635b2fdcbbc/datacontract.yaml \
1422
1573
  --server production \
1423
1574
  --publish https://api.datamesh-manager.com/api/test-results
1424
1575
  ```
@@ -1530,7 +1681,7 @@ Examples: Removing or renaming models and fields.
1530
1681
  ```
1531
1682
 
1532
1683
  ## Customizing Exporters and Importers
1533
-
1684
+
1534
1685
  ### Custom Exporter
1535
1686
  Using the exporter factory to add a new custom exporter
1536
1687
  ```python
@@ -1576,18 +1727,18 @@ if __name__ == "__main__":
1576
1727
  Output
1577
1728
  ```python
1578
1729
  {
1579
- 'title': 'Orders Unit Test',
1580
- 'version': '1.0.0',
1581
- 'description': 'The orders data contract',
1582
- 'email': 'team-orders@example.com',
1583
- 'url': 'https://wiki.example.com/teams/checkout',
1584
- 'model': 'orders',
1585
- 'model_columns': 'order_id, order_total, order_status',
1586
- 'export_args': {'server': 'production', 'custom_arg': 'my_custom_arg'},
1730
+ 'title': 'Orders Unit Test',
1731
+ 'version': '1.0.0',
1732
+ 'description': 'The orders data contract',
1733
+ 'email': 'team-orders@example.com',
1734
+ 'url': 'https://wiki.example.com/teams/checkout',
1735
+ 'model': 'orders',
1736
+ 'model_columns': 'order_id, order_total, order_status',
1737
+ 'export_args': {'server': 'production', 'custom_arg': 'my_custom_arg'},
1587
1738
  'custom_args': 'my_custom_arg'
1588
1739
  }
1589
1740
  ```
1590
-
1741
+
1591
1742
  ### Custom Importer
1592
1743
  Using the importer factory to add a new custom importer
1593
1744
  ```python
@@ -1609,31 +1760,31 @@ class CustomImporter(Importer):
1609
1760
  data_contract_specification.info.title = source_dict.get("title")
1610
1761
  data_contract_specification.info.version = source_dict.get("version")
1611
1762
  data_contract_specification.info.description = source_dict.get("description_from_app")
1612
-
1763
+
1613
1764
  for model in source_dict.get("models", []):
1614
1765
  fields = {}
1615
1766
  for column in model.get('columns'):
1616
1767
  field = Field(
1617
- description=column.get('column_description'),
1618
- type=column.get('type')
1768
+ description=column.get('column_description'),
1769
+ type=column.get('type')
1619
1770
  )
1620
- fields[column.get('name')] = field
1621
-
1771
+ fields[column.get('name')] = field
1772
+
1622
1773
  dc_model = Model(
1623
- description=model.get('description'),
1774
+ description=model.get('description'),
1624
1775
  fields= fields
1625
1776
  )
1626
1777
 
1627
1778
  data_contract_specification.models[model.get('name')] = dc_model
1628
1779
  return data_contract_specification
1629
-
1780
+
1630
1781
 
1631
1782
  # Register the new custom class into factory
1632
1783
  importer_factory.register_importer("custom_company_importer", CustomImporter)
1633
1784
 
1634
1785
 
1635
1786
  if __name__ == "__main__":
1636
- # Get a custom data from other app
1787
+ # Get a custom data from other app
1637
1788
  json_from_custom_app = '''
1638
1789
  {
1639
1790
  "id_custom": "uuid-custom",
@@ -1665,14 +1816,14 @@ if __name__ == "__main__":
1665
1816
 
1666
1817
  # Call import_from_source
1667
1818
  result = data_contract.import_from_source(
1668
- format="custom_company_importer",
1669
- data_contract_specification=DataContract.init(),
1819
+ format="custom_company_importer",
1820
+ data_contract_specification=DataContract.init(),
1670
1821
  source=json_from_custom_app
1671
- )
1822
+ )
1672
1823
  print(result.to_yaml() )
1673
1824
  ```
1674
1825
  Output
1675
-
1826
+
1676
1827
  ```yaml
1677
1828
  dataContractSpecification: 1.1.0
1678
1829
  id: uuid-custom