datacontract-cli 0.10.18__py3-none-any.whl → 0.10.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of datacontract-cli might be problematic. Click here for more details.

Files changed (28) hide show
  1. datacontract/cli.py +20 -27
  2. datacontract/data_contract.py +7 -8
  3. datacontract/engines/soda/connections/duckdb.py +22 -9
  4. datacontract/export/data_caterer_converter.py +20 -7
  5. datacontract/export/sodacl_converter.py +21 -4
  6. datacontract/export/sql_type_converter.py +7 -2
  7. datacontract/imports/csv_importer.py +89 -0
  8. datacontract/imports/importer.py +1 -0
  9. datacontract/imports/importer_factory.py +5 -0
  10. datacontract/init/init_template.py +20 -0
  11. datacontract/integration/datamesh_manager.py +5 -10
  12. datacontract/lint/linters/field_reference_linter.py +10 -1
  13. datacontract/lint/resolve.py +22 -1
  14. datacontract/lint/schema.py +10 -3
  15. datacontract/model/data_contract_specification.py +2 -0
  16. datacontract/schemas/datacontract-1.1.0.init.yaml +91 -0
  17. datacontract/schemas/datacontract-1.1.0.schema.json +1975 -0
  18. datacontract/schemas/odcs-3.0.1.schema.json +2634 -0
  19. datacontract/templates/datacontract.html +20 -1
  20. datacontract/templates/partials/definition.html +15 -5
  21. datacontract/templates/partials/model_field.html +9 -0
  22. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/METADATA +445 -295
  23. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/RECORD +27 -23
  24. datacontract/init/download_datacontract_file.py +0 -17
  25. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/LICENSE +0 -0
  26. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/WHEEL +0 -0
  27. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/entry_points.txt +0 -0
  28. {datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/top_level.txt +0 -0
{datacontract_cli-0.10.18.dist-info → datacontract_cli-0.10.19.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: datacontract-cli
3
- Version: 0.10.18
3
+ Version: 0.10.19
4
4
  Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
5
5
  Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
6
6
  Project-URL: Homepage, https://cli.datacontract.com
@@ -34,6 +34,9 @@ Provides-Extra: avro
34
34
  Requires-Dist: avro==1.12.0; extra == "avro"
35
35
  Provides-Extra: bigquery
36
36
  Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.20; extra == "bigquery"
37
+ Provides-Extra: csv
38
+ Requires-Dist: clevercsv>=0.8.2; extra == "csv"
39
+ Requires-Dist: pandas>=2.0.0; extra == "csv"
37
40
  Provides-Extra: databricks
38
41
  Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.20; extra == "databricks"
39
42
  Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.20; extra == "databricks"
@@ -63,7 +66,7 @@ Requires-Dist: pydbml>=1.1.1; extra == "dbml"
63
66
  Provides-Extra: parquet
64
67
  Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
65
68
  Provides-Extra: all
66
- Requires-Dist: datacontract-cli[bigquery,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
69
+ Requires-Dist: datacontract-cli[bigquery,csv,databricks,dbml,dbt,iceberg,kafka,parquet,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
67
70
  Provides-Extra: dev
68
71
  Requires-Dist: datacontract-cli[all]; extra == "dev"
69
72
  Requires-Dist: httpx==0.28.1; extra == "dev"
@@ -76,7 +79,7 @@ Requires-Dist: pytest-xdist; extra == "dev"
76
79
  Requires-Dist: pymssql==2.3.2; extra == "dev"
77
80
  Requires-Dist: ruff; extra == "dev"
78
81
  Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.9.0; extra == "dev"
79
- Requires-Dist: trino==0.331.0; extra == "dev"
82
+ Requires-Dist: trino==0.332.0; extra == "dev"
80
83
 
81
84
  # Data Contract CLI
82
85
 
@@ -304,71 +307,101 @@ Commands
304
307
  - [serve](#serve)
305
308
 
306
309
  ### init
307
-
308
310
  ```
309
- Usage: datacontract init [OPTIONS] [LOCATION]
310
-
311
- Download a datacontract.yaml template and write it to file.
311
+
312
+ Usage: datacontract init [OPTIONS] [LOCATION]
313
+
314
+ Download a datacontract.yaml template and write it to file.
315
+
316
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
317
+ │ location [LOCATION] The location (url or path) of the data contract │
318
+ │ yaml to create. │
319
+ │ [default: datacontract.yaml] │
320
+ ╰──────────────────────────────────────────────────────────────────────────────╯
321
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
322
+ │ --template TEXT URL of a template or data contract │
323
+ │ [default: │
324
+ │ https://datacontract.com/datacontrac… │
325
+ │ --overwrite --no-overwrite Replace the existing │
326
+ │ datacontract.yaml │
327
+ │ [default: no-overwrite] │
328
+ │ --help Show this message and exit. │
329
+ ╰──────────────────────────────────────────────────────────────────────────────╯
312
330
 
313
- ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────╮
314
- │ location [LOCATION] The location (url or path) of the data contract yaml to create. │
315
- │ [default: datacontract.yaml] │
316
- ╰──────────────────────────────────────────────────────────────────────────────────────────────╯
317
- ╭─ Options ────────────────────────────────────────────────────────────────────────────────────╮
318
- │ --template TEXT URL of a template or data contract │
319
- │ [default: │
320
- │ https://datacontract.com/datacontract.init.yaml] │
321
- │ --overwrite --no-overwrite Replace the existing datacontract.yaml │
322
- │ [default: no-overwrite] │
323
- │ --help Show this message and exit. │
324
- ╰──────────────────────────────────────────────────────────────────────────────────────────────╯
325
331
  ```
326
332
 
327
333
  ### lint
328
-
329
334
  ```
330
- Usage: datacontract lint [OPTIONS] [LOCATION]
335
+
336
+ Usage: datacontract lint [OPTIONS] [LOCATION]
337
+
338
+ Validate that the datacontract.yaml is correctly formatted.
339
+
340
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
341
+ │ location [LOCATION] The location (url or path) of the data contract │
342
+ │ yaml. │
343
+ │ [default: datacontract.yaml] │
344
+ ╰──────────────────────────────────────────────────────────────────────────────╯
345
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
346
+ │ --schema TEXT The location (url or path) of the Data Contract │
347
+ │ Specification JSON Schema │
348
+ │ [default: │
349
+ │ https://datacontract.com/datacontract.schema.json] │
350
+ │ --help Show this message and exit. │
351
+ ╰──────────────────────────────────────────────────────────────────────────────╯
331
352
 
332
- Validate that the datacontract.yaml is correctly formatted.
333
-
334
- ╭─ Arguments ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
335
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
336
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
337
- ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
338
- │ --schema TEXT The location (url or path) of the Data Contract Specification JSON Schema │
339
- │ [default: https://datacontract.com/datacontract.schema.json] │
340
- │ --help Show this message and exit. │
341
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
342
353
  ```
343
354
 
344
355
  ### test
345
-
346
356
  ```
347
- Usage: datacontract test [OPTIONS] [LOCATION]
348
-
349
- Run schema and quality tests on configured servers.
350
-
351
- ╭─ Arguments ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
352
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
353
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
354
- ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
355
- │ --schema TEXT The location (url or path) of the Data Contract │
356
- │ Specification JSON Schema │
357
- │ [default: │
358
- │ https://datacontract.com/datacontract.schema.json] │
359
- │ --server TEXT The server configuration to run the schema and quality │
360
- │ tests. Use the key of the server object in the data │
361
- │ contract yaml file to refer to a server, e.g., │
362
- │ `production`, or `all` for all servers (default). │
363
- │ [default: all] │
364
- │ --examples --no-examples Run the schema and quality tests on the example data │
365
- │ within the data contract. │
366
- │ [default: no-examples] │
367
- │ --publish TEXT The url to publish the results after the test │
368
- │ [default: None] │
369
- │ --logs --no-logs Print logs [default: no-logs] │
370
- │ --help Show this message and exit. │
371
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
357
+
358
+ Usage: datacontract test [OPTIONS] [LOCATION]
359
+
360
+ Run schema and quality tests on configured servers.
361
+
362
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
363
+ │ location [LOCATION] The location (url or path) of the data contract │
364
+ │ yaml. │
365
+ │ [default: datacontract.yaml] │
366
+ ╰──────────────────────────────────────────────────────────────────────────────╯
367
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
368
+ │ --schema TEXT The location (url or │
369
+ │ path) of the Data │
370
+ │ Contract Specification │
371
+ │ JSON Schema │
372
+ │ [default: │
373
+ │ https://datacontract.c… │
374
+ │ --server TEXT The server │
375
+ │ configuration to run │
376
+ │ the schema and quality │
377
+ │ tests. Use the key of │
378
+ │ the server object in │
379
+ │ the data contract yaml │
380
+ │ file to refer to a │
381
+ │ server, e.g., │
382
+ │ `production`, or `all` │
383
+ │ for all servers │
384
+ │ (default). │
385
+ │ [default: all] │
386
+ │ --examples --no-examples Run the schema and │
387
+ │ quality tests on the │
388
+ │ example data within the │
389
+ │ data contract. │
390
+ │ [default: no-examples] │
391
+ │ --publish TEXT The url to publish the │
392
+ │ results after the test │
393
+ │ [default: None] │
394
+ │ --logs --no-logs Print logs │
395
+ │ [default: no-logs] │
396
+ │ --ssl-verification --no-ssl-verification SSL verification when │
397
+ │ publishing the data │
398
+ │ contract. │
399
+ │ [default: │
400
+ │ ssl-verification] │
401
+ │ --help Show this message and │
402
+ │ exit. │
403
+ ╰──────────────────────────────────────────────────────────────────────────────╯
404
+
372
405
  ```
373
406
 
374
407
  Data Contract CLI connects to a data source and runs schema and quality tests to verify that the data contract is valid.
@@ -410,7 +443,7 @@ Supported formats:
410
443
 
411
444
  Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/issues), if you need support for an additional type and formats.
412
445
 
413
- ### S3
446
+ #### S3
414
447
 
415
448
  Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
416
449
 
@@ -420,9 +453,9 @@ Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant
420
453
  - Parquet
421
454
  - Iceberg (coming soon)
422
455
 
423
- #### Examples
456
+ ##### Examples
424
457
 
425
- ##### JSON
458
+ ###### JSON
426
459
 
427
460
  datacontract.yaml
428
461
  ```yaml
@@ -435,7 +468,7 @@ servers:
435
468
  delimiter: new_line # new_line, array, or none
436
469
  ```
437
470
 
438
- ##### Delta Tables
471
+ ###### Delta Tables
439
472
 
440
473
  datacontract.yaml
441
474
  ```yaml
@@ -447,7 +480,7 @@ servers:
447
480
  format: delta
448
481
  ```
449
482
 
450
- #### Environment Variables
483
+ ##### Environment Variables
451
484
 
452
485
  | Environment Variable | Example | Description |
453
486
  |-------------------------------------|---------------------------------|----------------------------------------|
@@ -458,12 +491,12 @@ servers:
458
491
 
459
492
 
460
493
 
461
- ### Google Cloud Storage (GCS)
494
+ #### Google Cloud Storage (GCS)
462
495
 
463
496
  The [S3](#S3) integration also works with files on Google Cloud Storage through its [interoperability](https://cloud.google.com/storage/docs/interoperability).
464
497
  Use `https://storage.googleapis.com` as the endpoint URL.
465
498
 
466
- #### Example
499
+ ##### Example
467
500
 
468
501
  datacontract.yaml
469
502
  ```yaml
@@ -476,7 +509,7 @@ servers:
476
509
  delimiter: new_line # new_line, array, or none
477
510
  ```
478
511
 
479
- #### Environment Variables
512
+ ##### Environment Variables
480
513
 
481
514
  | Environment Variable | Example | Description |
482
515
  |-------------------------------------|----------------|------------------------------------------------------------------------------------------|
@@ -484,14 +517,14 @@ servers:
484
517
  | `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `PDWWpb...` | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Secret |
485
518
 
486
519
 
487
- ### BigQuery
520
+ #### BigQuery
488
521
 
489
522
  We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
490
523
  * BigQuery Job User
491
524
  * BigQuery Data Viewer
492
525
 
493
526
 
494
- #### Example
527
+ ##### Example
495
528
 
496
529
  datacontract.yaml
497
530
  ```yaml
@@ -506,29 +539,30 @@ models:
506
539
  fields: ...
507
540
  ```
508
541
 
509
- #### Environment Variables
542
+ ##### Environment Variables
510
543
 
511
544
  | Environment Variable | Example | Description |
512
545
  |----------------------------------------------|---------------------------|---------------------------------------------------------|
513
546
  | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery. If this environment variable isn't set, the cli tries to use `GOOGLE_APPLICATION_CREDENTIALS` as a fallback, so if you have that set for using their Python library anyway, it should work seamlessly. |
514
547
 
515
548
 
516
- ### Azure
549
+ #### Azure
517
550
 
518
551
  Data Contract CLI can test data that is stored in Azure Blob storage or Azure Data Lake Storage (Gen2) (ADLS) in various formats.
519
552
 
520
- #### Example
553
+ ##### Example
521
554
 
522
555
  datacontract.yaml
523
556
  ```yaml
524
557
  servers:
525
558
  production:
526
559
  type: azure
527
- location: abfss://datameshdatabricksdemo.dfs.core.windows.net/dataproducts/inventory_events/*.parquet
560
+ storageAccount: datameshdatabricksdemo
561
+ location: abfss://dataproducts/inventory_events/*.parquet
528
562
  format: parquet
529
563
  ```
530
564
 
531
- #### Environment Variables
565
+ ##### Environment Variables
532
566
 
533
567
  Authentication works with an Azure Service Principal (SPN) aka App Registration with a secret.
534
568
 
@@ -540,11 +574,11 @@ Authentication works with an Azure Service Principal (SPN) aka App Registration
540
574
 
541
575
 
542
576
 
543
- ### Sqlserver
577
+ #### Sqlserver
544
578
 
545
579
  Data Contract CLI can test data in MS SQL Server (including Azure SQL, Synapse Analytics SQL Pool).
546
580
 
547
- #### Example
581
+ ##### Example
548
582
 
549
583
  datacontract.yaml
550
584
  ```yaml
@@ -564,7 +598,7 @@ models:
564
598
  type: varchar
565
599
  ```
566
600
 
567
- #### Environment Variables
601
+ ##### Environment Variables
568
602
 
569
603
  | Environment Variable | Example| Description |
570
604
  |---------------------------------------------------|--------|----------------------------------------------|
@@ -577,13 +611,13 @@ models:
577
611
 
578
612
 
579
613
 
580
- ### Databricks
614
+ #### Databricks
581
615
 
582
616
  Works with Unity Catalog and Hive metastore.
583
617
 
584
618
  Needs a running SQL warehouse or compute cluster.
585
619
 
586
- #### Example
620
+ ##### Example
587
621
 
588
622
  datacontract.yaml
589
623
  ```yaml
@@ -598,7 +632,7 @@ models:
598
632
  fields: ...
599
633
  ```
600
634
 
601
- #### Environment Variables
635
+ ##### Environment Variables
602
636
 
603
637
  | Environment Variable | Example | Description |
604
638
  |-------------------------------------------|--------------------------------------|-----------------------------------------------------------|
@@ -607,7 +641,7 @@ models:
607
641
  | `DATACONTRACT_DATABRICKS_SERVER_HOSTNAME` | `dbc-abcdefgh-1234.cloud.databricks.com` | The host name of the SQL warehouse or compute cluster |
608
642
 
609
643
 
610
- ### Databricks (programmatic)
644
+ #### Databricks (programmatic)
611
645
 
612
646
  Works with Unity Catalog and Hive metastore.
613
647
  When running in a notebook or pipeline, the provided `spark` session can be used.
@@ -615,7 +649,7 @@ An additional authentication is not required.
615
649
 
616
650
  Requires a Databricks Runtime with Python >= 3.10.
617
651
 
618
- #### Example
652
+ ##### Example
619
653
 
620
654
  datacontract.yaml
621
655
  ```yaml
@@ -645,7 +679,7 @@ run = data_contract.test()
645
679
  run.result
646
680
  ```
647
681
 
648
- ### Dataframe (programmatic)
682
+ #### Dataframe (programmatic)
649
683
 
650
684
  Works with Spark DataFrames.
651
685
  DataFrames need to be created as named temporary views.
@@ -653,7 +687,7 @@ Multiple temporary views are supported if your data contract contains multiple m
653
687
 
654
688
  Testing DataFrames is useful to test your datasets in a pipeline before writing them to a data source.
655
689
 
656
- #### Example
690
+ ##### Example
657
691
 
658
692
  datacontract.yaml
659
693
  ```yaml
@@ -681,11 +715,11 @@ assert run.result == "passed"
681
715
  ```
682
716
 
683
717
 
684
- ### Snowflake
718
+ #### Snowflake
685
719
 
686
720
  Data Contract CLI can test data in Snowflake.
687
721
 
688
- #### Example
722
+ ##### Example
689
723
 
690
724
  datacontract.yaml
691
725
  ```yaml
@@ -704,11 +738,11 @@ models:
704
738
  type: varchar
705
739
  ```
706
740
 
707
- #### Environment Variables
708
- All [parameters supported by Soda](https://docs.soda.io/soda/connect-snowflake.html), uppercased and prepended by `DATACONTRACT_SNOWFLAKE_` prefix.
741
+ ##### Environment Variables
742
+ All [parameters supported by Soda](https://docs.soda.io/soda/connect-snowflake.html), uppercased and prepended by `DATACONTRACT_SNOWFLAKE_` prefix.
709
743
  For example:
710
744
 
711
- | Soda parameter | Environment Variable |
745
+ | Soda parameter | Environment Variable |
712
746
  |----------------------|---------------------------------------------|
713
747
  | `username` | `DATACONTRACT_SNOWFLAKE_USERNAME` |
714
748
  | `password` | `DATACONTRACT_SNOWFLAKE_PASSWORD` |
@@ -721,7 +755,7 @@ Beware, that parameters:
721
755
  * `database`
722
756
  * `schema`
723
757
 
724
- are obtained from the `servers` section of the YAML-file.
758
+ are obtained from the `servers` section of the YAML-file.
725
759
  E.g. from the example above:
726
760
  ```yaml
727
761
  servers:
@@ -732,11 +766,11 @@ servers:
732
766
  ```
733
767
 
734
768
 
735
- ### Kafka
769
+ #### Kafka
736
770
 
737
771
  Kafka support is currently considered experimental.
738
772
 
739
- #### Example
773
+ ##### Example
740
774
 
741
775
  datacontract.yaml
742
776
  ```yaml
@@ -748,7 +782,7 @@ servers:
748
782
  format: json
749
783
  ```
750
784
 
751
- #### Environment Variables
785
+ ##### Environment Variables
752
786
 
753
787
  | Environment Variable | Example | Description |
754
788
  |-------------------------------------|---------|----------------------------------------------------------------------------------|
@@ -757,11 +791,11 @@ servers:
757
791
  | `DATACONTRACT_KAFKA_SASL_MECHANISM` | `PLAIN` | Default `PLAIN`. Other supported mechanisms: `SCRAM-SHA-256` and `SCRAM-SHA-512` |
758
792
 
759
793
 
760
- ### Postgres
794
+ #### Postgres
761
795
 
762
796
  Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
763
797
 
764
- #### Example
798
+ ##### Example
765
799
 
766
800
  datacontract.yaml
767
801
  ```yaml
@@ -780,7 +814,7 @@ models:
780
814
  type: varchar
781
815
  ```
782
816
 
783
- #### Environment Variables
817
+ ##### Environment Variables
784
818
 
785
819
  | Environment Variable | Example | Description |
786
820
  |----------------------------------|--------------------|-------------|
@@ -788,11 +822,11 @@ models:
788
822
  | `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
789
823
 
790
824
 
791
- ### Trino
825
+ #### Trino
792
826
 
793
827
  Data Contract CLI can test data in Trino.
794
828
 
795
- #### Example
829
+ ##### Example
796
830
 
797
831
  datacontract.yaml
798
832
  ```yaml
@@ -815,7 +849,7 @@ models:
815
849
  trinoType: row(en_us varchar, pt_br varchar)
816
850
  ```
817
851
 
818
- #### Environment Variables
852
+ ##### Environment Variables
819
853
 
820
854
  | Environment Variable | Example | Description |
821
855
  |-------------------------------|--------------------|-------------|
@@ -825,46 +859,64 @@ models:
825
859
 
826
860
 
827
861
  ### export
828
-
829
862
  ```
830
- Usage: datacontract export [OPTIONS] [LOCATION]
831
-
832
- Convert data contract to a specific format. Saves to file specified by `output` option if present, otherwise prints to stdout.
833
-
834
- ╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
835
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
836
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
837
- ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
838
- │ * --format [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db The export format. [default: None] [required] │
839
- │ t-staging-sql|odcs|odcs_v2|odcs_v3|rdf|avro|protobuf │
840
- │ |great-expectations|terraform|avro-idl|sql|sql-query │
841
- │ |html|go|bigquery|dbml|spark|sqlalchemy|data-caterer │
842
- │ |dcs|markdown|iceberg] │
843
- │ --output PATH Specify the file path where the exported data will be │
844
- │ saved. If no path is provided, the output will be │
845
- │ printed to stdout. │
846
- │ [default: None] │
847
- │ --server TEXT The server name to export. [default: None] │
848
- │ --model TEXT Use the key of the model in the data contract yaml │
849
- │ file to refer to a model, e.g., `orders`, or `all` │
850
- │ for all models (default). │
851
- │ [default: all] │
852
- │ --schema TEXT The location (url or path) of the Data Contract │
853
- │ Specification JSON Schema │
854
- │ [default: │
855
- │ https://datacontract.com/datacontract.schema.json] │
856
- │ --engine TEXT [engine] The engine used for great expection run. │
857
- │ [default: None] │
858
- │ --help Show this message and exit. │
859
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
860
- ╭─ RDF Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
861
- │ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. [default: None] │
862
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
863
- ╭─ SQL Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
864
- │ --sql-server-type TEXT [sql] The server type to determine the sql dialect. By default, it uses 'auto' to automatically │
865
- │ detect the sql dialect via the specified servers in the data contract. │
866
- │ [default: auto] │
867
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
863
+
864
+ Usage: datacontract export [OPTIONS] [LOCATION]
865
+
866
+ Convert data contract to a specific format. Saves to file specified by
867
+ `output` option if present, otherwise prints to stdout.
868
+
869
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
870
+ │ location [LOCATION] The location (url or path) of the data contract │
871
+ │ yaml. │
872
+ │ [default: datacontract.yaml] │
873
+ ╰──────────────────────────────────────────────────────────────────────────────╯
874
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
875
+ │ * --format [jsonschema|pydantic-model| The export format. │
876
+ │ sodacl|dbt|dbt-sources|dbt- [default: None] │
877
+ │ staging-sql|odcs|odcs_v2|od [required] │
878
+ │ cs_v3|rdf|avro|protobuf|gre │
879
+ │ at-expectations|terraform|a │
880
+ │ vro-idl|sql|sql-query|html| │
881
+ │ go|bigquery|dbml|spark|sqla │
882
+ │ lchemy|data-caterer|dcs|mar │
883
+ │ kdown|iceberg] │
884
+ │ --output PATH Specify the file path where │
885
+ │ the exported data will be │
886
+ │ saved. If no path is │
887
+ │ provided, the output will be │
888
+ │ printed to stdout. │
889
+ │ [default: None] │
890
+ │ --server TEXT The server name to export. │
891
+ │ [default: None] │
892
+ │ --model TEXT Use the key of the model in │
893
+ │ the data contract yaml file │
894
+ │ to refer to a model, e.g., │
895
+ │ `orders`, or `all` for all │
896
+ │ models (default). │
897
+ │ [default: all] │
898
+ │ --schema TEXT The location (url or path) │
899
+ │ of the Data Contract │
900
+ │ Specification JSON Schema │
901
+ │ [default: │
902
+ │ https://datacontract.com/da… │
903
+ │ --engine TEXT [engine] The engine used for │
904
+ │ great expection run. │
905
+ │ [default: None] │
906
+ │ --help Show this message and exit. │
907
+ ╰──────────────────────────────────────────────────────────────────────────────╯
908
+ ╭─ RDF Options ────────────────────────────────────────────────────────────────╮
909
+ │ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. │
910
+ │ [default: None] │
911
+ ╰──────────────────────────────────────────────────────────────────────────────╯
912
+ ╭─ SQL Options ────────────────────────────────────────────────────────────────╮
913
+ │ --sql-server-type TEXT [sql] The server type to determine the sql │
914
+ │ dialect. By default, it uses 'auto' to │
915
+ │ automatically detect the sql dialect via the │
916
+ │ specified servers in the data contract. │
917
+ │ [default: auto] │
918
+ ╰──────────────────────────────────────────────────────────────────────────────╯
919
+
868
920
  ```
869
921
 
870
922
  ```bash
@@ -919,7 +971,7 @@ The export creates a list of expectations by utilizing:
919
971
  - The data from the Model definition with a fixed mapping
920
972
  - The expectations provided in the quality field for each model (find here the expectations gallery: [Great Expectations Gallery](https://greatexpectations.io/expectations/))
921
973
 
922
- ### Additional Arguments
974
+ ##### Additional Arguments
923
975
 
924
976
  To further customize the export, the following optional arguments are available:
925
977
 
@@ -966,8 +1018,8 @@ logical data types are exported.
966
1018
 
967
1019
  #### Spark
968
1020
 
969
- The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
970
- Spark DataFrame schema is defined as StructType. For more details about Spark Data Types please see [the spark documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
1021
+ The export function converts the data contract specification into a StructType Spark schema. The returned value is a Python code picture of the model schemas.
1022
+ Spark DataFrame schema is defined as StructType. For more details about Spark Data Types please see [the spark documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
971
1023
 
972
1024
  #### Avro
973
1025
 
@@ -981,20 +1033,6 @@ To specify custom Avro properties in your data contract, you can define them wit
981
1033
 
982
1034
  >NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
983
1035
 
984
- #### Data Caterer
985
-
986
- The export function converts the data contract to a data generation task in YAML format that can be
987
- ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the
988
- ability to generate production-like data in any environment based off your data contract.
989
-
990
- ```shell
991
- datacontract export datacontract.yaml --format data-caterer --model orders
992
- ```
993
-
994
- You can further customise the way data is generated via adding
995
- [additional metadata in the YAML](https://data.catering/setup/generator/data-generator/)
996
- to suit your needs.
997
-
998
1036
  #### Example Configuration
999
1037
 
1000
1038
  ```yaml
@@ -1025,6 +1063,20 @@ models:
1025
1063
  - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
1026
1064
  - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
1027
1065
 
1066
+ #### Data Caterer
1067
+
1068
+ The export function converts the data contract to a data generation task in YAML format that can be
1069
+ ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the
1070
+ ability to generate production-like data in any environment based off your data contract.
1071
+
1072
+ ```shell
1073
+ datacontract export datacontract.yaml --format data-caterer --model orders
1074
+ ```
1075
+
1076
+ You can further customise the way data is generated via adding
1077
+ [additional metadata in the YAML](https://data.catering/setup/generator/data-generator/)
1078
+ to suit your needs.
1079
+
1028
1080
  #### Iceberg
1029
1081
 
1030
1082
  Exports to an [Iceberg Table Json Schema Definition](https://iceberg.apache.org/spec/#appendix-c-json-serialization).
@@ -1084,55 +1136,100 @@ to limit your contract export to a single model.
1084
1136
  ```
1085
1137
 
1086
1138
  ### import
1087
-
1088
1139
  ```
1089
- Usage: datacontract import [OPTIONS]
1090
-
1091
- Create a data contract from the given source location. Saves to file specified by `output` option if present, otherwise
1092
- prints to stdout.
1093
-
1094
- ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1095
- │ * --format [sql|avro|dbt|dbml|glue|jsonschema|bigquery The format of the source file. │
1096
- │ |odcs|unity|spark|iceberg|parquet] [default: None] │
1097
- │ [required] │
1098
- │ --output PATH Specify the file path where the Data │
1099
- │ Contract will be saved. If no path is │
1100
- │ provided, the output will be printed to │
1101
- │ stdout. │
1102
- │ [default: None] │
1103
- │ --source TEXT The path to the file or Glue Database that │
1104
- │ should be imported. │
1105
- │ [default: None] │
1106
- │ --glue-table TEXT List of table ids to import from the Glue │
1107
- │ Database (repeat for multiple table ids, │
1108
- │ leave empty for all tables in the dataset). │
1109
- │ [default: None] │
1110
- │ --bigquery-project TEXT The bigquery project id. [default: None] │
1111
- │ --bigquery-dataset TEXT The bigquery dataset id. [default: None] │
1112
- │ --bigquery-table TEXT List of table ids to import from the │
1113
- │ bigquery API (repeat for multiple table ids, │
1114
- │ leave empty for all tables in the dataset). │
1115
- │ [default: None] │
1116
- │ --unity-table-full-name TEXT Full name of a table in the unity catalog │
1117
- │ [default: None] │
1118
- │ --dbt-model TEXT List of models names to import from the dbt │
1119
- │ manifest file (repeat for multiple models │
1120
- │ names, leave empty for all models in the │
1121
- │ dataset). │
1122
- │ [default: None] │
1123
- │ --dbml-schema TEXT List of schema names to import from the DBML │
1124
- │ file (repeat for multiple schema names, │
1125
- │ leave empty for all tables in the file). │
1126
- │ [default: None] │
1127
- │ --dbml-table TEXT List of table names to import from the DBML │
1128
- │ file (repeat for multiple table names, leave │
1129
- │ empty for all tables in the file). │
1130
- │ [default: None] │
1131
- │ --iceberg-table TEXT Table name to assign to the model created │
1132
- │ from the Iceberg schema. │
1133
- │ [default: None] │
1134
- │ --help Show this message and exit. │
1135
- ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1140
+
1141
+ Usage: datacontract import [OPTIONS]
1142
+
1143
+ Create a data contract from the given source location. Saves to file specified
1144
+ by `output` option if present, otherwise prints to stdout.
1145
+
1146
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1147
+ │ * --format [sql|avro|dbt|dbml|gl The format of the │
1148
+ │ ue|jsonschema|bigquer source file. │
1149
+ │ y|odcs|unity|spark|ic [default: None] │
1150
+ │ eberg|parquet|csv] [required] │
1151
+ │ --output PATH Specify the file path │
1152
+ │ where the Data │
1153
+ │ Contract will be │
1154
+ │ saved. If no path is │
1155
+ │ provided, the output │
1156
+ │ will be printed to │
1157
+ │ stdout. │
1158
+ │ [default: None] │
1159
+ │ --source TEXT The path to the file │
1160
+ │ or Glue Database that │
1161
+ │ should be imported. │
1162
+ │ [default: None] │
1163
+ │ --glue-table TEXT List of table ids to │
1164
+ │ import from the Glue │
1165
+ │ Database (repeat for │
1166
+ │ multiple table ids, │
1167
+ │ leave empty for all │
1168
+ │ tables in the │
1169
+ │ dataset). │
1170
+ │ [default: None] │
1171
+ │ --bigquery-project TEXT The bigquery project │
1172
+ │ id. │
1173
+ │ [default: None] │
1174
+ │ --bigquery-dataset TEXT The bigquery dataset │
1175
+ │ id. │
1176
+ │ [default: None] │
1177
+ │ --bigquery-table TEXT List of table ids to │
1178
+ │ import from the │
1179
+ │ bigquery API (repeat │
1180
+ │ for multiple table │
1181
+ │ ids, leave empty for │
1182
+ │ all tables in the │
1183
+ │ dataset). │
1184
+ │ [default: None] │
1185
+ │ --unity-table-full-n… TEXT Full name of a table │
1186
+ │ in the unity catalog │
1187
+ │ [default: None] │
1188
+ │ --dbt-model TEXT List of models names │
1189
+ │ to import from the │
1190
+ │ dbt manifest file │
1191
+ │ (repeat for multiple │
1192
+ │ models names, leave │
1193
+ │ empty for all models │
1194
+ │ in the dataset). │
1195
+ │ [default: None] │
1196
+ │ --dbml-schema TEXT List of schema names │
1197
+ │ to import from the │
1198
+ │ DBML file (repeat for │
1199
+ │ multiple schema │
1200
+ │ names, leave empty │
1201
+ │ for all tables in the │
1202
+ │ file). │
1203
+ │ [default: None] │
1204
+ │ --dbml-table TEXT List of table names │
1205
+ │ to import from the │
1206
+ │ DBML file (repeat for │
1207
+ │ multiple table names, │
1208
+ │ leave empty for all │
1209
+ │ tables in the file). │
1210
+ │ [default: None] │
1211
+ │ --iceberg-table TEXT Table name to assign │
1212
+ │ to the model created │
1213
+ │ from the Iceberg │
1214
+ │ schema. │
1215
+ │ [default: None] │
1216
+ │ --template TEXT The location (url or │
1217
+ │ path) of the Data │
1218
+ │ Contract │
1219
+ │ Specification │
1220
+ │ Template │
1221
+ │ [default: None] │
1222
+ │ --schema TEXT The location (url or │
1223
+ │ path) of the Data │
1224
+ │ Contract │
1225
+ │ Specification JSON │
1226
+ │ Schema │
1227
+ │ [default: │
1228
+ │ https://datacontract… │
1229
+ │ --help Show this message and │
1230
+ │ exit. │
1231
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1232
+
1136
1233
  ```
1137
1234
 
1138
1235
  Example:
@@ -1157,6 +1254,7 @@ Available import options:
1157
1254
  | `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ |
1158
1255
  | `spark` | Import from Spark StructTypes | ✅ |
1159
1256
  | `dbml` | Import from DBML models | ✅ |
1257
+ | `csv` | Import from CSV File | ✅ |
1160
1258
  | `protobuf` | Import from Protobuf schemas | TBD |
1161
1259
  | `iceberg` | Import from an Iceberg JSON Schema Definition | partial |
1162
1260
  | `parquet` | Import from Parquet File Metadta | ✅ |
@@ -1299,67 +1397,106 @@ Examples:
1299
1397
  datacontract import --format iceberg --source ./tests/fixtures/iceberg/simple_schema.json --iceberg-table test-table
1300
1398
  ```
1301
1399
 
1302
- ### breaking
1400
+ #### CSV
1303
1401
 
1304
- ```
1305
- Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW
1402
+ Importing from CSV File. Specify file in `source` parameter. It does autodetection for encoding and csv dialect
1306
1403
 
1307
- Identifies breaking changes between data contracts. Prints to stdout.
1404
+ Example:
1308
1405
 
1309
- ╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1310
- │ * location_old TEXT The location (url or path) of the old data contract yaml. [default: None] [required] │
1311
- │ * location_new TEXT The location (url or path) of the new data contract yaml. [default: None] [required] │
1312
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1313
- ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1314
- │ --help Show this message and exit. │
1315
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1406
+ ```bash
1407
+ datacontract import --format csv --source "test.csv"
1316
1408
  ```
1317
1409
 
1318
- ### changelog
1319
1410
 
1411
+ ### breaking
1320
1412
  ```
1321
- Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW
1413
+
1414
+ Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW
1415
+
1416
+ Identifies breaking changes between data contracts. Prints to stdout.
1417
+
1418
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1419
+ │ * location_old TEXT The location (url or path) of the old data │
1420
+ │ contract yaml. │
1421
+ │ [default: None] │
1422
+ │ [required] │
1423
+ │ * location_new TEXT The location (url or path) of the new data │
1424
+ │ contract yaml. │
1425
+ │ [default: None] │
1426
+ │ [required] │
1427
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1428
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1429
+ │ --help Show this message and exit. │
1430
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1322
1431
 
1323
- Generate a changelog between data contracts. Prints to stdout.
1324
-
1325
- ╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1326
- │ * location_old TEXT The location (url or path) of the old data contract yaml. [default: None] [required] │
1327
- │ * location_new TEXT The location (url or path) of the new data contract yaml. [default: None] [required] │
1328
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1329
- ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1330
- │ --help Show this message and exit. │
1331
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1332
1432
  ```
1333
1433
 
1334
- ### diff
1434
+ ### changelog
1435
+ ```
1436
+
1437
+ Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW
1438
+
1439
+ Generate a changelog between data contracts. Prints to stdout.
1440
+
1441
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1442
+ │ * location_old TEXT The location (url or path) of the old data │
1443
+ │ contract yaml. │
1444
+ │ [default: None] │
1445
+ │ [required] │
1446
+ │ * location_new TEXT The location (url or path) of the new data │
1447
+ │ contract yaml. │
1448
+ │ [default: None] │
1449
+ │ [required] │
1450
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1451
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1452
+ │ --help Show this message and exit. │
1453
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1335
1454
 
1336
1455
  ```
1337
- Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW
1338
1456
 
1339
- PLACEHOLDER. Currently works as 'changelog' does.
1457
+ ### diff
1458
+ ```
1459
+
1460
+ Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW
1461
+
1462
+ PLACEHOLDER. Currently works as 'changelog' does.
1463
+
1464
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1465
+ │ * location_old TEXT The location (url or path) of the old data │
1466
+ │ contract yaml. │
1467
+ │ [default: None] │
1468
+ │ [required] │
1469
+ │ * location_new TEXT The location (url or path) of the new data │
1470
+ │ contract yaml. │
1471
+ │ [default: None] │
1472
+ │ [required] │
1473
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1474
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1475
+ │ --help Show this message and exit. │
1476
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1340
1477
 
1341
- ╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1342
- │ * location_old TEXT The location (url or path) of the old data contract yaml. [default: None] [required] │
1343
- │ * location_new TEXT The location (url or path) of the new data contract yaml. [default: None] [required] │
1344
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1345
- ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1346
- │ --help Show this message and exit. │
1347
- ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1348
1478
  ```
1349
1479
 
1350
1480
  ### catalog
1351
-
1352
1481
  ```
1482
+
1483
+ Usage: datacontract catalog [OPTIONS]
1484
+
1485
+ Create an html catalog of data contracts.
1486
+
1487
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1488
+ │ --files TEXT Glob pattern for the data contract files to include in │
1489
+ │ the catalog. Applies recursively to any subfolders. │
1490
+ │ [default: *.yaml] │
1491
+ │ --output TEXT Output directory for the catalog html files. │
1492
+ │ [default: catalog/] │
1493
+ │ --schema TEXT The location (url or path) of the Data Contract │
1494
+ │ Specification JSON Schema │
1495
+ │ [default: │
1496
+ │ https://datacontract.com/datacontract.schema.json] │
1497
+ │ --help Show this message and exit. │
1498
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1353
1499
 
1354
- Usage: datacontract catalog [OPTIONS]
1355
-
1356
- Create an html catalog of data contracts.
1357
-
1358
- ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1359
- │ --files TEXT Glob pattern for the data contract files to include in the catalog. [default: *.yaml] │
1360
- │ --output TEXT Output directory for the catalog html files. [default: catalog/] │
1361
- │ --help Show this message and exit. │
1362
- ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1363
1500
  ```
1364
1501
 
1365
1502
  Examples:
@@ -1372,35 +1509,48 @@ datacontract catalog --output "."
1372
1509
  datacontract catalog --files "*.odcs.yaml"
1373
1510
  ```
1374
1511
 
1375
- ### Publish
1376
-
1377
- ```
1378
-
1379
- Usage: datacontract publish [OPTIONS] [LOCATION]
1380
-
1381
- Publish the data contract to the Data Mesh Manager.
1382
-
1383
- ╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1384
- │ location [LOCATION] The location (url or path) of the data contract yaml. [default: datacontract.yaml] │
1385
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1386
- ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1387
- │ --help Show this message and exit. │
1388
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1389
- ```
1390
-
1391
- ### Serve
1392
-
1393
- ```
1394
-
1395
- Usage: datacontract serve [OPTIONS]
1396
-
1397
- Start the datacontract web server.
1398
-
1399
- ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
1400
- │ --port INTEGER Bind socket to this port. [default: 4242] │
1401
- │ --host TEXT Bind socket to this host. [default: 127.0.0.1] │
1402
- │ --help Show this message and exit. │
1403
- ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
1512
+ ### publish
1513
+ ```
1514
+
1515
+ Usage: datacontract publish [OPTIONS] [LOCATION]
1516
+
1517
+ Publish the data contract to the Data Mesh Manager.
1518
+
1519
+ ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
1520
+ │ location [LOCATION] The location (url or path) of the data contract │
1521
+ │ yaml. │
1522
+ │ [default: datacontract.yaml] │
1523
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1524
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1525
+ │ --schema TEXT The location (url or │
1526
+ │ path) of the Data │
1527
+ │ Contract Specification │
1528
+ │ JSON Schema │
1529
+ │ [default: │
1530
+ │ https://datacontract.c… │
1531
+ │ --ssl-verification --no-ssl-verification SSL verification when │
1532
+ │ publishing the data │
1533
+ │ contract. │
1534
+ │ [default: │
1535
+ │ ssl-verification] │
1536
+ │ --help Show this message and │
1537
+ │ exit. │
1538
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1539
+
1540
+ ```
1541
+
1542
+ ### serve
1543
+ ```
1544
+
1545
+ Usage: datacontract serve [OPTIONS]
1546
+
1547
+ Start the datacontract web server.
1548
+
1549
+ ╭─ Options ────────────────────────────────────────────────────────────────────╮
1550
+ │ --port INTEGER Bind socket to this port. [default: 4242] │
1551
+ │ --host TEXT Bind socket to this host. [default: 127.0.0.1] │
1552
+ │ --help Show this message and exit. │
1553
+ ╰──────────────────────────────────────────────────────────────────────────────╯
1404
1554
 
1405
1555
  ```
1406
1556
 
@@ -1418,7 +1568,7 @@ If you use [Data Mesh Manager](https://datamesh-manager.com/) or [Data Contract
1418
1568
  ```bash
1419
1569
  # Fetch current data contract, execute tests on production, and publish result to data mesh manager
1420
1570
  $ EXPORT DATAMESH_MANAGER_API_KEY=xxx
1421
- $ datacontract test https://demo.datamesh-manager.com/demo279750347121/datacontracts/4df9d6ee-e55d-4088-9598-b635b2fdcbbc/datacontract.yaml \
1571
+ $ datacontract test https://demo.datamesh-manager.com/demo279750347121/datacontracts/4df9d6ee-e55d-4088-9598-b635b2fdcbbc/datacontract.yaml \
1422
1572
  --server production \
1423
1573
  --publish https://api.datamesh-manager.com/api/test-results
1424
1574
  ```
@@ -1530,7 +1680,7 @@ Examples: Removing or renaming models and fields.
1530
1680
  ```
1531
1681
 
1532
1682
  ## Customizing Exporters and Importers
1533
-
1683
+
1534
1684
  ### Custom Exporter
1535
1685
  Using the exporter factory to add a new custom exporter
1536
1686
  ```python
@@ -1576,18 +1726,18 @@ if __name__ == "__main__":
1576
1726
  Output
1577
1727
  ```python
1578
1728
  {
1579
- 'title': 'Orders Unit Test',
1580
- 'version': '1.0.0',
1581
- 'description': 'The orders data contract',
1582
- 'email': 'team-orders@example.com',
1583
- 'url': 'https://wiki.example.com/teams/checkout',
1584
- 'model': 'orders',
1585
- 'model_columns': 'order_id, order_total, order_status',
1586
- 'export_args': {'server': 'production', 'custom_arg': 'my_custom_arg'},
1729
+ 'title': 'Orders Unit Test',
1730
+ 'version': '1.0.0',
1731
+ 'description': 'The orders data contract',
1732
+ 'email': 'team-orders@example.com',
1733
+ 'url': 'https://wiki.example.com/teams/checkout',
1734
+ 'model': 'orders',
1735
+ 'model_columns': 'order_id, order_total, order_status',
1736
+ 'export_args': {'server': 'production', 'custom_arg': 'my_custom_arg'},
1587
1737
  'custom_args': 'my_custom_arg'
1588
1738
  }
1589
1739
  ```
1590
-
1740
+
1591
1741
  ### Custom Importer
1592
1742
  Using the importer factory to add a new custom importer
1593
1743
  ```python
@@ -1609,31 +1759,31 @@ class CustomImporter(Importer):
1609
1759
  data_contract_specification.info.title = source_dict.get("title")
1610
1760
  data_contract_specification.info.version = source_dict.get("version")
1611
1761
  data_contract_specification.info.description = source_dict.get("description_from_app")
1612
-
1762
+
1613
1763
  for model in source_dict.get("models", []):
1614
1764
  fields = {}
1615
1765
  for column in model.get('columns'):
1616
1766
  field = Field(
1617
- description=column.get('column_description'),
1618
- type=column.get('type')
1767
+ description=column.get('column_description'),
1768
+ type=column.get('type')
1619
1769
  )
1620
- fields[column.get('name')] = field
1621
-
1770
+ fields[column.get('name')] = field
1771
+
1622
1772
  dc_model = Model(
1623
- description=model.get('description'),
1773
+ description=model.get('description'),
1624
1774
  fields= fields
1625
1775
  )
1626
1776
 
1627
1777
  data_contract_specification.models[model.get('name')] = dc_model
1628
1778
  return data_contract_specification
1629
-
1779
+
1630
1780
 
1631
1781
  # Register the new custom class into factory
1632
1782
  importer_factory.register_importer("custom_company_importer", CustomImporter)
1633
1783
 
1634
1784
 
1635
1785
  if __name__ == "__main__":
1636
- # Get a custom data from other app
1786
+ # Get a custom data from other app
1637
1787
  json_from_custom_app = '''
1638
1788
  {
1639
1789
  "id_custom": "uuid-custom",
@@ -1665,14 +1815,14 @@ if __name__ == "__main__":
1665
1815
 
1666
1816
  # Call import_from_source
1667
1817
  result = data_contract.import_from_source(
1668
- format="custom_company_importer",
1669
- data_contract_specification=DataContract.init(),
1818
+ format="custom_company_importer",
1819
+ data_contract_specification=DataContract.init(),
1670
1820
  source=json_from_custom_app
1671
- )
1821
+ )
1672
1822
  print(result.to_yaml() )
1673
1823
  ```
1674
1824
  Output
1675
-
1825
+
1676
1826
  ```yaml
1677
1827
  dataContractSpecification: 1.1.0
1678
1828
  id: uuid-custom