datacontract-cli 0.10.11__tar.gz → 0.10.13__tar.gz

{datacontract_cli-0.10.11 → datacontract_cli-0.10.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datacontract-cli
-Version: 0.10.11
+Version: 0.10.13
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 Project-URL: Homepage, https://cli.datacontract.com
@@ -11,69 +11,69 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer[all]<0.13,>=0.9
-Requires-Dist: pydantic<2.9.0,>=2.8.2
+Requires-Dist: typer<0.13,>=0.12
+Requires-Dist: pydantic<2.10.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.33,>=2.31
-Requires-Dist: fastapi==0.111.1
+Requires-Dist: fastapi==0.114.2
+Requires-Dist: uvicorn==0.30.6
+Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
 Requires-Dist: fastparquet==2024.5.0
 Requires-Dist: python-multipart==0.0.9
-Requires-Dist: rich~=13.7.0
-Requires-Dist: simple-ddl-parser==1.5.2
+Requires-Dist: rich<13.9,>=13.7
+Requires-Dist: simple-ddl-parser==1.6.1
+Requires-Dist: duckdb==1.0.0
 Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
 Requires-Dist: setuptools>=60
-Requires-Dist: duckdb==1.0.0
-Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
 Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: rdflib==7.0.0
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
 Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
-Requires-Dist: boto3<1.34.137,>=1.34.41
-Requires-Dist: botocore<1.34.137,>=1.34.41
+Requires-Dist: boto3<1.35.20,>=1.34.41
 Requires-Dist: jinja_partials>=0.2.1
 Provides-Extra: avro
-Requires-Dist: avro==1.11.3; extra == "avro"
+Requires-Dist: avro==1.12.0; extra == "avro"
 Provides-Extra: bigquery
 Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
 Provides-Extra: databricks
 Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
-Requires-Dist: databricks-sql-connector<3.4.0,>=3.1.2; extra == "databricks"
+Requires-Dist: databricks-sql-connector<3.5.0,>=3.1.2; extra == "databricks"
+Requires-Dist: databricks-sdk<0.33.0,>=0.32.0; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
-Provides-Extra: deltalake
-Requires-Dist: deltalake<0.19,>=0.17; extra == "deltalake"
 Provides-Extra: kafka
 Requires-Dist: datacontract-cli[avro]; extra == "kafka"
 Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
 Provides-Extra: postgres
 Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
 Provides-Extra: s3
-Requires-Dist: s3fs==2024.6.1; extra == "s3"
+Requires-Dist: s3fs==2024.9.0; extra == "s3"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python[pandas]<3.12,>=3.6; extra == "snowflake"
+Requires-Dist: snowflake-connector-python[pandas]<3.13,>=3.6; extra == "snowflake"
 Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
 Provides-Extra: sqlserver
 Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1; extra == "sqlserver"
 Provides-Extra: trino
 Requires-Dist: soda-core-trino<3.4.0,>=3.3.1; extra == "trino"
+Provides-Extra: dbt
+Requires-Dist: dbt-core>=1.8.0; extra == "dbt"
+Provides-Extra: dbml
+Requires-Dist: pydbml>=1.1.1; extra == "dbml"
 Provides-Extra: all
-Requires-Dist: datacontract-cli[bigquery,databricks,deltalake,kafka,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
+Requires-Dist: datacontract-cli[bigquery,databricks,dbml,dbt,kafka,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
-Requires-Dist: httpx==0.27.0; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
+Requires-Dist: httpx==0.27.2; extra == "dev"
+Requires-Dist: kafka-python; extra == "dev"
+Requires-Dist: moto==5.0.14; extra == "dev"
+Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<3.9.0,>=3.7.1; extra == "dev"
+Requires-Dist: pyarrow>=12.0.0; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: moto==5.0.11; extra == "dev"
-Requires-Dist: pymssql==2.3.0; extra == "dev"
-Requires-Dist: kafka-python; extra == "dev"
+Requires-Dist: pymssql==2.3.1; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.8.1; extra == "dev"
 Requires-Dist: trino==0.329.0; extra == "dev"
-Requires-Dist: testcontainers==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[core]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[minio]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[postgres]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[kafka]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[mssql]==4.7.2; extra == "dev"
 
 # Data Contract CLI
 
@@ -82,7 +82,7 @@ Requires-Dist: testcontainers[mssql]==4.7.2; extra == "dev"
     <img alt="Test Workflow" src="https://img.shields.io/github/actions/workflow/status/datacontract/datacontract-cli/ci.yaml?branch=main"></a>
   <a href="https://github.com/datacontract/datacontract-cli">
     <img alt="Stars" src="https://img.shields.io/github/stars/datacontract/datacontract-cli" /></a>
-  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://camo.githubusercontent.com/5ade1fd1e76a6ab860802cdd2941fe2501e2ca2cb534e5d8968dbf864c13d33d/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f736c61636b2d6a6f696e5f636861742d77686974652e7376673f6c6f676f3d736c61636b267374796c653d736f6369616c" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
+  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
 The `datacontract` CLI is an open source command-line tool for working with [Data Contracts](https://datacontract.com/).
@@ -267,13 +267,13 @@ A list of available extras:
 | Avro Support           | `pip install datacontract-cli[avro]`       |
 | Google BigQuery        | `pip install datacontract-cli[bigquery]`   |
 | Databricks Integration | `pip install datacontract-cli[databricks]` |
-| Deltalake Integration  | `pip install datacontract-cli[deltalake]`  |
 | Kafka Integration      | `pip install datacontract-cli[kafka]`      |
 | PostgreSQL Integration | `pip install datacontract-cli[postgres]`   |
 | S3 Integration         | `pip install datacontract-cli[s3]`         |
 | Snowflake Integration  | `pip install datacontract-cli[snowflake]`  |
 | Microsoft SQL Server   | `pip install datacontract-cli[sqlserver]`  |
 | Trino                  | `pip install datacontract-cli[trino]`      |
+| Dbt                    | `pip install datacontract-cli[dbt]`        |
 
 
 
@@ -406,6 +406,12 @@ Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/
 
 Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
 
+- CSV
+- JSON
+- Delta
+- Parquet
+- Iceberg (coming soon)
+
 #### Examples
 
 ##### JSON
@@ -444,6 +450,32 @@ servers:
 
 
 
+### Google Cloud Storage (GCS)
+
+The [S3](#S3) integration also works with files on Google Cloud Storage through its [interoperability](https://cloud.google.com/storage/docs/interoperability).
+Use `https://storage.googleapis.com` as the endpoint URL.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: s3
+    endpointUrl: https://storage.googleapis.com
+    location: s3://bucket-name/path/*/*.json # use s3:// schema instead of gs://
+    format: json
+    delimiter: new_line # new_line, array, or none
+```
+
+#### Environment Variables
+
+| Environment Variable                | Example        | Description                                                                              |
+|-------------------------------------|----------------|------------------------------------------------------------------------------------------|
+| `DATACONTRACT_S3_ACCESS_KEY_ID`     | `GOOG1EZZZ...` | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Key ID |
+| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `PDWWpb...`    | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Secret |
+
+
 ### BigQuery
 
 We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
@@ -794,7 +826,7 @@ models:
 │ *  --format        [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db  The export format. [default: None] [required]         │
 │                    t-staging-sql|odcs|rdf|avro|protobuf|great-expectati                                                        │
 │                    ons|terraform|avro-idl|sql|sql-query|html|go|bigquer                                                        │
-│                    y|dbml|spark]                                                                                               │
+│                    y|dbml|spark|sqlalchemy|data-caterer]                                                                       │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -845,6 +877,8 @@ Available export options:
 | `pydantic-model`     | Export to pydantic models                               | ✅     |
 | `DBML`               | Export to a DBML Diagram description                    | ✅     |
 | `spark`              | Export to a Spark StructType                            | ✅     |
+| `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅     |
+| `data-caterer`       | Export to Data Caterer in YAML format                   | ✅     |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 #### Great Expectations
@@ -908,6 +942,20 @@ To specify custom Avro properties in your data contract, you can define them wit
 
 >NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
 
+#### Data Caterer
+
+The export function converts the data contract to a data generation task in YAML format that can be 
+ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the 
+ability to generate production-like data in any environment based off your data contract.
+
+```shell
+datacontract export datacontract.yaml --format data-caterer --model orders
+```
+
+You can further customise the way data is generated via adding 
+[additional metadata in the YAML](https://data.catering/setup/generator/data-generator/) 
+to suit your needs.
+
 #### Example Configuration
 
 ```yaml
@@ -918,6 +966,7 @@ models:
         description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
         type: long
         example: 1672534861000000  # Equivalent to 2023-01-01 01:01:01 in microseconds
+        required: true
         config:
           avroLogicalType: local-timestamp-micros
           avroDefault: 1672534861000000
@@ -932,6 +981,7 @@ models:
   - **description**: A textual description of the field.
   - **type**: The data type of the field. In this example, it is `long`.
   - **example**: An example value for the field.
+  - **required**: Is this a required field (as opposed to optional/nullable).
   - **config**: Section to specify custom Avro properties.
     - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
     - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
@@ -968,6 +1018,14 @@ models:
 │                                                                                names, leave empty for all models in the     │
 │                                                                                dataset).                                    │
 │                                                                                [default: None]                              │
+│    --dbml-schema                  TEXT                                         List of schema names to import from the DBML │
+│                                                                                file (repeat for multiple schema names,      │
+│                                                                                leave empty for all tables in the file).     │
+│                                                                                [default: None]                              │
+│    --dbml-table                   TEXT                                         List of table names to import from the DBML  │
+│                                                                                file (repeat for multiple table names, leave │
+│                                                                                empty for all tables in the file).           │
+│                                                                                [default: None]                              │
 │    --help                                                                      Show this message and exit.                  │
 ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
@@ -991,6 +1049,7 @@ Available import options:
 | `dbt`              | Import from dbt models                         | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
 | `spark`            | Import from Spark StructTypes                  | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | TBD    |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
@@ -1078,6 +1137,38 @@ Example:
 datacontract import --format spark --source "users,orders"
 ```
 
+#### DBML
+
+Importing from DBML Documents.
+**NOTE:** Since DBML does _not_ have strict requirements on the types of columns, this import _may_ create non-valid datacontracts, as not all types of fields can be properly mapped. In this case you will have to adapt the generated document manually.
+We also assume, that the description for models and fields is stored in a Note within the DBML model.
+
+You may give the `dbml-table` or `dbml-schema` parameter to enumerate the tables or schemas that should be imported. 
+If no tables are given, _all_ available tables of the source will be imported. Likewise, if no schema is given, _all_ schemas are imported.
+
+Examples:
+
+```bash
+# Example import from DBML file, importing everything
+datacontract import --format dbml --source <file_path>
+```
+
+```bash
+# Example import from DBML file, filtering for tables from specific schemas
+datacontract import --format dbml --source <file_path> --dbml-schema <schema_1> --dbml-schema <schema_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-table <table_name_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names from a specific schema
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-schema <schema_1>
+```
+
+
 ### breaking
 
 ```
@@ -1427,7 +1518,7 @@ if __name__ == "__main__":
         "models": [
             {
             "name": "model1",
-            "desctiption": "model description from app",
+            "description": "model description from app",
             "columns": [
                 {
                 "name": "columnA",

datacontract_cli-0.10.11/datacontract_cli.egg-info/PKG-INFO → datacontract_cli-0.10.13/README.md

@@ -1,80 +1,3 @@
-Metadata-Version: 2.1
-Name: datacontract-cli
-Version: 0.10.11
-Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
-Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
-Project-URL: Homepage, https://cli.datacontract.com
-Project-URL: Issues, https://github.com/datacontract/datacontract-cli/issues
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: typer[all]<0.13,>=0.9
-Requires-Dist: pydantic<2.9.0,>=2.8.2
-Requires-Dist: pyyaml~=6.0.1
-Requires-Dist: requests<2.33,>=2.31
-Requires-Dist: fastapi==0.111.1
-Requires-Dist: fastparquet==2024.5.0
-Requires-Dist: python-multipart==0.0.9
-Requires-Dist: rich~=13.7.0
-Requires-Dist: simple-ddl-parser==1.5.2
-Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
-Requires-Dist: setuptools>=60
-Requires-Dist: duckdb==1.0.0
-Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
-Requires-Dist: python-dotenv~=1.0.0
-Requires-Dist: rdflib==7.0.0
-Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
-Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
-Requires-Dist: boto3<1.34.137,>=1.34.41
-Requires-Dist: botocore<1.34.137,>=1.34.41
-Requires-Dist: jinja_partials>=0.2.1
-Provides-Extra: avro
-Requires-Dist: avro==1.11.3; extra == "avro"
-Provides-Extra: bigquery
-Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
-Provides-Extra: databricks
-Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
-Requires-Dist: databricks-sql-connector<3.4.0,>=3.1.2; extra == "databricks"
-Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
-Provides-Extra: deltalake
-Requires-Dist: deltalake<0.19,>=0.17; extra == "deltalake"
-Provides-Extra: kafka
-Requires-Dist: datacontract-cli[avro]; extra == "kafka"
-Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
-Provides-Extra: postgres
-Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
-Provides-Extra: s3
-Requires-Dist: s3fs==2024.6.1; extra == "s3"
-Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python[pandas]<3.12,>=3.6; extra == "snowflake"
-Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
-Provides-Extra: sqlserver
-Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1; extra == "sqlserver"
-Provides-Extra: trino
-Requires-Dist: soda-core-trino<3.4.0,>=3.3.1; extra == "trino"
-Provides-Extra: all
-Requires-Dist: datacontract-cli[bigquery,databricks,deltalake,kafka,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
-Provides-Extra: dev
-Requires-Dist: datacontract-cli[all]; extra == "dev"
-Requires-Dist: httpx==0.27.0; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
-Requires-Dist: pre-commit<3.9.0,>=3.7.1; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: moto==5.0.11; extra == "dev"
-Requires-Dist: pymssql==2.3.0; extra == "dev"
-Requires-Dist: kafka-python; extra == "dev"
-Requires-Dist: trino==0.329.0; extra == "dev"
-Requires-Dist: testcontainers==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[core]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[minio]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[postgres]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[kafka]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[mssql]==4.7.2; extra == "dev"
-
 # Data Contract CLI
 
 <p>
@@ -82,7 +5,7 @@ Requires-Dist: testcontainers[mssql]==4.7.2; extra == "dev"
     <img alt="Test Workflow" src="https://img.shields.io/github/actions/workflow/status/datacontract/datacontract-cli/ci.yaml?branch=main"></a>
   <a href="https://github.com/datacontract/datacontract-cli">
     <img alt="Stars" src="https://img.shields.io/github/stars/datacontract/datacontract-cli" /></a>
-  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://camo.githubusercontent.com/5ade1fd1e76a6ab860802cdd2941fe2501e2ca2cb534e5d8968dbf864c13d33d/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f736c61636b2d6a6f696e5f636861742d77686974652e7376673f6c6f676f3d736c61636b267374796c653d736f6369616c" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
+  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
 The `datacontract` CLI is an open source command-line tool for working with [Data Contracts](https://datacontract.com/).
@@ -267,13 +190,13 @@ A list of available extras:
 | Avro Support           | `pip install datacontract-cli[avro]`       |
 | Google BigQuery        | `pip install datacontract-cli[bigquery]`   |
 | Databricks Integration | `pip install datacontract-cli[databricks]` |
-| Deltalake Integration  | `pip install datacontract-cli[deltalake]`  |
 | Kafka Integration      | `pip install datacontract-cli[kafka]`      |
 | PostgreSQL Integration | `pip install datacontract-cli[postgres]`   |
 | S3 Integration         | `pip install datacontract-cli[s3]`         |
 | Snowflake Integration  | `pip install datacontract-cli[snowflake]`  |
 | Microsoft SQL Server   | `pip install datacontract-cli[sqlserver]`  |
 | Trino                  | `pip install datacontract-cli[trino]`      |
+| Dbt                    | `pip install datacontract-cli[dbt]`        |
 
 
 
@@ -406,6 +329,12 @@ Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/
 
 Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
 
+- CSV
+- JSON
+- Delta
+- Parquet
+- Iceberg (coming soon)
+
 #### Examples
 
 ##### JSON
@@ -444,6 +373,32 @@ servers:
 
 
 
+### Google Cloud Storage (GCS)
+
+The [S3](#S3) integration also works with files on Google Cloud Storage through its [interoperability](https://cloud.google.com/storage/docs/interoperability).
+Use `https://storage.googleapis.com` as the endpoint URL.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: s3
+    endpointUrl: https://storage.googleapis.com
+    location: s3://bucket-name/path/*/*.json # use s3:// schema instead of gs://
+    format: json
+    delimiter: new_line # new_line, array, or none
+```
+
+#### Environment Variables
+
+| Environment Variable                | Example        | Description                                                                              |
+|-------------------------------------|----------------|------------------------------------------------------------------------------------------|
+| `DATACONTRACT_S3_ACCESS_KEY_ID`     | `GOOG1EZZZ...` | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Key ID |
+| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `PDWWpb...`    | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Secret |
+
+
 ### BigQuery
 
 We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
@@ -794,7 +749,7 @@ models:
 │ *  --format        [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db  The export format. [default: None] [required]         │
 │                    t-staging-sql|odcs|rdf|avro|protobuf|great-expectati                                                        │
 │                    ons|terraform|avro-idl|sql|sql-query|html|go|bigquer                                                        │
-│                    y|dbml|spark]                                                                                               │
+│                    y|dbml|spark|sqlalchemy|data-caterer]                                                                       │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -845,6 +800,8 @@ Available export options:
 | `pydantic-model`     | Export to pydantic models                               | ✅     |
 | `DBML`               | Export to a DBML Diagram description                    | ✅     |
 | `spark`              | Export to a Spark StructType                            | ✅     |
+| `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅     |
+| `data-caterer`       | Export to Data Caterer in YAML format                   | ✅     |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 #### Great Expectations
@@ -908,6 +865,20 @@ To specify custom Avro properties in your data contract, you can define them wit
 
 >NOTE: At this moment, we just support [logicalType](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types) and [default](https://avro.apache.org/docs/1.11.0/spec.htm)
 
+#### Data Caterer
+
+The export function converts the data contract to a data generation task in YAML format that can be 
+ingested by [Data Caterer](https://github.com/data-catering/data-caterer). This gives you the 
+ability to generate production-like data in any environment based off your data contract.
+
+```shell
+datacontract export datacontract.yaml --format data-caterer --model orders
+```
+
+You can further customise the way data is generated via adding 
+[additional metadata in the YAML](https://data.catering/setup/generator/data-generator/) 
+to suit your needs.
+
 #### Example Configuration
 
 ```yaml
@@ -918,6 +889,7 @@ models:
         description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
         type: long
         example: 1672534861000000  # Equivalent to 2023-01-01 01:01:01 in microseconds
+        required: true
         config:
           avroLogicalType: local-timestamp-micros
           avroDefault: 1672534861000000
@@ -932,6 +904,7 @@ models:
   - **description**: A textual description of the field.
   - **type**: The data type of the field. In this example, it is `long`.
   - **example**: An example value for the field.
+  - **required**: Is this a required field (as opposed to optional/nullable).
   - **config**: Section to specify custom Avro properties.
     - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
     - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
@@ -968,6 +941,14 @@ models:
 │                                                                                names, leave empty for all models in the     │
 │                                                                                dataset).                                    │
 │                                                                                [default: None]                              │
+│    --dbml-schema                  TEXT                                         List of schema names to import from the DBML │
+│                                                                                file (repeat for multiple schema names,      │
+│                                                                                leave empty for all tables in the file).     │
+│                                                                                [default: None]                              │
+│    --dbml-table                   TEXT                                         List of table names to import from the DBML  │
+│                                                                                file (repeat for multiple table names, leave │
+│                                                                                empty for all tables in the file).           │
+│                                                                                [default: None]                              │
 │    --help                                                                      Show this message and exit.                  │
 ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
@@ -991,6 +972,7 @@ Available import options:
 | `dbt`              | Import from dbt models                         | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
 | `spark`            | Import from Spark StructTypes                  | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | TBD    |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
@@ -1078,6 +1060,38 @@ Example:
 datacontract import --format spark --source "users,orders"
 ```
 
+#### DBML
+
+Importing from DBML Documents.
+**NOTE:** Since DBML does _not_ have strict requirements on the types of columns, this import _may_ create non-valid datacontracts, as not all types of fields can be properly mapped. In this case you will have to adapt the generated document manually.
+We also assume, that the description for models and fields is stored in a Note within the DBML model.
+
+You may give the `dbml-table` or `dbml-schema` parameter to enumerate the tables or schemas that should be imported. 
+If no tables are given, _all_ available tables of the source will be imported. Likewise, if no schema is given, _all_ schemas are imported.
+
+Examples:
+
+```bash
+# Example import from DBML file, importing everything
+datacontract import --format dbml --source <file_path>
+```
+
+```bash
+# Example import from DBML file, filtering for tables from specific schemas
+datacontract import --format dbml --source <file_path> --dbml-schema <schema_1> --dbml-schema <schema_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-table <table_name_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names from a specific schema
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-schema <schema_1>
+```
+
+
 ### breaking
 
 ```
@@ -1427,7 +1441,7 @@ if __name__ == "__main__":
         "models": [
             {
             "name": "model1",
-            "desctiption": "model description from app",
+            "description": "model description from app",
             "columns": [
                 {
                 "name": "columnA",

{datacontract_cli-0.10.11 → datacontract_cli-0.10.13}/datacontract/cli.py

@@ -17,7 +17,7 @@ from datacontract.catalog.catalog import create_index_html, create_data_contract
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.imports.importer import ImportFormat
 from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
-from datacontract.publish.publish import publish_to_datamesh_manager
+from datacontract.integration.datamesh_manager import publish_data_contract_to_datamesh_manager
 
 DEFAULT_DATA_CONTRACT_SCHEMA_URL = "https://datacontract.com/datacontract.schema.json"
 
@@ -232,6 +232,18 @@ def import_(
             help="List of models names to import from the dbt manifest file (repeat for multiple models names, leave empty for all models in the dataset)."
         ),
     ] = None,
+    dbml_schema: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            help="List of schema names to import from the DBML file (repeat for multiple schema names, leave empty for all tables in the file)."
+        ),
+    ] = None,
+    dbml_table: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            help="List of table names to import from the DBML file (repeat for multiple table names, leave empty for all tables in the file)."
+        ),
+    ] = None,
 ):
     """
     Create a data contract from the given source location. Prints to stdout.
@@ -245,6 +257,8 @@ def import_(
         bigquery_dataset=bigquery_dataset,
         unity_table_full_name=unity_table_full_name,
         dbt_model=dbt_model,
+        dbml_schema=dbml_schema,
+        dbml_table=dbml_table,
     )
     console.print(result.to_yaml())
 
@@ -261,8 +275,10 @@ def publish(
     """
     Publish the data contract to the Data Mesh Manager.
     """
-    publish_to_datamesh_manager(
-        data_contract=DataContract(data_contract_file=location, schema_location=schema),
+    publish_data_contract_to_datamesh_manager(
+        data_contract_specification=DataContract(
+            data_contract_file=location, schema_location=schema
+        ).get_data_contract_specification(),
     )
 
 

{datacontract_cli-0.10.11 → datacontract_cli-0.10.13}/datacontract/data_contract.py

@@ -18,8 +18,8 @@ from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
 
-from datacontract.integration.publish_datamesh_manager import publish_datamesh_manager
-from datacontract.integration.publish_opentelemetry import publish_opentelemetry
+from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
+from datacontract.integration.opentelemetry import publish_test_results_to_opentelemetry
 from datacontract.lint import resolve
 from datacontract.lint.linters.description_linter import DescriptionLinter
 from datacontract.lint.linters.example_model_linter import ExampleModelLinter
@@ -218,15 +218,10 @@ class DataContract:
         run.finish()
 
         if self._publish_url is not None:
-            try:
-                publish_datamesh_manager(run, self._publish_url)
-            except Exception:
-                run.log_error("Failed to publish to datamesh manager")
+            publish_test_results_to_datamesh_manager(run, self._publish_url)
+
         if self._publish_to_opentelemetry:
-            try:
-                publish_opentelemetry(run)
-            except Exception:
-                run.log_error("Failed to publish to opentelemetry")
+            publish_test_results_to_opentelemetry(run)
 
         return run