datacontract-cli 0.10.11__tar.gz → 0.10.12__tar.gz

{datacontract_cli-0.10.11/datacontract_cli.egg-info → datacontract_cli-0.10.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datacontract-cli
-Version: 0.10.11
+Version: 0.10.12
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 Project-URL: Homepage, https://cli.datacontract.com
@@ -11,36 +11,34 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer[all]<0.13,>=0.9
+Requires-Dist: typer<0.13,>=0.12
 Requires-Dist: pydantic<2.9.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.33,>=2.31
-Requires-Dist: fastapi==0.111.1
+Requires-Dist: fastapi==0.112.0
+Requires-Dist: uvicorn==0.30.5
+Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
 Requires-Dist: fastparquet==2024.5.0
 Requires-Dist: python-multipart==0.0.9
 Requires-Dist: rich~=13.7.0
-Requires-Dist: simple-ddl-parser==1.5.2
+Requires-Dist: simple-ddl-parser==1.6.0
+Requires-Dist: duckdb==1.0.0
 Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
 Requires-Dist: setuptools>=60
-Requires-Dist: duckdb==1.0.0
-Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
 Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: rdflib==7.0.0
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
 Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
-Requires-Dist: boto3<1.34.137,>=1.34.41
-Requires-Dist: botocore<1.34.137,>=1.34.41
+Requires-Dist: boto3<1.35.6,>=1.34.41
 Requires-Dist: jinja_partials>=0.2.1
 Provides-Extra: avro
-Requires-Dist: avro==1.11.3; extra == "avro"
+Requires-Dist: avro==1.12.0; extra == "avro"
 Provides-Extra: bigquery
 Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
 Provides-Extra: databricks
 Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
 Requires-Dist: databricks-sql-connector<3.4.0,>=3.1.2; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
-Provides-Extra: deltalake
-Requires-Dist: deltalake<0.19,>=0.17; extra == "deltalake"
 Provides-Extra: kafka
 Requires-Dist: datacontract-cli[avro]; extra == "kafka"
 Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
@@ -49,31 +47,32 @@ Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
 Provides-Extra: s3
 Requires-Dist: s3fs==2024.6.1; extra == "s3"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python[pandas]<3.12,>=3.6; extra == "snowflake"
+Requires-Dist: snowflake-connector-python[pandas]<3.13,>=3.6; extra == "snowflake"
 Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
 Provides-Extra: sqlserver
 Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1; extra == "sqlserver"
 Provides-Extra: trino
 Requires-Dist: soda-core-trino<3.4.0,>=3.3.1; extra == "trino"
+Provides-Extra: dbt
+Requires-Dist: dbt-core>=1.8.0; extra == "dbt"
+Provides-Extra: dbml
+Requires-Dist: pydbml>=1.1.1; extra == "dbml"
 Provides-Extra: all
-Requires-Dist: datacontract-cli[bigquery,databricks,deltalake,kafka,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
+Requires-Dist: datacontract-cli[bigquery,databricks,dbml,dbt,kafka,postgres,s3,snowflake,sqlserver,trino]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
-Requires-Dist: httpx==0.27.0; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
+Requires-Dist: httpx==0.27.2; extra == "dev"
+Requires-Dist: kafka-python; extra == "dev"
+Requires-Dist: moto==5.0.13; extra == "dev"
+Requires-Dist: pandas>=2.1.0; extra == "dev"
 Requires-Dist: pre-commit<3.9.0,>=3.7.1; extra == "dev"
+Requires-Dist: pyarrow>=12.0.0; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: moto==5.0.11; extra == "dev"
-Requires-Dist: pymssql==2.3.0; extra == "dev"
-Requires-Dist: kafka-python; extra == "dev"
+Requires-Dist: pymssql==2.3.1; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: testcontainers[kafka,minio,mssql,postgres]==4.8.1; extra == "dev"
 Requires-Dist: trino==0.329.0; extra == "dev"
-Requires-Dist: testcontainers==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[core]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[minio]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[postgres]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[kafka]==4.7.2; extra == "dev"
-Requires-Dist: testcontainers[mssql]==4.7.2; extra == "dev"
 
 # Data Contract CLI
 
@@ -82,7 +81,7 @@ Requires-Dist: testcontainers[mssql]==4.7.2; extra == "dev"
     <img alt="Test Workflow" src="https://img.shields.io/github/actions/workflow/status/datacontract/datacontract-cli/ci.yaml?branch=main"></a>
   <a href="https://github.com/datacontract/datacontract-cli">
     <img alt="Stars" src="https://img.shields.io/github/stars/datacontract/datacontract-cli" /></a>
-  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://camo.githubusercontent.com/5ade1fd1e76a6ab860802cdd2941fe2501e2ca2cb534e5d8968dbf864c13d33d/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f736c61636b2d6a6f696e5f636861742d77686974652e7376673f6c6f676f3d736c61636b267374796c653d736f6369616c" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
+  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
 The `datacontract` CLI is an open source command-line tool for working with [Data Contracts](https://datacontract.com/).
@@ -267,13 +266,13 @@ A list of available extras:
 | Avro Support           | `pip install datacontract-cli[avro]`       |
 | Google BigQuery        | `pip install datacontract-cli[bigquery]`   |
 | Databricks Integration | `pip install datacontract-cli[databricks]` |
-| Deltalake Integration  | `pip install datacontract-cli[deltalake]`  |
 | Kafka Integration      | `pip install datacontract-cli[kafka]`      |
 | PostgreSQL Integration | `pip install datacontract-cli[postgres]`   |
 | S3 Integration         | `pip install datacontract-cli[s3]`         |
 | Snowflake Integration  | `pip install datacontract-cli[snowflake]`  |
 | Microsoft SQL Server   | `pip install datacontract-cli[sqlserver]`  |
 | Trino                  | `pip install datacontract-cli[trino]`      |
+| Dbt                    | `pip install datacontract-cli[dbt]`        |
 
 
 
@@ -406,6 +405,12 @@ Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/
 
 Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
 
+- CSV
+- JSON
+- Delta
+- Parquet
+- Iceberg (coming soon)
+
 #### Examples
 
 ##### JSON
@@ -444,6 +449,32 @@ servers:
 
 
 
+### Google Cloud Storage (GCS)
+
+The [S3](#S3) integration also works with files on Google Cloud Storage through its [interoperability](https://cloud.google.com/storage/docs/interoperability).
+Use `https://storage.googleapis.com` as the endpoint URL.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: s3
+    endpointUrl: https://storage.googleapis.com
+    location: s3://bucket-name/path/*/*.json # use s3:// schema instead of gs://
+    format: json
+    delimiter: new_line # new_line, array, or none
+```
+
+#### Environment Variables
+
+| Environment Variable                | Example        | Description                                                                              |
+|-------------------------------------|----------------|------------------------------------------------------------------------------------------|
+| `DATACONTRACT_S3_ACCESS_KEY_ID`     | `GOOG1EZZZ...` | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Key ID |
+| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `PDWWpb...`    | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Secret |
+
+
 ### BigQuery
 
 We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
@@ -794,7 +825,7 @@ models:
 │ *  --format        [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db  The export format. [default: None] [required]         │
 │                    t-staging-sql|odcs|rdf|avro|protobuf|great-expectati                                                        │
 │                    ons|terraform|avro-idl|sql|sql-query|html|go|bigquer                                                        │
-│                    y|dbml|spark]                                                                                               │
+│                    y|dbml|spark|sqlalchemy]                                                                                    │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -845,6 +876,7 @@ Available export options:
 | `pydantic-model`     | Export to pydantic models                               | ✅     |
 | `DBML`               | Export to a DBML Diagram description                    | ✅     |
 | `spark`              | Export to a Spark StructType                            | ✅     |
+| `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅     |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 #### Great Expectations
@@ -918,6 +950,7 @@ models:
         description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
         type: long
         example: 1672534861000000  # Equivalent to 2023-01-01 01:01:01 in microseconds
+        required: true
         config:
           avroLogicalType: local-timestamp-micros
           avroDefault: 1672534861000000
@@ -932,6 +965,7 @@ models:
   - **description**: A textual description of the field.
   - **type**: The data type of the field. In this example, it is `long`.
   - **example**: An example value for the field.
+  - **required**: Is this a required field (as opposed to optional/nullable).
   - **config**: Section to specify custom Avro properties.
     - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
     - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
@@ -968,6 +1002,14 @@ models:
 │                                                                                names, leave empty for all models in the     │
 │                                                                                dataset).                                    │
 │                                                                                [default: None]                              │
+│    --dbml-schema                  TEXT                                         List of schema names to import from the DBML │
+│                                                                                file (repeat for multiple schema names,      │
+│                                                                                leave empty for all tables in the file).     │
+│                                                                                [default: None]                              │
+│    --dbml-table                   TEXT                                         List of table names to import from the DBML  │
+│                                                                                file (repeat for multiple table names, leave │
+│                                                                                empty for all tables in the file).           │
+│                                                                                [default: None]                              │
 │    --help                                                                      Show this message and exit.                  │
 ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
@@ -991,6 +1033,7 @@ Available import options:
 | `dbt`              | Import from dbt models                         | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
 | `spark`            | Import from Spark StructTypes                  | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | TBD    |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
@@ -1078,6 +1121,38 @@ Example:
 datacontract import --format spark --source "users,orders"
 ```
 
+#### DBML
+
+Importing from DBML Documents.
+**NOTE:** Since DBML does _not_ have strict requirements on the types of columns, this import _may_ create non-valid datacontracts, as not all types of fields can be properly mapped. In this case you will have to adapt the generated document manually.
+We also assume, that the description for models and fields is stored in a Note within the DBML model.
+
+You may give the `dbml-table` or `dbml-schema` parameter to enumerate the tables or schemas that should be imported. 
+If no tables are given, _all_ available tables of the source will be imported. Likewise, if no schema is given, _all_ schemas are imported.
+
+Examples:
+
+```bash
+# Example import from DBML file, importing everything
+datacontract import --format dbml --source <file_path>
+```
+
+```bash
+# Example import from DBML file, filtering for tables from specific schemas
+datacontract import --format dbml --source <file_path> --dbml-schema <schema_1> --dbml-schema <schema_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-table <table_name_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names from a specific schema
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-schema <schema_1>
+```
+
+
 ### breaking
 
 ```
@@ -1427,7 +1502,7 @@ if __name__ == "__main__":
         "models": [
             {
             "name": "model1",
-            "desctiption": "model description from app",
+            "description": "model description from app",
             "columns": [
                 {
                 "name": "columnA",

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/README.md

@@ -5,7 +5,7 @@
     <img alt="Test Workflow" src="https://img.shields.io/github/actions/workflow/status/datacontract/datacontract-cli/ci.yaml?branch=main"></a>
   <a href="https://github.com/datacontract/datacontract-cli">
     <img alt="Stars" src="https://img.shields.io/github/stars/datacontract/datacontract-cli" /></a>
-  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://camo.githubusercontent.com/5ade1fd1e76a6ab860802cdd2941fe2501e2ca2cb534e5d8968dbf864c13d33d/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f736c61636b2d6a6f696e5f636861742d77686974652e7376673f6c6f676f3d736c61636b267374796c653d736f6369616c" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
+  <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
 The `datacontract` CLI is an open source command-line tool for working with [Data Contracts](https://datacontract.com/).
@@ -190,13 +190,13 @@ A list of available extras:
 | Avro Support           | `pip install datacontract-cli[avro]`       |
 | Google BigQuery        | `pip install datacontract-cli[bigquery]`   |
 | Databricks Integration | `pip install datacontract-cli[databricks]` |
-| Deltalake Integration  | `pip install datacontract-cli[deltalake]`  |
 | Kafka Integration      | `pip install datacontract-cli[kafka]`      |
 | PostgreSQL Integration | `pip install datacontract-cli[postgres]`   |
 | S3 Integration         | `pip install datacontract-cli[s3]`         |
 | Snowflake Integration  | `pip install datacontract-cli[snowflake]`  |
 | Microsoft SQL Server   | `pip install datacontract-cli[sqlserver]`  |
 | Trino                  | `pip install datacontract-cli[trino]`      |
+| Dbt                    | `pip install datacontract-cli[dbt]`        |
 
 
 
@@ -329,6 +329,12 @@ Feel free to create an [issue](https://github.com/datacontract/datacontract-cli/
 
 Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
 
+- CSV
+- JSON
+- Delta
+- Parquet
+- Iceberg (coming soon)
+
 #### Examples
 
 ##### JSON
@@ -367,6 +373,32 @@ servers:
 
 
 
+### Google Cloud Storage (GCS)
+
+The [S3](#S3) integration also works with files on Google Cloud Storage through its [interoperability](https://cloud.google.com/storage/docs/interoperability).
+Use `https://storage.googleapis.com` as the endpoint URL.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: s3
+    endpointUrl: https://storage.googleapis.com
+    location: s3://bucket-name/path/*/*.json # use s3:// schema instead of gs://
+    format: json
+    delimiter: new_line # new_line, array, or none
+```
+
+#### Environment Variables
+
+| Environment Variable                | Example        | Description                                                                              |
+|-------------------------------------|----------------|------------------------------------------------------------------------------------------|
+| `DATACONTRACT_S3_ACCESS_KEY_ID`     | `GOOG1EZZZ...` | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Key ID |
+| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `PDWWpb...`    | The GCS [HMAC Key](https://cloud.google.com/storage/docs/authentication/hmackeys) Secret |
+
+
 ### BigQuery
 
 We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
@@ -717,7 +749,7 @@ models:
 │ *  --format        [jsonschema|pydantic-model|sodacl|dbt|dbt-sources|db  The export format. [default: None] [required]         │
 │                    t-staging-sql|odcs|rdf|avro|protobuf|great-expectati                                                        │
 │                    ons|terraform|avro-idl|sql|sql-query|html|go|bigquer                                                        │
-│                    y|dbml|spark]                                                                                               │
+│                    y|dbml|spark|sqlalchemy]                                                                                    │
 │    --output        PATH                                                  Specify the file path where the exported data will be │
 │                                                                          saved. If no path is provided, the output will be     │
 │                                                                          printed to stdout.                                    │
@@ -768,6 +800,7 @@ Available export options:
 | `pydantic-model`     | Export to pydantic models                               | ✅     |
 | `DBML`               | Export to a DBML Diagram description                    | ✅     |
 | `spark`              | Export to a Spark StructType                            | ✅     |
+| `sqlalchemy`         | Export to SQLAlchemy Models                             | ✅     |
 | Missing something?   | Please create an issue on GitHub                        | TBD    |
 
 #### Great Expectations
@@ -841,6 +874,7 @@ models:
         description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
         type: long
         example: 1672534861000000  # Equivalent to 2023-01-01 01:01:01 in microseconds
+        required: true
         config:
           avroLogicalType: local-timestamp-micros
           avroDefault: 1672534861000000
@@ -855,6 +889,7 @@ models:
   - **description**: A textual description of the field.
   - **type**: The data type of the field. In this example, it is `long`.
   - **example**: An example value for the field.
+  - **required**: Is this a required field (as opposed to optional/nullable).
   - **config**: Section to specify custom Avro properties.
     - **avroLogicalType**: Specifies the logical type of the field in Avro. In this example, it is `local-timestamp-micros`.
     - **avroDefault**: Specifies the default value for the field in Avro. In this example, it is 1672534861000000 which corresponds to ` 2023-01-01 01:01:01 UTC`.
@@ -891,6 +926,14 @@ models:
 │                                                                                names, leave empty for all models in the     │
 │                                                                                dataset).                                    │
 │                                                                                [default: None]                              │
+│    --dbml-schema                  TEXT                                         List of schema names to import from the DBML │
+│                                                                                file (repeat for multiple schema names,      │
+│                                                                                leave empty for all tables in the file).     │
+│                                                                                [default: None]                              │
+│    --dbml-table                   TEXT                                         List of table names to import from the DBML  │
+│                                                                                file (repeat for multiple table names, leave │
+│                                                                                empty for all tables in the file).           │
+│                                                                                [default: None]                              │
 │    --help                                                                      Show this message and exit.                  │
 ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
@@ -914,6 +957,7 @@ Available import options:
 | `dbt`              | Import from dbt models                         | ✅      |
 | `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅      |
 | `spark`            | Import from Spark StructTypes                  | ✅      |
+| `dbml`             | Import from DBML models                        | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | TBD    |
 | Missing something? | Please create an issue on GitHub               | TBD    |
 
@@ -1001,6 +1045,38 @@ Example:
 datacontract import --format spark --source "users,orders"
 ```
 
+#### DBML
+
+Importing from DBML Documents.
+**NOTE:** Since DBML does _not_ have strict requirements on the types of columns, this import _may_ create non-valid datacontracts, as not all types of fields can be properly mapped. In this case you will have to adapt the generated document manually.
+We also assume, that the description for models and fields is stored in a Note within the DBML model.
+
+You may give the `dbml-table` or `dbml-schema` parameter to enumerate the tables or schemas that should be imported. 
+If no tables are given, _all_ available tables of the source will be imported. Likewise, if no schema is given, _all_ schemas are imported.
+
+Examples:
+
+```bash
+# Example import from DBML file, importing everything
+datacontract import --format dbml --source <file_path>
+```
+
+```bash
+# Example import from DBML file, filtering for tables from specific schemas
+datacontract import --format dbml --source <file_path> --dbml-schema <schema_1> --dbml-schema <schema_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-table <table_name_2>
+```
+
+```bash
+# Example import from DBML file, filtering for tables with specific names from a specific schema
+datacontract import --format dbml --source <file_path> --dbml-table <table_name_1> --dbml-schema <schema_1>
+```
+
+
 ### breaking
 
 ```
@@ -1350,7 +1426,7 @@ if __name__ == "__main__":
         "models": [
             {
             "name": "model1",
-            "desctiption": "model description from app",
+            "description": "model description from app",
             "columns": [
                 {
                 "name": "columnA",

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/datacontract/cli.py

@@ -17,7 +17,7 @@ from datacontract.catalog.catalog import create_index_html, create_data_contract
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.imports.importer import ImportFormat
 from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
-from datacontract.publish.publish import publish_to_datamesh_manager
+from datacontract.integration.datamesh_manager import publish_data_contract_to_datamesh_manager
 
 DEFAULT_DATA_CONTRACT_SCHEMA_URL = "https://datacontract.com/datacontract.schema.json"
 
@@ -232,6 +232,18 @@ def import_(
             help="List of models names to import from the dbt manifest file (repeat for multiple models names, leave empty for all models in the dataset)."
         ),
     ] = None,
+    dbml_schema: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            help="List of schema names to import from the DBML file (repeat for multiple schema names, leave empty for all tables in the file)."
+        ),
+    ] = None,
+    dbml_table: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            help="List of table names to import from the DBML file (repeat for multiple table names, leave empty for all tables in the file)."
+        ),
+    ] = None,
 ):
     """
     Create a data contract from the given source location. Prints to stdout.
@@ -245,6 +257,8 @@ def import_(
         bigquery_dataset=bigquery_dataset,
         unity_table_full_name=unity_table_full_name,
         dbt_model=dbt_model,
+        dbml_schema=dbml_schema,
+        dbml_table=dbml_table,
     )
     console.print(result.to_yaml())
 
@@ -261,8 +275,10 @@ def publish(
     """
     Publish the data contract to the Data Mesh Manager.
     """
-    publish_to_datamesh_manager(
-        data_contract=DataContract(data_contract_file=location, schema_location=schema),
+    publish_data_contract_to_datamesh_manager(
+        data_contract_specification=DataContract(
+            data_contract_file=location, schema_location=schema
+        ).get_data_contract_specification(),
     )
 
 

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/datacontract/data_contract.py

@@ -18,8 +18,8 @@ from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
 
-from datacontract.integration.publish_datamesh_manager import publish_datamesh_manager
-from datacontract.integration.publish_opentelemetry import publish_opentelemetry
+from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
+from datacontract.integration.opentelemetry import publish_test_results_to_opentelemetry
 from datacontract.lint import resolve
 from datacontract.lint.linters.description_linter import DescriptionLinter
 from datacontract.lint.linters.example_model_linter import ExampleModelLinter
@@ -218,15 +218,10 @@ class DataContract:
         run.finish()
 
         if self._publish_url is not None:
-            try:
-                publish_datamesh_manager(run, self._publish_url)
-            except Exception:
-                run.log_error("Failed to publish to datamesh manager")
+            publish_test_results_to_datamesh_manager(run, self._publish_url)
+
         if self._publish_to_opentelemetry:
-            try:
-                publish_opentelemetry(run)
-            except Exception:
-                run.log_error("Failed to publish to opentelemetry")
+            publish_test_results_to_opentelemetry(run)
 
         return run
 

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -158,6 +158,17 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
             process_local_file(run, server, model_name, validate)
         elif server.type == "s3":
             process_s3_file(server, model_name, validate)
+        elif server.type == "gcs":
+            run.checks.append(
+                Check(
+                    type="schema",
+                    name="Check that JSON has valid schema",
+                    model=model_name,
+                    result="info",
+                    reason="JSON Schema check skipped for GCS, as GCS is currently not supported",
+                    engine="jsonschema",
+                )
+            )
         else:
             run.checks.append(
                 Check(

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/datacontract/engines/fastjsonschema/s3/s3_read_files.py

@@ -28,9 +28,11 @@ def s3_fs(s3_endpoint_url):
 
     aws_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
     aws_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
+    aws_session_token = os.getenv("DATACONTRACT_S3_SESSION_TOKEN")
     return s3fs.S3FileSystem(
         key=aws_access_key_id,
         secret=aws_secret_access_key,
+        token=aws_session_token,
         anon=aws_access_key_id is None,
         client_kwargs={"endpoint_url": s3_endpoint_url},
     )

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/datacontract/engines/soda/check_soda_execute.py

@@ -1,8 +1,4 @@
 import logging
-import typing
-
-if typing.TYPE_CHECKING:
-    from pyspark.sql import SparkSession
 
 from soda.scan import Scan
 
@@ -19,9 +15,7 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 from datacontract.model.run import Run, Check, Log
 
 
-def check_soda_execute(
-    run: Run, data_contract: DataContractSpecification, server: Server, spark: "SparkSession", tmp_dir
-):
+def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
     if data_contract is None:
         run.log_warn("Cannot run engine soda-core, as data contract is invalid")
         return
@@ -29,7 +23,7 @@ def check_soda_execute(
     run.log_info("Running engine soda-core")
     scan = Scan()
 
-    if server.type in ["s3", "azure", "local"]:
+    if server.type in ["s3", "gcs", "azure", "local"]:
         if server.format in ["json", "parquet", "csv", "delta"]:
             con = get_duckdb_connection(data_contract, server, run)
             scan.add_duckdb_connection(duckdb_connection=con, data_source_name=server.type)

{datacontract_cli-0.10.11 → datacontract_cli-0.10.12}/datacontract/engines/soda/connections/duckdb.py

@@ -1,7 +1,5 @@
 import os
 
-from deltalake import DeltaTable
-
 import duckdb
 from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
 from datacontract.model.run import Run
@@ -15,6 +13,9 @@ def get_duckdb_connection(data_contract, server, run: Run):
     if server.type == "s3":
         path = server.location
         setup_s3_connection(con, server)
+    if server.type == "gcs":
+        path = server.location
+        setup_gcs_connection(con, server)
     if server.type == "azure":
         path = server.location
         setup_azure_connection(con, server)
@@ -49,28 +50,8 @@ def get_duckdb_connection(data_contract, server, run: Run):
                     f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_csv('{model_path}', hive_partitioning=1, columns={columns});"""
                 )
         elif server.format == "delta":
-            if server.type == "local":
-                delta_table_arrow = DeltaTable(model_path).to_pyarrow_dataset()
-                con.register(model_name, delta_table_arrow)
-
-            if server.type == "azure":
-                # After switching to native delta table support
-                # in https://github.com/datacontract/datacontract-cli/issues/258,
-                # azure storage should also work
-                # https://github.com/duckdb/duckdb_delta/issues/21
-                raise NotImplementedError("Support for Delta Tables on Azure Storage is not implemented yet")
-            if server.type == "s3":
-                storage_options = {
-                    "AWS_ENDPOINT_URL": server.endpointUrl,
-                    "AWS_ACCESS_KEY_ID": os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID"),
-                    "AWS_SECRET_ACCESS_KEY": os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY"),
-                    "AWS_REGION": os.getenv("DATACONTRACT_S3_REGION", "us-east-1"),
-                    "AWS_ALLOW_HTTP": "True" if server.endpointUrl.startswith("http://") else "False",
-                }
-
-                delta_table_arrow = DeltaTable(model_path, storage_options=storage_options).to_pyarrow_dataset()
-
-                con.register(model_name, delta_table_arrow)
+            con.sql("update extensions;")  # Make sure we have the latest delta extension
+            con.sql(f"""CREATE VIEW "{model_name}" AS SELECT * FROM delta_scan('{model_path}');""")
     return con
 
 
@@ -142,6 +123,24 @@ def setup_s3_connection(con, server):
     # print(con.sql("SELECT * FROM duckdb_settings() WHERE name like 's3%'"))
 
 
+def setup_gcs_connection(con, server):
+    key_id = os.getenv("DATACONTRACT_GCS_KEY_ID")
+    secret = os.getenv("DATACONTRACT_GCS_SECRET")
+
+    if key_id is None:
+        raise ValueError("Error: Environment variable DATACONTRACT_GCS_KEY_ID is not set")
+    if secret is None:
+        raise ValueError("Error: Environment variable DATACONTRACT_GCS_SECRET is not set")
+
+    con.sql(f"""
+    CREATE SECRET gcs_secret (
+        TYPE GCS,
+        KEY_ID '{key_id}',
+        SECRET '{secret}'
+    );
+    """)
+
+
 def setup_azure_connection(con, server):
     tenant_id = os.getenv("DATACONTRACT_AZURE_TENANT_ID")
     client_id = os.getenv("DATACONTRACT_AZURE_CLIENT_ID")