datacontract-cli 0.10.6__tar.gz → 0.10.8__tar.gz

{datacontract_cli-0.10.6 → datacontract_cli-0.10.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datacontract-cli
-Version: 0.10.6
+Version: 0.10.8
 Summary: Test data contracts
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
 Project-URL: Homepage, https://cli.datacontract.com
@@ -20,39 +20,55 @@ Requires-Dist: fastparquet==2024.5.0
 Requires-Dist: python-multipart==0.0.9
 Requires-Dist: rich~=13.7.0
 Requires-Dist: simple-ddl-parser==1.5.1
-Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1
 Requires-Dist: soda-core-duckdb<3.4.0,>=3.3.1
-Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1
-Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1
-Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1
-Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1
-Requires-Dist: databricks-sql-connector<3.2.0,>=3.1.2
-Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1
 Requires-Dist: setuptools>=60
-Requires-Dist: snowflake-connector-python[pandas]<3.11,>=3.6
-Requires-Dist: duckdb==0.10.3
-Requires-Dist: fastjsonschema~=2.19.1
+Requires-Dist: duckdb==1.0.0
+Requires-Dist: fastjsonschema<2.21.0,>=2.19.1
 Requires-Dist: python-dotenv~=1.0.0
-Requires-Dist: s3fs==2024.5.0
 Requires-Dist: rdflib==7.0.0
-Requires-Dist: avro==1.11.3
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc~=1.16
 Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.16
-Requires-Dist: deltalake~=0.17.0
-Requires-Dist: boto3<1.34.114,>=1.34.41
-Requires-Dist: botocore<1.34.114,>=1.34.41
+Requires-Dist: boto3<1.34.130,>=1.34.41
+Requires-Dist: botocore<1.34.128,>=1.34.41
 Requires-Dist: jinja_partials>=0.2.1
+Provides-Extra: avro
+Requires-Dist: avro==1.11.3; extra == "avro"
+Provides-Extra: bigquery
+Requires-Dist: soda-core-bigquery<3.4.0,>=3.3.1; extra == "bigquery"
+Provides-Extra: databricks
+Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "databricks"
+Requires-Dist: databricks-sql-connector<3.2.0,>=3.1.2; extra == "databricks"
+Requires-Dist: soda-core-spark[databricks]<3.4.0,>=3.3.1; extra == "databricks"
+Provides-Extra: deltalake
+Requires-Dist: deltalake<0.19,>=0.17; extra == "deltalake"
+Provides-Extra: kafka
+Requires-Dist: datacontract-cli[avro]; extra == "kafka"
+Requires-Dist: soda-core-spark-df<3.4.0,>=3.3.1; extra == "kafka"
+Provides-Extra: postgres
+Requires-Dist: soda-core-postgres<3.4.0,>=3.3.1; extra == "postgres"
+Provides-Extra: s3
+Requires-Dist: s3fs==2024.6.0; extra == "s3"
+Provides-Extra: snowflake
+Requires-Dist: snowflake-connector-python[pandas]<3.11,>=3.6; extra == "snowflake"
+Requires-Dist: soda-core-snowflake<3.4.0,>=3.3.1; extra == "snowflake"
+Provides-Extra: sqlserver
+Requires-Dist: soda-core-sqlserver<3.4.0,>=3.3.1; extra == "sqlserver"
+Provides-Extra: all
+Requires-Dist: datacontract-cli[bigquery,databricks,deltalake,kafka,postgres,s3,snowflake,sqlserver]; extra == "all"
 Provides-Extra: dev
+Requires-Dist: datacontract-cli[all]; extra == "dev"
 Requires-Dist: httpx==0.27.0; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit~=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
 Requires-Dist: moto; extra == "dev"
 Requires-Dist: pymssql==2.3.0; extra == "dev"
-Requires-Dist: testcontainers<4.0.0; extra == "dev"
-Requires-Dist: testcontainers-minio; extra == "dev"
-Requires-Dist: testcontainers-postgres; extra == "dev"
-Requires-Dist: testcontainers-kafka; extra == "dev"
+Requires-Dist: kafka-python; extra == "dev"
+Requires-Dist: testcontainers~=4.5.0; extra == "dev"
+Requires-Dist: testcontainers[minio]; extra == "dev"
+Requires-Dist: testcontainers[postgres]; extra == "dev"
+Requires-Dist: testcontainers[kafka]; extra == "dev"
 Requires-Dist: testcontainers[mssql]; extra == "dev"
 
 # Data Contract CLI
@@ -82,9 +98,10 @@ This data contract contains all information to connect to S3 and check that the
 
 Let's use [pip](https://pip.pypa.io/en/stable/getting-started/) to install the CLI (or use the [Docker image](#docker), if you prefer).
 ```bash
-$ python3 -m pip install datacontract-cli
+$ python3 -m pip install datacontract-cli[all]
 ```
 
+
 We run the tests:
 
 ```bash
@@ -154,6 +171,7 @@ $ datacontract export --format html https://datacontract.com/examples/orders-lat
 
 which will create this [HTML export](https://datacontract.com/examples/orders-latest/datacontract.html).
 
+
 ## Usage
 
 ```bash
@@ -206,13 +224,13 @@ Python 3.11 recommended.
 Python 3.12 available as pre-release release candidate for 0.9.3
 
 ```bash
-python3 -m pip install datacontract-cli
+python3 -m pip install datacontract-cli[all]
 ```
 
 ### pipx
 pipx installs into an isolated environment.
 ```bash
-pipx install datacontract-cli
+pipx install datacontract-cli[all]
 ```
 
 ### Docker
@@ -228,6 +246,32 @@ Or via an alias that automatically uses the latest version:
 alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" datacontract/cli:latest'
 ```
 
+
+## Optional Dependencies
+
+The CLI tool defines several optional dependencies (also known as extras) that can be installed for using with specific servers types.
+With _all_, all server dependencies are included.
+
+```bash
+pip install datacontract-cli[all]
+```
+
+A list of available extras:
+
+| Dependency              | Installation Command                                        |
+|-------------------------|-------------------------------------------------------------|
+| Avro Support            | `pip install datacontract-cli[avro]`                        |
+| Google BigQuery         | `pip install datacontract-cli[bigquery]`                    |
+| Databricks Integration  | `pip install datacontract-cli[databricks]`                  |
+| Deltalake Integration   | `pip install datacontract-cli[deltalake]`                  |
+| Kafka Integration       | `pip install datacontract-cli[kafka]`                       |
+| PostgreSQL Integration  | `pip install datacontract-cli[postgres]`                    |
+| S3 Integration          | `pip install datacontract-cli[s3]`                          |
+| Snowflake Integration   | `pip install datacontract-cli[snowflake]`                   |
+| Microsoft SQL Server    | `pip install datacontract-cli[sqlserver]`                   |
+
+
+
 ## Documentation
 
 Commands
@@ -333,8 +377,10 @@ Supported server types:
 - [s3](#S3)
 - [bigquery](#bigquery)
 - [azure](#azure)
+- [sqlserver](#sqlserver)
 - [databricks](#databricks)
 - [databricks (programmatic)](#databricks-programmatic)
+- [dataframr (programmatic)](#dataframe-programmatic)
 - [snowflake](#snowflake)
 - [kafka](#kafka)
 - [postgres](#postgres)
@@ -417,7 +463,7 @@ models:
 
 | Environment Variable                         | Example                   | Description                                             |
 |----------------------------------------------|---------------------------|---------------------------------------------------------|
-| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
+| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery. If this environment variable isn't set, the cli tries to use `GOOGLE_APPLICATION_CREDENTIALS` as a fallback, so if you have that set for using their Python library anyway, it should work seamlessly. |
 
 
 
@@ -448,6 +494,43 @@ Authentication works with an Azure Service Principal (SPN) aka App Registration
 
 
 
+### Sqlserver
+
+Data Contract CLI can test data in MS SQL Server (including Azure SQL, Synapse Analytics SQL Pool).
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: sqlserver
+    host: localhost
+    port: 5432
+    database: tempdb
+    schema: dbo
+    driver: ODBC Driver 18 for SQL Server
+models:
+  my_table_1: # corresponds to a table
+    type: table
+    fields:
+      my_column_1: # corresponds to a column
+        type: varchar
+```
+
+#### Environment Variables
+
+| Environment Variable             | Example            | Description |
+|----------------------------------|--------------------|-------------|
+| `DATACONTRACT_SQLSERVER_USERNAME` | `root`         | Username    |
+| `DATACONTRACT_SQLSERVER_PASSWORD` | `toor` | Password    |
+| `DATACONTRACT_SQLSERVER_TRUSTED_CONNECTION` | `True` | Use windows authentication, instead of login    |
+| `DATACONTRACT_SQLSERVER_TRUST_SERVER_CERTIFICATE` | `True` | Trust self-signed certificate    |
+| `DATACONTRACT_SQLSERVER_ENCRYPTED_CONNECTION` | `True` | Use SSL    |
+
+
+
+
 ### Databricks
 
 Works with Unity Catalog and Hive metastore.
@@ -504,7 +587,7 @@ models:
 
 Notebook
 ```python
-%pip install datacontract-cli
+%pip install datacontract-cli[databricks]
 dbutils.library.restartPython()
 
 from datacontract.data_contract import DataContract
@@ -516,6 +599,41 @@ run = data_contract.test()
 run.result
 ```
 
+### Dataframe (programmatic)
+
+Works with Spark DataFrames.
+DataFrames need to be created as named temporary views.
+Multiple temporary views are suppored if your data contract contains multiple models.
+
+Testing DataFrames is useful to test your datasets in a pipeline before writing them to a data source.
+
+#### Example
+
+datacontract.yaml
+```yaml
+servers:
+  production:
+    type: dataframe
+models:
+  my_table: # corresponds to a temporary view
+    type: table
+    fields: ...
+```
+
+Example code
+```python
+from datacontract.data_contract import DataContract
+
+df.createOrReplaceTempView("my_table")
+
+data_contract = DataContract(
+  data_contract_file="datacontract.yaml",
+  spark=spark,
+)
+run = data_contract.test()
+assert run.result == "passed"
+```
+
 
 ### Snowflake
 
@@ -608,50 +726,15 @@ models:
 
 
 
-### Postgres
-
-Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
 
-#### Example
+### export
 
-datacontract.yaml
-```yaml
-servers:
-  postgres:
-    type: sqlserver
-    host: localhost
-    port: 5432
-    database: tempdb
-    schema: dbo
-    driver: ODBC Driver 18 for SQL Server
-models:
-  my_table_1: # corresponds to a table
-    type: table
-    fields:
-      my_column_1: # corresponds to a column
-        type: varchar
 ```
 
-#### Environment Variables
-
-| Environment Variable             | Example            | Description |
-|----------------------------------|--------------------|-------------|
-| `DATACONTRACT_SQLSERVER_USERNAME` | `root`         | Username    |
-| `DATACONTRACT_SQLSERVER_PASSWORD` | `toor` | Password    |
-| `DATACONTRACT_SQLSERVER_TRUSTED_CONNECTION` | `True` | Use windows authentication, instead of login    |
-| `DATACONTRACT_SQLSERVER_TRUST_SERVER_CERTIFICATE` | `True` | Trust self-signed certificate    |
-| `DATACONTRACT_SQLSERVER_ENCRYPTED_CONNECTION` | `True` | Use SSL    |
-
+ Usage: datacontract export [OPTIONS] [LOCATION]
 
+ Convert data contract to a specific format. Prints to stdout or to the specified output file.
 
-### export
-
-```
-
- Usage: datacontract export [OPTIONS] [LOCATION]                                                                                                                           
-                                                                                                                                                                           
- Convert data contract to a specific format. Prints to stdout or to the specified output file.                                                                                                     
-                                                                                                                                                                           
 ╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
 │   location      [LOCATION]  The location (url or path) of the data contract yaml. [default: datacontract.yaml]                 │
 ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
@@ -752,13 +835,13 @@ data products, find the true domain owner of a field attribute)
 #### DBML
 
 The export function converts the logical data types of the datacontract into the specific ones of a concrete Database
-if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the 
+if a server is selected via the `--server` option (based on the `type` of that server). If no server is selected, the
 logical data types are exported.
 
 
 #### Avro
 
-The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values. 
+The export function converts the data contract specification into an avro schema. It supports specifying custom avro properties for logicalTypes and default values.
 
 ##### Custom Avro Properties
 
@@ -778,7 +861,7 @@ models:
         description: Example for AVRO with Timestamp (microsecond precision) https://avro.apache.org/docs/current/spec.html#Local+timestamp+%28microsecond+precision%29
         type: long
         example: 1672534861000000  # Equivalent to 2023-01-01 01:01:01 in microseconds
-        config:          
+        config:
           avroLogicalType: local-timestamp-micros
           avroDefault: 1672534861000000
 ```
@@ -804,39 +887,28 @@ models:
 
  Create a data contract from the given source location. Prints to stdout.
 
-╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-│ *  --format                  [sql|avro|glue|bigquery|jsonschema]  The format of the source file. [default: None] [required]                                   │
-│    --source                  TEXT                      The path to the file or Glue Database that should be imported. [default: None]                         │
-│    --bigquery-project        TEXT                      The bigquery project id. [default: None]                                                               │
-│    --bigquery-dataset        TEXT                      The bigquery dataset id. [default: None]                                                               │
-│    --bigquery-table          TEXT                      List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all  │
-│                                                        tables in the dataset).                                                                                │
-│                                                        [default: None]                                                                                        │
-│    --help                                              Show this message and exit.                                                                            │
-╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+│ *  --format                  [sql|avro|glue|bigquery|jsonschema|unity]  The format of the source file. [default: None] [required]         │
+│    --source                  TEXT                                       The path to the file or Glue Database that should be imported.    │
+│                                                                         [default: None]                                                   │
+│    --glue-table              TEXT                                       List of table ids to import from the Glue Database (repeat for    │
+│                                                                         multiple table ids, leave empty for all tables in the dataset).   │
+│                                                                         [default: None]                                                   │
+│    --bigquery-project        TEXT                                       The bigquery project id. [default: None]                          │
+│    --bigquery-dataset        TEXT                                       The bigquery dataset id. [default: None]                          │
+│    --bigquery-table          TEXT                                       List of table ids to import from the bigquery API (repeat for     │
+│                                                                         multiple table ids, leave empty for all tables in the dataset).   │
+│                                                                         [default: None]                                                   │
+│    --help                                                               Show this message and exit.                                       │
+╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
 
-As shown, some options are only relevant in certain conditions: For `format` Bigtable we support to directly read off the Bigtable APIs.
-In this case there's no need to specify `source` but instead `bt-project-id`, `bt-dataset-id` and `table` must be specified.
-
-For providing authentication to the Client, please see [the google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to) or the one [about authorizing client libraries](https://cloud.google.com/bigquery/docs/authentication#client-libs).
-
 Example:
 ```bash
 # Example import from SQL DDL
 datacontract import --format sql --source my_ddl.sql
 ```
 
-```bash
-# Example import from Bigquery JSON
-datacontract import --format bigquery --source my_bigquery_table.json
-```
-
-```bash
-# Example import from Bigquery API
-datacontract import --format bigquery --btProjectId <project_id> --btDatasetId <dataset_id> --table <tableid_1> --table <tableid_2> --table <tableid_3>
-```
-
 Available import options:
 
 | Type               | Description                                    | Status  |
@@ -845,13 +917,71 @@ Available import options:
 | `avro`             | Import from AVRO schemas                       | ✅      |
 | `glue`             | Import from AWS Glue DataCatalog               | ✅      |
 | `protobuf`         | Import from Protobuf schemas                   | TBD     |
-| `jsonschema`       | Import from JSON Schemas                       | ✅      |
-| `bigquery`         | Import from BigQuery Schemas                   | ✅      |
+| `jsonschema`       | Import from JSON Schemas                       | ✅       |
+| `bigquery`         | Import from BigQuery Schemas                   | ✅       |
+| `unity`            | Import from Databricks Unity Catalog           | partial |
 | `dbt`              | Import from dbt models                         | TBD     |
-| `odcs`             | Import from Open Data Contract Standard (ODCS) | TBD     |
+| `odcs`             | Import from Open Data Contract Standard (ODCS) | ✅       |
 | Missing something? | Please create an issue on GitHub               | TBD     |
 
 
+#### BigQuery
+
+Bigquery data can either be imported off of JSON Files generated from the table descriptions or directly from the Bigquery API. In case you want to use JSON Files, specify the `source` parameter with a path to the JSON File.
+
+To import from the Bigquery API, you have to _omit_ `source` and instead need to provide `bigquery-project` and `bigquery-dataset`. Additionally you may specify `bigquery-table` to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the dataset will be imported.
+
+For providing authentication to the Client, please see [the google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to) or the one [about authorizing client libraries](https://cloud.google.com/bigquery/docs/authentication#client-libs).
+
+Examples:
+
+```bash
+# Example import from Bigquery JSON
+datacontract import --format bigquery --source my_bigquery_table.json
+```
+
+```bash
+# Example import from Bigquery API with specifying the tables to import
+datacontract import --format bigquery --bigquery-project <project_id> --bigquery-dataset <dataset_id> --bigquery-table <tableid_1> --bigquery-table <tableid_2> --bigquery-table <tableid_3>
+```
+
+```bash
+# Example import from Bigquery API importing all tables in the dataset
+datacontract import --format bigquery --bigquery-project <project_id> --bigquery-dataset <dataset_id>
+```
+
+#### Unity Catalog
+
+```bash
+# Example import from a Unity Catalog JSON file
+datacontract import --format unity --source my_unity_table.json
+```
+
+```bash
+# Example import single table from Unity Catalog via HTTP endpoint
+export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com"
+export DATABRICKS_IMPORT_ACCESS_TOKEN=<token>
+datacontract import --format unity --unity-table-full-name <table_full_name>
+```
+
+### Glue
+
+Importing from Glue reads the necessary Data directly off of the AWS API.
+You may give the `glue-table` parameter to enumerate the tables that should be imported. If no tables are given, _all_ available tables of the database will be imported.
+
+Examples:
+
+```bash
+# Example import from AWS Glue with specifying the tables to import
+datacontract import --format glue --source <database_name> --glue-table <table_name_1> --glue-table <table_name_2> --glue-table <table_name_3>
+```
+
+```bash
+# Example import from AWS Glue importing all tables in the database
+datacontract import --format glue --source <database_name>
+```
+
+
 ### breaking
 
 ```
@@ -1096,8 +1226,8 @@ source venv/bin/activate
 # Install Requirements
 pip install --upgrade pip setuptools wheel
 pip install -e '.[dev]'
-ruff check --fix
-ruff format
+pre-commit install
+pre-commit run --all-files
 pytest
 ```
 
@@ -1153,6 +1283,12 @@ We are happy to receive your contributions. Propose your change in an issue or d
 - [INNOQ](https://innoq.com)
 - And many more. To add your company, please create a pull request.
 
+## Related Tools
+
+- [Data Mesh Manager](https://www.datamesh-manager.com/) is a commercial tool to manage data products and data contracts. It supports the data contract specification and allows the user to import or export data contracts using this specification.
+- [Data Contract GPT](https://gpt.datacontract.com) is a custom GPT that can help you write data contracts.
+- [Data Contract Editor](https://editor.datacontract.com) is an editor for Data Contracts, including a live html preview.
+
 ## License
 
 [MIT License](LICENSE)