datacontract-cli 0.9.3__tar.gz → 0.9.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of datacontract-cli might be problematic. Click here for more details.

Files changed (63) hide show
  1. {datacontract-cli-0.9.3/datacontract_cli.egg-info → datacontract-cli-0.9.4}/PKG-INFO +161 -29
  2. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/README.md +153 -25
  3. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/cli.py +4 -2
  4. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/data_contract.py +62 -14
  5. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/fastjsonschema/check_jsonschema.py +23 -11
  6. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/fastjsonschema/s3/s3_read_files.py +1 -1
  7. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/check_soda_execute.py +21 -2
  8. datacontract-cli-0.9.4/datacontract/engines/soda/connections/databricks.py +20 -0
  9. datacontract-cli-0.9.4/datacontract/engines/soda/connections/postgres.py +21 -0
  10. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/model/data_contract_specification.py +5 -0
  11. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4/datacontract_cli.egg-info}/PKG-INFO +161 -29
  12. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/SOURCES.txt +7 -0
  13. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/requires.txt +5 -1
  14. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/pyproject.toml +11 -7
  15. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_bigquery.py +11 -10
  16. datacontract-cli-0.9.4/tests/test_examples_databricks.py +25 -0
  17. datacontract-cli-0.9.4/tests/test_examples_examples_csv.py +25 -0
  18. datacontract-cli-0.9.4/tests/test_examples_examples_inline.py +25 -0
  19. datacontract-cli-0.9.4/tests/test_examples_examples_json.py +25 -0
  20. datacontract-cli-0.9.4/tests/test_examples_postgres.py +63 -0
  21. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/LICENSE +0 -0
  22. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/__init__.py +0 -0
  23. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/__init__.py +0 -0
  24. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +0 -0
  25. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/datacontract/check_that_datacontract_file_exists.py +0 -0
  26. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py +0 -0
  27. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/__init__.py +0 -0
  28. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/bigquery.py +0 -0
  29. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/dask.py +0 -0
  30. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/duckdb.py +0 -0
  31. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/snowflake.py +0 -0
  32. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/export/jsonschema_converter.py +0 -0
  33. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/export/sodacl_converter.py +0 -0
  34. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/init/download_datacontract_file.py +0 -0
  35. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/integration/publish_datamesh_manager.py +0 -0
  36. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/files.py +0 -0
  37. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/lint.py +0 -0
  38. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/linters/__init__.py +0 -0
  39. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/linters/example_model_linter.py +0 -0
  40. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/resolve.py +0 -0
  41. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/schema.py +0 -0
  42. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/urls.py +0 -0
  43. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/model/exceptions.py +0 -0
  44. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/model/run.py +0 -0
  45. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/dependency_links.txt +0 -0
  46. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/entry_points.txt +0 -0
  47. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/top_level.txt +0 -0
  48. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/setup.cfg +0 -0
  49. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_cli.py +0 -0
  50. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_download_datacontract_file.py +0 -0
  51. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_example_parquet.py +0 -0
  52. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_local_json.py +0 -0
  53. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_csv.py +0 -0
  54. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json.py +0 -0
  55. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json_complex.py +0 -0
  56. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json_multiple_models.py +0 -0
  57. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json_remote.py +0 -0
  58. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_snowflake.py +0 -0
  59. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_export_jsonschema.py +0 -0
  60. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_export_sodacl.py +0 -0
  61. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_integration_datameshmanager.py +0 -0
  62. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_lint.py +0 -0
  63. {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_linters.py +0 -0
@@ -1,14 +1,14 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datacontract-cli
3
- Version: 0.9.3
4
- Summary: Validate data contracts
3
+ Version: 0.9.4
4
+ Summary: Test data contracts
5
5
  Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
6
6
  Project-URL: Homepage, https://cli.datacontract.com
7
7
  Project-URL: Issues, https://github.com/datacontract/cli/issues
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.11
11
+ Requires-Python: >=3.10
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
14
  Requires-Dist: typer[all]~=0.9.0
@@ -18,15 +18,19 @@ Requires-Dist: requests~=2.31.0
18
18
  Requires-Dist: fastparquet==2023.10.1
19
19
  Requires-Dist: soda-core-bigquery~=3.1.5
20
20
  Requires-Dist: soda-core-duckdb~=3.1.5
21
+ Requires-Dist: soda-core-postgres~=3.1.5
21
22
  Requires-Dist: soda-core-snowflake~=3.1.5
23
+ Requires-Dist: soda-core-spark[databricks]~=3.1.5
24
+ Requires-Dist: soda-core-spark-df~=3.1.5
22
25
  Requires-Dist: snowflake-connector-python[pandas]<3.8,>=3.6
23
- Requires-Dist: duckdb>=0.9.3.dev3920
26
+ Requires-Dist: duckdb==0.10.0
24
27
  Requires-Dist: fastjsonschema~=2.19.1
25
28
  Requires-Dist: python-dotenv~=1.0.0
26
29
  Requires-Dist: s3fs==2024.2.0
27
30
  Provides-Extra: dev
28
31
  Requires-Dist: pytest; extra == "dev"
29
32
  Requires-Dist: testcontainers-minio; extra == "dev"
33
+ Requires-Dist: testcontainers-postgres; extra == "dev"
30
34
 
31
35
  # Data Contract CLI
32
36
 
@@ -78,6 +82,9 @@ $ datacontract lint datacontract.yaml
78
82
  # execute schema and quality checks
79
83
  $ datacontract test datacontract.yaml
80
84
 
85
+ # execute schema and quality checks on the examples within the contract
86
+ $ datacontract test --examples datacontract.yaml
87
+
81
88
  # find differences between to data contracts (Coming Soon)
82
89
  $ datacontract diff datacontract-v1.yaml datacontract-v2.yaml
83
90
 
@@ -137,17 +144,17 @@ pipx installs into an isolated environment.
137
144
  pipx install datacontract-cli
138
145
  ```
139
146
 
140
- ### Homebrew (coming soon)
147
+ ### Docker
141
148
 
142
149
  ```bash
143
- brew install datacontract/brew/datacontract
150
+ docker pull --platform linux/amd64 datacontract/cli
151
+ docker run --rm --platform linux/amd64 -v ${PWD}:/home/datacontract datacontract/cli
144
152
  ```
145
153
 
146
- ### Docker (coming soon)
154
+ Or via an alias that automatically uses the latest version:
147
155
 
148
156
  ```bash
149
- docker pull datacontract/cli
150
- docker run --rm -v ${PWD}:/datacontract datacontract/cli
157
+ alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" --platform linux/amd64 datacontract/cli:latest'
151
158
  ```
152
159
 
153
160
  ## Documentation
@@ -170,11 +177,12 @@ The application uses different engines, based on the server `type`.
170
177
  | `s3` | `json` | Support for `new_line` delimited JSON files and one JSON record per file. | ✅ | fastjsonschema<br> soda-core-duckdb |
171
178
  | `s3` | `csv` | | ✅ | soda-core-duckdb |
172
179
  | `s3` | `delta` | | Coming soon | TBD |
173
- | `postgres` | n/a | | Coming soon | TBD |
174
- | `snowflake` | n/a | | ✅ | soda-core-snowflake |
175
- | `bigquery` | n/a | | ✅ | soda-core-bigquery |
180
+ | `postgres` | n/a | | | soda-core-postgres |
181
+ | `snowflake` | n/a | | ✅ | soda-core-snowflake |
182
+ | `bigquery` | n/a | | ✅ | soda-core-bigquery |
176
183
  | `redshift` | n/a | | Coming soon | TBD |
177
- | `databricks` | n/a | | Coming soon | TBD |
184
+ | `databricks` | n/a | Support for Databricks SQL with Unity catalog and Hive metastore. | | soda-core-spark |
185
+ | `databricks` | n/a | Support for Spark for programmatic use in Notebooks. | ✅ | soda-core-spark-df |
178
186
  | `kafka` | `json` | | Coming soon | TBD |
179
187
  | `kafka` | `avro` | | Coming soon | TBD |
180
188
  | `kafka` | `protobuf` | | Coming soon | TBD |
@@ -184,9 +192,11 @@ The application uses different engines, based on the server `type`.
184
192
 
185
193
  Feel free to create an issue, if you need support for an additional type.
186
194
 
187
- ### Server Type S3
195
+ ### S3
196
+
197
+ Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
188
198
 
189
- Example:
199
+ #### Example
190
200
 
191
201
  datacontract.yaml
192
202
  ```yaml
@@ -195,24 +205,58 @@ servers:
195
205
  type: s3
196
206
  endpointUrl: https://minio.example.com # not needed with AWS S3
197
207
  location: s3://bucket-name/path/*/*.json
198
- delimiter: new_line # new_line, array, or none
199
208
  format: json
209
+ delimiter: new_line # new_line, array, or none
200
210
  ```
201
211
 
202
- Environment variables
203
- ```bash
204
- export DATACONTRACT_S3_REGION=eu-central-1
205
- export DATACONTRACT_S3_ACCESS_KEY_ID=AKIAXV5Q5QABCDEFGH
206
- export DATACONTRACT_S3_SECRET_ACCESS_KEY=93S7LRrJcqLkdb2/XXXXXXXXXXXXX
212
+ #### Environment Variables
213
+
214
+ | Environment Variable | Example | Description |
215
+ |-----------------------------------|-------------------------------|-----------------------|
216
+ | `DATACONTRACT_S3_REGION` | `eu-central-1` | Region of S3 bucket |
217
+ | `DATACONTRACT_S3_ACCESS_KEY_ID` | `AKIAXV5Q5QABCDEFGH` | AWS Access Key ID |
218
+ | `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
219
+
220
+
221
+ ### Postgres
222
+
223
+ Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
224
+
225
+ #### Example
226
+
227
+ datacontract.yaml
228
+ ```yaml
229
+ servers:
230
+ postgres:
231
+ type: postgres
232
+ host: localhost
233
+ port: 5432
234
+ database: postgres
235
+ schema: public
236
+ models:
237
+ my_table_1: # corresponds to a table
238
+ type: table
239
+ fields:
240
+ my_column_1: # corresponds to a column
241
+ type: varchar
207
242
  ```
208
243
 
209
- ### Server Type BigQuery
244
+ #### Environment Variables
245
+
246
+ | Environment Variable | Example | Description |
247
+ |----------------------------------|--------------------|-------------|
248
+ | `DATACONTRACT_POSTGRES_USERNAME` | `postgres` | Username |
249
+ | `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
250
+
251
+
252
+ ### BigQuery
210
253
 
211
254
  We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
212
255
  * BigQuery Job User
213
256
  * BigQuery Data Viewer
214
257
 
215
- Example:
258
+
259
+ #### Example
216
260
 
217
261
  datacontract.yaml
218
262
  ```yaml
@@ -227,15 +271,99 @@ models:
227
271
  fields: ...
228
272
  ```
229
273
 
230
- Required environment variable:
231
- ```bash
232
- export DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH=~/service-access-key.json # as saved on key creation by BigQuery
274
+ #### Environment Variables
275
+
276
+ | Environment Variable | Example | Description |
277
+ |----------------------------------------------|---------------------------|---------------------------------------------------------|
278
+ | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
279
+
280
+
281
+ ### Databricks
282
+
283
+ Works with Unity Catalog and Hive metastore.
284
+
285
+ Needs a running SQL warehouse or compute cluster.
286
+
287
+ #### Example
288
+
289
+ datacontract.yaml
290
+ ```yaml
291
+ servers:
292
+ production:
293
+ type: databricks
294
+ host: dbc-abcdefgh-1234.cloud.databricks.com
295
+ catalog: acme_catalog_prod
296
+ schema: orders_latest
297
+ models:
298
+ orders: # corresponds to a table
299
+ type: table
300
+ fields: ...
301
+ ```
302
+
303
+ #### Environment Variables
304
+
305
+ | Environment Variable | Example | Description |
306
+ |----------------------------------------------|--------------------------------------|-------------------------------------------------------|
307
+ | `DATACONTRACT_DATABRICKS_TOKEN` | `dapia00000000000000000000000000000` | The personal access token to authenticate |
308
+ | `DATACONTRACT_DATABRICKS_HTTP_PATH` | `/sql/1.0/warehouses/b053a3ffffffff` | The HTTP path to the SQL warehouse or compute cluster |
309
+
310
+
311
+ ### Databricks (programmatic)
312
+
313
+ Works with Unity Catalog and Hive metastore.
314
+ When running in a notebook or pipeline, the provided `spark` session can be used.
315
+ An additional authentication is not required.
316
+
317
+ Requires a Databricks Runtime with Python >= 3.10.
318
+
319
+ #### Example
320
+
321
+ datacontract.yaml
322
+ ```yaml
323
+ servers:
324
+ production:
325
+ type: databricks
326
+ host: dbc-abcdefgh-1234.cloud.databricks.com # ignored, always use current host
327
+ catalog: acme_catalog_prod
328
+ schema: orders_latest
329
+ models:
330
+ orders: # corresponds to a table
331
+ type: table
332
+ fields: ...
333
+ ```
334
+
335
+ Notebook
336
+ ```python
337
+ %pip install git+https://github.com/datacontract/cli.git
338
+ dbutils.library.restartPython()
339
+
340
+ from datacontract.data_contract import DataContract
341
+
342
+ data_contract = DataContract(
343
+ data_contract_file="/Volumes/acme_catalog_prod/orders_latest/datacontract/datacontract.yaml",
344
+ spark=spark)
345
+ run = data_contract.test()
346
+ run.result
233
347
  ```
234
348
 
349
+
350
+ ### Exports
351
+
352
+ Available export options:
353
+
354
+ | Type | Description | Status |
355
+ |--------------|------------------------------------------------|--------|
356
+ | `jsonschema` | Export to JSON Schema | ✅ |
357
+ | `sodacl` | Export to SodaCL quality checks in YAML format | ✅ |
358
+ | `dbt` | Export to dbt model in YAML format | TBD |
359
+ | `avro` | Export to AVRO models | TBD |
360
+ | `pydantic` | Export to pydantic models | TBD |
361
+ | `sql` | Export to SQL DDL | TBD |
362
+ | `protobuf` | Export to Protobuf | TBD |
363
+
235
364
  ## Development Setup
236
365
 
237
- Python base interpreter should be 3.11.x (unless
238
- working on 3.12 release candidate).
366
+ Python base interpreter should be 3.11.x (unless working on 3.12 release candidate).
239
367
 
240
368
  ```bash
241
369
  # create venv
@@ -265,7 +393,7 @@ Docker Build
265
393
 
266
394
  ```
267
395
  docker build -t datacontract/cli .
268
- docker run --rm -v ${PWD}:/datacontract datacontract/cli
396
+ docker run --rm -v ${PWD}:/home/datacontract datacontract/cli
269
397
  ```
270
398
 
271
399
  ## Contribution
@@ -279,3 +407,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
279
407
  ## Credits
280
408
 
281
409
  Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/) and [Jochen Christ](https://www.linkedin.com/in/jochenchrist/).
410
+
411
+
412
+
413
+ <a href="https://github.com/datacontract/cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
@@ -48,6 +48,9 @@ $ datacontract lint datacontract.yaml
48
48
  # execute schema and quality checks
49
49
  $ datacontract test datacontract.yaml
50
50
 
51
+ # execute schema and quality checks on the examples within the contract
52
+ $ datacontract test --examples datacontract.yaml
53
+
51
54
  # find differences between to data contracts (Coming Soon)
52
55
  $ datacontract diff datacontract-v1.yaml datacontract-v2.yaml
53
56
 
@@ -107,17 +110,17 @@ pipx installs into an isolated environment.
107
110
  pipx install datacontract-cli
108
111
  ```
109
112
 
110
- ### Homebrew (coming soon)
113
+ ### Docker
111
114
 
112
115
  ```bash
113
- brew install datacontract/brew/datacontract
116
+ docker pull --platform linux/amd64 datacontract/cli
117
+ docker run --rm --platform linux/amd64 -v ${PWD}:/home/datacontract datacontract/cli
114
118
  ```
115
119
 
116
- ### Docker (coming soon)
120
+ Or via an alias that automatically uses the latest version:
117
121
 
118
122
  ```bash
119
- docker pull datacontract/cli
120
- docker run --rm -v ${PWD}:/datacontract datacontract/cli
123
+ alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" --platform linux/amd64 datacontract/cli:latest'
121
124
  ```
122
125
 
123
126
  ## Documentation
@@ -140,11 +143,12 @@ The application uses different engines, based on the server `type`.
140
143
  | `s3` | `json` | Support for `new_line` delimited JSON files and one JSON record per file. | ✅ | fastjsonschema<br> soda-core-duckdb |
141
144
  | `s3` | `csv` | | ✅ | soda-core-duckdb |
142
145
  | `s3` | `delta` | | Coming soon | TBD |
143
- | `postgres` | n/a | | Coming soon | TBD |
144
- | `snowflake` | n/a | | ✅ | soda-core-snowflake |
145
- | `bigquery` | n/a | | ✅ | soda-core-bigquery |
146
+ | `postgres` | n/a | | | soda-core-postgres |
147
+ | `snowflake` | n/a | | ✅ | soda-core-snowflake |
148
+ | `bigquery` | n/a | | ✅ | soda-core-bigquery |
146
149
  | `redshift` | n/a | | Coming soon | TBD |
147
- | `databricks` | n/a | | Coming soon | TBD |
150
+ | `databricks` | n/a | Support for Databricks SQL with Unity catalog and Hive metastore. | | soda-core-spark |
151
+ | `databricks` | n/a | Support for Spark for programmatic use in Notebooks. | ✅ | soda-core-spark-df |
148
152
  | `kafka` | `json` | | Coming soon | TBD |
149
153
  | `kafka` | `avro` | | Coming soon | TBD |
150
154
  | `kafka` | `protobuf` | | Coming soon | TBD |
@@ -154,9 +158,11 @@ The application uses different engines, based on the server `type`.
154
158
 
155
159
  Feel free to create an issue, if you need support for an additional type.
156
160
 
157
- ### Server Type S3
161
+ ### S3
162
+
163
+ Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
158
164
 
159
- Example:
165
+ #### Example
160
166
 
161
167
  datacontract.yaml
162
168
  ```yaml
@@ -165,24 +171,58 @@ servers:
165
171
  type: s3
166
172
  endpointUrl: https://minio.example.com # not needed with AWS S3
167
173
  location: s3://bucket-name/path/*/*.json
168
- delimiter: new_line # new_line, array, or none
169
174
  format: json
175
+ delimiter: new_line # new_line, array, or none
170
176
  ```
171
177
 
172
- Environment variables
173
- ```bash
174
- export DATACONTRACT_S3_REGION=eu-central-1
175
- export DATACONTRACT_S3_ACCESS_KEY_ID=AKIAXV5Q5QABCDEFGH
176
- export DATACONTRACT_S3_SECRET_ACCESS_KEY=93S7LRrJcqLkdb2/XXXXXXXXXXXXX
178
+ #### Environment Variables
179
+
180
+ | Environment Variable | Example | Description |
181
+ |-----------------------------------|-------------------------------|-----------------------|
182
+ | `DATACONTRACT_S3_REGION` | `eu-central-1` | Region of S3 bucket |
183
+ | `DATACONTRACT_S3_ACCESS_KEY_ID` | `AKIAXV5Q5QABCDEFGH` | AWS Access Key ID |
184
+ | `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
185
+
186
+
187
+ ### Postgres
188
+
189
+ Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
190
+
191
+ #### Example
192
+
193
+ datacontract.yaml
194
+ ```yaml
195
+ servers:
196
+ postgres:
197
+ type: postgres
198
+ host: localhost
199
+ port: 5432
200
+ database: postgres
201
+ schema: public
202
+ models:
203
+ my_table_1: # corresponds to a table
204
+ type: table
205
+ fields:
206
+ my_column_1: # corresponds to a column
207
+ type: varchar
177
208
  ```
178
209
 
179
- ### Server Type BigQuery
210
+ #### Environment Variables
211
+
212
+ | Environment Variable | Example | Description |
213
+ |----------------------------------|--------------------|-------------|
214
+ | `DATACONTRACT_POSTGRES_USERNAME` | `postgres` | Username |
215
+ | `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
216
+
217
+
218
+ ### BigQuery
180
219
 
181
220
  We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
182
221
  * BigQuery Job User
183
222
  * BigQuery Data Viewer
184
223
 
185
- Example:
224
+
225
+ #### Example
186
226
 
187
227
  datacontract.yaml
188
228
  ```yaml
@@ -197,15 +237,99 @@ models:
197
237
  fields: ...
198
238
  ```
199
239
 
200
- Required environment variable:
201
- ```bash
202
- export DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH=~/service-access-key.json # as saved on key creation by BigQuery
240
+ #### Environment Variables
241
+
242
+ | Environment Variable | Example | Description |
243
+ |----------------------------------------------|---------------------------|---------------------------------------------------------|
244
+ | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
245
+
246
+
247
+ ### Databricks
248
+
249
+ Works with Unity Catalog and Hive metastore.
250
+
251
+ Needs a running SQL warehouse or compute cluster.
252
+
253
+ #### Example
254
+
255
+ datacontract.yaml
256
+ ```yaml
257
+ servers:
258
+ production:
259
+ type: databricks
260
+ host: dbc-abcdefgh-1234.cloud.databricks.com
261
+ catalog: acme_catalog_prod
262
+ schema: orders_latest
263
+ models:
264
+ orders: # corresponds to a table
265
+ type: table
266
+ fields: ...
267
+ ```
268
+
269
+ #### Environment Variables
270
+
271
+ | Environment Variable | Example | Description |
272
+ |----------------------------------------------|--------------------------------------|-------------------------------------------------------|
273
+ | `DATACONTRACT_DATABRICKS_TOKEN` | `dapia00000000000000000000000000000` | The personal access token to authenticate |
274
+ | `DATACONTRACT_DATABRICKS_HTTP_PATH` | `/sql/1.0/warehouses/b053a3ffffffff` | The HTTP path to the SQL warehouse or compute cluster |
275
+
276
+
277
+ ### Databricks (programmatic)
278
+
279
+ Works with Unity Catalog and Hive metastore.
280
+ When running in a notebook or pipeline, the provided `spark` session can be used.
281
+ An additional authentication is not required.
282
+
283
+ Requires a Databricks Runtime with Python >= 3.10.
284
+
285
+ #### Example
286
+
287
+ datacontract.yaml
288
+ ```yaml
289
+ servers:
290
+ production:
291
+ type: databricks
292
+ host: dbc-abcdefgh-1234.cloud.databricks.com # ignored, always use current host
293
+ catalog: acme_catalog_prod
294
+ schema: orders_latest
295
+ models:
296
+ orders: # corresponds to a table
297
+ type: table
298
+ fields: ...
299
+ ```
300
+
301
+ Notebook
302
+ ```python
303
+ %pip install git+https://github.com/datacontract/cli.git
304
+ dbutils.library.restartPython()
305
+
306
+ from datacontract.data_contract import DataContract
307
+
308
+ data_contract = DataContract(
309
+ data_contract_file="/Volumes/acme_catalog_prod/orders_latest/datacontract/datacontract.yaml",
310
+ spark=spark)
311
+ run = data_contract.test()
312
+ run.result
203
313
  ```
204
314
 
315
+
316
+ ### Exports
317
+
318
+ Available export options:
319
+
320
+ | Type | Description | Status |
321
+ |--------------|------------------------------------------------|--------|
322
+ | `jsonschema` | Export to JSON Schema | ✅ |
323
+ | `sodacl` | Export to SodaCL quality checks in YAML format | ✅ |
324
+ | `dbt` | Export to dbt model in YAML format | TBD |
325
+ | `avro` | Export to AVRO models | TBD |
326
+ | `pydantic` | Export to pydantic models | TBD |
327
+ | `sql` | Export to SQL DDL | TBD |
328
+ | `protobuf` | Export to Protobuf | TBD |
329
+
205
330
  ## Development Setup
206
331
 
207
- Python base interpreter should be 3.11.x (unless
208
- working on 3.12 release candidate).
332
+ Python base interpreter should be 3.11.x (unless working on 3.12 release candidate).
209
333
 
210
334
  ```bash
211
335
  # create venv
@@ -235,7 +359,7 @@ Docker Build
235
359
 
236
360
  ```
237
361
  docker build -t datacontract/cli .
238
- docker run --rm -v ${PWD}:/datacontract datacontract/cli
362
+ docker run --rm -v ${PWD}:/home/datacontract datacontract/cli
239
363
  ```
240
364
 
241
365
  ## Contribution
@@ -249,3 +373,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
249
373
  ## Credits
250
374
 
251
375
  Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/) and [Jochen Christ](https://www.linkedin.com/in/jochenchrist/).
376
+
377
+
378
+
379
+ <a href="https://github.com/datacontract/cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
@@ -87,14 +87,16 @@ def test(
87
87
  "Use the key of the server object in the data contract yaml file "
88
88
  "to refer to a server, e.g., `production`, or `all` for all "
89
89
  "servers (default).")] = "all",
90
+ examples: Annotated[bool, typer.Option(
91
+ help="Run the schema and quality tests on the example data within the data contract.")] = None,
90
92
  publish: Annotated[str, typer.Option(
91
- help="")] = None,
93
+ help="The url to publish the results after the test")] = None,
92
94
  ):
93
95
  """
94
96
  Run schema and quality tests on configured servers.
95
97
  """
96
98
  print(f"Testing {location}")
97
- run = DataContract(data_contract_file=location, publish_url=publish).test()
99
+ run = DataContract(data_contract_file=location, publish_url=publish, examples=examples).test()
98
100
  _handle_result(run)
99
101
 
100
102