datacontract-cli 0.9.3__tar.gz → 0.9.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- {datacontract-cli-0.9.3/datacontract_cli.egg-info → datacontract-cli-0.9.4}/PKG-INFO +161 -29
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/README.md +153 -25
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/cli.py +4 -2
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/data_contract.py +62 -14
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/fastjsonschema/check_jsonschema.py +23 -11
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/fastjsonschema/s3/s3_read_files.py +1 -1
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/check_soda_execute.py +21 -2
- datacontract-cli-0.9.4/datacontract/engines/soda/connections/databricks.py +20 -0
- datacontract-cli-0.9.4/datacontract/engines/soda/connections/postgres.py +21 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/model/data_contract_specification.py +5 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4/datacontract_cli.egg-info}/PKG-INFO +161 -29
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/SOURCES.txt +7 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/requires.txt +5 -1
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/pyproject.toml +11 -7
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_bigquery.py +11 -10
- datacontract-cli-0.9.4/tests/test_examples_databricks.py +25 -0
- datacontract-cli-0.9.4/tests/test_examples_examples_csv.py +25 -0
- datacontract-cli-0.9.4/tests/test_examples_examples_inline.py +25 -0
- datacontract-cli-0.9.4/tests/test_examples_examples_json.py +25 -0
- datacontract-cli-0.9.4/tests/test_examples_postgres.py +63 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/LICENSE +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/__init__.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/__init__.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/datacontract/check_that_datacontract_file_exists.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/__init__.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/bigquery.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/dask.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/duckdb.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/engines/soda/connections/snowflake.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/export/jsonschema_converter.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/export/sodacl_converter.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/init/download_datacontract_file.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/integration/publish_datamesh_manager.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/files.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/lint.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/linters/__init__.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/linters/example_model_linter.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/resolve.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/schema.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/lint/urls.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/model/exceptions.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract/model/run.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/dependency_links.txt +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/entry_points.txt +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/datacontract_cli.egg-info/top_level.txt +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/setup.cfg +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_cli.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_download_datacontract_file.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_example_parquet.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_local_json.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_csv.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json_complex.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json_multiple_models.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_s3_json_remote.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_examples_snowflake.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_export_jsonschema.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_export_sodacl.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_integration_datameshmanager.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_lint.py +0 -0
- {datacontract-cli-0.9.3 → datacontract-cli-0.9.4}/tests/test_linters.py +0 -0
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datacontract-cli
|
|
3
|
-
Version: 0.9.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.9.4
|
|
4
|
+
Summary: Test data contracts
|
|
5
5
|
Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>
|
|
6
6
|
Project-URL: Homepage, https://cli.datacontract.com
|
|
7
7
|
Project-URL: Issues, https://github.com/datacontract/cli/issues
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: typer[all]~=0.9.0
|
|
@@ -18,15 +18,19 @@ Requires-Dist: requests~=2.31.0
|
|
|
18
18
|
Requires-Dist: fastparquet==2023.10.1
|
|
19
19
|
Requires-Dist: soda-core-bigquery~=3.1.5
|
|
20
20
|
Requires-Dist: soda-core-duckdb~=3.1.5
|
|
21
|
+
Requires-Dist: soda-core-postgres~=3.1.5
|
|
21
22
|
Requires-Dist: soda-core-snowflake~=3.1.5
|
|
23
|
+
Requires-Dist: soda-core-spark[databricks]~=3.1.5
|
|
24
|
+
Requires-Dist: soda-core-spark-df~=3.1.5
|
|
22
25
|
Requires-Dist: snowflake-connector-python[pandas]<3.8,>=3.6
|
|
23
|
-
Requires-Dist: duckdb
|
|
26
|
+
Requires-Dist: duckdb==0.10.0
|
|
24
27
|
Requires-Dist: fastjsonschema~=2.19.1
|
|
25
28
|
Requires-Dist: python-dotenv~=1.0.0
|
|
26
29
|
Requires-Dist: s3fs==2024.2.0
|
|
27
30
|
Provides-Extra: dev
|
|
28
31
|
Requires-Dist: pytest; extra == "dev"
|
|
29
32
|
Requires-Dist: testcontainers-minio; extra == "dev"
|
|
33
|
+
Requires-Dist: testcontainers-postgres; extra == "dev"
|
|
30
34
|
|
|
31
35
|
# Data Contract CLI
|
|
32
36
|
|
|
@@ -78,6 +82,9 @@ $ datacontract lint datacontract.yaml
|
|
|
78
82
|
# execute schema and quality checks
|
|
79
83
|
$ datacontract test datacontract.yaml
|
|
80
84
|
|
|
85
|
+
# execute schema and quality checks on the examples within the contract
|
|
86
|
+
$ datacontract test --examples datacontract.yaml
|
|
87
|
+
|
|
81
88
|
# find differences between to data contracts (Coming Soon)
|
|
82
89
|
$ datacontract diff datacontract-v1.yaml datacontract-v2.yaml
|
|
83
90
|
|
|
@@ -137,17 +144,17 @@ pipx installs into an isolated environment.
|
|
|
137
144
|
pipx install datacontract-cli
|
|
138
145
|
```
|
|
139
146
|
|
|
140
|
-
###
|
|
147
|
+
### Docker
|
|
141
148
|
|
|
142
149
|
```bash
|
|
143
|
-
|
|
150
|
+
docker pull --platform linux/amd64 datacontract/cli
|
|
151
|
+
docker run --rm --platform linux/amd64 -v ${PWD}:/home/datacontract datacontract/cli
|
|
144
152
|
```
|
|
145
153
|
|
|
146
|
-
|
|
154
|
+
Or via an alias that automatically uses the latest version:
|
|
147
155
|
|
|
148
156
|
```bash
|
|
149
|
-
docker
|
|
150
|
-
docker run --rm -v ${PWD}:/datacontract datacontract/cli
|
|
157
|
+
alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" --platform linux/amd64 datacontract/cli:latest'
|
|
151
158
|
```
|
|
152
159
|
|
|
153
160
|
## Documentation
|
|
@@ -170,11 +177,12 @@ The application uses different engines, based on the server `type`.
|
|
|
170
177
|
| `s3` | `json` | Support for `new_line` delimited JSON files and one JSON record per file. | ✅ | fastjsonschema<br> soda-core-duckdb |
|
|
171
178
|
| `s3` | `csv` | | ✅ | soda-core-duckdb |
|
|
172
179
|
| `s3` | `delta` | | Coming soon | TBD |
|
|
173
|
-
| `postgres` | n/a | |
|
|
174
|
-
| `snowflake` | n/a | | ✅
|
|
175
|
-
| `bigquery` | n/a
|
|
180
|
+
| `postgres` | n/a | | ✅ | soda-core-postgres |
|
|
181
|
+
| `snowflake` | n/a | | ✅ | soda-core-snowflake |
|
|
182
|
+
| `bigquery` | n/a | | ✅ | soda-core-bigquery |
|
|
176
183
|
| `redshift` | n/a | | Coming soon | TBD |
|
|
177
|
-
| `databricks` | n/a |
|
|
184
|
+
| `databricks` | n/a | Support for Databricks SQL with Unity catalog and Hive metastore. | ✅ | soda-core-spark |
|
|
185
|
+
| `databricks` | n/a | Support for Spark for programmatic use in Notebooks. | ✅ | soda-core-spark-df |
|
|
178
186
|
| `kafka` | `json` | | Coming soon | TBD |
|
|
179
187
|
| `kafka` | `avro` | | Coming soon | TBD |
|
|
180
188
|
| `kafka` | `protobuf` | | Coming soon | TBD |
|
|
@@ -184,9 +192,11 @@ The application uses different engines, based on the server `type`.
|
|
|
184
192
|
|
|
185
193
|
Feel free to create an issue, if you need support for an additional type.
|
|
186
194
|
|
|
187
|
-
###
|
|
195
|
+
### S3
|
|
196
|
+
|
|
197
|
+
Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
|
|
188
198
|
|
|
189
|
-
Example
|
|
199
|
+
#### Example
|
|
190
200
|
|
|
191
201
|
datacontract.yaml
|
|
192
202
|
```yaml
|
|
@@ -195,24 +205,58 @@ servers:
|
|
|
195
205
|
type: s3
|
|
196
206
|
endpointUrl: https://minio.example.com # not needed with AWS S3
|
|
197
207
|
location: s3://bucket-name/path/*/*.json
|
|
198
|
-
delimiter: new_line # new_line, array, or none
|
|
199
208
|
format: json
|
|
209
|
+
delimiter: new_line # new_line, array, or none
|
|
200
210
|
```
|
|
201
211
|
|
|
202
|
-
Environment
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
212
|
+
#### Environment Variables
|
|
213
|
+
|
|
214
|
+
| Environment Variable | Example | Description |
|
|
215
|
+
|-----------------------------------|-------------------------------|-----------------------|
|
|
216
|
+
| `DATACONTRACT_S3_REGION` | `eu-central-1` | Region of S3 bucket |
|
|
217
|
+
| `DATACONTRACT_S3_ACCESS_KEY_ID` | `AKIAXV5Q5QABCDEFGH` | AWS Access Key ID |
|
|
218
|
+
| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
### Postgres
|
|
222
|
+
|
|
223
|
+
Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
|
|
224
|
+
|
|
225
|
+
#### Example
|
|
226
|
+
|
|
227
|
+
datacontract.yaml
|
|
228
|
+
```yaml
|
|
229
|
+
servers:
|
|
230
|
+
postgres:
|
|
231
|
+
type: postgres
|
|
232
|
+
host: localhost
|
|
233
|
+
port: 5432
|
|
234
|
+
database: postgres
|
|
235
|
+
schema: public
|
|
236
|
+
models:
|
|
237
|
+
my_table_1: # corresponds to a table
|
|
238
|
+
type: table
|
|
239
|
+
fields:
|
|
240
|
+
my_column_1: # corresponds to a column
|
|
241
|
+
type: varchar
|
|
207
242
|
```
|
|
208
243
|
|
|
209
|
-
|
|
244
|
+
#### Environment Variables
|
|
245
|
+
|
|
246
|
+
| Environment Variable | Example | Description |
|
|
247
|
+
|----------------------------------|--------------------|-------------|
|
|
248
|
+
| `DATACONTRACT_POSTGRES_USERNAME` | `postgres` | Username |
|
|
249
|
+
| `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
### BigQuery
|
|
210
253
|
|
|
211
254
|
We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
|
|
212
255
|
* BigQuery Job User
|
|
213
256
|
* BigQuery Data Viewer
|
|
214
257
|
|
|
215
|
-
|
|
258
|
+
|
|
259
|
+
#### Example
|
|
216
260
|
|
|
217
261
|
datacontract.yaml
|
|
218
262
|
```yaml
|
|
@@ -227,15 +271,99 @@ models:
|
|
|
227
271
|
fields: ...
|
|
228
272
|
```
|
|
229
273
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
274
|
+
#### Environment Variables
|
|
275
|
+
|
|
276
|
+
| Environment Variable | Example | Description |
|
|
277
|
+
|----------------------------------------------|---------------------------|---------------------------------------------------------|
|
|
278
|
+
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
### Databricks
|
|
282
|
+
|
|
283
|
+
Works with Unity Catalog and Hive metastore.
|
|
284
|
+
|
|
285
|
+
Needs a running SQL warehouse or compute cluster.
|
|
286
|
+
|
|
287
|
+
#### Example
|
|
288
|
+
|
|
289
|
+
datacontract.yaml
|
|
290
|
+
```yaml
|
|
291
|
+
servers:
|
|
292
|
+
production:
|
|
293
|
+
type: databricks
|
|
294
|
+
host: dbc-abcdefgh-1234.cloud.databricks.com
|
|
295
|
+
catalog: acme_catalog_prod
|
|
296
|
+
schema: orders_latest
|
|
297
|
+
models:
|
|
298
|
+
orders: # corresponds to a table
|
|
299
|
+
type: table
|
|
300
|
+
fields: ...
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
#### Environment Variables
|
|
304
|
+
|
|
305
|
+
| Environment Variable | Example | Description |
|
|
306
|
+
|----------------------------------------------|--------------------------------------|-------------------------------------------------------|
|
|
307
|
+
| `DATACONTRACT_DATABRICKS_TOKEN` | `dapia00000000000000000000000000000` | The personal access token to authenticate |
|
|
308
|
+
| `DATACONTRACT_DATABRICKS_HTTP_PATH` | `/sql/1.0/warehouses/b053a3ffffffff` | The HTTP path to the SQL warehouse or compute cluster |
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
### Databricks (programmatic)
|
|
312
|
+
|
|
313
|
+
Works with Unity Catalog and Hive metastore.
|
|
314
|
+
When running in a notebook or pipeline, the provided `spark` session can be used.
|
|
315
|
+
An additional authentication is not required.
|
|
316
|
+
|
|
317
|
+
Requires a Databricks Runtime with Python >= 3.10.
|
|
318
|
+
|
|
319
|
+
#### Example
|
|
320
|
+
|
|
321
|
+
datacontract.yaml
|
|
322
|
+
```yaml
|
|
323
|
+
servers:
|
|
324
|
+
production:
|
|
325
|
+
type: databricks
|
|
326
|
+
host: dbc-abcdefgh-1234.cloud.databricks.com # ignored, always use current host
|
|
327
|
+
catalog: acme_catalog_prod
|
|
328
|
+
schema: orders_latest
|
|
329
|
+
models:
|
|
330
|
+
orders: # corresponds to a table
|
|
331
|
+
type: table
|
|
332
|
+
fields: ...
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
Notebook
|
|
336
|
+
```python
|
|
337
|
+
%pip install git+https://github.com/datacontract/cli.git
|
|
338
|
+
dbutils.library.restartPython()
|
|
339
|
+
|
|
340
|
+
from datacontract.data_contract import DataContract
|
|
341
|
+
|
|
342
|
+
data_contract = DataContract(
|
|
343
|
+
data_contract_file="/Volumes/acme_catalog_prod/orders_latest/datacontract/datacontract.yaml",
|
|
344
|
+
spark=spark)
|
|
345
|
+
run = data_contract.test()
|
|
346
|
+
run.result
|
|
233
347
|
```
|
|
234
348
|
|
|
349
|
+
|
|
350
|
+
### Exports
|
|
351
|
+
|
|
352
|
+
Available export options:
|
|
353
|
+
|
|
354
|
+
| Type | Description | Status |
|
|
355
|
+
|--------------|------------------------------------------------|--------|
|
|
356
|
+
| `jsonschema` | Export to JSON Schema | ✅ |
|
|
357
|
+
| `sodacl` | Export to SodaCL quality checks in YAML format | ✅ |
|
|
358
|
+
| `dbt` | Export to dbt model in YAML format | TBD |
|
|
359
|
+
| `avro` | Export to AVRO models | TBD |
|
|
360
|
+
| `pydantic` | Export to pydantic models | TBD |
|
|
361
|
+
| `sql` | Export to SQL DDL | TBD |
|
|
362
|
+
| `protobuf` | Export to Protobuf | TBD |
|
|
363
|
+
|
|
235
364
|
## Development Setup
|
|
236
365
|
|
|
237
|
-
Python base interpreter should be 3.11.x (unless
|
|
238
|
-
working on 3.12 release candidate).
|
|
366
|
+
Python base interpreter should be 3.11.x (unless working on 3.12 release candidate).
|
|
239
367
|
|
|
240
368
|
```bash
|
|
241
369
|
# create venv
|
|
@@ -265,7 +393,7 @@ Docker Build
|
|
|
265
393
|
|
|
266
394
|
```
|
|
267
395
|
docker build -t datacontract/cli .
|
|
268
|
-
docker run --rm -v ${PWD}:/datacontract datacontract/cli
|
|
396
|
+
docker run --rm -v ${PWD}:/home/datacontract datacontract/cli
|
|
269
397
|
```
|
|
270
398
|
|
|
271
399
|
## Contribution
|
|
@@ -279,3 +407,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
279
407
|
## Credits
|
|
280
408
|
|
|
281
409
|
Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/) and [Jochen Christ](https://www.linkedin.com/in/jochenchrist/).
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
<a href="https://github.com/datacontract/cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
|
|
@@ -48,6 +48,9 @@ $ datacontract lint datacontract.yaml
|
|
|
48
48
|
# execute schema and quality checks
|
|
49
49
|
$ datacontract test datacontract.yaml
|
|
50
50
|
|
|
51
|
+
# execute schema and quality checks on the examples within the contract
|
|
52
|
+
$ datacontract test --examples datacontract.yaml
|
|
53
|
+
|
|
51
54
|
# find differences between to data contracts (Coming Soon)
|
|
52
55
|
$ datacontract diff datacontract-v1.yaml datacontract-v2.yaml
|
|
53
56
|
|
|
@@ -107,17 +110,17 @@ pipx installs into an isolated environment.
|
|
|
107
110
|
pipx install datacontract-cli
|
|
108
111
|
```
|
|
109
112
|
|
|
110
|
-
###
|
|
113
|
+
### Docker
|
|
111
114
|
|
|
112
115
|
```bash
|
|
113
|
-
|
|
116
|
+
docker pull --platform linux/amd64 datacontract/cli
|
|
117
|
+
docker run --rm --platform linux/amd64 -v ${PWD}:/home/datacontract datacontract/cli
|
|
114
118
|
```
|
|
115
119
|
|
|
116
|
-
|
|
120
|
+
Or via an alias that automatically uses the latest version:
|
|
117
121
|
|
|
118
122
|
```bash
|
|
119
|
-
docker
|
|
120
|
-
docker run --rm -v ${PWD}:/datacontract datacontract/cli
|
|
123
|
+
alias datacontract='docker run --rm -v "${PWD}:/home/datacontract" --platform linux/amd64 datacontract/cli:latest'
|
|
121
124
|
```
|
|
122
125
|
|
|
123
126
|
## Documentation
|
|
@@ -140,11 +143,12 @@ The application uses different engines, based on the server `type`.
|
|
|
140
143
|
| `s3` | `json` | Support for `new_line` delimited JSON files and one JSON record per file. | ✅ | fastjsonschema<br> soda-core-duckdb |
|
|
141
144
|
| `s3` | `csv` | | ✅ | soda-core-duckdb |
|
|
142
145
|
| `s3` | `delta` | | Coming soon | TBD |
|
|
143
|
-
| `postgres` | n/a | |
|
|
144
|
-
| `snowflake` | n/a | | ✅
|
|
145
|
-
| `bigquery` | n/a
|
|
146
|
+
| `postgres` | n/a | | ✅ | soda-core-postgres |
|
|
147
|
+
| `snowflake` | n/a | | ✅ | soda-core-snowflake |
|
|
148
|
+
| `bigquery` | n/a | | ✅ | soda-core-bigquery |
|
|
146
149
|
| `redshift` | n/a | | Coming soon | TBD |
|
|
147
|
-
| `databricks` | n/a |
|
|
150
|
+
| `databricks` | n/a | Support for Databricks SQL with Unity catalog and Hive metastore. | ✅ | soda-core-spark |
|
|
151
|
+
| `databricks` | n/a | Support for Spark for programmatic use in Notebooks. | ✅ | soda-core-spark-df |
|
|
148
152
|
| `kafka` | `json` | | Coming soon | TBD |
|
|
149
153
|
| `kafka` | `avro` | | Coming soon | TBD |
|
|
150
154
|
| `kafka` | `protobuf` | | Coming soon | TBD |
|
|
@@ -154,9 +158,11 @@ The application uses different engines, based on the server `type`.
|
|
|
154
158
|
|
|
155
159
|
Feel free to create an issue, if you need support for an additional type.
|
|
156
160
|
|
|
157
|
-
###
|
|
161
|
+
### S3
|
|
162
|
+
|
|
163
|
+
Data Contract CLI can test data that is stored in S3 buckets or any S3-compliant endpoints in various formats.
|
|
158
164
|
|
|
159
|
-
Example
|
|
165
|
+
#### Example
|
|
160
166
|
|
|
161
167
|
datacontract.yaml
|
|
162
168
|
```yaml
|
|
@@ -165,24 +171,58 @@ servers:
|
|
|
165
171
|
type: s3
|
|
166
172
|
endpointUrl: https://minio.example.com # not needed with AWS S3
|
|
167
173
|
location: s3://bucket-name/path/*/*.json
|
|
168
|
-
delimiter: new_line # new_line, array, or none
|
|
169
174
|
format: json
|
|
175
|
+
delimiter: new_line # new_line, array, or none
|
|
170
176
|
```
|
|
171
177
|
|
|
172
|
-
Environment
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
178
|
+
#### Environment Variables
|
|
179
|
+
|
|
180
|
+
| Environment Variable | Example | Description |
|
|
181
|
+
|-----------------------------------|-------------------------------|-----------------------|
|
|
182
|
+
| `DATACONTRACT_S3_REGION` | `eu-central-1` | Region of S3 bucket |
|
|
183
|
+
| `DATACONTRACT_S3_ACCESS_KEY_ID` | `AKIAXV5Q5QABCDEFGH` | AWS Access Key ID |
|
|
184
|
+
| `DATACONTRACT_S3_SECRET_ACCESS_KEY` | `93S7LRrJcqLaaaa/XXXXXXXXXXXXX` | AWS Secret Access Key |
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
### Postgres
|
|
188
|
+
|
|
189
|
+
Data Contract CLI can test data in Postgres or Postgres-compliant databases (e.g., RisingWave).
|
|
190
|
+
|
|
191
|
+
#### Example
|
|
192
|
+
|
|
193
|
+
datacontract.yaml
|
|
194
|
+
```yaml
|
|
195
|
+
servers:
|
|
196
|
+
postgres:
|
|
197
|
+
type: postgres
|
|
198
|
+
host: localhost
|
|
199
|
+
port: 5432
|
|
200
|
+
database: postgres
|
|
201
|
+
schema: public
|
|
202
|
+
models:
|
|
203
|
+
my_table_1: # corresponds to a table
|
|
204
|
+
type: table
|
|
205
|
+
fields:
|
|
206
|
+
my_column_1: # corresponds to a column
|
|
207
|
+
type: varchar
|
|
177
208
|
```
|
|
178
209
|
|
|
179
|
-
|
|
210
|
+
#### Environment Variables
|
|
211
|
+
|
|
212
|
+
| Environment Variable | Example | Description |
|
|
213
|
+
|----------------------------------|--------------------|-------------|
|
|
214
|
+
| `DATACONTRACT_POSTGRES_USERNAME` | `postgres` | Username |
|
|
215
|
+
| `DATACONTRACT_POSTGRES_PASSWORD` | `mysecretpassword` | Password |
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
### BigQuery
|
|
180
219
|
|
|
181
220
|
We support authentication to BigQuery using Service Account Key. The used Service Account should include the roles:
|
|
182
221
|
* BigQuery Job User
|
|
183
222
|
* BigQuery Data Viewer
|
|
184
223
|
|
|
185
|
-
|
|
224
|
+
|
|
225
|
+
#### Example
|
|
186
226
|
|
|
187
227
|
datacontract.yaml
|
|
188
228
|
```yaml
|
|
@@ -197,15 +237,99 @@ models:
|
|
|
197
237
|
fields: ...
|
|
198
238
|
```
|
|
199
239
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
240
|
+
#### Environment Variables
|
|
241
|
+
|
|
242
|
+
| Environment Variable | Example | Description |
|
|
243
|
+
|----------------------------------------------|---------------------------|---------------------------------------------------------|
|
|
244
|
+
| `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Access key as saved on key creation by BigQuery |
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
### Databricks
|
|
248
|
+
|
|
249
|
+
Works with Unity Catalog and Hive metastore.
|
|
250
|
+
|
|
251
|
+
Needs a running SQL warehouse or compute cluster.
|
|
252
|
+
|
|
253
|
+
#### Example
|
|
254
|
+
|
|
255
|
+
datacontract.yaml
|
|
256
|
+
```yaml
|
|
257
|
+
servers:
|
|
258
|
+
production:
|
|
259
|
+
type: databricks
|
|
260
|
+
host: dbc-abcdefgh-1234.cloud.databricks.com
|
|
261
|
+
catalog: acme_catalog_prod
|
|
262
|
+
schema: orders_latest
|
|
263
|
+
models:
|
|
264
|
+
orders: # corresponds to a table
|
|
265
|
+
type: table
|
|
266
|
+
fields: ...
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
#### Environment Variables
|
|
270
|
+
|
|
271
|
+
| Environment Variable | Example | Description |
|
|
272
|
+
|----------------------------------------------|--------------------------------------|-------------------------------------------------------|
|
|
273
|
+
| `DATACONTRACT_DATABRICKS_TOKEN` | `dapia00000000000000000000000000000` | The personal access token to authenticate |
|
|
274
|
+
| `DATACONTRACT_DATABRICKS_HTTP_PATH` | `/sql/1.0/warehouses/b053a3ffffffff` | The HTTP path to the SQL warehouse or compute cluster |
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
### Databricks (programmatic)
|
|
278
|
+
|
|
279
|
+
Works with Unity Catalog and Hive metastore.
|
|
280
|
+
When running in a notebook or pipeline, the provided `spark` session can be used.
|
|
281
|
+
An additional authentication is not required.
|
|
282
|
+
|
|
283
|
+
Requires a Databricks Runtime with Python >= 3.10.
|
|
284
|
+
|
|
285
|
+
#### Example
|
|
286
|
+
|
|
287
|
+
datacontract.yaml
|
|
288
|
+
```yaml
|
|
289
|
+
servers:
|
|
290
|
+
production:
|
|
291
|
+
type: databricks
|
|
292
|
+
host: dbc-abcdefgh-1234.cloud.databricks.com # ignored, always use current host
|
|
293
|
+
catalog: acme_catalog_prod
|
|
294
|
+
schema: orders_latest
|
|
295
|
+
models:
|
|
296
|
+
orders: # corresponds to a table
|
|
297
|
+
type: table
|
|
298
|
+
fields: ...
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Notebook
|
|
302
|
+
```python
|
|
303
|
+
%pip install git+https://github.com/datacontract/cli.git
|
|
304
|
+
dbutils.library.restartPython()
|
|
305
|
+
|
|
306
|
+
from datacontract.data_contract import DataContract
|
|
307
|
+
|
|
308
|
+
data_contract = DataContract(
|
|
309
|
+
data_contract_file="/Volumes/acme_catalog_prod/orders_latest/datacontract/datacontract.yaml",
|
|
310
|
+
spark=spark)
|
|
311
|
+
run = data_contract.test()
|
|
312
|
+
run.result
|
|
203
313
|
```
|
|
204
314
|
|
|
315
|
+
|
|
316
|
+
### Exports
|
|
317
|
+
|
|
318
|
+
Available export options:
|
|
319
|
+
|
|
320
|
+
| Type | Description | Status |
|
|
321
|
+
|--------------|------------------------------------------------|--------|
|
|
322
|
+
| `jsonschema` | Export to JSON Schema | ✅ |
|
|
323
|
+
| `sodacl` | Export to SodaCL quality checks in YAML format | ✅ |
|
|
324
|
+
| `dbt` | Export to dbt model in YAML format | TBD |
|
|
325
|
+
| `avro` | Export to AVRO models | TBD |
|
|
326
|
+
| `pydantic` | Export to pydantic models | TBD |
|
|
327
|
+
| `sql` | Export to SQL DDL | TBD |
|
|
328
|
+
| `protobuf` | Export to Protobuf | TBD |
|
|
329
|
+
|
|
205
330
|
## Development Setup
|
|
206
331
|
|
|
207
|
-
Python base interpreter should be 3.11.x (unless
|
|
208
|
-
working on 3.12 release candidate).
|
|
332
|
+
Python base interpreter should be 3.11.x (unless working on 3.12 release candidate).
|
|
209
333
|
|
|
210
334
|
```bash
|
|
211
335
|
# create venv
|
|
@@ -235,7 +359,7 @@ Docker Build
|
|
|
235
359
|
|
|
236
360
|
```
|
|
237
361
|
docker build -t datacontract/cli .
|
|
238
|
-
docker run --rm -v ${PWD}:/datacontract datacontract/cli
|
|
362
|
+
docker run --rm -v ${PWD}:/home/datacontract datacontract/cli
|
|
239
363
|
```
|
|
240
364
|
|
|
241
365
|
## Contribution
|
|
@@ -249,3 +373,7 @@ We are happy to receive your contributions. Propose your change in an issue or d
|
|
|
249
373
|
## Credits
|
|
250
374
|
|
|
251
375
|
Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/) and [Jochen Christ](https://www.linkedin.com/in/jochenchrist/).
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
<a href="https://github.com/datacontract/cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
|
|
@@ -87,14 +87,16 @@ def test(
|
|
|
87
87
|
"Use the key of the server object in the data contract yaml file "
|
|
88
88
|
"to refer to a server, e.g., `production`, or `all` for all "
|
|
89
89
|
"servers (default).")] = "all",
|
|
90
|
+
examples: Annotated[bool, typer.Option(
|
|
91
|
+
help="Run the schema and quality tests on the example data within the data contract.")] = None,
|
|
90
92
|
publish: Annotated[str, typer.Option(
|
|
91
|
-
help="")] = None,
|
|
93
|
+
help="The url to publish the results after the test")] = None,
|
|
92
94
|
):
|
|
93
95
|
"""
|
|
94
96
|
Run schema and quality tests on configured servers.
|
|
95
97
|
"""
|
|
96
98
|
print(f"Testing {location}")
|
|
97
|
-
run = DataContract(data_contract_file=location, publish_url=publish).test()
|
|
99
|
+
run = DataContract(data_contract_file=location, publish_url=publish, examples=examples).test()
|
|
98
100
|
_handle_result(run)
|
|
99
101
|
|
|
100
102
|
|