datacontract-cli 0.10.23__py3-none-any.whl → 0.10.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacontract/__init__.py +13 -0
- datacontract/api.py +12 -5
- datacontract/catalog/catalog.py +5 -3
- datacontract/cli.py +119 -13
- datacontract/data_contract.py +145 -67
- datacontract/engines/data_contract_checks.py +366 -60
- datacontract/engines/data_contract_test.py +50 -4
- datacontract/engines/fastjsonschema/check_jsonschema.py +37 -19
- datacontract/engines/fastjsonschema/s3/s3_read_files.py +3 -2
- datacontract/engines/soda/check_soda_execute.py +27 -3
- datacontract/engines/soda/connections/athena.py +79 -0
- datacontract/engines/soda/connections/duckdb_connection.py +65 -6
- datacontract/engines/soda/connections/kafka.py +4 -2
- datacontract/engines/soda/connections/oracle.py +50 -0
- datacontract/export/avro_converter.py +20 -3
- datacontract/export/bigquery_converter.py +1 -1
- datacontract/export/dbt_converter.py +36 -7
- datacontract/export/dqx_converter.py +126 -0
- datacontract/export/duckdb_type_converter.py +57 -0
- datacontract/export/excel_exporter.py +923 -0
- datacontract/export/exporter.py +3 -0
- datacontract/export/exporter_factory.py +17 -1
- datacontract/export/great_expectations_converter.py +55 -5
- datacontract/export/{html_export.py → html_exporter.py} +31 -20
- datacontract/export/markdown_converter.py +134 -5
- datacontract/export/mermaid_exporter.py +110 -0
- datacontract/export/odcs_v3_exporter.py +193 -149
- datacontract/export/protobuf_converter.py +163 -69
- datacontract/export/rdf_converter.py +2 -2
- datacontract/export/sodacl_converter.py +9 -1
- datacontract/export/spark_converter.py +31 -4
- datacontract/export/sql_converter.py +6 -2
- datacontract/export/sql_type_converter.py +124 -8
- datacontract/imports/avro_importer.py +63 -12
- datacontract/imports/csv_importer.py +111 -57
- datacontract/imports/excel_importer.py +1112 -0
- datacontract/imports/importer.py +16 -3
- datacontract/imports/importer_factory.py +17 -0
- datacontract/imports/json_importer.py +325 -0
- datacontract/imports/odcs_importer.py +2 -2
- datacontract/imports/odcs_v3_importer.py +367 -151
- datacontract/imports/protobuf_importer.py +264 -0
- datacontract/imports/spark_importer.py +117 -13
- datacontract/imports/sql_importer.py +32 -16
- datacontract/imports/unity_importer.py +84 -38
- datacontract/init/init_template.py +1 -1
- datacontract/integration/entropy_data.py +126 -0
- datacontract/lint/resolve.py +112 -23
- datacontract/lint/schema.py +24 -15
- datacontract/lint/urls.py +17 -3
- datacontract/model/data_contract_specification/__init__.py +1 -0
- datacontract/model/odcs.py +13 -0
- datacontract/model/run.py +3 -0
- datacontract/output/junit_test_results.py +3 -3
- datacontract/schemas/datacontract-1.1.0.init.yaml +1 -1
- datacontract/schemas/datacontract-1.2.0.init.yaml +91 -0
- datacontract/schemas/datacontract-1.2.0.schema.json +2029 -0
- datacontract/schemas/datacontract-1.2.1.init.yaml +91 -0
- datacontract/schemas/datacontract-1.2.1.schema.json +2058 -0
- datacontract/schemas/odcs-3.0.2.schema.json +2382 -0
- datacontract/schemas/odcs-3.1.0.schema.json +2809 -0
- datacontract/templates/datacontract.html +54 -3
- datacontract/templates/datacontract_odcs.html +685 -0
- datacontract/templates/index.html +5 -2
- datacontract/templates/partials/server.html +2 -0
- datacontract/templates/style/output.css +319 -145
- {datacontract_cli-0.10.23.dist-info → datacontract_cli-0.10.40.dist-info}/METADATA +711 -433
- datacontract_cli-0.10.40.dist-info/RECORD +121 -0
- {datacontract_cli-0.10.23.dist-info → datacontract_cli-0.10.40.dist-info}/WHEEL +1 -1
- {datacontract_cli-0.10.23.dist-info → datacontract_cli-0.10.40.dist-info/licenses}/LICENSE +1 -1
- datacontract/export/csv_type_converter.py +0 -36
- datacontract/integration/datamesh_manager.py +0 -72
- datacontract/lint/lint.py +0 -142
- datacontract/lint/linters/description_linter.py +0 -35
- datacontract/lint/linters/field_pattern_linter.py +0 -34
- datacontract/lint/linters/field_reference_linter.py +0 -48
- datacontract/lint/linters/notice_period_linter.py +0 -55
- datacontract/lint/linters/quality_schema_linter.py +0 -52
- datacontract/lint/linters/valid_constraints_linter.py +0 -100
- datacontract/model/data_contract_specification.py +0 -327
- datacontract_cli-0.10.23.dist-info/RECORD +0 -113
- /datacontract/{lint/linters → output}/__init__.py +0 -0
- {datacontract_cli-0.10.23.dist-info → datacontract_cli-0.10.40.dist-info}/entry_points.txt +0 -0
- {datacontract_cli-0.10.23.dist-info → datacontract_cli-0.10.40.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from datacontract_specification.model import *
|
datacontract/model/odcs.py
CHANGED
|
@@ -9,3 +9,16 @@ def is_open_data_contract_standard(odcs: dict) -> bool:
|
|
|
9
9
|
bool: True if the dictionary is an OpenDataContractStandard, False otherwise.
|
|
10
10
|
"""
|
|
11
11
|
return odcs.get("kind") == "DataContract" and odcs.get("apiVersion", "").startswith("v3")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def is_open_data_product_standard(odcs: dict) -> bool:
|
|
15
|
+
"""
|
|
16
|
+
Check if the given dictionary is an open data product standard.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
odcs (dict): The dictionary to check.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
bool: True if the dictionary is an open data product standard, False otherwise.
|
|
23
|
+
"""
|
|
24
|
+
return odcs.get("kind") == "DataProduct" and odcs.get("apiVersion", "").startswith("v1")
|
datacontract/model/run.py
CHANGED
|
@@ -89,6 +89,9 @@ class Run(BaseModel):
|
|
|
89
89
|
def pretty(self):
|
|
90
90
|
return self.model_dump_json(indent=2)
|
|
91
91
|
|
|
92
|
+
def pretty_logs(self) -> str:
|
|
93
|
+
return "\n".join(f"[{log.timestamp.isoformat()}] {log.level}: {log.message}" for log in self.logs)
|
|
94
|
+
|
|
92
95
|
@staticmethod
|
|
93
96
|
def create_run():
|
|
94
97
|
"""
|
|
@@ -56,19 +56,19 @@ def write_junit_test_results(run: Run, console, output_path: Path):
|
|
|
56
56
|
type=check.category if check.category else "General",
|
|
57
57
|
)
|
|
58
58
|
error.text = to_failure_text(check)
|
|
59
|
-
elif check.result
|
|
59
|
+
elif check.result == ResultEnum.warning:
|
|
60
60
|
skipped = ET.SubElement(
|
|
61
61
|
testcase,
|
|
62
62
|
"skipped",
|
|
63
63
|
message=check.reason if check.reason else "Warning",
|
|
64
64
|
type=check.category if check.category else "General",
|
|
65
65
|
)
|
|
66
|
-
skipped.
|
|
66
|
+
skipped.text = to_failure_text(check)
|
|
67
67
|
else:
|
|
68
68
|
ET.SubElement(
|
|
69
69
|
testcase,
|
|
70
70
|
"skipped",
|
|
71
|
-
message=check.reason if check.reason else "
|
|
71
|
+
message=check.reason if check.reason else "Skipped",
|
|
72
72
|
type=check.category if check.category else "General",
|
|
73
73
|
)
|
|
74
74
|
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
dataContractSpecification: 1.2.1
|
|
2
|
+
id: my-data-contract-id
|
|
3
|
+
info:
|
|
4
|
+
title: My Data Contract
|
|
5
|
+
version: 0.0.1
|
|
6
|
+
# description:
|
|
7
|
+
# owner:
|
|
8
|
+
# contact:
|
|
9
|
+
# name:
|
|
10
|
+
# url:
|
|
11
|
+
# email:
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### servers
|
|
15
|
+
|
|
16
|
+
#servers:
|
|
17
|
+
# production:
|
|
18
|
+
# type: s3
|
|
19
|
+
# location: s3://
|
|
20
|
+
# format: parquet
|
|
21
|
+
# delimiter: new_line
|
|
22
|
+
|
|
23
|
+
### terms
|
|
24
|
+
|
|
25
|
+
#terms:
|
|
26
|
+
# usage:
|
|
27
|
+
# limitations:
|
|
28
|
+
# billing:
|
|
29
|
+
# noticePeriod:
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
### models
|
|
33
|
+
|
|
34
|
+
# models:
|
|
35
|
+
# my_model:
|
|
36
|
+
# description:
|
|
37
|
+
# type:
|
|
38
|
+
# fields:
|
|
39
|
+
# my_field:
|
|
40
|
+
# type:
|
|
41
|
+
# description:
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
### definitions
|
|
45
|
+
|
|
46
|
+
# definitions:
|
|
47
|
+
# my_field:
|
|
48
|
+
# domain:
|
|
49
|
+
# name:
|
|
50
|
+
# title:
|
|
51
|
+
# type:
|
|
52
|
+
# description:
|
|
53
|
+
# example:
|
|
54
|
+
# pii:
|
|
55
|
+
# classification:
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
### servicelevels
|
|
59
|
+
|
|
60
|
+
#servicelevels:
|
|
61
|
+
# availability:
|
|
62
|
+
# description: The server is available during support hours
|
|
63
|
+
# percentage: 99.9%
|
|
64
|
+
# retention:
|
|
65
|
+
# description: Data is retained for one year because!
|
|
66
|
+
# period: P1Y
|
|
67
|
+
# unlimited: false
|
|
68
|
+
# latency:
|
|
69
|
+
# description: Data is available within 25 hours after the order was placed
|
|
70
|
+
# threshold: 25h
|
|
71
|
+
# sourceTimestampField: orders.order_timestamp
|
|
72
|
+
# processedTimestampField: orders.processed_timestamp
|
|
73
|
+
# freshness:
|
|
74
|
+
# description: The age of the youngest row in a table.
|
|
75
|
+
# threshold: 25h
|
|
76
|
+
# timestampField: orders.order_timestamp
|
|
77
|
+
# frequency:
|
|
78
|
+
# description: Data is delivered once a day
|
|
79
|
+
# type: batch # or streaming
|
|
80
|
+
# interval: daily # for batch, either or cron
|
|
81
|
+
# cron: 0 0 * * * # for batch, either or interval
|
|
82
|
+
# support:
|
|
83
|
+
# description: The data is available during typical business hours at headquarters
|
|
84
|
+
# time: 9am to 5pm in EST on business days
|
|
85
|
+
# responseTime: 1h
|
|
86
|
+
# backup:
|
|
87
|
+
# description: Data is backed up once a week, every Sunday at 0:00 UTC.
|
|
88
|
+
# interval: weekly
|
|
89
|
+
# cron: 0 0 * * 0
|
|
90
|
+
# recoveryTime: 24 hours
|
|
91
|
+
# recoveryPoint: 1 week
|