datacontract-cli 0.9.6.post2__py3-none-any.whl → 0.9.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of datacontract-cli might be problematic. Click here for more details.

Files changed (60) hide show
  1. datacontract/breaking/breaking.py +139 -63
  2. datacontract/breaking/breaking_rules.py +71 -54
  3. datacontract/cli.py +138 -45
  4. datacontract/data_contract.py +316 -78
  5. datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py +5 -1
  6. datacontract/engines/datacontract/check_that_datacontract_file_exists.py +9 -8
  7. datacontract/engines/datacontract/check_that_datacontract_str_is_valid.py +26 -22
  8. datacontract/engines/fastjsonschema/check_jsonschema.py +31 -25
  9. datacontract/engines/fastjsonschema/s3/s3_read_files.py +8 -6
  10. datacontract/engines/soda/check_soda_execute.py +46 -35
  11. datacontract/engines/soda/connections/bigquery.py +5 -3
  12. datacontract/engines/soda/connections/dask.py +0 -1
  13. datacontract/engines/soda/connections/databricks.py +2 -2
  14. datacontract/engines/soda/connections/duckdb.py +4 -4
  15. datacontract/engines/soda/connections/kafka.py +36 -17
  16. datacontract/engines/soda/connections/postgres.py +3 -3
  17. datacontract/engines/soda/connections/snowflake.py +4 -4
  18. datacontract/export/avro_converter.py +3 -7
  19. datacontract/export/avro_idl_converter.py +280 -0
  20. datacontract/export/dbt_converter.py +55 -80
  21. datacontract/export/great_expectations_converter.py +141 -0
  22. datacontract/export/jsonschema_converter.py +3 -1
  23. datacontract/export/odcs_converter.py +10 -12
  24. datacontract/export/protobuf_converter.py +99 -0
  25. datacontract/export/pydantic_converter.py +140 -0
  26. datacontract/export/rdf_converter.py +35 -12
  27. datacontract/export/sodacl_converter.py +24 -24
  28. datacontract/export/sql_converter.py +93 -0
  29. datacontract/export/sql_type_converter.py +131 -0
  30. datacontract/export/terraform_converter.py +71 -0
  31. datacontract/imports/avro_importer.py +106 -0
  32. datacontract/imports/sql_importer.py +0 -2
  33. datacontract/init/download_datacontract_file.py +2 -2
  34. datacontract/integration/publish_datamesh_manager.py +4 -9
  35. datacontract/integration/publish_opentelemetry.py +107 -0
  36. datacontract/lint/files.py +2 -2
  37. datacontract/lint/lint.py +46 -31
  38. datacontract/lint/linters/description_linter.py +34 -0
  39. datacontract/lint/linters/example_model_linter.py +67 -43
  40. datacontract/lint/linters/field_pattern_linter.py +34 -0
  41. datacontract/lint/linters/field_reference_linter.py +38 -0
  42. datacontract/lint/linters/notice_period_linter.py +55 -0
  43. datacontract/lint/linters/primary_field_linter.py +28 -0
  44. datacontract/lint/linters/quality_schema_linter.py +52 -0
  45. datacontract/lint/linters/valid_constraints_linter.py +99 -0
  46. datacontract/lint/resolve.py +53 -8
  47. datacontract/lint/schema.py +2 -3
  48. datacontract/lint/urls.py +4 -5
  49. datacontract/model/breaking_change.py +27 -5
  50. datacontract/model/data_contract_specification.py +45 -25
  51. datacontract/model/exceptions.py +13 -2
  52. datacontract/model/run.py +1 -1
  53. datacontract/web.py +5 -8
  54. {datacontract_cli-0.9.6.post2.dist-info → datacontract_cli-0.9.8.dist-info}/METADATA +207 -35
  55. datacontract_cli-0.9.8.dist-info/RECORD +63 -0
  56. {datacontract_cli-0.9.6.post2.dist-info → datacontract_cli-0.9.8.dist-info}/WHEEL +1 -1
  57. datacontract_cli-0.9.6.post2.dist-info/RECORD +0 -47
  58. {datacontract_cli-0.9.6.post2.dist-info → datacontract_cli-0.9.8.dist-info}/LICENSE +0 -0
  59. {datacontract_cli-0.9.6.post2.dist-info → datacontract_cli-0.9.8.dist-info}/entry_points.txt +0 -0
  60. {datacontract_cli-0.9.6.post2.dist-info → datacontract_cli-0.9.8.dist-info}/top_level.txt +0 -0
@@ -1,13 +1,84 @@
1
1
  from datacontract.breaking.breaking_rules import BreakingRules
2
- from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Location
3
- from datacontract.model.data_contract_specification import Field, Model
2
+ from datacontract.model.breaking_change import BreakingChange, Location, Severity
3
+ from datacontract.model.data_contract_specification import Field, Model, Quality
4
+
5
+
6
+ def quality_breaking_changes(
7
+ old_quality: Quality,
8
+ new_quality: Quality,
9
+ new_path: str,
10
+ include_severities: [Severity],
11
+ ) -> list[BreakingChange]:
12
+ results = list[BreakingChange]()
13
+
14
+ if not old_quality and new_quality:
15
+ rule_name = "quality_added"
16
+ severity = _get_rule(rule_name)
17
+ description = "added quality"
18
+
19
+ if severity in include_severities:
20
+ results.append(
21
+ BreakingChange(
22
+ description=description,
23
+ check_name=rule_name,
24
+ severity=severity,
25
+ location=Location(path=new_path, composition=["quality"]),
26
+ )
27
+ )
28
+ elif old_quality and not new_quality:
29
+ rule_name = "quality_removed"
30
+ severity = _get_rule(rule_name)
31
+ description = "removed quality"
32
+
33
+ if severity in include_severities:
34
+ results.append(
35
+ BreakingChange(
36
+ description=description,
37
+ check_name=rule_name,
38
+ severity=severity,
39
+ location=Location(path=new_path, composition=["quality"]),
40
+ )
41
+ )
42
+
43
+ elif old_quality and new_quality:
44
+ if old_quality.type != new_quality.type:
45
+ rule_name = "quality_type_updated"
46
+ severity = _get_rule(rule_name)
47
+ description = f"changed from `{old_quality.type}` to `{new_quality.type}`"
48
+
49
+ if severity in include_severities:
50
+ results.append(
51
+ BreakingChange(
52
+ description=description,
53
+ check_name=rule_name,
54
+ severity=severity,
55
+ location=Location(path=new_path, composition=["quality", "type"]),
56
+ )
57
+ )
58
+
59
+ if old_quality.specification != new_quality.specification:
60
+ rule_name = "quality_specification_updated"
61
+ severity = _get_rule(rule_name)
62
+ description = f"changed from `{old_quality.specification}` to `{new_quality.specification}`"
63
+ if severity in include_severities:
64
+ results.append(
65
+ BreakingChange(
66
+ description=description,
67
+ check_name=rule_name,
68
+ severity=severity,
69
+ location=Location(path=new_path, composition=["quality", "specification"]),
70
+ )
71
+ )
72
+
73
+ return results
4
74
 
5
75
 
6
76
  def models_breaking_changes(
7
77
  old_models: dict[str, Model],
8
78
  new_models: dict[str, Model],
9
- new_path: str
10
- ) -> BreakingChanges:
79
+ new_path: str,
80
+ include_severities: [Severity],
81
+ ) -> list[BreakingChange]:
11
82
  composition = ["models"]
12
83
  results = list[BreakingChange]()
13
84
 
@@ -15,31 +86,29 @@ def models_breaking_changes(
15
86
  if model_name not in old_models.keys():
16
87
  rule_name = "model_added"
17
88
  severity = _get_rule(rule_name)
18
- if severity != "info":
89
+ if severity in include_severities:
19
90
  results.append(
20
91
  BreakingChange(
21
92
  description="added the model",
22
93
  check_name=rule_name,
23
94
  severity=severity,
24
- location=Location(
25
- path=new_path,
26
- composition=composition + [model_name]
27
- )))
95
+ location=Location(path=new_path, composition=composition + [model_name]),
96
+ )
97
+ )
28
98
 
29
99
  for model_name, old_model in old_models.items():
30
100
  if model_name not in new_models.keys():
31
101
  rule_name = "model_removed"
32
102
  severity = _get_rule(rule_name)
33
- if severity != "info":
103
+ if severity in include_severities:
34
104
  results.append(
35
105
  BreakingChange(
36
106
  description="removed the model",
37
107
  check_name=rule_name,
38
108
  severity=severity,
39
- location=Location(
40
- path=new_path,
41
- composition=composition + [model_name]
42
- )))
109
+ location=Location(path=new_path, composition=composition + [model_name]),
110
+ )
111
+ )
43
112
  continue
44
113
 
45
114
  results.extend(
@@ -47,17 +116,16 @@ def models_breaking_changes(
47
116
  old_model=old_model,
48
117
  new_model=new_models[model_name],
49
118
  new_path=new_path,
50
- composition=composition + [model_name]
51
- ))
119
+ composition=composition + [model_name],
120
+ include_severities=include_severities,
121
+ )
122
+ )
52
123
 
53
- return BreakingChanges(breaking_changes=results)
124
+ return results
54
125
 
55
126
 
56
127
  def model_breaking_changes(
57
- old_model: Model,
58
- new_model: Model,
59
- new_path: str,
60
- composition: list[str]
128
+ old_model: Model, new_model: Model, new_path: str, composition: list[str], include_severities: [Severity]
61
129
  ) -> list[BreakingChange]:
62
130
  results = list[BreakingChange]()
63
131
 
@@ -87,24 +155,25 @@ def model_breaking_changes(
87
155
 
88
156
  if rule_name is not None:
89
157
  severity = _get_rule(rule_name)
90
- if severity != "info":
158
+ if severity in include_severities:
91
159
  results.append(
92
160
  BreakingChange(
93
161
  description=description,
94
162
  check_name=rule_name,
95
163
  severity=severity,
96
- location=Location(
97
- path=new_path,
98
- composition=composition + [model_definition_field]
99
- )))
164
+ location=Location(path=new_path, composition=composition + [model_definition_field]),
165
+ )
166
+ )
100
167
 
101
168
  results.extend(
102
169
  fields_breaking_changes(
103
170
  old_fields=old_model.fields,
104
171
  new_fields=new_model.fields,
105
172
  new_path=new_path,
106
- composition=composition
107
- ))
173
+ composition=composition + ["fields"],
174
+ include_severities=include_severities,
175
+ )
176
+ )
108
177
 
109
178
  return results
110
179
 
@@ -113,7 +182,8 @@ def fields_breaking_changes(
113
182
  old_fields: dict[str, Field],
114
183
  new_fields: dict[str, Field],
115
184
  new_path: str,
116
- composition: list[str]
185
+ composition: list[str],
186
+ include_severities: [Severity],
117
187
  ) -> list[BreakingChange]:
118
188
  results = list[BreakingChange]()
119
189
 
@@ -121,31 +191,29 @@ def fields_breaking_changes(
121
191
  if field_name not in old_fields.keys():
122
192
  rule_name = "field_added"
123
193
  severity = _get_rule(rule_name)
124
- if severity != "info":
194
+ if severity in include_severities:
125
195
  results.append(
126
196
  BreakingChange(
127
197
  description="added the field",
128
198
  check_name=rule_name,
129
199
  severity=severity,
130
- location=Location(
131
- path=new_path,
132
- composition=composition + [field_name]
133
- )))
200
+ location=Location(path=new_path, composition=composition + [field_name]),
201
+ )
202
+ )
134
203
 
135
204
  for field_name, old_field in old_fields.items():
136
205
  if field_name not in new_fields.keys():
137
206
  rule_name = "field_removed"
138
207
  severity = _get_rule(rule_name)
139
- if severity != "info":
208
+ if severity in include_severities:
140
209
  results.append(
141
210
  BreakingChange(
142
- description='removed the field',
211
+ description="removed the field",
143
212
  check_name=rule_name,
144
213
  severity=severity,
145
- location=Location(
146
- path=new_path,
147
- composition=composition + [field_name]
148
- )))
214
+ location=Location(path=new_path, composition=composition + [field_name]),
215
+ )
216
+ )
149
217
  continue
150
218
 
151
219
  results.extend(
@@ -154,7 +222,9 @@ def fields_breaking_changes(
154
222
  new_field=new_fields[field_name],
155
223
  composition=composition + [field_name],
156
224
  new_path=new_path,
157
- ))
225
+ include_severities=include_severities,
226
+ )
227
+ )
158
228
  return results
159
229
 
160
230
 
@@ -163,14 +233,13 @@ def field_breaking_changes(
163
233
  new_field: Field,
164
234
  composition: list[str],
165
235
  new_path: str,
236
+ include_severities: [Severity],
166
237
  ) -> list[BreakingChange]:
167
238
  results = list[BreakingChange]()
168
239
 
169
240
  field_definition_fields = vars(new_field)
170
241
  for field_definition_field in field_definition_fields.keys():
171
-
172
- # TODO(torbenkeller): handle ref case
173
- if field_definition_field == "ref":
242
+ if field_definition_field == "ref_obj":
174
243
  continue
175
244
 
176
245
  old_value = getattr(old_field, field_definition_field)
@@ -182,7 +251,8 @@ def field_breaking_changes(
182
251
  old_fields=old_field.fields,
183
252
  new_fields=new_field.fields,
184
253
  new_path=new_path,
185
- composition=composition + [field_definition_field]
254
+ composition=composition + [field_definition_field],
255
+ include_severities=include_severities,
186
256
  )
187
257
  )
188
258
  continue
@@ -191,50 +261,56 @@ def field_breaking_changes(
191
261
  description = None
192
262
 
193
263
  # logic for enum, tags and other arrays
194
- if type(old_value) is list and type(new_value) is list:
264
+ if isinstance(old_value, list) and isinstance(new_value, list):
195
265
  if not old_value and new_value:
196
- rule_name = f"field_{field_definition_field}_added"
266
+ rule_name = f"field_{_camel_to_snake(field_definition_field)}_added"
197
267
  description = f"added with value: `{new_value}`"
198
268
  elif old_value and not new_value:
199
- rule_name = f"field_{field_definition_field}_removed"
269
+ rule_name = f"field_{_camel_to_snake(field_definition_field)}_removed"
200
270
  description = "removed field property"
201
271
  elif sorted(old_value) != sorted(new_value):
202
- rule_name = f"field_{field_definition_field}_updated"
272
+ rule_name = f"field_{_camel_to_snake(field_definition_field)}_updated"
203
273
  description = f"changed from `{old_value}` to `{new_value}`"
204
274
 
205
275
  # logic for normal fields
206
276
  elif old_value is None and new_value is not None:
207
- rule_name = f"field_{field_definition_field}_added"
208
- description = f"added with value: `{str(new_value).lower() if type(new_value) is bool else new_value}`"
277
+ rule_name = f"field_{_camel_to_snake(field_definition_field)}_added"
278
+ description = f"added with value: `{str(new_value).lower() if isinstance(new_value, bool) else new_value}`"
209
279
 
210
280
  elif old_value is not None and new_value is None:
211
- rule_name = f"field_{field_definition_field}_removed"
281
+ rule_name = f"field_{_camel_to_snake(field_definition_field)}_removed"
212
282
  description = "removed field property"
213
283
 
214
284
  elif old_value != new_value:
215
- rule_name = f"field_{field_definition_field}_updated"
216
- description = (f"changed from `{str(old_value).lower() if type(old_value) is bool else old_value}` "
217
- f"to `{str(new_value).lower() if type(new_value) is bool else new_value}`")
285
+ rule_name = f"field_{_camel_to_snake(field_definition_field)}_updated"
286
+ description = (
287
+ f"changed from `{str(old_value).lower() if isinstance(old_value, bool) else old_value}` "
288
+ f"to `{str(new_value).lower() if isinstance(new_value, bool) else new_value}`"
289
+ )
218
290
 
219
291
  if rule_name is not None:
220
292
  severity = _get_rule(rule_name)
221
- if severity != "info":
293
+ field_schema_name = "$ref" if field_definition_field == "ref" else field_definition_field
294
+ if severity in include_severities:
222
295
  results.append(
223
296
  BreakingChange(
224
297
  description=description,
225
298
  check_name=rule_name,
226
299
  severity=severity,
227
- location=Location(
228
- path=new_path,
229
- composition=composition + [field_definition_field]
230
- )))
300
+ location=Location(path=new_path, composition=composition + [field_schema_name]),
301
+ )
302
+ )
231
303
 
232
304
  return results
233
305
 
234
306
 
235
- def _get_rule(rule_name):
307
+ def _get_rule(rule_name) -> Severity:
236
308
  try:
237
309
  return getattr(BreakingRules, rule_name)
238
310
  except AttributeError:
239
- print(f'WARNING: Breaking Rule not found for {rule_name}!')
240
- return 'error'
311
+ print(f"WARNING: Breaking Rule not found for {rule_name}!")
312
+ return Severity.ERROR
313
+
314
+
315
+ def _camel_to_snake(s):
316
+ return "".join(["_" + c.lower() if c.isupper() else c for c in s]).lstrip("_")
@@ -1,77 +1,94 @@
1
+ from datacontract.model.breaking_change import Severity
2
+
3
+
1
4
  class BreakingRules:
2
5
  # model rules
3
- model_added = 'info'
4
- model_removed = 'error'
6
+ model_added = Severity.INFO
7
+ model_removed = Severity.ERROR
5
8
 
6
- model_description_added = 'info'
7
- model_description_removed = 'info'
8
- model_description_updated = 'info'
9
+ model_description_added = Severity.INFO
10
+ model_description_removed = Severity.INFO
11
+ model_description_updated = Severity.INFO
9
12
 
10
- model_type_updated = 'error'
13
+ model_type_updated = Severity.ERROR
11
14
 
12
15
  # field rules
13
- field_added = 'info'
14
- field_removed = 'error'
16
+ field_added = Severity.INFO
17
+ field_removed = Severity.ERROR
18
+
19
+ field_ref_added = Severity.WARNING
20
+ field_ref_removed = Severity.WARNING
21
+ field_ref_updated = Severity.WARNING
22
+
23
+ field_type_added = Severity.WARNING
24
+ field_type_removed = Severity.WARNING
25
+ field_type_updated = Severity.ERROR
26
+
27
+ field_format_added = Severity.WARNING
28
+ field_format_removed = Severity.WARNING
29
+ field_format_updated = Severity.ERROR
30
+
31
+ field_required_updated = Severity.ERROR
15
32
 
16
- field_ref_added = 'warning'
17
- field_ref_removed = 'error'
33
+ field_primary_updated = Severity.WARNING
18
34
 
19
- field_type_added = 'warning'
20
- field_type_removed = 'warning'
21
- field_type_updated = 'error'
35
+ field_references_added = Severity.WARNING
36
+ field_references_removed = Severity.WARNING
37
+ field_references_updated = Severity.WARNING
22
38
 
23
- field_format_added = 'warning'
24
- field_format_removed = 'warning'
25
- field_format_updated = 'error'
39
+ field_unique_updated = Severity.ERROR
26
40
 
27
- field_required_updated = 'error'
41
+ field_description_added = Severity.INFO
42
+ field_description_removed = Severity.INFO
43
+ field_description_updated = Severity.INFO
28
44
 
29
- field_unique_updated = 'error'
45
+ field_pii_added = Severity.WARNING
46
+ field_pii_removed = Severity.ERROR
47
+ field_pii_updated = Severity.ERROR
30
48
 
31
- field_description_added = 'info'
32
- field_description_removed = 'info'
33
- field_description_updated = 'info'
49
+ field_classification_added = Severity.WARNING
50
+ field_classification_removed = Severity.ERROR
51
+ field_classification_updated = Severity.ERROR
34
52
 
35
- field_pii_added = 'warning'
36
- field_pii_removed = 'error'
37
- field_pii_updated = 'error'
53
+ field_pattern_added = Severity.WARNING
54
+ field_pattern_removed = Severity.ERROR
55
+ field_pattern_updated = Severity.ERROR
38
56
 
39
- field_classification_added = 'warning'
40
- field_classification_removed = 'error'
41
- field_classification_updated = 'error'
57
+ field_min_length_added = Severity.WARNING
58
+ field_min_length_removed = Severity.WARNING
59
+ field_min_length_updated = Severity.ERROR
42
60
 
43
- field_pattern_added = 'warning'
44
- field_pattern_removed = 'error'
45
- field_pattern_updated = 'error'
61
+ field_max_length_added = Severity.WARNING
62
+ field_max_length_removed = Severity.WARNING
63
+ field_max_length_updated = Severity.ERROR
46
64
 
47
- field_minLength_added = 'warning'
48
- field_minLength_removed = 'warning'
49
- field_minLength_updated = 'error'
65
+ field_minimum_added = Severity.WARNING
66
+ field_minimum_removed = Severity.WARNING
67
+ field_minimum_updated = Severity.ERROR
50
68
 
51
- field_maxLength_added = 'warning'
52
- field_maxLength_removed = 'warning'
53
- field_maxLength_updated = 'error'
69
+ field_exclusive_minimum_added = Severity.WARNING
70
+ field_exclusive_minimum_removed = Severity.WARNING
71
+ field_exclusive_minimum_updated = Severity.ERROR
54
72
 
55
- field_minimum_added = 'warning'
56
- field_minimum_removed = 'warning'
57
- field_minimum_updated = 'error'
73
+ field_maximum_added = Severity.WARNING
74
+ field_maximum_removed = Severity.WARNING
75
+ field_maximum_updated = Severity.ERROR
58
76
 
59
- field_minimumExclusive_added = 'warning'
60
- field_minimumExclusive_removed = 'warning'
61
- field_minimumExclusive_updated = 'error'
77
+ field_exclusive_maximum_added = Severity.WARNING
78
+ field_exclusive_maximum_removed = Severity.WARNING
79
+ field_exclusive_maximum_updated = Severity.ERROR
62
80
 
63
- field_maximum_added = 'warning'
64
- field_maximum_removed = 'warning'
65
- field_maximum_updated = 'error'
81
+ field_enum_added = Severity.WARNING
82
+ field_enum_removed = Severity.INFO
83
+ field_enum_updated = Severity.ERROR
66
84
 
67
- field_maximumExclusive_added = 'warning'
68
- field_maximumExclusive_removed = 'warning'
69
- field_maximumExclusive_updated = 'error'
85
+ field_tags_added = Severity.INFO
86
+ field_tags_removed = Severity.INFO
87
+ field_tags_updated = Severity.INFO
70
88
 
71
- field_enum_added = 'warning'
72
- field_enum_removed = 'info'
73
- field_enum_updated = 'error'
89
+ # quality Rules
90
+ quality_added = Severity.INFO
91
+ quality_removed = Severity.WARNING
74
92
 
75
- field_tags_added = 'info'
76
- field_tags_removed = 'info'
77
- field_tags_updated = 'info'
93
+ quality_type_updated = Severity.WARNING
94
+ quality_specification_updated = Severity.WARNING