datacontract-cli 0.10.26__py3-none-any.whl → 0.10.28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- datacontract/catalog/catalog.py +1 -1
- datacontract/cli.py +20 -3
- datacontract/data_contract.py +125 -22
- datacontract/engines/data_contract_checks.py +2 -0
- datacontract/export/dbt_converter.py +6 -3
- datacontract/export/exporter.py +1 -0
- datacontract/export/exporter_factory.py +7 -1
- datacontract/export/{html_export.py → html_exporter.py} +31 -20
- datacontract/export/mermaid_exporter.py +97 -0
- datacontract/export/odcs_v3_exporter.py +8 -10
- datacontract/export/sodacl_converter.py +9 -1
- datacontract/export/sql_converter.py +2 -2
- datacontract/export/sql_type_converter.py +6 -2
- datacontract/imports/excel_importer.py +5 -2
- datacontract/imports/importer.py +10 -1
- datacontract/imports/odcs_importer.py +2 -2
- datacontract/imports/odcs_v3_importer.py +9 -9
- datacontract/imports/spark_importer.py +103 -12
- datacontract/imports/sql_importer.py +4 -2
- datacontract/imports/unity_importer.py +77 -37
- datacontract/integration/datamesh_manager.py +16 -2
- datacontract/lint/resolve.py +60 -6
- datacontract/templates/datacontract.html +52 -2
- datacontract/templates/datacontract_odcs.html +666 -0
- datacontract/templates/index.html +2 -0
- datacontract/templates/partials/server.html +2 -0
- datacontract/templates/style/output.css +319 -145
- {datacontract_cli-0.10.26.dist-info → datacontract_cli-0.10.28.dist-info}/METADATA +364 -381
- {datacontract_cli-0.10.26.dist-info → datacontract_cli-0.10.28.dist-info}/RECORD +33 -31
- {datacontract_cli-0.10.26.dist-info → datacontract_cli-0.10.28.dist-info}/WHEEL +1 -1
- {datacontract_cli-0.10.26.dist-info → datacontract_cli-0.10.28.dist-info}/entry_points.txt +0 -0
- {datacontract_cli-0.10.26.dist-info → datacontract_cli-0.10.28.dist-info}/licenses/LICENSE +0 -0
- {datacontract_cli-0.10.26.dist-info → datacontract_cli-0.10.28.dist-info}/top_level.txt +0 -0
datacontract/catalog/catalog.py
CHANGED
|
@@ -6,7 +6,7 @@ import pytz
|
|
|
6
6
|
from jinja2 import Environment, PackageLoader, select_autoescape
|
|
7
7
|
|
|
8
8
|
from datacontract.data_contract import DataContract
|
|
9
|
-
from datacontract.export.
|
|
9
|
+
from datacontract.export.html_exporter import get_version
|
|
10
10
|
from datacontract.model.data_contract_specification import DataContractSpecification
|
|
11
11
|
|
|
12
12
|
|
datacontract/cli.py
CHANGED
|
@@ -11,7 +11,7 @@ from typing_extensions import Annotated
|
|
|
11
11
|
|
|
12
12
|
from datacontract.catalog.catalog import create_data_contract_html, create_index_html
|
|
13
13
|
from datacontract.data_contract import DataContract, ExportFormat
|
|
14
|
-
from datacontract.imports.importer import ImportFormat
|
|
14
|
+
from datacontract.imports.importer import ImportFormat, Spec
|
|
15
15
|
from datacontract.init.init_template import get_init_template
|
|
16
16
|
from datacontract.integration.datamesh_manager import (
|
|
17
17
|
publish_data_contract_to_datamesh_manager,
|
|
@@ -126,7 +126,8 @@ def test(
|
|
|
126
126
|
"servers (default)."
|
|
127
127
|
),
|
|
128
128
|
] = "all",
|
|
129
|
-
|
|
129
|
+
publish_test_results: Annotated[bool, typer.Option(help="Publish the results after the test")] = False,
|
|
130
|
+
publish: Annotated[str, typer.Option(help="DEPRECATED. The url to publish the results after the test.")] = None,
|
|
130
131
|
output: Annotated[
|
|
131
132
|
Path,
|
|
132
133
|
typer.Option(
|
|
@@ -149,6 +150,7 @@ def test(
|
|
|
149
150
|
run = DataContract(
|
|
150
151
|
data_contract_file=location,
|
|
151
152
|
schema_location=schema,
|
|
153
|
+
publish_test_results=publish_test_results,
|
|
152
154
|
publish_url=publish,
|
|
153
155
|
server=server,
|
|
154
156
|
ssl_verification=ssl_verification,
|
|
@@ -246,6 +248,10 @@ def import_(
|
|
|
246
248
|
Optional[str],
|
|
247
249
|
typer.Option(help="The path to the file that should be imported."),
|
|
248
250
|
] = None,
|
|
251
|
+
spec: Annotated[
|
|
252
|
+
Spec,
|
|
253
|
+
typer.Option(help="The format of the data contract to import. "),
|
|
254
|
+
] = Spec.datacontract_specification,
|
|
249
255
|
dialect: Annotated[
|
|
250
256
|
Optional[str],
|
|
251
257
|
typer.Option(help="The SQL dialect to use when importing SQL files, e.g., postgres, tsql, bigquery."),
|
|
@@ -265,7 +271,7 @@ def import_(
|
|
|
265
271
|
),
|
|
266
272
|
] = None,
|
|
267
273
|
unity_table_full_name: Annotated[
|
|
268
|
-
Optional[str], typer.Option(help="Full name of a table in the unity catalog")
|
|
274
|
+
Optional[List[str]], typer.Option(help="Full name of a table in the unity catalog")
|
|
269
275
|
] = None,
|
|
270
276
|
dbt_model: Annotated[
|
|
271
277
|
Optional[List[str]],
|
|
@@ -297,6 +303,14 @@ def import_(
|
|
|
297
303
|
str,
|
|
298
304
|
typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
|
|
299
305
|
] = None,
|
|
306
|
+
owner: Annotated[
|
|
307
|
+
Optional[str],
|
|
308
|
+
typer.Option(help="The owner or team responsible for managing the data contract."),
|
|
309
|
+
] = None,
|
|
310
|
+
id: Annotated[
|
|
311
|
+
Optional[str],
|
|
312
|
+
typer.Option(help="The identifier for the the data contract."),
|
|
313
|
+
] = None,
|
|
300
314
|
):
|
|
301
315
|
"""
|
|
302
316
|
Create a data contract from the given source location. Saves to file specified by `output` option if present, otherwise prints to stdout.
|
|
@@ -304,6 +318,7 @@ def import_(
|
|
|
304
318
|
result = DataContract().import_from_source(
|
|
305
319
|
format=format,
|
|
306
320
|
source=source,
|
|
321
|
+
spec=spec,
|
|
307
322
|
template=template,
|
|
308
323
|
schema=schema,
|
|
309
324
|
dialect=dialect,
|
|
@@ -316,6 +331,8 @@ def import_(
|
|
|
316
331
|
dbml_schema=dbml_schema,
|
|
317
332
|
dbml_table=dbml_table,
|
|
318
333
|
iceberg_table=iceberg_table,
|
|
334
|
+
owner=owner,
|
|
335
|
+
id=id,
|
|
319
336
|
)
|
|
320
337
|
if output is None:
|
|
321
338
|
console.print(result.to_yaml(), markup=False, soft_wrap=True)
|
datacontract/data_contract.py
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import typing
|
|
3
3
|
|
|
4
|
+
from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard
|
|
5
|
+
|
|
6
|
+
from datacontract.export.odcs_v3_exporter import to_odcs_v3
|
|
7
|
+
from datacontract.imports.importer import Spec
|
|
8
|
+
from datacontract.imports.odcs_v3_importer import import_from_odcs
|
|
9
|
+
|
|
4
10
|
if typing.TYPE_CHECKING:
|
|
5
11
|
from pyspark.sql import SparkSession
|
|
6
12
|
|
|
@@ -25,7 +31,7 @@ from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
|
|
|
25
31
|
from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
|
|
26
32
|
from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
|
|
27
33
|
from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
|
|
28
|
-
from datacontract.model.data_contract_specification import DataContractSpecification
|
|
34
|
+
from datacontract.model.data_contract_specification import DataContractSpecification, Info
|
|
29
35
|
from datacontract.model.exceptions import DataContractException
|
|
30
36
|
from datacontract.model.run import Check, ResultEnum, Run
|
|
31
37
|
|
|
@@ -44,6 +50,7 @@ class DataContract:
|
|
|
44
50
|
inline_definitions: bool = True,
|
|
45
51
|
inline_quality: bool = True,
|
|
46
52
|
ssl_verification: bool = True,
|
|
53
|
+
publish_test_results: bool = False,
|
|
47
54
|
):
|
|
48
55
|
self._data_contract_file = data_contract_file
|
|
49
56
|
self._data_contract_str = data_contract_str
|
|
@@ -51,6 +58,7 @@ class DataContract:
|
|
|
51
58
|
self._schema_location = schema_location
|
|
52
59
|
self._server = server
|
|
53
60
|
self._publish_url = publish_url
|
|
61
|
+
self._publish_test_results = publish_test_results
|
|
54
62
|
self._spark = spark
|
|
55
63
|
self._duckdb_connection = duckdb_connection
|
|
56
64
|
self._inline_definitions = inline_definitions
|
|
@@ -178,7 +186,7 @@ class DataContract:
|
|
|
178
186
|
|
|
179
187
|
run.finish()
|
|
180
188
|
|
|
181
|
-
if self._publish_url is not None:
|
|
189
|
+
if self._publish_url is not None or self._publish_test_results:
|
|
182
190
|
publish_test_results_to_datamesh_manager(run, self._publish_url, self._ssl_verification)
|
|
183
191
|
|
|
184
192
|
return run
|
|
@@ -243,33 +251,128 @@ class DataContract:
|
|
|
243
251
|
)
|
|
244
252
|
|
|
245
253
|
def export(self, export_format: ExportFormat, model: str = "all", sql_server_type: str = "auto", **kwargs) -> str:
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
+
if export_format == ExportFormat.html or export_format == ExportFormat.mermaid:
|
|
255
|
+
data_contract = resolve.resolve_data_contract_v2(
|
|
256
|
+
self._data_contract_file,
|
|
257
|
+
self._data_contract_str,
|
|
258
|
+
self._data_contract,
|
|
259
|
+
schema_location=self._schema_location,
|
|
260
|
+
inline_definitions=self._inline_definitions,
|
|
261
|
+
inline_quality=self._inline_quality,
|
|
262
|
+
)
|
|
254
263
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
264
|
+
return exporter_factory.create(export_format).export(
|
|
265
|
+
data_contract=data_contract,
|
|
266
|
+
model=model,
|
|
267
|
+
server=self._server,
|
|
268
|
+
sql_server_type=sql_server_type,
|
|
269
|
+
export_args=kwargs,
|
|
270
|
+
)
|
|
271
|
+
else:
|
|
272
|
+
data_contract = resolve.resolve_data_contract(
|
|
273
|
+
self._data_contract_file,
|
|
274
|
+
self._data_contract_str,
|
|
275
|
+
self._data_contract,
|
|
276
|
+
schema_location=self._schema_location,
|
|
277
|
+
inline_definitions=self._inline_definitions,
|
|
278
|
+
inline_quality=self._inline_quality,
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
return exporter_factory.create(export_format).export(
|
|
282
|
+
data_contract=data_contract,
|
|
283
|
+
model=model,
|
|
284
|
+
server=self._server,
|
|
285
|
+
sql_server_type=sql_server_type,
|
|
286
|
+
export_args=kwargs,
|
|
287
|
+
)
|
|
262
288
|
|
|
289
|
+
# REFACTOR THIS
|
|
290
|
+
# could be a class method, not using anything from the instance
|
|
263
291
|
def import_from_source(
|
|
264
292
|
self,
|
|
265
293
|
format: str,
|
|
266
294
|
source: typing.Optional[str] = None,
|
|
267
295
|
template: typing.Optional[str] = None,
|
|
268
296
|
schema: typing.Optional[str] = None,
|
|
297
|
+
spec: Spec = Spec.datacontract_specification,
|
|
269
298
|
**kwargs,
|
|
270
|
-
) -> DataContractSpecification:
|
|
271
|
-
|
|
299
|
+
) -> DataContractSpecification | OpenDataContractStandard:
|
|
300
|
+
id = kwargs.get("id")
|
|
301
|
+
owner = kwargs.get("owner")
|
|
272
302
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
303
|
+
if spec == Spec.odcs:
|
|
304
|
+
data_contract_specification_initial = DataContract.init(template=template, schema=schema)
|
|
305
|
+
|
|
306
|
+
odcs_imported = importer_factory.create(format).import_source(
|
|
307
|
+
data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
if isinstance(odcs_imported, DataContractSpecification):
|
|
311
|
+
# convert automatically
|
|
312
|
+
odcs_imported = to_odcs_v3(odcs_imported)
|
|
313
|
+
|
|
314
|
+
self._overwrite_id_in_odcs(odcs_imported, id)
|
|
315
|
+
self._overwrite_owner_in_odcs(odcs_imported, owner)
|
|
316
|
+
|
|
317
|
+
return odcs_imported
|
|
318
|
+
elif spec == Spec.datacontract_specification:
|
|
319
|
+
data_contract_specification_initial = DataContract.init(template=template, schema=schema)
|
|
320
|
+
|
|
321
|
+
data_contract_specification_imported = importer_factory.create(format).import_source(
|
|
322
|
+
data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
if isinstance(data_contract_specification_imported, OpenDataContractStandard):
|
|
326
|
+
# convert automatically
|
|
327
|
+
data_contract_specification_imported = import_from_odcs(
|
|
328
|
+
data_contract_specification_initial, data_contract_specification_imported
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
self._overwrite_id_in_data_contract_specification(data_contract_specification_imported, id)
|
|
332
|
+
self._overwrite_owner_in_data_contract_specification(data_contract_specification_imported, owner)
|
|
333
|
+
|
|
334
|
+
return data_contract_specification_imported
|
|
335
|
+
else:
|
|
336
|
+
raise DataContractException(
|
|
337
|
+
type="general",
|
|
338
|
+
result=ResultEnum.error,
|
|
339
|
+
name="Import Data Contract",
|
|
340
|
+
reason=f"Unsupported data contract format: {spec}",
|
|
341
|
+
engine="datacontract",
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
def _overwrite_id_in_data_contract_specification(
|
|
345
|
+
self, data_contract_specification: DataContractSpecification, id: str | None
|
|
346
|
+
):
|
|
347
|
+
if not id:
|
|
348
|
+
return
|
|
349
|
+
|
|
350
|
+
data_contract_specification.id = id
|
|
351
|
+
|
|
352
|
+
def _overwrite_owner_in_data_contract_specification(
|
|
353
|
+
self, data_contract_specification: DataContractSpecification, owner: str | None
|
|
354
|
+
):
|
|
355
|
+
if not owner:
|
|
356
|
+
return
|
|
357
|
+
|
|
358
|
+
if data_contract_specification.info is None:
|
|
359
|
+
data_contract_specification.info = Info()
|
|
360
|
+
data_contract_specification.info.owner = owner
|
|
361
|
+
|
|
362
|
+
def _overwrite_owner_in_odcs(self, odcs: OpenDataContractStandard, owner: str | None):
|
|
363
|
+
if not owner:
|
|
364
|
+
return
|
|
365
|
+
|
|
366
|
+
if odcs.customProperties is None:
|
|
367
|
+
odcs.customProperties = []
|
|
368
|
+
for customProperty in odcs.customProperties:
|
|
369
|
+
if customProperty.name == "owner":
|
|
370
|
+
customProperty.value = owner
|
|
371
|
+
return
|
|
372
|
+
odcs.customProperties.append(CustomProperty(property="owner", value=owner))
|
|
373
|
+
|
|
374
|
+
def _overwrite_id_in_odcs(self, odcs: OpenDataContractStandard, id: str | None):
|
|
375
|
+
if not id:
|
|
376
|
+
return
|
|
377
|
+
|
|
378
|
+
odcs.id = id
|
|
@@ -502,11 +502,13 @@ def prepare_query(quality: Quality, model_name: str, field_name: str = None) ->
|
|
|
502
502
|
query = quality.query
|
|
503
503
|
|
|
504
504
|
query = query.replace("{model}", model_name)
|
|
505
|
+
query = query.replace("{schema}", model_name)
|
|
505
506
|
query = query.replace("{table}", model_name)
|
|
506
507
|
|
|
507
508
|
if field_name is not None:
|
|
508
509
|
query = query.replace("{field}", field_name)
|
|
509
510
|
query = query.replace("{column}", field_name)
|
|
511
|
+
query = query.replace("{property}", field_name)
|
|
510
512
|
|
|
511
513
|
return query
|
|
512
514
|
|
|
@@ -27,7 +27,7 @@ class DbtStageExporter(Exporter):
|
|
|
27
27
|
)
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
def to_dbt_models_yaml(data_contract_spec: DataContractSpecification, server: str = None):
|
|
30
|
+
def to_dbt_models_yaml(data_contract_spec: DataContractSpecification, server: str = None) -> str:
|
|
31
31
|
dbt = {
|
|
32
32
|
"version": 2,
|
|
33
33
|
"models": [],
|
|
@@ -102,8 +102,11 @@ def _to_dbt_model(
|
|
|
102
102
|
"name": model_key,
|
|
103
103
|
}
|
|
104
104
|
model_type = _to_dbt_model_type(model_value.type)
|
|
105
|
+
|
|
105
106
|
dbt_model["config"] = {"meta": {"data_contract": data_contract_spec.id}}
|
|
106
|
-
|
|
107
|
+
|
|
108
|
+
if model_type:
|
|
109
|
+
dbt_model["config"]["materialized"] = model_type
|
|
107
110
|
|
|
108
111
|
if data_contract_spec.info.owner is not None:
|
|
109
112
|
dbt_model["config"]["meta"]["owner"] = data_contract_spec.info.owner
|
|
@@ -123,7 +126,7 @@ def _to_dbt_model_type(model_type):
|
|
|
123
126
|
# Allowed values: table, view, incremental, ephemeral, materialized view
|
|
124
127
|
# Custom values also possible
|
|
125
128
|
if model_type is None:
|
|
126
|
-
return
|
|
129
|
+
return None
|
|
127
130
|
if model_type.lower() == "table":
|
|
128
131
|
return "table"
|
|
129
132
|
if model_type.lower() == "view":
|
datacontract/export/exporter.py
CHANGED
|
@@ -89,6 +89,12 @@ exporter_factory.register_lazy_exporter(
|
|
|
89
89
|
class_name="DbtExporter",
|
|
90
90
|
)
|
|
91
91
|
|
|
92
|
+
exporter_factory.register_lazy_exporter(
|
|
93
|
+
name=ExportFormat.mermaid,
|
|
94
|
+
module_path="datacontract.export.mermaid_exporter",
|
|
95
|
+
class_name="MermaidExporter",
|
|
96
|
+
)
|
|
97
|
+
|
|
92
98
|
exporter_factory.register_lazy_exporter(
|
|
93
99
|
name=ExportFormat.dbt_sources,
|
|
94
100
|
module_path="datacontract.export.dbt_converter",
|
|
@@ -127,7 +133,7 @@ exporter_factory.register_lazy_exporter(
|
|
|
127
133
|
|
|
128
134
|
exporter_factory.register_lazy_exporter(
|
|
129
135
|
name=ExportFormat.html,
|
|
130
|
-
module_path="datacontract.export.
|
|
136
|
+
module_path="datacontract.export.html_exporter",
|
|
131
137
|
class_name="HtmlExporter",
|
|
132
138
|
)
|
|
133
139
|
|
|
@@ -6,8 +6,10 @@ import jinja_partials
|
|
|
6
6
|
import pytz
|
|
7
7
|
import yaml
|
|
8
8
|
from jinja2 import Environment, PackageLoader, select_autoescape
|
|
9
|
+
from open_data_contract_standard.model import OpenDataContractStandard
|
|
9
10
|
|
|
10
11
|
from datacontract.export.exporter import Exporter
|
|
12
|
+
from datacontract.export.mermaid_exporter import to_mermaid
|
|
11
13
|
from datacontract.model.data_contract_specification import DataContractSpecification
|
|
12
14
|
|
|
13
15
|
|
|
@@ -16,7 +18,7 @@ class HtmlExporter(Exporter):
|
|
|
16
18
|
return to_html(data_contract)
|
|
17
19
|
|
|
18
20
|
|
|
19
|
-
def to_html(data_contract_spec: DataContractSpecification) -> str:
|
|
21
|
+
def to_html(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str:
|
|
20
22
|
# Load templates from templates folder
|
|
21
23
|
package_loader = PackageLoader("datacontract", "templates")
|
|
22
24
|
env = Environment(
|
|
@@ -31,28 +33,30 @@ def to_html(data_contract_spec: DataContractSpecification) -> str:
|
|
|
31
33
|
|
|
32
34
|
# Load the required template
|
|
33
35
|
# needs to be included in /MANIFEST.in
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
if data_contract_spec.quality.type == "great-expectations":
|
|
40
|
-
quality_specification = yaml.dump(
|
|
41
|
-
data_contract_spec.quality.specification, sort_keys=False, default_style="|"
|
|
42
|
-
)
|
|
43
|
-
else:
|
|
44
|
-
quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
|
|
45
|
-
else:
|
|
46
|
-
quality_specification = None
|
|
36
|
+
template_file = "datacontract.html"
|
|
37
|
+
if isinstance(data_contract_spec, OpenDataContractStandard):
|
|
38
|
+
template_file = "datacontract_odcs.html"
|
|
39
|
+
|
|
40
|
+
template = env.get_template(template_file)
|
|
47
41
|
|
|
48
42
|
style_content, _, _ = package_loader.get_source(env, "style/output.css")
|
|
49
43
|
|
|
44
|
+
quality_specification = None
|
|
45
|
+
if isinstance(data_contract_spec, DataContractSpecification):
|
|
46
|
+
if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
|
|
47
|
+
quality_specification = data_contract_spec.quality.specification
|
|
48
|
+
elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
|
|
49
|
+
if data_contract_spec.quality.type == "great-expectations":
|
|
50
|
+
quality_specification = yaml.dump(
|
|
51
|
+
data_contract_spec.quality.specification, sort_keys=False, default_style="|"
|
|
52
|
+
)
|
|
53
|
+
else:
|
|
54
|
+
quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
|
|
55
|
+
|
|
50
56
|
datacontract_yaml = data_contract_spec.to_yaml()
|
|
51
57
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
formatted_date = now.strftime("%d %b %Y %H:%M:%S UTC")
|
|
55
|
-
datacontract_cli_version = get_version()
|
|
58
|
+
# Get the mermaid diagram
|
|
59
|
+
mermaid_diagram = to_mermaid(data_contract_spec)
|
|
56
60
|
|
|
57
61
|
# Render the template with necessary data
|
|
58
62
|
html_string = template.render(
|
|
@@ -60,13 +64,20 @@ def to_html(data_contract_spec: DataContractSpecification) -> str:
|
|
|
60
64
|
quality_specification=quality_specification,
|
|
61
65
|
style=style_content,
|
|
62
66
|
datacontract_yaml=datacontract_yaml,
|
|
63
|
-
formatted_date=
|
|
64
|
-
datacontract_cli_version=
|
|
67
|
+
formatted_date=_formatted_date(),
|
|
68
|
+
datacontract_cli_version=get_version(),
|
|
69
|
+
mermaid_diagram=mermaid_diagram,
|
|
65
70
|
)
|
|
66
71
|
|
|
67
72
|
return html_string
|
|
68
73
|
|
|
69
74
|
|
|
75
|
+
def _formatted_date() -> str:
|
|
76
|
+
tz = pytz.timezone("UTC")
|
|
77
|
+
now = datetime.datetime.now(tz)
|
|
78
|
+
return now.strftime("%d %b %Y %H:%M:%S UTC")
|
|
79
|
+
|
|
80
|
+
|
|
70
81
|
def get_version() -> str:
|
|
71
82
|
try:
|
|
72
83
|
return version("datacontract_cli")
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from open_data_contract_standard.model import OpenDataContractStandard
|
|
2
|
+
|
|
3
|
+
from datacontract.export.exporter import Exporter
|
|
4
|
+
from datacontract.model.data_contract_specification import DataContractSpecification
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class MermaidExporter(Exporter):
|
|
8
|
+
def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
|
|
9
|
+
return to_mermaid(data_contract)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def to_mermaid(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str | None:
|
|
13
|
+
if isinstance(data_contract_spec, DataContractSpecification):
|
|
14
|
+
return dcs_to_mermaid(data_contract_spec)
|
|
15
|
+
elif isinstance(data_contract_spec, OpenDataContractStandard):
|
|
16
|
+
return odcs_to_mermaid(data_contract_spec)
|
|
17
|
+
else:
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def dcs_to_mermaid(data_contract_spec: DataContractSpecification) -> str | None:
|
|
22
|
+
try:
|
|
23
|
+
if not data_contract_spec.models:
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
mmd_entity = "erDiagram\n"
|
|
27
|
+
mmd_references = []
|
|
28
|
+
|
|
29
|
+
for model_name, model in data_contract_spec.models.items():
|
|
30
|
+
entity_block = ""
|
|
31
|
+
|
|
32
|
+
for field_name, field in model.fields.items():
|
|
33
|
+
clean_name = _sanitize_name(field_name)
|
|
34
|
+
indicators = ""
|
|
35
|
+
|
|
36
|
+
if field.primaryKey or (field.unique and field.required):
|
|
37
|
+
indicators += "🔑"
|
|
38
|
+
if field.references:
|
|
39
|
+
indicators += "⌘"
|
|
40
|
+
|
|
41
|
+
field_type = field.type or "unknown"
|
|
42
|
+
entity_block += f"\t{clean_name}{indicators} {field_type}\n"
|
|
43
|
+
|
|
44
|
+
if field.references:
|
|
45
|
+
referenced_model = field.references.split(".")[0] if "." in field.references else ""
|
|
46
|
+
if referenced_model:
|
|
47
|
+
mmd_references.append(f'"📑{referenced_model}"' + "}o--{ ||" + f'"📑{model_name}"')
|
|
48
|
+
|
|
49
|
+
mmd_entity += f'\t"**{model_name}**"' + "{\n" + entity_block + "}\n"
|
|
50
|
+
|
|
51
|
+
if mmd_references:
|
|
52
|
+
mmd_entity += "\n" + "\n".join(mmd_references)
|
|
53
|
+
|
|
54
|
+
return f"{mmd_entity}\n"
|
|
55
|
+
|
|
56
|
+
except Exception as e:
|
|
57
|
+
print(f"Error generating DCS mermaid diagram: {e}")
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def odcs_to_mermaid(data_contract_spec: OpenDataContractStandard) -> str | None:
|
|
62
|
+
try:
|
|
63
|
+
if not data_contract_spec.schema_:
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
mmd_entity = "erDiagram\n"
|
|
67
|
+
|
|
68
|
+
for schema in data_contract_spec.schema_:
|
|
69
|
+
schema_name = schema.name or schema.physicalName
|
|
70
|
+
entity_block = ""
|
|
71
|
+
|
|
72
|
+
if schema.properties:
|
|
73
|
+
for prop in schema.properties:
|
|
74
|
+
clean_name = _sanitize_name(prop.name)
|
|
75
|
+
indicators = ""
|
|
76
|
+
|
|
77
|
+
if prop.primaryKey:
|
|
78
|
+
indicators += "🔑"
|
|
79
|
+
if getattr(prop, "partitioned", False):
|
|
80
|
+
indicators += "🔀"
|
|
81
|
+
if getattr(prop, "criticalDataElement", False):
|
|
82
|
+
indicators += "⚠️"
|
|
83
|
+
|
|
84
|
+
prop_type = prop.logicalType or prop.physicalType or "unknown"
|
|
85
|
+
entity_block += f"\t{clean_name}{indicators} {prop_type}\n"
|
|
86
|
+
|
|
87
|
+
mmd_entity += f'\t"**{schema_name}**"' + "{\n" + entity_block + "}\n"
|
|
88
|
+
|
|
89
|
+
return f"{mmd_entity}\n"
|
|
90
|
+
|
|
91
|
+
except Exception as e:
|
|
92
|
+
print(f"Error generating ODCS mermaid diagram: {e}")
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _sanitize_name(name: str) -> str:
|
|
97
|
+
return name.replace("#", "Nb").replace(" ", "_").replace("/", "by")
|
|
@@ -23,6 +23,12 @@ class OdcsV3Exporter(Exporter):
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
|
|
26
|
+
result = to_odcs_v3(data_contract_spec)
|
|
27
|
+
|
|
28
|
+
return result.to_yaml()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def to_odcs_v3(data_contract_spec: DataContractSpecification) -> OpenDataContractStandard:
|
|
26
32
|
result = OpenDataContractStandard(
|
|
27
33
|
apiVersion="v3.0.1",
|
|
28
34
|
kind="DataContract",
|
|
@@ -31,7 +37,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
|
|
|
31
37
|
version=data_contract_spec.info.version,
|
|
32
38
|
status=to_status(data_contract_spec.info.status),
|
|
33
39
|
)
|
|
34
|
-
|
|
35
40
|
if data_contract_spec.terms is not None:
|
|
36
41
|
result.description = Description(
|
|
37
42
|
purpose=data_contract_spec.terms.description.strip()
|
|
@@ -42,12 +47,10 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
|
|
|
42
47
|
if data_contract_spec.terms.limitations is not None
|
|
43
48
|
else None,
|
|
44
49
|
)
|
|
45
|
-
|
|
46
50
|
result.schema_ = []
|
|
47
51
|
for model_key, model_value in data_contract_spec.models.items():
|
|
48
52
|
odcs_schema = to_odcs_schema(model_key, model_value)
|
|
49
53
|
result.schema_.append(odcs_schema)
|
|
50
|
-
|
|
51
54
|
if data_contract_spec.servicelevels is not None:
|
|
52
55
|
slas = []
|
|
53
56
|
if data_contract_spec.servicelevels.availability is not None:
|
|
@@ -65,7 +68,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
|
|
|
65
68
|
|
|
66
69
|
if len(slas) > 0:
|
|
67
70
|
result.slaProperties = slas
|
|
68
|
-
|
|
69
71
|
if data_contract_spec.info.contact is not None:
|
|
70
72
|
support = []
|
|
71
73
|
if data_contract_spec.info.contact.email is not None:
|
|
@@ -74,7 +76,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
|
|
|
74
76
|
support.append(Support(channel="other", url=data_contract_spec.info.contact.url))
|
|
75
77
|
if len(support) > 0:
|
|
76
78
|
result.support = support
|
|
77
|
-
|
|
78
79
|
if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0:
|
|
79
80
|
servers = []
|
|
80
81
|
|
|
@@ -126,18 +127,15 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
|
|
|
126
127
|
|
|
127
128
|
if len(servers) > 0:
|
|
128
129
|
result.servers = servers
|
|
129
|
-
|
|
130
130
|
custom_properties = []
|
|
131
131
|
if data_contract_spec.info.owner is not None:
|
|
132
132
|
custom_properties.append(CustomProperty(property="owner", value=data_contract_spec.info.owner))
|
|
133
133
|
if data_contract_spec.info.model_extra is not None:
|
|
134
134
|
for key, value in data_contract_spec.info.model_extra.items():
|
|
135
135
|
custom_properties.append(CustomProperty(property=key, value=value))
|
|
136
|
-
|
|
137
136
|
if len(custom_properties) > 0:
|
|
138
137
|
result.customProperties = custom_properties
|
|
139
|
-
|
|
140
|
-
return result.to_yaml()
|
|
138
|
+
return result
|
|
141
139
|
|
|
142
140
|
|
|
143
141
|
def to_odcs_schema(model_key, model_value: Model) -> SchemaObject:
|
|
@@ -249,7 +247,7 @@ def to_property(field_name: str, field: Field) -> SchemaProperty:
|
|
|
249
247
|
|
|
250
248
|
if field.type is not None:
|
|
251
249
|
property.logicalType = to_logical_type(field.type)
|
|
252
|
-
property.physicalType = to_physical_type(field.config)
|
|
250
|
+
property.physicalType = to_physical_type(field.config) or field.type
|
|
253
251
|
|
|
254
252
|
if field.description is not None:
|
|
255
253
|
property.description = field.description
|
|
@@ -2,12 +2,14 @@ import yaml
|
|
|
2
2
|
|
|
3
3
|
from datacontract.engines.data_contract_checks import create_checks
|
|
4
4
|
from datacontract.export.exporter import Exporter
|
|
5
|
+
from datacontract.model.data_contract_specification import DataContractSpecification, Server
|
|
5
6
|
from datacontract.model.run import Run
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
class SodaExporter(Exporter):
|
|
9
|
-
def export(self, data_contract, model, server, sql_server_type, export_args) ->
|
|
10
|
+
def export(self, data_contract, model, server, sql_server_type, export_args) -> str:
|
|
10
11
|
run = Run.create_run()
|
|
12
|
+
server = get_server(data_contract, server)
|
|
11
13
|
run.checks.extend(create_checks(data_contract, server))
|
|
12
14
|
return to_sodacl_yaml(run)
|
|
13
15
|
|
|
@@ -28,3 +30,9 @@ def to_sodacl_yaml(run: Run) -> str:
|
|
|
28
30
|
else:
|
|
29
31
|
sodacl_dict[key] = value
|
|
30
32
|
return yaml.dump(sodacl_dict)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_server(data_contract_specification: DataContractSpecification, server_name: str = None) -> Server | None:
|
|
36
|
+
if server_name is None:
|
|
37
|
+
return None
|
|
38
|
+
return data_contract_specification.servers.get(server_name)
|
|
@@ -4,7 +4,7 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class SqlExporter(Exporter):
|
|
7
|
-
def export(self, data_contract, model, server, sql_server_type, export_args) ->
|
|
7
|
+
def export(self, data_contract, model, server, sql_server_type, export_args) -> str:
|
|
8
8
|
server_type = _determine_sql_server_type(
|
|
9
9
|
data_contract,
|
|
10
10
|
sql_server_type,
|
|
@@ -13,7 +13,7 @@ class SqlExporter(Exporter):
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class SqlQueryExporter(Exporter):
|
|
16
|
-
def export(self, data_contract, model, server, sql_server_type, export_args) ->
|
|
16
|
+
def export(self, data_contract, model, server, sql_server_type, export_args) -> str:
|
|
17
17
|
model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
|
|
18
18
|
server_type = _determine_sql_server_type(data_contract, sql_server_type, export_args.get("server"))
|
|
19
19
|
return to_sql_query(
|