datacontract-cli 0.10.2__py3-none-any.whl → 0.10.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- datacontract/breaking/breaking.py +12 -0
- datacontract/breaking/breaking_rules.py +4 -0
- datacontract/catalog/catalog.py +3 -0
- datacontract/cli.py +36 -8
- datacontract/data_contract.py +62 -128
- datacontract/export/avro_converter.py +16 -2
- datacontract/export/bigquery_converter.py +106 -0
- datacontract/export/go_converter.py +98 -0
- datacontract/export/html_export.py +6 -1
- datacontract/export/jsonschema_converter.py +45 -5
- datacontract/export/sql_converter.py +1 -0
- datacontract/export/sql_type_converter.py +42 -1
- datacontract/imports/avro_importer.py +14 -1
- datacontract/imports/bigquery_importer.py +166 -0
- datacontract/imports/jsonschema_importer.py +150 -0
- datacontract/model/data_contract_specification.py +55 -1
- datacontract/publish/publish.py +32 -0
- datacontract/templates/datacontract.html +37 -346
- datacontract/templates/index.html +233 -0
- datacontract/templates/partials/datacontract_information.html +66 -0
- datacontract/templates/partials/datacontract_servicelevels.html +253 -0
- datacontract/templates/partials/datacontract_terms.html +44 -0
- datacontract/templates/partials/definition.html +99 -0
- datacontract/templates/partials/example.html +27 -0
- datacontract/templates/partials/model_field.html +97 -0
- datacontract/templates/partials/server.html +144 -0
- datacontract/templates/style/output.css +94 -13
- {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/METADATA +139 -96
- {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/RECORD +33 -20
- {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/LICENSE +0 -0
- {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/WHEEL +0 -0
- {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/entry_points.txt +0 -0
- {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/top_level.txt +0 -0
|
@@ -256,6 +256,18 @@ def field_breaking_changes(
|
|
|
256
256
|
)
|
|
257
257
|
)
|
|
258
258
|
continue
|
|
259
|
+
|
|
260
|
+
if field_definition_field == "items" and old_field.type == 'array' and new_field.type == 'array':
|
|
261
|
+
results.extend(
|
|
262
|
+
field_breaking_changes(
|
|
263
|
+
old_field=old_value,
|
|
264
|
+
new_field=new_value,
|
|
265
|
+
composition=composition + ['items'],
|
|
266
|
+
new_path=new_path,
|
|
267
|
+
include_severities=include_severities,
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
continue
|
|
259
271
|
|
|
260
272
|
rule_name = None
|
|
261
273
|
description = None
|
|
@@ -90,6 +90,10 @@ class BreakingRules:
|
|
|
90
90
|
field_tags_removed = Severity.INFO
|
|
91
91
|
field_tags_updated = Severity.INFO
|
|
92
92
|
|
|
93
|
+
field_example_added = Severity.INFO
|
|
94
|
+
field_example_updated = Severity.INFO
|
|
95
|
+
field_example_removed = Severity.INFO
|
|
96
|
+
|
|
93
97
|
# quality Rules
|
|
94
98
|
quality_added = Severity.INFO
|
|
95
99
|
quality_removed = Severity.WARNING
|
datacontract/catalog/catalog.py
CHANGED
|
@@ -53,8 +53,10 @@ def create_index_html(contracts, path):
|
|
|
53
53
|
)
|
|
54
54
|
|
|
55
55
|
# Load the required template
|
|
56
|
+
# needs to be included in /MANIFEST.in
|
|
56
57
|
template = env.get_template("index.html")
|
|
57
58
|
|
|
59
|
+
# needs to be included in /MANIFEST.in
|
|
58
60
|
style_content, _, _ = package_loader.get_source(env, "style/output.css")
|
|
59
61
|
|
|
60
62
|
tz = pytz.timezone("UTC")
|
|
@@ -69,6 +71,7 @@ def create_index_html(contracts, path):
|
|
|
69
71
|
datacontract_cli_version=datacontract_cli_version,
|
|
70
72
|
contracts=contracts,
|
|
71
73
|
contracts_size=len(contracts),
|
|
74
|
+
owners=sorted(set(dc.spec.info.owner for dc in contracts if dc.spec.info.owner)),
|
|
72
75
|
)
|
|
73
76
|
f.write(html_string)
|
|
74
77
|
print(f"Created {index_filepath}")
|
datacontract/cli.py
CHANGED
|
@@ -10,12 +10,14 @@ from rich.console import Console
|
|
|
10
10
|
from rich.table import Table
|
|
11
11
|
from typer.core import TyperGroup
|
|
12
12
|
from typing_extensions import Annotated
|
|
13
|
+
from typing import List
|
|
13
14
|
|
|
14
|
-
from datacontract.catalog.catalog import create_index_html,
|
|
15
|
-
create_data_contract_html
|
|
15
|
+
from datacontract.catalog.catalog import create_index_html, create_data_contract_html
|
|
16
16
|
from datacontract.data_contract import DataContract
|
|
17
|
-
from datacontract.init.download_datacontract_file import
|
|
18
|
-
|
|
17
|
+
from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
|
|
18
|
+
|
|
19
|
+
from datacontract.publish.publish import publish_to_datamesh_manager
|
|
20
|
+
|
|
19
21
|
|
|
20
22
|
console = Console()
|
|
21
23
|
|
|
@@ -158,12 +160,19 @@ class ExportFormat(str, Enum):
|
|
|
158
160
|
sql = "sql"
|
|
159
161
|
sql_query = "sql-query"
|
|
160
162
|
html = "html"
|
|
163
|
+
go = "go"
|
|
164
|
+
bigquery = "bigquery"
|
|
161
165
|
|
|
162
166
|
|
|
163
167
|
@app.command()
|
|
164
168
|
def export(
|
|
165
169
|
format: Annotated[ExportFormat, typer.Option(help="The export format.")],
|
|
166
|
-
output: Annotated[
|
|
170
|
+
output: Annotated[
|
|
171
|
+
Path,
|
|
172
|
+
typer.Option(
|
|
173
|
+
help="Specify the file path where the exported data will be saved. If no path is provided, the output will be printed to stdout."
|
|
174
|
+
),
|
|
175
|
+
] = None,
|
|
167
176
|
server: Annotated[str, typer.Option(help="The server name to export.")] = None,
|
|
168
177
|
model: Annotated[
|
|
169
178
|
str,
|
|
@@ -204,7 +213,7 @@ def export(
|
|
|
204
213
|
if output is None:
|
|
205
214
|
console.print(result, markup=False)
|
|
206
215
|
else:
|
|
207
|
-
with output.open(
|
|
216
|
+
with output.open("w") as f:
|
|
208
217
|
f.write(result)
|
|
209
218
|
console.print(f"Written result to {output}")
|
|
210
219
|
|
|
@@ -213,20 +222,39 @@ class ImportFormat(str, Enum):
|
|
|
213
222
|
sql = "sql"
|
|
214
223
|
avro = "avro"
|
|
215
224
|
glue = "glue"
|
|
225
|
+
bigquery = "bigquery"
|
|
226
|
+
jsonschema = "jsonschema"
|
|
216
227
|
|
|
217
228
|
|
|
218
229
|
@app.command(name="import")
|
|
219
230
|
def import_(
|
|
220
231
|
format: Annotated[ImportFormat, typer.Option(help="The format of the source file.")],
|
|
221
|
-
source: Annotated[str, typer.Option(help="The path to the file or Glue Database that should be imported.")],
|
|
232
|
+
source: Annotated[Optional[str], typer.Option(help="The path to the file or Glue Database that should be imported.")] = None,
|
|
233
|
+
bigquery_project: Annotated[Optional[str], typer.Option(help="The bigquery project id.")] = None,
|
|
234
|
+
bigquery_dataset: Annotated[Optional[str], typer.Option(help="The bigquery dataset id.")] = None,
|
|
235
|
+
bigquery_table: Annotated[Optional[List[str]], typer.Option(help="List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all tables in the dataset).")] = None,
|
|
222
236
|
):
|
|
223
237
|
"""
|
|
224
238
|
Create a data contract from the given source location. Prints to stdout.
|
|
225
239
|
"""
|
|
226
|
-
result = DataContract().import_from_source(format, source)
|
|
240
|
+
result = DataContract().import_from_source(format, source, bigquery_table, bigquery_project, bigquery_dataset)
|
|
227
241
|
console.print(result.to_yaml())
|
|
228
242
|
|
|
229
243
|
|
|
244
|
+
@app.command(name="publish")
|
|
245
|
+
def publish(
|
|
246
|
+
location: Annotated[
|
|
247
|
+
str, typer.Argument(help="The location (url or path) of the data contract yaml.")
|
|
248
|
+
] = "datacontract.yaml",
|
|
249
|
+
):
|
|
250
|
+
"""
|
|
251
|
+
Publish the data contract to the Data Mesh Manager.
|
|
252
|
+
"""
|
|
253
|
+
publish_to_datamesh_manager(
|
|
254
|
+
data_contract=DataContract(data_contract_file=location),
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
230
258
|
@app.command(name="catalog")
|
|
231
259
|
def catalog(
|
|
232
260
|
files: Annotated[
|
datacontract/data_contract.py
CHANGED
|
@@ -6,16 +6,15 @@ import typing
|
|
|
6
6
|
import yaml
|
|
7
7
|
from pyspark.sql import SparkSession
|
|
8
8
|
|
|
9
|
-
from datacontract.breaking.breaking import models_breaking_changes,
|
|
10
|
-
quality_breaking_changes
|
|
9
|
+
from datacontract.breaking.breaking import models_breaking_changes, quality_breaking_changes
|
|
11
10
|
from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
|
|
12
11
|
check_that_datacontract_contains_valid_server_configuration,
|
|
13
12
|
)
|
|
14
|
-
from datacontract.engines.fastjsonschema.check_jsonschema import
|
|
15
|
-
check_jsonschema
|
|
13
|
+
from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
|
|
16
14
|
from datacontract.engines.soda.check_soda_execute import check_soda_execute
|
|
17
15
|
from datacontract.export.avro_converter import to_avro_schema_json
|
|
18
16
|
from datacontract.export.avro_idl_converter import to_avro_idl
|
|
17
|
+
from datacontract.export.bigquery_converter import to_bigquery_json
|
|
19
18
|
from datacontract.export.dbt_converter import to_dbt_models_yaml, \
|
|
20
19
|
to_dbt_sources_yaml, to_dbt_staging_sql
|
|
21
20
|
from datacontract.export.great_expectations_converter import \
|
|
@@ -25,13 +24,16 @@ from datacontract.export.jsonschema_converter import to_jsonschema_json
|
|
|
25
24
|
from datacontract.export.odcs_converter import to_odcs_yaml
|
|
26
25
|
from datacontract.export.protobuf_converter import to_protobuf
|
|
27
26
|
from datacontract.export.pydantic_converter import to_pydantic_model_str
|
|
27
|
+
from datacontract.export.go_converter import to_go_types
|
|
28
28
|
from datacontract.export.rdf_converter import to_rdf_n3
|
|
29
29
|
from datacontract.export.sodacl_converter import to_sodacl_yaml
|
|
30
30
|
from datacontract.export.sql_converter import to_sql_ddl, to_sql_query
|
|
31
31
|
from datacontract.export.terraform_converter import to_terraform
|
|
32
32
|
from datacontract.imports.avro_importer import import_avro
|
|
33
|
+
from datacontract.imports.bigquery_importer import import_bigquery_from_api, import_bigquery_from_json
|
|
33
34
|
from datacontract.imports.glue_importer import import_glue
|
|
34
35
|
from datacontract.imports.sql_importer import import_sql
|
|
36
|
+
from datacontract.imports.jsonschema_importer import import_jsonschema
|
|
35
37
|
from datacontract.integration.publish_datamesh_manager import \
|
|
36
38
|
publish_datamesh_manager
|
|
37
39
|
from datacontract.integration.publish_opentelemetry import publish_opentelemetry
|
|
@@ -39,17 +41,12 @@ from datacontract.lint import resolve
|
|
|
39
41
|
from datacontract.lint.linters.description_linter import DescriptionLinter
|
|
40
42
|
from datacontract.lint.linters.example_model_linter import ExampleModelLinter
|
|
41
43
|
from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
|
|
42
|
-
from datacontract.lint.linters.field_reference_linter import
|
|
43
|
-
FieldReferenceLinter
|
|
44
|
+
from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
|
|
44
45
|
from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
|
|
45
|
-
from datacontract.lint.linters.quality_schema_linter import
|
|
46
|
-
|
|
47
|
-
from datacontract.
|
|
48
|
-
|
|
49
|
-
from datacontract.model.breaking_change import BreakingChanges, BreakingChange, \
|
|
50
|
-
Severity
|
|
51
|
-
from datacontract.model.data_contract_specification import \
|
|
52
|
-
DataContractSpecification, Server
|
|
46
|
+
from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
|
|
47
|
+
from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
|
|
48
|
+
from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Severity
|
|
49
|
+
from datacontract.model.data_contract_specification import DataContractSpecification, Server
|
|
53
50
|
from datacontract.model.exceptions import DataContractException
|
|
54
51
|
from datacontract.model.run import Run, Check
|
|
55
52
|
|
|
@@ -289,28 +286,8 @@ class DataContract:
|
|
|
289
286
|
inline_quality=True,
|
|
290
287
|
)
|
|
291
288
|
if export_format == "jsonschema":
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
model_names = list(data_contract.models.keys())
|
|
296
|
-
|
|
297
|
-
if model == "all":
|
|
298
|
-
if len(data_contract.models.items()) != 1:
|
|
299
|
-
raise RuntimeError(
|
|
300
|
-
f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
|
|
301
|
-
)
|
|
302
|
-
|
|
303
|
-
model_name, model_value = next(iter(data_contract.models.items()))
|
|
304
|
-
return to_jsonschema_json(model_name, model_value)
|
|
305
|
-
else:
|
|
306
|
-
model_name = model
|
|
307
|
-
model_value = data_contract.models.get(model_name)
|
|
308
|
-
if model_value is None:
|
|
309
|
-
raise RuntimeError(
|
|
310
|
-
f"Model {model_name} not found in the data contract. Available models: {model_names}"
|
|
311
|
-
)
|
|
312
|
-
|
|
313
|
-
return to_jsonschema_json(model_name, model_value)
|
|
289
|
+
model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
|
|
290
|
+
return to_jsonschema_json(model_name, model_value)
|
|
314
291
|
if export_format == "sodacl":
|
|
315
292
|
return to_sodacl_yaml(data_contract)
|
|
316
293
|
if export_format == "dbt":
|
|
@@ -318,28 +295,8 @@ class DataContract:
|
|
|
318
295
|
if export_format == "dbt-sources":
|
|
319
296
|
return to_dbt_sources_yaml(data_contract, self._server)
|
|
320
297
|
if export_format == "dbt-staging-sql":
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
model_names = list(data_contract.models.keys())
|
|
325
|
-
|
|
326
|
-
if model == "all":
|
|
327
|
-
if len(data_contract.models.items()) != 1:
|
|
328
|
-
raise RuntimeError(
|
|
329
|
-
f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
|
|
330
|
-
)
|
|
331
|
-
|
|
332
|
-
model_name, model_value = next(iter(data_contract.models.items()))
|
|
333
|
-
return to_dbt_staging_sql(data_contract, model_name, model_value)
|
|
334
|
-
else:
|
|
335
|
-
model_name = model
|
|
336
|
-
model_value = data_contract.models.get(model_name)
|
|
337
|
-
if model_value is None:
|
|
338
|
-
raise RuntimeError(
|
|
339
|
-
f"Model {model_name} not found in the data contract. Available models: {model_names}"
|
|
340
|
-
)
|
|
341
|
-
|
|
342
|
-
return to_dbt_staging_sql(data_contract, model_name, model_value)
|
|
298
|
+
model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
|
|
299
|
+
return to_dbt_staging_sql(data_contract, model_name, model_value)
|
|
343
300
|
if export_format == "odcs":
|
|
344
301
|
return to_odcs_yaml(data_contract)
|
|
345
302
|
if export_format == "rdf":
|
|
@@ -347,28 +304,8 @@ class DataContract:
|
|
|
347
304
|
if export_format == "protobuf":
|
|
348
305
|
return to_protobuf(data_contract)
|
|
349
306
|
if export_format == "avro":
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
model_names = list(data_contract.models.keys())
|
|
354
|
-
|
|
355
|
-
if model == "all":
|
|
356
|
-
if len(data_contract.models.items()) != 1:
|
|
357
|
-
raise RuntimeError(
|
|
358
|
-
f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
|
|
359
|
-
)
|
|
360
|
-
|
|
361
|
-
model_name, model_value = next(iter(data_contract.models.items()))
|
|
362
|
-
return to_avro_schema_json(model_name, model_value)
|
|
363
|
-
else:
|
|
364
|
-
model_name = model
|
|
365
|
-
model_value = data_contract.models.get(model_name)
|
|
366
|
-
if model_value is None:
|
|
367
|
-
raise RuntimeError(
|
|
368
|
-
f"Model {model_name} not found in the data contract. Available models: {model_names}"
|
|
369
|
-
)
|
|
370
|
-
|
|
371
|
-
return to_avro_schema_json(model_name, model_value)
|
|
307
|
+
model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
|
|
308
|
+
return to_avro_schema_json(model_name, model_value)
|
|
372
309
|
if export_format == "avro-idl":
|
|
373
310
|
return to_avro_idl(data_contract)
|
|
374
311
|
if export_format == "terraform":
|
|
@@ -377,59 +314,26 @@ class DataContract:
|
|
|
377
314
|
server_type = self._determine_sql_server_type(data_contract, sql_server_type)
|
|
378
315
|
return to_sql_ddl(data_contract, server_type=server_type)
|
|
379
316
|
if export_format == "sql-query":
|
|
380
|
-
|
|
381
|
-
raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
|
|
382
|
-
|
|
317
|
+
model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
|
|
383
318
|
server_type = self._determine_sql_server_type(data_contract, sql_server_type)
|
|
384
|
-
|
|
385
|
-
model_names = list(data_contract.models.keys())
|
|
386
|
-
|
|
387
|
-
if model == "all":
|
|
388
|
-
if len(data_contract.models.items()) != 1:
|
|
389
|
-
raise RuntimeError(
|
|
390
|
-
f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
|
|
391
|
-
)
|
|
392
|
-
|
|
393
|
-
model_name, model_value = next(iter(data_contract.models.items()))
|
|
394
|
-
return to_sql_query(data_contract, model_name, model_value, server_type)
|
|
395
|
-
else:
|
|
396
|
-
model_name = model
|
|
397
|
-
model_value = data_contract.models.get(model_name)
|
|
398
|
-
if model_value is None:
|
|
399
|
-
raise RuntimeError(
|
|
400
|
-
f"Model {model_name} not found in the data contract. Available models: {model_names}"
|
|
401
|
-
)
|
|
402
|
-
|
|
403
|
-
return to_sql_query(data_contract, model_name, model_value, server_type)
|
|
404
|
-
|
|
319
|
+
return to_sql_query(data_contract, model_name, model_value, server_type)
|
|
405
320
|
if export_format == "great-expectations":
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
model_names = list(data_contract.models.keys())
|
|
410
|
-
|
|
411
|
-
if model == "all":
|
|
412
|
-
if len(data_contract.models.items()) != 1:
|
|
413
|
-
raise RuntimeError(
|
|
414
|
-
f"Export to {export_format} is model specific. Specify the model via --model "
|
|
415
|
-
f"$MODEL_NAME. Available models: {model_names}"
|
|
416
|
-
)
|
|
417
|
-
|
|
418
|
-
model_name, model_value = next(iter(data_contract.models.items()))
|
|
419
|
-
return to_great_expectations(data_contract, model_name)
|
|
420
|
-
else:
|
|
421
|
-
model_name = model
|
|
422
|
-
model_value = data_contract.models.get(model_name)
|
|
423
|
-
if model_value is None:
|
|
424
|
-
raise RuntimeError(
|
|
425
|
-
f"Model {model_name} not found in the data contract. " f"Available models: {model_names}"
|
|
426
|
-
)
|
|
427
|
-
|
|
428
|
-
return to_great_expectations(data_contract, model_name)
|
|
321
|
+
model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
|
|
322
|
+
return to_great_expectations(data_contract, model_name)
|
|
429
323
|
if export_format == "pydantic-model":
|
|
430
324
|
return to_pydantic_model_str(data_contract)
|
|
431
325
|
if export_format == "html":
|
|
432
326
|
return to_html(data_contract)
|
|
327
|
+
if export_format == "go":
|
|
328
|
+
return to_go_types(data_contract)
|
|
329
|
+
if export_format == "bigquery":
|
|
330
|
+
model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
|
|
331
|
+
found_server = data_contract.servers.get(self._server)
|
|
332
|
+
if found_server is None:
|
|
333
|
+
raise RuntimeError(f"Export to {export_format} requires selecting a bigquery server from the data contract.")
|
|
334
|
+
if found_server.type != 'bigquery':
|
|
335
|
+
raise RuntimeError(f"Export to {export_format} requires selecting a bigquery server from the data contract.")
|
|
336
|
+
return to_bigquery_json(model_name, model_value, found_server)
|
|
433
337
|
else:
|
|
434
338
|
print(f"Export format {export_format} not supported.")
|
|
435
339
|
return ""
|
|
@@ -483,8 +387,31 @@ class DataContract:
|
|
|
483
387
|
)
|
|
484
388
|
run.log_info(f"Using {server} for testing the examples")
|
|
485
389
|
return server
|
|
390
|
+
|
|
391
|
+
def _check_models_for_export(self, data_contract: DataContractSpecification, model: str, export_format: str) -> typing.Tuple[str, str]:
|
|
392
|
+
if data_contract.models is None:
|
|
393
|
+
raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
|
|
394
|
+
|
|
395
|
+
model_names = list(data_contract.models.keys())
|
|
486
396
|
|
|
487
|
-
|
|
397
|
+
if model == "all":
|
|
398
|
+
if len(data_contract.models.items()) != 1:
|
|
399
|
+
raise RuntimeError(
|
|
400
|
+
f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
model_name, model_value = next(iter(data_contract.models.items()))
|
|
404
|
+
else:
|
|
405
|
+
model_name = model
|
|
406
|
+
model_value = data_contract.models.get(model_name)
|
|
407
|
+
if model_value is None:
|
|
408
|
+
raise RuntimeError(
|
|
409
|
+
f"Model {model_name} not found in the data contract. Available models: {model_names}"
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
return model_name, model_value
|
|
413
|
+
|
|
414
|
+
def import_from_source(self, format: str, source: typing.Optional[str] = None, bigquery_tables: typing.Optional[typing.List[str]] = None, bigquery_project: typing.Optional[str] = None, bigquery_dataset: typing.Optional[str] = None) -> DataContractSpecification:
|
|
488
415
|
data_contract_specification = DataContract.init()
|
|
489
416
|
|
|
490
417
|
if format == "sql":
|
|
@@ -493,6 +420,13 @@ class DataContract:
|
|
|
493
420
|
data_contract_specification = import_avro(data_contract_specification, source)
|
|
494
421
|
elif format == "glue":
|
|
495
422
|
data_contract_specification = import_glue(data_contract_specification, source)
|
|
423
|
+
elif format == "jsonschema":
|
|
424
|
+
data_contract_specification = import_jsonschema(data_contract_specification, source)
|
|
425
|
+
elif format == "bigquery":
|
|
426
|
+
if source is not None:
|
|
427
|
+
data_contract_specification = import_bigquery_from_json(data_contract_specification, source)
|
|
428
|
+
else:
|
|
429
|
+
data_contract_specification = import_bigquery_from_api(data_contract_specification, bigquery_tables, bigquery_project, bigquery_dataset)
|
|
496
430
|
else:
|
|
497
431
|
print(f"Import format {format} not supported.")
|
|
498
432
|
|
|
@@ -34,6 +34,10 @@ def to_avro_field(field, field_name):
|
|
|
34
34
|
if field.description is not None:
|
|
35
35
|
avro_field["doc"] = field.description
|
|
36
36
|
avro_field["type"] = to_avro_type(field, field_name)
|
|
37
|
+
# add logical type definitions for any of the date type fields
|
|
38
|
+
if field.type in ["timestamp", "timestamp_tz", "timestamp_ntz", "date"]:
|
|
39
|
+
avro_field["logicalType"] = to_avro_logical_type(field.type)
|
|
40
|
+
|
|
37
41
|
return avro_field
|
|
38
42
|
|
|
39
43
|
|
|
@@ -54,9 +58,9 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
|
|
|
54
58
|
elif field.type in ["boolean"]:
|
|
55
59
|
return "boolean"
|
|
56
60
|
elif field.type in ["timestamp", "timestamp_tz"]:
|
|
57
|
-
return "
|
|
61
|
+
return "long"
|
|
58
62
|
elif field.type in ["timestamp_ntz"]:
|
|
59
|
-
return "
|
|
63
|
+
return "long"
|
|
60
64
|
elif field.type in ["date"]:
|
|
61
65
|
return "int"
|
|
62
66
|
elif field.type in ["time"]:
|
|
@@ -72,3 +76,13 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
|
|
|
72
76
|
return "null"
|
|
73
77
|
else:
|
|
74
78
|
return "bytes"
|
|
79
|
+
|
|
80
|
+
def to_avro_logical_type(type: str) -> str:
|
|
81
|
+
if type in ["timestamp", "timestamp_tz"]:
|
|
82
|
+
return "timestamp-millis"
|
|
83
|
+
elif type in ["timestamp_ntz"]:
|
|
84
|
+
return "local-timestamp-millis"
|
|
85
|
+
elif type in ["date"]:
|
|
86
|
+
return "date"
|
|
87
|
+
else:
|
|
88
|
+
return ""
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Dict, List
|
|
4
|
+
|
|
5
|
+
from datacontract.model.data_contract_specification import Model, Field, Server
|
|
6
|
+
from datacontract.model.exceptions import DataContractException
|
|
7
|
+
|
|
8
|
+
def to_bigquery_json(model_name: str, model_value: Model, server: Server) -> str:
|
|
9
|
+
bigquery_table = to_bigquery_schema(model_name, model_value, server)
|
|
10
|
+
return json.dumps(bigquery_table, indent=2)
|
|
11
|
+
|
|
12
|
+
def to_bigquery_schema(model_name: str, model_value: Model, server: Server) -> dict:
|
|
13
|
+
return {
|
|
14
|
+
"kind": "bigquery#table",
|
|
15
|
+
"tableReference": {
|
|
16
|
+
"datasetId": server.dataset,
|
|
17
|
+
"projectId": server.project,
|
|
18
|
+
"tableId": model_name
|
|
19
|
+
},
|
|
20
|
+
"description": model_value.description,
|
|
21
|
+
"schema": {
|
|
22
|
+
"fields": to_fields_array(model_value.fields)
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
def to_fields_array(fields: Dict[str, Field]) -> List[Dict[str, Field]]:
|
|
27
|
+
bq_fields = []
|
|
28
|
+
for field_name, field in fields.items():
|
|
29
|
+
bq_fields.append(to_field(field_name, field))
|
|
30
|
+
|
|
31
|
+
return bq_fields
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def to_field(field_name: str, field: Field) -> dict:
|
|
35
|
+
|
|
36
|
+
bq_type = map_type_to_bigquery(field.type, field_name)
|
|
37
|
+
bq_field = {
|
|
38
|
+
"name": field_name,
|
|
39
|
+
"type": bq_type,
|
|
40
|
+
"mode": "REQUIRED" if field.required else "NULLABLE",
|
|
41
|
+
"description": field.description
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
# handle arrays
|
|
45
|
+
if field.type == 'array':
|
|
46
|
+
bq_field["mode"] = 'REPEATED'
|
|
47
|
+
if field.items.type == 'object':
|
|
48
|
+
# in case the array type is a complex object, we want to copy all its fields
|
|
49
|
+
bq_field["fields"] = to_fields_array(field.items.fields)
|
|
50
|
+
else:
|
|
51
|
+
# otherwise we make up a structure that gets us a single field of the specified type
|
|
52
|
+
bq_field["fields"] = to_fields_array({ f"{field_name}_1": Field(type=field.items.type, required=False, description="")})
|
|
53
|
+
# all of these can carry other fields
|
|
54
|
+
elif bq_type.lower() in ["record", "struct"]:
|
|
55
|
+
bq_field["fields"] = to_fields_array(field.fields)
|
|
56
|
+
|
|
57
|
+
# strings can have a maxlength
|
|
58
|
+
if bq_type.lower() == "string":
|
|
59
|
+
bq_field["maxLength"] = field.maxLength
|
|
60
|
+
|
|
61
|
+
# number types have precision and scale
|
|
62
|
+
if bq_type.lower() in ["numeric", "bignumeric"]:
|
|
63
|
+
bq_field["precision"] = field.precision
|
|
64
|
+
bq_field["scale"] = field.scale
|
|
65
|
+
|
|
66
|
+
return bq_field
|
|
67
|
+
|
|
68
|
+
def map_type_to_bigquery(type_str: str, field_name: str) -> str:
|
|
69
|
+
logger = logging.getLogger(__name__)
|
|
70
|
+
if type_str.lower() in ["string", "varchar", "text"]:
|
|
71
|
+
return "STRING"
|
|
72
|
+
elif type_str == "bytes":
|
|
73
|
+
return "BYTES"
|
|
74
|
+
elif type_str.lower() in ["int", "integer"]:
|
|
75
|
+
return "INTEGER"
|
|
76
|
+
elif type_str.lower() in ["long", "bigint"]:
|
|
77
|
+
return "INT64"
|
|
78
|
+
elif type_str == "float":
|
|
79
|
+
return "FLOAT"
|
|
80
|
+
elif type_str == "boolean":
|
|
81
|
+
return "BOOLEAN"
|
|
82
|
+
elif type_str.lower() in ["timestamp", "timestamp_tz"]:
|
|
83
|
+
return "TIMESTAMP"
|
|
84
|
+
elif type_str == "date":
|
|
85
|
+
return "DATE"
|
|
86
|
+
elif type_str == "timestamp_ntz":
|
|
87
|
+
return "TIME"
|
|
88
|
+
elif type_str.lower() in ["number", "decimal", "numeric"]:
|
|
89
|
+
return "NUMERIC"
|
|
90
|
+
elif type_str == "double":
|
|
91
|
+
return "BIGNUMERIC"
|
|
92
|
+
elif type_str.lower() in ["object", "record", "array"]:
|
|
93
|
+
return "RECORD"
|
|
94
|
+
elif type_str == "struct":
|
|
95
|
+
return "STRUCT"
|
|
96
|
+
elif type_str == "null":
|
|
97
|
+
logger.info(f"Can't properly map {field_name} to bigquery Schema, as 'null' is not supported as a type. Mapping it to STRING.")
|
|
98
|
+
return "STRING"
|
|
99
|
+
else:
|
|
100
|
+
raise DataContractException(
|
|
101
|
+
type="schema",
|
|
102
|
+
result="failed",
|
|
103
|
+
name="Map datacontract type to bigquery data type",
|
|
104
|
+
reason=f"Unsupported type {type_str} in data contract definition.",
|
|
105
|
+
engine="datacontract",
|
|
106
|
+
)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import datacontract.model.data_contract_specification as spec
|
|
2
|
+
from typing import List
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def to_go_types(contract: spec.DataContractSpecification) -> str:
|
|
7
|
+
result = "package main\n\n"
|
|
8
|
+
|
|
9
|
+
for key in contract.models.keys():
|
|
10
|
+
go_types = generate_go_type(contract.models[key], key)
|
|
11
|
+
for go_type in go_types:
|
|
12
|
+
# print(go_type + "\n\n")
|
|
13
|
+
result += f"\n{go_type}\n"
|
|
14
|
+
|
|
15
|
+
return result
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def python_type_to_go_type(py_type) -> str:
|
|
19
|
+
match py_type:
|
|
20
|
+
case "text":
|
|
21
|
+
return "string"
|
|
22
|
+
case "timestamp":
|
|
23
|
+
return "time.Time"
|
|
24
|
+
case "long":
|
|
25
|
+
return "int64"
|
|
26
|
+
case "int":
|
|
27
|
+
return "int"
|
|
28
|
+
case "float":
|
|
29
|
+
return "float64"
|
|
30
|
+
case "boolean":
|
|
31
|
+
return "bool"
|
|
32
|
+
case _:
|
|
33
|
+
return "interface{}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def to_camel_case(snake_str) -> str:
|
|
37
|
+
return "".join(word.capitalize() for word in re.split(r"_|(?<!^)(?=[A-Z])", snake_str))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_subtype(field_info, nested_types, type_name, camel_case_name) -> str:
|
|
41
|
+
go_type = "interface{}"
|
|
42
|
+
if field_info.fields:
|
|
43
|
+
nested_type_name = to_camel_case(f"{type_name}_{camel_case_name}")
|
|
44
|
+
nested_types[nested_type_name] = field_info.fields
|
|
45
|
+
go_type = nested_type_name
|
|
46
|
+
|
|
47
|
+
match field_info.type:
|
|
48
|
+
case "array":
|
|
49
|
+
if field_info.items:
|
|
50
|
+
item_type = get_subtype(field_info.items, nested_types, type_name, camel_case_name + "Item")
|
|
51
|
+
go_type = f"[]{item_type}"
|
|
52
|
+
else:
|
|
53
|
+
go_type = "[]interface{}"
|
|
54
|
+
case "record":
|
|
55
|
+
if field_info.fields:
|
|
56
|
+
nested_type_name = to_camel_case(f"{type_name}_{camel_case_name}")
|
|
57
|
+
nested_types[nested_type_name] = field_info.fields
|
|
58
|
+
go_type = nested_type_name
|
|
59
|
+
else:
|
|
60
|
+
go_type = "interface{}"
|
|
61
|
+
case "object":
|
|
62
|
+
pass
|
|
63
|
+
case _:
|
|
64
|
+
go_type = field_info.type
|
|
65
|
+
|
|
66
|
+
return go_type
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def generate_go_type(model, model_name) -> List[str]:
|
|
70
|
+
go_types = []
|
|
71
|
+
type_name = to_camel_case(model_name)
|
|
72
|
+
lines = [f"type {type_name} struct {{"]
|
|
73
|
+
|
|
74
|
+
nested_types = {}
|
|
75
|
+
|
|
76
|
+
for field_name, field_info in model.fields.items():
|
|
77
|
+
go_type = python_type_to_go_type(field_info.type)
|
|
78
|
+
camel_case_name = to_camel_case(field_name)
|
|
79
|
+
json_tag = field_name if field_info.required else f"{field_name},omitempty"
|
|
80
|
+
avro_tag = field_name
|
|
81
|
+
|
|
82
|
+
if go_type == "interface{}":
|
|
83
|
+
go_type = get_subtype(field_info, nested_types, type_name, camel_case_name)
|
|
84
|
+
|
|
85
|
+
go_type = go_type if field_info.required else f"*{go_type}"
|
|
86
|
+
|
|
87
|
+
lines.append(
|
|
88
|
+
f' {camel_case_name} {go_type} `json:"{json_tag}" avro:"{avro_tag}"` // {field_info.description}'
|
|
89
|
+
)
|
|
90
|
+
lines.append("}")
|
|
91
|
+
go_types.append("\n".join(lines))
|
|
92
|
+
|
|
93
|
+
for nested_type_name, nested_fields in nested_types.items():
|
|
94
|
+
nested_model = spec.Model(fields=nested_fields)
|
|
95
|
+
nested_go_types = generate_go_type(nested_model, nested_type_name)
|
|
96
|
+
go_types.extend(nested_go_types)
|
|
97
|
+
|
|
98
|
+
return go_types
|