datacontract-cli 0.10.2__py3-none-any.whl → 0.10.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of datacontract-cli might be problematic. Click here for more details.

Files changed (33) hide show
  1. datacontract/breaking/breaking.py +12 -0
  2. datacontract/breaking/breaking_rules.py +4 -0
  3. datacontract/catalog/catalog.py +3 -0
  4. datacontract/cli.py +36 -8
  5. datacontract/data_contract.py +62 -128
  6. datacontract/export/avro_converter.py +16 -2
  7. datacontract/export/bigquery_converter.py +106 -0
  8. datacontract/export/go_converter.py +98 -0
  9. datacontract/export/html_export.py +6 -1
  10. datacontract/export/jsonschema_converter.py +45 -5
  11. datacontract/export/sql_converter.py +1 -0
  12. datacontract/export/sql_type_converter.py +42 -1
  13. datacontract/imports/avro_importer.py +14 -1
  14. datacontract/imports/bigquery_importer.py +166 -0
  15. datacontract/imports/jsonschema_importer.py +150 -0
  16. datacontract/model/data_contract_specification.py +55 -1
  17. datacontract/publish/publish.py +32 -0
  18. datacontract/templates/datacontract.html +37 -346
  19. datacontract/templates/index.html +233 -0
  20. datacontract/templates/partials/datacontract_information.html +66 -0
  21. datacontract/templates/partials/datacontract_servicelevels.html +253 -0
  22. datacontract/templates/partials/datacontract_terms.html +44 -0
  23. datacontract/templates/partials/definition.html +99 -0
  24. datacontract/templates/partials/example.html +27 -0
  25. datacontract/templates/partials/model_field.html +97 -0
  26. datacontract/templates/partials/server.html +144 -0
  27. datacontract/templates/style/output.css +94 -13
  28. {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/METADATA +139 -96
  29. {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/RECORD +33 -20
  30. {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/LICENSE +0 -0
  31. {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/WHEEL +0 -0
  32. {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/entry_points.txt +0 -0
  33. {datacontract_cli-0.10.2.dist-info → datacontract_cli-0.10.4.dist-info}/top_level.txt +0 -0
@@ -256,6 +256,18 @@ def field_breaking_changes(
256
256
  )
257
257
  )
258
258
  continue
259
+
260
+ if field_definition_field == "items" and old_field.type == 'array' and new_field.type == 'array':
261
+ results.extend(
262
+ field_breaking_changes(
263
+ old_field=old_value,
264
+ new_field=new_value,
265
+ composition=composition + ['items'],
266
+ new_path=new_path,
267
+ include_severities=include_severities,
268
+ )
269
+ )
270
+ continue
259
271
 
260
272
  rule_name = None
261
273
  description = None
@@ -90,6 +90,10 @@ class BreakingRules:
90
90
  field_tags_removed = Severity.INFO
91
91
  field_tags_updated = Severity.INFO
92
92
 
93
+ field_example_added = Severity.INFO
94
+ field_example_updated = Severity.INFO
95
+ field_example_removed = Severity.INFO
96
+
93
97
  # quality Rules
94
98
  quality_added = Severity.INFO
95
99
  quality_removed = Severity.WARNING
@@ -53,8 +53,10 @@ def create_index_html(contracts, path):
53
53
  )
54
54
 
55
55
  # Load the required template
56
+ # needs to be included in /MANIFEST.in
56
57
  template = env.get_template("index.html")
57
58
 
59
+ # needs to be included in /MANIFEST.in
58
60
  style_content, _, _ = package_loader.get_source(env, "style/output.css")
59
61
 
60
62
  tz = pytz.timezone("UTC")
@@ -69,6 +71,7 @@ def create_index_html(contracts, path):
69
71
  datacontract_cli_version=datacontract_cli_version,
70
72
  contracts=contracts,
71
73
  contracts_size=len(contracts),
74
+ owners=sorted(set(dc.spec.info.owner for dc in contracts if dc.spec.info.owner)),
72
75
  )
73
76
  f.write(html_string)
74
77
  print(f"Created {index_filepath}")
datacontract/cli.py CHANGED
@@ -10,12 +10,14 @@ from rich.console import Console
10
10
  from rich.table import Table
11
11
  from typer.core import TyperGroup
12
12
  from typing_extensions import Annotated
13
+ from typing import List
13
14
 
14
- from datacontract.catalog.catalog import create_index_html, \
15
- create_data_contract_html
15
+ from datacontract.catalog.catalog import create_index_html, create_data_contract_html
16
16
  from datacontract.data_contract import DataContract
17
- from datacontract.init.download_datacontract_file import \
18
- download_datacontract_file, FileExistsException
17
+ from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
18
+
19
+ from datacontract.publish.publish import publish_to_datamesh_manager
20
+
19
21
 
20
22
  console = Console()
21
23
 
@@ -158,12 +160,19 @@ class ExportFormat(str, Enum):
158
160
  sql = "sql"
159
161
  sql_query = "sql-query"
160
162
  html = "html"
163
+ go = "go"
164
+ bigquery = "bigquery"
161
165
 
162
166
 
163
167
  @app.command()
164
168
  def export(
165
169
  format: Annotated[ExportFormat, typer.Option(help="The export format.")],
166
- output: Annotated[Path, typer.Option(help="Specify the file path where the exported data will be saved. If no path is provided, the output will be printed to stdout.")] = None,
170
+ output: Annotated[
171
+ Path,
172
+ typer.Option(
173
+ help="Specify the file path where the exported data will be saved. If no path is provided, the output will be printed to stdout."
174
+ ),
175
+ ] = None,
167
176
  server: Annotated[str, typer.Option(help="The server name to export.")] = None,
168
177
  model: Annotated[
169
178
  str,
@@ -204,7 +213,7 @@ def export(
204
213
  if output is None:
205
214
  console.print(result, markup=False)
206
215
  else:
207
- with output.open('w') as f:
216
+ with output.open("w") as f:
208
217
  f.write(result)
209
218
  console.print(f"Written result to {output}")
210
219
 
@@ -213,20 +222,39 @@ class ImportFormat(str, Enum):
213
222
  sql = "sql"
214
223
  avro = "avro"
215
224
  glue = "glue"
225
+ bigquery = "bigquery"
226
+ jsonschema = "jsonschema"
216
227
 
217
228
 
218
229
  @app.command(name="import")
219
230
  def import_(
220
231
  format: Annotated[ImportFormat, typer.Option(help="The format of the source file.")],
221
- source: Annotated[str, typer.Option(help="The path to the file or Glue Database that should be imported.")],
232
+ source: Annotated[Optional[str], typer.Option(help="The path to the file or Glue Database that should be imported.")] = None,
233
+ bigquery_project: Annotated[Optional[str], typer.Option(help="The bigquery project id.")] = None,
234
+ bigquery_dataset: Annotated[Optional[str], typer.Option(help="The bigquery dataset id.")] = None,
235
+ bigquery_table: Annotated[Optional[List[str]], typer.Option(help="List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all tables in the dataset).")] = None,
222
236
  ):
223
237
  """
224
238
  Create a data contract from the given source location. Prints to stdout.
225
239
  """
226
- result = DataContract().import_from_source(format, source)
240
+ result = DataContract().import_from_source(format, source, bigquery_table, bigquery_project, bigquery_dataset)
227
241
  console.print(result.to_yaml())
228
242
 
229
243
 
244
+ @app.command(name="publish")
245
+ def publish(
246
+ location: Annotated[
247
+ str, typer.Argument(help="The location (url or path) of the data contract yaml.")
248
+ ] = "datacontract.yaml",
249
+ ):
250
+ """
251
+ Publish the data contract to the Data Mesh Manager.
252
+ """
253
+ publish_to_datamesh_manager(
254
+ data_contract=DataContract(data_contract_file=location),
255
+ )
256
+
257
+
230
258
  @app.command(name="catalog")
231
259
  def catalog(
232
260
  files: Annotated[
@@ -6,16 +6,15 @@ import typing
6
6
  import yaml
7
7
  from pyspark.sql import SparkSession
8
8
 
9
- from datacontract.breaking.breaking import models_breaking_changes, \
10
- quality_breaking_changes
9
+ from datacontract.breaking.breaking import models_breaking_changes, quality_breaking_changes
11
10
  from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
12
11
  check_that_datacontract_contains_valid_server_configuration,
13
12
  )
14
- from datacontract.engines.fastjsonschema.check_jsonschema import \
15
- check_jsonschema
13
+ from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
16
14
  from datacontract.engines.soda.check_soda_execute import check_soda_execute
17
15
  from datacontract.export.avro_converter import to_avro_schema_json
18
16
  from datacontract.export.avro_idl_converter import to_avro_idl
17
+ from datacontract.export.bigquery_converter import to_bigquery_json
19
18
  from datacontract.export.dbt_converter import to_dbt_models_yaml, \
20
19
  to_dbt_sources_yaml, to_dbt_staging_sql
21
20
  from datacontract.export.great_expectations_converter import \
@@ -25,13 +24,16 @@ from datacontract.export.jsonschema_converter import to_jsonschema_json
25
24
  from datacontract.export.odcs_converter import to_odcs_yaml
26
25
  from datacontract.export.protobuf_converter import to_protobuf
27
26
  from datacontract.export.pydantic_converter import to_pydantic_model_str
27
+ from datacontract.export.go_converter import to_go_types
28
28
  from datacontract.export.rdf_converter import to_rdf_n3
29
29
  from datacontract.export.sodacl_converter import to_sodacl_yaml
30
30
  from datacontract.export.sql_converter import to_sql_ddl, to_sql_query
31
31
  from datacontract.export.terraform_converter import to_terraform
32
32
  from datacontract.imports.avro_importer import import_avro
33
+ from datacontract.imports.bigquery_importer import import_bigquery_from_api, import_bigquery_from_json
33
34
  from datacontract.imports.glue_importer import import_glue
34
35
  from datacontract.imports.sql_importer import import_sql
36
+ from datacontract.imports.jsonschema_importer import import_jsonschema
35
37
  from datacontract.integration.publish_datamesh_manager import \
36
38
  publish_datamesh_manager
37
39
  from datacontract.integration.publish_opentelemetry import publish_opentelemetry
@@ -39,17 +41,12 @@ from datacontract.lint import resolve
39
41
  from datacontract.lint.linters.description_linter import DescriptionLinter
40
42
  from datacontract.lint.linters.example_model_linter import ExampleModelLinter
41
43
  from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
42
- from datacontract.lint.linters.field_reference_linter import \
43
- FieldReferenceLinter
44
+ from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
44
45
  from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
45
- from datacontract.lint.linters.quality_schema_linter import \
46
- QualityUsesSchemaLinter
47
- from datacontract.lint.linters.valid_constraints_linter import \
48
- ValidFieldConstraintsLinter
49
- from datacontract.model.breaking_change import BreakingChanges, BreakingChange, \
50
- Severity
51
- from datacontract.model.data_contract_specification import \
52
- DataContractSpecification, Server
46
+ from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
47
+ from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
48
+ from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Severity
49
+ from datacontract.model.data_contract_specification import DataContractSpecification, Server
53
50
  from datacontract.model.exceptions import DataContractException
54
51
  from datacontract.model.run import Run, Check
55
52
 
@@ -289,28 +286,8 @@ class DataContract:
289
286
  inline_quality=True,
290
287
  )
291
288
  if export_format == "jsonschema":
292
- if data_contract.models is None:
293
- raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
294
-
295
- model_names = list(data_contract.models.keys())
296
-
297
- if model == "all":
298
- if len(data_contract.models.items()) != 1:
299
- raise RuntimeError(
300
- f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
301
- )
302
-
303
- model_name, model_value = next(iter(data_contract.models.items()))
304
- return to_jsonschema_json(model_name, model_value)
305
- else:
306
- model_name = model
307
- model_value = data_contract.models.get(model_name)
308
- if model_value is None:
309
- raise RuntimeError(
310
- f"Model {model_name} not found in the data contract. Available models: {model_names}"
311
- )
312
-
313
- return to_jsonschema_json(model_name, model_value)
289
+ model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
290
+ return to_jsonschema_json(model_name, model_value)
314
291
  if export_format == "sodacl":
315
292
  return to_sodacl_yaml(data_contract)
316
293
  if export_format == "dbt":
@@ -318,28 +295,8 @@ class DataContract:
318
295
  if export_format == "dbt-sources":
319
296
  return to_dbt_sources_yaml(data_contract, self._server)
320
297
  if export_format == "dbt-staging-sql":
321
- if data_contract.models is None:
322
- raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
323
-
324
- model_names = list(data_contract.models.keys())
325
-
326
- if model == "all":
327
- if len(data_contract.models.items()) != 1:
328
- raise RuntimeError(
329
- f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
330
- )
331
-
332
- model_name, model_value = next(iter(data_contract.models.items()))
333
- return to_dbt_staging_sql(data_contract, model_name, model_value)
334
- else:
335
- model_name = model
336
- model_value = data_contract.models.get(model_name)
337
- if model_value is None:
338
- raise RuntimeError(
339
- f"Model {model_name} not found in the data contract. Available models: {model_names}"
340
- )
341
-
342
- return to_dbt_staging_sql(data_contract, model_name, model_value)
298
+ model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
299
+ return to_dbt_staging_sql(data_contract, model_name, model_value)
343
300
  if export_format == "odcs":
344
301
  return to_odcs_yaml(data_contract)
345
302
  if export_format == "rdf":
@@ -347,28 +304,8 @@ class DataContract:
347
304
  if export_format == "protobuf":
348
305
  return to_protobuf(data_contract)
349
306
  if export_format == "avro":
350
- if data_contract.models is None:
351
- raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
352
-
353
- model_names = list(data_contract.models.keys())
354
-
355
- if model == "all":
356
- if len(data_contract.models.items()) != 1:
357
- raise RuntimeError(
358
- f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
359
- )
360
-
361
- model_name, model_value = next(iter(data_contract.models.items()))
362
- return to_avro_schema_json(model_name, model_value)
363
- else:
364
- model_name = model
365
- model_value = data_contract.models.get(model_name)
366
- if model_value is None:
367
- raise RuntimeError(
368
- f"Model {model_name} not found in the data contract. Available models: {model_names}"
369
- )
370
-
371
- return to_avro_schema_json(model_name, model_value)
307
+ model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
308
+ return to_avro_schema_json(model_name, model_value)
372
309
  if export_format == "avro-idl":
373
310
  return to_avro_idl(data_contract)
374
311
  if export_format == "terraform":
@@ -377,59 +314,26 @@ class DataContract:
377
314
  server_type = self._determine_sql_server_type(data_contract, sql_server_type)
378
315
  return to_sql_ddl(data_contract, server_type=server_type)
379
316
  if export_format == "sql-query":
380
- if data_contract.models is None:
381
- raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
382
-
317
+ model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
383
318
  server_type = self._determine_sql_server_type(data_contract, sql_server_type)
384
-
385
- model_names = list(data_contract.models.keys())
386
-
387
- if model == "all":
388
- if len(data_contract.models.items()) != 1:
389
- raise RuntimeError(
390
- f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
391
- )
392
-
393
- model_name, model_value = next(iter(data_contract.models.items()))
394
- return to_sql_query(data_contract, model_name, model_value, server_type)
395
- else:
396
- model_name = model
397
- model_value = data_contract.models.get(model_name)
398
- if model_value is None:
399
- raise RuntimeError(
400
- f"Model {model_name} not found in the data contract. Available models: {model_names}"
401
- )
402
-
403
- return to_sql_query(data_contract, model_name, model_value, server_type)
404
-
319
+ return to_sql_query(data_contract, model_name, model_value, server_type)
405
320
  if export_format == "great-expectations":
406
- if data_contract.models is None:
407
- raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
408
-
409
- model_names = list(data_contract.models.keys())
410
-
411
- if model == "all":
412
- if len(data_contract.models.items()) != 1:
413
- raise RuntimeError(
414
- f"Export to {export_format} is model specific. Specify the model via --model "
415
- f"$MODEL_NAME. Available models: {model_names}"
416
- )
417
-
418
- model_name, model_value = next(iter(data_contract.models.items()))
419
- return to_great_expectations(data_contract, model_name)
420
- else:
421
- model_name = model
422
- model_value = data_contract.models.get(model_name)
423
- if model_value is None:
424
- raise RuntimeError(
425
- f"Model {model_name} not found in the data contract. " f"Available models: {model_names}"
426
- )
427
-
428
- return to_great_expectations(data_contract, model_name)
321
+ model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
322
+ return to_great_expectations(data_contract, model_name)
429
323
  if export_format == "pydantic-model":
430
324
  return to_pydantic_model_str(data_contract)
431
325
  if export_format == "html":
432
326
  return to_html(data_contract)
327
+ if export_format == "go":
328
+ return to_go_types(data_contract)
329
+ if export_format == "bigquery":
330
+ model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
331
+ found_server = data_contract.servers.get(self._server)
332
+ if found_server is None:
333
+ raise RuntimeError(f"Export to {export_format} requires selecting a bigquery server from the data contract.")
334
+ if found_server.type != 'bigquery':
335
+ raise RuntimeError(f"Export to {export_format} requires selecting a bigquery server from the data contract.")
336
+ return to_bigquery_json(model_name, model_value, found_server)
433
337
  else:
434
338
  print(f"Export format {export_format} not supported.")
435
339
  return ""
@@ -483,8 +387,31 @@ class DataContract:
483
387
  )
484
388
  run.log_info(f"Using {server} for testing the examples")
485
389
  return server
390
+
391
+ def _check_models_for_export(self, data_contract: DataContractSpecification, model: str, export_format: str) -> typing.Tuple[str, str]:
392
+ if data_contract.models is None:
393
+ raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
394
+
395
+ model_names = list(data_contract.models.keys())
486
396
 
487
- def import_from_source(self, format: str, source: str) -> DataContractSpecification:
397
+ if model == "all":
398
+ if len(data_contract.models.items()) != 1:
399
+ raise RuntimeError(
400
+ f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
401
+ )
402
+
403
+ model_name, model_value = next(iter(data_contract.models.items()))
404
+ else:
405
+ model_name = model
406
+ model_value = data_contract.models.get(model_name)
407
+ if model_value is None:
408
+ raise RuntimeError(
409
+ f"Model {model_name} not found in the data contract. Available models: {model_names}"
410
+ )
411
+
412
+ return model_name, model_value
413
+
414
+ def import_from_source(self, format: str, source: typing.Optional[str] = None, bigquery_tables: typing.Optional[typing.List[str]] = None, bigquery_project: typing.Optional[str] = None, bigquery_dataset: typing.Optional[str] = None) -> DataContractSpecification:
488
415
  data_contract_specification = DataContract.init()
489
416
 
490
417
  if format == "sql":
@@ -493,6 +420,13 @@ class DataContract:
493
420
  data_contract_specification = import_avro(data_contract_specification, source)
494
421
  elif format == "glue":
495
422
  data_contract_specification = import_glue(data_contract_specification, source)
423
+ elif format == "jsonschema":
424
+ data_contract_specification = import_jsonschema(data_contract_specification, source)
425
+ elif format == "bigquery":
426
+ if source is not None:
427
+ data_contract_specification = import_bigquery_from_json(data_contract_specification, source)
428
+ else:
429
+ data_contract_specification = import_bigquery_from_api(data_contract_specification, bigquery_tables, bigquery_project, bigquery_dataset)
496
430
  else:
497
431
  print(f"Import format {format} not supported.")
498
432
 
@@ -34,6 +34,10 @@ def to_avro_field(field, field_name):
34
34
  if field.description is not None:
35
35
  avro_field["doc"] = field.description
36
36
  avro_field["type"] = to_avro_type(field, field_name)
37
+ # add logical type definitions for any of the date type fields
38
+ if field.type in ["timestamp", "timestamp_tz", "timestamp_ntz", "date"]:
39
+ avro_field["logicalType"] = to_avro_logical_type(field.type)
40
+
37
41
  return avro_field
38
42
 
39
43
 
@@ -54,9 +58,9 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
54
58
  elif field.type in ["boolean"]:
55
59
  return "boolean"
56
60
  elif field.type in ["timestamp", "timestamp_tz"]:
57
- return "string"
61
+ return "long"
58
62
  elif field.type in ["timestamp_ntz"]:
59
- return "string"
63
+ return "long"
60
64
  elif field.type in ["date"]:
61
65
  return "int"
62
66
  elif field.type in ["time"]:
@@ -72,3 +76,13 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
72
76
  return "null"
73
77
  else:
74
78
  return "bytes"
79
+
80
+ def to_avro_logical_type(type: str) -> str:
81
+ if type in ["timestamp", "timestamp_tz"]:
82
+ return "timestamp-millis"
83
+ elif type in ["timestamp_ntz"]:
84
+ return "local-timestamp-millis"
85
+ elif type in ["date"]:
86
+ return "date"
87
+ else:
88
+ return ""
@@ -0,0 +1,106 @@
1
+ import json
2
+ import logging
3
+ from typing import Dict, List
4
+
5
+ from datacontract.model.data_contract_specification import Model, Field, Server
6
+ from datacontract.model.exceptions import DataContractException
7
+
8
+ def to_bigquery_json(model_name: str, model_value: Model, server: Server) -> str:
9
+ bigquery_table = to_bigquery_schema(model_name, model_value, server)
10
+ return json.dumps(bigquery_table, indent=2)
11
+
12
+ def to_bigquery_schema(model_name: str, model_value: Model, server: Server) -> dict:
13
+ return {
14
+ "kind": "bigquery#table",
15
+ "tableReference": {
16
+ "datasetId": server.dataset,
17
+ "projectId": server.project,
18
+ "tableId": model_name
19
+ },
20
+ "description": model_value.description,
21
+ "schema": {
22
+ "fields": to_fields_array(model_value.fields)
23
+ }
24
+ }
25
+
26
+ def to_fields_array(fields: Dict[str, Field]) -> List[Dict[str, Field]]:
27
+ bq_fields = []
28
+ for field_name, field in fields.items():
29
+ bq_fields.append(to_field(field_name, field))
30
+
31
+ return bq_fields
32
+
33
+
34
+ def to_field(field_name: str, field: Field) -> dict:
35
+
36
+ bq_type = map_type_to_bigquery(field.type, field_name)
37
+ bq_field = {
38
+ "name": field_name,
39
+ "type": bq_type,
40
+ "mode": "REQUIRED" if field.required else "NULLABLE",
41
+ "description": field.description
42
+ }
43
+
44
+ # handle arrays
45
+ if field.type == 'array':
46
+ bq_field["mode"] = 'REPEATED'
47
+ if field.items.type == 'object':
48
+ # in case the array type is a complex object, we want to copy all its fields
49
+ bq_field["fields"] = to_fields_array(field.items.fields)
50
+ else:
51
+ # otherwise we make up a structure that gets us a single field of the specified type
52
+ bq_field["fields"] = to_fields_array({ f"{field_name}_1": Field(type=field.items.type, required=False, description="")})
53
+ # all of these can carry other fields
54
+ elif bq_type.lower() in ["record", "struct"]:
55
+ bq_field["fields"] = to_fields_array(field.fields)
56
+
57
+ # strings can have a maxlength
58
+ if bq_type.lower() == "string":
59
+ bq_field["maxLength"] = field.maxLength
60
+
61
+ # number types have precision and scale
62
+ if bq_type.lower() in ["numeric", "bignumeric"]:
63
+ bq_field["precision"] = field.precision
64
+ bq_field["scale"] = field.scale
65
+
66
+ return bq_field
67
+
68
+ def map_type_to_bigquery(type_str: str, field_name: str) -> str:
69
+ logger = logging.getLogger(__name__)
70
+ if type_str.lower() in ["string", "varchar", "text"]:
71
+ return "STRING"
72
+ elif type_str == "bytes":
73
+ return "BYTES"
74
+ elif type_str.lower() in ["int", "integer"]:
75
+ return "INTEGER"
76
+ elif type_str.lower() in ["long", "bigint"]:
77
+ return "INT64"
78
+ elif type_str == "float":
79
+ return "FLOAT"
80
+ elif type_str == "boolean":
81
+ return "BOOLEAN"
82
+ elif type_str.lower() in ["timestamp", "timestamp_tz"]:
83
+ return "TIMESTAMP"
84
+ elif type_str == "date":
85
+ return "DATE"
86
+ elif type_str == "timestamp_ntz":
87
+ return "TIME"
88
+ elif type_str.lower() in ["number", "decimal", "numeric"]:
89
+ return "NUMERIC"
90
+ elif type_str == "double":
91
+ return "BIGNUMERIC"
92
+ elif type_str.lower() in ["object", "record", "array"]:
93
+ return "RECORD"
94
+ elif type_str == "struct":
95
+ return "STRUCT"
96
+ elif type_str == "null":
97
+ logger.info(f"Can't properly map {field_name} to bigquery Schema, as 'null' is not supported as a type. Mapping it to STRING.")
98
+ return "STRING"
99
+ else:
100
+ raise DataContractException(
101
+ type="schema",
102
+ result="failed",
103
+ name="Map datacontract type to bigquery data type",
104
+ reason=f"Unsupported type {type_str} in data contract definition.",
105
+ engine="datacontract",
106
+ )
@@ -0,0 +1,98 @@
1
+ import datacontract.model.data_contract_specification as spec
2
+ from typing import List
3
+ import re
4
+
5
+
6
+ def to_go_types(contract: spec.DataContractSpecification) -> str:
7
+ result = "package main\n\n"
8
+
9
+ for key in contract.models.keys():
10
+ go_types = generate_go_type(contract.models[key], key)
11
+ for go_type in go_types:
12
+ # print(go_type + "\n\n")
13
+ result += f"\n{go_type}\n"
14
+
15
+ return result
16
+
17
+
18
+ def python_type_to_go_type(py_type) -> str:
19
+ match py_type:
20
+ case "text":
21
+ return "string"
22
+ case "timestamp":
23
+ return "time.Time"
24
+ case "long":
25
+ return "int64"
26
+ case "int":
27
+ return "int"
28
+ case "float":
29
+ return "float64"
30
+ case "boolean":
31
+ return "bool"
32
+ case _:
33
+ return "interface{}"
34
+
35
+
36
+ def to_camel_case(snake_str) -> str:
37
+ return "".join(word.capitalize() for word in re.split(r"_|(?<!^)(?=[A-Z])", snake_str))
38
+
39
+
40
+ def get_subtype(field_info, nested_types, type_name, camel_case_name) -> str:
41
+ go_type = "interface{}"
42
+ if field_info.fields:
43
+ nested_type_name = to_camel_case(f"{type_name}_{camel_case_name}")
44
+ nested_types[nested_type_name] = field_info.fields
45
+ go_type = nested_type_name
46
+
47
+ match field_info.type:
48
+ case "array":
49
+ if field_info.items:
50
+ item_type = get_subtype(field_info.items, nested_types, type_name, camel_case_name + "Item")
51
+ go_type = f"[]{item_type}"
52
+ else:
53
+ go_type = "[]interface{}"
54
+ case "record":
55
+ if field_info.fields:
56
+ nested_type_name = to_camel_case(f"{type_name}_{camel_case_name}")
57
+ nested_types[nested_type_name] = field_info.fields
58
+ go_type = nested_type_name
59
+ else:
60
+ go_type = "interface{}"
61
+ case "object":
62
+ pass
63
+ case _:
64
+ go_type = field_info.type
65
+
66
+ return go_type
67
+
68
+
69
+ def generate_go_type(model, model_name) -> List[str]:
70
+ go_types = []
71
+ type_name = to_camel_case(model_name)
72
+ lines = [f"type {type_name} struct {{"]
73
+
74
+ nested_types = {}
75
+
76
+ for field_name, field_info in model.fields.items():
77
+ go_type = python_type_to_go_type(field_info.type)
78
+ camel_case_name = to_camel_case(field_name)
79
+ json_tag = field_name if field_info.required else f"{field_name},omitempty"
80
+ avro_tag = field_name
81
+
82
+ if go_type == "interface{}":
83
+ go_type = get_subtype(field_info, nested_types, type_name, camel_case_name)
84
+
85
+ go_type = go_type if field_info.required else f"*{go_type}"
86
+
87
+ lines.append(
88
+ f' {camel_case_name} {go_type} `json:"{json_tag}" avro:"{avro_tag}"` // {field_info.description}'
89
+ )
90
+ lines.append("}")
91
+ go_types.append("\n".join(lines))
92
+
93
+ for nested_type_name, nested_fields in nested_types.items():
94
+ nested_model = spec.Model(fields=nested_fields)
95
+ nested_go_types = generate_go_type(nested_model, nested_type_name)
96
+ go_types.extend(nested_go_types)
97
+
98
+ return go_types