datacontract-cli 0.10.24__py3-none-any.whl → 0.10.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacontract-cli might be problematic. Click here for more details.
- datacontract/api.py +3 -3
- datacontract/cli.py +1 -1
- datacontract/engines/soda/connections/kafka.py +2 -1
- datacontract/export/great_expectations_converter.py +49 -2
- datacontract/export/odcs_v3_exporter.py +183 -140
- datacontract/export/spark_converter.py +1 -1
- datacontract/export/sql_converter.py +4 -0
- datacontract/export/sql_type_converter.py +2 -0
- datacontract/imports/avro_importer.py +23 -23
- datacontract/imports/csv_importer.py +2 -2
- datacontract/imports/excel_importer.py +850 -0
- datacontract/imports/importer.py +4 -2
- datacontract/imports/importer_factory.py +5 -0
- datacontract/imports/odcs_v3_importer.py +202 -145
- datacontract/imports/protobuf_importer.py +0 -2
- datacontract/imports/spark_importer.py +2 -0
- datacontract/lint/linters/description_linter.py +1 -3
- datacontract/lint/linters/field_reference_linter.py +1 -2
- datacontract/lint/linters/notice_period_linter.py +2 -2
- datacontract/lint/linters/valid_constraints_linter.py +3 -3
- datacontract/model/data_contract_specification/__init__.py +1 -0
- {datacontract_cli-0.10.24.dist-info → datacontract_cli-0.10.26.dist-info}/METADATA +59 -18
- {datacontract_cli-0.10.24.dist-info → datacontract_cli-0.10.26.dist-info}/RECORD +27 -26
- {datacontract_cli-0.10.24.dist-info → datacontract_cli-0.10.26.dist-info}/WHEEL +1 -1
- datacontract/model/data_contract_specification.py +0 -327
- {datacontract_cli-0.10.24.dist-info → datacontract_cli-0.10.26.dist-info}/entry_points.txt +0 -0
- {datacontract_cli-0.10.24.dist-info → datacontract_cli-0.10.26.dist-info}/licenses/LICENSE +0 -0
- {datacontract_cli-0.10.24.dist-info → datacontract_cli-0.10.26.dist-info}/top_level.txt +0 -0
|
@@ -55,7 +55,7 @@ def import_avro(data_contract_specification: DataContractSpecification, source:
|
|
|
55
55
|
engine="datacontract",
|
|
56
56
|
original_exception=e,
|
|
57
57
|
)
|
|
58
|
-
|
|
58
|
+
# type record is being used for both the table and the object types in data contract
|
|
59
59
|
# -> CONSTRAINT: one table per .avsc input, all nested records are interpreted as objects
|
|
60
60
|
fields = import_record_fields(avro_schema.fields)
|
|
61
61
|
|
|
@@ -92,19 +92,19 @@ def handle_config_avro_custom_properties(field: avro.schema.Field, imported_fiel
|
|
|
92
92
|
|
|
93
93
|
|
|
94
94
|
LOGICAL_TYPE_MAPPING = {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
95
|
+
"decimal": "decimal",
|
|
96
|
+
"date": "date",
|
|
97
|
+
"time-millis": "time",
|
|
98
|
+
"time-micros": "time",
|
|
99
|
+
"timestamp-millis": "timestamp_tz",
|
|
100
|
+
"timestamp-micros": "timestamp_tz",
|
|
101
|
+
"local-timestamp-micros": "timestamp_ntz",
|
|
102
|
+
"local-timestamp-millis": "timestamp_ntz",
|
|
103
|
+
"duration": "string",
|
|
104
|
+
"uuid": "string",
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
108
|
def import_record_fields(record_fields: List[avro.schema.Field]) -> Dict[str, Field]:
|
|
109
109
|
"""
|
|
110
110
|
Import Avro record fields and convert them to data contract fields.
|
|
@@ -150,15 +150,15 @@ def import_record_fields(record_fields: List[avro.schema.Field]) -> Dict[str, Fi
|
|
|
150
150
|
if not imported_field.config:
|
|
151
151
|
imported_field.config = {}
|
|
152
152
|
imported_field.config["avroType"] = "enum"
|
|
153
|
-
else:
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
153
|
+
else:
|
|
154
|
+
logical_type = field.type.get_prop("logicalType")
|
|
155
|
+
if logical_type in LOGICAL_TYPE_MAPPING:
|
|
156
|
+
imported_field.type = LOGICAL_TYPE_MAPPING[logical_type]
|
|
157
|
+
if logical_type == "decimal":
|
|
158
|
+
imported_field.precision = field.type.precision
|
|
159
|
+
imported_field.scale = field.type.scale
|
|
160
|
+
else:
|
|
161
|
+
imported_field.type = map_type_from_avro(field.type.type)
|
|
162
162
|
imported_fields[field.name] = imported_field
|
|
163
163
|
|
|
164
164
|
return imported_fields
|
|
@@ -31,10 +31,10 @@ def import_csv(
|
|
|
31
31
|
if data_contract_specification.servers is None:
|
|
32
32
|
data_contract_specification.servers = {}
|
|
33
33
|
|
|
34
|
-
delimiter = None if dialect is None else dialect[
|
|
34
|
+
delimiter = None if dialect is None else dialect["Delimiter"][0]
|
|
35
35
|
|
|
36
36
|
if dialect is not None:
|
|
37
|
-
dc_types = [map_type_from_duckdb(x["type"]) for x in dialect[
|
|
37
|
+
dc_types = [map_type_from_duckdb(x["type"]) for x in dialect["Columns"][0]]
|
|
38
38
|
else:
|
|
39
39
|
dc_types = [map_type_from_duckdb(str(x)) for x in tbl.dtypes]
|
|
40
40
|
|