structurize 3.0.1__py3-none-any.whl → 3.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- avrotize/_version.py +3 -3
- avrotize/avrotoiceberg.py +111 -13
- avrotize/commands.json +20 -2
- avrotize/structuretodb.py +1 -1
- avrotize/structuretoiceberg.py +113 -13
- {structurize-3.0.1.dist-info → structurize-3.0.2.dist-info}/METADATA +1 -1
- {structurize-3.0.1.dist-info → structurize-3.0.2.dist-info}/RECORD +11 -11
- {structurize-3.0.1.dist-info → structurize-3.0.2.dist-info}/WHEEL +0 -0
- {structurize-3.0.1.dist-info → structurize-3.0.2.dist-info}/entry_points.txt +0 -0
- {structurize-3.0.1.dist-info → structurize-3.0.2.dist-info}/licenses/LICENSE +0 -0
- {structurize-3.0.1.dist-info → structurize-3.0.2.dist-info}/top_level.txt +0 -0
avrotize/_version.py
CHANGED
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '3.0.
|
|
32
|
-
__version_tuple__ = version_tuple = (3, 0,
|
|
31
|
+
__version__ = version = '3.0.2'
|
|
32
|
+
__version_tuple__ = version_tuple = (3, 0, 2)
|
|
33
33
|
|
|
34
|
-
__commit_id__ = commit_id = '
|
|
34
|
+
__commit_id__ = commit_id = 'g3ee5f8f67'
|
avrotize/avrotoiceberg.py
CHANGED
|
@@ -5,6 +5,7 @@ import sys
|
|
|
5
5
|
from typing import Dict, List
|
|
6
6
|
import pyarrow as pa
|
|
7
7
|
from pyiceberg.schema import Schema, NestedField
|
|
8
|
+
from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
|
|
8
9
|
from pyiceberg.types import (
|
|
9
10
|
BooleanType,
|
|
10
11
|
IntegerType,
|
|
@@ -21,11 +22,74 @@ from pyiceberg.types import (
|
|
|
21
22
|
MapType,
|
|
22
23
|
StructType
|
|
23
24
|
)
|
|
24
|
-
from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
|
|
25
25
|
|
|
26
26
|
JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
|
|
27
27
|
|
|
28
28
|
|
|
29
|
+
def iceberg_type_to_json(iceberg_type) -> str | Dict:
|
|
30
|
+
"""
|
|
31
|
+
Serialize an Iceberg type to JSON per Iceberg Table Spec Appendix C.
|
|
32
|
+
|
|
33
|
+
Primitive types are serialized as strings. Complex types (struct, list, map)
|
|
34
|
+
are serialized as JSON objects with their nested structure.
|
|
35
|
+
"""
|
|
36
|
+
# Primitive types map to simple strings
|
|
37
|
+
if isinstance(iceberg_type, BooleanType):
|
|
38
|
+
return "boolean"
|
|
39
|
+
elif isinstance(iceberg_type, IntegerType):
|
|
40
|
+
return "int"
|
|
41
|
+
elif isinstance(iceberg_type, LongType):
|
|
42
|
+
return "long"
|
|
43
|
+
elif isinstance(iceberg_type, FloatType):
|
|
44
|
+
return "float"
|
|
45
|
+
elif isinstance(iceberg_type, DoubleType):
|
|
46
|
+
return "double"
|
|
47
|
+
elif isinstance(iceberg_type, StringType):
|
|
48
|
+
return "string"
|
|
49
|
+
elif isinstance(iceberg_type, BinaryType):
|
|
50
|
+
return "binary"
|
|
51
|
+
elif isinstance(iceberg_type, DateType):
|
|
52
|
+
return "date"
|
|
53
|
+
elif isinstance(iceberg_type, TimestampType):
|
|
54
|
+
return "timestamp"
|
|
55
|
+
elif isinstance(iceberg_type, DecimalType):
|
|
56
|
+
return f"decimal({iceberg_type.precision},{iceberg_type.scale})"
|
|
57
|
+
elif isinstance(iceberg_type, FixedType):
|
|
58
|
+
return f"fixed[{iceberg_type.length}]"
|
|
59
|
+
elif isinstance(iceberg_type, ListType):
|
|
60
|
+
return {
|
|
61
|
+
"type": "list",
|
|
62
|
+
"element-id": iceberg_type.element_id,
|
|
63
|
+
"element-required": iceberg_type.element_required,
|
|
64
|
+
"element": iceberg_type_to_json(iceberg_type.element_type)
|
|
65
|
+
}
|
|
66
|
+
elif isinstance(iceberg_type, MapType):
|
|
67
|
+
return {
|
|
68
|
+
"type": "map",
|
|
69
|
+
"key-id": iceberg_type.key_id,
|
|
70
|
+
"key": iceberg_type_to_json(iceberg_type.key_type),
|
|
71
|
+
"value-id": iceberg_type.value_id,
|
|
72
|
+
"value-required": iceberg_type.value_required,
|
|
73
|
+
"value": iceberg_type_to_json(iceberg_type.value_type)
|
|
74
|
+
}
|
|
75
|
+
elif isinstance(iceberg_type, StructType):
|
|
76
|
+
return {
|
|
77
|
+
"type": "struct",
|
|
78
|
+
"fields": [
|
|
79
|
+
{
|
|
80
|
+
"id": field.field_id,
|
|
81
|
+
"name": field.name,
|
|
82
|
+
"required": field.required,
|
|
83
|
+
"type": iceberg_type_to_json(field.field_type)
|
|
84
|
+
}
|
|
85
|
+
for field in iceberg_type.fields
|
|
86
|
+
]
|
|
87
|
+
}
|
|
88
|
+
else:
|
|
89
|
+
# Fallback for unknown types
|
|
90
|
+
return str(iceberg_type)
|
|
91
|
+
|
|
92
|
+
|
|
29
93
|
class AvroToIcebergConverter:
|
|
30
94
|
"""Class to convert Avro schema to Iceberg schema."""
|
|
31
95
|
|
|
@@ -42,8 +106,16 @@ class AvroToIcebergConverter:
|
|
|
42
106
|
"""Get the full name of a record type."""
|
|
43
107
|
return f"{namespace}.{name}" if namespace else name
|
|
44
108
|
|
|
45
|
-
def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False):
|
|
46
|
-
"""Convert an Avro schema to an Iceberg schema.
|
|
109
|
+
def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False, output_format: str="arrow"):
|
|
110
|
+
"""Convert an Avro schema to an Iceberg schema.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
avro_schema_path: Path to the Avro schema file
|
|
114
|
+
avro_record_type: Record type to convert (or None for the root)
|
|
115
|
+
output_path: Path to write the Iceberg schema
|
|
116
|
+
emit_cloudevents_columns: Whether to add CloudEvents columns
|
|
117
|
+
output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
|
|
118
|
+
"""
|
|
47
119
|
schema_file = avro_schema_path
|
|
48
120
|
if not schema_file:
|
|
49
121
|
print("Please specify the avro schema file")
|
|
@@ -96,14 +168,32 @@ class AvroToIcebergConverter:
|
|
|
96
168
|
])
|
|
97
169
|
|
|
98
170
|
iceberg_schema = Schema(*iceberg_fields)
|
|
99
|
-
|
|
100
|
-
print(f"Iceberg schema created: {arrow_schema}")
|
|
171
|
+
print(f"Iceberg schema created: {iceberg_schema}")
|
|
101
172
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
173
|
+
if output_format == "arrow":
|
|
174
|
+
# Write as binary PyArrow schema
|
|
175
|
+
arrow_schema = schema_to_pyarrow(iceberg_schema)
|
|
176
|
+
file_io = PyArrowFileIO()
|
|
177
|
+
output_file = file_io.new_output("file://" + output_path)
|
|
178
|
+
with output_file.create(overwrite=True) as f:
|
|
179
|
+
pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
|
|
180
|
+
else:
|
|
181
|
+
# Write Iceberg schema as spec-compliant JSON (per Iceberg Table Spec Appendix C)
|
|
182
|
+
schema_json = {
|
|
183
|
+
"type": "struct",
|
|
184
|
+
"schema-id": 0,
|
|
185
|
+
"fields": [
|
|
186
|
+
{
|
|
187
|
+
"id": field.field_id,
|
|
188
|
+
"name": field.name,
|
|
189
|
+
"required": field.required,
|
|
190
|
+
"type": iceberg_type_to_json(field.field_type)
|
|
191
|
+
}
|
|
192
|
+
for field in iceberg_schema.fields
|
|
193
|
+
]
|
|
194
|
+
}
|
|
195
|
+
with open(output_path, "w", encoding="utf-8") as f:
|
|
196
|
+
json.dump(schema_json, f, indent=2)
|
|
107
197
|
|
|
108
198
|
def convert_avro_type_to_iceberg_type(self, avro_type):
|
|
109
199
|
"""Convert an Avro type to an Iceberg type."""
|
|
@@ -203,8 +293,16 @@ class AvroToIcebergConverter:
|
|
|
203
293
|
return StringType()
|
|
204
294
|
|
|
205
295
|
|
|
206
|
-
def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False):
|
|
207
|
-
"""Convert an Avro schema to an Iceberg schema.
|
|
296
|
+
def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False, output_format="arrow"):
|
|
297
|
+
"""Convert an Avro schema to an Iceberg schema.
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
avro_schema_path: Path to the Avro schema file
|
|
301
|
+
avro_record_type: Record type to convert (or None for the root)
|
|
302
|
+
output_path: Path to write the Iceberg schema
|
|
303
|
+
emit_cloudevents_columns: Whether to add CloudEvents columns
|
|
304
|
+
output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
|
|
305
|
+
"""
|
|
208
306
|
converter = AvroToIcebergConverter()
|
|
209
307
|
converter.convert_avro_to_iceberg(
|
|
210
|
-
avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns)
|
|
308
|
+
avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns, output_format)
|
avrotize/commands.json
CHANGED
|
@@ -1115,7 +1115,8 @@
|
|
|
1115
1115
|
"avro_schema_path": "input_file_path",
|
|
1116
1116
|
"output_path": "output_file_path",
|
|
1117
1117
|
"avro_record_type": "args.record_type",
|
|
1118
|
-
"emit_cloudevents_columns": "args.emit_cloudevents_columns"
|
|
1118
|
+
"emit_cloudevents_columns": "args.emit_cloudevents_columns",
|
|
1119
|
+
"output_format": "args.format"
|
|
1119
1120
|
}
|
|
1120
1121
|
},
|
|
1121
1122
|
"extensions": [
|
|
@@ -1147,6 +1148,14 @@
|
|
|
1147
1148
|
"help": "Add CloudEvents columns to the Iceberg schema",
|
|
1148
1149
|
"default": false,
|
|
1149
1150
|
"required": false
|
|
1151
|
+
},
|
|
1152
|
+
{
|
|
1153
|
+
"name": "--format",
|
|
1154
|
+
"type": "str",
|
|
1155
|
+
"help": "Output format: 'arrow' for binary Arrow IPC (default), 'schema' for JSON",
|
|
1156
|
+
"choices": ["schema", "arrow"],
|
|
1157
|
+
"default": "arrow",
|
|
1158
|
+
"required": false
|
|
1150
1159
|
}
|
|
1151
1160
|
],
|
|
1152
1161
|
"suggested_output_file_path": "{input_file_name}.iceberg",
|
|
@@ -1169,7 +1178,8 @@
|
|
|
1169
1178
|
"structure_schema_path": "input_file_path",
|
|
1170
1179
|
"output_path": "output_file_path",
|
|
1171
1180
|
"structure_record_type": "args.record_type",
|
|
1172
|
-
"emit_cloudevents_columns": "args.emit_cloudevents_columns"
|
|
1181
|
+
"emit_cloudevents_columns": "args.emit_cloudevents_columns",
|
|
1182
|
+
"output_format": "args.format"
|
|
1173
1183
|
}
|
|
1174
1184
|
},
|
|
1175
1185
|
"extensions": [
|
|
@@ -1202,6 +1212,14 @@
|
|
|
1202
1212
|
"help": "Add CloudEvents columns to the Iceberg schema",
|
|
1203
1213
|
"default": false,
|
|
1204
1214
|
"required": false
|
|
1215
|
+
},
|
|
1216
|
+
{
|
|
1217
|
+
"name": "--format",
|
|
1218
|
+
"type": "str",
|
|
1219
|
+
"help": "Output format: 'arrow' for binary Arrow IPC (default), 'schema' for JSON",
|
|
1220
|
+
"choices": ["schema", "arrow"],
|
|
1221
|
+
"default": "arrow",
|
|
1222
|
+
"required": false
|
|
1205
1223
|
}
|
|
1206
1224
|
],
|
|
1207
1225
|
"suggested_output_file_path": "{input_file_name}.iceberg",
|
avrotize/structuretodb.py
CHANGED
|
@@ -426,7 +426,7 @@ def structure_type_to_sql_type(structure_type: Any, dialect: str) -> str:
|
|
|
426
426
|
type_map["sqlanywhere"] = type_map["sqlserver"].copy()
|
|
427
427
|
type_map["bigquery"] = {k: v.replace("VARCHAR", "STRING").replace("JSONB", "STRING").replace("BYTEA", "BYTES") for k, v in type_map["postgres"].items()}
|
|
428
428
|
type_map["snowflake"] = {k: v.replace("JSONB", "VARIANT").replace("BYTEA", "BINARY") for k, v in type_map["postgres"].items()}
|
|
429
|
-
type_map["redshift"] = type_map["postgres"].
|
|
429
|
+
type_map["redshift"] = {k: v.replace("JSONB", "SUPER").replace("BYTEA", "VARBYTE") for k, v in type_map["postgres"].items()}
|
|
430
430
|
|
|
431
431
|
# Handle type resolution
|
|
432
432
|
if isinstance(structure_type, str):
|
avrotize/structuretoiceberg.py
CHANGED
|
@@ -5,6 +5,7 @@ import sys
|
|
|
5
5
|
from typing import Dict, List, Any, Optional
|
|
6
6
|
import pyarrow as pa
|
|
7
7
|
from pyiceberg.schema import Schema, NestedField
|
|
8
|
+
from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
|
|
8
9
|
from pyiceberg.types import (
|
|
9
10
|
BooleanType,
|
|
10
11
|
IntegerType,
|
|
@@ -22,11 +23,76 @@ from pyiceberg.types import (
|
|
|
22
23
|
StructType,
|
|
23
24
|
TimeType
|
|
24
25
|
)
|
|
25
|
-
from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
|
|
26
26
|
|
|
27
27
|
JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
def iceberg_type_to_json(iceberg_type) -> str | Dict:
|
|
31
|
+
"""
|
|
32
|
+
Serialize an Iceberg type to JSON per Iceberg Table Spec Appendix C.
|
|
33
|
+
|
|
34
|
+
Primitive types are serialized as strings. Complex types (struct, list, map)
|
|
35
|
+
are serialized as JSON objects with their nested structure.
|
|
36
|
+
"""
|
|
37
|
+
# Primitive types map to simple strings
|
|
38
|
+
if isinstance(iceberg_type, BooleanType):
|
|
39
|
+
return "boolean"
|
|
40
|
+
elif isinstance(iceberg_type, IntegerType):
|
|
41
|
+
return "int"
|
|
42
|
+
elif isinstance(iceberg_type, LongType):
|
|
43
|
+
return "long"
|
|
44
|
+
elif isinstance(iceberg_type, FloatType):
|
|
45
|
+
return "float"
|
|
46
|
+
elif isinstance(iceberg_type, DoubleType):
|
|
47
|
+
return "double"
|
|
48
|
+
elif isinstance(iceberg_type, StringType):
|
|
49
|
+
return "string"
|
|
50
|
+
elif isinstance(iceberg_type, BinaryType):
|
|
51
|
+
return "binary"
|
|
52
|
+
elif isinstance(iceberg_type, DateType):
|
|
53
|
+
return "date"
|
|
54
|
+
elif isinstance(iceberg_type, TimeType):
|
|
55
|
+
return "time"
|
|
56
|
+
elif isinstance(iceberg_type, TimestampType):
|
|
57
|
+
return "timestamp"
|
|
58
|
+
elif isinstance(iceberg_type, DecimalType):
|
|
59
|
+
return f"decimal({iceberg_type.precision},{iceberg_type.scale})"
|
|
60
|
+
elif isinstance(iceberg_type, FixedType):
|
|
61
|
+
return f"fixed[{iceberg_type.length}]"
|
|
62
|
+
elif isinstance(iceberg_type, ListType):
|
|
63
|
+
return {
|
|
64
|
+
"type": "list",
|
|
65
|
+
"element-id": iceberg_type.element_id,
|
|
66
|
+
"element-required": iceberg_type.element_required,
|
|
67
|
+
"element": iceberg_type_to_json(iceberg_type.element_type)
|
|
68
|
+
}
|
|
69
|
+
elif isinstance(iceberg_type, MapType):
|
|
70
|
+
return {
|
|
71
|
+
"type": "map",
|
|
72
|
+
"key-id": iceberg_type.key_id,
|
|
73
|
+
"key": iceberg_type_to_json(iceberg_type.key_type),
|
|
74
|
+
"value-id": iceberg_type.value_id,
|
|
75
|
+
"value-required": iceberg_type.value_required,
|
|
76
|
+
"value": iceberg_type_to_json(iceberg_type.value_type)
|
|
77
|
+
}
|
|
78
|
+
elif isinstance(iceberg_type, StructType):
|
|
79
|
+
return {
|
|
80
|
+
"type": "struct",
|
|
81
|
+
"fields": [
|
|
82
|
+
{
|
|
83
|
+
"id": field.field_id,
|
|
84
|
+
"name": field.name,
|
|
85
|
+
"required": field.required,
|
|
86
|
+
"type": iceberg_type_to_json(field.field_type)
|
|
87
|
+
}
|
|
88
|
+
for field in iceberg_type.fields
|
|
89
|
+
]
|
|
90
|
+
}
|
|
91
|
+
else:
|
|
92
|
+
# Fallback for unknown types
|
|
93
|
+
return str(iceberg_type)
|
|
94
|
+
|
|
95
|
+
|
|
30
96
|
class StructureToIcebergConverter:
|
|
31
97
|
"""Class to convert JSON Structure schema to Iceberg schema."""
|
|
32
98
|
|
|
@@ -45,8 +111,16 @@ class StructureToIcebergConverter:
|
|
|
45
111
|
"""Get the full name of a record type."""
|
|
46
112
|
return f"{namespace}.{name}" if namespace else name
|
|
47
113
|
|
|
48
|
-
def convert_structure_to_iceberg(self, structure_schema_path: str, structure_record_type: Optional[str], output_path: str, emit_cloudevents_columns: bool=False):
|
|
49
|
-
"""Convert a JSON Structure schema to an Iceberg schema.
|
|
114
|
+
def convert_structure_to_iceberg(self, structure_schema_path: str, structure_record_type: Optional[str], output_path: str, emit_cloudevents_columns: bool=False, output_format: str="arrow"):
|
|
115
|
+
"""Convert a JSON Structure schema to an Iceberg schema.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
structure_schema_path: Path to the JSON Structure schema file
|
|
119
|
+
structure_record_type: Record type to convert (or None for the root)
|
|
120
|
+
output_path: Path to write the Iceberg schema
|
|
121
|
+
emit_cloudevents_columns: Whether to add CloudEvents columns
|
|
122
|
+
output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
|
|
123
|
+
"""
|
|
50
124
|
schema_file = structure_schema_path
|
|
51
125
|
if not schema_file:
|
|
52
126
|
print("Please specify the JSON Structure schema file")
|
|
@@ -114,14 +188,32 @@ class StructureToIcebergConverter:
|
|
|
114
188
|
])
|
|
115
189
|
|
|
116
190
|
iceberg_schema = Schema(*iceberg_fields)
|
|
117
|
-
|
|
118
|
-
print(f"Iceberg schema created: {arrow_schema}")
|
|
191
|
+
print(f"Iceberg schema created: {iceberg_schema}")
|
|
119
192
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
193
|
+
if output_format == "arrow":
|
|
194
|
+
# Write as binary PyArrow schema
|
|
195
|
+
arrow_schema = schema_to_pyarrow(iceberg_schema)
|
|
196
|
+
file_io = PyArrowFileIO()
|
|
197
|
+
output_file = file_io.new_output("file://" + output_path)
|
|
198
|
+
with output_file.create(overwrite=True) as f:
|
|
199
|
+
pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
|
|
200
|
+
else:
|
|
201
|
+
# Write Iceberg schema as spec-compliant JSON (per Iceberg Table Spec Appendix C)
|
|
202
|
+
schema_json = {
|
|
203
|
+
"type": "struct",
|
|
204
|
+
"schema-id": 0,
|
|
205
|
+
"fields": [
|
|
206
|
+
{
|
|
207
|
+
"id": field.field_id,
|
|
208
|
+
"name": field.name,
|
|
209
|
+
"required": field.required,
|
|
210
|
+
"type": iceberg_type_to_json(field.field_type)
|
|
211
|
+
}
|
|
212
|
+
for field in iceberg_schema.fields
|
|
213
|
+
]
|
|
214
|
+
}
|
|
215
|
+
with open(output_path, "w", encoding="utf-8") as f:
|
|
216
|
+
json.dump(schema_json, f, indent=2)
|
|
125
217
|
|
|
126
218
|
def resolve_ref(self, ref: str) -> Dict[str, Any]:
|
|
127
219
|
"""Resolve a $ref reference."""
|
|
@@ -348,8 +440,16 @@ class StructureToIcebergConverter:
|
|
|
348
440
|
return type_mapping.get(type_name, StringType())
|
|
349
441
|
|
|
350
442
|
|
|
351
|
-
def convert_structure_to_iceberg(structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns=False):
|
|
352
|
-
"""Convert a JSON Structure schema to an Iceberg schema.
|
|
443
|
+
def convert_structure_to_iceberg(structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns=False, output_format="arrow"):
|
|
444
|
+
"""Convert a JSON Structure schema to an Iceberg schema.
|
|
445
|
+
|
|
446
|
+
Args:
|
|
447
|
+
structure_schema_path: Path to the JSON Structure schema file
|
|
448
|
+
structure_record_type: Record type to convert (or None for the root)
|
|
449
|
+
output_path: Path to write the Iceberg schema
|
|
450
|
+
emit_cloudevents_columns: Whether to add CloudEvents columns
|
|
451
|
+
output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
|
|
452
|
+
"""
|
|
353
453
|
converter = StructureToIcebergConverter()
|
|
354
454
|
converter.convert_structure_to_iceberg(
|
|
355
|
-
structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns)
|
|
455
|
+
structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns, output_format)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: structurize
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.2
|
|
4
4
|
Summary: Tools to convert from and to JSON Structure from various other schema languages.
|
|
5
5
|
Author-email: Clemens Vasters <clemensv@microsoft.com>
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
avrotize/__init__.py,sha256=t5h5wkHXr6M0mmHAB5rhjZ3Gxy9xutGTGIfojfao9rI,3820
|
|
2
2
|
avrotize/__main__.py,sha256=5pY8dYAURcOnFRvgb6fgaOIa_SOzPLIWbU8-ZTQ0jG4,88
|
|
3
|
-
avrotize/_version.py,sha256=
|
|
3
|
+
avrotize/_version.py,sha256=oe-LtbcDFd-ELWKNVft1PeQ3Tp3QrrPnPsINIJYW4ZE,712
|
|
4
4
|
avrotize/asn1toavro.py,sha256=QDNwfBfXMxSH-k487CA3CaGCGDzOLs4PpVbbENm5uF0,8386
|
|
5
5
|
avrotize/avrotize.py,sha256=VHFpBltMVBpyt0ju3ZWW725BKjQ4Fk-nrAy8udW-X44,5713
|
|
6
6
|
avrotize/avrotocpp.py,sha256=hRZV247_TDD7Sm6_8sFx-UH5SueLLx2Wg6TvAVUX0iE,25693
|
|
@@ -10,7 +10,7 @@ avrotize/avrotodatapackage.py,sha256=zSCphLvCYiBKRAUCdccsr-4JysH3PyAS6fSgwa65Tss
|
|
|
10
10
|
avrotize/avrotodb.py,sha256=IXqH5nHNgIErEIosxxSE5DIGzKJGk2FU2HIzmSt5eIs,47102
|
|
11
11
|
avrotize/avrotogo.py,sha256=9URV0SG8vvjadiNhXW12x_uRplO-JoHEqjIgViWWb4w,22545
|
|
12
12
|
avrotize/avrotographql.py,sha256=i6G7xWjH_Lsn_CLiM4BCPb8OyZuCCpsYjXwXNTRMwEE,7394
|
|
13
|
-
avrotize/avrotoiceberg.py,sha256=
|
|
13
|
+
avrotize/avrotoiceberg.py,sha256=DCzUqtUYyECIrwq9QRopgiON0QcsXIxI-r_RkPJpxnk,12960
|
|
14
14
|
avrotize/avrotojava.py,sha256=NZZ7mUFVzvp7HBsU0XPiCwl4GVXE1RtS86pfNFJsIq8,135427
|
|
15
15
|
avrotize/avrotojs.py,sha256=QjB6XjFnDrpZBZrrWqS0TN8fQfRXBfhHabfG73FOIo8,12249
|
|
16
16
|
avrotize/avrotojsons.py,sha256=I7EJz_IaCYLNzcycYXDOLAexHRHQfyb4yoWcxJ02vb0,22697
|
|
@@ -25,7 +25,7 @@ avrotize/avrotorust.py,sha256=HEcDirRBCbXQNNs_FmkT-sp1dWQgZ8A23qkQYUxVuXE,24255
|
|
|
25
25
|
avrotize/avrotots.py,sha256=u_XLjlHN0Gof5QYlpqK4X9WoX9rL30TjQMPg4TiyYnI,33241
|
|
26
26
|
avrotize/avrotoxsd.py,sha256=iGQq_8kC0kfKsqvqS6s_mO-kJ8N5G8vXOwqRI_DZUxc,17744
|
|
27
27
|
avrotize/cddltostructure.py,sha256=MA2c-P3CIEAxEaBX-FF299gR55xcLEV3FrfTr2QfayM,74491
|
|
28
|
-
avrotize/commands.json,sha256=
|
|
28
|
+
avrotize/commands.json,sha256=c6yV5vSey4GuoLVllPvL0Zb5rfmMQ8dgXUzKAtgKMZw,98904
|
|
29
29
|
avrotize/common.py,sha256=enqNR1I9-SbW7fNJE3w7N2R87kiN6_9Oa7VB4b2AUBc,31913
|
|
30
30
|
avrotize/constants.py,sha256=LlgHrvT6RsRPrhFGRNHmFuIj3b1bSd45yC4rBCIGGVA,2753
|
|
31
31
|
avrotize/csvtoavro.py,sha256=TuIYm_Xv8gioEHl1YgWQKOYkFGGHfuwmK5RuEAEXbt8,4293
|
|
@@ -47,10 +47,10 @@ avrotize/structuretocpp.py,sha256=tBWOvyZPYQ1CHN6RgDnWlmzJ1giOyQ9SlHBHWvhPyiw,35
|
|
|
47
47
|
avrotize/structuretocsharp.py,sha256=NnHzeJcOWDbZ_LnGV6AW9hXgP7rQW6Lld9utzxO8l-g,128208
|
|
48
48
|
avrotize/structuretocsv.py,sha256=w9cwXAnnakKaeTtXsLWWO8KwYnXUxyXvC7a-ZKs-E94,13851
|
|
49
49
|
avrotize/structuretodatapackage.py,sha256=NEHRt30KfVDWH1EQakvuMdRZTtfVXx8fsaYud0ofb2g,29768
|
|
50
|
-
avrotize/structuretodb.py,sha256=
|
|
50
|
+
avrotize/structuretodb.py,sha256=a1lEQliACdLaN5FWJSQBPEA-25aFTyDbmzs7xUr34bI,44934
|
|
51
51
|
avrotize/structuretogo.py,sha256=EtU7C5IqKzPrYaxKUQJMDMpk-rlc8eRwFzHe489jHp4,34100
|
|
52
52
|
avrotize/structuretographql.py,sha256=wcGXnrup5v5saRa1BhR6o-X8q8ujsQMVqrFHQTBPjww,20468
|
|
53
|
-
avrotize/structuretoiceberg.py,sha256=
|
|
53
|
+
avrotize/structuretoiceberg.py,sha256=MNk17eIbvhZ5p7s-goNr-DBjN6GcOEksSDLNnrpWbMQ,19354
|
|
54
54
|
avrotize/structuretojava.py,sha256=r6YfI-_HtdeGqotL5W31WR5sz-TE1VLdDegctpjeiC4,47886
|
|
55
55
|
avrotize/structuretojs.py,sha256=TJuhUGqn0F2omjPVcrPeFPHY2KynRceFg9gMGvEenxA,29608
|
|
56
56
|
avrotize/structuretojsons.py,sha256=PJrQBaf6yQHu5eFkePxbjPBEmL-fYfX2wj6OmH1jsWw,22495
|
|
@@ -64,9 +64,9 @@ avrotize/structuretoxsd.py,sha256=01VpasyWSMOx04sILHLP7H-WkhGdXAEGKohUUfgrNf0,32
|
|
|
64
64
|
avrotize/xsdtoavro.py,sha256=nQtNH_3pEZBp67oUCPqzhvItEExHTe-8obsIfNRXt8Y,19064
|
|
65
65
|
avrotize/dependencies/cpp/vcpkg/vcpkg.json,sha256=se5qnUVQ1Q6wN_DqgIioqKg_y7ouh9oly2iBAJJXkgw,414
|
|
66
66
|
avrotize/dependencies/typescript/node22/package.json,sha256=oAW_2X-b715kV7aajwuONZEMkyQUJGviG3GwnoUa7hU,387
|
|
67
|
-
structurize-3.0.
|
|
68
|
-
structurize-3.0.
|
|
69
|
-
structurize-3.0.
|
|
70
|
-
structurize-3.0.
|
|
71
|
-
structurize-3.0.
|
|
72
|
-
structurize-3.0.
|
|
67
|
+
structurize-3.0.2.dist-info/licenses/LICENSE,sha256=xGtQGygTETTtDQJafZCUbpsed3GxO6grmqig-jGEuSk,11348
|
|
68
|
+
structurize-3.0.2.dist-info/METADATA,sha256=3c5_rlTtVt9hq44svI_ve5VVxLzIMEZH98HwWIm8BWw,3669
|
|
69
|
+
structurize-3.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
70
|
+
structurize-3.0.2.dist-info/entry_points.txt,sha256=biIH7jA5auhVqfbwHVk2gmD_gvrPYKgjpCAn0JWZ-Rs,55
|
|
71
|
+
structurize-3.0.2.dist-info/top_level.txt,sha256=yn-yQ0Cm1O9fbF8KJgv4IIvX4YRGelKgPqZF1wS5P50,9
|
|
72
|
+
structurize-3.0.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|