structurize 3.5.2__tar.gz → 3.5.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. {structurize-3.5.2/structurize.egg-info → structurize-3.5.3}/PKG-INFO +1 -1
  2. {structurize-3.5.2 → structurize-3.5.3}/avrotize/_version.py +3 -3
  3. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotokusto.py +40 -24
  4. {structurize-3.5.2 → structurize-3.5.3}/avrotize/commands.json +32 -2
  5. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretokusto.py +94 -37
  6. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretopython.py +6 -2
  7. {structurize-3.5.2 → structurize-3.5.3/structurize.egg-info}/PKG-INFO +1 -1
  8. {structurize-3.5.2 → structurize-3.5.3}/.gitignore +0 -0
  9. {structurize-3.5.2 → structurize-3.5.3}/LICENSE +0 -0
  10. {structurize-3.5.2 → structurize-3.5.3}/MANIFEST.in +0 -0
  11. {structurize-3.5.2 → structurize-3.5.3}/README.md +0 -0
  12. {structurize-3.5.2 → structurize-3.5.3}/avrotize/__init__.py +0 -0
  13. {structurize-3.5.2 → structurize-3.5.3}/avrotize/__main__.py +0 -0
  14. {structurize-3.5.2 → structurize-3.5.3}/avrotize/asn1toavro.py +0 -0
  15. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotize.py +0 -0
  16. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotocpp.py +0 -0
  17. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotocsharp.py +0 -0
  18. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotocsv.py +0 -0
  19. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotodatapackage.py +0 -0
  20. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotodb.py +0 -0
  21. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotogo.py +0 -0
  22. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotographql.py +0 -0
  23. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotoiceberg.py +0 -0
  24. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotojava.py +0 -0
  25. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotojs.py +0 -0
  26. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotojsons.py +0 -0
  27. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotojstruct.py +0 -0
  28. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotomd.py +0 -0
  29. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotools.py +0 -0
  30. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotoparquet.py +0 -0
  31. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotoproto.py +0 -0
  32. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotopython.py +0 -0
  33. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotorust.py +0 -0
  34. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotots.py +0 -0
  35. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrotoxsd.py +0 -0
  36. {structurize-3.5.2 → structurize-3.5.3}/avrotize/avrovalidator.py +0 -0
  37. {structurize-3.5.2 → structurize-3.5.3}/avrotize/cddltostructure.py +0 -0
  38. {structurize-3.5.2 → structurize-3.5.3}/avrotize/choice_inference.py +0 -0
  39. {structurize-3.5.2 → structurize-3.5.3}/avrotize/common.py +0 -0
  40. {structurize-3.5.2 → structurize-3.5.3}/avrotize/constants.py +0 -0
  41. {structurize-3.5.2 → structurize-3.5.3}/avrotize/csvtoavro.py +0 -0
  42. {structurize-3.5.2 → structurize-3.5.3}/avrotize/datapackagetoavro.py +0 -0
  43. {structurize-3.5.2 → structurize-3.5.3}/avrotize/dependencies/cpp/vcpkg/vcpkg.json +0 -0
  44. {structurize-3.5.2 → structurize-3.5.3}/avrotize/dependencies/typescript/node22/package.json +0 -0
  45. {structurize-3.5.2 → structurize-3.5.3}/avrotize/dependency_resolver.py +0 -0
  46. {structurize-3.5.2 → structurize-3.5.3}/avrotize/dependency_version.py +0 -0
  47. {structurize-3.5.2 → structurize-3.5.3}/avrotize/jsonstoavro.py +0 -0
  48. {structurize-3.5.2 → structurize-3.5.3}/avrotize/jsonstostructure.py +0 -0
  49. {structurize-3.5.2 → structurize-3.5.3}/avrotize/jsontoschema.py +0 -0
  50. {structurize-3.5.2 → structurize-3.5.3}/avrotize/jstructtoavro.py +0 -0
  51. {structurize-3.5.2 → structurize-3.5.3}/avrotize/kstructtoavro.py +0 -0
  52. {structurize-3.5.2 → structurize-3.5.3}/avrotize/kustotoavro.py +0 -0
  53. {structurize-3.5.2 → structurize-3.5.3}/avrotize/kustotojstruct.py +0 -0
  54. {structurize-3.5.2 → structurize-3.5.3}/avrotize/mcp_server.py +0 -0
  55. {structurize-3.5.2 → structurize-3.5.3}/avrotize/openapitostructure.py +0 -0
  56. {structurize-3.5.2 → structurize-3.5.3}/avrotize/parquettoavro.py +0 -0
  57. {structurize-3.5.2 → structurize-3.5.3}/avrotize/proto2parser.py +0 -0
  58. {structurize-3.5.2 → structurize-3.5.3}/avrotize/proto3parser.py +0 -0
  59. {structurize-3.5.2 → structurize-3.5.3}/avrotize/prototoavro.py +0 -0
  60. {structurize-3.5.2 → structurize-3.5.3}/avrotize/schema_inference.py +0 -0
  61. {structurize-3.5.2 → structurize-3.5.3}/avrotize/sqltoavro.py +0 -0
  62. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretocddl.py +0 -0
  63. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretocpp.py +0 -0
  64. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretocsharp.py +0 -0
  65. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretocsv.py +0 -0
  66. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretodatapackage.py +0 -0
  67. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretodb.py +0 -0
  68. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretogo.py +0 -0
  69. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretographql.py +0 -0
  70. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretoiceberg.py +0 -0
  71. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretojava.py +0 -0
  72. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretojs.py +0 -0
  73. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretojsons.py +0 -0
  74. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretomd.py +0 -0
  75. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretoproto.py +0 -0
  76. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretorust.py +0 -0
  77. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretots.py +0 -0
  78. {structurize-3.5.2 → structurize-3.5.3}/avrotize/structuretoxsd.py +0 -0
  79. {structurize-3.5.2 → structurize-3.5.3}/avrotize/validate.py +0 -0
  80. {structurize-3.5.2 → structurize-3.5.3}/avrotize/xmltoschema.py +0 -0
  81. {structurize-3.5.2 → structurize-3.5.3}/avrotize/xsdtoavro.py +0 -0
  82. {structurize-3.5.2 → structurize-3.5.3}/build.ps1 +0 -0
  83. {structurize-3.5.2 → structurize-3.5.3}/build.sh +0 -0
  84. {structurize-3.5.2 → structurize-3.5.3}/pyproject.toml +0 -0
  85. {structurize-3.5.2 → structurize-3.5.3}/setup.cfg +0 -0
  86. {structurize-3.5.2 → structurize-3.5.3}/structurize.egg-info/SOURCES.txt +0 -0
  87. {structurize-3.5.2 → structurize-3.5.3}/structurize.egg-info/dependency_links.txt +0 -0
  88. {structurize-3.5.2 → structurize-3.5.3}/structurize.egg-info/entry_points.txt +0 -0
  89. {structurize-3.5.2 → structurize-3.5.3}/structurize.egg-info/requires.txt +0 -0
  90. {structurize-3.5.2 → structurize-3.5.3}/structurize.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: structurize
3
- Version: 3.5.2
3
+ Version: 3.5.3
4
4
  Summary: Tools to convert from and to JSON Structure from various other schema languages.
5
5
  Author-email: Clemens Vasters <clemensv@microsoft.com>
6
6
  Classifier: Programming Language :: Python :: 3
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '3.5.2'
22
- __version_tuple__ = version_tuple = (3, 5, 2)
21
+ __version__ = version = '3.5.3'
22
+ __version_tuple__ = version_tuple = (3, 5, 3)
23
23
 
24
- __commit_id__ = commit_id = 'gff224504a'
24
+ __commit_id__ = commit_id = 'g4c9a891f7'
@@ -14,17 +14,33 @@ class AvroToKusto:
14
14
  """Initializes a new instance of the AvroToKusto class."""
15
15
  pass
16
16
 
17
- def convert_record_to_kusto(self, type_dict:dict, recordschema: dict, emit_cloudevents_columns: bool, emit_cloudevents_dispatch_table: bool) -> List[str]:
17
+ def convert_record_to_kusto(self, type_dict:dict, recordschema: dict, emit_cloudevents_columns: bool, emit_cloudevents_dispatch_table: bool, qualified_table_names: bool = False, namespace_override: str | None = None) -> List[str]:
18
18
  """Converts an Avro record schema to a Kusto table schema."""
19
19
  # Get the name and fields of the top-level record
20
- table_name = recordschema["name"]
20
+ simple_name = recordschema["name"]
21
+ record_namespace = namespace_override if namespace_override else recordschema.get("namespace")
22
+ if qualified_table_names and record_namespace:
23
+ table_name = f"{record_namespace}.{simple_name}"
24
+ else:
25
+ table_name = simple_name
26
+ # Kusto identifiers containing dots must be quoted as ['name.with.dots'].
27
+ if '.' in table_name:
28
+ table_ref = f"['{table_name}']"
29
+ table_in_query = f"['{table_name}']"
30
+ mv_ref = f"['{table_name}Latest']"
31
+ else:
32
+ table_ref = f"[{table_name}]"
33
+ table_in_query = table_name
34
+ mv_ref = f"{table_name}Latest"
35
+ # The JSON mapping name is a string literal, so dots are always fine.
36
+ mapping_base = table_name
21
37
  fields = recordschema["fields"]
22
38
 
23
39
  # Create a StringBuilder to store the kusto statements
24
40
  kusto = []
25
41
 
26
42
  # Append the create table statement with the column names and types
27
- kusto.append(f".create-merge table [{table_name}] (")
43
+ kusto.append(f".create-merge table {table_ref} (")
28
44
  columns = []
29
45
  for field in fields:
30
46
  column_name = field["name"]
@@ -48,7 +64,7 @@ class AvroToKusto:
48
64
  "description": doc_data
49
65
  }))
50
66
  kusto.append(
51
- f".alter table [{table_name}] docstring {doc_string};")
67
+ f".alter table {table_ref} docstring {doc_string};")
52
68
  kusto.append("")
53
69
 
54
70
  doc_string_statement = []
@@ -84,7 +100,7 @@ class AvroToKusto:
84
100
  " [___subject]: 'Context subject of the event'"
85
101
  ])
86
102
  if doc_string_statement:
87
- kusto.append(f".alter table [{table_name}] column-docstrings (")
103
+ kusto.append(f".alter table {table_ref} column-docstrings (")
88
104
  kusto.append(",\n".join(doc_string_statement))
89
105
  kusto.append(");")
90
106
  kusto.append("")
@@ -92,7 +108,7 @@ class AvroToKusto:
92
108
  # add the JSON mapping for the table
93
109
  # .create-or-alter table dfl_data_events ingestion json mapping
94
110
  kusto.append(
95
- f".create-or-alter table [{table_name}] ingestion json mapping \"{table_name}_json_flat\"")
111
+ f".create-or-alter table {table_ref} ingestion json mapping \"{mapping_base}_json_flat\"")
96
112
  kusto.append("```\n[")
97
113
  if emit_cloudevents_columns:
98
114
  kusto.append(" {\"column\": \"___type\", \"path\": \"$.type\"},")
@@ -115,7 +131,7 @@ class AvroToKusto:
115
131
 
116
132
  if emit_cloudevents_columns:
117
133
  kusto.append(
118
- f".create-or-alter table [{table_name}] ingestion json mapping \"{table_name}_json_ce_structured\"")
134
+ f".create-or-alter table {table_ref} ingestion json mapping \"{mapping_base}_json_ce_structured\"")
119
135
  kusto.append("```\n[")
120
136
  kusto.append(" {\"column\": \"___type\", \"path\": \"$.type\"},")
121
137
  kusto.append(
@@ -137,18 +153,18 @@ class AvroToKusto:
137
153
 
138
154
  if emit_cloudevents_columns:
139
155
  kusto.append(
140
- f".drop materialized-view {table_name}Latest ifexists;")
156
+ f".drop materialized-view {mv_ref} ifexists;")
141
157
  kusto.append("")
142
158
  kusto.append(
143
- f".create materialized-view with (backfill=true) {table_name}Latest on table {table_name} {{")
159
+ f".create materialized-view with (backfill=true) {mv_ref} on table {table_in_query} {{")
144
160
  kusto.append(
145
- f" {table_name} | summarize arg_max(___time, *) by ___type, ___source, ___subject")
161
+ f" {table_in_query} | summarize arg_max(___time, *) by ___type, ___source, ___subject")
146
162
  kusto.append("}")
147
163
  kusto.append("")
148
164
 
149
165
  if emit_cloudevents_dispatch_table:
150
- event_type = recordschema["namespace"] + "." + \
151
- recordschema["name"] if "namespace" in recordschema else recordschema["name"]
166
+ event_type = record_namespace + "." + \
167
+ simple_name if record_namespace else simple_name
152
168
 
153
169
  query = f"_cloudevents_dispatch | where (specversion == '1.0' and type == '{event_type}') | " + \
154
170
  "project"
@@ -162,7 +178,7 @@ class AvroToKusto:
162
178
  query += "___type = type,___source = source,___id = ['id'],___time = ['time'],___subject = subject"
163
179
 
164
180
  # build an update policy for the table that gets triggered by updates to the dispatch table and extracts the event
165
- kusto.append(f".alter table [{table_name}] policy update")
181
+ kusto.append(f".alter table {table_ref} policy update")
166
182
  kusto.append("```")
167
183
  kusto.append("[{")
168
184
  kusto.append(" \"IsEnabled\": true,")
@@ -176,7 +192,7 @@ class AvroToKusto:
176
192
 
177
193
  return kusto
178
194
 
179
- def convert_avro_to_kusto_script(self, avro_schema_path, avro_record_type, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False) -> str:
195
+ def convert_avro_to_kusto_script(self, avro_schema_path, avro_record_type, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, qualified_table_names: bool = False, namespace_override: str | None = None) -> str:
180
196
  """Converts an Avro schema to a Kusto table schema."""
181
197
  if emit_cloudevents_dispatch_table:
182
198
  emit_cloudevents_columns = True
@@ -248,13 +264,13 @@ class AvroToKusto:
248
264
  if not isinstance(record, dict) or "type" not in record or record["type"] != "record":
249
265
  continue
250
266
  kusto_script.extend(self.convert_record_to_kusto(type_dict,
251
- record, emit_cloudevents_columns, emit_cloudevents_dispatch_table))
267
+ record, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace_override))
252
268
  return "\n".join(kusto_script)
253
269
 
254
- def convert_avro_to_kusto_file(self, avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False):
270
+ def convert_avro_to_kusto_file(self, avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, qualified_table_names: bool = False, namespace_override: str | None = None):
255
271
  """Converts an Avro schema to a Kusto table schema."""
256
272
  script = self.convert_avro_to_kusto_script(
257
- avro_schema_path, avro_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
273
+ avro_schema_path, avro_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace_override)
258
274
  with open(kusto_file_path, "w", encoding="utf-8") as kusto_file:
259
275
  kusto_file.write(script)
260
276
 
@@ -331,18 +347,18 @@ class AvroToKusto:
331
347
  return "dynamic"
332
348
 
333
349
 
334
- def convert_avro_to_kusto_file(avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False):
350
+ def convert_avro_to_kusto_file(avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, qualified_table_names: bool = False, namespace: str | None = None):
335
351
  """Converts an Avro schema to a Kusto table schema."""
336
352
  avro_to_kusto = AvroToKusto()
337
353
  avro_to_kusto.convert_avro_to_kusto_file(
338
- avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
354
+ avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace)
339
355
 
340
356
 
341
- def convert_avro_to_kusto_db(avro_schema_path, avro_record_type, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None):
357
+ def convert_avro_to_kusto_db(avro_schema_path, avro_record_type, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None, qualified_table_names: bool = False, namespace: str | None = None):
342
358
  """Converts an Avro schema to a Kusto table schema."""
343
359
  avro_to_kusto = AvroToKusto()
344
360
  script = avro_to_kusto.convert_avro_to_kusto_script(
345
- avro_schema_path, avro_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
361
+ avro_schema_path, avro_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace)
346
362
  kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(
347
363
  kusto_uri) if not token_provider else KustoConnectionStringBuilder.with_token_provider(kusto_uri, token_provider)
348
364
  client = KustoClient(kcsb)
@@ -354,11 +370,11 @@ def convert_avro_to_kusto_db(avro_schema_path, avro_record_type, kusto_uri, kust
354
370
  print(e)
355
371
  sys.exit(1)
356
372
 
357
- def convert_avro_to_kusto(avro_schema_path, avro_record_type, kusto_file_path, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None):
373
+ def convert_avro_to_kusto(avro_schema_path, avro_record_type, kusto_file_path, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None, qualified_table_names: bool = False, namespace: str | None = None):
358
374
  """Converts an Avro schema to a Kusto table schema."""
359
375
  if not kusto_uri and not kusto_database:
360
376
  convert_avro_to_kusto_file(
361
- avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
377
+ avro_schema_path, avro_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace)
362
378
  else:
363
379
  convert_avro_to_kusto_db(
364
- avro_schema_path, avro_record_type, kusto_uri, kusto_database, emit_cloudevents_columns, emit_cloudevents_dispatch_table, token_provider)
380
+ avro_schema_path, avro_record_type, kusto_uri, kusto_database, emit_cloudevents_columns, emit_cloudevents_dispatch_table, token_provider, qualified_table_names, namespace)
@@ -561,7 +561,9 @@
561
561
  "kusto_database": "args.kusto_database",
562
562
  "avro_record_type": "args.record_type",
563
563
  "emit_cloudevents_columns": "args.emit_cloudevents_columns",
564
- "emit_cloudevents_dispatch_table": "args.emit_cloudevents_dispatch"
564
+ "emit_cloudevents_dispatch_table": "args.emit_cloudevents_dispatch",
565
+ "qualified_table_names": "args.qualified_table_names",
566
+ "namespace": "args.namespace"
565
567
  }
566
568
  },
567
569
  "extensions": [
@@ -617,6 +619,19 @@
617
619
  "type": "bool",
618
620
  "help": "Emit a _cloudevents_dispatch ingestion table and update policies for each generated table",
619
621
  "required": false
622
+ },
623
+ {
624
+ "name": "--qualified-table-names",
625
+ "type": "bool",
626
+ "help": "Generate table names using the fully qualified namespace name of the entity, if available",
627
+ "default": false,
628
+ "required": false
629
+ },
630
+ {
631
+ "name": "--namespace",
632
+ "type": "str",
633
+ "help": "Set or override the namespace used for qualifying table names and CloudEvents type identifiers",
634
+ "required": false
620
635
  }
621
636
  ],
622
637
  "suggested_output_file_path": "{input_file_name}.kql",
@@ -662,7 +677,9 @@
662
677
  "kusto_database": "args.kusto_database",
663
678
  "structure_record_type": "args.record_type",
664
679
  "emit_cloudevents_columns": "args.emit_cloudevents_columns",
665
- "emit_cloudevents_dispatch_table": "args.emit_cloudevents_dispatch"
680
+ "emit_cloudevents_dispatch_table": "args.emit_cloudevents_dispatch",
681
+ "qualified_table_names": "args.qualified_table_names",
682
+ "namespace": "args.namespace"
666
683
  }
667
684
  },
668
685
  "extensions": [
@@ -719,6 +736,19 @@
719
736
  "type": "bool",
720
737
  "help": "Emit a _cloudevents_dispatch ingestion table and update policies for each generated table",
721
738
  "required": false
739
+ },
740
+ {
741
+ "name": "--qualified-table-names",
742
+ "type": "bool",
743
+ "help": "Generate table names using the fully qualified namespace name of the entity, if available",
744
+ "default": false,
745
+ "required": false
746
+ },
747
+ {
748
+ "name": "--namespace",
749
+ "type": "str",
750
+ "help": "Set or override the namespace used for qualifying table names and CloudEvents type identifiers",
751
+ "required": false
722
752
  }
723
753
  ],
724
754
  "suggested_output_file_path": "{input_file_name}.kql",
@@ -139,59 +139,109 @@ class StructureToKusto:
139
139
  """
140
140
  Find all concrete object types in the schema, including those in definitions.
141
141
  Filters out abstract types and includes flattened versions of types with inheritance.
142
+ Tracks the JSON Structure namespace path (definitions/<seg>/<seg>/<type>) on each
143
+ returned schema via a synthetic '_kusto_namespace' key.
142
144
  """
143
145
  object_types = []
144
-
145
- def process_schema(s: Dict, path: str = ""):
146
+
147
+ def process_schema(s: Dict, namespace_path: str = ""):
146
148
  if not isinstance(s, dict):
147
149
  return
148
-
150
+
149
151
  # Check if this is an object type
150
152
  if s.get('type') == 'object':
151
153
  # Only include concrete types
152
154
  if self.is_concrete_type(s):
153
155
  # Flatten inheritance if present
154
156
  flattened = self.flatten_inheritance(s, schema_doc)
157
+ if namespace_path and '_kusto_namespace' not in flattened:
158
+ flattened = dict(flattened)
159
+ flattened['_kusto_namespace'] = namespace_path
155
160
  object_types.append(flattened)
156
-
161
+
157
162
  # Recursively process definitions
158
163
  if 'definitions' in s:
159
164
  for def_name, def_schema in s['definitions'].items():
160
165
  if isinstance(def_schema, dict):
161
166
  # Handle nested definitions
162
167
  if def_schema.get('type') == 'object':
163
- process_schema(def_schema, f"{path}/{def_name}")
168
+ process_schema(def_schema, namespace_path)
164
169
  else:
165
- # Recurse into nested namespaces
170
+ # Recurse into nested namespaces; the def_name is a
171
+ # namespace segment when the value isn't itself an object type.
172
+ nested_ns = f"{namespace_path}.{def_name}" if namespace_path else def_name
166
173
  for nested_key, nested_val in def_schema.items():
167
174
  if isinstance(nested_val, dict):
168
- process_schema(nested_val, f"{path}/{def_name}/{nested_key}")
169
-
175
+ if nested_val.get('type') == 'object':
176
+ process_schema(nested_val, nested_ns)
177
+ else:
178
+ deeper_ns = f"{nested_ns}.{nested_key}"
179
+ process_schema(nested_val, deeper_ns)
180
+
170
181
  # Process top-level schema
171
182
  if isinstance(schema, dict):
172
183
  if '$root' in schema:
173
184
  root_ref = schema['$root']
174
185
  root_schema = self.resolve_ref(root_ref, schema, schema)
175
186
  if root_schema:
176
- process_schema(root_schema)
187
+ # Derive namespace from the $root path (drop leading '#/definitions/'
188
+ # and the trailing type name segment).
189
+ root_ns = ''
190
+ if isinstance(root_ref, str) and root_ref.startswith('#/definitions/'):
191
+ parts = root_ref[len('#/definitions/'):].split('/')
192
+ if len(parts) > 1:
193
+ root_ns = '.'.join(parts[:-1])
194
+ flattened = self.flatten_inheritance(root_schema, schema_doc)
195
+ if root_ns and '_kusto_namespace' not in flattened:
196
+ flattened = dict(flattened)
197
+ flattened['_kusto_namespace'] = root_ns
198
+ if self.is_concrete_type(flattened):
199
+ object_types.append(flattened)
177
200
  elif 'type' in schema and schema['type'] == 'object':
178
201
  process_schema(schema)
179
-
202
+
180
203
  # Always process definitions
181
204
  if 'definitions' in schema:
182
205
  process_schema(schema)
183
-
206
+
184
207
  elif isinstance(schema, list):
185
208
  for s in schema:
186
209
  if isinstance(s, dict):
187
210
  process_schema(s)
188
-
211
+
189
212
  return object_types
190
213
 
191
- def convert_record_to_kusto(self, recordschema: dict, schema_doc: dict, emit_cloudevents_columns: bool, emit_cloudevents_dispatch_table: bool) -> List[str]:
214
+ def convert_record_to_kusto(self, recordschema: dict, schema_doc: dict, emit_cloudevents_columns: bool, emit_cloudevents_dispatch_table: bool, qualified_table_names: bool = False, namespace_override: str | None = None) -> List[str]:
192
215
  """Converts a JSON Structure object schema to a Kusto table schema."""
193
216
  # Get the name and fields of the top-level record
194
- table_name = recordschema.get("name", "UnnamedTable")
217
+ simple_name = recordschema.get("name", "UnnamedTable")
218
+ record_namespace: str | None
219
+ if namespace_override:
220
+ record_namespace = namespace_override
221
+ elif recordschema.get("namespace"):
222
+ record_namespace = recordschema.get("namespace")
223
+ elif qualified_table_names:
224
+ # Only fall back to the derived JSON Structure namespace path when
225
+ # qualified table names are explicitly requested, to preserve
226
+ # backward-compatible output otherwise.
227
+ record_namespace = recordschema.get("_kusto_namespace")
228
+ else:
229
+ record_namespace = None
230
+ if qualified_table_names and record_namespace:
231
+ table_name = f"{record_namespace}.{simple_name}"
232
+ else:
233
+ table_name = simple_name
234
+ # Kusto identifiers containing dots must be quoted as ['name.with.dots'].
235
+ if '.' in table_name:
236
+ table_ref = f"['{table_name}']"
237
+ table_in_query = f"['{table_name}']"
238
+ mv_ref = f"['{table_name}Latest']"
239
+ else:
240
+ table_ref = f"[{table_name}]"
241
+ table_in_query = table_name
242
+ mv_ref = f"{table_name}Latest"
243
+ # The JSON mapping name is a string literal, so dots are always fine.
244
+ mapping_base = table_name
195
245
 
196
246
  # Handle properties from JSON Structure
197
247
  properties = recordschema.get("properties", {})
@@ -200,7 +250,7 @@ class StructureToKusto:
200
250
  kusto = []
201
251
 
202
252
  # Append the create table statement with the column names and types
203
- kusto.append(f".create-merge table [{table_name}] (")
253
+ kusto.append(f".create-merge table {table_ref} (")
204
254
  columns = []
205
255
  for prop_name, prop_schema in properties.items():
206
256
  column_name = prop_name
@@ -238,7 +288,7 @@ class StructureToKusto:
238
288
  "description": doc_data
239
289
  }))
240
290
  kusto.append(
241
- f".alter table [{table_name}] docstring {doc_string};")
291
+ f".alter table {table_ref} docstring {doc_string};")
242
292
  kusto.append("")
243
293
 
244
294
  doc_string_statement = []
@@ -284,14 +334,14 @@ class StructureToKusto:
284
334
  " [___subject]: 'Context subject of the event'"
285
335
  ])
286
336
  if doc_string_statement:
287
- kusto.append(f".alter table [{table_name}] column-docstrings (")
337
+ kusto.append(f".alter table {table_ref} column-docstrings (")
288
338
  kusto.append(",\n".join(doc_string_statement))
289
339
  kusto.append(");")
290
340
  kusto.append("")
291
341
 
292
342
  # add the JSON mapping for the table
293
343
  kusto.append(
294
- f".create-or-alter table [{table_name}] ingestion json mapping \"{table_name}_json_flat\"")
344
+ f".create-or-alter table {table_ref} ingestion json mapping \"{mapping_base}_json_flat\"")
295
345
  kusto.append("```\n[")
296
346
  if emit_cloudevents_columns:
297
347
  kusto.append(" {\"column\": \"___type\", \"path\": \"$.type\"},")
@@ -312,7 +362,7 @@ class StructureToKusto:
312
362
 
313
363
  if emit_cloudevents_columns:
314
364
  kusto.append(
315
- f".create-or-alter table [{table_name}] ingestion json mapping \"{table_name}_json_ce_structured\"")
365
+ f".create-or-alter table {table_ref} ingestion json mapping \"{mapping_base}_json_ce_structured\"")
316
366
  kusto.append("```\n[")
317
367
  kusto.append(" {\"column\": \"___type\", \"path\": \"$.type\"},")
318
368
  kusto.append(
@@ -332,18 +382,17 @@ class StructureToKusto:
332
382
 
333
383
  if emit_cloudevents_columns:
334
384
  kusto.append(
335
- f".drop materialized-view {table_name}Latest ifexists;")
385
+ f".drop materialized-view {mv_ref} ifexists;")
336
386
  kusto.append("")
337
387
  kusto.append(
338
- f".create materialized-view with (backfill=true) {table_name}Latest on table {table_name} {{")
388
+ f".create materialized-view with (backfill=true) {mv_ref} on table {table_in_query} {{")
339
389
  kusto.append(
340
- f" {table_name} | summarize arg_max(___time, *) by ___type, ___source, ___subject")
390
+ f" {table_in_query} | summarize arg_max(___time, *) by ___type, ___source, ___subject")
341
391
  kusto.append("}")
342
392
  kusto.append("")
343
393
 
344
394
  if emit_cloudevents_dispatch_table:
345
- namespace = recordschema.get("namespace", "")
346
- event_type = namespace + "." + table_name if namespace else table_name
395
+ event_type = record_namespace + "." + simple_name if record_namespace else simple_name
347
396
 
348
397
  query = f"_cloudevents_dispatch | where (specversion == '1.0' and type == '{event_type}') | " + \
349
398
  "project"
@@ -354,7 +403,7 @@ class StructureToKusto:
354
403
  query += "___type = type,___source = source,___id = ['id'],___time = ['time'],___subject = subject"
355
404
 
356
405
  # build an update policy for the table that gets triggered by updates to the dispatch table and extracts the event
357
- kusto.append(f".alter table [{table_name}] policy update")
406
+ kusto.append(f".alter table {table_ref} policy update")
358
407
  kusto.append("```")
359
408
  kusto.append("[{")
360
409
  kusto.append(" \"IsEnabled\": true,")
@@ -368,7 +417,7 @@ class StructureToKusto:
368
417
 
369
418
  return kusto
370
419
 
371
- def convert_structure_to_kusto_script(self, structure_schema_path, structure_record_type, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False) -> str:
420
+ def convert_structure_to_kusto_script(self, structure_schema_path, structure_record_type, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, qualified_table_names: bool = False, namespace_override: str | None = None) -> str:
372
421
  """Converts a JSON Structure schema to a Kusto table schema."""
373
422
  if emit_cloudevents_dispatch_table:
374
423
  emit_cloudevents_columns = True
@@ -426,7 +475,15 @@ class StructureToKusto:
426
475
  record_schema = self.resolve_ref(root_ref, schema, schema)
427
476
  if record_schema:
428
477
  # Flatten inheritance
429
- record_schemas = [self.flatten_inheritance(record_schema, schema_doc)]
478
+ flat = self.flatten_inheritance(record_schema, schema_doc)
479
+ # Derive namespace from the $root path so qualified table
480
+ # naming works when --qualified-table-names is set.
481
+ if isinstance(root_ref, str) and root_ref.startswith('#/definitions/'):
482
+ parts = root_ref[len('#/definitions/'):].split('/')
483
+ if len(parts) > 1 and '_kusto_namespace' not in flat:
484
+ flat = dict(flat)
485
+ flat['_kusto_namespace'] = '.'.join(parts[:-1])
486
+ record_schemas = [flat]
430
487
  elif 'type' in schema and schema['type'] == 'object':
431
488
  # Flatten inheritance
432
489
  record_schemas = [self.flatten_inheritance(schema, schema_doc)]
@@ -515,17 +572,17 @@ class StructureToKusto:
515
572
  continue
516
573
 
517
574
  kusto_script.extend(self.convert_record_to_kusto(
518
- record_schema, schema_doc, emit_cloudevents_columns, emit_cloudevents_dispatch_table))
575
+ record_schema, schema_doc, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace_override))
519
576
 
520
577
  # Join and clean up extra blank lines at the end
521
578
  result = "\n".join(kusto_script)
522
579
  # Remove trailing whitespace while preserving intentional blank lines
523
580
  return result.rstrip() + "\n" if result else ""
524
581
 
525
- def convert_structure_to_kusto_file(self, structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False):
582
+ def convert_structure_to_kusto_file(self, structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, qualified_table_names: bool = False, namespace_override: str | None = None):
526
583
  """Converts a JSON Structure schema to a Kusto table schema."""
527
584
  script = self.convert_structure_to_kusto_script(
528
- structure_schema_path, structure_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
585
+ structure_schema_path, structure_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace_override)
529
586
  with open(kusto_file_path, "w", encoding="utf-8") as kusto_file:
530
587
  kusto_file.write(script)
531
588
 
@@ -621,18 +678,18 @@ class StructureToKusto:
621
678
  return mapping.get(type_value, 'dynamic')
622
679
 
623
680
 
624
- def convert_structure_to_kusto_file(structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False):
681
+ def convert_structure_to_kusto_file(structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, qualified_table_names: bool = False, namespace: str | None = None):
625
682
  """Converts a JSON Structure schema to a Kusto table schema."""
626
683
  structure_to_kusto = StructureToKusto()
627
684
  structure_to_kusto.convert_structure_to_kusto_file(
628
- structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
685
+ structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace)
629
686
 
630
687
 
631
- def convert_structure_to_kusto_db(structure_schema_path, structure_record_type, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None):
688
+ def convert_structure_to_kusto_db(structure_schema_path, structure_record_type, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None, qualified_table_names: bool = False, namespace: str | None = None):
632
689
  """Converts a JSON Structure schema to a Kusto table schema."""
633
690
  structure_to_kusto = StructureToKusto()
634
691
  script = structure_to_kusto.convert_structure_to_kusto_script(
635
- structure_schema_path, structure_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
692
+ structure_schema_path, structure_record_type, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace)
636
693
  kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(
637
694
  kusto_uri) if not token_provider else KustoConnectionStringBuilder.with_token_provider(kusto_uri, token_provider)
638
695
  client = KustoClient(kcsb)
@@ -645,11 +702,11 @@ def convert_structure_to_kusto_db(structure_schema_path, structure_record_type,
645
702
  sys.exit(1)
646
703
 
647
704
 
648
- def convert_structure_to_kusto(structure_schema_path, structure_record_type, kusto_file_path, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None):
705
+ def convert_structure_to_kusto(structure_schema_path, structure_record_type, kusto_file_path, kusto_uri, kusto_database, emit_cloudevents_columns=False, emit_cloudevents_dispatch_table=False, token_provider=None, qualified_table_names: bool = False, namespace: str | None = None):
649
706
  """Converts a JSON Structure schema to a Kusto table schema."""
650
707
  if not kusto_uri and not kusto_database:
651
708
  convert_structure_to_kusto_file(
652
- structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table)
709
+ structure_schema_path, structure_record_type, kusto_file_path, emit_cloudevents_columns, emit_cloudevents_dispatch_table, qualified_table_names, namespace)
653
710
  else:
654
711
  convert_structure_to_kusto_db(
655
- structure_schema_path, structure_record_type, kusto_uri, kusto_database, emit_cloudevents_columns, emit_cloudevents_dispatch_table, token_provider)
712
+ structure_schema_path, structure_record_type, kusto_uri, kusto_database, emit_cloudevents_columns, emit_cloudevents_dispatch_table, token_provider, qualified_table_names, namespace)
@@ -863,10 +863,14 @@ def convert_structure_to_python(structure_schema_path, py_file_path, package_nam
863
863
  structure_to_python.convert(structure_schema_path, py_file_path)
864
864
 
865
865
 
866
- def convert_structure_schema_to_python(structure_schema, py_file_path, package_name='', dataclasses_json_annotation=False):
866
+ def convert_structure_schema_to_python(structure_schema, py_file_path, package_name='', dataclasses_json_annotation=False, avro_annotation=False):
867
867
  """Converts JSON Structure schema to Python dataclasses"""
868
868
  package_name = safe_package_name(package_name) if package_name else package_name
869
- structure_to_python = StructureToPython(package_name, dataclasses_json_annotation=dataclasses_json_annotation)
869
+ structure_to_python = StructureToPython(
870
+ package_name,
871
+ dataclasses_json_annotation=dataclasses_json_annotation,
872
+ avro_annotation=avro_annotation,
873
+ )
870
874
  if isinstance(structure_schema, dict):
871
875
  structure_schema = [structure_schema]
872
876
  structure_to_python.convert_schemas(structure_schema, py_file_path)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: structurize
3
- Version: 3.5.2
3
+ Version: 3.5.3
4
4
  Summary: Tools to convert from and to JSON Structure from various other schema languages.
5
5
  Author-email: Clemens Vasters <clemensv@microsoft.com>
6
6
  Classifier: Programming Language :: Python :: 3
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes