structurize 3.5.3__tar.gz → 3.5.5__tar.gz

{structurize-3.5.3/structurize.egg-info → structurize-3.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structurize
-Version: 3.5.3
+Version: 3.5.5
 Summary: Tools to convert from and to JSON Structure from various other schema languages.
 Author-email: Clemens Vasters <clemensv@microsoft.com>
 Classifier: Programming Language :: Python :: 3

{structurize-3.5.3 → structurize-3.5.5}/avrotize/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '3.5.3'
-__version_tuple__ = version_tuple = (3, 5, 3)
+__version__ = version = '3.5.5'
+__version_tuple__ = version_tuple = (3, 5, 5)
 
-__commit_id__ = commit_id = 'g4c9a891f7'
+__commit_id__ = commit_id = 'g2c742c4cc'

{structurize-3.5.3 → structurize-3.5.5}/avrotize/kustotojstruct.py

@@ -85,7 +85,11 @@ class KustoToJsonStructure:
         return self.type_map.get(kusto_type, "string")
 
     def kusto_to_jstruct_schema(self, kusto_schema: dict, table_name: str) -> JsonNode:
-        """ Converts a Kusto schema to JSON Structure schema."""
+        """ Converts a Kusto schema to JSON Structure schema.
+        
+        Returns a list of tuples (type_value, schema) where type_value is the CloudEvents 
+        type value (or None for non-CloudEvents tables).
+        """
         column_names = set([column['Name'].lstrip('_')
                            for column in kusto_schema['OrderedColumns']])
         type_values: List[str|None] = []
@@ -105,7 +109,7 @@ class KustoToJsonStructure:
         if len(type_values) == 0:
             type_values.append(None)
 
-        schemas: List[JsonNode] = []
+        schemas: List[tuple] = []
         for type_value in type_values:
             schema: JsonNode = {}
             properties: Dict[str, JsonNode] = {}
@@ -120,10 +124,11 @@ class KustoToJsonStructure:
                     data_schemas = [data_schemas]
                 if isinstance(data_schemas, list):
                     for schema in data_schemas:
-                        if not isinstance(schema, dict) or "type" not in schema or schema["type"] != "object":
-                            schema = self.wrap_schema_in_root_record(schema, type_name)
+                        if isinstance(schema, dict):
+                            # Remove inferrer artifacts that don't belong in the final schema
+                            schema.pop("root", None)
                         self.apply_schema_attributes(schema, kusto_schema, table_name, type_value)
-                        schemas.append(schema)
+                        schemas.append((type_value, schema))
             else:
                 for column in kusto_schema['OrderedColumns']:
                     jstruct_type = self.map_kusto_type_to_jstruct_type(
@@ -147,9 +152,9 @@ class KustoToJsonStructure:
                     "properties": properties
                 }
                 self.apply_schema_attributes(schema, kusto_schema, table_name, type_value)
-                schemas.append(schema)
+                schemas.append((type_value, schema))
 
-        return schemas if len(schemas) > 1 else schemas[0]
+        return schemas
 
 
     def wrap_schema_in_root_record(self, schema: JsonNode, type_name: str):
@@ -188,6 +193,16 @@ class KustoToJsonStructure:
         table_names = [row['TableName'] for row in rows.primary_results[0]]
         return table_names
 
+    def _derive_schema_id(self, type_value: str | None, table_name: str) -> str:
+        """Derives a schema ID from the event type value or table name."""
+        if type_value:
+            return type_value
+        return table_name
+
+    def _derive_group_id(self, table_name: str) -> str:
+        """Derives a message group ID from the table name."""
+        return table_name
+
     def process_all_tables(self):
         """ Processes all tables in the Kusto database."""
         if self.single_table_name:
@@ -195,15 +210,13 @@ class KustoToJsonStructure:
         else:
             table_names = self.fetch_all_table_names()
         
-        all_schemas = []
+        # Collect all (type_value, schema) tuples grouped by table
+        all_entries: List[tuple] = []
         for table_name in table_names:
             kusto_schema = self.fetch_table_schema_and_docs(table_name)
-            jstruct_schema = self.kusto_to_jstruct_schema(kusto_schema, table_name)
-            
-            if isinstance(jstruct_schema, list):
-                all_schemas.extend(jstruct_schema)
-            else:
-                all_schemas.append(jstruct_schema)
+            entries = self.kusto_to_jstruct_schema(kusto_schema, table_name)
+            for entry in entries:
+                all_entries.append((table_name, entry[0], entry[1]))
         
         # Write output
         output_dir = os.path.dirname(self.jstruct_schema_path)
@@ -211,26 +224,105 @@ class KustoToJsonStructure:
             os.makedirs(output_dir)
             
         if self.emit_xregistry:
-            # Create xRegistry manifest
-            output = {
-                "$schema": "https://cloudevents.io/schemas/registry",
-                "specversion": "0.5-wip",
-                "endpoints": {},
-                "messagegroups": {},
-                "schemagroups": {
-                    f"{self.kusto_database}": {
-                        "schemas": {schema.get("$id", f"schema_{i}"): {"format": "JSONStructure/1.0", "schema": schema} 
-                                   for i, schema in enumerate(all_schemas)}
-                    }
-                }
-            }
+            output = self._build_xregistry_manifest(all_entries)
         else:
             # Single schema or array of schemas
+            all_schemas = [entry[2] for entry in all_entries]
             output = all_schemas if len(all_schemas) > 1 else all_schemas[0] if all_schemas else {}
         
         with open(self.jstruct_schema_path, 'w', encoding='utf-8') as jstruct_file:
             json.dump(output, jstruct_file, indent=4)
 
+    def _build_xregistry_manifest(self, all_entries: List[tuple]) -> dict:
+        """Builds an xRegistry manifest with messagegroups and schemagroups.
+        
+        Args:
+            all_entries: List of (table_name, type_value, schema) tuples.
+        """
+        messagegroups: Dict[str, JsonNode] = {}
+        schemagroups: Dict[str, JsonNode] = {}
+
+        # Group entries by table
+        tables: Dict[str, List[tuple]] = {}
+        for table_name, type_value, schema in all_entries:
+            if table_name not in tables:
+                tables[table_name] = []
+            tables[table_name].append((type_value, schema))
+
+        for table_name, entries in tables.items():
+            group_id = self._derive_group_id(table_name)
+            has_cloudevents = any(tv is not None for tv, _ in entries)
+
+            # Build schema group
+            schemas: Dict[str, JsonNode] = {}
+            for type_value, schema in entries:
+                schema_id = self._derive_schema_id(type_value, table_name)
+                schemas[schema_id] = {
+                    "format": "JSONStructure/1.0",
+                    "versions": {
+                        "1": {
+                            "format": "JSONStructure/1.0",
+                            "schema": schema
+                        }
+                    }
+                }
+            schemagroups[group_id] = {"schemas": schemas}
+
+            if has_cloudevents:
+                # Build message group with CloudEvents messages
+                messages: Dict[str, JsonNode] = {}
+                for type_value, schema in entries:
+                    if type_value is None:
+                        continue
+                    schema_id = self._derive_schema_id(type_value, table_name)
+                    messages[type_value] = {
+                        "description": f"CloudEvent of type {type_value}",
+                        "envelope": "CloudEvents/1.0",
+                        "envelopemetadata": {
+                            "id": {
+                                "required": True
+                            },
+                            "type": {
+                                "value": type_value
+                            },
+                            "source": {
+                                "required": True
+                            },
+                            "time": {
+                                "required": True
+                            }
+                        },
+                        "datacontenttype": "application/json",
+                        "dataschemaformat": "JSONStructure/1.0",
+                        "dataschemauri": f"#/schemagroups/{group_id}/schemas/{schema_id}"
+                    }
+                messagegroups[group_id] = {
+                    "envelope": "CloudEvents/1.0",
+                    "description": f"Events from {table_name}",
+                    "messages": messages
+                }
+            else:
+                # Non-CloudEvents table: add a simple message group entry
+                messages = {}
+                for type_value, schema in entries:
+                    schema_id = self._derive_schema_id(type_value, table_name)
+                    messages[schema_id] = {
+                        "description": f"Message from {table_name}",
+                        "dataschemaformat": "JSONStructure/1.0",
+                        "dataschemauri": f"#/schemagroups/{group_id}/schemas/{schema_id}"
+                    }
+                messagegroups[group_id] = {
+                    "description": f"Messages from {table_name}",
+                    "messages": messages
+                }
+
+        return {
+            "$schema": "https://cloudevents.io/schemas/registry",
+            "specversion": "0.5-wip",
+            "messagegroups": messagegroups,
+            "schemagroups": schemagroups
+        }
+
 
 def convert_kusto_to_jstruct(kusto_uri: str, kusto_database: str, table_name: str | None, base_id: str, jstruct_schema_file: str, emit_cloudevents: bool, emit_cloudevents_xregistry: bool, token_provider=None, sample_size: int = 100, infer_choices: bool = False, choice_depth: int = 1, infer_enums: bool = False):
     """ Converts Kusto table schemas to JSON Structure schema format."""

{structurize-3.5.3 → structurize-3.5.5}/avrotize/structuretopython.py

@@ -812,6 +812,32 @@ class StructureToPython:
         with open(os.path.join(self.output_dir, 'pyproject.toml'), 'w', encoding='utf-8') as file:
             file.write(pyproject_content)
 
+    def process_definitions(self, definitions: Dict, namespace_path: str) -> None:
+        """ Recursively walks the definitions tree and generates a Python type for each leaf.
+
+        Leaves are sub-schemas that carry an ``enum`` keyword or a ``type`` of
+        ``object``/``choice``/``map``. Other dict entries are treated as nested
+        namespace segments (matching the path-as-namespace convention used in
+        ``$root`` JSON pointers like ``#/definitions/de/wsv/pegelonline/Station``).
+        """
+        for name, definition in definitions.items():
+            if not isinstance(definition, dict):
+                continue
+            if 'enum' in definition:
+                self.generate_enum(definition, name, namespace_path, write_file=True)
+            elif definition.get('type') == 'object':
+                self.generate_class(definition, namespace_path, write_file=True, explicit_name=name)
+            elif definition.get('type') == 'choice':
+                self.generate_choice(definition, namespace_path, write_file=True, explicit_name=name)
+            elif definition.get('type') == 'map':
+                # generate_map_alias doesn't accept explicit_name; stamp the name on a copy.
+                map_schema = dict(definition)
+                map_schema.setdefault('name', name)
+                self.generate_map_alias(map_schema, namespace_path, write_file=True)
+            else:
+                new_namespace = f"{namespace_path}.{name}" if namespace_path else name
+                self.process_definitions(definition, new_namespace)
+
     def convert_schemas(self, structure_schemas: List, output_dir: str):
         """ Converts JSON Structure schemas to Python dataclasses"""
         self.output_dir = output_dir
@@ -827,15 +853,48 @@ class StructureToPython:
             if 'definitions' in structure_schema:
                 self.definitions = structure_schema['definitions']
 
+            handled = False
             if 'enum' in structure_schema:
-                self.generate_enum(structure_schema, structure_schema.get('name', 'Enum'), 
+                self.generate_enum(structure_schema, structure_schema.get('name', 'Enum'),
                                  structure_schema.get('namespace', ''), write_file=True)
+                handled = True
             elif structure_schema.get('type') == 'object':
                 self.generate_class(structure_schema, structure_schema.get('namespace', ''), write_file=True)
+                handled = True
             elif structure_schema.get('type') == 'choice':
                 self.generate_choice(structure_schema, structure_schema.get('namespace', ''), write_file=True)
+                handled = True
             elif structure_schema.get('type') == 'map':
                 self.generate_map_alias(structure_schema, structure_schema.get('namespace', ''), write_file=True)
+                handled = True
+            elif '$root' in structure_schema:
+                # $root + definitions wrapper: resolve the pointer, derive the namespace
+                # from the pointer path (segments between 'definitions' and the type name),
+                # and emit the targeted class explicitly. Sibling types in `definitions`
+                # are picked up below by process_definitions.
+                root_ref = structure_schema['$root']
+                root_schema = self.resolve_ref(root_ref, structure_schema)
+                if root_schema:
+                    ref_path = root_ref.split('/')
+                    type_name = ref_path[-1]
+                    ref_namespace = '.'.join(ref_path[2:-1]) if len(ref_path) > 3 else ''
+                    self.generate_class_or_choice(root_schema, ref_namespace, write_file=True, explicit_name=type_name)
+                    handled = True
+
+            # Recursively walk the definitions tree so every named type is emitted,
+            # regardless of whether the top-level schema also dispatched something.
+            if 'definitions' in structure_schema:
+                self.process_definitions(structure_schema['definitions'], '')
+                handled = True
+
+            if not handled:
+                schema_id = structure_schema.get('$id', '<no $id>')
+                schema_name = structure_schema.get('name', '<no name>')
+                print(
+                    f"Warning: structure schema (id={schema_id}, name={schema_name}) did not "
+                    "match any recognized top-level shape (enum, type=object/choice/map, "
+                    "$root, or definitions); no Python output generated for it."
+                )
 
         self.write_init_files()
         self.write_pyproject_toml()

{structurize-3.5.3 → structurize-3.5.5/structurize.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structurize
-Version: 3.5.3
+Version: 3.5.5
 Summary: Tools to convert from and to JSON Structure from various other schema languages.
 Author-email: Clemens Vasters <clemensv@microsoft.com>
 Classifier: Programming Language :: Python :: 3