structurize 3.5.11__tar.gz → 3.6.0__tar.gz

{structurize-3.5.11/structurize.egg-info → structurize-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structurize
-Version: 3.5.11
+Version: 3.6.0
 Summary: Tools to convert from and to JSON Structure from various other schema languages.
 Author-email: Clemens Vasters <clemensv@microsoft.com>
 Classifier: Programming Language :: Python :: 3

{structurize-3.5.11 → structurize-3.6.0}/avrotize/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '3.5.11'
-__version_tuple__ = version_tuple = (3, 5, 11)
+__version__ = version = '3.6.0'
+__version_tuple__ = version_tuple = (3, 6, 0)
 
-__commit_id__ = commit_id = 'g9e1128f35'
+__commit_id__ = commit_id = 'g1329b8c45'

{structurize-3.5.11 → structurize-3.6.0}/avrotize/avrotojsons.py

@@ -98,22 +98,34 @@ class AvroToJsonSchemaConverter:
                     json_type['type'] = 'integer'
                     json_type['format'] = 'int64'
                     return json_type
-                # Avrotize schema extensions: string-based logical types
-                elif base_type == 'string' and logical_type == 'date':
+                # Avrotize schema extensions: string-based logical types.
+                # The rfc3339-* family is the current emission; the bare reserved
+                # names on a string base are still recognized for backward compat.
+                elif base_type == 'string' and logical_type in ['date', 'rfc3339-date']:
                     # Avrotize extension: string with date logicalType
                     json_type['type'] = 'string'
                     json_type['format'] = 'date'
                     return json_type
-                elif base_type == 'string' and logical_type in ['timestamp-millis', 'timestamp-micros', 'datetime']:
+                elif base_type == 'string' and logical_type in [
+                        'timestamp-millis', 'timestamp-micros', 'datetime',
+                        'rfc3339-timestamp-millis', 'rfc3339-timestamp-micros',
+                        'rfc3339-local-timestamp-millis', 'rfc3339-local-timestamp-micros']:
                     # Avrotize extension: string with datetime logicalType
                     json_type['type'] = 'string'
                     json_type['format'] = 'date-time'
                     return json_type
-                elif base_type == 'string' and logical_type in ['time-millis', 'time-micros', 'time']:
+                elif base_type == 'string' and logical_type in [
+                        'time-millis', 'time-micros', 'time',
+                        'rfc3339-time-millis', 'rfc3339-time-micros']:
                     # Avrotize extension: string with time logicalType
                     json_type['type'] = 'string'
                     json_type['format'] = 'time'
                     return json_type
+                elif base_type == 'string' and logical_type in ['duration', 'rfc3339-duration']:
+                    # Avrotize extension: string with duration logicalType (RFC 3339)
+                    json_type['type'] = 'string'
+                    json_type['format'] = 'duration'
+                    return json_type
                 elif logical_type == 'decimal':
                     json_type['type'] = 'number'
                     return json_type

{structurize-3.5.11 → structurize-3.6.0}/avrotize/avrotojstruct.py

@@ -305,6 +305,16 @@ class AvroToJsonStructure:
             "timestamp-millis": {"type": "int64", "logicalType": "timestampMillis"},
             "date": {"type": "int32", "logicalType": "date"},
             "uuid": {"type": "string", "format": "uuid"},
+            # Avrotize Schema rfc3339-* string temporal family -> JSON Structure native
+            # temporal types (round-trips with jstructtoavro). See issue #335.
+            "rfc3339-date": {"type": "date"},
+            "rfc3339-time-millis": {"type": "time"},
+            "rfc3339-time-micros": {"type": "time"},
+            "rfc3339-timestamp-millis": {"type": "datetime"},
+            "rfc3339-timestamp-micros": {"type": "datetime"},
+            "rfc3339-local-timestamp-millis": {"type": "datetime"},
+            "rfc3339-local-timestamp-micros": {"type": "datetime"},
+            "rfc3339-duration": {"type": "duration"},
         }
         return mapping.get(logical_type, {"type": "string"})
 

{structurize-3.5.11 → structurize-3.6.0}/avrotize/avrotokusto.py

@@ -7,6 +7,16 @@ from avrotize.common import build_flat_type_dict, inline_avro_references, strip_
 from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
 
 
+def kusto_string_literal(value: str) -> str:
+    """Return a KQL verbatim string literal for a raw Python string."""
+    return '@"' + value.replace('"', '""') + '"'
+
+
+def kusto_json_literal(value: Any) -> str:
+    """Return a KQL string literal containing one JSON encoding of value."""
+    return kusto_string_literal(json.dumps(value))
+
+
 class AvroToKusto:
     """Converts an Avro schema to a Kusto table schema."""
 
@@ -60,9 +70,9 @@ class AvroToKusto:
         if "doc" in recordschema:
             doc_data = recordschema["doc"]
             doc_data = (doc_data[:997] + "...") if len(doc_data) > 1000 else doc_data
-            doc_string = json.dumps(json.dumps({
+            doc_string = kusto_json_literal({
                 "description": doc_data
-            }))
+            })
             kusto.append(
                 f".alter table {table_ref} docstring {doc_string};")
             kusto.append("")
@@ -89,7 +99,7 @@ class AvroToKusto:
                             doc_content["schema"] = inline_schema
                     else:
                         doc_content["schema"] = inline_schema
-                doc = json.dumps(json.dumps(doc_content))
+                doc = kusto_json_literal(doc_content)
                 doc_string_statement.append(f"   [{column_name}]: {doc}")
         if doc_string_statement and emit_cloudevents_columns:
             doc_string_statement.extend([
@@ -109,15 +119,13 @@ class AvroToKusto:
         # .create-or-alter table dfl_data_events ingestion json mapping
         kusto.append(
             f".create-or-alter table {table_ref} ingestion json mapping \"{mapping_base}_json_flat\"")
-        kusto.append("```\n[")
+        mapping_entries = []
         if emit_cloudevents_columns:
-            kusto.append("  {\"column\": \"___type\", \"path\": \"$.type\"},")
-            kusto.append(
-                "  {\"column\": \"___source\", \"path\": \"$.source\"},")
-            kusto.append("  {\"column\": \"___id\", \"path\": \"$.id\"},")
-            kusto.append("  {\"column\": \"___time\", \"path\": \"$.time\"},")
-            kusto.append(
-                "  {\"column\": \"___subject\", \"path\": \"$.subject\"},")
+            mapping_entries.append("  {\"column\": \"___type\", \"path\": \"$.type\"}")
+            mapping_entries.append("  {\"column\": \"___source\", \"path\": \"$.source\"}")
+            mapping_entries.append("  {\"column\": \"___id\", \"path\": \"$.id\"}")
+            mapping_entries.append("  {\"column\": \"___time\", \"path\": \"$.time\"}")
+            mapping_entries.append("  {\"column\": \"___subject\", \"path\": \"$.subject\"}")
         for field in fields:
             json_name = column_name = field["name"]
             if 'altnames' in field:
@@ -125,21 +133,19 @@ class AvroToKusto:
                     column_name = field['altnames']['kql']
                 if 'json' in field['altnames']:
                     json_name = field['altnames']['json']
-            kusto.append(
-                f"  {{\"column\": \"{column_name}\", \"path\": \"$.{json_name}\"}},")
-        kusto.append("]\n```\n\n")
+            mapping_entries.append(
+                f"  {{\"column\": \"{column_name}\", \"path\": \"$.{json_name}\"}}")
+        kusto.append("```\n[\n" + ",\n".join(mapping_entries) + "\n]\n```\n\n")
 
         if emit_cloudevents_columns:
             kusto.append(
                 f".create-or-alter table {table_ref} ingestion json mapping \"{mapping_base}_json_ce_structured\"")
-            kusto.append("```\n[")
-            kusto.append("  {\"column\": \"___type\", \"path\": \"$.type\"},")
-            kusto.append(
-                "  {\"column\": \"___source\", \"path\": \"$.source\"},")
-            kusto.append("  {\"column\": \"___id\", \"path\": \"$.id\"},")
-            kusto.append("  {\"column\": \"___time\", \"path\": \"$.time\"},")
-            kusto.append(
-                "  {\"column\": \"___subject\", \"path\": \"$.subject\"},")
+            ce_entries = []
+            ce_entries.append("  {\"column\": \"___type\", \"path\": \"$.type\"}")
+            ce_entries.append("  {\"column\": \"___source\", \"path\": \"$.source\"}")
+            ce_entries.append("  {\"column\": \"___id\", \"path\": \"$.id\"}")
+            ce_entries.append("  {\"column\": \"___time\", \"path\": \"$.time\"}")
+            ce_entries.append("  {\"column\": \"___subject\", \"path\": \"$.subject\"}")
             for field in fields:
                 json_name = column_name = field["name"]
                 if 'altnames' in field:
@@ -147,9 +153,9 @@ class AvroToKusto:
                         column_name = field['altnames']['kql']
                     if 'json' in field['altnames']:
                         json_name = field['altnames']['json']
-                kusto.append(
-                    f"  {{\"column\": \"{column_name}\", \"path\": \"$.data.{json_name}\"}},")
-            kusto.append("]\n```\n\n")
+                ce_entries.append(
+                    f"  {{\"column\": \"{column_name}\", \"path\": \"$.data.{json_name}\"}}")
+            kusto.append("```\n[\n" + ",\n".join(ce_entries) + "\n]\n```\n\n")
 
         if emit_cloudevents_columns:
             kusto.append(
@@ -186,7 +192,7 @@ class AvroToKusto:
             kusto.append(
                 f"  \"Query\": \"{query}\",")
             kusto.append("  \"IsTransactional\": false,")
-            kusto.append("  \"PropagateIngestionProperties\": true,")
+            kusto.append("  \"PropagateIngestionProperties\": true")
             kusto.append("}]")
             kusto.append("```\n")
 

{structurize-3.5.11 → structurize-3.6.0}/avrotize/avrotopython.py

@@ -90,6 +90,38 @@ class AvroToPython:
             return name + "_"
         return name
 
+    def safe_enum_symbols(self, symbols: List[str]) -> List[Dict[str, str]]:
+        """Converts Avro enum symbols to collision-safe Python enum members."""
+        enum_symbols = []
+        used_names = set()
+
+        for symbol in symbols:
+            if symbol.isidentifier() and not is_python_reserved_word(symbol):
+                member_name = symbol
+            else:
+                member_name = re.sub(r'[^0-9A-Za-z_]', '_', symbol).strip('_').upper()
+                if not member_name:
+                    member_name = "VALUE"
+                if member_name[0].isdigit():
+                    member_name = f"VALUE_{member_name}"
+                if is_python_reserved_word(symbol) or is_python_reserved_word(member_name.lower()) or is_python_reserved_word(member_name):
+                    member_name = f"{member_name}_"
+
+            base_member_name = member_name
+            suffix = 2
+            while member_name in used_names or not member_name.isidentifier() or is_python_reserved_word(member_name):
+                member_name = f"{base_member_name}_{suffix}"
+                suffix += 1
+
+            used_names.add(member_name)
+            enum_symbols.append({
+                'name': member_name,
+                'value': symbol,
+                'value_literal': repr(symbol)
+            })
+
+        return enum_symbols
+
     def pascal_type_name(self, ref: str) -> str:
         """Converts a reference to a type name"""
         return '_'.join([pascal(part) for part in ref.split('.')[-1].split('_')])
@@ -350,9 +382,12 @@ class AvroToPython:
         if python_qualified_name in self.generated_types:
             return python_qualified_name
 
-        symbols = [symbol if not is_python_reserved_word(
-            symbol) else symbol + "_" for symbol in avro_schema.get('symbols', [])]
-        ordinals =  avro_schema.get('ordinals', {})
+        symbols = self.safe_enum_symbols(avro_schema.get('symbols', []))
+        symbol_names_by_value = {symbol['value']: symbol['name'] for symbol in symbols}
+        ordinals = {
+            symbol_names_by_value[symbol]: ordinal
+            for symbol, ordinal in avro_schema.get('ordinals', {}).items()
+        }
 
         enum_definition = process_template(
             "avrotopython/enum_core.jinja",

{structurize-3.5.11 → structurize-3.6.0}/avrotize/avrotots.py

@@ -16,7 +16,6 @@ def is_typescript_reserved_word(word: str) -> bool:
         'for', 'function', 'if', 'import', 'in', 'instanceof', 'new', 'return',
         'super', 'switch', 'this', 'throw', 'try', 'typeof', 'var', 'void',
         'while', 'with', 'yield', 'enum', 'string', 'number', 'boolean', 'symbol',
-        'type', 'namespace', 'module', 'declare', 'abstract', 'readonly',
     ]
     return word in reserved_words
 
@@ -32,6 +31,7 @@ class AvroToTypeScript:
         self.src_dir = os.path.join(self.output_dir, "src")
         self.generated_types: Dict[str, str] = {}
         self.generated_type_fields: Dict[str, List[Dict]] = {}  # Store fields for test generation
+        self.generated_type_imports: Dict[str, List[str]] = {}  # Store import type FQNs for test generation
         self.main_schema = None
         self.type_dict = None
         self.INDENT = ' ' * 4
@@ -207,6 +207,10 @@ class AvroToTypeScript:
             self.write_to_file(namespace, class_name, class_definition)
         self.generated_types[ts_qualified_name] = 'class'
         self.generated_type_fields[ts_qualified_name] = fields  # Store fields for test generation
+        # Store the fully-qualified import names so generated test files can resolve
+        # sub-namespaced types (e.g. enums placed in a "<Record>_types" namespace)
+        # to their real source path instead of a flattened, non-existent one.
+        self.generated_type_imports[ts_qualified_name] = sorted(import_types)
         return ts_qualified_name
 
     def generate_enum(self, avro_schema: Dict, parent_namespace: str, write_file: bool = True) -> str:
@@ -670,17 +674,29 @@ class AvroToTypeScript:
         
         fields = self.generated_type_fields.get(qualified_name, [])
         
-        # Build imports for nested types
+        # Build imports for nested types. Reuse the fully-qualified import names
+        # captured during class generation so that types living in sub-namespaces
+        # (e.g. enums placed in a "<Record>_types" namespace) resolve to their real
+        # source path instead of a flattened, non-existent one (issue #338).
         imports_with_paths: Dict[str, str] = {}
+        import_fqns = self.generated_type_imports.get(qualified_name, [])
+        path_by_leaf: Dict[str, str] = {}
+        for fqn in import_fqns:
+            fqn_parts = fqn.split('.')
+            leaf = pascal(fqn_parts[-1])
+            path_by_leaf[leaf] = '../src/' + '/'.join(fqn_parts) + '.js'
         for field in fields:
             field_type = field.get('type_no_null', '')
             if not self.is_typescript_primitive(field_type.replace('[]', '')):
                 # It's a reference type, need to import it
                 type_name = field_type.replace('[]', '').replace('?', '')
                 if type_name and type_name not in ['null', 'any', 'Date']:
-                    # Build relative path from test dir to src dir
-                    src_path = '/'.join(parts)
-                    imports_with_paths[type_name] = f'../src/{src_path.rsplit("/", 1)[0]}/{type_name}.js' if '/' in src_path else f'../src/{parts[0]}/{type_name}.js'
+                    if type_name in path_by_leaf:
+                        imports_with_paths[type_name] = path_by_leaf[type_name]
+                    else:
+                        # Fallback: best-effort path relative to the class namespace
+                        src_path = '/'.join(parts)
+                        imports_with_paths[type_name] = f'../src/{src_path.rsplit("/", 1)[0]}/{type_name}.js' if '/' in src_path else f'../src/{parts[0]}/{type_name}.js'
         
         # Calculate relative path from test directory to class file
         class_path_parts = namespace.split('.') if namespace else []

{structurize-3.5.11 → structurize-3.6.0}/avrotize/avrovalidator.py

@@ -414,6 +414,33 @@ class AvroValidator:
         """
         base_type = schema.get('type')
 
+        # Avrotize Schema rfc3339-* string temporal family (issue #335). These are
+        # valid standard Avro because rfc3339-* is not a reserved Avro logical type;
+        # the value is an RFC 3339 textual string on a string base.
+        if isinstance(logical_type, str) and logical_type.startswith('rfc3339-'):
+            if base_type != 'string':
+                raise AvroValidationError(
+                    f"{logical_type} logical type requires string base type", path)
+            if not isinstance(instance, str):
+                raise AvroValidationError(
+                    f"Expected string for {logical_type}, got {type(instance).__name__}", path)
+            core = logical_type[len('rfc3339-'):]
+            if core == 'date':
+                pattern = self.DATE_PATTERN
+            elif core in ('time-millis', 'time-micros'):
+                pattern = self.TIME_PATTERN
+            elif core in ('timestamp-millis', 'timestamp-micros',
+                          'local-timestamp-millis', 'local-timestamp-micros'):
+                pattern = self.DATETIME_PATTERN
+            elif core == 'duration':
+                pattern = self.DURATION_PATTERN
+            else:
+                pattern = None
+            if pattern is not None and not pattern.match(instance):
+                raise AvroValidationError(
+                    f"Invalid {logical_type} format: {instance}", path)
+            return
+
         if logical_type == 'decimal':
             self._validate_decimal(instance, schema, path)
 

{structurize-3.5.11 → structurize-3.6.0}/avrotize/common.py

@@ -830,6 +830,52 @@ def altname(schema_obj: dict, purpose: str):
     return schema_obj["name"]
 
 
+def json_wire_name(prop_name: str, prop_schema: Any) -> str:
+    """
+    Resolve the JSON wire key for a property, honoring JSON Structure ``altnames.json``.
+
+    JSON Structure requires property keys to be identifiers. A non-identifier wire key
+    (e.g. ``dr-type``) is modeled as an identifier property name (``dr_type``) carrying
+    ``altnames: {"json": "dr-type"}``. The ``json`` purpose is the canonical wire name.
+
+    Args:
+        prop_name (str): The property name (identifier) used as the schema key.
+        prop_schema (Any): The property schema; may carry an ``altnames`` map.
+
+    Returns:
+        str: ``altnames.json`` when present, otherwise ``prop_name``.
+    """
+    if isinstance(prop_schema, dict):
+        altnames = prop_schema.get("altnames")
+        if isinstance(altnames, dict) and "json" in altnames:
+            return altnames["json"]
+    return prop_name
+
+
+def json_enum_wire_value(value: Any, enum_schema: Any) -> str:
+    """
+    Resolve the JSON wire string for an enum value, honoring JSON Structure ``altenums.json``.
+
+    ``altenums.json`` is a map keyed by the original enum value; the language member name
+    stays derived from the original value, while the wire value uses the mapping when present.
+    Unmapped values serialize verbatim (identity fallback).
+
+    Args:
+        value: The original enum value (schema key).
+        enum_schema (Any): The enum schema; may carry an ``altenums`` map.
+
+    Returns:
+        str: ``altenums.json[value]`` when present, otherwise ``str(value)``.
+    """
+    if isinstance(enum_schema, dict):
+        altenums = enum_schema.get("altenums")
+        if isinstance(altenums, dict):
+            j = altenums.get("json")
+            if isinstance(j, dict) and str(value) in j:
+                return j[str(value)]
+    return str(value)
+
+
 def process_template(file_path: str, **kvargs) -> str:
     """
     Process a file as a Jinja2 template with the given object as input.

{structurize-3.5.11 → structurize-3.6.0}/avrotize/dependencies/typescript/node22/package.json

@@ -8,7 +8,7 @@
     "kafkajs": "^2.2.4"
   },
   "devDependencies": {
-    "@types/node": "^25.0.3",
+    "@types/node": "^26.0.0",
     "typescript": "^6.0.2",
     "jest": "^30.2.0",
     "@types/jest": "^30.0.0"

{structurize-3.5.11 → structurize-3.6.0}/avrotize/jsonstoavro.py

@@ -591,14 +591,16 @@ class JsonToAvroConverter:
 
         # if you've got { 'type': 'string', 'format': ['date-time', 'duration'] }, I'm sorry
         if format and isinstance(format, str):
-            if format in ('date-time', 'date'):
+            if format == 'date-time':
+                avro_primitive = {'type': 'long', 'logicalType': 'timestamp-millis'}
+            elif format == 'date':
                 avro_primitive = {'type': 'int', 'logicalType': 'date'}
-            elif format in ('time'):
+            elif format == 'time':
                 avro_primitive = {'type': 'int', 'logicalType': 'time-millis'}
-            elif format in ('duration'):
+            elif format == 'duration':
                 avro_primitive = {'type': 'fixed',
                                   'size': 12, 'logicalType': 'duration'}
-            elif format in ('uuid'):
+            elif format == 'uuid':
                 avro_primitive = {'type': 'string', 'logicalType': 'uuid'}
 
         return avro_primitive
@@ -1357,7 +1359,14 @@ class JsonToAvroConverter:
         if isinstance(avro_type, dict) and 'type' in avro_type and (isinstance(avro_type, list) or not avro_type['type'] in ['array', 'map', 'record', 'enum', 'fixed']):
             if 'dependencies' in avro_type:
                 dependencies.extend(avro_type['dependencies'])
-            avro_type = avro_type['type']
+            # Preserve logical-type annotations (e.g. timestamp-millis, date,
+            # time-millis, uuid). Collapsing these to the bare base type would
+            # silently discard the logical meaning (and, for date-time, the
+            # required 64-bit width). See issue #337.
+            if 'logicalType' in avro_type:
+                avro_type = {k: v for k, v in avro_type.items() if k != 'dependencies'}
+            else:
+                avro_type = avro_type['type']
         return avro_type
 
     def register_type(self, avro_schema, avro_type) -> bool:
@@ -1733,8 +1742,9 @@ class JsonToAvroConverter:
                             # None type is a problem
                             raise ValueError(
                                 f"avro_field_type is None for field {field_name}")
-                        if isinstance(avro_field_type, dict) and 'type' in avro_field_type and not self.is_avro_complex_type(avro_field_type):
+                        if isinstance(avro_field_type, dict) and 'type' in avro_field_type and 'logicalType' not in avro_field_type and not self.is_avro_complex_type(avro_field_type):
                             # if the field type is a basic type, inline it
+                            # (logical types are kept intact, see issue #337)
                             avro_field_type = avro_field_type['type']
                         field_type_list.append(avro_field_type)
                         field_ref_type_list.append(avro_field_ref_type)