structurize 2.19.0__py3-none-any.whl

avrotize/__init__.py

@@ -0,0 +1,64 @@
+import importlib
+
+mod = "avrotize"
+class LazyLoader:
+    """    
+    Lazy loader for the avrotize functions to speed up startup time.    
+    """
+    def __init__(self, mappings):
+        self._modules = {}
+        self._mappings = mappings
+
+    def _load_module(self, module_name):
+        if module_name not in self._modules:
+            self._modules[module_name] = importlib.import_module(module_name)
+        return self._modules[module_name]
+
+    def __getattr__(self, item):
+        if item in self._mappings:
+            module_name, func_name = self._mappings[item]
+            module = self._load_module(module_name)
+            return getattr(module, func_name)
+        else:
+            return self._load_module(f"{mod}.{item}")
+
+# Define the functions and their corresponding module paths
+_mappings = {
+    "convert_proto_to_avro": (f"{mod}.prototoavro", "convert_proto_to_avro"),
+    "convert_jsons_to_avro": (f"{mod}.jsonstoavro", "convert_jsons_to_avro"),
+    "convert_kafka_struct_to_avro_schema": (f"{mod}.kstructtoavro", "convert_kafka_struct_to_avro_schema"),
+    "convert_kusto_to_avro": (f"{mod}.kustotoavro", "convert_kusto_to_avro"),
+    "convert_asn1_to_avro": (f"{mod}.asn1toavro", "convert_asn1_to_avro"),
+    "convert_xsd_to_avro": (f"{mod}.xsdtoavro", "convert_xsd_to_avro"),
+    "convert_avro_to_json_schema": (f"{mod}.avrotojsons", "convert_avro_to_json_schema"),
+    "convert_avro_to_kusto_file": (f"{mod}.avrotokusto", "convert_avro_to_kusto_file"),
+    "convert_avro_to_kusto_db": (f"{mod}.avrotokusto", "convert_avro_to_kusto_db"),
+    "convert_avro_to_parquet": (f"{mod}.avrotoparquet", "convert_avro_to_parquet"),
+    "convert_avro_to_proto": (f"{mod}.avrotoproto", "convert_avro_to_proto"),
+    "convert_avro_to_sql": (f"{mod}.avrotodb", "convert_avro_to_sql"),
+    "convert_avro_to_xsd": (f"{mod}.avrotoxsd", "convert_avro_to_xsd"),
+    "convert_avro_to_java": (f"{mod}.avrotojava", "convert_avro_to_java"),
+    "convert_avro_schema_to_java": (f"{mod}.avrotojava", "convert_avro_schema_to_java"),
+    "convert_avro_to_csharp": (f"{mod}.avrotocsharp", "convert_avro_to_csharp"),
+    "convert_avro_schema_to_csharp": (f"{mod}.avrotocsharp", "convert_avro_schema_to_csharp"),
+    "convert_avro_to_python": (f"{mod}.avrotopython", "convert_avro_to_python"),
+    "convert_avro_schema_to_python": (f"{mod}.avrotopython", "convert_avro_schema_to_python"),
+    "convert_avro_to_typescript": (f"{mod}.avrotots", "convert_avro_to_typescript"),
+    "convert_avro_schema_to_typescript": (f"{mod}.avrotots", "convert_avro_schema_to_typescript"),
+    "convert_avro_to_javascript": (f"{mod}.avrotojs", "convert_avro_to_javascript"),
+    "convert_avro_schema_to_javascript": (f"{mod}.avrotojs", "convert_avro_schema_to_javascript"),
+    "convert_avro_to_markdown": (f"{mod}.avrotomd", "convert_avro_to_markdown"),
+    "convert_structure_to_markdown": (f"{mod}.structuretomd", "convert_structure_to_markdown"),
+    "convert_avro_to_cpp": (f"{mod}.avrotocpp", "convert_avro_to_cpp"),
+    "convert_avro_schema_to_cpp": (f"{mod}.avrotocpp", "convert_avro_schema_to_cpp"),
+    "convert_avro_to_go": (f"{mod}.avrotogo", "convert_avro_to_go"),
+    "convert_avro_schema_to_go": (f"{mod}.avrotogo", "convert_avro_schema_to_go"),
+    "convert_avro_to_rust": (f"{mod}.avrotorust", "convert_avro_to_rust"),
+    "convert_avro_schema_to_rust": (f"{mod}.avrotorust", "convert_avro_schema_to_rust"),
+    "convert_avro_to_datapackage": (f"{mod}.avrotodatapackage", "convert_avro_to_datapackage"),
+}
+
+_lazy_loader = LazyLoader(_mappings)
+
+def __getattr__(name):
+    return getattr(_lazy_loader, name)

avrotize/__main__.py

@@ -0,0 +1,6 @@
+# __main__.py 
+
+from . import avrotize
+
+if __name__ == '__main__':
+      avrotize.main()

avrotize/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '2.19.0'
+__version_tuple__ = version_tuple = (2, 19, 0)
+
+__commit_id__ = commit_id = 'geec8f9840'

avrotize/asn1toavro.py

@@ -0,0 +1,160 @@
+import json
+from typing import List
+import asn1tools
+from asn1tools.codecs.ber import Sequence, SequenceOf, Integer, Boolean, Enumerated, OctetString, IA5String, UTF8String, Date, Real, Choice, Null, SetOf, Recursive, ExplicitTag
+from asn1tools.codecs.ber import BitString, VisibleString, BMPString, GeneralString, GraphicString, NumericString, PrintableString, TeletexString, UniversalString, ObjectIdentifier, Set
+
+from avrotize.common import avro_name
+from avrotize.dependency_resolver import sort_messages_by_dependencies
+
+def asn1_type_to_avro_type(asn1_type: dict, avro_schema: list, namespace: str, parent_type_name: str | None, parent_member_name: str | None, dependencies: list)-> str | dict | None:
+    """Convert an ASN.1 type to an Avro type."""
+    avro_type: str | dict | list | None = None
+    deps: List[str] = []
+
+    if isinstance(asn1_type,Integer):
+        avro_type = 'int'
+    elif isinstance(asn1_type, Boolean):
+        avro_type = 'boolean'
+    elif isinstance(asn1_type, Enumerated):
+        symbols = [avro_name(member) for member in asn1_type.data_to_value.keys()]
+        avro_type = { 
+            'type': 'enum', 
+            'name': avro_name(parent_member_name if parent_member_name else asn1_type.name),
+            'namespace': namespace, 
+            'symbols': symbols
+            }
+    elif isinstance(asn1_type, Sequence) or isinstance(asn1_type, Choice) or isinstance(asn1_type, Set):
+        if avro_schema and next((s for s in avro_schema if s.get('name') == asn1_type.type_name), []):
+            return namespace + '.' + asn1_type.type_name
+        else:
+            record_name = asn1_type.type_name if asn1_type.type_name else asn1_type.name
+            if record_name == 'CHOICE' or record_name == 'SEQUENCE' or record_name == 'SET':
+                if parent_member_name and parent_type_name:
+                    record_name = parent_type_name + parent_member_name         
+                elif parent_type_name:
+                    record_name = parent_type_name 
+                else:
+                    raise ValueError(f"Can't name record without a type name and member name")      
+
+            fields = []
+            for member in asn1_type.members if isinstance(asn1_type, Choice) else asn1_type.root_members:
+                field_type = asn1_type_to_avro_type(member, avro_schema, namespace, record_name, member.name, deps)
+                if isinstance(field_type, dict) and (field_type.get('type') == 'record' or field_type.get('type') == 'enum'):
+                    existing_type = next((t for t in avro_schema if (isinstance(t,dict) and t.get('name') == field_type['name'] and t.get('namespace') == field_type.get('namespace')) ), None)
+                    if not existing_type:
+                        field_type['dependencies'] = [dep for dep in deps if dep != field_type['namespace']+'.'+field_type['name']]
+                        avro_schema.append(field_type)
+                    field_type = namespace + '.' + field_type.get('name','')
+                    dependencies.append(field_type)
+                if isinstance(asn1_type, Choice):
+                    field_type = [field_type, 'null']
+                fields.append({
+                    'name': avro_name(member.name),
+                    'type': field_type
+                })
+            
+            avro_type = {
+                'type': 'record', 
+                'name': avro_name(record_name),
+                'namespace': namespace,
+                'fields': fields
+                }
+    elif isinstance(asn1_type, BitString):
+        avro_type = {
+            'type': 'bytes',
+            'logicalType': 'bitstring'
+        }
+    elif isinstance(asn1_type, ObjectIdentifier):
+        avro_type = 'string'
+    elif isinstance(asn1_type, SequenceOf):
+        record_name = asn1_type.name if asn1_type.name else parent_member_name
+        if record_name == 'SEQUENCE OF':
+            record_name = parent_member_name if parent_member_name else ''
+        if parent_type_name:
+            record_name = parent_type_name + record_name
+        item_type = asn1_type_to_avro_type(asn1_type.element_type, avro_schema, namespace, record_name, 'Item', deps)
+        if isinstance(item_type, dict) and not item_type.get('name'):
+            item_type['name'] = asn1_type.name + 'Item'
+        avro_type = {
+            'type': 'array',
+            'namespace': namespace, 
+            'name': avro_name(record_name), 
+            'items': item_type
+            }
+    elif isinstance(asn1_type, SetOf):
+        record_name = asn1_type.name if asn1_type.name else parent_member_name
+        if record_name == 'SET OF':
+            record_name = parent_member_name if parent_member_name else ''                
+        if parent_type_name:
+            record_name = parent_type_name + record_name
+        item_type = asn1_type_to_avro_type(asn1_type.element_type, avro_schema, namespace, record_name, 'Item', deps)
+        if isinstance(item_type, dict) and not item_type.get('name'):
+            item_type['name'] = asn1_type.name + 'Item'               
+        avro_type = {
+            'type': 'array',
+            'namespace': namespace, 
+            'name': avro_name(record_name),
+            'items': item_type
+            }
+    elif isinstance(asn1_type, OctetString):
+        avro_type = 'bytes'
+    elif isinstance(asn1_type, IA5String) or isinstance(asn1_type, UTF8String) or isinstance(asn1_type, VisibleString) or isinstance(asn1_type, BMPString) or isinstance(asn1_type, GeneralString) or isinstance(asn1_type, GraphicString) or isinstance(asn1_type, NumericString) or isinstance(asn1_type, PrintableString) or isinstance(asn1_type, TeletexString) or isinstance(asn1_type, UniversalString):
+        avro_type = 'string'
+    elif isinstance(asn1_type, Date):
+        avro_type = {'type': 'int', 'logicalType': 'date'}
+    elif isinstance(asn1_type, Real):
+        avro_type = 'double'
+    elif isinstance(asn1_type, Null):
+        avro_type = 'null'
+    elif isinstance(asn1_type, Recursive):
+        avro_type = asn1_type.type_name
+    elif isinstance(asn1_type, ExplicitTag):
+        avro_type = asn1_type_to_avro_type(asn1_type.inner, avro_schema, namespace, parent_type_name, parent_member_name, dependencies)
+        if isinstance(avro_type, dict):
+            avro_type['name'] = asn1_type.name if asn1_type.name else asn1_type.type_name
+    else:
+        raise ValueError(f"Don't know how to translate ASN.1 type '{type(asn1_type)}' to Avro")
+
+    if len(avro_schema) > 0 and isinstance(avro_type, dict) and 'name' in avro_type:
+        existing_type = next((t for t in avro_schema if (isinstance(t,dict) and t.get('name') == avro_type['name'] and t.get('namespace') == avro_type.get('namespace')) ), None)
+        if existing_type:
+            qualified_name = namespace + '.' + existing_type.get('name','')
+            dependencies.append(qualified_name)
+            return qualified_name
+            
+    return avro_type
+
+def convert_asn1_to_avro_schema(asn1_spec_list: List[str]):
+    """Convert ASN.1 specification to Avro schema."""
+    
+    try:
+        spec = asn1tools.compile_files(asn1_spec_list)
+    except asn1tools.ParseError as e:
+        print(f"Error parsing ASN.1 files: {e.args[0]}")
+        for file in asn1_spec_list:
+            print(f"  {file}")
+        return None
+    
+    avro_schema: List[dict] = []
+    for module_name, module in spec.modules.items():
+        for type_name, asn1_type in module.items():
+            dependencies: List [str] = []
+            avro_type = asn1_type_to_avro_type(asn1_type.type, avro_schema, avro_name(module_name), type_name, None, dependencies)
+            if avro_type and not isinstance(avro_type, str):
+                avro_type['dependencies'] = [dep for dep in dependencies if dep != avro_type['namespace'] + '.' + avro_type['name']]
+                avro_schema.append(avro_type)
+    
+    avro_schema = sort_messages_by_dependencies(avro_schema)
+
+    if len(avro_schema) == 1:
+        return avro_schema[0]
+    return avro_schema
+
+def convert_asn1_to_avro(asn1_spec_list: List[str], avro_file_path: str):
+    """Convert ASN.1 specification to Avro schema and save it to a file."""
+    avro_schema = convert_asn1_to_avro_schema(asn1_spec_list)
+    if avro_schema is None:
+        return
+    with open(avro_file_path, 'w') as file:
+        json.dump(avro_schema, file, indent=4)

avrotize/avrotize.py

@@ -0,0 +1,152 @@
+"""
+
+Command line utility to convert a variety of schema formats to Avrotize schema and vice versa.
+
+"""
+
+
+import argparse
+import tempfile
+import sys
+import os
+import json
+try:
+    from avrotize import _version
+    VERSION = _version.version
+except ImportError:
+    VERSION = "dev"
+
+def load_commands():
+    """Load the commands from the commands.json file."""
+    commands_path = os.path.join(os.path.dirname(__file__), 'commands.json')
+    with open(commands_path, 'r', encoding='utf-8') as f:
+        return json.load(f)
+
+def create_subparsers(subparsers, commands):
+    """Create subparsers for the commands."""
+    for command in commands:
+        cmd_parser = subparsers.add_parser(command['command'], help=command['description'])
+        for arg in command['args']:
+            kwargs = {
+                'type': eval(arg['type']),
+                'help': arg['help'],
+            }
+            
+            if 'nargs' in arg:
+                kwargs['nargs'] = arg['nargs']
+            if 'choices' in arg:
+                kwargs['choices'] = arg['choices']
+            if 'default' in arg:
+                kwargs['default'] = arg['default']
+            if arg['type'] == 'bool':
+                kwargs['action'] = 'store_true'
+                del kwargs['type']
+                argname = arg['name']
+                if '_' in argname:
+                    argname2 = argname.replace('_', '-')
+                    carg = cmd_parser.add_argument(argname, argname2, **kwargs)
+                else:
+                    carg = cmd_parser.add_argument(arg['name'], **kwargs)
+            else:
+                if '_' in arg['name']:
+                    argname2 = arg['name'].replace('_', '-')
+                    carg = cmd_parser.add_argument(arg['name'], argname2, **kwargs)
+                else:
+                    carg = cmd_parser.add_argument(arg['name'], **kwargs)
+            carg.required = arg.get('required', True)
+
+def dynamic_import(module, func):
+    """Dynamically import a module and function."""
+    mod = __import__(module, fromlist=[func])
+    return getattr(mod, func)
+
+def main():
+    """Main function for the command line utility."""
+    commands = load_commands()
+    parser = argparse.ArgumentParser(description='Convert a variety of schema formats to Avrotize schema and vice versa.')
+    parser.add_argument('--version', action='store_true', help='Print the version of Avrotize.')
+
+    subparsers = parser.add_subparsers(dest='command')
+    create_subparsers(subparsers, commands)
+
+    args = parser.parse_args()
+
+    if 'version' in args and args.version:
+        print(f'Avrotize {VERSION}')
+        return
+    
+    if args.command is None:
+        parser.print_help()
+        return
+    
+    try:
+        command = next((cmd for cmd in commands if cmd['command'] == args.command), None)
+        if not command:
+            print(f"Error: Command {args.command} not found.")
+            exit(1)
+        
+        input_file_path = args.input or getattr(args, 'avsc', None) or getattr(args, 'proto', None) or getattr(args, 'jsons', None) or getattr(args, 'xsd', None) or getattr(args, 'kusto_uri', None) or getattr(args, 'parquet', None) or getattr(args, 'asn', None) or getattr(args, 'kstruct', None)
+        temp_input = None
+        skip_input_file_handling = command.get('skip_input_file_handling', False)
+        if not skip_input_file_handling:
+            if input_file_path is None:
+                temp_input = tempfile.NamedTemporaryFile(delete=False, mode='w', encoding='utf-8')
+                input_file_path = temp_input.name
+                # read to EOF
+                s = sys.stdin.read()
+                while s:
+                    temp_input.write(s)
+                    s = sys.stdin.read()
+                temp_input.flush()
+                temp_input.close()
+
+        suppress_print = False
+        temp_output = None
+        output_file_path = ''
+        if 'out' in args:
+            output_file_path = args.out
+            if output_file_path is None:
+                suppress_print = True
+                temp_output = tempfile.NamedTemporaryFile(delete=False)
+                output_file_path = temp_output.name
+        
+        def printmsg(s):
+            if not suppress_print:
+                print(s)
+    
+        module_name, func_name = command['function']['name'].rsplit('.', 1)
+        func = dynamic_import(module_name, func_name)
+        func_args = {}
+        for arg in command['function']['args']:
+            if command['function']['args'][arg] == 'input_file_path':
+                func_args[arg] = input_file_path
+            elif output_file_path and command['function']['args'][arg] == 'output_file_path':
+                func_args[arg] = output_file_path
+            else:
+                val = command['function']['args'][arg]
+                if val.startswith('args.'):
+                    if hasattr(args, val[5:]):
+                        func_args[arg] = getattr(args, val[5:])
+                else:
+                    func_args[arg] = val
+        if output_file_path:
+            printmsg(f'Executing {command["description"]} with input {input_file_path} and output {output_file_path}')
+        func(**func_args)
+        
+        if temp_output:
+            with open(output_file_path, 'r', encoding='utf-8') as f:
+                sys.stdout.write(f.read())
+            temp_output.close()
+
+    except Exception as e:
+        print("Error: ", str(e))
+        exit(1)
+    finally:
+        if temp_input:
+            try:
+                os.remove(temp_input.name)
+            except OSError as e:
+                print(f"Error: Could not delete temporary input file {temp_input.name}. {e}")
+
+if __name__ == "__main__":
+    main()