structurize 3.6.1__tar.gz → 3.7.1__tar.gz

{structurize-3.6.1/structurize.egg-info → structurize-3.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structurize
-Version: 3.6.1
+Version: 3.7.1
 Summary: Tools to convert from and to JSON Structure from various other schema languages.
 Author-email: Clemens Vasters <clemensv@microsoft.com>
 Classifier: Programming Language :: Python :: 3

{structurize-3.6.1 → structurize-3.7.1}/avrotize/__init__.py

@@ -25,6 +25,28 @@ class LazyLoader:
 # Define the functions and their corresponding module paths
 _mappings = {
     "convert_proto_to_avro": (f"{mod}.prototoavro", "convert_proto_to_avro"),
+    "convert_jtd_to_avro": (f"{mod}.jtdtoavro", "convert_jtd_to_avro"),
+    "convert_avro_to_jtd": (f"{mod}.avrotojtd", "convert_avro_to_jtd"),
+    "convert_jtd_to_structure": (f"{mod}.jtdtostructure", "convert_jtd_to_structure"),
+    "convert_structure_to_jtd": (f"{mod}.structuretojtd", "convert_structure_to_jtd"),
+    "convert_surrealql_to_avro": (f"{mod}.surrealtoavro", "convert_surrealql_to_avro"),
+    "convert_avro_to_surrealql": (f"{mod}.avrotosurreal", "convert_avro_to_surrealql"),
+    "convert_flatbuffers_to_avro": (f"{mod}.flatbufferstoavro", "convert_flatbuffers_to_avro"),
+    "convert_avro_to_flatbuffers": (f"{mod}.avrotoflatbuffers", "convert_avro_to_flatbuffers"),
+    "convert_flatbuffers_to_json_structure": (f"{mod}.flatbufferstojstruct", "convert_flatbuffers_to_json_structure"),
+    "convert_json_structure_to_flatbuffers": (f"{mod}.jstructtoflatbuffers", "convert_json_structure_to_flatbuffers"),
+    "convert_thrift_to_avro": (f"{mod}.thrifttoavro", "convert_thrift_to_avro"),
+    "convert_avro_to_thrift": (f"{mod}.avrotothrift", "convert_avro_to_thrift"),
+    "convert_thrift_to_structure": (f"{mod}.thriftstructure", "convert_thrift_to_structure"),
+    "convert_structure_to_thrift": (f"{mod}.thriftstructure", "convert_structure_to_thrift"),
+    "convert_smithy_to_avro": (f"{mod}.smithytoavro", "convert_smithy_to_avro"),
+    "convert_avro_to_smithy": (f"{mod}.avrotosmithy", "convert_avro_to_smithy"),
+    "convert_smithy_to_json_structure": (f"{mod}.smithytojstruct", "convert_smithy_to_json_structure"),
+    "convert_json_structure_to_smithy": (f"{mod}.jstructtosmithy", "convert_json_structure_to_smithy"),
+    "convert_raml_to_avro": (f"{mod}.ramltoavro", "convert_raml_to_avro"),
+    "convert_avro_to_raml": (f"{mod}.avrotoraml", "convert_avro_to_raml"),
+    "convert_raml_to_json_structure": (f"{mod}.ramltojstruct", "convert_raml_to_json_structure"),
+    "convert_json_structure_to_raml": (f"{mod}.jstructtoraml", "convert_json_structure_to_raml"),
     "convert_jsons_to_avro": (f"{mod}.jsonstoavro", "convert_jsons_to_avro"),
     "convert_kafka_struct_to_avro_schema": (f"{mod}.kstructtoavro", "convert_kafka_struct_to_avro_schema"),
     "convert_kusto_to_avro": (f"{mod}.kustotoavro", "convert_kusto_to_avro"),
@@ -35,6 +57,14 @@ _mappings = {
     "convert_avro_to_kusto_db": (f"{mod}.avrotokusto", "convert_avro_to_kusto_db"),
     "convert_avro_to_parquet": (f"{mod}.avrotoparquet", "convert_avro_to_parquet"),
     "convert_avro_to_proto": (f"{mod}.avrotoproto", "convert_avro_to_proto"),
+    "convert_cue_to_avro": (f"{mod}.cuetoavro", "convert_cue_to_avro"),
+    "convert_avro_to_cue": (f"{mod}.avrotocue", "convert_avro_to_cue"),
+    "convert_cue_to_json_structure": (f"{mod}.cuetostructure", "convert_cue_to_json_structure"),
+    "convert_json_structure_to_cue": (f"{mod}.structuretocue", "convert_json_structure_to_cue"),
+    "convert_capnproto_to_avro": (f"{mod}.capnprototoavro", "convert_capnproto_to_avro"),
+    "convert_avro_to_capnproto": (f"{mod}.avrotocapnproto", "convert_avro_to_capnproto"),
+    "convert_capnproto_to_json_structure": (f"{mod}.capnprototoavro", "convert_capnproto_to_json_structure"),
+    "convert_json_structure_to_capnproto": (f"{mod}.avrotocapnproto", "convert_json_structure_to_capnproto"),
     "convert_avro_to_sql": (f"{mod}.avrotodb", "convert_avro_to_sql"),
     "convert_avro_to_xsd": (f"{mod}.avrotoxsd", "convert_avro_to_xsd"),
     "convert_avro_to_java": (f"{mod}.avrotojava", "convert_avro_to_java"),

{structurize-3.6.1 → structurize-3.7.1}/avrotize/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '3.6.1'
-__version_tuple__ = version_tuple = (3, 6, 1)
+__version__ = version = '3.7.1'
+__version_tuple__ = version_tuple = (3, 7, 1)
 
-__commit_id__ = commit_id = 'ga0404973f'
+__commit_id__ = commit_id = 'g2f9460efb'

structurize-3.7.1/avrotize/avrotocapnproto.py

@@ -0,0 +1,207 @@
+"""Convert Avrotize/Avro schemas to Cap'n Proto .capnp schemas."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import tempfile
+from typing import Any
+
+from avrotize.common import unique_name
+
+INDENT = "  "
+FILE_ID = "0xd12f8a7e9b0c4a6f"
+
+
+def _safe_ident(name: str) -> str:
+    safe = re.sub(r"[^A-Za-z0-9_]", "_", name)
+    if not safe or safe[0].isdigit():
+        safe = f"_{safe}"
+    return safe
+
+
+def _type_name(schema: dict[str, Any]) -> str:
+    namespace = schema.get("namespace")
+    name = schema["name"]
+    return f"{namespace}.{name}" if namespace else name
+
+
+class AvroToCapnpConverter:
+    primitive_map = {
+        "null": "Void",
+        "boolean": "Bool",
+        "int": "Int32",
+        "long": "Int64",
+        "float": "Float32",
+        "double": "Float64",
+        "bytes": "Data",
+        "string": "Text",
+    }
+
+    def __init__(self, namespace: str | None = None) -> None:
+        self.namespace = namespace
+        self.named: dict[str, dict[str, Any]] = {}
+        self.emitted: set[str] = set()
+        self.type_names: dict[int, str] = {}
+        self.top_level_type_names: set[str] = set()
+
+    def convert(self, avro_schema: Any) -> str:
+        schemas = avro_schema if isinstance(avro_schema, list) else [avro_schema]
+        for schema in schemas:
+            if isinstance(schema, dict) and schema.get("type") in {"record", "enum"}:
+                self.named[schema["name"]] = schema
+                self.named[_type_name(schema)] = schema
+                self._assign_type_name(schema, self.top_level_type_names)
+        lines = [f"@{FILE_ID};", ""]
+        if self.namespace:
+            lines.append(f"# namespace: {self.namespace}")
+        lines.append("# Generated by avrotize. File id is stable for deterministic output; replace if needed.")
+        lines.append("")
+        for schema in schemas:
+            if isinstance(schema, dict) and schema.get("type") in {"enum", "record"}:
+                name = _type_name(schema)
+                if name not in self.emitted:
+                    lines.extend(self._render_named(schema, 0, self.top_level_type_names))
+                    lines.append("")
+        return "\n".join(lines).rstrip() + "\n"
+
+    def _assign_type_name(self, schema: dict[str, Any], used: set[str] | None = None) -> str:
+        key = id(schema)
+        if key not in self.type_names:
+            self.type_names[key] = unique_name(_safe_ident(schema["name"]), used if used is not None else self.top_level_type_names)
+        return self.type_names[key]
+
+    def _render_named(self, schema: dict[str, Any], level: int, used_type_names: set[str] | None = None) -> list[str]:
+        if schema.get("type") == "enum":
+            return self._render_enum(schema, level, used_type_names)
+        if schema.get("type") == "record":
+            return self._render_struct(schema, level, used_type_names)
+        return []
+
+    def _render_enum(self, schema: dict[str, Any], level: int, used_type_names: set[str] | None = None) -> list[str]:
+        self.emitted.add(_type_name(schema))
+        lines = [f"{INDENT * level}enum {self._assign_type_name(schema, used_type_names)} {{"]
+        used_symbols: set[str] = set()
+        for index, symbol in enumerate(schema.get("symbols", [])):
+            lines.append(f"{INDENT * (level + 1)}{unique_name(_safe_ident(symbol), used_symbols)} @{index};")
+        lines.append(f"{INDENT * level}}}")
+        return lines
+
+    def _render_struct(self, schema: dict[str, Any], level: int, used_type_names: set[str] | None = None) -> list[str]:
+        self.emitted.add(_type_name(schema))
+        lines = [f"{INDENT * level}struct {self._assign_type_name(schema, used_type_names)} {{"]
+        fields = schema.get("fields", [])
+        grouped: dict[str, list[tuple[int, dict[str, Any], Any, str]]] = {}
+        normal: list[tuple[int, dict[str, Any], Any, str]] = []
+        used_fields: set[str] = set()
+        for index, field in enumerate(fields):
+            avro_type = field.get("type")
+            union_name = field.get("capnpUnion")
+            safe_field_name = unique_name(_safe_ident(field.get("name", f"field{index}")), used_fields)
+            item = (index, field, avro_type, safe_field_name)
+            if union_name:
+                grouped.setdefault(str(union_name), []).append(item)
+            else:
+                normal.append(item)
+        nested_type_names: set[str] = set()
+        for index, field, avro_type, safe_field_name in normal:
+            lines.extend(self._render_field(field, avro_type, index, level + 1, safe_field_name, nested_type_names))
+        for _, items in grouped.items():
+            lines.append(f"{INDENT * (level + 1)}union {{")
+            for index, field, avro_type, safe_field_name in items:
+                lines.extend(self._render_field(field, avro_type, index, level + 2, safe_field_name, nested_type_names))
+            lines.append(f"{INDENT * (level + 1)}}}")
+        lines.append(f"{INDENT * level}}}")
+        return lines
+
+    def _render_field(
+        self,
+        field: dict[str, Any],
+        avro_type: Any,
+        index: int,
+        level: int,
+        safe_field_name: str,
+        used_type_names: set[str],
+    ) -> list[str]:
+        capnp_type, nested = self._capnp_type(avro_type, safe_field_name, used_type_names)
+        ordinal = index
+        lines: list[str] = []
+        lines.extend(self._render_inline_nested(nested, level, used_type_names))
+        lines.append(f"{INDENT * level}{safe_field_name} @{ordinal} :{capnp_type};")
+        return lines
+
+    def _render_inline_nested(self, nested: list[dict[str, Any]], level: int, used_type_names: set[str]) -> list[str]:
+        lines: list[str] = []
+        for schema in nested:
+            lines.extend(self._render_named(schema, level, used_type_names))
+        return lines
+
+    def _non_null_union(self, avro_type: list[Any]) -> list[Any]:
+        return [item for item in avro_type if item != "null" and not (isinstance(item, dict) and item.get("type") == "null")]
+
+    def _capnp_type(self, avro_type: Any, field_name: str, used_type_names: set[str] | None = None) -> tuple[str, list[dict[str, Any]]]:
+        used_type_names = used_type_names if used_type_names is not None else self.top_level_type_names
+        if isinstance(avro_type, list):
+            non_null = self._non_null_union(avro_type)
+            if len(non_null) == 1:
+                return self._capnp_type(non_null[0], field_name, used_type_names)
+            union_schema = {
+                "type": "record",
+                "name": f"{field_name[:1].upper()}{field_name[1:]}Choice",
+                "fields": [{"name": f"choice{i}", "type": t, "capnpUnion": "union"} for i, t in enumerate(non_null)],
+            }
+            return self._assign_type_name(union_schema, used_type_names), [union_schema]
+        if isinstance(avro_type, dict):
+            avro_kind = avro_type.get("type")
+            if isinstance(avro_kind, list):
+                return self._capnp_type(avro_kind, field_name, used_type_names)
+            if avro_kind == "array":
+                item_type, nested = self._capnp_type(avro_type.get("items", "string"), f"{field_name}Item", used_type_names)
+                return f"List({item_type})", nested
+            if avro_kind == "map":
+                entry = {
+                    "type": "record",
+                    "name": f"{field_name[:1].upper()}{field_name[1:]}Entry",
+                    "fields": [
+                        {"name": "key", "type": "string"},
+                        {"name": "value", "type": avro_type.get("values", "string")},
+                    ],
+                }
+                return f"List({self._assign_type_name(entry, used_type_names)})", [entry]
+            if avro_kind == "record":
+                return self._assign_type_name(avro_type, used_type_names), [avro_type]
+            if avro_kind == "enum":
+                return self._assign_type_name(avro_type, used_type_names), [avro_type]
+            if avro_kind == "fixed":
+                return "Data", []
+            return self._capnp_type(avro_kind, field_name, used_type_names)
+        if isinstance(avro_type, str):
+            named = self.named.get(avro_type)
+            if named:
+                return self._assign_type_name(named), []
+            return self.primitive_map.get(avro_type, _safe_ident(avro_type.split(".")[-1])), []
+        return "Text", []
+
+
+def convert_avro_to_capnproto(avro_schema_path: str, capnp_file_path: str, namespace: str | None = None) -> None:
+    """Convert an Avrotize/Avro schema file to a Cap'n Proto .capnp schema file."""
+    with open(avro_schema_path, "r", encoding="utf-8") as avro_file:
+        avro_schema = json.load(avro_file)
+    converter = AvroToCapnpConverter(namespace=namespace)
+    capnp_schema = converter.convert(avro_schema)
+    os.makedirs(os.path.dirname(os.path.abspath(capnp_file_path)) or ".", exist_ok=True)
+    with open(capnp_file_path, "w", encoding="utf-8") as capnp_file:
+        capnp_file.write(capnp_schema)
+
+
+def convert_json_structure_to_capnproto(structure_file: str, capnp_file_path: str, namespace: str | None = None) -> None:
+    """Convert JSON Structure to Cap'n Proto by bridging through Avrotize/Avro."""
+    from avrotize.jstructtoavro import convert_json_structure_to_avro
+
+    temp_parent = os.path.dirname(os.path.abspath(capnp_file_path)) or os.getcwd()
+    os.makedirs(temp_parent, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="avrotize-capnp-", dir=temp_parent) as temp_dir:
+        avro_file = os.path.join(temp_dir, "schema.avsc")
+        convert_json_structure_to_avro(structure_file, avro_file)
+        convert_avro_to_capnproto(avro_file, capnp_file_path, namespace=namespace)