structurize 3.0.2__py3-none-any.whl → 3.1.1__py3-none-any.whl

avrotize/commands.json

@@ -997,6 +997,472 @@
     "suggested_output_file_path": "{input_file_name}.sql",
     "prompts": []
   },
+  {
+    "command": "sql2a",
+    "description": "Convert SQL schema to Avrotize schema (connects to live database)",
+    "group": "5_SQL",
+    "skip_input_file_handling": true,
+    "function": {
+      "name": "avrotize.sqltoavro.convert_sql_to_avro",
+      "args": {
+        "connection_string": "args.connection_string",
+        "database": "args.database",
+        "table_name": "args.table_name",
+        "avro_namespace": "args.namespace",
+        "avro_schema_file": "output_file_path",
+        "dialect": "args.dialect",
+        "emit_cloudevents": "args.emit_cloudevents",
+        "emit_cloudevents_xregistry": "args.emit_xregistry",
+        "sample_size": "args.sample_size",
+        "infer_json_schema": "args.infer_json",
+        "infer_xml_schema": "args.infer_xml",
+        "username": "args.username",
+        "password": "args.password"
+      }
+    },
+    "extensions": [],
+    "args": [
+      {
+        "name": "input",
+        "type": "str",
+        "nargs": "?",
+        "help": "Not used (database connection is live)",
+        "required": false
+      },
+      {
+        "name": "--out",
+        "type": "str",
+        "help": "Path to the Avrotize schema file",
+        "required": false
+      },
+      {
+        "name": "--connection-string",
+        "type": "str",
+        "help": "Database connection string. Examples: postgresql://host:port/db (credentials via --username/--password), postgresql://user:pass@host:port/db?sslmode=require (PostgreSQL with SSL), mysql://user:pass@host/db?ssl=true (MySQL with SSL), mssql://@host/db (SQL Server with Windows Auth)",
+        "required": true
+      },
+      {
+        "name": "--username",
+        "type": "str",
+        "help": "Database username (overrides credentials in connection string, avoids credentials in command history)",
+        "required": false
+      },
+      {
+        "name": "--password",
+        "type": "str",
+        "help": "Database password (overrides credentials in connection string, avoids credentials in command history)",
+        "required": false
+      },
+      {
+        "name": "--database",
+        "type": "str",
+        "help": "Database name (if not in connection string)",
+        "required": false
+      },
+      {
+        "name": "--table-name",
+        "type": "str",
+        "help": "Specific table name (omit for all tables)",
+        "required": false
+      },
+      {
+        "name": "--namespace",
+        "type": "str",
+        "help": "Namespace for the Avrotize schema",
+        "required": false
+      },
+      {
+        "name": "--dialect",
+        "type": "str",
+        "help": "SQL dialect",
+        "choices": [
+          "postgres",
+          "mysql",
+          "sqlserver",
+          "oracle",
+          "sqlite"
+        ],
+        "default": "postgres",
+        "required": false
+      },
+      {
+        "name": "--emit-cloudevents",
+        "type": "bool",
+        "help": "Emit CloudEvents declarations for tables with type/source/data/id columns",
+        "required": false
+      },
+      {
+        "name": "--emit-xregistry",
+        "type": "bool",
+        "help": "Emit an xRegistry manifest instead of a single Avrotize schema",
+        "required": false
+      },
+      {
+        "name": "--sample-size",
+        "type": "int",
+        "help": "Number of rows to sample for JSON/XML inference",
+        "default": 100,
+        "required": false
+      },
+      {
+        "name": "--infer-json",
+        "type": "bool",
+        "help": "Infer schema for JSON/JSONB columns",
+        "default": true,
+        "required": false
+      },
+      {
+        "name": "--infer-xml",
+        "type": "bool",
+        "help": "Infer schema for XML columns",
+        "default": true,
+        "required": false
+      }
+    ],
+    "suggested_output_file_path": "{database}.avsc",
+    "prompts": [
+      {
+        "name": "--namespace",
+        "message": "Enter the namespace for the Avro schema",
+        "type": "str",
+        "required": false
+      },
+      {
+        "name": "--dialect",
+        "message": "Select the SQL dialect",
+        "choices": [
+          "postgres",
+          "mysql",
+          "sqlserver",
+          "oracle",
+          "sqlite"
+        ],
+        "default": "postgres",
+        "required": true
+      }
+    ]
+  },
+  {
+    "command": "json2a",
+    "description": "Infer Avro schema from JSON files",
+    "group": "6_Inference",
+    "skip_input_file_handling": true,
+    "function": {
+      "name": "avrotize.jsontoschema.convert_json_to_avro",
+      "args": {
+        "input_files": "args.input",
+        "avro_schema_file": "output_file_path",
+        "type_name": "args.type_name",
+        "avro_namespace": "args.namespace",
+        "sample_size": "args.sample_size"
+      }
+    },
+    "extensions": [".json", ".jsonl", ".ndjson"],
+    "args": [
+      {
+        "name": "input",
+        "type": "str",
+        "nargs": "+",
+        "help": "JSON file(s) to analyze for schema inference",
+        "required": true
+      },
+      {
+        "name": "--out",
+        "type": "str",
+        "help": "Path to the output Avro schema file",
+        "required": false
+      },
+      {
+        "name": "--type-name",
+        "type": "str",
+        "help": "Name for the root type",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--namespace",
+        "type": "str",
+        "help": "Namespace for the Avro schema",
+        "required": false
+      },
+      {
+        "name": "--sample-size",
+        "type": "int",
+        "help": "Maximum number of records to sample (0 = all)",
+        "default": 0,
+        "required": false
+      }
+    ],
+    "suggested_output_file_path": "{input_file_name}.avsc",
+    "prompts": [
+      {
+        "name": "--type-name",
+        "message": "Enter the name for the root type",
+        "type": "str",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--namespace",
+        "message": "Enter the namespace for the Avro schema",
+        "type": "str",
+        "required": false
+      }
+    ]
+  },
+  {
+    "command": "json2s",
+    "description": "Infer JSON Structure schema from JSON files",
+    "group": "6_Inference",
+    "skip_input_file_handling": true,
+    "function": {
+      "name": "avrotize.jsontoschema.convert_json_to_jstruct",
+      "args": {
+        "input_files": "args.input",
+        "jstruct_schema_file": "output_file_path",
+        "type_name": "args.type_name",
+        "base_id": "args.base_id",
+        "sample_size": "args.sample_size"
+      }
+    },
+    "extensions": [".json", ".jsonl", ".ndjson"],
+    "args": [
+      {
+        "name": "input",
+        "type": "str",
+        "nargs": "+",
+        "help": "JSON file(s) to analyze for schema inference",
+        "required": true
+      },
+      {
+        "name": "--out",
+        "type": "str",
+        "help": "Path to the output JSON Structure schema file",
+        "required": false
+      },
+      {
+        "name": "--type-name",
+        "type": "str",
+        "help": "Name for the root type",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--base-id",
+        "type": "str",
+        "help": "Base URI for $id generation",
+        "default": "https://example.com/",
+        "required": false
+      },
+      {
+        "name": "--sample-size",
+        "type": "int",
+        "help": "Maximum number of records to sample (0 = all)",
+        "default": 0,
+        "required": false
+      }
+    ],
+    "suggested_output_file_path": "{input_file_name}.jstruct.json",
+    "prompts": [
+      {
+        "name": "--type-name",
+        "message": "Enter the name for the root type",
+        "type": "str",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--base-id",
+        "message": "Enter the base URI for $id generation",
+        "type": "str",
+        "default": "https://example.com/",
+        "required": false
+      }
+    ]
+  },
+  {
+    "command": "xml2a",
+    "description": "Infer Avro schema from XML files",
+    "group": "6_Inference",
+    "skip_input_file_handling": true,
+    "function": {
+      "name": "avrotize.xmltoschema.convert_xml_to_avro",
+      "args": {
+        "input_files": "args.input",
+        "avro_schema_file": "output_file_path",
+        "type_name": "args.type_name",
+        "avro_namespace": "args.namespace",
+        "sample_size": "args.sample_size"
+      }
+    },
+    "extensions": [".xml"],
+    "args": [
+      {
+        "name": "input",
+        "type": "str",
+        "nargs": "+",
+        "help": "XML file(s) to analyze for schema inference",
+        "required": true
+      },
+      {
+        "name": "--out",
+        "type": "str",
+        "help": "Path to the output Avro schema file",
+        "required": false
+      },
+      {
+        "name": "--type-name",
+        "type": "str",
+        "help": "Name for the root type",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--namespace",
+        "type": "str",
+        "help": "Namespace for the Avro schema",
+        "required": false
+      },
+      {
+        "name": "--sample-size",
+        "type": "int",
+        "help": "Maximum number of documents to sample (0 = all)",
+        "default": 0,
+        "required": false
+      }
+    ],
+    "suggested_output_file_path": "{input_file_name}.avsc",
+    "prompts": [
+      {
+        "name": "--type-name",
+        "message": "Enter the name for the root type",
+        "type": "str",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--namespace",
+        "message": "Enter the namespace for the Avro schema",
+        "type": "str",
+        "required": false
+      }
+    ]
+  },
+  {
+    "command": "xml2s",
+    "description": "Infer JSON Structure schema from XML files",
+    "group": "6_Inference",
+    "skip_input_file_handling": true,
+    "function": {
+      "name": "avrotize.xmltoschema.convert_xml_to_jstruct",
+      "args": {
+        "input_files": "args.input",
+        "jstruct_schema_file": "output_file_path",
+        "type_name": "args.type_name",
+        "base_id": "args.base_id",
+        "sample_size": "args.sample_size"
+      }
+    },
+    "extensions": [".xml"],
+    "args": [
+      {
+        "name": "input",
+        "type": "str",
+        "nargs": "+",
+        "help": "XML file(s) to analyze for schema inference",
+        "required": true
+      },
+      {
+        "name": "--out",
+        "type": "str",
+        "help": "Path to the output JSON Structure schema file",
+        "required": false
+      },
+      {
+        "name": "--type-name",
+        "type": "str",
+        "help": "Name for the root type",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--base-id",
+        "type": "str",
+        "help": "Base URI for $id generation",
+        "default": "https://example.com/",
+        "required": false
+      },
+      {
+        "name": "--sample-size",
+        "type": "int",
+        "help": "Maximum number of documents to sample (0 = all)",
+        "default": 0,
+        "required": false
+      }
+    ],
+    "suggested_output_file_path": "{input_file_name}.jstruct.json",
+    "prompts": [
+      {
+        "name": "--type-name",
+        "message": "Enter the name for the root type",
+        "type": "str",
+        "default": "Document",
+        "required": false
+      },
+      {
+        "name": "--base-id",
+        "message": "Enter the base URI for $id generation",
+        "type": "str",
+        "default": "https://example.com/",
+        "required": false
+      }
+    ]
+  },
+  {
+    "command": "validate",
+    "description": "Validate JSON instances against Avro or JSON Structure schemas",
+    "group": "6_Inference",
+    "skip_input_file_handling": true,
+    "function": {
+      "name": "avrotize.validate.validate",
+      "args": {
+        "input": "args.input",
+        "schema": "args.schema",
+        "schema_type": "args.schema_type",
+        "quiet": "args.quiet"
+      }
+    },
+    "extensions": [".json", ".jsonl"],
+    "args": [
+      {
+        "name": "input",
+        "type": "str",
+        "nargs": "+",
+        "help": "JSON file(s) to validate",
+        "required": true
+      },
+      {
+        "name": "--schema",
+        "type": "str",
+        "help": "Path to schema file (.avsc for Avro, .jstruct.json for JSON Structure)",
+        "required": true
+      },
+      {
+        "name": "--schema-type",
+        "type": "str",
+        "help": "Schema type: 'avro' or 'jstruct'. Auto-detected from file extension if omitted.",
+        "required": false
+      },
+      {
+        "name": "--quiet",
+        "type": "bool",
+        "help": "Suppress output. Exit code 0 if valid, 1 if invalid.",
+        "default": false,
+        "required": false
+      }
+    ],
+    "prompts": []
+  },
   {
     "command": "a2mongo",
     "description": "Convert Avrotize schema to MongoDB schema",

avrotize/dependencies/typescript/node22/package.json

@@ -8,7 +8,7 @@
     "kafkajs": "^2.2.4"
   },
   "devDependencies": {
-    "@types/node": "^24.10.1",
+    "@types/node": "^25.0.3",
     "typescript": "^5.7.2",
     "jest": "^30.2.0",
     "@types/jest": "^30.0.0"

avrotize/jsontoschema.py

@@ -0,0 +1,151 @@
+"""Infers schema from JSON files and converts to Avro or JSON Structure format.
+
+This module provides:
+- json2a: Infer Avro schema from JSON files
+- json2s: Infer JSON Structure schema from JSON files
+"""
+
+import json
+import os
+from typing import Any, Dict, List
+
+from avrotize.schema_inference import (
+    AvroSchemaInferrer,
+    JsonStructureSchemaInferrer,
+    JsonNode
+)
+
+
+def convert_json_to_avro(
+    input_files: List[str],
+    avro_schema_file: str,
+    type_name: str = 'Document',
+    avro_namespace: str = '',
+    sample_size: int = 0
+) -> None:
+    """Infers Avro schema from JSON files.
+
+    Reads JSON files, analyzes their structure, and generates an Avro schema
+    that can represent all the data. Multiple files are analyzed together to
+    produce a unified schema.
+
+    Args:
+        input_files: List of JSON file paths to analyze
+        avro_schema_file: Output path for the Avro schema
+        type_name: Name for the root type
+        avro_namespace: Namespace for generated Avro types
+        sample_size: Maximum number of records to sample (0 = all)
+    """
+    if not input_files:
+        raise ValueError("At least one input file is required")
+
+    values = _load_json_values(input_files, sample_size)
+
+    if not values:
+        raise ValueError("No valid JSON data found in input files")
+
+    inferrer = AvroSchemaInferrer(namespace=avro_namespace)
+    schema = inferrer.infer_from_json_values(type_name, values)
+
+    # Ensure output directory exists
+    output_dir = os.path.dirname(avro_schema_file)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    with open(avro_schema_file, 'w', encoding='utf-8') as f:
+        json.dump(schema, f, indent=2)
+
+
+def convert_json_to_jstruct(
+    input_files: List[str],
+    jstruct_schema_file: str,
+    type_name: str = 'Document',
+    base_id: str = 'https://example.com/',
+    sample_size: int = 0
+) -> None:
+    """Infers JSON Structure schema from JSON files.
+
+    Reads JSON files, analyzes their structure, and generates a JSON Structure
+    schema that validates with the official JSON Structure SDK.
+
+    Args:
+        input_files: List of JSON file paths to analyze
+        jstruct_schema_file: Output path for the JSON Structure schema
+        type_name: Name for the root type
+        base_id: Base URI for $id generation
+        sample_size: Maximum number of records to sample (0 = all)
+    """
+    if not input_files:
+        raise ValueError("At least one input file is required")
+
+    values = _load_json_values(input_files, sample_size)
+
+    if not values:
+        raise ValueError("No valid JSON data found in input files")
+
+    inferrer = JsonStructureSchemaInferrer(base_id=base_id)
+    schema = inferrer.infer_from_json_values(type_name, values)
+
+    # Ensure output directory exists
+    output_dir = os.path.dirname(jstruct_schema_file)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    with open(jstruct_schema_file, 'w', encoding='utf-8') as f:
+        json.dump(schema, f, indent=2)
+
+
+def _load_json_values(input_files: List[str], sample_size: int) -> List[Any]:
+    """Loads JSON values from files.
+
+    Handles both single JSON documents and JSON Lines (JSONL) files.
+    Arrays at the root level are flattened into individual values.
+
+    Args:
+        input_files: List of file paths
+        sample_size: Maximum values to load (0 = all)
+
+    Returns:
+        List of parsed JSON values
+    """
+    values: List[Any] = []
+
+    for file_path in input_files:
+        if sample_size > 0 and len(values) >= sample_size:
+            break
+
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read().strip()
+
+        if not content:
+            continue
+
+        # Try parsing as a single JSON document first
+        try:
+            data = json.loads(content)
+            if isinstance(data, list):
+                # Root-level array: each element is a separate value
+                for item in data:
+                    values.append(item)
+                    if sample_size > 0 and len(values) >= sample_size:
+                        break
+            else:
+                values.append(data)
+            continue
+        except json.JSONDecodeError:
+            pass
+
+        # Try parsing as JSON Lines (JSONL)
+        for line in content.split('\n'):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                data = json.loads(line)
+                values.append(data)
+                if sample_size > 0 and len(values) >= sample_size:
+                    break
+            except json.JSONDecodeError:
+                pass
+
+    return values