avrotize 2.21.1__py3-none-any.whl

avrotize/constants.py

@@ -0,0 +1,87 @@
+"""Constants for the avrotize package.
+
+These constants are now derived from central dependency files in avrotize/dependencies/
+which are monitored by Dependabot. This ensures versions stay up-to-date.
+"""
+
+from avrotize.dependency_version import get_dependency_version
+
+# Java/JVM dependencies
+try:
+    AVRO_VERSION = get_dependency_version('java', 'jdk21', 'org.apache.avro:avro')
+except ValueError:
+    AVRO_VERSION = '1.12.0'  # Fallback
+
+try:
+    JACKSON_VERSION = get_dependency_version('java', 'jdk21', 'com.fasterxml.jackson.core:jackson-core')
+except ValueError:
+    JACKSON_VERSION = '2.18.2'  # Fallback
+
+JDK_VERSION = '21'
+
+# C# dependencies  
+try:
+    CSHARP_AVRO_VERSION = get_dependency_version('cs', 'net90', 'Apache.Avro')
+except ValueError:
+    CSHARP_AVRO_VERSION = '1.12.0'
+
+try:
+    NEWTONSOFT_JSON_VERSION = get_dependency_version('cs', 'net90', 'Newtonsoft.Json')
+except ValueError:
+    NEWTONSOFT_JSON_VERSION = '13.0.3'
+
+try:
+    SYSTEM_TEXT_JSON_VERSION = get_dependency_version('cs', 'net90', 'System.Text.Json')
+except ValueError:
+    SYSTEM_TEXT_JSON_VERSION = '9.0.3'
+
+try:
+    SYSTEM_MEMORY_DATA_VERSION = get_dependency_version('cs', 'net90', 'System.Memory.Data')
+except ValueError:
+    SYSTEM_MEMORY_DATA_VERSION = '9.0.3'
+
+try:
+    PROTOBUF_NET_VERSION = get_dependency_version('cs', 'net90', 'protobuf-net')
+except ValueError:
+    PROTOBUF_NET_VERSION = '3.2.30'
+
+try:
+    NUNIT_VERSION = get_dependency_version('cs', 'net90', 'NUnit')
+except ValueError:
+    NUNIT_VERSION = '4.3.2'
+
+try:
+    NUNIT_ADAPTER_VERSION = get_dependency_version('cs', 'net90', 'NUnit3TestAdapter')
+except ValueError:
+    NUNIT_ADAPTER_VERSION = '5.0.0'
+
+try:
+    MSTEST_SDK_VERSION = get_dependency_version('cs', 'net90', 'Microsoft.NET.Test.Sdk')
+except ValueError:
+    MSTEST_SDK_VERSION = '17.13.0'
+
+try:
+    COVERLET_VERSION = get_dependency_version('cs', 'net90', 'coverlet.collector')
+except ValueError:
+    COVERLET_VERSION = '6.0.4'
+
+try:
+    MSGPACK_VERSION = get_dependency_version('cs', 'net90', 'MessagePack')
+except ValueError:
+    MSGPACK_VERSION = '2.5.187'
+
+# Java test dependencies
+try:
+    JUNIT_VERSION = get_dependency_version('java', 'jdk21', 'org.junit.jupiter:junit-jupiter-api')
+except ValueError:
+    JUNIT_VERSION = '5.11.4'
+
+try:
+    MAVEN_SUREFIRE_VERSION = get_dependency_version('java', 'jdk21', 'org.apache.maven.plugins:maven-surefire-plugin')
+except ValueError:
+    MAVEN_SUREFIRE_VERSION = '3.5.2'
+
+try:
+    MAVEN_COMPILER_VERSION = get_dependency_version('java', 'jdk21', 'org.apache.maven.plugins:maven-compiler-plugin')
+except ValueError:
+    MAVEN_COMPILER_VERSION = '3.13.0'

avrotize/csvtoavro.py

@@ -0,0 +1,132 @@
+# coding: utf-8
+"""
+Module to convert CSV schema to Avro schema.
+"""
+
+import json
+import os
+import pandas as pd
+
+from avrotize.common import pascal
+
+class CSVToAvroConverter:
+    """
+    Class to convert CSV schema to Avro schema.
+    """
+
+    def __init__(self, csv_file_path, avro_schema_path, namespace=""):
+        """
+        Initialize the converter with file paths and namespace.
+
+        :param csv_file_path: Path to the CSV file.
+        :param avro_schema_path: Path to save the Avro schema file.
+        :param csv_schema_path: Optional path to CSV schema file.
+        :param namespace: Namespace for Avro records.
+        """
+        self.csv_file_path = csv_file_path
+        self.avro_schema_path = avro_schema_path
+        self.namespace = namespace
+
+    def convert(self):
+        """
+        Convert CSV schema to Avro schema and save to file.
+        """
+        schema = self.infer_schema()
+
+        # Infer the name of the schema from the CSV file name
+        schema_name = os.path.splitext(os.path.basename(self.csv_file_path))[0].replace(" ", "_")
+
+        avro_schema = {}
+        avro_schema["type"] = "record"
+        avro_schema["name"] = schema_name
+        if self.namespace:
+            avro_schema["namespace"] = self.namespace
+        avro_schema["fields"] = schema
+
+        with open(self.avro_schema_path, "w", encoding="utf-8") as file:
+            json.dump(avro_schema, file, indent=2)
+
+    def infer_schema(self):
+        """
+        Infer the schema from CSV headers or data.
+        :return: List of fields in Avro schema format.
+        """
+        df = pd.read_csv(self.csv_file_path)
+        schema = []
+        for column in df.columns:
+            avro_field = {
+                "name": pascal(column),
+                "type": self.infer_avro_type(df[column])
+            }
+            if avro_field["name"] != column:
+                avro_field["altnames"] = { "csv": column}
+            schema.append(avro_field)
+        return schema
+
+    def infer_avro_type(self, series):
+        """
+        Infer Avro type from pandas series.
+        :param series: Pandas series to infer type from.
+        :return: Avro type as string.
+        """
+        if pd.api.types.is_integer_dtype(series):
+            return "int"
+        if pd.api.types.is_float_dtype(series):
+            return "double"
+        if pd.api.types.is_bool_dtype(series):
+            return "boolean"
+        if pd.api.types.is_datetime64_any_dtype(series):
+            return {"type": "long", "logicalType": "timestamp-millis"}
+        if pd.api.types.is_object_dtype(series):
+            return "string"
+        return "string"
+
+    def map_csv_type_to_avro(self, csv_type):
+        """
+        Map CSV type to Avro type.
+        :param csv_type: CSV type as string.
+        :return: Avro type as string.
+        """
+        type_mapping = {
+            "string": "string",
+            "number": "double",
+            "integer": "int",
+            "boolean": "boolean",
+            "date": {"type": "int", "logicalType": "date"},
+            "timestamp": {"type": "long", "logicalType": "timestamp-millis"}
+        }
+        return type_mapping.get(csv_type.lower(), "string")
+
+    def handle_string_format(self, format_type):
+        """
+        Handle string format types.
+        :param format_type: Format type as string.
+        :return: Avro type as string or dict.
+        """
+        format_mapping = {
+            "email": "string",
+            "uri": "string",
+            "uuid": "string",
+            "ipv4": "string",
+            "ipv6": "string",
+            "hostname": "string",
+            "datetime": {"type": "long", "logicalType": "timestamp-millis"}
+        }
+        return format_mapping.get(format_type.lower(), "string")
+
+
+def convert_csv_to_avro(csv_file_path, avro_file_path, namespace=""):
+    """
+    Convert a CSV file to an Avro schema file.
+
+    :param csv_file_path: Path to the CSV file.
+    :param avro_file_path: Path to save the Avro schema file.
+    :param namespace: Namespace for Avro records.
+    """
+    
+    if not os.path.exists(csv_file_path):
+        raise FileNotFoundError(f"CSV file not found at: {csv_file_path}")
+    
+    converter = CSVToAvroConverter(
+        csv_file_path, avro_file_path, namespace)
+    converter.convert()

avrotize/datapackagetoavro.py

@@ -0,0 +1,76 @@
+"""Convert a Data Package to an Avro schema."""
+
+import json
+import sys
+from typing import Dict, List
+from datapackage import Package
+
+JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
+
+
+class DataPackageToAvroConverter:
+    """Class to convert Data Package to Avro schema."""
+
+    def convert_datapackage_to_avro(self, datapackage_path, avro_schema_path):
+        """Convert a Data Package to an Avro schema."""
+        package = Package(datapackage_path)
+        resources = package.resources
+
+        avro_schemas = []
+
+        for resource in resources:
+            table_name = resource.descriptor['name']
+            fields = resource.descriptor['schema']['fields']
+
+            avro_fields = []
+            for field in fields:
+                avro_field = {
+                    "name": field["name"],
+                    "type": self.convert_datapackage_type_to_avro_type(field["type"])
+                }
+                avro_fields.append(avro_field)
+
+            avro_schema = {
+                "type": "record",
+                "name": table_name,
+                "fields": avro_fields
+            }
+            avro_schemas.append(avro_schema)
+
+        # If there's only one schema, write it directly
+        if len(avro_schemas) == 1:
+            avro_schema = avro_schemas[0]
+        else:
+            # If there are multiple schemas, create a union
+            avro_schema = avro_schemas
+
+        with open(avro_schema_path, "w", encoding="utf-8") as f:
+            json.dump(avro_schema, f, indent=2)
+
+    def convert_datapackage_type_to_avro_type(self, datapackage_type):
+        """Convert a Data Package type to an Avro type."""
+        if datapackage_type == "string":
+            return "string"
+        elif datapackage_type == "number":
+            return "double"
+        elif datapackage_type == "integer":
+            return "int"
+        elif datapackage_type == "boolean":
+            return "boolean"
+        elif datapackage_type == "array":
+            return {"type": "array", "items": "string"}
+        elif datapackage_type == "object":
+            return {"type": "map", "values": "string"}
+        else:
+            print(f"WARNING: Unsupported data package type: {datapackage_type}")
+            return "string"
+
+
+def convert_datapackage_to_avro(datapackage_path, avro_schema_path):
+    """Convert a Data Package to an Avro schema."""
+    converter = DataPackageToAvroConverter()
+    converter.convert_datapackage_to_avro(datapackage_path, avro_schema_path)
+
+
+# Example usage:
+# convert_datapackage_to_avro("datapackage.json", "schema.avsc")

avrotize/dependencies/cpp/vcpkg/vcpkg.json

@@ -0,0 +1,19 @@
+{
+  "$comment": "Central dependency versions for Avrotize C++ code generation templates. This file is monitored by Dependabot.",
+  "name": "avrotize-dependencies",
+  "version": "1.0.0",
+  "dependencies": [
+    {
+      "name": "avro-cpp",
+      "version>=": "1.11.1"
+    },
+    {
+      "name": "nlohmann-json",
+      "version>=": "3.11.3"
+    },
+    {
+      "name": "gtest",
+      "version>=": "1.15.2"
+    }
+  ]
+}

avrotize/dependencies/cs/net90/dependencies.csproj

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Central dependency file for .NET 9.0 code generation templates.
+  Dependabot will detect and propose updates to these dependencies.
+  The dependency() function in templates reads from this file.
+-->
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <!-- Serialization -->
+    <ItemGroup>
+        <PackageReference Include="Apache.Avro" Version="1.12.0" />
+        <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
+        <PackageReference Include="System.Text.Json" Version="9.0.3" />
+        <PackageReference Include="System.Memory.Data" Version="9.0.3" />
+        <PackageReference Include="protobuf-net" Version="3.2.30" />
+        <PackageReference Include="MessagePack" Version="3.1.4" />
+    </ItemGroup>
+
+    <!-- Testing -->
+    <ItemGroup>
+        <PackageReference Include="NUnit" Version="4.4.0" />
+        <PackageReference Include="NUnit3TestAdapter" Version="5.2.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+    </ItemGroup>
+</Project>

avrotize/dependencies/go/go121/go.mod

@@ -0,0 +1,6 @@
+// Central dependency versions for Avrotize Go code generation templates
+// This file is monitored by Dependabot and read by the dependency() function in templates
+
+module avrotize_dependencies
+
+go 1.21

avrotize/dependencies/java/jdk21/pom.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Central dependency file for Java JDK 21 code generation templates.
+  Dependabot will detect and propose updates to these dependencies.
+  The dependency() function in templates reads from this file.
+  
+  Note: XML namespace is intentionally omitted for simpler parsing.
+  Dependabot still recognizes dependencies without the namespace.
+-->
+<project>
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>io.avrotize</groupId>
+    <artifactId>dependencies</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <name>Avrotize Java Dependencies Reference</name>
+    <description>Reference project for dependency version management</description>
+
+    <properties>
+        <maven.compiler.source>21</maven.compiler.source>
+        <maven.compiler.target>21</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <!-- Apache Avro -->
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+            <version>1.12.1</version>
+        </dependency>
+
+        <!-- Jackson BOM for version consistency -->
+        <dependency>
+            <groupId>com.fasterxml.jackson</groupId>
+            <artifactId>jackson-bom</artifactId>
+            <version>2.20.1</version>
+            <type>pom</type>
+            <scope>import</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>2.18.2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.20.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>2.20</version>
+        </dependency>
+
+        <!-- Testing - JUnit 5 -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>6.0.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>6.0.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>6.0.1</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Maven Plugins -->
+        <dependency>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>3.5.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-compiler-plugin</artifactId>
+            <version>3.14.1</version>
+        </dependency>
+    </dependencies>
+</project>

avrotize/dependencies/python/py312/requirements.txt

@@ -0,0 +1,13 @@
+# Central dependency versions for Avrotize Python code generation templates
+# This file is monitored by Dependabot and read by the dependency() function in templates
+
+# Runtime dependencies
+avro>=1.12.0
+fastavro>=1.10.0
+confluent-kafka>=2.6.1
+dataclasses-json>=0.6.7
+
+# Development dependencies
+pytest>=8.3.4
+mypy>=1.13.0
+pylint>=3.3.2

avrotize/dependencies/rust/stable/Cargo.toml

@@ -0,0 +1,17 @@
+# Central dependency versions for Avrotize Rust code generation templates
+# This file is monitored by Dependabot and read by the dependency() function in templates
+
+[package]
+name = "avrotize-dependencies"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+apache-avro = "0.21.0"
+serde = { version = "1.0.215", features = ["derive"] }
+serde_json = "1.0.133"
+chrono = { version = "0.4.38", features = ["serde"] }
+uuid = { version = "1.11.0", features = ["v4", "serde"] }
+
+[dev-dependencies]

avrotize/dependencies/typescript/node22/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "avrotize-dependencies",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Central dependency versions for Avrotize TypeScript code generation templates",
+  "dependencies": {
+    "avsc": "^5.7.7",
+    "kafkajs": "^2.2.4"
+  },
+  "devDependencies": {
+    "@types/node": "^24.10.1",
+    "typescript": "^5.7.2",
+    "jest": "^30.2.0",
+    "@types/jest": "^30.0.0"
+  }
+}