avrokit 0.0.7__tar.gz → 0.1.0__tar.gz

{avrokit-0.0.7 → avrokit-0.1.0}/Makefile

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-.PHONY: install test test-coverage lint lint-fix typecheck format build publish lock check license-check bump help
+.PHONY: install test test-coverage lint fix typecheck format build publish lock check license-check bump help
 
 help: ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
@@ -25,9 +25,6 @@ test-coverage: ## Run tests with coverage report
 lint: ## Run ruff linter
 	uv run ruff check .
 
-lint-fix: ## Run ruff linter and apply fixes
-	uv run ruff check --fix .
-
 typecheck: ## Run type checker
 	uv run ty check
 
@@ -40,6 +37,10 @@ bump: ## Bump version: make bump part=patch|minor|major
 lock: ## Update the lockfile
 	uv lock
 
+fix: ## Run ruff linter and apply fixes
+	uv run ruff format .
+	uv run ruff check --fix .
+
 check: lint typecheck format test license-check ## Run all checks (lint, typecheck, format, test)
 
 license-check: ## Check that all files have REUSE license headers

{avrokit-0.0.7 → avrokit-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avrokit
-Version: 0.0.7
+Version: 0.1.0
 Summary: Python utilities for working with Avro data files
 Project-URL: Homepage, https://github.com/brandtg/avrokit
 Project-URL: Repository, https://github.com/brandtg/avrokit
@@ -12,10 +12,12 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Requires-Dist: avro>=1.12.1
 Requires-Dist: boto3-stubs[s3]>=1.38.8
 Requires-Dist: datetime-truncate>=1.1.1
@@ -23,6 +25,7 @@ Requires-Dist: filelock>=3.18.0
 Requires-Dist: pyarrow>=22.0.0
 Requires-Dist: requests>=2.32.3
 Requires-Dist: types-requests>=2.32.0.20250328
+Requires-Dist: typing-extensions>=4.12.0
 Provides-Extra: all
 Requires-Dist: boto3>=1.38.7; extra == 'all'
 Requires-Dist: google-cloud-storage>=3.6.0; extra == 'all'
@@ -80,7 +83,6 @@ pip install avrokit[all]
 ```bash
 git clone https://github.com/brandtg/avrokit.git
 cd avrokit
-poetry env use python3.12
 make install
 ```
 
@@ -321,15 +323,12 @@ avrokit toparquet INPUT.avro OUTPUT.parquet
 ### Prerequisites
 
 - Python 3.12+
-- Poetry
+- uv
 - Docker (for GCS/S3 integration tests)
 
 ### Setup
 
 ```bash
-# Set up Python environment
-poetry env use python3.12
-
 # Install dependencies
 make install
 
@@ -347,7 +346,7 @@ make test
 make test-coverage
 
 # Run tests in parallel (default)
-poetry run pytest -n auto
+uv run pytest -n auto
 ```
 
 ### Code Quality

{avrokit-0.0.7 → avrokit-0.1.0}/README.md

@@ -44,7 +44,6 @@ pip install avrokit[all]
 ```bash
 git clone https://github.com/brandtg/avrokit.git
 cd avrokit
-poetry env use python3.12
 make install
 ```
 
@@ -285,15 +284,12 @@ avrokit toparquet INPUT.avro OUTPUT.parquet
 ### Prerequisites
 
 - Python 3.12+
-- Poetry
+- uv
 - Docker (for GCS/S3 integration tests)
 
 ### Setup
 
 ```bash
-# Set up Python environment
-poetry env use python3.12
-
 # Install dependencies
 make install
 
@@ -311,7 +307,7 @@ make test
 make test-coverage
 
 # Run tests in parallel (default)
-poetry run pytest -n auto
+uv run pytest -n auto
 ```
 
 ### Code Quality

{avrokit-0.0.7 → avrokit-0.1.0}/avrokit/io/reader.py

@@ -6,10 +6,11 @@ from __future__ import annotations
 
 from collections.abc import Generator, Iterator, Sequence
 from contextlib import contextmanager
-from typing import IO, Any, Self, cast
+from typing import IO, Any, cast
 
 from avro.datafile import DataFileReader
 from avro.io import DatumReader
+from typing_extensions import Self
 
 from ..url import URL
 

{avrokit-0.0.7 → avrokit-0.1.0}/avrokit/io/writer.py

@@ -10,18 +10,13 @@ from collections.abc import Generator, Mapping, Sequence
 from contextlib import contextmanager
 from datetime import datetime
 from itertools import groupby
-from typing import (
-    IO,
-    Any,
-    Literal,
-    Protocol,
-    Self,
-)
+from typing import IO, Any, Literal, Protocol
 
 from avro.datafile import NULL_CODEC, DataFileWriter
 from avro.io import DatumWriter
 from avro.schema import Schema
 from datetime_truncate import truncate as truncate_datetime
+from typing_extensions import Self
 
 from avrokit.url.utils import flatten_urls
 

{avrokit-0.0.7 → avrokit-0.1.0}/avrokit/url/file.py

@@ -6,7 +6,9 @@ import glob
 import os
 import shutil
 from collections.abc import Sequence
-from typing import IO, Any, override
+from typing import IO, Any
+
+from typing_extensions import override
 
 from .base import URL
 

{avrokit-0.0.7 → avrokit-0.1.0}/avrokit/url/google.py

@@ -7,11 +7,12 @@ import os
 import tempfile
 from collections.abc import Generator, Sequence
 from contextlib import contextmanager
-from typing import IO, Any, cast, override
+from typing import IO, Any, cast
 
 from google.api_core.exceptions import NotFound
 from google.auth.credentials import AnonymousCredentials
 from google.cloud import storage
+from typing_extensions import override
 
 from .base import URL
 

{avrokit-0.0.7 → avrokit-0.1.0}/avrokit/url/http.py

@@ -5,9 +5,10 @@
 import io
 import tempfile
 from collections.abc import Sequence
-from typing import IO, Any, override
+from typing import IO, Any
 
 import requests
+from typing_extensions import override
 
 from .base import URL
 
@@ -57,9 +58,12 @@ class HttpURL(URL):
         try:
             response = requests.head(self.url, timeout=5)
             if response.ok:
-                return int(response.headers.get("Content-Length", 0))
+                content_length = response.headers.get("Content-Length")
+                if content_length is not None:
+                    return int(content_length)
+                return 0
             return 0
-        except requests.RequestException:
+        except (requests.RequestException, ValueError):
             return 0
 
     @override

{avrokit-0.0.7 → avrokit-0.1.0}/avrokit/url/s3.py

@@ -6,10 +6,11 @@ import io
 import tempfile
 from collections.abc import Sequence
 from contextlib import contextmanager
-from typing import IO, Any, cast, override
+from typing import IO, Any, cast
 
 import boto3
 from mypy_boto3_s3 import S3Client
+from typing_extensions import override
 
 from .base import URL
 

{avrokit-0.0.7 → avrokit-0.1.0}/pyproject.toml

@@ -4,18 +4,20 @@
 
 [project]
 name = "avrokit"
-version = "0.0.7"
+version = "0.1.0"
 description = "Python utilities for working with Avro data files"
 authors = [{ name = "Greg Brandt", email = "brandt.greg@gmail.com" }]
 license = "Apache-2.0"
 readme = "README.md"
-requires-python = ">=3.12"
+requires-python = ">=3.10"
 keywords = ["avro", "data", "serialization", "gcs", "s3", "parquet"]
 classifiers = [
   "Development Status :: 3 - Alpha",
   "Intended Audience :: Developers",
   "License :: OSI Approved :: Apache Software License",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Topic :: Software Development :: Libraries :: Python Modules",
   "Typing :: Typed",
@@ -28,6 +30,7 @@ dependencies = [
   "boto3-stubs[s3]>=1.38.8",
   "types-requests>=2.32.0.20250328",
   "pyarrow>=22.0.0",
+  "typing-extensions>=4.12.0",
 ]
 
 [project.optional-dependencies]
@@ -66,6 +69,7 @@ filterwarnings = ["ignore::DeprecationWarning:botocore.*"]
 
 [tool.ruff]
 exclude = []
+target-version = "py310"
 
 [tool.ruff.lint]
 select = ["E", "F", "I", "UP"]

{avrokit-0.0.7 → avrokit-0.1.0}/tests/test_io_schema.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# SPDX-License-Identifier: Apache-2.0
+
 import pytest
 from avro.schema import RecordSchema
 
@@ -107,10 +109,10 @@ from avrokit.io.schema import add_avro_schema_fields, validate_avro_schema_evolu
                 "fields": [
                     {
                         "name": "status",
+                        # Added a new symbol 'PENDING'
                         "type": {
                             "type": "enum",
                             "name": "Status",
-                            # Added a new symbol 'PENDING'
                             "symbols": ["ACTIVE", "INACTIVE", "PENDING"],
                         },
                     }
@@ -123,21 +125,15 @@ from avrokit.io.schema import add_avro_schema_fields, validate_avro_schema_evolu
                 "name": "Record",
                 "type": "record",
                 "fields": [
-                    {
-                        "name": "status",
-                        "type": ["null", "string"],
-                    }
+                    {"name": "status", "type": ["null", "string"]},
                 ],
             },
             {
                 "name": "Record",
                 "type": "record",
                 "fields": [
-                    {
-                        "name": "status",
-                        # Added a new type 'int' to the union
-                        "type": ["null", "string", "int"],
-                    }
+                    # Added a new type 'int' to the union
+                    {"name": "status", "type": ["null", "string", "int"]},
                 ],
             },
         ),
@@ -223,10 +219,10 @@ def test_validate_avro_schema_evolution_valid(schema_a: dict, schema_b: dict):
                 "fields": [
                     {
                         "name": "status",
+                        # Removed the symbol 'PENDING'
                         "type": {
                             "type": "enum",
                             "name": "Status",
-                            # Removed the symbol 'PENDING'
                             "symbols": ["ACTIVE", "INACTIVE"],
                         },
                     }
@@ -239,21 +235,15 @@ def test_validate_avro_schema_evolution_valid(schema_a: dict, schema_b: dict):
                 "name": "Record",
                 "type": "record",
                 "fields": [
-                    {
-                        "name": "status",
-                        "type": ["null", "string", "int"],
-                    }
+                    {"name": "status", "type": ["null", "string", "int"]},
                 ],
             },
             {
                 "name": "Record",
                 "type": "record",
                 "fields": [
-                    {
-                        "name": "status",
-                        # Removed the type 'int' from the union
-                        "type": ["null", "string"],
-                    }
+                    # Removed the type 'int' from the union
+                    {"name": "status", "type": ["null", "string"]},
                 ],
             },
         ),
@@ -310,3 +300,216 @@ def test_add_avro_schema_fields():
         {"name": "address", "type": "string"},
         {"name": "phone", "type": "string"},
     ]
+
+
+class TestSchemaEvolutionEdgeCases:
+    def test_enum_symbol_removal_raises_error(self):
+        """Test that narrowing enum symbols raises ValueError."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {
+                    "name": "status",
+                    "type": {
+                        "type": "enum",
+                        "name": "Status",
+                        "symbols": ["ACTIVE", "INACTIVE", "PENDING"],
+                    },
+                },
+            ],
+        }
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {
+                    "name": "status",
+                    "type": {"type": "enum", "name": "Status", "symbols": ["ACTIVE"]},
+                },
+            ],
+        }
+
+        with pytest.raises(ValueError, match="Field status enum has changed"):
+            validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))
+
+    def test_union_type_removal_raises_error(self):
+        """Test that removing types from union raises ValueError."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [{"name": "status", "type": ["null", "string", "int"]}],
+        }
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [{"name": "status", "type": ["null", "string"]}],
+        }
+
+        with pytest.raises(ValueError, match="Field status union has changed"):
+            validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))
+
+    def test_deeply_nested_record_addition(self):
+        """Test schema evolution through 3+ levels of nested records."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {"name": "user_id", "type": "int"},
+                {
+                    "name": "profile",
+                    "type": {
+                        "name": "Profile",
+                        "type": "record",
+                        "fields": [
+                            {"name": "email", "type": "string"},
+                            {
+                                "name": "address",
+                                "type": {
+                                    "name": "Address",
+                                    "type": "record",
+                                    "fields": [{"name": "city", "type": "string"}],
+                                },
+                            },
+                        ],
+                    },
+                },
+            ],
+        }
+
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {"name": "user_id", "type": "int"},
+                {
+                    "name": "profile",
+                    "type": {
+                        "name": "Profile",
+                        "type": "record",
+                        "fields": [
+                            {"name": "email", "type": "string"},
+                            {
+                                "name": "address",
+                                "type": {
+                                    "name": "Address",
+                                    "type": "record",
+                                    "fields": [{"name": "city", "type": "string"}],
+                                },
+                            },
+                            {
+                                "name": "country",
+                                "type": ["null", "string"],
+                                "default": None,
+                            },
+                        ],
+                    },
+                },
+            ],
+        }
+
+        validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))
+
+    def test_deeply_nested_record_field_removal(self):
+        """Test that removing deeply nested field without default raises error."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {"name": "user_id", "type": "int"},
+                {
+                    "name": "profile",
+                    "type": {
+                        "name": "Profile",
+                        "type": "record",
+                        "fields": [{"name": "email", "type": "string"}],
+                    },
+                },
+            ],
+        }
+
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {"name": "user_id", "type": "int"},
+                {
+                    "name": "profile",
+                    "type": {
+                        "name": "Profile",
+                        "type": "record",
+                        "fields": [],
+                    },
+                },
+            ],
+        }
+
+        with pytest.raises(
+            ValueError, match="Field profile.email is missing a default value"
+        ):
+            validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))
+
+    def test_mixed_union_complex_type_changes(self):
+        """Test evolution of unions containing records and enums."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {
+                    "name": "value",
+                    "type": [
+                        "null",
+                        {"type": "enum", "name": "Color", "symbols": ["RED", "GREEN"]},
+                    ],
+                },
+            ],
+        }
+
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [
+                {
+                    "name": "value",
+                    "type": [
+                        "null",
+                        {"type": "enum", "name": "Color", "symbols": ["RED"]},
+                    ],
+                },
+            ],
+        }
+
+        with pytest.raises(ValueError, match="Field value union has changed"):
+            validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))
+
+    def test_field_type_change_to_non_union_raises_error(self):
+        """Test that changing field type to non-union raises error."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [{"name": "count", "type": "int"}],
+        }
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [{"name": "count", "type": "string"}],
+        }
+
+        with pytest.raises(ValueError, match="Field count type has changed"):
+            validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))
+
+    def test_field_type_change_to_union_without_null_raises_error(self):
+        """Test that changing to union without null raises error."""
+        schema_a = {
+            "name": "Record",
+            "type": "record",
+            "fields": [{"name": "count", "type": "int"}],
+        }
+        schema_b = {
+            "name": "Record",
+            "type": "record",
+            "fields": [{"name": "count", "type": ["string", "long"]}],
+        }
+
+        with pytest.raises(ValueError, match="Field count type has changed"):
+            validate_avro_schema_evolution(avro_schema(schema_a), avro_schema(schema_b))

{avrokit-0.0.7 → avrokit-0.1.0}/tests/test_tools_httpserver.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import concurrent.futures as futures
 import json
 import tempfile
 import threading
@@ -158,3 +159,51 @@ class TestHttpServerTool:
         # Check that the file is gone
         res = requests.get(f"{http_url}/newfile.avro")
         assert res.status_code == 404
+
+    def test_invalid_accept_header(self, tmpserver):
+        http_url, _ = tmpserver
+        res = requests.get(
+            f"{http_url}/test.avro", headers={"Accept": "application/unknown"}
+        )
+        assert res.status_code in [406, 200]
+
+    def test_concurrent_requests(self, tmpserver):
+        http_url, _ = tmpserver
+
+        def make_request():
+            res = requests.get(
+                f"{http_url}/test.avro", headers={"Accept": "application/json"}
+            )
+            data = [
+                json.loads(line) for line in res.content.decode("utf-8").splitlines()
+            ]
+            return len(data)
+
+        with futures.ThreadPoolExecutor(max_workers=10) as executor:
+            results = list(executor.map(lambda _: make_request(), range(5)))
+
+        assert all(isinstance(r, int) for r in results), (
+            "All results should be integers"
+        )
+        assert len(results) == 5, f"Expected 5 results, got {len(results)}"
+        expected_count = 10
+        for result in results:
+            assert result == expected_count, (
+                f"Expected count {expected_count}, got {result}"
+            )
+
+    def test_file_not_found(self, tmpserver):
+        http_url, _ = tmpserver
+        res = requests.get(f"{http_url}/nonexistent.avro")
+        assert res.status_code == 404
+
+    def test_put_invalid_schema_json(self, tmpserver):
+        http_url, root_url = tmpserver
+        res = requests.put(f"{http_url}/invalid_file.avro", json={"type": "recordy"})
+        assert res.status_code in [400, 500]
+
+
+def test_post_to_nonexistent_file(tmpserver):
+    http_url, root_url = tmpserver
+    res = requests.post(f"{http_url}/newfile.avro", json={"name": "test"})
+    assert res.status_code in [201, 405, 404]