avrokit 0.1.3__tar.gz → 0.1.4__tar.gz

{avrokit-0.1.3 → avrokit-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avrokit
-Version: 0.1.3
+Version: 0.1.4
 Summary: Python utilities for working with Avro data files
 Project-URL: Homepage, https://github.com/brandtg/avrokit
 Project-URL: Repository, https://github.com/brandtg/avrokit
@@ -116,24 +116,23 @@ avrokit concat file1.avro file2.avro s3://bucket/file3.avro output.avro
 from avrokit import avro_reader, avro_records, avro_writer, avro_schema, parse_url
 
 # Read records from any URL
-url = parse_url('gs://bucket/data.avro', mode='rb')
+url = parse_url("gs://bucket/data.avro", mode="rb")
 for record in avro_records(url):
     print(record)
 
 # Write Avro data
-schema = avro_schema({
-    'type': 'record',
-    'name': 'User',
-    'fields': [
-        {'name': 'name', 'type': 'string'},
-        {'name': 'age', 'type': 'int'}
-    ]
-})
-
-url = parse_url('output.avro', mode='wb')
+schema = avro_schema(
+    {
+        "type": "record",
+        "name": "User",
+        "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}],
+    }
+)
+
+url = parse_url("output.avro", mode="wb")
 with avro_writer(url, schema) as writer:
-    writer.append({'name': 'Alice', 'age': 30})
-    writer.append({'name': 'Bob', 'age': 25})
+    writer.append({"name": "Alice", "age": 30})
+    writer.append({"name": "Bob", "age": 25})
 ```
 
 ## Architecture
@@ -156,10 +155,10 @@ The `parse_url()` factory function automatically instantiates the correct URL cl
 from avrokit.url import parse_url
 
 # All of these return the appropriate URL subclass
-url1 = parse_url('file:///path/to/data.avro')      # FileURL
-url2 = parse_url('gs://bucket/data.avro')           # GCSURL
-url3 = parse_url('s3://bucket/data.avro')           # S3URL
-url4 = parse_url('https://example.com/data.avro')   # HTTPURL
+url1 = parse_url("file:///path/to/data.avro")  # FileURL
+url2 = parse_url("gs://bucket/data.avro")  # GCSURL
+url3 = parse_url("s3://bucket/data.avro")  # S3URL
+url4 = parse_url("https://example.com/data.avro")  # HTTPURL
 ```
 
 ### 2. I/O Layer (`avrokit.io`)
@@ -401,13 +400,13 @@ The GitHub Actions workflow will automatically build and publish to PyPI.
 from avrokit import PartitionedAvroReader, PartitionedAvroWriter, parse_url
 
 # Read from multiple files as single stream
-url = parse_url('data-*.avro', mode='rb')
+url = parse_url("data-*.avro", mode="rb")
 with PartitionedAvroReader(url) as reader:
     for record in reader:
         process(record)
 
 # Write to multiple files (roll every 10000 records)
-url = parse_url('output/', mode='wb')
+url = parse_url("output/", mode="wb")
 with PartitionedAvroWriter(url, schema, max_records=10000) as writer:
     for record in records:
         writer.append(record)
@@ -422,15 +421,12 @@ from avrokit import TimePartitionedAvroWriter, parse_url
 from datetime import datetime
 
 # Creates hourly files: output/2024/01/15/10.avro
-url = parse_url('output/', mode='wb')
+url = parse_url("output/", mode="wb")
 with TimePartitionedAvroWriter(
-    url,
-    schema,
-    time_granularity='hour',
-    time_field='timestamp'
+    url, schema, time_granularity="hour", time_field="timestamp"
 ) as writer:
     for record in records:
-        writer.append(record, timestamp=record['timestamp'])
+        writer.append(record, timestamp=record["timestamp"])
 ```
 
 ### Async Operations
@@ -440,8 +436,9 @@ import asyncio
 from avrokit.asyncio import DeferredAvroWriter
 from avrokit import parse_url, avro_schema
 
+
 async def write_async():
-    url = parse_url('output.avro', mode='wb')
+    url = parse_url("output.avro", mode="wb")
     schema = avro_schema({...})
 
     async with DeferredAvroWriter(url, schema) as writer:
@@ -449,6 +446,7 @@ async def write_async():
             await writer.append(record)
         # Flushes happen automatically in background
 
+
 asyncio.run(write_async())
 ```
 
@@ -459,13 +457,13 @@ from avrokit import validate_avro_schema_evolution, read_avro_schema
 from avrokit.url import parse_url
 
 # Check if new schema is backward compatible
-reader_schema = read_avro_schema(parse_url('old.avro', 'rb'))
-writer_schema = read_avro_schema(parse_url('new.avro', 'rb'))
+reader_schema = read_avro_schema(parse_url("old.avro", "rb"))
+writer_schema = read_avro_schema(parse_url("new.avro", "rb"))
 
 is_valid = validate_avro_schema_evolution(
     reader_schema,
     writer_schema,
-    strategy='backward'  # or 'forward', 'full'
+    strategy="backward",  # or 'forward', 'full'
 )
 ```
 

{avrokit-0.1.3 → avrokit-0.1.4}/README.md

@@ -77,24 +77,23 @@ avrokit concat file1.avro file2.avro s3://bucket/file3.avro output.avro
 from avrokit import avro_reader, avro_records, avro_writer, avro_schema, parse_url
 
 # Read records from any URL
-url = parse_url('gs://bucket/data.avro', mode='rb')
+url = parse_url("gs://bucket/data.avro", mode="rb")
 for record in avro_records(url):
     print(record)
 
 # Write Avro data
-schema = avro_schema({
-    'type': 'record',
-    'name': 'User',
-    'fields': [
-        {'name': 'name', 'type': 'string'},
-        {'name': 'age', 'type': 'int'}
-    ]
-})
-
-url = parse_url('output.avro', mode='wb')
+schema = avro_schema(
+    {
+        "type": "record",
+        "name": "User",
+        "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}],
+    }
+)
+
+url = parse_url("output.avro", mode="wb")
 with avro_writer(url, schema) as writer:
-    writer.append({'name': 'Alice', 'age': 30})
-    writer.append({'name': 'Bob', 'age': 25})
+    writer.append({"name": "Alice", "age": 30})
+    writer.append({"name": "Bob", "age": 25})
 ```
 
 ## Architecture
@@ -117,10 +116,10 @@ The `parse_url()` factory function automatically instantiates the correct URL cl
 from avrokit.url import parse_url
 
 # All of these return the appropriate URL subclass
-url1 = parse_url('file:///path/to/data.avro')      # FileURL
-url2 = parse_url('gs://bucket/data.avro')           # GCSURL
-url3 = parse_url('s3://bucket/data.avro')           # S3URL
-url4 = parse_url('https://example.com/data.avro')   # HTTPURL
+url1 = parse_url("file:///path/to/data.avro")  # FileURL
+url2 = parse_url("gs://bucket/data.avro")  # GCSURL
+url3 = parse_url("s3://bucket/data.avro")  # S3URL
+url4 = parse_url("https://example.com/data.avro")  # HTTPURL
 ```
 
 ### 2. I/O Layer (`avrokit.io`)
@@ -362,13 +361,13 @@ The GitHub Actions workflow will automatically build and publish to PyPI.
 from avrokit import PartitionedAvroReader, PartitionedAvroWriter, parse_url
 
 # Read from multiple files as single stream
-url = parse_url('data-*.avro', mode='rb')
+url = parse_url("data-*.avro", mode="rb")
 with PartitionedAvroReader(url) as reader:
     for record in reader:
         process(record)
 
 # Write to multiple files (roll every 10000 records)
-url = parse_url('output/', mode='wb')
+url = parse_url("output/", mode="wb")
 with PartitionedAvroWriter(url, schema, max_records=10000) as writer:
     for record in records:
         writer.append(record)
@@ -383,15 +382,12 @@ from avrokit import TimePartitionedAvroWriter, parse_url
 from datetime import datetime
 
 # Creates hourly files: output/2024/01/15/10.avro
-url = parse_url('output/', mode='wb')
+url = parse_url("output/", mode="wb")
 with TimePartitionedAvroWriter(
-    url,
-    schema,
-    time_granularity='hour',
-    time_field='timestamp'
+    url, schema, time_granularity="hour", time_field="timestamp"
 ) as writer:
     for record in records:
-        writer.append(record, timestamp=record['timestamp'])
+        writer.append(record, timestamp=record["timestamp"])
 ```
 
 ### Async Operations
@@ -401,8 +397,9 @@ import asyncio
 from avrokit.asyncio import DeferredAvroWriter
 from avrokit import parse_url, avro_schema
 
+
 async def write_async():
-    url = parse_url('output.avro', mode='wb')
+    url = parse_url("output.avro", mode="wb")
     schema = avro_schema({...})
 
     async with DeferredAvroWriter(url, schema) as writer:
@@ -410,6 +407,7 @@ async def write_async():
             await writer.append(record)
         # Flushes happen automatically in background
 
+
 asyncio.run(write_async())
 ```
 
@@ -420,13 +418,13 @@ from avrokit import validate_avro_schema_evolution, read_avro_schema
 from avrokit.url import parse_url
 
 # Check if new schema is backward compatible
-reader_schema = read_avro_schema(parse_url('old.avro', 'rb'))
-writer_schema = read_avro_schema(parse_url('new.avro', 'rb'))
+reader_schema = read_avro_schema(parse_url("old.avro", "rb"))
+writer_schema = read_avro_schema(parse_url("new.avro", "rb"))
 
 is_valid = validate_avro_schema_evolution(
     reader_schema,
     writer_schema,
-    strategy='backward'  # or 'forward', 'full'
+    strategy="backward",  # or 'forward', 'full'
 )
 ```
 

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/asyncio/writer.py

@@ -43,6 +43,20 @@ class DeferredAvroWriter:
         self._writer_thread_done.set()
         self._writer_thread.join()
 
+    def close(self) -> None:
+        self.stop()
+        # Close the underlying writer if it supports it (e.g. DataFileWriter)
+        close_fn = getattr(self.writer, "close", None)
+        if close_fn:
+            close_fn()
+
+    def __enter__(self) -> "DeferredAvroWriter":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
     def append(
         self, datum: object, block: bool = True, timeout: int | None = None
     ) -> None:

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/io/reader.py

@@ -85,6 +85,8 @@ class PartitionedAvroReader:
         if self.current_reader:
             self.current_reader.close()
             self.current_reader = None
+        if self.current_url:
+            self.current_url.close()
         self.current_url_stream = None
         self.current_index = 0
 

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/io/writer.py

@@ -128,6 +128,8 @@ class PartitionedAvroWriter:
         if self.current_writer:
             self.current_writer.close()
             self.current_writer = None
+        if self.current_url:
+            self.current_url.close()
         self.current_url_stream = None
         self.current_url = None
 

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/tools/concat.py

@@ -80,11 +80,13 @@ class ConcatTool:
 
         output_sync_marker = None
 
-        with output_url.with_mode("wb").open() as out_f:
+        out_url = output_url.with_mode("wb")
+        with out_url as out_f:
             writer = BinaryEncoder(out_f)
 
             for i, url in enumerate(input_urls):
-                with url.with_mode("rb").open() as in_f:
+                in_url = url.with_mode("rb")
+                with in_url as in_f:
                     # Initialize DataFileReader to parse the header/metadata and
                     # position the file pointer at the start of the first block.
                     reader = DataFileReader(in_f, DatumReader())

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/tools/filesort.py

@@ -90,7 +90,7 @@ class FileSortTool:
         schema: Schema,
         root: str,
         batch_id: int,
-        records: list[object],
+        records: list[dict[str, Any]],
         sort_key: Callable[[Any], Any],
         reverse: bool = False,
     ) -> None:

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/tools/repair.py

@@ -147,6 +147,7 @@ class RepairTool:
                 if dry_run:
                     tmp = tempfile.NamedTemporaryFile(delete=False)
                     output_url = parse_url(tmp.name, mode="wb")
+                    tmp.close()  # Close the handle; the URL will open it fresh
                     logger.debug("Using temporary file %s for dry run", output_url)
                 # Initialize the output file with the same schema and codec
                 with avro_writer(output_url, schema, codec=codec) as writer:

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/url/http.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import io
+import os
 import tempfile
 from collections.abc import Sequence
 from typing import IO, Any
@@ -34,6 +35,7 @@ class HttpURL(URL):
         self.content_type = content_type
         self._current_response: requests.Response | None = None
         self._current_request_buffer: IO[Any] | None = None
+        self._current_request_temp_path: str | None = None
 
     @override
     def _expand(self) -> Sequence[URL]:
@@ -84,9 +86,11 @@ class HttpURL(URL):
             if self.spill_request_to_file:
                 # Use a temporary file to store the request body
                 buf = tempfile.NamedTemporaryFile(delete=False)
+                self._current_request_temp_path = buf.name
             else:
                 # Use an in-memory buffer
                 buf = io.BytesIO()
+                self._current_request_temp_path = None
             # Wrap in TextIOWrapper if not binary mode
             if "b" not in self.mode:
                 buf = io.TextIOWrapper(buf, encoding="utf-8")
@@ -112,8 +116,15 @@ class HttpURL(URL):
                 headers={"Content-Type": self.content_type},
             )
             res.raise_for_status()
-            # Close the buffer (will delete temp file if used)
+            # Close the buffer
             self._current_request_buffer.close()
+            # Unlink temp file if used
+            if self._current_request_temp_path is not None:
+                try:
+                    os.unlink(self._current_request_temp_path)
+                except FileNotFoundError:
+                    pass
+                self._current_request_temp_path = None
 
     @override
     def with_mode(self, mode) -> URL:

{avrokit-0.1.3 → avrokit-0.1.4}/avrokit/url/s3.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import io
+import os
 import tempfile
 from collections.abc import Sequence
 from contextlib import contextmanager
@@ -125,7 +126,16 @@ class S3URL(URL):
                         self._current_client.upload_fileobj(f, self.bucket, self.path)
             finally:
                 # Cleanup resources
-                self._current_local.close()
+                local = self._current_local
+                name = local.name
+                local.close()
+                # N.b. the temp file is created with delete=False so it survives an external
+                # close (e.g. DataFileWriter closing the stream before URL.close() uploads).
+                # Unlink it explicitly here to avoid leaking a file per open/close cycle.
+                try:
+                    os.unlink(name)
+                except FileNotFoundError:
+                    pass
                 self._current_client.close()
                 self._current_local = None
                 self._current_client = None

{avrokit-0.1.3 → avrokit-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@
 
 [project]
 name = "avrokit"
-version = "0.1.3"
+version = "0.1.4"
 description = "Python utilities for working with Avro data files"
 authors = [{ name = "Greg Brandt", email = "brandt.greg@gmail.com" }]
 license = "Apache-2.0"