avrokit 0.1.3__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. avrokit-0.1.5/.github/workflows/release.yml +65 -0
  2. {avrokit-0.1.3 → avrokit-0.1.5}/PKG-INFO +45 -33
  3. {avrokit-0.1.3 → avrokit-0.1.5}/README.md +40 -31
  4. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/asyncio/writer.py +14 -0
  5. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/reader.py +2 -0
  6. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/writer.py +2 -0
  7. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/concat.py +4 -2
  8. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/filesort.py +1 -1
  9. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/repair.py +1 -0
  10. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/http.py +12 -1
  11. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/s3.py +11 -1
  12. {avrokit-0.1.3 → avrokit-0.1.5}/pyproject.toml +3 -2
  13. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_io.py +19 -0
  14. avrokit-0.1.5/uv.lock +1793 -0
  15. avrokit-0.1.3/.github/workflows/release.yml +0 -31
  16. avrokit-0.1.3/uv.lock +0 -1533
  17. {avrokit-0.1.3 → avrokit-0.1.5}/.github/workflows/check.yml +0 -0
  18. {avrokit-0.1.3 → avrokit-0.1.5}/.gitignore +0 -0
  19. {avrokit-0.1.3 → avrokit-0.1.5}/LICENSE +0 -0
  20. {avrokit-0.1.3 → avrokit-0.1.5}/LICENSES/Apache-2.0.txt +0 -0
  21. {avrokit-0.1.3 → avrokit-0.1.5}/Makefile +0 -0
  22. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/__init__.py +0 -0
  23. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/asyncio/__init__.py +0 -0
  24. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/asyncio/reader.py +0 -0
  25. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/__init__.py +0 -0
  26. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/compact.py +0 -0
  27. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/schema.py +0 -0
  28. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/py.typed +0 -0
  29. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/__init__.py +0 -0
  30. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/__main__.py +0 -0
  31. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/base.py +0 -0
  32. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/cat.py +0 -0
  33. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/count.py +0 -0
  34. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/fromparquet.py +0 -0
  35. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/getmeta.py +0 -0
  36. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/getschema.py +0 -0
  37. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/httpserver.py +0 -0
  38. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/partition.py +0 -0
  39. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/stats.py +0 -0
  40. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/tojson.py +0 -0
  41. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/toparquet.py +0 -0
  42. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/__init__.py +0 -0
  43. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/base.py +0 -0
  44. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/factory.py +0 -0
  45. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/file.py +0 -0
  46. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/google.py +0 -0
  47. {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/utils.py +0 -0
  48. {avrokit-0.1.3 → avrokit-0.1.5}/tests/__init__.py +0 -0
  49. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_additional_coverage.py +0 -0
  50. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_asyncio.py +0 -0
  51. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_exports.py +0 -0
  52. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_io_schema.py +0 -0
  53. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_plan_coverage.py +0 -0
  54. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_property_based.py +0 -0
  55. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_schema_advanced.py +0 -0
  56. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_tools_httpserver.py +0 -0
  57. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_tools_integration.py +0 -0
  58. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_tools_repair.py +0 -0
  59. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url.py +0 -0
  60. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_contract.py +0 -0
  61. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_file.py +0 -0
  62. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_google.py +0 -0
  63. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_http.py +0 -0
  64. {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_s3.py +0 -0
  65. {avrokit-0.1.3 → avrokit-0.1.5}/uv.lock.license +0 -0
@@ -0,0 +1,65 @@
1
+ # SPDX-FileCopyrightText: 2026 Greg Brandt <brandt.greg@gmail.com>
2
+ #
3
+ # SPDX-License-Identifier: Apache-2.0
4
+
5
+ name: Publish to PyPI
6
+
7
+ on:
8
+ release:
9
+ types: [published]
10
+
11
+ permissions:
12
+ id-token: write # This allows the OIDC authentication
13
+ contents: read
14
+
15
+ jobs:
16
+ build-and-publish:
17
+ runs-on: ubuntu-latest
18
+ environment:
19
+ name: pypi
20
+ url: https://pypi.org/p/avrokit
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+
24
+ - name: Set up uv
25
+ uses: astral-sh/setup-uv@v5
26
+
27
+ - name: Build package
28
+ run: uv build
29
+
30
+ - name: Verify wheel extras match pyproject.toml
31
+ run: |
32
+ uv run --no-project --python 3.12 python - <<'EOF'
33
+ import email
34
+ import sys
35
+ import tomllib
36
+ import zipfile
37
+ from pathlib import Path
38
+
39
+ expected = set(
40
+ tomllib.loads(Path("pyproject.toml").read_text())["project"][
41
+ "optional-dependencies"
42
+ ]
43
+ )
44
+
45
+ wheels = sorted(Path("dist").glob("*.whl"))
46
+ if not wheels:
47
+ print("No wheel found in dist/")
48
+ sys.exit(1)
49
+ wheel = wheels[0]
50
+ with zipfile.ZipFile(wheel) as zf:
51
+ name = next(n for n in zf.namelist() if n.endswith(".dist-info/METADATA"))
52
+ metadata = email.message_from_bytes(zf.read(name))
53
+ actual = set(metadata.get_all("Provides-Extra") or [])
54
+
55
+ if actual != expected:
56
+ print(
57
+ f"Extras mismatch for {wheel.name}: "
58
+ f"pyproject.toml={sorted(expected)} wheel={sorted(actual)}"
59
+ )
60
+ sys.exit(1)
61
+ print(f"Wheel extras match pyproject.toml: {sorted(actual)}")
62
+ EOF
63
+
64
+ - name: Publish to PyPI
65
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: avrokit
3
- Version: 0.1.3
3
+ Version: 0.1.5
4
4
  Summary: Python utilities for working with Avro data files
5
5
  Project-URL: Homepage, https://github.com/brandtg/avrokit
6
6
  Project-URL: Repository, https://github.com/brandtg/avrokit
@@ -30,11 +30,14 @@ Provides-Extra: all
30
30
  Requires-Dist: boto3>=1.38.7; extra == 'all'
31
31
  Requires-Dist: google-cloud-storage>=3.6.0; extra == 'all'
32
32
  Requires-Dist: google-crc32c>=1.7.1; extra == 'all'
33
+ Requires-Dist: python-snappy>=0.7.3; extra == 'all'
33
34
  Provides-Extra: aws
34
35
  Requires-Dist: boto3>=1.38.7; extra == 'aws'
35
36
  Provides-Extra: gcp
36
37
  Requires-Dist: google-cloud-storage>=3.6.0; extra == 'gcp'
37
38
  Requires-Dist: google-crc32c>=1.7.1; extra == 'gcp'
39
+ Provides-Extra: snappy
40
+ Requires-Dist: python-snappy>=0.7.3; extra == 'snappy'
38
41
  Description-Content-Type: text/markdown
39
42
 
40
43
  <!--
@@ -65,7 +68,15 @@ Python utilities for working with Avro data files with support for local files,
65
68
  pip install avrokit
66
69
  ```
67
70
 
68
- ### With Cloud Storage Support
71
+ ### With pipx (CLI tools)
72
+
73
+ ```bash
74
+ # Installs the avrokit CLI in an isolated environment,
75
+ # with support for snappy-compressed Avro files
76
+ pipx install avrokit[snappy]
77
+ ```
78
+
79
+ ### With Optional Extras
69
80
 
70
81
  ```bash
71
82
  # AWS S3 support
@@ -74,7 +85,10 @@ pip install avrokit[aws]
74
85
  # Google Cloud Storage support
75
86
  pip install avrokit[gcp]
76
87
 
77
- # All extras (S3 + GCS)
88
+ # Snappy codec support (for reading snappy-compressed files)
89
+ pip install avrokit[snappy]
90
+
91
+ # All extras (S3 + GCS + snappy)
78
92
  pip install avrokit[all]
79
93
  ```
80
94
 
@@ -116,24 +130,23 @@ avrokit concat file1.avro file2.avro s3://bucket/file3.avro output.avro
116
130
  from avrokit import avro_reader, avro_records, avro_writer, avro_schema, parse_url
117
131
 
118
132
  # Read records from any URL
119
- url = parse_url('gs://bucket/data.avro', mode='rb')
133
+ url = parse_url("gs://bucket/data.avro", mode="rb")
120
134
  for record in avro_records(url):
121
135
  print(record)
122
136
 
123
137
  # Write Avro data
124
- schema = avro_schema({
125
- 'type': 'record',
126
- 'name': 'User',
127
- 'fields': [
128
- {'name': 'name', 'type': 'string'},
129
- {'name': 'age', 'type': 'int'}
130
- ]
131
- })
132
-
133
- url = parse_url('output.avro', mode='wb')
138
+ schema = avro_schema(
139
+ {
140
+ "type": "record",
141
+ "name": "User",
142
+ "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}],
143
+ }
144
+ )
145
+
146
+ url = parse_url("output.avro", mode="wb")
134
147
  with avro_writer(url, schema) as writer:
135
- writer.append({'name': 'Alice', 'age': 30})
136
- writer.append({'name': 'Bob', 'age': 25})
148
+ writer.append({"name": "Alice", "age": 30})
149
+ writer.append({"name": "Bob", "age": 25})
137
150
  ```
138
151
 
139
152
  ## Architecture
@@ -156,10 +169,10 @@ The `parse_url()` factory function automatically instantiates the correct URL cl
156
169
  from avrokit.url import parse_url
157
170
 
158
171
  # All of these return the appropriate URL subclass
159
- url1 = parse_url('file:///path/to/data.avro') # FileURL
160
- url2 = parse_url('gs://bucket/data.avro') # GCSURL
161
- url3 = parse_url('s3://bucket/data.avro') # S3URL
162
- url4 = parse_url('https://example.com/data.avro') # HTTPURL
172
+ url1 = parse_url("file:///path/to/data.avro") # FileURL
173
+ url2 = parse_url("gs://bucket/data.avro") # GCSURL
174
+ url3 = parse_url("s3://bucket/data.avro") # S3URL
175
+ url4 = parse_url("https://example.com/data.avro") # HTTPURL
163
176
  ```
164
177
 
165
178
  ### 2. I/O Layer (`avrokit.io`)
@@ -401,13 +414,13 @@ The GitHub Actions workflow will automatically build and publish to PyPI.
401
414
  from avrokit import PartitionedAvroReader, PartitionedAvroWriter, parse_url
402
415
 
403
416
  # Read from multiple files as single stream
404
- url = parse_url('data-*.avro', mode='rb')
417
+ url = parse_url("data-*.avro", mode="rb")
405
418
  with PartitionedAvroReader(url) as reader:
406
419
  for record in reader:
407
420
  process(record)
408
421
 
409
422
  # Write to multiple files (roll every 10000 records)
410
- url = parse_url('output/', mode='wb')
423
+ url = parse_url("output/", mode="wb")
411
424
  with PartitionedAvroWriter(url, schema, max_records=10000) as writer:
412
425
  for record in records:
413
426
  writer.append(record)
@@ -422,15 +435,12 @@ from avrokit import TimePartitionedAvroWriter, parse_url
422
435
  from datetime import datetime
423
436
 
424
437
  # Creates hourly files: output/2024/01/15/10.avro
425
- url = parse_url('output/', mode='wb')
438
+ url = parse_url("output/", mode="wb")
426
439
  with TimePartitionedAvroWriter(
427
- url,
428
- schema,
429
- time_granularity='hour',
430
- time_field='timestamp'
440
+ url, schema, time_granularity="hour", time_field="timestamp"
431
441
  ) as writer:
432
442
  for record in records:
433
- writer.append(record, timestamp=record['timestamp'])
443
+ writer.append(record, timestamp=record["timestamp"])
434
444
  ```
435
445
 
436
446
  ### Async Operations
@@ -440,8 +450,9 @@ import asyncio
440
450
  from avrokit.asyncio import DeferredAvroWriter
441
451
  from avrokit import parse_url, avro_schema
442
452
 
453
+
443
454
  async def write_async():
444
- url = parse_url('output.avro', mode='wb')
455
+ url = parse_url("output.avro", mode="wb")
445
456
  schema = avro_schema({...})
446
457
 
447
458
  async with DeferredAvroWriter(url, schema) as writer:
@@ -449,6 +460,7 @@ async def write_async():
449
460
  await writer.append(record)
450
461
  # Flushes happen automatically in background
451
462
 
463
+
452
464
  asyncio.run(write_async())
453
465
  ```
454
466
 
@@ -459,13 +471,13 @@ from avrokit import validate_avro_schema_evolution, read_avro_schema
459
471
  from avrokit.url import parse_url
460
472
 
461
473
  # Check if new schema is backward compatible
462
- reader_schema = read_avro_schema(parse_url('old.avro', 'rb'))
463
- writer_schema = read_avro_schema(parse_url('new.avro', 'rb'))
474
+ reader_schema = read_avro_schema(parse_url("old.avro", "rb"))
475
+ writer_schema = read_avro_schema(parse_url("new.avro", "rb"))
464
476
 
465
477
  is_valid = validate_avro_schema_evolution(
466
478
  reader_schema,
467
479
  writer_schema,
468
- strategy='backward' # or 'forward', 'full'
480
+ strategy="backward", # or 'forward', 'full'
469
481
  )
470
482
  ```
471
483
 
@@ -26,7 +26,15 @@ Python utilities for working with Avro data files with support for local files,
26
26
  pip install avrokit
27
27
  ```
28
28
 
29
- ### With Cloud Storage Support
29
+ ### With pipx (CLI tools)
30
+
31
+ ```bash
32
+ # Installs the avrokit CLI in an isolated environment,
33
+ # with support for snappy-compressed Avro files
34
+ pipx install avrokit[snappy]
35
+ ```
36
+
37
+ ### With Optional Extras
30
38
 
31
39
  ```bash
32
40
  # AWS S3 support
@@ -35,7 +43,10 @@ pip install avrokit[aws]
35
43
  # Google Cloud Storage support
36
44
  pip install avrokit[gcp]
37
45
 
38
- # All extras (S3 + GCS)
46
+ # Snappy codec support (for reading snappy-compressed files)
47
+ pip install avrokit[snappy]
48
+
49
+ # All extras (S3 + GCS + snappy)
39
50
  pip install avrokit[all]
40
51
  ```
41
52
 
@@ -77,24 +88,23 @@ avrokit concat file1.avro file2.avro s3://bucket/file3.avro output.avro
77
88
  from avrokit import avro_reader, avro_records, avro_writer, avro_schema, parse_url
78
89
 
79
90
  # Read records from any URL
80
- url = parse_url('gs://bucket/data.avro', mode='rb')
91
+ url = parse_url("gs://bucket/data.avro", mode="rb")
81
92
  for record in avro_records(url):
82
93
  print(record)
83
94
 
84
95
  # Write Avro data
85
- schema = avro_schema({
86
- 'type': 'record',
87
- 'name': 'User',
88
- 'fields': [
89
- {'name': 'name', 'type': 'string'},
90
- {'name': 'age', 'type': 'int'}
91
- ]
92
- })
93
-
94
- url = parse_url('output.avro', mode='wb')
96
+ schema = avro_schema(
97
+ {
98
+ "type": "record",
99
+ "name": "User",
100
+ "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}],
101
+ }
102
+ )
103
+
104
+ url = parse_url("output.avro", mode="wb")
95
105
  with avro_writer(url, schema) as writer:
96
- writer.append({'name': 'Alice', 'age': 30})
97
- writer.append({'name': 'Bob', 'age': 25})
106
+ writer.append({"name": "Alice", "age": 30})
107
+ writer.append({"name": "Bob", "age": 25})
98
108
  ```
99
109
 
100
110
  ## Architecture
@@ -117,10 +127,10 @@ The `parse_url()` factory function automatically instantiates the correct URL cl
117
127
  from avrokit.url import parse_url
118
128
 
119
129
  # All of these return the appropriate URL subclass
120
- url1 = parse_url('file:///path/to/data.avro') # FileURL
121
- url2 = parse_url('gs://bucket/data.avro') # GCSURL
122
- url3 = parse_url('s3://bucket/data.avro') # S3URL
123
- url4 = parse_url('https://example.com/data.avro') # HTTPURL
130
+ url1 = parse_url("file:///path/to/data.avro") # FileURL
131
+ url2 = parse_url("gs://bucket/data.avro") # GCSURL
132
+ url3 = parse_url("s3://bucket/data.avro") # S3URL
133
+ url4 = parse_url("https://example.com/data.avro") # HTTPURL
124
134
  ```
125
135
 
126
136
  ### 2. I/O Layer (`avrokit.io`)
@@ -362,13 +372,13 @@ The GitHub Actions workflow will automatically build and publish to PyPI.
362
372
  from avrokit import PartitionedAvroReader, PartitionedAvroWriter, parse_url
363
373
 
364
374
  # Read from multiple files as single stream
365
- url = parse_url('data-*.avro', mode='rb')
375
+ url = parse_url("data-*.avro", mode="rb")
366
376
  with PartitionedAvroReader(url) as reader:
367
377
  for record in reader:
368
378
  process(record)
369
379
 
370
380
  # Write to multiple files (roll every 10000 records)
371
- url = parse_url('output/', mode='wb')
381
+ url = parse_url("output/", mode="wb")
372
382
  with PartitionedAvroWriter(url, schema, max_records=10000) as writer:
373
383
  for record in records:
374
384
  writer.append(record)
@@ -383,15 +393,12 @@ from avrokit import TimePartitionedAvroWriter, parse_url
383
393
  from datetime import datetime
384
394
 
385
395
  # Creates hourly files: output/2024/01/15/10.avro
386
- url = parse_url('output/', mode='wb')
396
+ url = parse_url("output/", mode="wb")
387
397
  with TimePartitionedAvroWriter(
388
- url,
389
- schema,
390
- time_granularity='hour',
391
- time_field='timestamp'
398
+ url, schema, time_granularity="hour", time_field="timestamp"
392
399
  ) as writer:
393
400
  for record in records:
394
- writer.append(record, timestamp=record['timestamp'])
401
+ writer.append(record, timestamp=record["timestamp"])
395
402
  ```
396
403
 
397
404
  ### Async Operations
@@ -401,8 +408,9 @@ import asyncio
401
408
  from avrokit.asyncio import DeferredAvroWriter
402
409
  from avrokit import parse_url, avro_schema
403
410
 
411
+
404
412
  async def write_async():
405
- url = parse_url('output.avro', mode='wb')
413
+ url = parse_url("output.avro", mode="wb")
406
414
  schema = avro_schema({...})
407
415
 
408
416
  async with DeferredAvroWriter(url, schema) as writer:
@@ -410,6 +418,7 @@ async def write_async():
410
418
  await writer.append(record)
411
419
  # Flushes happen automatically in background
412
420
 
421
+
413
422
  asyncio.run(write_async())
414
423
  ```
415
424
 
@@ -420,13 +429,13 @@ from avrokit import validate_avro_schema_evolution, read_avro_schema
420
429
  from avrokit.url import parse_url
421
430
 
422
431
  # Check if new schema is backward compatible
423
- reader_schema = read_avro_schema(parse_url('old.avro', 'rb'))
424
- writer_schema = read_avro_schema(parse_url('new.avro', 'rb'))
432
+ reader_schema = read_avro_schema(parse_url("old.avro", "rb"))
433
+ writer_schema = read_avro_schema(parse_url("new.avro", "rb"))
425
434
 
426
435
  is_valid = validate_avro_schema_evolution(
427
436
  reader_schema,
428
437
  writer_schema,
429
- strategy='backward' # or 'forward', 'full'
438
+ strategy="backward", # or 'forward', 'full'
430
439
  )
431
440
  ```
432
441
 
@@ -43,6 +43,20 @@ class DeferredAvroWriter:
43
43
  self._writer_thread_done.set()
44
44
  self._writer_thread.join()
45
45
 
46
+ def close(self) -> None:
47
+ self.stop()
48
+ # Close the underlying writer if it supports it (e.g. DataFileWriter)
49
+ close_fn = getattr(self.writer, "close", None)
50
+ if close_fn:
51
+ close_fn()
52
+
53
+ def __enter__(self) -> "DeferredAvroWriter":
54
+ self.start()
55
+ return self
56
+
57
+ def __exit__(self, exc_type, exc_value, traceback):
58
+ self.close()
59
+
46
60
  def append(
47
61
  self, datum: object, block: bool = True, timeout: int | None = None
48
62
  ) -> None:
@@ -85,6 +85,8 @@ class PartitionedAvroReader:
85
85
  if self.current_reader:
86
86
  self.current_reader.close()
87
87
  self.current_reader = None
88
+ if self.current_url:
89
+ self.current_url.close()
88
90
  self.current_url_stream = None
89
91
  self.current_index = 0
90
92
 
@@ -128,6 +128,8 @@ class PartitionedAvroWriter:
128
128
  if self.current_writer:
129
129
  self.current_writer.close()
130
130
  self.current_writer = None
131
+ if self.current_url:
132
+ self.current_url.close()
131
133
  self.current_url_stream = None
132
134
  self.current_url = None
133
135
 
@@ -80,11 +80,13 @@ class ConcatTool:
80
80
 
81
81
  output_sync_marker = None
82
82
 
83
- with output_url.with_mode("wb").open() as out_f:
83
+ out_url = output_url.with_mode("wb")
84
+ with out_url as out_f:
84
85
  writer = BinaryEncoder(out_f)
85
86
 
86
87
  for i, url in enumerate(input_urls):
87
- with url.with_mode("rb").open() as in_f:
88
+ in_url = url.with_mode("rb")
89
+ with in_url as in_f:
88
90
  # Initialize DataFileReader to parse the header/metadata and
89
91
  # position the file pointer at the start of the first block.
90
92
  reader = DataFileReader(in_f, DatumReader())
@@ -90,7 +90,7 @@ class FileSortTool:
90
90
  schema: Schema,
91
91
  root: str,
92
92
  batch_id: int,
93
- records: list[object],
93
+ records: list[dict[str, Any]],
94
94
  sort_key: Callable[[Any], Any],
95
95
  reverse: bool = False,
96
96
  ) -> None:
@@ -147,6 +147,7 @@ class RepairTool:
147
147
  if dry_run:
148
148
  tmp = tempfile.NamedTemporaryFile(delete=False)
149
149
  output_url = parse_url(tmp.name, mode="wb")
150
+ tmp.close() # Close the handle; the URL will open it fresh
150
151
  logger.debug("Using temporary file %s for dry run", output_url)
151
152
  # Initialize the output file with the same schema and codec
152
153
  with avro_writer(output_url, schema, codec=codec) as writer:
@@ -3,6 +3,7 @@
3
3
  # SPDX-License-Identifier: Apache-2.0
4
4
 
5
5
  import io
6
+ import os
6
7
  import tempfile
7
8
  from collections.abc import Sequence
8
9
  from typing import IO, Any
@@ -34,6 +35,7 @@ class HttpURL(URL):
34
35
  self.content_type = content_type
35
36
  self._current_response: requests.Response | None = None
36
37
  self._current_request_buffer: IO[Any] | None = None
38
+ self._current_request_temp_path: str | None = None
37
39
 
38
40
  @override
39
41
  def _expand(self) -> Sequence[URL]:
@@ -84,9 +86,11 @@ class HttpURL(URL):
84
86
  if self.spill_request_to_file:
85
87
  # Use a temporary file to store the request body
86
88
  buf = tempfile.NamedTemporaryFile(delete=False)
89
+ self._current_request_temp_path = buf.name
87
90
  else:
88
91
  # Use an in-memory buffer
89
92
  buf = io.BytesIO()
93
+ self._current_request_temp_path = None
90
94
  # Wrap in TextIOWrapper if not binary mode
91
95
  if "b" not in self.mode:
92
96
  buf = io.TextIOWrapper(buf, encoding="utf-8")
@@ -112,8 +116,15 @@ class HttpURL(URL):
112
116
  headers={"Content-Type": self.content_type},
113
117
  )
114
118
  res.raise_for_status()
115
- # Close the buffer (will delete temp file if used)
119
+ # Close the buffer
116
120
  self._current_request_buffer.close()
121
+ # Unlink temp file if used
122
+ if self._current_request_temp_path is not None:
123
+ try:
124
+ os.unlink(self._current_request_temp_path)
125
+ except FileNotFoundError:
126
+ pass
127
+ self._current_request_temp_path = None
117
128
 
118
129
  @override
119
130
  def with_mode(self, mode) -> URL:
@@ -3,6 +3,7 @@
3
3
  # SPDX-License-Identifier: Apache-2.0
4
4
 
5
5
  import io
6
+ import os
6
7
  import tempfile
7
8
  from collections.abc import Sequence
8
9
  from contextlib import contextmanager
@@ -125,7 +126,16 @@ class S3URL(URL):
125
126
  self._current_client.upload_fileobj(f, self.bucket, self.path)
126
127
  finally:
127
128
  # Cleanup resources
128
- self._current_local.close()
129
+ local = self._current_local
130
+ name = local.name
131
+ local.close()
132
+ # N.b. the temp file is created with delete=False so it survives an external
133
+ # close (e.g. DataFileWriter closing the stream before URL.close() uploads).
134
+ # Unlink it explicitly here to avoid leaking a file per open/close cycle.
135
+ try:
136
+ os.unlink(name)
137
+ except FileNotFoundError:
138
+ pass
129
139
  self._current_client.close()
130
140
  self._current_local = None
131
141
  self._current_client = None
@@ -4,7 +4,7 @@
4
4
 
5
5
  [project]
6
6
  name = "avrokit"
7
- version = "0.1.3"
7
+ version = "0.1.5"
8
8
  description = "Python utilities for working with Avro data files"
9
9
  authors = [{ name = "Greg Brandt", email = "brandt.greg@gmail.com" }]
10
10
  license = "Apache-2.0"
@@ -36,7 +36,8 @@ dependencies = [
36
36
  [project.optional-dependencies]
37
37
  aws = ["boto3>=1.38.7"]
38
38
  gcp = ["google-cloud-storage>=3.6.0", "google-crc32c>=1.7.1"]
39
- all = ["boto3>=1.38.7", "google-cloud-storage>=3.6.0", "google-crc32c>=1.7.1"]
39
+ snappy = ["python-snappy>=0.7.3"]
40
+ all = ["boto3>=1.38.7", "google-cloud-storage>=3.6.0", "google-crc32c>=1.7.1", "python-snappy>=0.7.3"]
40
41
 
41
42
  [project.urls]
42
43
  Homepage = "https://github.com/brandtg/avrokit"
@@ -61,6 +61,25 @@ class TestSingleFileIO:
61
61
  count += 1
62
62
  assert count == 100
63
63
 
64
+ def test_read_write_snappy_codec(self):
65
+ with tempfile.NamedTemporaryFile() as tmp:
66
+ url = parse_url(tmp.name)
67
+ with avro_writer(url.with_mode("wb"), SCHEMA, codec="snappy") as writer:
68
+ for _ in range(100):
69
+ writer.append(
70
+ {
71
+ "name": faker.name(),
72
+ "age": faker.random_int(min=18, max=80),
73
+ "emails": [faker.email() for _ in range(3)],
74
+ }
75
+ )
76
+ # Codec is recorded in the file header metadata
77
+ with avro_reader(url.with_mode("rb")) as reader:
78
+ assert reader.meta["avro.codec"] == b"snappy"
79
+ # Roundtrip: all records readable after snappy compression
80
+ with avro_reader(url.with_mode("rb")) as reader:
81
+ assert sum(1 for _ in reader) == 100
82
+
64
83
  def test_write_text_mode(self):
65
84
  with tempfile.NamedTemporaryFile() as tmp, pytest.raises(ValueError):
66
85
  # Attempt to write to a file opened in text mode