avrokit 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- avrokit-0.1.5/.github/workflows/release.yml +65 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/PKG-INFO +45 -33
- {avrokit-0.1.3 → avrokit-0.1.5}/README.md +40 -31
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/asyncio/writer.py +14 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/reader.py +2 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/writer.py +2 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/concat.py +4 -2
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/filesort.py +1 -1
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/repair.py +1 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/http.py +12 -1
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/s3.py +11 -1
- {avrokit-0.1.3 → avrokit-0.1.5}/pyproject.toml +3 -2
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_io.py +19 -0
- avrokit-0.1.5/uv.lock +1793 -0
- avrokit-0.1.3/.github/workflows/release.yml +0 -31
- avrokit-0.1.3/uv.lock +0 -1533
- {avrokit-0.1.3 → avrokit-0.1.5}/.github/workflows/check.yml +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/.gitignore +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/LICENSE +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/LICENSES/Apache-2.0.txt +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/Makefile +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/__init__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/asyncio/__init__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/asyncio/reader.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/__init__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/compact.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/io/schema.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/py.typed +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/__init__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/__main__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/base.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/cat.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/count.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/fromparquet.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/getmeta.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/getschema.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/httpserver.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/partition.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/stats.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/tojson.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/tools/toparquet.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/__init__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/base.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/factory.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/file.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/google.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/avrokit/url/utils.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/__init__.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_additional_coverage.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_asyncio.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_exports.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_io_schema.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_plan_coverage.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_property_based.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_schema_advanced.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_tools_httpserver.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_tools_integration.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_tools_repair.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_contract.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_file.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_google.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_http.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/tests/test_url_s3.py +0 -0
- {avrokit-0.1.3 → avrokit-0.1.5}/uv.lock.license +0 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Greg Brandt <brandt.greg@gmail.com>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
name: Publish to PyPI
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
release:
|
|
9
|
+
types: [published]
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write # This allows the OIDC authentication
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
build-and-publish:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
environment:
|
|
19
|
+
name: pypi
|
|
20
|
+
url: https://pypi.org/p/avrokit
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Set up uv
|
|
25
|
+
uses: astral-sh/setup-uv@v5
|
|
26
|
+
|
|
27
|
+
- name: Build package
|
|
28
|
+
run: uv build
|
|
29
|
+
|
|
30
|
+
- name: Verify wheel extras match pyproject.toml
|
|
31
|
+
run: |
|
|
32
|
+
uv run --no-project --python 3.12 python - <<'EOF'
|
|
33
|
+
import email
|
|
34
|
+
import sys
|
|
35
|
+
import tomllib
|
|
36
|
+
import zipfile
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
|
|
39
|
+
expected = set(
|
|
40
|
+
tomllib.loads(Path("pyproject.toml").read_text())["project"][
|
|
41
|
+
"optional-dependencies"
|
|
42
|
+
]
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
wheels = sorted(Path("dist").glob("*.whl"))
|
|
46
|
+
if not wheels:
|
|
47
|
+
print("No wheel found in dist/")
|
|
48
|
+
sys.exit(1)
|
|
49
|
+
wheel = wheels[0]
|
|
50
|
+
with zipfile.ZipFile(wheel) as zf:
|
|
51
|
+
name = next(n for n in zf.namelist() if n.endswith(".dist-info/METADATA"))
|
|
52
|
+
metadata = email.message_from_bytes(zf.read(name))
|
|
53
|
+
actual = set(metadata.get_all("Provides-Extra") or [])
|
|
54
|
+
|
|
55
|
+
if actual != expected:
|
|
56
|
+
print(
|
|
57
|
+
f"Extras mismatch for {wheel.name}: "
|
|
58
|
+
f"pyproject.toml={sorted(expected)} wheel={sorted(actual)}"
|
|
59
|
+
)
|
|
60
|
+
sys.exit(1)
|
|
61
|
+
print(f"Wheel extras match pyproject.toml: {sorted(actual)}")
|
|
62
|
+
EOF
|
|
63
|
+
|
|
64
|
+
- name: Publish to PyPI
|
|
65
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: avrokit
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Python utilities for working with Avro data files
|
|
5
5
|
Project-URL: Homepage, https://github.com/brandtg/avrokit
|
|
6
6
|
Project-URL: Repository, https://github.com/brandtg/avrokit
|
|
@@ -30,11 +30,14 @@ Provides-Extra: all
|
|
|
30
30
|
Requires-Dist: boto3>=1.38.7; extra == 'all'
|
|
31
31
|
Requires-Dist: google-cloud-storage>=3.6.0; extra == 'all'
|
|
32
32
|
Requires-Dist: google-crc32c>=1.7.1; extra == 'all'
|
|
33
|
+
Requires-Dist: python-snappy>=0.7.3; extra == 'all'
|
|
33
34
|
Provides-Extra: aws
|
|
34
35
|
Requires-Dist: boto3>=1.38.7; extra == 'aws'
|
|
35
36
|
Provides-Extra: gcp
|
|
36
37
|
Requires-Dist: google-cloud-storage>=3.6.0; extra == 'gcp'
|
|
37
38
|
Requires-Dist: google-crc32c>=1.7.1; extra == 'gcp'
|
|
39
|
+
Provides-Extra: snappy
|
|
40
|
+
Requires-Dist: python-snappy>=0.7.3; extra == 'snappy'
|
|
38
41
|
Description-Content-Type: text/markdown
|
|
39
42
|
|
|
40
43
|
<!--
|
|
@@ -65,7 +68,15 @@ Python utilities for working with Avro data files with support for local files,
|
|
|
65
68
|
pip install avrokit
|
|
66
69
|
```
|
|
67
70
|
|
|
68
|
-
### With
|
|
71
|
+
### With pipx (CLI tools)
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Installs the avrokit CLI in an isolated environment,
|
|
75
|
+
# with support for snappy-compressed Avro files
|
|
76
|
+
pipx install avrokit[snappy]
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### With Optional Extras
|
|
69
80
|
|
|
70
81
|
```bash
|
|
71
82
|
# AWS S3 support
|
|
@@ -74,7 +85,10 @@ pip install avrokit[aws]
|
|
|
74
85
|
# Google Cloud Storage support
|
|
75
86
|
pip install avrokit[gcp]
|
|
76
87
|
|
|
77
|
-
#
|
|
88
|
+
# Snappy codec support (for reading snappy-compressed files)
|
|
89
|
+
pip install avrokit[snappy]
|
|
90
|
+
|
|
91
|
+
# All extras (S3 + GCS + snappy)
|
|
78
92
|
pip install avrokit[all]
|
|
79
93
|
```
|
|
80
94
|
|
|
@@ -116,24 +130,23 @@ avrokit concat file1.avro file2.avro s3://bucket/file3.avro output.avro
|
|
|
116
130
|
from avrokit import avro_reader, avro_records, avro_writer, avro_schema, parse_url
|
|
117
131
|
|
|
118
132
|
# Read records from any URL
|
|
119
|
-
url = parse_url(
|
|
133
|
+
url = parse_url("gs://bucket/data.avro", mode="rb")
|
|
120
134
|
for record in avro_records(url):
|
|
121
135
|
print(record)
|
|
122
136
|
|
|
123
137
|
# Write Avro data
|
|
124
|
-
schema = avro_schema(
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
{
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
url = parse_url('output.avro', mode='wb')
|
|
138
|
+
schema = avro_schema(
|
|
139
|
+
{
|
|
140
|
+
"type": "record",
|
|
141
|
+
"name": "User",
|
|
142
|
+
"fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}],
|
|
143
|
+
}
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
url = parse_url("output.avro", mode="wb")
|
|
134
147
|
with avro_writer(url, schema) as writer:
|
|
135
|
-
writer.append({
|
|
136
|
-
writer.append({
|
|
148
|
+
writer.append({"name": "Alice", "age": 30})
|
|
149
|
+
writer.append({"name": "Bob", "age": 25})
|
|
137
150
|
```
|
|
138
151
|
|
|
139
152
|
## Architecture
|
|
@@ -156,10 +169,10 @@ The `parse_url()` factory function automatically instantiates the correct URL cl
|
|
|
156
169
|
from avrokit.url import parse_url
|
|
157
170
|
|
|
158
171
|
# All of these return the appropriate URL subclass
|
|
159
|
-
url1 = parse_url(
|
|
160
|
-
url2 = parse_url(
|
|
161
|
-
url3 = parse_url(
|
|
162
|
-
url4 = parse_url(
|
|
172
|
+
url1 = parse_url("file:///path/to/data.avro") # FileURL
|
|
173
|
+
url2 = parse_url("gs://bucket/data.avro") # GCSURL
|
|
174
|
+
url3 = parse_url("s3://bucket/data.avro") # S3URL
|
|
175
|
+
url4 = parse_url("https://example.com/data.avro") # HTTPURL
|
|
163
176
|
```
|
|
164
177
|
|
|
165
178
|
### 2. I/O Layer (`avrokit.io`)
|
|
@@ -401,13 +414,13 @@ The GitHub Actions workflow will automatically build and publish to PyPI.
|
|
|
401
414
|
from avrokit import PartitionedAvroReader, PartitionedAvroWriter, parse_url
|
|
402
415
|
|
|
403
416
|
# Read from multiple files as single stream
|
|
404
|
-
url = parse_url(
|
|
417
|
+
url = parse_url("data-*.avro", mode="rb")
|
|
405
418
|
with PartitionedAvroReader(url) as reader:
|
|
406
419
|
for record in reader:
|
|
407
420
|
process(record)
|
|
408
421
|
|
|
409
422
|
# Write to multiple files (roll every 10000 records)
|
|
410
|
-
url = parse_url(
|
|
423
|
+
url = parse_url("output/", mode="wb")
|
|
411
424
|
with PartitionedAvroWriter(url, schema, max_records=10000) as writer:
|
|
412
425
|
for record in records:
|
|
413
426
|
writer.append(record)
|
|
@@ -422,15 +435,12 @@ from avrokit import TimePartitionedAvroWriter, parse_url
|
|
|
422
435
|
from datetime import datetime
|
|
423
436
|
|
|
424
437
|
# Creates hourly files: output/2024/01/15/10.avro
|
|
425
|
-
url = parse_url(
|
|
438
|
+
url = parse_url("output/", mode="wb")
|
|
426
439
|
with TimePartitionedAvroWriter(
|
|
427
|
-
url,
|
|
428
|
-
schema,
|
|
429
|
-
time_granularity='hour',
|
|
430
|
-
time_field='timestamp'
|
|
440
|
+
url, schema, time_granularity="hour", time_field="timestamp"
|
|
431
441
|
) as writer:
|
|
432
442
|
for record in records:
|
|
433
|
-
writer.append(record, timestamp=record[
|
|
443
|
+
writer.append(record, timestamp=record["timestamp"])
|
|
434
444
|
```
|
|
435
445
|
|
|
436
446
|
### Async Operations
|
|
@@ -440,8 +450,9 @@ import asyncio
|
|
|
440
450
|
from avrokit.asyncio import DeferredAvroWriter
|
|
441
451
|
from avrokit import parse_url, avro_schema
|
|
442
452
|
|
|
453
|
+
|
|
443
454
|
async def write_async():
|
|
444
|
-
url = parse_url(
|
|
455
|
+
url = parse_url("output.avro", mode="wb")
|
|
445
456
|
schema = avro_schema({...})
|
|
446
457
|
|
|
447
458
|
async with DeferredAvroWriter(url, schema) as writer:
|
|
@@ -449,6 +460,7 @@ async def write_async():
|
|
|
449
460
|
await writer.append(record)
|
|
450
461
|
# Flushes happen automatically in background
|
|
451
462
|
|
|
463
|
+
|
|
452
464
|
asyncio.run(write_async())
|
|
453
465
|
```
|
|
454
466
|
|
|
@@ -459,13 +471,13 @@ from avrokit import validate_avro_schema_evolution, read_avro_schema
|
|
|
459
471
|
from avrokit.url import parse_url
|
|
460
472
|
|
|
461
473
|
# Check if new schema is backward compatible
|
|
462
|
-
reader_schema = read_avro_schema(parse_url(
|
|
463
|
-
writer_schema = read_avro_schema(parse_url(
|
|
474
|
+
reader_schema = read_avro_schema(parse_url("old.avro", "rb"))
|
|
475
|
+
writer_schema = read_avro_schema(parse_url("new.avro", "rb"))
|
|
464
476
|
|
|
465
477
|
is_valid = validate_avro_schema_evolution(
|
|
466
478
|
reader_schema,
|
|
467
479
|
writer_schema,
|
|
468
|
-
strategy=
|
|
480
|
+
strategy="backward", # or 'forward', 'full'
|
|
469
481
|
)
|
|
470
482
|
```
|
|
471
483
|
|
|
@@ -26,7 +26,15 @@ Python utilities for working with Avro data files with support for local files,
|
|
|
26
26
|
pip install avrokit
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
### With
|
|
29
|
+
### With pipx (CLI tools)
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# Installs the avrokit CLI in an isolated environment,
|
|
33
|
+
# with support for snappy-compressed Avro files
|
|
34
|
+
pipx install avrokit[snappy]
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### With Optional Extras
|
|
30
38
|
|
|
31
39
|
```bash
|
|
32
40
|
# AWS S3 support
|
|
@@ -35,7 +43,10 @@ pip install avrokit[aws]
|
|
|
35
43
|
# Google Cloud Storage support
|
|
36
44
|
pip install avrokit[gcp]
|
|
37
45
|
|
|
38
|
-
#
|
|
46
|
+
# Snappy codec support (for reading snappy-compressed files)
|
|
47
|
+
pip install avrokit[snappy]
|
|
48
|
+
|
|
49
|
+
# All extras (S3 + GCS + snappy)
|
|
39
50
|
pip install avrokit[all]
|
|
40
51
|
```
|
|
41
52
|
|
|
@@ -77,24 +88,23 @@ avrokit concat file1.avro file2.avro s3://bucket/file3.avro output.avro
|
|
|
77
88
|
from avrokit import avro_reader, avro_records, avro_writer, avro_schema, parse_url
|
|
78
89
|
|
|
79
90
|
# Read records from any URL
|
|
80
|
-
url = parse_url(
|
|
91
|
+
url = parse_url("gs://bucket/data.avro", mode="rb")
|
|
81
92
|
for record in avro_records(url):
|
|
82
93
|
print(record)
|
|
83
94
|
|
|
84
95
|
# Write Avro data
|
|
85
|
-
schema = avro_schema(
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
{
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
url = parse_url('output.avro', mode='wb')
|
|
96
|
+
schema = avro_schema(
|
|
97
|
+
{
|
|
98
|
+
"type": "record",
|
|
99
|
+
"name": "User",
|
|
100
|
+
"fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}],
|
|
101
|
+
}
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
url = parse_url("output.avro", mode="wb")
|
|
95
105
|
with avro_writer(url, schema) as writer:
|
|
96
|
-
writer.append({
|
|
97
|
-
writer.append({
|
|
106
|
+
writer.append({"name": "Alice", "age": 30})
|
|
107
|
+
writer.append({"name": "Bob", "age": 25})
|
|
98
108
|
```
|
|
99
109
|
|
|
100
110
|
## Architecture
|
|
@@ -117,10 +127,10 @@ The `parse_url()` factory function automatically instantiates the correct URL cl
|
|
|
117
127
|
from avrokit.url import parse_url
|
|
118
128
|
|
|
119
129
|
# All of these return the appropriate URL subclass
|
|
120
|
-
url1 = parse_url(
|
|
121
|
-
url2 = parse_url(
|
|
122
|
-
url3 = parse_url(
|
|
123
|
-
url4 = parse_url(
|
|
130
|
+
url1 = parse_url("file:///path/to/data.avro") # FileURL
|
|
131
|
+
url2 = parse_url("gs://bucket/data.avro") # GCSURL
|
|
132
|
+
url3 = parse_url("s3://bucket/data.avro") # S3URL
|
|
133
|
+
url4 = parse_url("https://example.com/data.avro") # HTTPURL
|
|
124
134
|
```
|
|
125
135
|
|
|
126
136
|
### 2. I/O Layer (`avrokit.io`)
|
|
@@ -362,13 +372,13 @@ The GitHub Actions workflow will automatically build and publish to PyPI.
|
|
|
362
372
|
from avrokit import PartitionedAvroReader, PartitionedAvroWriter, parse_url
|
|
363
373
|
|
|
364
374
|
# Read from multiple files as single stream
|
|
365
|
-
url = parse_url(
|
|
375
|
+
url = parse_url("data-*.avro", mode="rb")
|
|
366
376
|
with PartitionedAvroReader(url) as reader:
|
|
367
377
|
for record in reader:
|
|
368
378
|
process(record)
|
|
369
379
|
|
|
370
380
|
# Write to multiple files (roll every 10000 records)
|
|
371
|
-
url = parse_url(
|
|
381
|
+
url = parse_url("output/", mode="wb")
|
|
372
382
|
with PartitionedAvroWriter(url, schema, max_records=10000) as writer:
|
|
373
383
|
for record in records:
|
|
374
384
|
writer.append(record)
|
|
@@ -383,15 +393,12 @@ from avrokit import TimePartitionedAvroWriter, parse_url
|
|
|
383
393
|
from datetime import datetime
|
|
384
394
|
|
|
385
395
|
# Creates hourly files: output/2024/01/15/10.avro
|
|
386
|
-
url = parse_url(
|
|
396
|
+
url = parse_url("output/", mode="wb")
|
|
387
397
|
with TimePartitionedAvroWriter(
|
|
388
|
-
url,
|
|
389
|
-
schema,
|
|
390
|
-
time_granularity='hour',
|
|
391
|
-
time_field='timestamp'
|
|
398
|
+
url, schema, time_granularity="hour", time_field="timestamp"
|
|
392
399
|
) as writer:
|
|
393
400
|
for record in records:
|
|
394
|
-
writer.append(record, timestamp=record[
|
|
401
|
+
writer.append(record, timestamp=record["timestamp"])
|
|
395
402
|
```
|
|
396
403
|
|
|
397
404
|
### Async Operations
|
|
@@ -401,8 +408,9 @@ import asyncio
|
|
|
401
408
|
from avrokit.asyncio import DeferredAvroWriter
|
|
402
409
|
from avrokit import parse_url, avro_schema
|
|
403
410
|
|
|
411
|
+
|
|
404
412
|
async def write_async():
|
|
405
|
-
url = parse_url(
|
|
413
|
+
url = parse_url("output.avro", mode="wb")
|
|
406
414
|
schema = avro_schema({...})
|
|
407
415
|
|
|
408
416
|
async with DeferredAvroWriter(url, schema) as writer:
|
|
@@ -410,6 +418,7 @@ async def write_async():
|
|
|
410
418
|
await writer.append(record)
|
|
411
419
|
# Flushes happen automatically in background
|
|
412
420
|
|
|
421
|
+
|
|
413
422
|
asyncio.run(write_async())
|
|
414
423
|
```
|
|
415
424
|
|
|
@@ -420,13 +429,13 @@ from avrokit import validate_avro_schema_evolution, read_avro_schema
|
|
|
420
429
|
from avrokit.url import parse_url
|
|
421
430
|
|
|
422
431
|
# Check if new schema is backward compatible
|
|
423
|
-
reader_schema = read_avro_schema(parse_url(
|
|
424
|
-
writer_schema = read_avro_schema(parse_url(
|
|
432
|
+
reader_schema = read_avro_schema(parse_url("old.avro", "rb"))
|
|
433
|
+
writer_schema = read_avro_schema(parse_url("new.avro", "rb"))
|
|
425
434
|
|
|
426
435
|
is_valid = validate_avro_schema_evolution(
|
|
427
436
|
reader_schema,
|
|
428
437
|
writer_schema,
|
|
429
|
-
strategy=
|
|
438
|
+
strategy="backward", # or 'forward', 'full'
|
|
430
439
|
)
|
|
431
440
|
```
|
|
432
441
|
|
|
@@ -43,6 +43,20 @@ class DeferredAvroWriter:
|
|
|
43
43
|
self._writer_thread_done.set()
|
|
44
44
|
self._writer_thread.join()
|
|
45
45
|
|
|
46
|
+
def close(self) -> None:
|
|
47
|
+
self.stop()
|
|
48
|
+
# Close the underlying writer if it supports it (e.g. DataFileWriter)
|
|
49
|
+
close_fn = getattr(self.writer, "close", None)
|
|
50
|
+
if close_fn:
|
|
51
|
+
close_fn()
|
|
52
|
+
|
|
53
|
+
def __enter__(self) -> "DeferredAvroWriter":
|
|
54
|
+
self.start()
|
|
55
|
+
return self
|
|
56
|
+
|
|
57
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
58
|
+
self.close()
|
|
59
|
+
|
|
46
60
|
def append(
|
|
47
61
|
self, datum: object, block: bool = True, timeout: int | None = None
|
|
48
62
|
) -> None:
|
|
@@ -80,11 +80,13 @@ class ConcatTool:
|
|
|
80
80
|
|
|
81
81
|
output_sync_marker = None
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
out_url = output_url.with_mode("wb")
|
|
84
|
+
with out_url as out_f:
|
|
84
85
|
writer = BinaryEncoder(out_f)
|
|
85
86
|
|
|
86
87
|
for i, url in enumerate(input_urls):
|
|
87
|
-
|
|
88
|
+
in_url = url.with_mode("rb")
|
|
89
|
+
with in_url as in_f:
|
|
88
90
|
# Initialize DataFileReader to parse the header/metadata and
|
|
89
91
|
# position the file pointer at the start of the first block.
|
|
90
92
|
reader = DataFileReader(in_f, DatumReader())
|
|
@@ -147,6 +147,7 @@ class RepairTool:
|
|
|
147
147
|
if dry_run:
|
|
148
148
|
tmp = tempfile.NamedTemporaryFile(delete=False)
|
|
149
149
|
output_url = parse_url(tmp.name, mode="wb")
|
|
150
|
+
tmp.close() # Close the handle; the URL will open it fresh
|
|
150
151
|
logger.debug("Using temporary file %s for dry run", output_url)
|
|
151
152
|
# Initialize the output file with the same schema and codec
|
|
152
153
|
with avro_writer(output_url, schema, codec=codec) as writer:
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
import io
|
|
6
|
+
import os
|
|
6
7
|
import tempfile
|
|
7
8
|
from collections.abc import Sequence
|
|
8
9
|
from typing import IO, Any
|
|
@@ -34,6 +35,7 @@ class HttpURL(URL):
|
|
|
34
35
|
self.content_type = content_type
|
|
35
36
|
self._current_response: requests.Response | None = None
|
|
36
37
|
self._current_request_buffer: IO[Any] | None = None
|
|
38
|
+
self._current_request_temp_path: str | None = None
|
|
37
39
|
|
|
38
40
|
@override
|
|
39
41
|
def _expand(self) -> Sequence[URL]:
|
|
@@ -84,9 +86,11 @@ class HttpURL(URL):
|
|
|
84
86
|
if self.spill_request_to_file:
|
|
85
87
|
# Use a temporary file to store the request body
|
|
86
88
|
buf = tempfile.NamedTemporaryFile(delete=False)
|
|
89
|
+
self._current_request_temp_path = buf.name
|
|
87
90
|
else:
|
|
88
91
|
# Use an in-memory buffer
|
|
89
92
|
buf = io.BytesIO()
|
|
93
|
+
self._current_request_temp_path = None
|
|
90
94
|
# Wrap in TextIOWrapper if not binary mode
|
|
91
95
|
if "b" not in self.mode:
|
|
92
96
|
buf = io.TextIOWrapper(buf, encoding="utf-8")
|
|
@@ -112,8 +116,15 @@ class HttpURL(URL):
|
|
|
112
116
|
headers={"Content-Type": self.content_type},
|
|
113
117
|
)
|
|
114
118
|
res.raise_for_status()
|
|
115
|
-
# Close the buffer
|
|
119
|
+
# Close the buffer
|
|
116
120
|
self._current_request_buffer.close()
|
|
121
|
+
# Unlink temp file if used
|
|
122
|
+
if self._current_request_temp_path is not None:
|
|
123
|
+
try:
|
|
124
|
+
os.unlink(self._current_request_temp_path)
|
|
125
|
+
except FileNotFoundError:
|
|
126
|
+
pass
|
|
127
|
+
self._current_request_temp_path = None
|
|
117
128
|
|
|
118
129
|
@override
|
|
119
130
|
def with_mode(self, mode) -> URL:
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
import io
|
|
6
|
+
import os
|
|
6
7
|
import tempfile
|
|
7
8
|
from collections.abc import Sequence
|
|
8
9
|
from contextlib import contextmanager
|
|
@@ -125,7 +126,16 @@ class S3URL(URL):
|
|
|
125
126
|
self._current_client.upload_fileobj(f, self.bucket, self.path)
|
|
126
127
|
finally:
|
|
127
128
|
# Cleanup resources
|
|
128
|
-
self._current_local
|
|
129
|
+
local = self._current_local
|
|
130
|
+
name = local.name
|
|
131
|
+
local.close()
|
|
132
|
+
# N.b. the temp file is created with delete=False so it survives an external
|
|
133
|
+
# close (e.g. DataFileWriter closing the stream before URL.close() uploads).
|
|
134
|
+
# Unlink it explicitly here to avoid leaking a file per open/close cycle.
|
|
135
|
+
try:
|
|
136
|
+
os.unlink(name)
|
|
137
|
+
except FileNotFoundError:
|
|
138
|
+
pass
|
|
129
139
|
self._current_client.close()
|
|
130
140
|
self._current_local = None
|
|
131
141
|
self._current_client = None
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "avrokit"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.5"
|
|
8
8
|
description = "Python utilities for working with Avro data files"
|
|
9
9
|
authors = [{ name = "Greg Brandt", email = "brandt.greg@gmail.com" }]
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -36,7 +36,8 @@ dependencies = [
|
|
|
36
36
|
[project.optional-dependencies]
|
|
37
37
|
aws = ["boto3>=1.38.7"]
|
|
38
38
|
gcp = ["google-cloud-storage>=3.6.0", "google-crc32c>=1.7.1"]
|
|
39
|
-
|
|
39
|
+
snappy = ["python-snappy>=0.7.3"]
|
|
40
|
+
all = ["boto3>=1.38.7", "google-cloud-storage>=3.6.0", "google-crc32c>=1.7.1", "python-snappy>=0.7.3"]
|
|
40
41
|
|
|
41
42
|
[project.urls]
|
|
42
43
|
Homepage = "https://github.com/brandtg/avrokit"
|
|
@@ -61,6 +61,25 @@ class TestSingleFileIO:
|
|
|
61
61
|
count += 1
|
|
62
62
|
assert count == 100
|
|
63
63
|
|
|
64
|
+
def test_read_write_snappy_codec(self):
|
|
65
|
+
with tempfile.NamedTemporaryFile() as tmp:
|
|
66
|
+
url = parse_url(tmp.name)
|
|
67
|
+
with avro_writer(url.with_mode("wb"), SCHEMA, codec="snappy") as writer:
|
|
68
|
+
for _ in range(100):
|
|
69
|
+
writer.append(
|
|
70
|
+
{
|
|
71
|
+
"name": faker.name(),
|
|
72
|
+
"age": faker.random_int(min=18, max=80),
|
|
73
|
+
"emails": [faker.email() for _ in range(3)],
|
|
74
|
+
}
|
|
75
|
+
)
|
|
76
|
+
# Codec is recorded in the file header metadata
|
|
77
|
+
with avro_reader(url.with_mode("rb")) as reader:
|
|
78
|
+
assert reader.meta["avro.codec"] == b"snappy"
|
|
79
|
+
# Roundtrip: all records readable after snappy compression
|
|
80
|
+
with avro_reader(url.with_mode("rb")) as reader:
|
|
81
|
+
assert sum(1 for _ in reader) == 100
|
|
82
|
+
|
|
64
83
|
def test_write_text_mode(self):
|
|
65
84
|
with tempfile.NamedTemporaryFile() as tmp, pytest.raises(ValueError):
|
|
66
85
|
# Attempt to write to a file opened in text mode
|