avrokit 0.1.0__tar.gz → 0.1.3__tar.gz

{avrokit-0.1.0 → avrokit-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avrokit
-Version: 0.1.0
+Version: 0.1.3
 Summary: Python utilities for working with Avro data files
 Project-URL: Homepage, https://github.com/brandtg/avrokit
 Project-URL: Repository, https://github.com/brandtg/avrokit

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/tools/httpserver.py

@@ -17,6 +17,7 @@ from avro.io import DatumReader
 
 from ..io import avro_reader, avro_schema, avro_writer, read_avro_schema
 from ..url import URL, parse_url
+from .tojson import AvroJsonEncoder
 
 logger = logging.getLogger(__name__)
 
@@ -195,7 +196,9 @@ class AvroHTTPRequestHandler(BaseHTTPRequestHandler):
                     self.send_header("Content-type", "application/jsonl")
                     self.end_headers()
                     for record in reader:
-                        self.wfile.write(json.dumps(record).encode("utf-8"))
+                        self.wfile.write(
+                            json.dumps(record, cls=AvroJsonEncoder).encode("utf-8")
+                        )
                         self.wfile.write(b"\n")
             elif self._is_avro_binary(accept):
                 with data_url as f:

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/tools/tojson.py

@@ -5,11 +5,24 @@
 import argparse
 import json
 import sys
+from datetime import date, datetime, time
+from decimal import Decimal
 
 from ..io import avro_reader
 from ..url import flatten_urls, parse_url
 
 
+class AvroJsonEncoder(json.JSONEncoder):
+    """Encodes Avro logical types (datetime, date, time, decimal) as JSON."""
+
+    def default(self, o: object) -> object:
+        if isinstance(o, datetime | date | time):
+            return o.isoformat()
+        if isinstance(o, Decimal):
+            return str(o)
+        return super().default(o)
+
+
 class ToJsonTool:
     """
     Dumps Avro data file(s) as JSON, record per line.
@@ -30,5 +43,5 @@ class ToJsonTool:
         for url in urls:
             with avro_reader(url) as reader:
                 for record in reader:
-                    json.dump(record, sys.stdout)
+                    json.dump(record, sys.stdout, cls=AvroJsonEncoder)
                     sys.stdout.write("\n")

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/url/base.py

@@ -4,10 +4,11 @@
 
 from __future__ import annotations
 
+import fnmatch
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
 from typing import IO, Any
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunparse
 
 
 class URL(ABC):
@@ -22,13 +23,44 @@ class URL(ABC):
     def __str__(self) -> str:
         return self.url
 
-    @abstractmethod
     def expand(self) -> Sequence[URL]:
         """
         Expands the URL into all the URLs that represent concrete resources.
+
+        If the final path element contains glob wildcards, the parent path is
+        expanded and the results are filtered client-side with fnmatch.
+        """
+        if self._has_glob():
+            return self._expand_glob()
+        return self._expand()
+
+    @abstractmethod
+    def _expand(self) -> Sequence[URL]:
+        """
+        Expands the URL into all the URLs that represent concrete resources.
         """
         ...
 
+    def _has_glob(self) -> bool:
+        return any(c in self.parsed_url.path for c in "*?[")
+
+    def _expand_glob(self) -> Sequence[URL]:
+        path = self.parsed_url.path
+        wildcard_idx = min(idx for idx in (path.find(c) for c in "*?[") if idx != -1)
+        prefix = path[:wildcard_idx]
+        parent = self._with_path(prefix)
+        return [
+            url
+            for url in parent.expand()
+            if url.url != parent.url and fnmatch.fnmatch(url.parsed_url.path, path)
+        ]
+
+    def _with_path(self, path: str) -> URL:
+        new_url = urlunparse(
+            (self.parsed_url.scheme, self.parsed_url.netloc, path, "", "", "")
+        )
+        return type(self)(new_url, mode=self.mode)
+
     @abstractmethod
     def delete(self) -> None:
         """

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/url/file.py

@@ -21,20 +21,7 @@ class FileURL(URL):
         self.fh: IO[Any] | None = None
 
     @override
-    def expand(self) -> Sequence[URL]:
-        # Check if the path contains wildcard characters
-        if "*" in self.path or "?" in self.path or "[" in self.path:
-            # Use glob to expand the pattern
-            matches = glob.glob(self.path)
-            if not matches:
-                # No matches found, return empty list
-                return []
-            # Return sorted list of FileURL objects for all matches
-            return [
-                FileURL(self.prefix + match, mode=self.mode)
-                for match in sorted(matches)
-            ]
-
+    def _expand(self) -> Sequence[URL]:
         # If it's a regular file or doesn't exist (no expansion needed)
         if os.path.isfile(self.path) or not os.path.exists(self.path):
             return [self]
@@ -48,6 +35,15 @@ class FileURL(URL):
                 )
         return acc
 
+    @override
+    def _expand_glob(self) -> Sequence[URL]:
+        matches = glob.glob(self.path)
+        if not matches:
+            return []
+        return [
+            FileURL(self.prefix + match, mode=self.mode) for match in sorted(matches)
+        ]
+
     @override
     def delete(self) -> None:
         # If it's a wildcard pattern, expand and delete all matching files

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/url/google.py

@@ -59,7 +59,7 @@ class GoogleCloudStorageURL(URL):
         return self.url == value.url
 
     @override
-    def expand(self) -> Sequence[URL]:
+    def _expand(self) -> Sequence[URL]:
         with google_cloud_storage_client() as client:
             try:
                 path = self.path
@@ -152,7 +152,16 @@ class GoogleCloudStorageURL(URL):
                     self._current_remote.upload_from_filename(self._current_local.name)
             finally:
                 # Cleanup resources
-                self._current_local.close()  # N.b. deletes on close
+                local = self._current_local
+                name = local.name
+                local.close()
+                # N.b. the temp file is created with delete=False so it survives an external
+                # close (e.g. DataFileWriter closing the stream before URL.close() uploads).
+                # Unlink it explicitly here to avoid leaking a file per open/close cycle.
+                try:
+                    os.unlink(name)
+                except FileNotFoundError:
+                    pass
                 self._current_client.close()
                 self._current_local = None
                 self._current_local_stream = None

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/url/http.py

@@ -36,7 +36,7 @@ class HttpURL(URL):
         self._current_request_buffer: IO[Any] | None = None
 
     @override
-    def expand(self) -> Sequence[URL]:
+    def _expand(self) -> Sequence[URL]:
         # N.b. no way to "expand" an HTTP URL, i.e. discover sub resources
         return [self]
 

{avrokit-0.1.0 → avrokit-0.1.3}/avrokit/url/s3.py

@@ -34,7 +34,7 @@ class S3URL(URL):
         self._current_local_stream: IO[Any] | None = None
 
     @override
-    def expand(self) -> Sequence[URL]:
+    def _expand(self) -> Sequence[URL]:
         with s3_client() as client:
             path = self.path
             if not path.endswith("/"):
@@ -49,7 +49,9 @@ class S3URL(URL):
                     for obj in page["Contents"]:
                         key = obj.get("Key")
                         if key:
-                            urls.append(S3URL(self._append_path(key), mode=self.mode))
+                            urls.append(
+                                S3URL(f"s3://{self.bucket}/{key}", mode=self.mode)
+                            )
             if not urls:
                 # N.b. we also include URLs that don't exist yet here because there is nothing to expand
                 return [self]

{avrokit-0.1.0 → avrokit-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@
 
 [project]
 name = "avrokit"
-version = "0.1.0"
+version = "0.1.3"
 description = "Python utilities for working with Avro data files"
 authors = [{ name = "Greg Brandt", email = "brandt.greg@gmail.com" }]
 license = "Apache-2.0"

{avrokit-0.1.0 → avrokit-0.1.3}/tests/test_tools_httpserver.py

@@ -6,6 +6,7 @@ import concurrent.futures as futures
 import json
 import tempfile
 import threading
+from datetime import datetime, timezone
 
 import pytest
 import requests
@@ -99,6 +100,34 @@ class TestHttpServerTool:
                 assert isinstance(email, str)
                 assert "@" in email
 
+    def test_get_json_logical_types(self, tmpserver):
+        http_url, root_url = tmpserver
+        schema = avro_schema(
+            {
+                "type": "record",
+                "name": "Event",
+                "fields": [
+                    {
+                        "name": "created_at",
+                        "type": {"type": "long", "logicalType": "timestamp-millis"},
+                    },
+                ],
+            }
+        )
+        with avro_writer(
+            root_url.with_path("logical.avro").with_mode("wb"), schema
+        ) as writer:
+            writer.append(
+                {"created_at": datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)}
+            )
+        res = requests.get(
+            f"{http_url}/logical.avro", headers={"Accept": "application/json"}
+        )
+        res.raise_for_status()
+        data = [json.loads(line) for line in res.content.decode("utf-8").splitlines()]
+        assert len(data) == 1
+        assert data[0]["created_at"] == "2025-01-02T03:04:05+00:00"
+
     def test_get_avro(self, tmpserver):
         http_url, _ = tmpserver
         res = requests.get(

{avrokit-0.1.0 → avrokit-0.1.3}/tests/test_tools_integration.py

@@ -9,6 +9,8 @@ Integration tests for command-line tools in avrokit.
 import json
 import os
 import tempfile
+from datetime import date, datetime, timezone
+from decimal import Decimal
 from typing import cast
 
 import pytest
@@ -445,6 +447,74 @@ class TestToJsonTool:
                     assert record["id"] == i
                     assert record["name"] == f"name_{i}"
 
+    def test_tojson_logical_types(self):
+        """Test that Avro logical types (timestamps, dates, decimals) are serialized."""
+        schema = avro_schema(
+            {
+                "type": "record",
+                "name": "Test",
+                "fields": [
+                    {"name": "id", "type": "int"},
+                    {
+                        "name": "created_at",
+                        "type": {"type": "long", "logicalType": "timestamp-millis"},
+                    },
+                    {"name": "day", "type": {"type": "int", "logicalType": "date"}},
+                    {
+                        "name": "amount",
+                        "type": {
+                            "type": "bytes",
+                            "logicalType": "decimal",
+                            "precision": 10,
+                            "scale": 2,
+                        },
+                    },
+                ],
+            }
+        )
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            avro_file = os.path.join(tmpdir, "input.avro")
+            json_file = os.path.join(tmpdir, "output.json")
+            avro_url = parse_url(avro_file)
+
+            with avro_writer(avro_url.with_mode("wb"), schema) as writer:
+                writer.append(
+                    {
+                        "id": 1,
+                        "created_at": datetime(
+                            2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc
+                        ),
+                        "day": date(2025, 1, 2),
+                        "amount": Decimal("12.34"),
+                    }
+                )
+
+            tool = ToJsonTool()
+
+            import argparse
+
+            args = argparse.Namespace(url=[avro_file])
+
+            with open(json_file, "w") as f:
+                import sys
+
+                old_stdout = sys.stdout
+                sys.stdout = f
+                try:
+                    tool.run(args)
+                finally:
+                    sys.stdout = old_stdout
+
+            with open(json_file) as f:
+                lines = f.readlines()
+                assert len(lines) == 1
+                record = json.loads(lines[0])
+                assert record["id"] == 1
+                assert record["created_at"] == "2025-01-02T03:04:05+00:00"
+                assert record["day"] == "2025-01-02"
+                assert record["amount"] == "12.34"
+
 
 class TestGetSchemaTool:
     """Tests for GetSchema tool."""

{avrokit-0.1.0 → avrokit-0.1.3}/tests/test_url_file.py

@@ -36,6 +36,19 @@ class TestFileURL:
         assert isinstance(expanded_urls[0], FileURL)
         assert expanded_urls[0].url == f"file://{tmpdir}/file_0.txt"
 
+    def test_expand_glob(self, tmpdir):
+        url = FileURL(f"file://{tmpdir}/file_*.txt")
+        expanded_urls = url.expand()
+        assert len(expanded_urls) == 10
+        assert all(isinstance(u, FileURL) for u in expanded_urls)
+        assert [u.url for u in expanded_urls] == [
+            f"file://{tmpdir}/file_{i}.txt" for i in range(10)
+        ]
+
+    def test_expand_glob_no_match(self, tmpdir):
+        url = FileURL(f"file://{tmpdir}/nonexistent_*.txt")
+        assert url.expand() == []
+
     def test_exists_directory(self, tmpdir):
         url = FileURL(f"file://{tmpdir}")
         assert url.exists()

{avrokit-0.1.0 → avrokit-0.1.3}/tests/test_url_google.py

@@ -13,7 +13,7 @@ from filelock import FileLock
 
 from avrokit.io import avro_reader, avro_schema, avro_writer
 from avrokit.url.factory import parse_url
-from avrokit.url.google import google_cloud_storage_client
+from avrokit.url.google import GoogleCloudStorageURL, google_cloud_storage_client
 
 faker = Faker()
 
@@ -144,3 +144,69 @@ class TestGoogleCloudStorageURL:
         # Delete the file
         url.delete()
         assert url.exists() is False
+
+    def test_expand_glob(self, gcs_bucket):
+        base = f"{gcs_bucket}/test/glob"
+        for i in range(5):
+            url = parse_url(f"{base}/part_{i}.avro")
+            if url.exists():
+                url.delete()
+            with url.with_mode("w") as f:
+                f.write(f"Content {i}")
+        # Also create a non-matching file
+        other = parse_url(f"{base}/other.txt")
+        with other.with_mode("w") as f:
+            f.write("other")
+        expanded = parse_url(f"{base}/*.avro").expand()
+        assert sorted(u.url for u in expanded) == [
+            f"{base}/part_{i}.avro" for i in range(5)
+        ]
+        # No matches returns empty list
+        assert parse_url(f"{base}/nonexistent_*.avro").expand() == []
+
+    def test_no_temp_file_leak(self, monkeypatch):
+        class FakeBlob:
+            def __init__(self):
+                self.content = b""
+
+            def download_to_file(self, f):
+                f.write(self.content)
+                f.flush()
+
+            def upload_from_filename(self, name):
+                with open(name, "rb") as f:
+                    self.content = f.read()
+
+        class FakeBucket:
+            def __init__(self):
+                self.blob_obj = FakeBlob()
+
+            def blob(self, path):
+                return self.blob_obj
+
+        class FakeClient:
+            def __init__(self):
+                self.bucket_obj = FakeBucket()
+
+            def bucket(self, name):
+                return self.bucket_obj
+
+            def close(self):
+                pass
+
+        fake_client = FakeClient()
+        monkeypatch.setattr("avrokit.url.google._create_client", lambda: fake_client)
+
+        tmpdir = tempfile.mkdtemp()
+        monkeypatch.setattr(tempfile, "tempdir", tmpdir)
+
+        url = GoogleCloudStorageURL("gs://test-bucket/test/leak.txt", mode="w")
+        with url as f:
+            f.write("hello")
+        assert fake_client.bucket_obj.blob_obj.content == b"hello"
+        assert os.listdir(tmpdir) == []
+
+        url = GoogleCloudStorageURL("gs://test-bucket/test/leak.txt", mode="r")
+        with url as f:
+            assert f.read() == "hello"
+        assert os.listdir(tmpdir) == []

{avrokit-0.1.0 → avrokit-0.1.3}/tests/test_url_s3.py

@@ -58,6 +58,25 @@ class TestS3URL:
         url.delete()
         assert url.exists() is False
 
+    def test_expand_glob(self, s3_bucket):
+        base = f"{s3_bucket}/test/glob"
+        for i in range(5):
+            url = parse_url(f"{base}/part_{i}.avro")
+            if url.exists():
+                url.delete()
+            with url.with_mode("w") as f:
+                f.write(f"Content {i}")
+        # Also create a non-matching file
+        other = parse_url(f"{base}/other.txt")
+        with other.with_mode("w") as f:
+            f.write("other")
+        expanded = parse_url(f"{base}/*.avro").expand()
+        assert sorted(u.url for u in expanded) == [
+            f"{base}/part_{i}.avro" for i in range(5)
+        ]
+        # No matches returns empty list
+        assert parse_url(f"{base}/nonexistent_*.avro").expand() == []
+
     def test_avro(self, s3_bucket):
         url = parse_url(f"{s3_bucket}/test/avro/file.avro")
         # Clean up any existing file

{avrokit-0.1.0 → avrokit-0.1.3}/uv.lock

@@ -26,7 +26,7 @@ wheels = [
 
 [[package]]
 name = "avrokit"
-version = "0.1.0"
+version = "0.1.2"
 source = { editable = "." }
 dependencies = [
     { name = "avro" },