vecorel-cli 0.3.0__tar.gz → 0.3.1__tar.gz

{vecorel_cli-0.3.0/vecorel_cli.egg-info → vecorel_cli-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.3.0
+Version: 0.3.1
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -24,6 +24,7 @@ Requires-Dist: shapely<3.0,>=2.1
 Requires-Dist: numpy<3.0,>=2.0
 Requires-Dist: pyarrow<24.0,>=21.0
 Requires-Dist: py7zr<2.0,>=1.0
+Requires-Dist: inflate64<2.0,>=1.0
 Requires-Dist: multivolumefile<1.0,>=0.2.3
 Requires-Dist: fsspec>=2025.7.0
 Requires-Dist: jsonschema[format]<5.0,>=4.20

{vecorel_cli-0.3.0 → vecorel_cli-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vecorel-cli"
-version = "0.3.0"
+version = "0.3.1"
 description = "CLI tools such as validation and file format conversion for vecorel."
 readme = "README.md"
 license = "Apache-2.0"
@@ -30,6 +30,7 @@ dependencies = [
     "numpy>=2.0,<3.0",
     "pyarrow>=21.0,<24.0",
     "py7zr>=1.0,<2.0",
+    "inflate64>=1.0,<2.0",
     "multivolumefile>=0.2.3,<1.0",
     "fsspec>=2025.7.0",
     "jsonschema[format]>=4.20,<5.0",
@@ -72,6 +73,7 @@ shapely = ">=2.1,<3.0"
 numpy = ">=2.0,<3.0"
 pyarrow = ">=21.0,<24.0"
 py7zr = ">=1.0,<2.0"
+inflate64 = ">=1.0,<2.0"
 multivolumefile = ">=0.2.3,<1.0"
 unrar = ">=0.4"
 
@@ -89,6 +91,12 @@ GitPython = ">=3.0.0,<4.0.0"
 [tool.pixi.feature.dev.pypi-dependencies]
 build = ">=1.0.0,<2.0.0"
 
+[tool.pixi.feature.py311.dependencies]
+python = "3.11.*"
+
+[tool.pixi.feature.py314.dependencies]
+python = "3.14.*"
+
 [tool.pixi.feature.s3.dependencies]
 s3fs = ">=2025.7.0"
 
@@ -102,6 +110,9 @@ s3 = {features = ["s3"], solve-group = "default"}
 gcs = {features = ["gcs"], solve-group = "default"}
 cloud = {features = ["s3", "gcs"], solve-group = "default"}
 all = {features = ["dev", "s3", "gcs"], solve-group = "default"}
+# CI tests the oldest and newest supported Python; pixi, not setup-python, decides the version
+dev-py311 = {features = ["dev", "py311"], solve-group = "py311"}
+dev-py314 = {features = ["dev", "py314"], solve-group = "py314"}
 
 [tool.pixi.tasks]
 # Development tasks

vecorel_cli-0.3.1/tests/test_deflate64.py

@@ -0,0 +1,68 @@
+import random
+import struct
+import zipfile
+import zlib
+
+import inflate64
+from pytest import mark, raises
+
+from vecorel_cli.conversion.base import BaseConverter
+from vecorel_cli.conversion.deflate64 import ZIP_DEFLATED64, deflate64_zip
+
+# Compressible content fits one read; random bytes need many, which exercises the
+# streaming decompressor protocol of ZipExtFile.
+CONTENTS = {
+    "compressible": b"deflate64 " * 20_000,
+    "incompressible": random.Random(64).randbytes(2_000_000),
+}
+
+
+def _write_deflate64_zip(path, data, name="member.txt"):
+    """A single-member ZIP compressed with Deflate64, which zipfile cannot write."""
+    deflater = inflate64.Deflater()
+    compressed = deflater.deflate(data) + deflater.flush()
+    crc = zlib.crc32(data)
+    encoded = name.encode()
+    sizes = (crc, len(compressed), len(data), len(encoded))
+    local = struct.pack("<4s5H3L2H", b"PK\x03\x04", 21, 0, ZIP_DEFLATED64, 0, 0, *sizes, 0)
+    central = struct.pack(
+        "<4s6H3L5H2L", b"PK\x01\x02", 21, 21, 0, ZIP_DEFLATED64, 0, 0, *sizes, 0, 0, 0, 0, 0, 0
+    )
+    offset = len(local) + len(encoded) + len(compressed)
+    end = struct.pack("<4s4H2LH", b"PK\x05\x06", 0, 0, 1, 1, len(central) + len(encoded), offset, 0)
+    path.write_bytes(local + encoded + compressed + central + encoded + end)
+    return path
+
+
+@mark.parametrize("kind", CONTENTS)
+def test_zipfile_reads_deflate64_only_inside_the_block(tmp_path, kind):
+    archive = _write_deflate64_zip(tmp_path / "a.zip", CONTENTS[kind])
+    with raises(NotImplementedError):
+        zipfile.ZipFile(archive).read("member.txt")
+    with deflate64_zip():
+        # read() verifies the CRC, so a truncated or corrupt stream fails here
+        assert zipfile.ZipFile(archive).read("member.txt") == CONTENTS[kind]
+    with raises(NotImplementedError):
+        zipfile.ZipFile(archive).read("member.txt")
+
+
+def test_overlapping_blocks_restore_zipfile_whatever_order_they_exit_in():
+    original = zipfile._get_decompressor
+    first, second = deflate64_zip(), deflate64_zip()
+    first.__enter__()
+    second.__enter__()
+    first.__exit__(None, None, None)
+    assert zipfile._get_decompressor is not original  # the second block still needs it
+    second.__exit__(None, None, None)
+    assert zipfile._get_decompressor is original
+
+
+@mark.parametrize("kind", CONTENTS)
+def test_download_files_extracts_deflate64_archives(tmp_path, kind):
+    archive = _write_deflate64_zip(tmp_path / "source.zip", CONTENTS[kind])
+    paths = BaseConverter().download_files(
+        {str(archive): ["member.txt"]}, cache_folder=str(tmp_path / "cache")
+    )
+    assert len(paths) == 1
+    with open(paths[0][0], "rb") as f:
+        assert f.read() == CONTENTS[kind]

{vecorel_cli-0.3.0 → vecorel_cli-0.3.1}/vecorel_cli/conversion/base.py

@@ -34,6 +34,7 @@ from ..vecorel.hilbert import hilbert_sort_geodataframe
 from ..vecorel.schemas import Schemas
 from ..vecorel.typing import Sources
 from ..vecorel.util import get_fs, name_from_uri, stream_file, suffix_duplicate_ids
+from .deflate64 import deflate64_zip
 from .flatdict import FlatDict
 
 # a source column named `id` moves here when the converter generates its own id,
@@ -399,16 +400,8 @@ class BaseConverter(LoggerMixin):
 
             if must_extract:
                 if zipfile.is_zipfile(cache_file):
-                    try:
-                        with zipfile.ZipFile(cache_file, "r") as zip_file:
-                            zip_file.extractall(zip_folder)
-                    except NotImplementedError as e:
-                        if str(e) != "That compression method is not supported":
-                            raise e
-                        import zipfile_deflate64
-
-                        with zipfile_deflate64.ZipFile(cache_file, "r") as zip_file:
-                            zip_file.extractall(zip_folder)
+                    with deflate64_zip(), zipfile.ZipFile(cache_file, "r") as zip_file:
+                        zip_file.extractall(zip_folder)
                 elif py7zr.is_7zfile(cache_file):
                     with py7zr.SevenZipFile(cache_file, "r") as sz_file:
                         sz_file.extractall(zip_folder)

vecorel_cli-0.3.1/vecorel_cli/conversion/deflate64.py

@@ -0,0 +1,54 @@
+import threading
+import zipfile
+from contextlib import contextmanager
+
+import inflate64
+
+# Deflate64 ("enhanced deflate", ZIP method 9) is what Windows writes for large archives,
+# and python's zipfile cannot read it. inflate64, a dependency of py7zr, decodes it.
+ZIP_DEFLATED64 = 9
+
+_lock = threading.Lock()
+_users = 0
+_original_get_decompressor = zipfile._get_decompressor
+
+
+class _Deflate64Decompressor:
+    """The interface zipfile.ZipExtFile expects of a decompressor other than zlib's
+    (Python 3.11 to 3.14): decompress(data) and eof."""
+
+    def __init__(self):
+        self._inflater = inflate64.Inflater()
+
+    @property
+    def eof(self):
+        return self._inflater.eof
+
+    def decompress(self, data):
+        return self._inflater.inflate(data)
+
+
+def _get_decompressor(compress_type):
+    if compress_type == ZIP_DEFLATED64:
+        return _Deflate64Decompressor()
+    return _original_get_decompressor(compress_type)
+
+
+@contextmanager
+def deflate64_zip():
+    """Let zipfile read Deflate64 members for the duration of the block.
+
+    The hook is process-global, so it is counted: installed by the first block to enter
+    and restored by the last to exit, whatever order overlapping blocks exit in."""
+    global _users
+    with _lock:
+        if _users == 0:
+            zipfile._get_decompressor = _get_decompressor
+        _users += 1
+    try:
+        yield
+    finally:
+        with _lock:
+            _users -= 1
+            if _users == 0:
+                zipfile._get_decompressor = _original_get_decompressor

{vecorel_cli-0.3.0 → vecorel_cli-0.3.1/vecorel_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.3.0
+Version: 0.3.1
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -24,6 +24,7 @@ Requires-Dist: shapely<3.0,>=2.1
 Requires-Dist: numpy<3.0,>=2.0
 Requires-Dist: pyarrow<24.0,>=21.0
 Requires-Dist: py7zr<2.0,>=1.0
+Requires-Dist: inflate64<2.0,>=1.0
 Requires-Dist: multivolumefile<1.0,>=0.2.3
 Requires-Dist: fsspec>=2025.7.0
 Requires-Dist: jsonschema[format]<5.0,>=4.20

{vecorel_cli-0.3.0 → vecorel_cli-0.3.1}/vecorel_cli.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ tests/test_converters.py
 tests/test_create_geojson.py
 tests/test_create_geoparquet.py
 tests/test_create_stac.py
+tests/test_deflate64.py
 tests/test_describe.py
 tests/test_download.py
 tests/test_encoding_auto.py
@@ -51,6 +52,7 @@ vecorel_cli/cli/util.py
 vecorel_cli/conversion/__init__.py
 vecorel_cli/conversion/admin.py
 vecorel_cli/conversion/base.py
+vecorel_cli/conversion/deflate64.py
 vecorel_cli/conversion/duckdb.py
 vecorel_cli/conversion/flatdict.py
 vecorel_cli/datasets/__init__.py

{vecorel_cli-0.3.0 → vecorel_cli-0.3.1}/vecorel_cli.egg-info/requires.txt

@@ -6,6 +6,7 @@ shapely<3.0,>=2.1
 numpy<3.0,>=2.0
 pyarrow<24.0,>=21.0
 py7zr<2.0,>=1.0
+inflate64<2.0,>=1.0
 multivolumefile<1.0,>=0.2.3
 fsspec>=2025.7.0
 jsonschema[format]<5.0,>=4.20