bw-processing 1.1.1__tar.gz → 1.2__tar.gz

{bw_processing-1.1.1/src/bw_processing.egg-info → bw_processing-1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bw_processing
-Version: 1.1.1
+Version: 1.2
 Summary: Foo
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>
@@ -240,6 +240,26 @@ Datapackages are created using `create_datapackage`, which takes the following a
 * overwrite: `bool`, default `False`. Overwrite an existing resource with the same `dirpath` and `name`.
 * compress: `bool`, default `False`. Save to a zipfile, if saving to disk.
 
+### Zip file compression
+
+When writing a datapackage as a zip file via `generic_zipfile_filesystem`, compression is enabled by default (`zipfile.ZIP_DEFLATED`). Index arrays in particular compress extremely well (often to <10% of their original size) because they contain a limited set of repeated integer values. Float data arrays compress less, but the overall file size reduction is substantial.
+
+```python
+import zipfile
+from bw_processing.io_helpers import generic_zipfile_filesystem
+
+# Default: ZIP_DEFLATED (recommended — good compression, fast)
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip")
+
+# Uncompressed — fastest write/read, largest file
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip", compression=zipfile.ZIP_STORED)
+
+# LZMA — best compression ratio, but much slower to write
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip", compression=zipfile.ZIP_LZMA)
+```
+
+The `compression` and `compresslevel` arguments are passed directly to Python's `zipfile.ZipFile`. See the [Python docs](https://docs.python.org/3/library/zipfile.html#zipfile.ZipFile) for all options.
+
 Calling this function return an instance of `Datapackage`. You still need to add data.
 
 ## Contributing

{bw_processing-1.1.1 → bw_processing-1.2}/README.md

@@ -198,6 +198,26 @@ Datapackages are created using `create_datapackage`, which takes the following a
 * overwrite: `bool`, default `False`. Overwrite an existing resource with the same `dirpath` and `name`.
 * compress: `bool`, default `False`. Save to a zipfile, if saving to disk.
 
+### Zip file compression
+
+When writing a datapackage as a zip file via `generic_zipfile_filesystem`, compression is enabled by default (`zipfile.ZIP_DEFLATED`). Index arrays in particular compress extremely well (often to <10% of their original size) because they contain a limited set of repeated integer values. Float data arrays compress less, but the overall file size reduction is substantial.
+
+```python
+import zipfile
+from bw_processing.io_helpers import generic_zipfile_filesystem
+
+# Default: ZIP_DEFLATED (recommended — good compression, fast)
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip")
+
+# Uncompressed — fastest write/read, largest file
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip", compression=zipfile.ZIP_STORED)
+
+# LZMA — best compression ratio, but much slower to write
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip", compression=zipfile.ZIP_LZMA)
+```
+
+The `compression` and `compresslevel` arguments are passed directly to Python's `zipfile.ZipFile`. See the [Python docs](https://docs.python.org/3/library/zipfile.html#zipfile.ZipFile) for all options.
+
 Calling this function return an instance of `Datapackage`. You still need to add data.
 
 ## Contributing

{bw_processing-1.1.1 → bw_processing-1.2}/src/bw_processing/__init__.py

@@ -29,7 +29,7 @@ __all__ = (
     "UndefinedInterface",
 )
 
-__version__ = "1.1.1"
+__version__ = "1.2"
 
 
 from bw_processing.array_creation import create_array, create_structured_array

{bw_processing-1.1.1 → bw_processing-1.2}/src/bw_processing/io_helpers.py

@@ -1,4 +1,5 @@
 import json
+import zipfile
 from mimetypes import guess_type
 from pathlib import Path
 from typing import Any, Optional, Union
@@ -35,12 +36,22 @@ def generic_directory_filesystem(*, dirpath: Path) -> DirFileSystem:
 
 
 def generic_zipfile_filesystem(
-    *, dirpath: Path, filename: str, write: bool = True
+    *,
+    dirpath: Path,
+    filename: str,
+    write: bool = True,
+    compression: int = zipfile.ZIP_DEFLATED,
+    compresslevel: Optional[int] = None,
 ) -> ZipFileSystem:
     assert isinstance(dirpath, Path), "`dirpath` must be a `pathlib.Path` instance"
     if not dirpath.is_dir():
         raise ValueError("Destination directory `{}` doesn't exist".format(dirpath))
-    return ZipFileSystem(dirpath / filename, mode="w" if write else "r")
+    return ZipFileSystem(
+        dirpath / filename,
+        mode="w" if write else "r",
+        compression=compression,
+        compresslevel=compresslevel,
+    )
 
 
 def file_reader(

{bw_processing-1.1.1 → bw_processing-1.2/src/bw_processing.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bw_processing
-Version: 1.1.1
+Version: 1.2
 Summary: Foo
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>
@@ -240,6 +240,26 @@ Datapackages are created using `create_datapackage`, which takes the following a
 * overwrite: `bool`, default `False`. Overwrite an existing resource with the same `dirpath` and `name`.
 * compress: `bool`, default `False`. Save to a zipfile, if saving to disk.
 
+### Zip file compression
+
+When writing a datapackage as a zip file via `generic_zipfile_filesystem`, compression is enabled by default (`zipfile.ZIP_DEFLATED`). Index arrays in particular compress extremely well (often to <10% of their original size) because they contain a limited set of repeated integer values. Float data arrays compress less, but the overall file size reduction is substantial.
+
+```python
+import zipfile
+from bw_processing.io_helpers import generic_zipfile_filesystem
+
+# Default: ZIP_DEFLATED (recommended — good compression, fast)
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip")
+
+# Uncompressed — fastest write/read, largest file
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip", compression=zipfile.ZIP_STORED)
+
+# LZMA — best compression ratio, but much slower to write
+fs = generic_zipfile_filesystem(dirpath=some_path, filename="my.zip", compression=zipfile.ZIP_LZMA)
+```
+
+The `compression` and `compresslevel` arguments are passed directly to Python's `zipfile.ZipFile`. See the [Python docs](https://docs.python.org/3/library/zipfile.html#zipfile.ZipFile) for all options.
+
 Calling this function return an instance of `Datapackage`. You still need to add data.
 
 ## Contributing

{bw_processing-1.1.1 → bw_processing-1.2}/tests/test_datapackage.py

@@ -1,4 +1,5 @@
 import shutil
+import zipfile
 from pathlib import Path
 
 import numpy as np
@@ -10,7 +11,7 @@ from morefs.dict import DictFS
 from bw_processing import create_datapackage, load_datapackage, simple_graph
 from bw_processing.constants import INDICES_DTYPE, UNCERTAINTY_DTYPE, MAX_SIGNED_64BIT_INT, MAX_SIGNED_32BIT_INT
 from bw_processing.errors import NonUnique, PotentialInconsistency, ShapeMismatch, WrongDatatype
-from bw_processing.io_helpers import generic_directory_filesystem
+from bw_processing.io_helpers import generic_directory_filesystem, generic_zipfile_filesystem
 
 dirpath = Path(__file__).parent.resolve() / "fixtures"
 
@@ -497,3 +498,41 @@ def test_simple_graph():
     assert d["col"].sum() == 1102
     d, _ = dp.get_resource("a different life-data.data")
     assert np.allclose(d, np.array([101.234, 777]))
+
+
+def _write_zip_datapackage(dirpath, filename="test.zip", **kwargs):
+    fs = generic_zipfile_filesystem(dirpath=dirpath, filename=filename, **kwargs)
+    dp = create_datapackage(fs=fs, name="test-zip-compression")
+    add_data(dp)
+    dp.finalize_serialization()
+    return dirpath / filename
+
+
+def test_zipfile_default_compression(tmp_path):
+    """Default compression is ZIP_DEFLATED; indices compress well so file is smaller than uncompressed."""
+    compressed_path = _write_zip_datapackage(tmp_path, filename="compressed.zip")
+    uncompressed_path = _write_zip_datapackage(tmp_path, filename="uncompressed.zip", compression=zipfile.ZIP_STORED)
+
+    with zipfile.ZipFile(compressed_path) as zf:
+        for info in zf.infolist():
+            if info.filename.endswith(".indices.npy"):
+                assert info.compress_type == zipfile.ZIP_DEFLATED
+
+    assert compressed_path.stat().st_size < uncompressed_path.stat().st_size
+
+
+def test_zipfile_stored_compression(tmp_path):
+    """ZIP_STORED writes uncompressed entries."""
+    zip_path = _write_zip_datapackage(tmp_path, compression=zipfile.ZIP_STORED)
+
+    with zipfile.ZipFile(zip_path) as zf:
+        for info in zf.infolist():
+            assert info.compress_type == zipfile.ZIP_STORED
+
+
+def test_zipfile_compressed_roundtrip(tmp_path):
+    """Data written with ZIP_DEFLATED can be read back correctly."""
+    zip_path = _write_zip_datapackage(tmp_path)
+    dp = load_datapackage(ZipFileSystem(str(zip_path)))
+    data, _ = dp.get_resource("sa-data-vector.data")
+    assert np.allclose(data, [2, 7, 12])