zipremove 0.2.0__tar.gz → 0.3.0__tar.gz

{zipremove-0.2.0/src/zipremove.egg-info → zipremove-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zipremove
-Version: 0.2.0
+Version: 0.3.0
 Summary: Extend `zipfile` with `remove`-related functionalities
 Home-page: https://github.com/danny0838/zipremove
 Author: Danny Lin
@@ -10,6 +10,7 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: System :: Archiving :: Compression
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -31,6 +32,12 @@ Requires-Dist: flake8-isort>=6.0; extra == "dev"
 Requires-Dist: isort>=5.5; extra == "dev"
 Dynamic: license-file
 
+![PyPI version](https://img.shields.io/pypi/v/zipremove.svg)
+![Python Versions](https://img.shields.io/pypi/pyversions/zipremove)
+![Status](https://img.shields.io/pypi/status/zipremove)
+![License](https://img.shields.io/github/license/danny0838/zipremove)
+[![Downloads](https://static.pepy.tech/personalized-badge/zipremove?period=month&left_text=Downloads)](https://pepy.tech/project/zipremove)
+
 This package extends `zipfile` with `remove`-related functionalities.
 
 ## API

{zipremove-0.2.0 → zipremove-0.3.0}/README.md

@@ -1,3 +1,9 @@
+![PyPI version](https://img.shields.io/pypi/v/zipremove.svg)
+![Python Versions](https://img.shields.io/pypi/pyversions/zipremove)
+![Status](https://img.shields.io/pypi/status/zipremove)
+![License](https://img.shields.io/github/license/danny0838/zipremove)
+[![Downloads](https://static.pepy.tech/personalized-badge/zipremove?period=month&left_text=Downloads)](https://pepy.tech/project/zipremove)
+
 This package extends `zipfile` with `remove`-related functionalities.
 
 ## API

{zipremove-0.2.0 → zipremove-0.3.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = zipremove
-version = 0.2.0
+version = 0.3.0
 author = Danny Lin
 author_email = danny0838@gmail.com
 url = https://github.com/danny0838/zipremove
@@ -13,6 +13,7 @@ classifiers =
 	Intended Audience :: Developers
 	Topic :: System :: Archiving :: Compression
 	Programming Language :: Python :: 3
+	Programming Language :: Python :: 3 :: Only
 	Programming Language :: Python :: 3.9
 	Programming Language :: Python :: 3.10
 	Programming Language :: Python :: 3.11

{zipremove-0.2.0 → zipremove-0.3.0}/src/zipremove/__init__.py

@@ -3,7 +3,9 @@ import io
 import os
 import struct
 from zipfile import *
-from zipfile import (  # noqa: F401
+from zipfile import __all__  # noqa: F401
+from zipfile import _get_compressor  # noqa: F401
+from zipfile import (
     _DD_SIGNATURE,
     _FH_COMPRESSED_SIZE,
     _FH_COMPRESSION_METHOD,
@@ -13,8 +15,6 @@ from zipfile import (  # noqa: F401
     _FH_GENERAL_PURPOSE_FLAG_BITS,
     _FH_SIGNATURE,
     _FH_UNCOMPRESSED_SIZE,
-    LZMADecompressor,
-    _get_compressor,
     _get_decompressor,
     crc32,
     sizeFileHeader,
@@ -49,6 +49,20 @@ except ImportError:
             filename = filename.replace(os.altsep, "/")
         return filename
 
+try:
+    import zipfile
+    zipfile.LZMADecompressor().unused_data
+except AttributeError:
+    # polyfill to support LZMADecompressor().unused_data
+    class LZMADecompressor(zipfile.LZMADecompressor):
+        @property
+        def unused_data(self):
+            try:
+                return self._decomp.unused_data
+            except AttributeError:
+                return b''
+    zipfile.LZMADecompressor = LZMADecompressor
+
 
 class _ZipRepacker:
     """Class for ZipFile repacking."""
@@ -471,11 +485,6 @@ class _ZipRepacker:
         if decompressor is None:
             return False
 
-        # Current LZMADecompressor is unreliable since it's `.eof` is usually
-        # not set as expected.
-        if isinstance(decompressor, LZMADecompressor):
-            return False
-
         dd_fmt = '<LQQ' if zip64 else '<LLL'
         dd_size = struct.calcsize(dd_fmt)
 

{zipremove-0.2.0 → zipremove-0.3.0/src/zipremove.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zipremove
-Version: 0.2.0
+Version: 0.3.0
 Summary: Extend `zipfile` with `remove`-related functionalities
 Home-page: https://github.com/danny0838/zipremove
 Author: Danny Lin
@@ -10,6 +10,7 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: System :: Archiving :: Compression
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -31,6 +32,12 @@ Requires-Dist: flake8-isort>=6.0; extra == "dev"
 Requires-Dist: isort>=5.5; extra == "dev"
 Dynamic: license-file
 
+![PyPI version](https://img.shields.io/pypi/v/zipremove.svg)
+![Python Versions](https://img.shields.io/pypi/pyversions/zipremove)
+![Status](https://img.shields.io/pypi/status/zipremove)
+![License](https://img.shields.io/github/license/danny0838/zipremove)
+[![Downloads](https://static.pepy.tech/personalized-badge/zipremove?period=month&left_text=Downloads)](https://pepy.tech/project/zipremove)
+
 This package extends `zipfile` with `remove`-related functionalities.
 
 ## API

{zipremove-0.2.0 → zipremove-0.3.0}/tests/test_zipfile.py

@@ -1623,50 +1623,51 @@ class ZipRepackerTests(unittest.TestCase):
 
     def test_scan_data_descriptor(self):
         repacker = zipfile._ZipRepacker()
+        SIG = zipfile._DD_SIGNATURE
 
         # basic
-        bytes_ = b'dummyPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<4L', SIG, 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False),
             (0x4ff4f23f, 5, 5, 16),
         )
 
         # return None if no signature
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False),
             None,
         )
 
         # return None if not unpackable
-        bytes_ = b'PK\x07\x08'
+        bytes_ = struct.pack('<L', SIG)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False),
             None,
         )
 
         # return None if compressed size not match
-        bytes_ = b'dummPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dumm' + struct.pack('<4L', SIG, 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False),
             None,
         )
 
         # zip64
-        bytes_ = b'dummyPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<2L2Q', SIG, 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), True),
             (0x4ff4f23f, 5, 5, 24),
         )
 
         # offset
-        bytes_ = b'dummyPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<4L', SIG, 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 1, len(bytes_), False),
             None,
         )
 
-        bytes_ = b'123dummyPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'123dummy' + struct.pack('<4L', SIG, 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False),
             None,
@@ -1677,13 +1678,13 @@ class ZipRepackerTests(unittest.TestCase):
         )
 
         # end_offset
-        bytes_ = b'dummyPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<4L', SIG, 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_) - 1, False),
             None,
         )
 
-        bytes_ = b'dummyPK\x07\x08\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00123'
+        bytes_ = b'dummy' + struct.pack('<4L', SIG, 0x4ff4f23f, 5, 5) + b'123'
         self.assertEqual(
             repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_) - 3, False),
             (0x4ff4f23f, 5, 5, 16),
@@ -1693,34 +1694,34 @@ class ZipRepackerTests(unittest.TestCase):
         repacker = zipfile._ZipRepacker()
 
         # basic
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False),
             (0x4ff4f23f, 5, 5, 12),
         )
 
         # return None if compressed size not match
-        bytes_ = b'dumm\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dumm' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False),
             None,
         )
 
         # zip64
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<L2Q', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), True),
             (0x4ff4f23f, 5, 5, 20),
         )
 
         # offset
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 1, len(bytes_), False),
             None,
         )
 
-        bytes_ = b'123dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'123dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False),
             None,
@@ -1731,20 +1732,20 @@ class ZipRepackerTests(unittest.TestCase):
         )
 
         # end_offset
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_) - 1, False),
             None,
         )
 
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00123'
+        bytes_ = b'dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5) + b'123'
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_) - 3, False),
             (0x4ff4f23f, 5, 5, 12),
         )
 
         # chunk_size
-        bytes_ = b'dummy\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = b'dummy' + struct.pack('<3L', 0x4ff4f23f, 5, 5)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False, 12),
             (0x4ff4f23f, 5, 5, 12),
@@ -1767,7 +1768,7 @@ class ZipRepackerTests(unittest.TestCase):
 
     @requires_lzma()
     def test_scan_data_descriptor_no_sig_by_decompression_lzma(self):
-        self._test_scan_data_descriptor_no_sig_by_decompression_invalid(zipfile.ZIP_LZMA)
+        self._test_scan_data_descriptor_no_sig_by_decompression(zipfile.ZIP_LZMA)
 
     @requires_zstd()
     def test_scan_data_descriptor_no_sig_by_decompression_zstd(self):
@@ -1782,27 +1783,34 @@ class ZipRepackerTests(unittest.TestCase):
 
         compressor = zipfile._get_compressor(method)
 
-        comp_bytes = compressor.compress(b'dummy')
+        raw_bytes = b'dummy'
+        raw_len = len(raw_bytes)
+        comp_bytes = compressor.compress(raw_bytes)
         comp_bytes += compressor.flush()
         comp_len = len(comp_bytes)
 
         # basic
-        bytes_ = comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<L', comp_len) + b'\x05\x00\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_), False, method),
-            (0x4ff4f23f, comp_len, 5, 12),
+            (0x4ff4f23f, comp_len, raw_len, 12),
         )
 
         # return None if insufficient data length
-        bytes_ = b'\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_) - 1, False, method),
             None,
         )
 
-        bytes_ = b'\x3f\xf2\xf4\x4f\x05\x00\x00\x00\x05\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len)[:-1]
+        self.assertEqual(
+            repacker._scan_data_descriptor_no_sig_by_decompression(
+                io.BytesIO(bytes_), 0, len(bytes_), False, method),
+            None,
+        )
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_) + 1, False, method),
@@ -1810,7 +1818,7 @@ class ZipRepackerTests(unittest.TestCase):
         )
 
         # return None if compressed size not match
-        bytes_ = comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<L', comp_len - 1) + b'\x05\x00\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len - 1, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_), False, method),
@@ -1818,41 +1826,41 @@ class ZipRepackerTests(unittest.TestCase):
         )
 
         # zip64
-        bytes_ = comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<Q', comp_len) + b'\x05\x00\x00\x00\x00\x00\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<L2Q', 0x4ff4f23f, comp_len, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_), True, method),
-            (0x4ff4f23f, comp_len, 5, 20),
+            (0x4ff4f23f, comp_len, raw_len, 20),
         )
 
         # offset
-        bytes_ = comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<L', comp_len) + b'\x05\x00\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 1, len(bytes_), False, method),
             None,
         )
 
-        bytes_ = b'123' + comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<L', comp_len) + b'\x05\x00\x00\x00'
+        bytes_ = b'123' + comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 3, len(bytes_), False, method),
-            (0x4ff4f23f, comp_len, 5, 12),
+            (0x4ff4f23f, comp_len, raw_len, 12),
         )
 
         # end_offset
-        bytes_ = comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<L', comp_len) + b'\x05\x00\x00\x00'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len)
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_) - 2, False, method),
             None,
         )
 
-        bytes_ = comp_bytes + b'\x3f\xf2\xf4\x4f' + struct.pack('<L', comp_len) + b'\x05\x00\x00\x00123'
+        bytes_ = comp_bytes + struct.pack('<3L', 0x4ff4f23f, comp_len, raw_len) + b'123'
         self.assertEqual(
             repacker._scan_data_descriptor_no_sig_by_decompression(
                 io.BytesIO(bytes_), 0, len(bytes_) - 2, False, method),
-            (0x4ff4f23f, comp_len, 5, 12),
+            (0x4ff4f23f, comp_len, raw_len, 12),
         )
 
     def _test_scan_data_descriptor_no_sig_by_decompression_invalid(self, method):

{zipremove-0.2.0 → zipremove-0.3.0}/tests/test_zipfile64.py

@@ -13,10 +13,10 @@ from .test_zipfile import struct_pack_no_dd_sig
 
 # polyfills
 try:
-    from test.test_zipfile.test_core import Unseekable
+    from test.test_zipfile.test_core import Unseekable, requires_zlib
 except ImportError:
     # polyfill for Python < 3.12
-    from test.test_zipfile import Unseekable
+    from test.test_zipfile import Unseekable, requires_zlib
 
 ENABLED_RESOURCES = set(os.environ.get("TEST_RESOURCES", "").split(","))
 
@@ -172,5 +172,39 @@ class TestRepack(unittest.TestCase):
             zh.repack()
             self.assertIsNone(zh.testzip())
 
+    @requires_zlib()
+    def test_strip_removed_large_file_with_dd_no_sig_by_decompression(self):
+        """Should scan for the data descriptor (without signature) of a removed
+        large file without causing a memory issue."""
+        # Try the temp file.  If we do TESTFN2, then it hogs
+        # gigabytes of disk space for the duration of the test.
+        with TemporaryFile() as f:
+            tracemalloc.start()
+            self._test_strip_removed_large_file_with_dd_no_sig_by_decompression(
+                f, zipfile.ZIP_DEFLATED)
+            self.assertFalse(f.closed)
+            current, peak = tracemalloc.get_traced_memory()
+            tracemalloc.stop()
+            self.assertLess(peak, self.allowed_memory)
+
+    def _test_strip_removed_large_file_with_dd_no_sig_by_decompression(self, f, method):
+        file = 'file.txt'
+        file1 = 'largefile.txt'
+        data = b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem'
+        with mock.patch('zipfile.struct.pack', side_effect=struct_pack_no_dd_sig):
+            with zipfile.ZipFile(Unseekable(f), 'w', compression=method) as zh:
+                with zh.open(file1, 'w', force_zip64=True) as fh:
+                    self._write_large_file(fh)
+                zh.writestr(file, data)
+
+        with zipfile.ZipFile(f, 'a') as zh:
+            # make sure data descriptor bit is really set (by making zip file unseekable)
+            for zi in zh.infolist():
+                self.assertTrue(zi.flag_bits & 8, f'data descriptor flag not set: {zi.filename}')
+
+            zh.remove(file1)
+            zh.repack()
+            self.assertIsNone(zh.testzip())
+
 if __name__ == "__main__":
     unittest.main()