diffoscope 262__tar.gz → 263__tar.gz

{diffoscope-262 → diffoscope-263}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: diffoscope
-Version: 262
+Version: 263
 Summary: in-depth comparison of files, archives, and directories
 Home-page: https://diffoscope.org/
 Author: Diffoscope developers

{diffoscope-262 → diffoscope-263}/debian/changelog

@@ -1,8 +1,25 @@
-diffoscope (263) UNRELEASED; urgency=medium
+diffoscope (264) UNRELEASED; urgency=medium
 
   * WIP (generated upon release).
 
- -- Chris Lamb <lamby@debian.org>  Fri, 29 Mar 2024 09:44:15 +0000
+ -- Chris Lamb <lamby@debian.org>  Fri, 05 Apr 2024 12:24:24 +0100
+
+diffoscope (263) unstable; urgency=medium
+
+  [ Chris Lamb ]
+  * Add support for the zipdetails(1) tool included in the Perl distribution.
+    Thanks to Larry Doolittle et al. for the pointer to this tool.
+  * Don't use parenthesis within test "skipping…" messages; PyTest adds its own
+    parenthesis, so we were ending up with double nested parens.
+  * Fix the .epub tests after supporting zipdetails(1).
+  * Update copyright years and debian/tests/control.
+
+  [ FC (Fay) Stegerman ]
+  * Fix MozillaZipContainer's monkeypatch after Python's zipfile module changed
+    to detect potentially insecure overlapping entries within .zip files.
+    (Closes: reproducible-builds/diffoscope#362)
+
+ -- Chris Lamb <lamby@debian.org>  Fri, 05 Apr 2024 12:21:10 +0100
 
 diffoscope (262) unstable; urgency=medium
 

{diffoscope-262 → diffoscope-263}/debian/tests/control

@@ -7,7 +7,7 @@
 #   $ mv debian/tests/control.tmp debian/tests/control
 
 Tests: pytest-with-recommends
-Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], 7zip, aapt [amd64 arm64 armel armhf i386 mips64el mipsel], abootimg, acl, apksigcopier, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, dexdump [amd64 arm64 armhf i386], docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg-utils, gpg, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-utils, llvm, lz4 | liblz4-tool, lzip, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, androguard, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
+Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], 7zip, aapt [amd64 arm64 armel armhf i386 mips64el mipsel], abootimg, acl, apksigcopier, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, dexdump [amd64 arm64 armhf i386], docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg-utils, gpg, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-utils, llvm, lz4 | liblz4-tool, lzip, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, perl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, androguard, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
 
 Tests: pytest
 Depends: python3-all, diffoscope, python3-pytest, python3-h5py, file, python3-tlsh

{diffoscope-262 → diffoscope-263}/diffoscope/__init__.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
 
-VERSION = "262"
+VERSION = "263"

{diffoscope-262 → diffoscope-263}/diffoscope/comparators/zip.py

@@ -2,7 +2,7 @@
 # diffoscope: in-depth comparison of files, archives, and directories
 #
 # Copyright © 2014-2015 Jérémy Bobbio <lunar@debian.org>
-# Copyright © 2015-2022 Chris Lamb <lamby@debian.org>
+# Copyright © 2015-2022, 2024 Chris Lamb <lamby@debian.org>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -154,6 +154,12 @@ def zipinfo_differences(file, other):
     return []
 
 
+class Zipdetails(Command):
+    @tool_required("zipdetails")
+    def cmdline(self):
+        return ["zipdetails", "--redact", "--scan", "--utc", self.path]
+
+
 class ZipDirectory(Directory, ArchiveMember):
     def __init__(self, archive, member_name):
         ArchiveMember.__init__(self, archive, member_name)
@@ -251,12 +257,13 @@ class ZipFile(ZipFileBase):
         if Config().exclude_directory_metadata != "recursive":
             differences.extend(zipinfo_differences(self, other))
 
-        try:
-            differences.append(
-                Difference.from_operation(Zipnote, self.path, other.path)
-            )
-        except RequiredToolNotFound:  # noqa
-            pass
+        for op in (Zipnote, Zipdetails):
+            try:
+                differences.append(
+                    Difference.from_operation(op, self.path, other.path)
+                )
+            except RequiredToolNotFound:  # noqa
+                pass
 
         return differences
 
@@ -266,10 +273,13 @@ class MozillaZipContainer(ZipContainer):
         # This is gross: Monkeypatch zipfile._EndRecData to work with
         # Mozilla-optimized ZIPs
         _orig_EndRecData = zipfile._EndRecData
+        eocd_offset = None
 
         def _EndRecData(fh):
             endrec = _orig_EndRecData(fh)
             if endrec:
+                nonlocal eocd_offset
+                eocd_offset = endrec[zipfile._ECD_LOCATION]
                 endrec[zipfile._ECD_LOCATION] = (
                     endrec[zipfile._ECD_OFFSET] + endrec[zipfile._ECD_SIZE]
                 )
@@ -278,6 +288,17 @@ class MozillaZipContainer(ZipContainer):
         zipfile._EndRecData = _EndRecData
         result = super(MozillaZipContainer, self).open_archive()
         zipfile._EndRecData = _orig_EndRecData
+        # fix _end_offset after https://github.com/python/cpython/pull/110016
+        # added a check that fails because the central directory comes before
+        # the entries in these files
+        zinfos = sorted(
+            result.filelist,
+            key=lambda zinfo: zinfo.header_offset,
+            reverse=True,
+        )
+        if zinfos:
+            if hasattr(zinfos[0], "_end_offset"):
+                zinfos[0]._end_offset = eocd_offset
         return result
 
 

{diffoscope-262 → diffoscope-263}/diffoscope/external_tools.py

@@ -251,6 +251,7 @@ EXTERNAL_TOOLS = {
         "arch": "libxmlb",
         "guix": "libxmlb",
     },
+    "zipdetails": {"debian": "perl"},
     "zstd": {"debian": "zstd", "guix": "zstd"},
 }
 

{diffoscope-262 → diffoscope-263}/diffoscope.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: diffoscope
-Version: 262
+Version: 263
 Summary: in-depth comparison of files, archives, and directories
 Home-page: https://diffoscope.org/
 Author: Diffoscope developers

{diffoscope-262 → diffoscope-263}/diffoscope.egg-info/SOURCES.txt

@@ -296,6 +296,7 @@ tests/data/bug903401_1.deb
 tests/data/bug903401_2.deb
 tests/data/bug903565_1.deb
 tests/data/bug903565_2.deb
+tests/data/comment_zipdetails_expected_diff
 tests/data/comment_zipinfo_expected_diff
 tests/data/cpio_listing_expected_diff
 tests/data/deb_metadata_expected_diff
@@ -368,6 +369,7 @@ tests/data/iso9660_rockridge_expected_diff_cdrtools
 tests/data/javap_14_class_expected_diff
 tests/data/javap_class_expected_diff
 tests/data/javascript_expected_diff
+tests/data/jmod_zipdetails_expected_diff
 tests/data/jmod_zipinfo_expected_diff
 tests/data/jpeg_image_expected_diff
 tests/data/jpeg_image_meta_expected_diff
@@ -645,7 +647,9 @@ tests/data/uimage_expected_diff
 tests/data/uimage_expected_diff_pre_5_41
 tests/data/wasm_expected_diff
 tests/data/xsb_expected_diff
+tests/data/zip2_zipdetails_expected_diff
 tests/data/zip_bsdtar_expected_diff
+tests/data/zip_zipdetails_expected_diff
 tests/data/zip_zipinfo_expected_diff
 tests/data/containers/a.tar.bz2
 tests/data/containers/a.tar.gz

{diffoscope-262 → diffoscope-263}/tests/comparators/test_epub.py

@@ -2,7 +2,7 @@
 # diffoscope: in-depth comparison of files, archives, and directories
 #
 # Copyright © 2015 Jérémy Bobbio <lunar@debian.org>
-# Copyright © 2016-2017, 2020-2023 Chris Lamb <lamby@debian.org>
+# Copyright © 2016-2017, 2020-2024 Chris Lamb <lamby@debian.org>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -45,16 +45,18 @@ def differences(epub1, epub2):
     return epub1.compare(epub2).details
 
 
-@skip_unless_tools_exist("zipinfo")
+@skip_unless_tools_exist("zipinfo", "zipdetails")
 def test_differences(differences):
     assert differences[0].source1 == "zipinfo {}"
     assert differences[0].source2 == "zipinfo {}"
-    assert differences[1].source1 == "content.opf"
-    assert differences[1].source2 == "content.opf"
-    assert differences[2].source1 == "toc.ncx"
-    assert differences[2].source2 == "toc.ncx"
-    assert differences[3].source1 == "ch001.xhtml"
-    assert differences[3].source2 == "ch001.xhtml"
+    assert differences[1].source1.startswith("zipdetails ")
+    assert differences[1].source2.startswith("zipdetails ")
+    assert differences[2].source1 == "content.opf"
+    assert differences[2].source2 == "content.opf"
+    assert differences[3].source1 == "toc.ncx"
+    assert differences[3].source2 == "toc.ncx"
+    assert differences[4].source1 == "ch001.xhtml"
+    assert differences[4].source2 == "ch001.xhtml"
 
     # Flatten everything recursively, as XMLFile will contain reformatted data
     # under Difference.details.

{diffoscope-262 → diffoscope-263}/tests/comparators/test_zip.py

@@ -18,12 +18,11 @@
 # along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
 
 import pytest
-import sys
 
 from diffoscope.comparators.zip import ZipFile, MozillaZipFile, JmodJavaModule
 
 from ..utils.data import load_fixture, assert_diff
-from ..utils.tools import skip_unless_tools_exist, skipif
+from ..utils.tools import skip_unless_tools_exist
 from ..utils.nonexisting import assert_non_existing
 
 
@@ -39,12 +38,6 @@ jmod2 = load_fixture("test2.jmod")
 test_comment1 = load_fixture("test_comment1.zip")
 test_comment2 = load_fixture("test_comment2.zip")
 
-# See #362
-skip_if_python_zip_regression = pytest.mark.skipif(
-    sys.version_info >= (3, 10, 14),
-    reason="Test fails under 3.10.14+ (and 3.11.8+); possible regression",
-)
-
 
 def test_identification(zip1):
     assert isinstance(zip1, ZipFile)
@@ -65,21 +58,23 @@ def differences2(zip1, zip3):
     return zip1.compare(zip3).details
 
 
-@skip_unless_tools_exist("zipinfo")
+@skip_unless_tools_exist("zipinfo", "zipdetails")
 def test_metadata(differences):
     assert_diff(differences[0], "zip_zipinfo_expected_diff")
+    assert_diff(differences[1], "zip_zipdetails_expected_diff")
 
 
-@skip_unless_tools_exist("zipinfo")
+@skip_unless_tools_exist("zipinfo", "zipdetails")
 def test_compressed_files(differences):
-    assert differences[1].source1 == "dir/text"
-    assert differences[1].source2 == "dir/text"
-    assert_diff(differences[1], "text_ascii_expected_diff")
+    assert differences[2].source1 == "dir/text"
+    assert differences[2].source2 == "dir/text"
+    assert_diff(differences[2], "text_ascii_expected_diff")
 
 
-@skip_unless_tools_exist("zipinfo", "bsdtar")
+@skip_unless_tools_exist("zipinfo", "bsdtar", "zipdetails")
 def test_extra_fields(differences2):
     assert_diff(differences2[0], "zip_bsdtar_expected_diff")
+    assert_diff(differences2[1], "zip2_zipdetails_expected_diff")
 
 
 @skip_unless_tools_exist("zipinfo")
@@ -91,7 +86,6 @@ def test_mozzip_identification(mozzip1):
     assert isinstance(mozzip1, MozillaZipFile)
 
 
-@skip_if_python_zip_regression
 def test_mozzip_no_differences(mozzip1):
     difference = mozzip1.compare(mozzip1)
     assert difference is None
@@ -103,13 +97,11 @@ def mozzip_differences(mozzip1, mozzip2):
 
 
 @skip_unless_tools_exist("zipinfo")
-@skip_if_python_zip_regression
 def test_mozzip_metadata(mozzip_differences, mozzip1, mozzip2):
     assert_diff(mozzip_differences[0], "mozzip_zipinfo_expected_diff")
 
 
 @skip_unless_tools_exist("zipinfo")
-@skip_if_python_zip_regression
 def test_mozzip_compressed_files(mozzip_differences):
     assert mozzip_differences[-1].source1 == "dir/text"
     assert mozzip_differences[-1].source2 == "dir/text"
@@ -117,7 +109,6 @@ def test_mozzip_compressed_files(mozzip_differences):
 
 
 @skip_unless_tools_exist("zipinfo")
-@skip_if_python_zip_regression
 def test_mozzip_compare_non_existing(monkeypatch, mozzip1):
     assert_non_existing(monkeypatch, mozzip1)
 
@@ -136,9 +127,10 @@ def jmod_differences(jmod1, jmod2):
     return jmod1.compare(jmod2).details
 
 
-@skip_unless_tools_exist("zipinfo")
+@skip_unless_tools_exist("zipinfo", "zipdetails")
 def test_jmod_metadata(jmod_differences, jmod1, jmod2):
     assert_diff(jmod_differences[0], "jmod_zipinfo_expected_diff")
+    assert_diff(jmod_differences[1], "jmod_zipdetails_expected_diff")
 
 
 def test_encrypted(encrypted_zip1, encrypted_zip2):
@@ -151,6 +143,7 @@ def comment_differences(test_comment1, test_comment2):
     return test_comment1.compare(test_comment2).details
 
 
-@skip_unless_tools_exist("zipnote")
+@skip_unless_tools_exist("zipnote", "zipdetails")
 def test_commented(comment_differences):
     assert_diff(comment_differences[1], "comment_zipinfo_expected_diff")
+    assert_diff(comment_differences[2], "comment_zipdetails_expected_diff")

diffoscope-263/tests/data/comment_zipdetails_expected_diff

@@ -0,0 +1,45 @@
+@@ -33,15 +33,15 @@
+ 004B Compression Method    0000 'Stored'
+ 004D Last Mod Time         4D869485 'Thu Dec  6 18:36:10 2018'
+ 0051 CRC                   00000000
+ 0055 Compressed Length     00000000
+ 0059 Uncompressed Length   00000000
+ 005D Filename Length       0003
+ 005F Extra Length          0018
+-0061 Comment Length        0000
++0061 Comment Length        0005
+ 0063 Disk Start            0000
+ 0065 Int File Attributes   0000
+      [Bit 0]               0 'Binary Data'
+ 0067 Ext File Attributes   81A40000
+ 006B Local Header Offset   00000000
+ 006F Filename              'XXX'
+ 0072 Extra ID #0001        5455 'UT: Extended Timestamp'
+@@ -51,17 +51,19 @@
+ 007B Extra ID #0002        7875 'ux: Unix Extra Type 3'
+ 007D   Length              000B
+ 007F   Version             01
+ 0080   UID Size            04
+ 0081   UID                 000003E8
+ 0085   GID Size            04
+ 0086   GID                 000003E8
++008A Comment               'hello'
+ 
+-0086 END CENTRAL HEADER    06054B50
+-008A Number of this disk   0000
+-008C Central Dir Disk no   0000
+-008E Entries in this disk  0001
+-0090 Total Entries         0001
+-0092 Size of Central Dir   00000049
+-0096 Offset to Central Dir 0000003D
+-009A Comment Length        0000
++008B END CENTRAL HEADER    06054B50
++008F Number of this disk   0000
++0091 Central Dir Disk no   0000
++0093 Entries in this disk  0001
++0095 Total Entries         0001
++0097 Size of Central Dir   0000004E
++009B Offset to Central Dir 0000003D
++009F Comment Length        0007
++00A1 Comment               'goodbye'
+ Done