duplicity 2.2.2__tar.gz → 2.2.3__tar.gz

{duplicity-2.2.2 → duplicity-2.2.3}/CHANGELOG.md

@@ -1,6 +1,89 @@
 # Changelog
 
 
+## rel.2.2.3 (2024-03-20)
+
+### New
+
+* Add test case issue725.sh. [Kenneth Loafman]
+
+### Changes
+
+* Remove setuptools\_scm and usages. [Kenneth Loafman]
+
+* Run po/update-pot. [Kenneth Loafman]
+
+* !Minor changes to pip build and test. [Kenneth Loafman]
+
+* Pip install duplicity fails with "AssertionError: es\_PR" (or other
+language) [Kenneth Loafman]
+
+* Launchpad PPAs not built correctly for 2.2.[0,1], PyPI pips for
+2.2.[0,1,2] [Kenneth Loafman]
+
+    - tested on:
+      - Ubuntu 20.04, 22.04, 24.04
+      - Python 3.8, 3.9, 3.10, 3.11, 3.12
+    - added tools/testpip
+    - added wheel builds to .gitlab-ci.yml
+      - build wheels for manylinux
+    - added tools/testpip
+    - remove disutils includes in setup.py
+    - remove old test command in setup.py
+    - moved use of setuptools-scm to scmversion command
+    - add tools to build pyenv & pythons & more.
+      - installpyenv - install pyenv and environ
+      - installpythons - install pythons and environ
+      - pushpip - push pip sdist and wheels
+     - tools/list_python_files - removed, unused
+
+    Fixes #797
+
+* Change ngettext to \_ and fix wording. [Kenneth Loafman]
+
+* Move validate to backend. [Thomas Laubrock]
+
+* Run only one pipeline on merge request. [Kenneth Loafman]
+
+* Small fixes to the test system. [Kenneth Loafman]
+
+    - run test cases with the same Python as pytest
+    - fix some comments that pointed to wrong config file
+    - code tests now run without RUN_CODE_TESTS=1
+
+* Change from --capture=no to --capture=fd. [Kenneth Loafman]
+
+    - Should only print on test failure.
+
+* Warn on symmetric encrypt with --use-agent. [Kenneth Loafman]
+
+    Fixes #799
+
+* Warn on symmetric encrypt with --use-agent. [Kenneth Loafman]
+
+    Fixes #799
+
+* Remove last of bin/duplicity. [Kenneth Loafman]
+
+### Fix
+
+* Add fail open condition, for backends with no support. [Thomas Laubrock]
+
+* Update homepage. [sblondon]
+
+* Change path logging from Info to Debug. [Kenneth Loafman]
+
+    Fixes #804
+
+* Whoops! Remove parens in fix of #803. [Kenneth Loafman]
+
+* Exception using --exclude-older-than. [Kenneth Loafman]
+
+    fixes #803
+
+* Manpage remove superfluous indentions, some formatting fixes. [ede]
+
+
 ## rel.2.2.2 (2024-02-03)
 
 ### Changes

{duplicity-2.2.2/duplicity.egg-info → duplicity-2.2.3}/PKG-INFO

@@ -1,7 +1,5 @@
 Metadata-Version: 2.1
 Name: duplicity
-Version: 2.2.2
-Home-page: http://duplicity.us
-Platform: any
+Version: 2.2.3
 License-File: COPYING
 License-File: AUTHORS.md

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/__init__.py

@@ -21,7 +21,7 @@
 
 import gettext
 
-__version__: str = "2.2.2"
-__reldate__: str = "February 03, 2024"
+__version__: str = "2.2.3"
+__reldate__: str = "March 20, 2024"
 
 gettext.install("duplicity", names=["ngettext"])

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/backend.py

@@ -24,13 +24,14 @@ Provides a common interface to all backends and certain sevices
 intended to be used by the backends themselves.
 """
 
+from datetime import datetime
 import errno
 import getpass
 import os
 import re
-import socket
 import sys
 import time
+from typing import Tuple
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -604,6 +605,42 @@ class BackendWrapper(object):
         else:
             raise NotImplementedError()
 
+    def validate(self, remote_filename, expected_size, source_path=None) -> Tuple[bool, str]:
+        """
+        validates a file transferred to the backend by comparing the size.
+        source_path is optional to allow the backend do further validation, e.g. by
+        calulating a hash or so.
+        Returns: return a tuple of (result, reason)
+            - result: bool: True if file validation successful otherwise False
+            - reason: str: If False a reason why validation failed; True: emtpy str or description
+        """
+        msg = "Validation fail for unknow reaons."
+        if hasattr(self.backend, "_validate"):
+            return self.backend._validate(remote_filename, expected_size, source_path=source_path)
+        else:
+            # as some backends take some time refresh file attributes after upload
+            # retry config.num_retries times.
+            for attempt in range(0, config.num_retries + 1):
+                info = self.query_info([remote_filename])[remote_filename]
+                size = info["size"]
+                if size is None:
+                    log.Warn(
+                        "File size can't be validated, because of missing capabilities of the backend. "
+                        "Please verify the backup separately."
+                    )
+                    return (True, "Backend has no capabilities to check filesize, skip validation.")
+                if size == expected_size:
+                    return (True, "Validation OK, file size matches.")
+                msg = _("%s Remote filesize %d for %s does not match local size %d") % (
+                    datetime.now(),
+                    size,
+                    util.escape(remote_filename),
+                    expected_size,
+                )
+                log.Notice(f"{msg}, retrying.")
+                time.sleep(2**attempt)
+        return (False, f"{msg}.")
+
     def pre_process_download(self, remote_filename):
         """
         Manages remote access before downloading files

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/backends/README

@@ -19,6 +19,16 @@ _delete
 
 There are other methods you may optionally implement:
 
+_validate
+- minimum, compare the file size with the given value
+- has two pos args: remote_filename and size in bytes
+- accept **kwargs, which will held following arguments:
+   - source_path as path.Path object pointing to the source file in the local fs.
+   - (future arguments, that should be ignored.)
+- must handle retries on its own if e.g. size need some time to be refreshed after upload.
+- return a tuple of (bool, str)
+    - bool: True if file validation successful otherwise False
+    - str: If False a description why validation failed; True: emtpy str or description
 _delete_list
  - Delete list of files
  - This is used in preference of _delete if defined

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/backends/_testbackend.py

@@ -20,6 +20,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
 import glob
+import hashlib
 import inspect
 import json
 import logging
@@ -39,12 +40,14 @@ ERROR_ON = {}
 
 
 class BackendErrors:
-    FAIL_WITH_EXCEPTION = "DUP_FAIL_WITH_EXCEPTION"  # set to substring mached by the filename.
-    WAIT_FOR_OTHER_VOLUME = (
-        "DUP_FAIL_WAIT_FOR_VOLUME"  # set to json string: ["file_to_delay", "file_wait_for"] (substing match)
-    )
-    LAST_BYTE_MISSING = "DUP_FAIL_LAST_BYTE_MISSING"  # set to substring mached by the filename.
-    SKIP_PUT_SILENT = "DUP_FAIL_SKIP_PUT_SILENT"  # Don't put file contains string
+    # FAIL_WITH_EXCEPTION: set to substring mached by the filename.
+    FAIL_WITH_EXCEPTION = "DUP_FAIL_WITH_EXCEPTION"
+    # WAIT_FOR_OTHER_VOLUME: set to json string: ["file_to_delay", "file_wait_for"] (substing match)
+    WAIT_FOR_OTHER_VOLUME = "DUP_FAIL_WAIT_FOR_VOLUME"
+    # LAST_BYTE_MISSING: set to substring mached by the filename.
+    LAST_BYTE_MISSING = "DUP_FAIL_LAST_BYTE_MISSING"
+    # SKIP_PUT_SILENT: Don't put file if name contains string
+    SKIP_PUT_SILENT = "DUP_FAIL_SKIP_PUT_SILENT"
 
 
 class _TestBackend(duplicity.backend.Backend):
@@ -145,6 +148,43 @@ class _TestBackend(duplicity.backend.Backend):
     def _list(self):
         return self.remote_pathdir.listdir()
 
+    def _validate(self, remote_filename, size, source_path=None, **kwargs):
+        # poc to show that validate can do additional things.
+
+        self._remove_last_byte(remote_filename)
+
+        results_str = []
+        results_bool = []
+        try:
+            if source_path:
+                target_path = self.remote_pathdir.append(remote_filename)
+                target_hash = self.__hash_fileobj(target_path.open())
+                source_hash = self.__hash_fileobj(source_path.open())
+                if target_hash == source_hash:
+                    results_str.append(f"file hash {target_hash} matches")
+                    results_bool.append(True)
+                else:
+                    results_str.append(f"expected hash {source_hash} doesn't match file hash {target_hash}")
+                    results_bool.append(False)
+            if size == self._query(remote_filename)["size"]:
+                results_str.append(f"file size {size}")
+                results_bool.append(True)
+            else:
+                results_str.append(
+                    f'expected size {size} and file size {self._query(remote_filename)["size"]} don\'t match'
+                )
+                results_bool.append(False)
+        except FileNotFoundError as e:
+            results_bool.append(False)
+            results_str.append(f"FileNotFoundError: {e}")
+        except Exception as e:
+            log.FatalError(
+                _("Unexpected exception while validate %s.") % os.fsdecode(remote_filename),
+                log.ErrorCode.backend_verification_failed,
+                extra=f"Exception: {e}",
+            )
+        return (all(results_bool), ", ".join(results_str))
+
     def _delete(self, filename):
         self._fail_with_exception(filename)
         self.remote_pathdir.append(filename).delete()
@@ -161,5 +201,33 @@ class _TestBackend(duplicity.backend.Backend):
         size = target_file.getsize() if target_file.exists() else -1
         return {"size": size}
 
+    def _query_list(self, filename_list):
+        return {x: self._query(x) for x in filename_list}
+
+    def _retry_cleanup(self):
+        pass
+
+    def _close(self):
+        pass
+
+    def _error_code(self, operation, e):  # pylint: disable=unused-argument
+        return log.ErrorCode.backend_error
+
+    def __hash_file(self, filename):
+        self.__hash_fileobj(open(filename, "rb"))
+
+    def __hash_fileobj(self, fileobj):
+        h = hashlib.sha1()
+
+        # loop till the end of the file
+        chunk = 0
+        while chunk != b"":
+            # read only 1024 bytes at a time
+            chunk = fileobj.read(1024)
+            h.update(chunk)
+        fileobj.close()
+        # return the hex representation of digest
+        return h.hexdigest()
+
 
 duplicity.backend.register_backend("fortestsonly", _TestBackend)

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/backends/b2backend.py

@@ -217,9 +217,11 @@ class B2Backend(duplicity.backend.Backend):
         log.Log(f"Query: {self.path}{os.fsdecode(filename)}", log.INFO)
         file_version_info = self.file_info(quote_plus(self.path + os.fsdecode(filename), "/"))
         return {
-            "size": int(file_version_info.size)
-            if file_version_info is not None and file_version_info.size is not None
-            else -1
+            "size": (
+                int(file_version_info.size)
+                if file_version_info is not None and file_version_info.size is not None
+                else -1
+            )
         }
 
     def file_info(self, filename):

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/cli_data.py

@@ -224,6 +224,7 @@ OptionKwargs = dict(
     ),
     exclude_older_than=dict(
         metavar=_("time"),
+        type=check_time,
         action=AddSelectionAction,
         help="Exclude files older than time",
         default=dflt(None),

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/cli_main.py

@@ -298,6 +298,19 @@ def process_command_line(cmdline_list):
     gpg_version = ".".join(map(str, config.gpg_profile.gpg_version))
     log.Info(_(f"GPG binary is {config.gpg_binary}, version {gpg_version}"))
 
+    # --use-agent is not valid for symmetric encryption.  Turn it off and notify user.
+    if (
+        config.use_agent
+        and config.gpg_profile.sign_key is None
+        and len(config.gpg_profile.recipients) == 0
+        and len(config.gpg_profile.hidden_recipients) == 0
+    ):
+        config.use_agent = False
+        log.Warn(
+            "Option --gpg-agent is unsafe with symmetric encryption.  Ignoring --gpg-agent.\n"
+            "Refer to https://gitlab.com/duplicity/duplicity/-/issues/799 for more information."
+        )
+
     # shorten incremental to inc, replace backup with inc
     if config.action in ("incremental", "backup"):
         if config.action == "backup":

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/dup_collections.py

@@ -767,14 +767,7 @@ class CollectionsStatus(object):
 
         # get local filename list
         local_filename_list = self.archive_dir_path.listdir()
-        log.Debug(
-            ngettext(
-                "%d file exists in cache",
-                "%d files exist in cache",
-                len(local_filename_list),
-            )
-            % len(local_filename_list)
-        )
+        log.Debug(_("%d file(s) exist in cache") % len(local_filename_list))
 
         # check for partial backups
         partials = []

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/dup_main.py

@@ -314,31 +314,6 @@ def write_multivol(backup_type, tarblock_iter, man_outfp, sig_outfp, backend):
             end_block -= 1
         return start_index, start_block, end_index, end_block
 
-    def validate_block(orig_size, dest_filename):
-        info = backend.query_info([dest_filename])[dest_filename]
-        size = info["size"]
-        if size is None:
-            return  # error querying file
-        for attempt in range(1, config.num_retries + 1):
-            info = backend.query_info([dest_filename])[dest_filename]
-            size = info["size"]
-            if size == orig_size:
-                break
-            if size is None:
-                return
-            log.Notice(
-                _("%s Remote filesize %d for %s does not match local size %d, retrying.")
-                % (datetime.now(), size, util.escape(dest_filename), orig_size)
-            )
-            time.sleep(2**attempt)
-        if size != orig_size:
-            code_extra = f"{util.escape(dest_filename)} {int(orig_size)} {int(size)}"
-            log.FatalError(
-                _("File %s was corrupted during upload.") % os.fsdecode(dest_filename),
-                log.ErrorCode.volume_wrong_size,
-                code_extra,
-            )
-
     def put(tdp, dest_filename, vol_num):
         """
         Retrieve file size *before* calling backend.put(), which may (at least
@@ -348,7 +323,15 @@ def write_multivol(backup_type, tarblock_iter, man_outfp, sig_outfp, backend):
         putsize = tdp.getsize()
         if config.skip_volume != vol_num:  # for testing purposes only
             backend.put(tdp, dest_filename)
-        validate_block(putsize, dest_filename)
+        res, msg = backend.validate(dest_filename, putsize, source_path=tdp)
+        if not res:
+            code_extra = f"{util.escape(dest_filename)}: {msg}"
+            log.FatalError(
+                _("File %s was corrupted during upload.") % os.fsdecode(dest_filename),
+                log.ErrorCode.backend_verification_failed,
+                code_extra,
+            )
+
         if tdp.stat:
             tdp.delete()
         return putsize

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/log.py

@@ -346,6 +346,7 @@ class ErrorCode(object):
     backend_no_space = 53
     backend_command_error = 54
     backend_code_error = 55
+    backend_verification_failed = 56
 
     # file selection filter mode errors
     redundant_filter = 70

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity/path.py

@@ -602,7 +602,7 @@ class Path(ROPath):
 
     def mkdir(self):
         """Make directory(s) at specified path"""
-        log.Info(_("Making directory %s") % self.uc_name)
+        log.Debug(_("Making directory %s") % self.uc_name)
         try:
             os.makedirs(self.name)
         except OSError:
@@ -612,7 +612,7 @@ class Path(ROPath):
 
     def delete(self):
         """Remove this file"""
-        log.Info(_("Deleting %s") % self.uc_name)
+        log.Debug(_("Deleting %s") % self.uc_name)
         if self.isdir():
             util.ignore_missing(os.rmdir, self.name)
         else:
@@ -621,7 +621,7 @@ class Path(ROPath):
 
     def touch(self):
         """Open the file, write 0 bytes, close"""
-        log.Info(_("Touching %s") % self.uc_name)
+        log.Debug(_("Touching %s") % self.uc_name)
         fp = self.open("wb")
         fp.close()
 
@@ -629,7 +629,7 @@ class Path(ROPath):
         """Remove self by recursively deleting files under it"""
         from duplicity import selection  # TODO: avoid circ. dep. issue
 
-        log.Info(_("Deleting tree %s") % self.uc_name)
+        log.Debug(_("Deleting tree %s") % self.uc_name)
         itr = IterTreeReducer(PathDeleter, [])
         for path in selection.Select(self).set_iter():
             itr(path.index, path)

{duplicity-2.2.2 → duplicity-2.2.3/duplicity.egg-info}/PKG-INFO

@@ -1,7 +1,5 @@
 Metadata-Version: 2.1
 Name: duplicity
-Version: 2.2.2
-Home-page: http://duplicity.us
-Platform: any
+Version: 2.2.3
 License-File: COPYING
 License-File: AUTHORS.md

{duplicity-2.2.2 → duplicity-2.2.3}/duplicity.egg-info/SOURCES.txt

@@ -280,6 +280,7 @@ testing/regression/issue25.json
 testing/regression/issue25.sh
 testing/regression/issue26.sh
 testing/regression/issue683.sh
+testing/regression/issue725.sh
 testing/regression/issue73.env
 testing/regression/issue73.sh
 testing/regression/issue781.json
@@ -308,11 +309,14 @@ testing/unit/test_statistics.py
 testing/unit/test_tarfile.py
 testing/unit/test_tempdir.py
 testing/unit/test_util.py
+tools/installpyenv
+tools/installpythons
 tools/installsnap
-tools/list_python_files
 tools/makechangelog
 tools/makepip
 tools/makesnap
+tools/pushpip
 tools/pushsnap
 tools/release-prep
+tools/testpip
 tools/testsnap