muninn 6.2__tar.gz → 6.3__tar.gz

{muninn-6.2 → muninn-6.3}/CHANGES

@@ -1,3 +1,23 @@
+6.3 2023-12-13
+~~~~~~~~~~~~~~
+
+* Added support for extension-specific configuration sections in archive
+  configuration files.
+
+* Introduced `timeout` and `retries` options for the
+  `extension:muninn.remote` section in an archive configuration.
+  The number of retries now defaults back to 0.
+
+* Added option to muninn-info to show all archives if no archive parameter
+  is provided.
+
+* Added muninn.list_archives() function.
+
+* Improved support for using muninn on Windows (e.g. '\' in paths are now
+  translated to '/' when stored as muninn metadata)
+
+* Fixed issue where S3 bucket prefix object was created with one '/' too many.
+
 6.2 2023-09-20
 ~~~~~~~~~~~~~~
 

{muninn-6.2 → muninn-6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: muninn
-Version: 6.2
+Version: 6.3
 Summary: Data product catalogue and archive system
 Home-page: https://github.com/stcorp/muninn
 Author: S[&]T

{muninn-6.2 → muninn-6.3}/muninn/__init__.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function
 
-__version__ = "6.2"
+__version__ = "6.3"
 __copyright__ = "Copyright (C) 2014-2023 S[&]T, The Netherlands."
 
 __all__ = ["Error", "InternalError", "Struct", "Archive", "open", "config_path"]
@@ -43,6 +43,19 @@ def open(id=None, **kwargs):
 
     return Archive.create(configuration, id)
 
+def list_archives():
+    """Search locations found in the 'MUNINN_CONFIG_PATH' environment variable
+    and return names of all found archives."""
+
+    result = []
+    for path in filter(None, config_path().split(":")):
+        if _os.path.isfile(path):
+            result.append(_os.path.basename(path)[:-4])
+        else:
+            result.extend([f[:-4] for f in _os.listdir(path) if f.endswith('.cfg')])
+
+    return result
+
 
 def _read_archive_config_file(path):
     try:

{muninn-6.2 → muninn-6.3}/muninn/archive.py

@@ -100,6 +100,34 @@ def _inspect_nargs(func):
     return len(getargspec(func).args)
 
 
+def _register_extension(archive, extension, name, plugin_type, list_name=None, get_name=None):
+    if list_name is None:
+        list_name = plugin_type + 's'
+    if get_name is None:
+        get_name = plugin_type
+
+    try:
+        list_function = getattr(extension, list_name)
+        get_function = getattr(extension, get_name)
+    except AttributeError:
+        raise Error("extension %r does not implement the namespace extension API" % name)
+
+    config_section = archive._configuration.get('extension:' + extension.__name__)
+
+    list_args = []
+    if _inspect_nargs(list_function) == 1:
+        list_args.append(config_section)
+
+    for plugin_name in list_function(*list_args):
+
+        get_args = [plugin_name]
+        if _inspect_nargs(get_function) == 2:
+            get_args.append(config_section)
+
+        plugin = get_function(*get_args)
+        getattr(archive, 'register_' + plugin_type)(plugin_name, plugin)
+
+
 _CORE_PROP_NAMES = [
     'uuid',
     'active',
@@ -151,58 +179,43 @@ class Archive(object):
         product_type_extensions = options.pop("product_type_extensions", [])
         remote_backend_extensions = options.pop("remote_backend_extensions", [])
         hook_extensions = options.pop("hook_extensions", [])
-        archive = Archive(database=database, storage=storage, id=id, **options)
+        archive = Archive(database=database, storage=storage, id=id,
+                          configuration=configuration, **options)
 
         # Register core namespace.
         archive.register_namespace("core", Core)
 
-        # Register custom namespaces.
+        # Register custom extensions.
         for name in namespace_extensions:
             extension = _load_extension(name)
-            try:
-                for namespace in extension.namespaces():
-                    archive.register_namespace(namespace, extension.namespace(namespace))
-            except AttributeError:
-                raise Error("extension %r does not implement the namespace extension API" % name)
+            _register_extension(archive, extension, name, 'namespace')
 
-        # Register product types.
         for name in product_type_extensions:
             extension = _load_extension(name)
-            try:
-                for product_type in extension.product_types():
-                    archive.register_product_type(product_type, extension.product_type_plugin(product_type))
-            except AttributeError:
-                raise Error("extension %r does not implement the product type extension API" % name)
+            _register_extension(archive, extension, name, 'product_type', get_name='product_type_plugin')
 
-        # Register custom remote backends.
         for name in remote_backend_extensions:
             extension = _load_extension(name)
-            try:
-                for remote_backend in extension.remote_backends():
-                    archive.register_remote_backend(remote_backend, extension.remote_backend(remote_backend))
-            except AttributeError:
-                raise Error("extension %r does not implement the remote backend extension API" % name)
+            _register_extension(archive, extension, name, 'remote_backend')
 
-        # Register hook extensions.
         for name in hook_extensions:
             extension = _load_extension(name)
-            try:
-                for hook_extension in extension.hook_extensions():
-                    archive.register_hook_extension(hook_extension, extension.hook_extension(hook_extension))
-            except AttributeError:
-                raise Error("extension %r does not implement the hook extension API" % name)
+            _register_extension(archive, extension, name, 'hook_extension')
 
         return archive
 
     def __init__(self, database, storage, cascade_grace_period=0,
-                 max_cascade_cycles=25, auth_file=None, id=None, tempdir=None):
+                 max_cascade_cycles=25, auth_file=None, id=None, tempdir=None,
+                 configuration=None):
+        self._configuration = configuration
+
         self._cascade_grace_period = datetime.timedelta(minutes=cascade_grace_period)
         self._max_cascade_cycles = max_cascade_cycles
         self._auth_file = auth_file
 
         self._namespace_schemas = {}
         self._product_type_plugins = {}
-        self._remote_backend_plugins = copy.copy(remote.REMOTE_BACKENDS)
+        self._remote_backend_plugins = {}
         self._hook_extensions = collections.OrderedDict()
 
         self._export_formats = set()
@@ -214,6 +227,8 @@ class Archive(object):
         self._tempdir = tempdir
         self.id = id  # Archive id (usually name of configuration file)
 
+        _register_extension(self, remote, 'muninn.remote', 'remote_backend')
+
     def __enter__(self):
         return self
 
@@ -311,6 +326,10 @@ class Archive(object):
         if remote_backend in self._remote_backend_plugins:
             raise Error("redefinition of remote backend: \"%s\"" % remote_backend)
 
+        for method in ['identify', 'pull']:
+            if not hasattr(plugin, method):
+                raise Error("missing '%s' method in plugin for remote backend \"%s\"" % (method, remote_backend))
+
         self._remote_backend_plugins[remote_backend] = plugin
 
     def remote_backend(self, remote_backend):
@@ -394,7 +413,7 @@ class Archive(object):
         if len(set(basenames)) < len(basenames):
             raise Error("basename of each part should be unique for multi-part products")
 
-        return paths
+        return [path.replace('\\', '/') for path in paths]
 
     def _extract_hash_type(self, hash_value):
         prefix, middle, _ = hash_value.partition(':')
@@ -1097,7 +1116,7 @@ class Archive(object):
                         ingest_path = os.path.dirname(ingest_path)
                     current_path = self.root()
                     if existing.core.archive_path:
-                        current_path = os.path.join(current_path, existing.core.archive_path)
+                        current_path = util.fwd_join(current_path, existing.core.archive_path)
                     if existing.core.archive_path != properties.core.archive_path:
                         raise Error('cannot force ingest because of archive_path mismatch')
                     if ingest_path == current_path:
@@ -1132,6 +1151,7 @@ class Archive(object):
                     properties.core.remote_url = 'file://' + os.path.realpath(paths[0])
                 else:
                     properties.core.remote_url = 'file://' + os.path.realpath(os.path.dirname(paths[0]))  # TODO test
+                properties.core.remote_url = properties.core.remote_url.replace('\\', '/')
                 self.update_properties(Struct({'core': {'remote_url': properties.core.remote_url}}),
                                        properties.core.uuid)
 
@@ -1533,7 +1553,7 @@ class Archive(object):
     def root(self):
         """Return the archive root path."""
         if self._storage is not None:
-            return self._storage._root
+            return self._storage._root.replace('\\', '/')
 
     def search(self, where="", order_by=[], limit=None, parameters={}, namespaces=[], property_names=[]):
         """Search the product catalogue for products matching the specified search expression.

muninn-6.3/muninn/database/base.py

@@ -0,0 +1,6 @@
+#
+# Copyright (C) 2014-2023 S[&]T, The Netherlands.
+#
+
+class DatabaseBackend(object):
+    pass

{muninn-6.2 → muninn-6.3}/muninn/database/postgresql.py

@@ -4,8 +4,6 @@
 
 from __future__ import absolute_import, division, print_function
 
-from muninn._compat import dictkeys, dictvalues, is_python2_unicode
-
 import re
 import functools
 import inspect
@@ -24,6 +22,8 @@ try:
 except ImportError:
     pass
 
+from .base import DatabaseBackend
+
 import muninn.config as config
 import muninn.database.sql as sql
 import muninn.database.ewkb as ewkb
@@ -34,6 +34,8 @@ from muninn.function import Prototype
 from muninn.schema import *
 from muninn.struct import Struct
 
+from muninn._compat import dictkeys, dictvalues, is_python2_unicode
+
 PG_UNIQUE_VIOLATION = '23505'
 
 
@@ -74,6 +76,8 @@ def geometry_send_hex(geometry):
 
 def _connect_pg8000(connection_string):
     kwargs = dict(arg.split('=') for arg in connection_string.split(' '))
+    if 'dbname' in kwargs:
+        kwargs['database'] = kwargs.pop('dbname')
     _connection = pg8000.connect(**kwargs)
 
     geography_oid = _get_db_type_id(_connection, "geography")
@@ -282,7 +286,7 @@ class PostgresqlConnection(object):
         return self._connection.encoding
 
 
-class PostgresqlBackend(object):
+class PostgresqlBackend(DatabaseBackend):
     def __init__(self, connection_string="", table_prefix="", library="psycopg2"):
         self._connection = PostgresqlConnection(connection_string, library)
         self._library = library
@@ -330,7 +334,7 @@ class PostgresqlBackend(object):
             result.append("ALTER TABLE %s ADD COLUMN uuid UUID PRIMARY KEY;" % self._table_name(namespace))
             result.append("ALTER TABLE %s ADD CONSTRAINT %s_uuid_fkey FOREIGN KEY (uuid) REFERENCES %s (uuid) ON "
                           "DELETE CASCADE;" % (self._table_name(namespace), self._table_name(namespace),
-                                              self._core_table_name))
+                                               self._core_table_name))
 
             schema = self._namespace_schema(namespace)
             for name in schema:

{muninn-6.2 → muninn-6.3}/muninn/database/sql.py

@@ -455,6 +455,7 @@ class SQLBuilder(object):
 
         join_set = set()
         result_fields = []
+
         for field in group_by + ['count'] + aggregates:
             ident = Identifier(field, self._namespace_schemas)
             result_fields.append(ident.resolve)

{muninn-6.2 → muninn-6.3}/muninn/database/sqlite.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, division, print_function
 
-from muninn._compat import dictkeys, dictvalues
 import os
 import re
 import datetime
@@ -12,6 +11,10 @@ import functools
 import json
 import uuid
 
+from .base import DatabaseBackend
+
+from muninn._compat import dictkeys, dictvalues
+
 # Select a version of dbapi2 that's available.
 # Since we need the spatialite module, we need a version of sqlite3 that supports extensions.
 # The defaulft version of sqlite3 that comes with Python does not support this, so we prefer pysqlite2

{muninn-6.2 → muninn-6.3}/muninn/remote.py

@@ -34,7 +34,7 @@ def get_credentials(archive, url):
     return None
 
 
-def download_http_oath2(url, target_dir, credentials, timeout=60):
+def download_http_oath2(url, target_dir, credentials, timeout, retries):
     import requests
     from requests_oauthlib import OAuth2Session
     from oauthlib.oauth2 import LegacyApplicationClient
@@ -46,7 +46,6 @@ def download_http_oath2(url, target_dir, credentials, timeout=60):
                         password=credentials['password'], client_id=credentials['client_id'],
                         client_secret=credentials['client_secret'])
     try:
-        retry = 1
         while True:
             try:
                 r = session.get(url, timeout=timeout, stream=True)
@@ -60,9 +59,9 @@ def download_http_oath2(url, target_dir, credentials, timeout=60):
                     for block in r.iter_content(1048576):  # use 1MB blocks
                         output.write(block)
             except requests.exceptions.ReadTimeout:
-                if retry <= 0:
+                if retries <= 0:
                     raise
-                retry -= 1
+                retries -= 1
             else:
                 break
     except Exception as e:
@@ -70,13 +69,12 @@ def download_http_oath2(url, target_dir, credentials, timeout=60):
     return local_file
 
 
-def download_http(url, target_dir, credentials=None, timeout=60):
+def download_http(url, target_dir, credentials, timeout, retries):
     import requests
     auth = None
     if credentials is not None:
         auth = (credentials['username'], credentials['password'])
     try:
-        retry = 1
         while True:
             try:
                 r = requests.get(url, timeout=timeout, stream=True, auth=auth)
@@ -90,9 +88,9 @@ def download_http(url, target_dir, credentials=None, timeout=60):
                     for block in r.iter_content(1048576):  # use 1MB blocks
                         output.write(block)
             except requests.exceptions.ReadTimeout:
-                if retry <= 0:
+                if retries <= 0:
                     raise
-                retry -= 1
+                retries -= 1
             else:
                 break
     except Exception as e:
@@ -100,7 +98,7 @@ def download_http(url, target_dir, credentials=None, timeout=60):
     return local_file
 
 
-def download_ftp(url, target_dir, credentials=None, timeout=60):
+def download_ftp(url, target_dir, credentials, timeout):
     url = urlparse(url)
     if url.username:
         username = url.username
@@ -176,7 +174,7 @@ def download_s3(url, target_dir, credentials=None):
     return paths
 
 
-def download_sftp(url, target_dir, credentials=None, timeout=60):
+def download_sftp(url, target_dir, credentials, timeout):
     import paramiko
     logging.getLogger("paramiko").setLevel(logging.WARNING)
     url = urlparse(url)
@@ -203,10 +201,15 @@ def download_sftp(url, target_dir, credentials=None, timeout=60):
 
 
 class RemoteBackend(object):
-    def __init__(self, prefix):
+    def __init__(self, prefix, config=None):
         self.prefix = prefix
+        self.timeout = 60
+        self.retries = 0
+        if config is not None:
+            self.timeout = int(config.get('timeout', self.timeout))
+            self.retries = int(config.get('retries', self.retries))
 
-    def indentify(self, url):
+    def identify(self, url):
         result = False
         if self.prefix:
             result = url.startswith(self.prefix)
@@ -260,16 +263,17 @@ class HTTPBackend(RemoteBackend):
     def pull(self, archive, product, target_dir):
         credentials = get_credentials(archive, product.core.remote_url)
         if credentials and 'auth_type' in credentials and credentials['auth_type'] == "oauth2":
-            file_path = download_http_oath2(product.core.remote_url, target_dir, credentials)
+            file_path = download_http_oath2(product.core.remote_url, target_dir, credentials, self.timeout,
+                                            self.retries)
         else:
-            file_path = download_http(product.core.remote_url, target_dir, credentials=credentials)
+            file_path = download_http(product.core.remote_url, target_dir, credentials, self.timeout, self.retries)
         return self.auto_extract(file_path, product)
 
 
 class FTPBackend(RemoteBackend):
     def pull(self, archive, product, target_dir):
         credentials = get_credentials(archive, product.core.remote_url)
-        file_path = download_ftp(product.core.remote_url, target_dir, credentials=credentials)
+        file_path = download_ftp(product.core.remote_url, target_dir, credentials, self.timeout)
         return self.auto_extract(file_path, product)
 
 
@@ -285,17 +289,17 @@ class S3Backend(RemoteBackend):
 class SFTPBackend(RemoteBackend):
     def pull(self, archive, product, target_dir):
         credentials = get_credentials(archive, product.core.remote_url)
-        file_path = download_sftp(product.core.remote_url, target_dir, credentials=credentials)
+        file_path = download_sftp(product.core.remote_url, target_dir, credentials, self.timeout)
         return self.auto_extract(file_path, product)
 
 
 REMOTE_BACKENDS = {
-    'http': HTTPBackend(prefix='http://'),
-    'https': HTTPBackend(prefix='https://'),
-    'file': FileBackend(prefix='file://'),
-    'ftp': FTPBackend(prefix='ftp://'),
-    's3': S3Backend(prefix='s3://'),
-    'sftp': SFTPBackend(prefix='sftp://'),
+    'http':  (HTTPBackend, 'http://'),
+    'https': (HTTPBackend, 'https://'),
+    'file':  (FileBackend, 'file://'),
+    'ftp':   (FTPBackend,  'ftp://'),
+    's3':    (S3Backend,   's3://'),
+    'sftp':  (SFTPBackend, 'sftp://'),
 }
 
 
@@ -308,7 +312,7 @@ def retrieve_function(archive, product, verify_hash_download):
     url = product.core.remote_url
     for prot in archive.remote_backends():
         _backend = archive.remote_backend(prot)
-        if _backend.indentify(url):
+        if _backend.identify(url):
             backend = _backend
     if backend is None:
         raise Error("The protocol of '%s' is not supported" % url)
@@ -331,3 +335,12 @@ def retrieve_function(archive, product, verify_hash_download):
         return paths
 
     return retrieve_files
+
+
+def remote_backends():
+    return list(REMOTE_BACKENDS)
+
+
+def remote_backend(name, configuration):
+    backend, prefix = REMOTE_BACKENDS[name]
+    return backend(prefix, configuration)

{muninn-6.2 → muninn-6.3}/muninn/storage/fs.py

@@ -152,7 +152,7 @@ class FilesystemStorageBackend(StorageBackend):
                         if use_enclosing_directory:
                             os.rename(tmp_path, abs_product_path)
                         else:
-                            assert(len(paths) == 1 and os.path.basename(paths[0]) == physical_name)
+                            assert len(paths) == 1 and os.path.basename(paths[0]) == physical_name
                             tmp_product_path = os.path.join(tmp_path, physical_name)
                             os.rename(tmp_product_path, abs_product_path)
                         anything_stored = True

{muninn-6.2 → muninn-6.3}/muninn/storage/s3.py

@@ -86,7 +86,7 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
                     endpoint_url += ':%d' % port
         elif port is not None:
             endpoint_url += ':%d' % port
-        self.global_prefix = os.path.join(endpoint_url, bucket, prefix)
+        self.global_prefix = util.fwd_join(endpoint_url, bucket, prefix)
 
         self._root = bucket
 
@@ -150,6 +150,7 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
         raise Error("S3 storage backend does not support ingesting already archived products")
 
     def _upload_file(self, key, path):
+        key = key.replace('\\', '/')
         obj = self._resource.Object(self.bucket, key)
         if os.path.getsize(path) == 0:  # upload_file can hang on empty files
             self._resource.Object(self.bucket, key).put()
@@ -158,7 +159,10 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
 
     def _create_dir(self, key):
         # using put, as upload_file/upload_fileobj do not like the trailing slash
-        self._resource.Object(self.bucket, key+'/').put()
+        key = key.replace('\\', '/')
+        if not key.endswith('/'):
+            key = key + '/'
+        self._resource.Object(self.bucket, key).put()
 
     def put(self, paths, properties, use_enclosing_directory, use_symlinks=None,
             retrieve_files=None, run_for_product=None):
@@ -173,7 +177,7 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
             physical_name = properties.core.physical_name
 
             if not use_enclosing_directory and retrieve_files is None:
-                assert(len(paths) == 1 and os.path.basename(paths[0]) == physical_name)
+                assert len(paths) == 1 and os.path.basename(paths[0]) == physical_name
 
             tmp_root = self.get_tmp_root(properties)
             with util.TemporaryDirectory(dir=tmp_root, prefix=".put-",
@@ -183,11 +187,11 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
 
                 # Upload file(s)
                 for path in paths:
-                    key = self._prefix + os.path.join(archive_path, physical_name)
+                    key = self._prefix + util.fwd_join(archive_path, physical_name)
 
                     # Add enclosing dir
                     if use_enclosing_directory:
-                        key = os.path.join(key, os.path.basename(path))
+                        key = util.fwd_join(key, os.path.basename(path))
 
                     if os.path.isdir(path):
                         self._create_dir(key)
@@ -203,7 +207,7 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
 
                             for filename in files:
                                 filekey = os.path.normpath(os.path.join(key, rel_root, filename))
-                                filepath = os.path.join(root, filename)
+                                filepath = util.fwd_join(root, filename)
                                 self._upload_file(filekey, filepath)
                                 anything_stored = True
 
@@ -218,13 +222,13 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
             raise StorageError(e, anything_stored)
 
     def get(self, product, target_path, use_enclosing_directory, use_symlinks=None):
-        paths = []
+        paths = set()
 
         if use_symlinks:
             raise Error("S3 storage backend does not support symlinks")
 
         archive_path = product.core.archive_path
-        product_path = os.path.join(archive_path, product.core.physical_name)
+        product_path = util.fwd_join(archive_path, product.core.physical_name)
         prefix = self._prefix + product_path
 
         objs = list(self._resource.Bucket(self.bucket).objects.filter(Prefix=prefix))
@@ -232,12 +236,11 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
             raise Error("no data for product '%s' (%s)" % (product.core.product_name, product.core.uuid))
 
         for obj in objs:
-            rel_path = os.path.relpath(obj.key, self._prefix + archive_path)
+            rel_path = os.path.relpath(obj.key, self._prefix + archive_path).replace('\\', '/')
             if use_enclosing_directory:
                 rel_path = '/'.join(rel_path.split('/')[1:])
+            paths.add(os.path.normpath(os.path.join(target_path, rel_path.split('/')[0])))
             target = os.path.normpath(os.path.join(target_path, rel_path))
-            if os.path.dirname(rel_path) == '':
-                paths.append(target)
 
             if obj.key.endswith('/'):
                 util.make_path(target)
@@ -248,16 +251,18 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
                 self._resource.Object(self.bucket, obj.key).download_file(target, ExtraArgs=self._download_args,
                                                                           Config=self._transfer_config)
 
-        return paths
+        return list(paths)
 
     def delete(self, product_path, properties):
         prefix = self._prefix + product_path
+        prefix = prefix.replace('\\', '/')
         for obj in self._resource.Bucket(self.bucket).objects.filter(Prefix=prefix):
             obj.delete()
 
     def size(self, product_path):
         total = 0
         prefix = self._prefix + product_path
+        prefix = prefix.replace('\\', '/')
         for obj in self._resource.Bucket(self.bucket).objects.filter(Prefix=prefix):
             total += obj.size
         return total
@@ -267,8 +272,8 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
         if product.core.archive_path == archive_path:
             return paths
 
-        product_path = os.path.join(self._prefix, product.core.archive_path, product.core.physical_name)
-        new_product_path = os.path.join(self._prefix, archive_path, product.core.physical_name)
+        product_path = util.fwd_join(self._prefix, product.core.archive_path, product.core.physical_name)
+        new_product_path = util.fwd_join(self._prefix, archive_path, product.core.physical_name)
 
         objs = list(self._resource.Bucket(self.bucket).objects.filter(Prefix=product_path))
         if not objs:
@@ -276,11 +281,13 @@ class S3StorageBackend(StorageBackend):  # TODO '/' in keys to indicate director
 
         for obj in objs:
             new_key = os.path.normpath(os.path.join(new_product_path, os.path.relpath(obj.key, product_path)))
+            new_key = new_key.replace('\\', '/')
             if obj.key.endswith('/'):
                 self._create_dir(new_key)
             else:
                 self._resource.Object(self.bucket, new_key).copy(CopySource={'Bucket': self.bucket, 'Key': obj.key},
-                                                             ExtraArgs=self._copy_args, Config=self._transfer_config)
+                                                                 ExtraArgs=self._copy_args,
+                                                                 Config=self._transfer_config)
             self._resource.Object(self.bucket, obj.key).delete()
 
         return paths