arvados-python-client 3.0.0__tar.gz → 3.1.0__tar.gz

{arvados-python-client-3.0.0/arvados_python_client.egg-info → arvados-python-client-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: arvados-python-client
-Version: 3.0.0
+Version: 3.1.0
 Summary: Arvados client library
 Home-page: https://arvados.org
 Author: Arvados

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/_internal/__init__.py

@@ -10,10 +10,15 @@ time.
 """
 
 import functools
+import operator
 import re
 import time
 import warnings
 
+import typing as t
+
+HT = t.TypeVar('HT', bound=t.Hashable)
+
 class Timer:
     def __init__(self, verbose=False):
         self.verbose = verbose
@@ -81,3 +86,29 @@ def deprecated(version=None, preferred=None):
         deprecated_wrapper.__doc__ = docstring
         return deprecated_wrapper
     return deprecated_decorator
+
+
+def parse_seq(
+        s: str,
+        split: t.Callable[[str], t.Iterable[str]]=operator.methodcaller('split', ','),
+        clean: t.Callable[[str], str]=operator.methodcaller('strip'),
+        check: t.Callable[[str], bool]=bool,
+) -> t.Iterator[str]:
+    """Split, clean, and filter a string into multiple items
+
+    The default arguments split on commas, strip substrings, and skip empty
+    items.
+    """
+    return (word for substr in split(s) if check(word := clean(substr)))
+
+
+def uniq(it: t.Iterable[HT]) -> t.Iterator[HT]:
+    """Yield only unique items from an iterable
+
+    The items must be hashable.
+    """
+    seen = set()
+    for item in it:
+        if item not in seen:
+            seen.add(item)
+            yield item

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/_internal/basedirs.py

@@ -126,12 +126,14 @@ class BaseDirectories:
         self._env = env
         self._xdg_subdir = PurePath(xdg_subdir)
 
+    def search_paths(self) -> Iterator[Path]:
+        return itertools.chain(
+                self._spec.iter_systemd(self._env),
+                self._spec.iter_xdg(self._env, self._xdg_subdir))
+
     def search(self, name: str) -> Iterator[Path]:
         any_found = False
-        for search_path in itertools.chain(
-                self._spec.iter_systemd(self._env),
-                self._spec.iter_xdg(self._env, self._xdg_subdir),
-        ):
+        for search_path in self.search_paths():
             path = search_path / name
             if path.exists():
                 yield path

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/_internal/http_to_keep.py

@@ -16,6 +16,7 @@ import pycurl
 
 import arvados
 import arvados.collection
+import arvados._internal
 from .pycurl import PyCurlHelper
 
 logger = logging.getLogger('arvados.http_import')
@@ -250,11 +251,8 @@ def check_cached_url(api, project_uuid, url, etags,
                      utcnow=datetime.datetime.utcnow,
                      varying_url_params="",
                      prefer_cached_downloads=False):
-
     logger.info("Checking Keep for %s", url)
-
-    varying_params = [s.strip() for s in varying_url_params.split(",")]
-
+    varying_params = set(arvados._internal.parse_seq(varying_url_params))
     parsed = urllib.parse.urlparse(url)
     query = [q for q in urllib.parse.parse_qsl(parsed.query)
              if q[0] not in varying_params]

arvados-python-client-3.1.0/arvados/_version.py

@@ -0,0 +1 @@
+__version__ = '3.1.0'

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/arvfile.py

@@ -23,8 +23,16 @@ from .errors import KeepWriteError, AssertionError, ArgumentError
 from .keep import KeepLocator
 from .retry import retry_method
 
+ADD = "add"
+"""Argument value for `Collection` methods to represent an added item"""
+DEL = "del"
+"""Argument value for `Collection` methods to represent a removed item"""
 MOD = "mod"
+"""Argument value for `Collection` methods to represent a modified item"""
+TOK = "tok"
+"""Argument value for `Collection` methods to represent an item with token differences"""
 WRITE = "write"
+"""Argument value for `Collection` methods to represent that a file was written to"""
 
 _logger = logging.getLogger('arvados.arvfile')
 
@@ -841,6 +849,8 @@ class ArvadosFile(object):
     def replace_contents(self, other):
         """Replace segments of this file with segments from another `ArvadosFile` object."""
 
+        eventtype = TOK if self == other else MOD
+
         map_loc = {}
         self._segments = []
         for other_segment in other.segments():
@@ -857,6 +867,7 @@ class ArvadosFile(object):
             self._segments.append(streams.Range(new_loc, other_segment.range_start, other_segment.range_size, other_segment.segment_offset))
 
         self.set_committed(False)
+        self.parent.notify(eventtype, self.parent, self.name, (self, self))
 
     def __eq__(self, other):
         if other is self:

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/collection.py

@@ -30,7 +30,7 @@ from stat import *
 
 from ._internal import streams
 from .api import ThreadSafeAPIClient
-from .arvfile import split, _FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, WrappableFile, _BlockManager, synchronized, must_be_writable, NoopLock
+from .arvfile import split, _FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, WrappableFile, _BlockManager, synchronized, must_be_writable, NoopLock, ADD, DEL, MOD, TOK, WRITE
 from .keep import KeepLocator, KeepClient
 import arvados.config as config
 import arvados.errors as errors
@@ -58,14 +58,7 @@ else:
 
 _logger = logging.getLogger('arvados.collection')
 
-ADD = "add"
-"""Argument value for `Collection` methods to represent an added item"""
-DEL = "del"
-"""Argument value for `Collection` methods to represent a removed item"""
-MOD = "mod"
-"""Argument value for `Collection` methods to represent a modified item"""
-TOK = "tok"
-"""Argument value for `Collection` methods to represent an item with token differences"""
+
 FILE = "file"
 """`create_type` value for `Collection.find_or_create`"""
 COLLECTION = "collection"
@@ -922,9 +915,12 @@ class RichCollectionBase(CollectionBase):
                         # Overwrite path with new item; this can happen if
                         # path was a file and is now a collection or vice versa
                         self.copy(final, path, overwrite=True)
-                else:
-                    # Local is missing (presumably deleted) or local doesn't
-                    # match the "start" value, so save change to conflict file
+                elif event_type == MOD:
+                    # Local doesn't match the "start" value or local
+                    # is missing (presumably deleted) so save change
+                    # to conflict file.  Don't do this for TOK events
+                    # which means the file didn't change but only had
+                    # tokens updated.
                     self.copy(final, conflictpath)
             elif event_type == DEL:
                 if local == initial:
@@ -992,8 +988,13 @@ class RichCollectionBase(CollectionBase):
           was modified.
 
         * item: arvados.arvfile.ArvadosFile |
-          arvados.collection.Subcollection --- The new contents at `name`
-          within `collection`.
+          arvados.collection.Subcollection --- For ADD events, the new
+          contents at `name` within `collection`; for DEL events, the
+          item that was removed.  For MOD and TOK events, a 2-tuple of
+          the previous item and the new item (may be the same object
+          or different, depending on whether the action involved it
+          being modified in place or replaced).
+
         """
         if self._callback:
             self._callback(event, collection, name, item)
@@ -1134,7 +1135,7 @@ class Collection(RichCollectionBase):
         self._manifest_text = None
         self._portable_data_hash = None
         self._api_response = None
-        self._past_versions = set()
+        self._token_refresh_timestamp = 0
 
         self.lock = threading.RLock()
         self.events = None
@@ -1200,20 +1201,6 @@ class Collection(RichCollectionBase):
     def writable(self) -> bool:
         return True
 
-    @synchronized
-    def known_past_version(
-            self,
-            modified_at_and_portable_data_hash: Tuple[Optional[str], Optional[str]]
-    ) -> bool:
-        """Indicate whether an API record for this collection has been seen before
-
-        As this collection object loads records from the API server, it records
-        their `modified_at` and `portable_data_hash` fields. This method accepts
-        a 2-tuple with values for those fields, and returns `True` if the
-        combination was previously loaded.
-        """
-        return modified_at_and_portable_data_hash in self._past_versions
-
     @synchronized
     @retry_method
     def update(
@@ -1245,23 +1232,61 @@ class Collection(RichCollectionBase):
           the collection's API record from the API server. If not specified,
           uses the `num_retries` provided when this instance was constructed.
         """
+
+        token_refresh_period = 60*60
+        time_since_last_token_refresh = (time.time() - self._token_refresh_timestamp)
+        upstream_response = None
+
         if other is None:
             if self._manifest_locator is None:
                 raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
-            response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
-            if (self.known_past_version((response.get("modified_at"), response.get("portable_data_hash"))) and
-                response.get("portable_data_hash") != self.portable_data_hash()):
-                # The record on the server is different from our current one, but we've seen it before,
-                # so ignore it because it's already been merged.
-                # However, if it's the same as our current record, proceed with the update, because we want to update
-                # our tokens.
+
+            if re.match(arvados.util.portable_data_hash_pattern, self._manifest_locator) and time_since_last_token_refresh < token_refresh_period:
                 return
-            else:
-                self._remember_api_response(response)
-            other = CollectionReader(response["manifest_text"])
-        baseline = CollectionReader(self._manifest_text)
+
+            upstream_response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
+            other = CollectionReader(upstream_response["manifest_text"])
+
+        if self.committed():
+            # 1st case, no local changes, content is the same
+            if self.portable_data_hash() == other.portable_data_hash() and time_since_last_token_refresh < token_refresh_period:
+                # No difference in content.  Remember the API record
+                # (metadata such as name or properties may have changed)
+                # but don't update the token refresh timestamp.
+                if upstream_response is not None:
+                    self._remember_api_response(upstream_response)
+                return
+
+            # 2nd case, no local changes, but either upstream changed
+            # or we want to refresh tokens.
+
+            self.apply(self.diff(other))
+            if upstream_response is not None:
+                self._remember_api_response(upstream_response)
+            self._update_token_timestamp()
+            self.set_committed(True)
+            return
+
+        # 3rd case, upstream changed, but we also have uncommitted
+        # changes that we want to incorporate so they don't get lost.
+
+        # _manifest_text stores the text from last time we received a
+        # record from the API server.  This is the state of the
+        # collection before our uncommitted changes.
+        baseline = Collection(self._manifest_text)
+
+        # Get the set of changes between our baseline and the other
+        # collection and apply them to self.
+        #
+        # If a file was modified in both 'self' and 'other', the
+        # 'apply' method keeps the contents of 'self' and creates a
+        # conflict file with the contents of 'other'.
         self.apply(baseline.diff(other))
-        self._manifest_text = self.manifest_text()
+
+        # Remember the new baseline, changes to a file
+        if upstream_response is not None:
+            self._remember_api_response(upstream_response)
+
 
     @synchronized
     def _my_api(self):
@@ -1295,7 +1320,11 @@ class Collection(RichCollectionBase):
 
     def _remember_api_response(self, response):
         self._api_response = response
-        self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))
+        self._manifest_text = self._api_response['manifest_text']
+        self._portable_data_hash = self._api_response['portable_data_hash']
+
+    def _update_token_timestamp(self):
+        self._token_refresh_timestamp = time.time()
 
     def _populate_from_api_server(self):
         # As in KeepClient itself, we must wait until the last
@@ -1308,8 +1337,7 @@ class Collection(RichCollectionBase):
         self._remember_api_response(self._my_api().collections().get(
             uuid=self._manifest_locator).execute(
                 num_retries=self.num_retries))
-        self._manifest_text = self._api_response['manifest_text']
-        self._portable_data_hash = self._api_response['portable_data_hash']
+
         # If not overriden via kwargs, we should try to load the
         # replication_desired and storage_classes_desired from the API server
         if self.replication_desired is None:
@@ -1534,8 +1562,6 @@ class Collection(RichCollectionBase):
                 uuid=self._manifest_locator,
                 body=body
                 ).execute(num_retries=num_retries))
-            self._manifest_text = self._api_response["manifest_text"]
-            self._portable_data_hash = self._api_response["portable_data_hash"]
             self.set_committed(True)
         elif body:
             self._remember_api_response(self._my_api().collections().update(
@@ -1654,12 +1680,7 @@ class Collection(RichCollectionBase):
                 body["preserve_version"] = preserve_version
 
             self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
-            text = self._api_response["manifest_text"]
-
             self._manifest_locator = self._api_response["uuid"]
-            self._portable_data_hash = self._api_response["portable_data_hash"]
-
-            self._manifest_text = text
             self.set_committed(True)
 
         return text
@@ -1743,6 +1764,7 @@ class Collection(RichCollectionBase):
                 stream_name = None
                 state = STREAM_NAME
 
+        self._update_token_timestamp()
         self.set_committed(True)
 
     @synchronized

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/commands/_util.py

@@ -3,14 +3,20 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
+import dataclasses
 import errno
 import json
 import logging
+import operator
 import os
 import re
 import signal
 import sys
 
+import typing as t
+
+from .. import _internal
+
 FILTER_STR_RE = re.compile(r'''
 ^\(
 \ *(\w+)
@@ -19,16 +25,41 @@ FILTER_STR_RE = re.compile(r'''
 \ *\)$
 ''', re.ASCII | re.VERBOSE)
 
-def _pos_int(s):
-    num = int(s)
-    if num < 0:
-        raise ValueError("can't accept negative value: %s" % (num,))
-    return num
+T = t.TypeVar('T')
+
+@dataclasses.dataclass(unsafe_hash=True)
+class RangedValue(t.Generic[T]):
+    """Validate that an argument string is within a valid range of values"""
+    parse_func: t.Callable[[str], T]
+    valid_range: t.Container[T]
+
+    def __call__(self, s: str) -> T:
+        value = self.parse_func(s)
+        if value in self.valid_range:
+            return value
+        else:
+            raise ValueError(f"{value!r} is not a valid value")
+
+
+@dataclasses.dataclass(unsafe_hash=True)
+class UniqueSplit(t.Generic[T]):
+    """Parse a string into a list of unique values"""
+    split: t.Callable[[str], t.Iterable[str]]=operator.methodcaller('split', ',')
+    clean: t.Callable[[str], str]=operator.methodcaller('strip')
+    check: t.Callable[[str], bool]=bool
+
+    def __call__(self, s: str) -> T:
+        return list(_internal.uniq(_internal.parse_seq(s, self.split, self.clean, self.check)))
+
 
 retry_opt = argparse.ArgumentParser(add_help=False)
-retry_opt.add_argument('--retries', type=_pos_int, default=10, help="""
-Maximum number of times to retry server requests that encounter temporary
-failures (e.g., server down).  Default 10.""")
+retry_opt.add_argument(
+    '--retries',
+    type=RangedValue(int, range(0, sys.maxsize)),
+    default=10,
+    help="""Maximum number of times to retry server requests that encounter
+temporary failures (e.g., server down).  Default %(default)r.
+""")
 
 def _ignore_error(error):
     return None

{arvados-python-client-3.0.0 → arvados-python-client-3.1.0}/arvados/commands/arv_copy.py

@@ -33,6 +33,10 @@ import io
 import json
 import queue
 import threading
+import errno
+
+import httplib2.error
+import googleapiclient
 
 import arvados
 import arvados.config
@@ -40,6 +44,7 @@ import arvados.keep
 import arvados.util
 import arvados.commands._util as arv_cmd
 import arvados.commands.keepdocker
+from arvados.logging import log_handler
 
 from arvados._internal import basedirs, http_to_keep
 from arvados._version import __version__
@@ -48,6 +53,9 @@ COMMIT_HASH_RE = re.compile(r'^[0-9a-f]{1,40}$')
 
 logger = logging.getLogger('arvados.arv-copy')
 
+# Set this up so connection errors get logged.
+googleapi_logger = logging.getLogger('googleapiclient.http')
+
 # local_repo_dir records which git repositories from the Arvados source
 # instance have been checked out locally during this run, and to which
 # directories.
@@ -112,7 +120,18 @@ If not provided, will use the default client configuration from the environment
         '--project-uuid', dest='project_uuid',
         help='The UUID of the project at the destination to which the collection or workflow should be copied.')
     copy_opts.add_argument(
-        '--storage-classes', dest='storage_classes',
+        '--replication',
+        type=arv_cmd.RangedValue(int, range(1, sys.maxsize)),
+        metavar='N',
+        help="""
+Number of replicas per storage class for the copied collections at the destination.
+If not provided (or if provided with invalid value),
+use the destination's default replication-level setting (if found),
+or the fallback value 2.
+""")
+    copy_opts.add_argument(
+        '--storage-classes',
+        type=arv_cmd.UniqueSplit(),
         help='Comma separated list of storage classes to be used when saving data to the destinaton Arvados instance.')
     copy_opts.add_argument("--varying-url-params", type=str, default="",
                         help="A comma separated list of URL query parameters that should be ignored when storing HTTP URLs in Keep.")
@@ -131,9 +150,6 @@ If not provided, will use the default client configuration from the environment
         parents=[copy_opts, arv_cmd.retry_opt])
     args = parser.parse_args()
 
-    if args.storage_classes:
-        args.storage_classes = [x for x in args.storage_classes.strip().replace(' ', '').split(',') if x]
-
     if args.verbose:
         logger.setLevel(logging.DEBUG)
     else:
@@ -142,10 +158,29 @@ If not provided, will use the default client configuration from the environment
     if not args.source_arvados and arvados.util.uuid_pattern.match(args.object_uuid):
         args.source_arvados = args.object_uuid[:5]
 
+    if not args.destination_arvados and args.project_uuid:
+        args.destination_arvados = args.project_uuid[:5]
+
+    # Make sure errors trying to connect to clusters get logged.
+    googleapi_logger.setLevel(logging.WARN)
+    googleapi_logger.addHandler(log_handler)
+
     # Create API clients for the source and destination instances
     src_arv = api_for_instance(args.source_arvados, args.retries)
     dst_arv = api_for_instance(args.destination_arvados, args.retries)
 
+    # Once we've successfully contacted the clusters, we probably
+    # don't want to see logging about retries (unless the user asked
+    # for verbose output).
+    if not args.verbose:
+        googleapi_logger.setLevel(logging.ERROR)
+
+    if src_arv.config()["ClusterID"] == dst_arv.config()["ClusterID"]:
+        logger.info("Copying within cluster %s", src_arv.config()["ClusterID"])
+    else:
+        logger.info("Source cluster is %s", src_arv.config()["ClusterID"])
+        logger.info("Destination cluster is %s", dst_arv.config()["ClusterID"])
+
     if not args.project_uuid:
         args.project_uuid = dst_arv.users().current().execute(num_retries=args.retries)["uuid"]
 
@@ -213,43 +248,64 @@ def set_src_owner_uuid(resource, uuid, args):
 #     configuration directory.
 #
 def api_for_instance(instance_name, num_retries):
-    if not instance_name:
-        # Use environment
-        return arvados.api('v1')
-
-    if '/' in instance_name:
-        config_file = instance_name
-    else:
-        dirs = basedirs.BaseDirectories('CONFIG')
-        config_file = next(dirs.search(f'{instance_name}.conf'), '')
+    msg = []
+    if instance_name:
+        if '/' in instance_name:
+            config_file = instance_name
+        else:
+            dirs = basedirs.BaseDirectories('CONFIG')
+            config_file = next(dirs.search(f'{instance_name}.conf'), '')
 
+        try:
+            cfg = arvados.config.load(config_file)
+
+            if 'ARVADOS_API_HOST' in cfg and 'ARVADOS_API_TOKEN' in cfg:
+                api_is_insecure = (
+                    cfg.get('ARVADOS_API_HOST_INSECURE', '').lower() in set(
+                        ['1', 't', 'true', 'y', 'yes']))
+                return arvados.api('v1',
+                                     host=cfg['ARVADOS_API_HOST'],
+                                     token=cfg['ARVADOS_API_TOKEN'],
+                                     insecure=api_is_insecure,
+                                     num_retries=num_retries,
+                                     )
+            else:
+                msg.append('missing ARVADOS_API_HOST or ARVADOS_API_TOKEN for {} in config file {}'.format(instance_name, config_file))
+        except OSError as e:
+            if e.errno in (errno.EHOSTUNREACH, errno.ECONNREFUSED, errno.ECONNRESET, errno.ENETUNREACH):
+                verb = 'connect to instance from'
+            elif config_file:
+                verb = 'open'
+            else:
+                verb = 'find'
+                searchlist = ":".join(str(p) for p in dirs.search_paths())
+                config_file = f'{instance_name}.conf in path {searchlist}'
+            msg.append(("Could not {} config file {}: {}").format(
+                       verb, config_file, e.strerror))
+        except (httplib2.error.HttpLib2Error, googleapiclient.errors.Error) as e:
+            msg.append("Failed to connect to instance {} at {}, error was {}".format(instance_name, cfg['ARVADOS_API_HOST'], e))
+
+    default_api = None
+    default_instance = None
     try:
-        cfg = arvados.config.load(config_file)
-    except OSError as e:
-        if config_file:
-            verb = 'open'
-        else:
-            verb = 'find'
-            config_file = f'{instance_name}.conf'
-        abort(("Could not {} config file {}: {}\n" +
-               "You must make sure that your configuration tokens\n" +
-               "for Arvados instance {} are in {} and that this\n" +
-               "file is readable.").format(
-                   verb, config_file, e.strerror, instance_name, config_file))
-
-    if 'ARVADOS_API_HOST' in cfg and 'ARVADOS_API_TOKEN' in cfg:
-        api_is_insecure = (
-            cfg.get('ARVADOS_API_HOST_INSECURE', '').lower() in set(
-                ['1', 't', 'true', 'y', 'yes']))
-        client = arvados.api('v1',
-                             host=cfg['ARVADOS_API_HOST'],
-                             token=cfg['ARVADOS_API_TOKEN'],
-                             insecure=api_is_insecure,
-                             num_retries=num_retries,
-                             )
-    else:
-        abort('need ARVADOS_API_HOST and ARVADOS_API_TOKEN for {}'.format(instance_name))
-    return client
+        default_api = arvados.api('v1', num_retries=num_retries)
+        default_instance = default_api.config()["ClusterID"]
+    except ValueError:
+        pass
+    except (httplib2.error.HttpLib2Error, googleapiclient.errors.Error, OSError) as e:
+        msg.append("Failed to connect to default instance, error was {}".format(e))
+
+    if default_api is not None and (not instance_name or instance_name == default_instance):
+        # Use default settings
+        return default_api
+
+    if instance_name and default_instance and instance_name != default_instance:
+        msg.append("Default credentials are for {} but need to connect to {}".format(default_instance, instance_name))
+
+    for m in msg:
+        logger.error(m)
+
+    abort('Unable to find usable ARVADOS_API_HOST and ARVADOS_API_TOKEN')
 
 # Check if git is available
 def check_git_availability():
@@ -587,6 +643,14 @@ def copy_collection(obj_uuid, src, dst, args):
             ).execute(num_retries=args.retries)['manifest_text']
             return create_collection_from(c, src, dst, args)
 
+    if args.replication is None:
+        # Obtain default or fallback collection replication setting on the
+        # destination
+        try:
+            args.replication = int(dst.config()["Collections"]["DefaultReplication"])
+        except (KeyError, TypeError, ValueError):
+            args.replication = 2
+
     # Fetch the collection's manifest.
     manifest = c['manifest_text']
     logger.debug("Copying collection %s with manifest: <%s>", obj_uuid, manifest)
@@ -678,7 +742,7 @@ def copy_collection(obj_uuid, src, dst, args):
 
             try:
                 logger.debug("Putting block %s (%s bytes)", blockhash, loc.size)
-                dst_locator = dst_keep.put(data, classes=(args.storage_classes or []))
+                dst_locator = dst_keep.put(data, copies=args.replication, classes=(args.storage_classes or []))
                 with lock:
                     dst_locators[blockhash] = dst_locator
                     bytes_written += loc.size
@@ -870,17 +934,17 @@ def uuid_type(api, object_uuid):
 def copy_from_http(url, src, dst, args):
 
     project_uuid = args.project_uuid
-    varying_url_params = args.varying_url_params
+    # Ensure string of varying parameters is well-formed
     prefer_cached_downloads = args.prefer_cached_downloads
 
     cached = http_to_keep.check_cached_url(src, project_uuid, url, {},
-                                           varying_url_params=varying_url_params,
+                                           varying_url_params=args.varying_url_params,
                                            prefer_cached_downloads=prefer_cached_downloads)
     if cached[2] is not None:
         return copy_collection(cached[2], src, dst, args)
 
     cached = http_to_keep.http_to_keep(dst, project_uuid, url,
-                                       varying_url_params=varying_url_params,
+                                       varying_url_params=args.varying_url_params,
                                        prefer_cached_downloads=prefer_cached_downloads)
 
     if cached is not None: