arvados-python-client 2.7.1__tar.gz → 2.7.3__tar.gz

{arvados-python-client-2.7.1/arvados_python_client.egg-info → arvados-python-client-2.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: arvados-python-client
-Version: 2.7.1
+Version: 2.7.3
 Summary: Arvados client library
 Home-page: https://arvados.org
 Author: Arvados

{arvados-python-client-2.7.1 → arvados-python-client-2.7.3}/arvados/__init__.py

@@ -6,8 +6,8 @@
 This module provides the entire Python SDK for Arvados. The most useful modules
 include:
 
-* arvados.api - After you `import arvados`, you can call `arvados.api.api` as
-  `arvados.api` to construct a client object.
+* arvados.api - After you `import arvados`, you can call `arvados.api` as a
+  shortcut to the client constructor function `arvados.api.api`.
 
 * arvados.collection - The `arvados.collection.Collection` class provides a
   high-level interface to read and write collections. It coordinates sending
@@ -26,15 +26,24 @@ import types
 
 from collections import UserDict
 
-from .api import api, api_from_config, http_cache
+from . import api, errors, util
+from .api import api_from_config, http_cache
 from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
 from arvados.keep import *
 from arvados.stream import *
 from .arvfile import StreamFileReader
 from .logging import log_format, log_date_format, log_handler
 from .retry import RetryLoop
-import arvados.errors as errors
-import arvados.util as util
+
+# Previous versions of the PySDK used to say `from .api import api`.  This
+# made it convenient to call the API client constructor, but difficult to
+# access the rest of the `arvados.api` module. The magic below fixes that
+# bug while retaining backwards compatibility: `arvados.api` is now the
+# module and you can import it normally, but we make that module callable so
+# all the existing code that says `arvados.api('v1', ...)` still works.
+class _CallableAPIModule(api.__class__):
+    __call__ = staticmethod(api.api)
+api.__class__ = _CallableAPIModule
 
 # Override logging module pulled in via `from ... import *`
 # so users can `import arvados.logging`.

arvados-python-client-2.7.3/arvados/_version.py

@@ -0,0 +1 @@
+__version__ = '2.7.3'

{arvados-python-client-2.7.1 → arvados-python-client-2.7.3}/arvados/arvfile.py

@@ -491,7 +491,7 @@ class _BlockManager(object):
         self._put_queue = None
         self._put_threads = None
         self.lock = threading.Lock()
-        self.prefetch_enabled = True
+        self.prefetch_lookahead = self._keep.num_prefetch_threads
         self.num_put_threads = put_threads or _BlockManager.DEFAULT_PUT_THREADS
         self.copies = copies
         self.storage_classes = storage_classes_func or (lambda: [])
@@ -803,7 +803,7 @@ class _BlockManager(object):
         """Initiate a background download of a block.
         """
 
-        if not self.prefetch_enabled:
+        if not self.prefetch_lookahead:
             return
 
         with self.lock:
@@ -825,7 +825,7 @@ class ArvadosFile(object):
     """
 
     __slots__ = ('parent', 'name', '_writers', '_committed',
-                 '_segments', 'lock', '_current_bblock', 'fuse_entry')
+                 '_segments', 'lock', '_current_bblock', 'fuse_entry', '_read_counter')
 
     def __init__(self, parent, name, stream=[], segments=[]):
         """
@@ -846,6 +846,7 @@ class ArvadosFile(object):
         for s in segments:
             self._add_segment(stream, s.locator, s.range_size)
         self._current_bblock = None
+        self._read_counter = 0
 
     def writable(self):
         return self.parent.writable()
@@ -1047,20 +1048,47 @@ class ArvadosFile(object):
             # size == self.size()
             pass
 
-    def readfrom(self, offset, size, num_retries, exact=False):
+    def readfrom(self, offset, size, num_retries, exact=False, return_memoryview=False):
         """Read up to `size` bytes from the file starting at `offset`.
 
-        :exact:
-         If False (default), return less data than requested if the read
-         crosses a block boundary and the next block isn't cached.  If True,
-         only return less data than requested when hitting EOF.
+        Arguments:
+
+        * exact: bool --- If False (default), return less data than
+         requested if the read crosses a block boundary and the next
+         block isn't cached.  If True, only return less data than
+         requested when hitting EOF.
+
+        * return_memoryview: bool --- If False (default) return a
+          `bytes` object, which may entail making a copy in some
+          situations.  If True, return a `memoryview` object which may
+          avoid making a copy, but may be incompatible with code
+          expecting a `bytes` object.
+
         """
 
         with self.lock:
             if size == 0 or offset >= self.size():
-                return b''
+                return memoryview(b'') if return_memoryview else b''
             readsegs = locators_and_ranges(self._segments, offset, size)
-            prefetch = locators_and_ranges(self._segments, offset + size, config.KEEP_BLOCK_SIZE * self.parent._my_block_manager()._keep.num_prefetch_threads, limit=32)
+
+            prefetch = None
+            prefetch_lookahead = self.parent._my_block_manager().prefetch_lookahead
+            if prefetch_lookahead:
+                # Doing prefetch on every read() call is surprisingly expensive
+                # when we're trying to deliver data at 600+ MiBps and want
+                # the read() fast path to be as lightweight as possible.
+                #
+                # Only prefetching every 128 read operations
+                # dramatically reduces the overhead while still
+                # getting the benefit of prefetching (e.g. when
+                # reading 128 KiB at a time, it checks for prefetch
+                # every 16 MiB).
+                self._read_counter = (self._read_counter+1) % 128
+                if self._read_counter == 1:
+                    prefetch = locators_and_ranges(self._segments,
+                                                   offset + size,
+                                                   config.KEEP_BLOCK_SIZE * prefetch_lookahead,
+                                                   limit=(1+prefetch_lookahead))
 
         locs = set()
         data = []
@@ -1068,17 +1096,22 @@ class ArvadosFile(object):
             block = self.parent._my_block_manager().get_block_contents(lr.locator, num_retries=num_retries, cache_only=(bool(data) and not exact))
             if block:
                 blockview = memoryview(block)
-                data.append(blockview[lr.segment_offset:lr.segment_offset+lr.segment_size].tobytes())
+                data.append(blockview[lr.segment_offset:lr.segment_offset+lr.segment_size])
                 locs.add(lr.locator)
             else:
                 break
 
-        for lr in prefetch:
-            if lr.locator not in locs:
-                self.parent._my_block_manager().block_prefetch(lr.locator)
-                locs.add(lr.locator)
+        if prefetch:
+            for lr in prefetch:
+                if lr.locator not in locs:
+                    self.parent._my_block_manager().block_prefetch(lr.locator)
+                    locs.add(lr.locator)
+
+        if len(data) == 1:
+            return data[0] if return_memoryview else data[0].tobytes()
+        else:
+            return memoryview(b''.join(data)) if return_memoryview else b''.join(data)
 
-        return b''.join(data)
 
     @must_be_writable
     @synchronized
@@ -1243,33 +1276,49 @@ class ArvadosFileReader(ArvadosFileReaderBase):
 
     @_FileLikeObjectBase._before_close
     @retry_method
-    def read(self, size=None, num_retries=None):
+    def read(self, size=-1, num_retries=None, return_memoryview=False):
         """Read up to `size` bytes from the file and return the result.
 
-        Starts at the current file position.  If `size` is None, read the
-        entire remainder of the file.
+        Starts at the current file position.  If `size` is negative or None,
+        read the entire remainder of the file.
+
+        Returns None if the file pointer is at the end of the file.
+
+        Returns a `bytes` object, unless `return_memoryview` is True,
+        in which case it returns a memory view, which may avoid an
+        unnecessary data copy in some situations.
+
         """
-        if size is None:
+        if size < 0 or size is None:
             data = []
-            rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+            #
+            # specify exact=False, return_memoryview=True here so that we
+            # only copy data once into the final buffer.
+            #
+            rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries, exact=False, return_memoryview=True)
             while rd:
                 data.append(rd)
                 self._filepos += len(rd)
-                rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
-            return b''.join(data)
+                rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries, exact=False, return_memoryview=True)
+            return memoryview(b''.join(data)) if return_memoryview else b''.join(data)
         else:
-            data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
+            data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True, return_memoryview=return_memoryview)
             self._filepos += len(data)
             return data
 
     @_FileLikeObjectBase._before_close
     @retry_method
-    def readfrom(self, offset, size, num_retries=None):
+    def readfrom(self, offset, size, num_retries=None, return_memoryview=False):
         """Read up to `size` bytes from the stream, starting at the specified file offset.
 
         This method does not change the file position.
+
+        Returns a `bytes` object, unless `return_memoryview` is True,
+        in which case it returns a memory view, which may avoid an
+        unnecessary data copy in some situations.
+
         """
-        return self.arvadosfile.readfrom(offset, size, num_retries)
+        return self.arvadosfile.readfrom(offset, size, num_retries, exact=True, return_memoryview=return_memoryview)
 
     def flush(self):
         pass

{arvados-python-client-2.7.1 → arvados-python-client-2.7.3}/arvados/collection.py

@@ -341,7 +341,7 @@ class RichCollectionBase(CollectionBase):
             self,
             path: str,
             mode: str="r",
-            encoding: Optional[str]=None,
+            encoding: Optional[str]=None
     ) -> IO:
         """Open a file-like object within the collection
 
@@ -361,6 +361,7 @@ class RichCollectionBase(CollectionBase):
         * encoding: str | None --- The text encoding of the file. Only used
           when the file is opened in text mode. The default is
           platform-dependent.
+
         """
         if not re.search(r'^[rwa][bt]?\+?$', mode):
             raise errors.ArgumentError("Invalid mode {!r}".format(mode))

arvados-python-client-2.7.3/arvados/commands/_util.py

@@ -0,0 +1,158 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import errno
+import json
+import logging
+import os
+import re
+import signal
+import sys
+
+FILTER_STR_RE = re.compile(r'''
+^\(
+\ *(\w+)
+\ *(<|<=|=|>=|>)
+\ *(\w+)
+\ *\)$
+''', re.ASCII | re.VERBOSE)
+
+def _pos_int(s):
+    num = int(s)
+    if num < 0:
+        raise ValueError("can't accept negative value: %s" % (num,))
+    return num
+
+retry_opt = argparse.ArgumentParser(add_help=False)
+retry_opt.add_argument('--retries', type=_pos_int, default=10, help="""
+Maximum number of times to retry server requests that encounter temporary
+failures (e.g., server down).  Default 10.""")
+
+def _ignore_error(error):
+    return None
+
+def _raise_error(error):
+    raise error
+
+def make_home_conf_dir(path, mode=None, errors='ignore'):
+    # Make the directory path under the user's home directory, making parent
+    # directories as needed.
+    # If the directory is newly created, and a mode is specified, chmod it
+    # with those permissions.
+    # If there's an error, return None if errors is 'ignore', else raise an
+    # exception.
+    error_handler = _ignore_error if (errors == 'ignore') else _raise_error
+    tilde_path = os.path.join('~', path)
+    abs_path = os.path.expanduser(tilde_path)
+    if abs_path == tilde_path:
+        return error_handler(ValueError("no home directory available"))
+    try:
+        os.makedirs(abs_path)
+    except OSError as error:
+        if error.errno != errno.EEXIST:
+            return error_handler(error)
+    else:
+        if mode is not None:
+            os.chmod(abs_path, mode)
+    return abs_path
+
+CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
+
+def exit_signal_handler(sigcode, frame):
+    logging.getLogger('arvados').error("Caught signal {}, exiting.".format(sigcode))
+    sys.exit(-sigcode)
+
+def install_signal_handlers():
+    global orig_signal_handlers
+    orig_signal_handlers = {sigcode: signal.signal(sigcode, exit_signal_handler)
+                            for sigcode in CAUGHT_SIGNALS}
+
+def restore_signal_handlers():
+    for sigcode, orig_handler in orig_signal_handlers.items():
+        signal.signal(sigcode, orig_handler)
+
+def validate_filters(filters):
+    """Validate user-provided filters
+
+    This function validates that a user-defined object represents valid
+    Arvados filters that can be passed to an API client: that it's a list of
+    3-element lists with the field name and operator given as strings. If any
+    of these conditions are not true, it raises a ValueError with details about
+    the problem.
+
+    It returns validated filters. Currently the provided filters are returned
+    unmodified. Future versions of this function may clean up the filters with
+    "obvious" type conversions, so callers SHOULD use the returned value for
+    Arvados API calls.
+    """
+    if not isinstance(filters, list):
+        raise ValueError(f"filters are not a list: {filters!r}")
+    for index, f in enumerate(filters):
+        if isinstance(f, str):
+            match = FILTER_STR_RE.fullmatch(f)
+            if match is None:
+                raise ValueError(f"filter at index {index} has invalid syntax: {f!r}")
+            s, op, o = match.groups()
+            if s[0].isdigit():
+                raise ValueError(f"filter at index {index} has invalid syntax: bad field name {s!r}")
+            if o[0].isdigit():
+                raise ValueError(f"filter at index {index} has invalid syntax: bad field name {o!r}")
+            continue
+        elif not isinstance(f, list):
+            raise ValueError(f"filter at index {index} is not a string or list: {f!r}")
+        try:
+            s, op, o = f
+        except ValueError:
+            raise ValueError(
+                f"filter at index {index} does not have three items (field name, operator, operand): {f!r}",
+            ) from None
+        if not isinstance(s, str):
+            raise ValueError(f"filter at index {index} field name is not a string: {s!r}")
+        if not isinstance(op, str):
+            raise ValueError(f"filter at index {index} operator is not a string: {op!r}")
+    return filters
+
+
+class JSONArgument:
+    """Parse a JSON file from a command line argument string or path
+
+    JSONArgument objects can be called with a string and return an arbitrary
+    object. First it will try to decode the string as JSON. If that fails, it
+    will try to open a file at the path named by the string, and decode it as
+    JSON. If that fails, it raises ValueError with more detail.
+
+    This is designed to be used as an argparse argument type.
+    Typical usage looks like:
+
+        parser = argparse.ArgumentParser()
+        parser.add_argument('--object', type=JSONArgument(), ...)
+
+    You can construct JSONArgument with an optional validation function. If
+    given, it is called with the object decoded from user input, and its
+    return value replaces it. It should raise ValueError if there is a problem
+    with the input. (argparse turns ValueError into a useful error message.)
+
+        filters_type = JSONArgument(validate_filters)
+        parser.add_argument('--filters', type=filters_type, ...)
+    """
+    def __init__(self, validator=None):
+        self.validator = validator
+
+    def __call__(self, value):
+        try:
+            retval = json.loads(value)
+        except json.JSONDecodeError:
+            try:
+                with open(value, 'rb') as json_file:
+                    retval = json.load(json_file)
+            except json.JSONDecodeError as error:
+                raise ValueError(f"error decoding JSON from file {value!r}: {error}") from None
+            except (FileNotFoundError, ValueError):
+                raise ValueError(f"not a valid JSON string or file path: {value!r}") from None
+            except OSError as error:
+                raise ValueError(f"error reading JSON file path {value!r}: {error.strerror}") from None
+        if self.validator is not None:
+            retval = self.validator(retval)
+        return retval

{arvados-python-client-2.7.1 → arvados-python-client-2.7.3}/arvados/commands/keepdocker.py

@@ -2,34 +2,29 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from builtins import next
 import argparse
 import collections
 import datetime
 import errno
+import fcntl
 import json
+import logging
 import os
 import re
+import subprocess
 import sys
 import tarfile
 import tempfile
-import shutil
-import _strptime
-import fcntl
+
+import ciso8601
 from operator import itemgetter
 from stat import *
 
-import subprocess
-
 import arvados
+import arvados.config
 import arvados.util
 import arvados.commands._util as arv_cmd
 import arvados.commands.put as arv_put
-from arvados.collection import CollectionReader
-import ciso8601
-import logging
-import arvados.config
-
 from arvados._version import __version__
 
 logger = logging.getLogger('arvados.keepdocker')
@@ -356,6 +351,25 @@ def _uuid2pdh(api, uuid):
         select=['portable_data_hash'],
     ).execute()['items'][0]['portable_data_hash']
 
+def load_image_metadata(image_file):
+    """Load an image manifest and config from an archive
+
+    Given an image archive as an open binary file object, this function loads
+    the image manifest and configuration, deserializing each from JSON and
+    returning them in a 2-tuple of dicts.
+    """
+    image_file.seek(0)
+    with tarfile.open(fileobj=image_file) as image_tar:
+        with image_tar.extractfile('manifest.json') as manifest_file:
+            image_manifest_list = json.load(manifest_file)
+        # Because arv-keepdocker only saves one image, there should only be
+        # one manifest.  This extracts that from the list and raises
+        # ValueError if there's not exactly one.
+        image_manifest, = image_manifest_list
+        with image_tar.extractfile(image_manifest['Config']) as config_file:
+            image_config = json.load(config_file)
+    return image_manifest, image_config
+
 def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None):
     args = arg_parser.parse_args(arguments)
     if api is None:
@@ -532,21 +546,9 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
         # Managed properties could be already set
         coll_properties = api.collections().get(uuid=coll_uuid).execute(num_retries=args.retries).get('properties', {})
         coll_properties.update({"docker-image-repo-tag": image_repo_tag})
-
         api.collections().update(uuid=coll_uuid, body={"properties": coll_properties}).execute(num_retries=args.retries)
 
-        # Read the image metadata and make Arvados links from it.
-        image_file.seek(0)
-        image_tar = tarfile.open(fileobj=image_file)
-        image_hash_type, _, raw_image_hash = image_hash.rpartition(':')
-        if image_hash_type:
-            json_filename = raw_image_hash + '.json'
-        else:
-            json_filename = raw_image_hash + '/json'
-        json_file = image_tar.extractfile(image_tar.getmember(json_filename))
-        image_metadata = json.loads(json_file.read().decode('utf-8'))
-        json_file.close()
-        image_tar.close()
+        _, image_metadata = load_image_metadata(image_file)
         link_base = {'head_uuid': coll_uuid, 'properties': {}}
         if 'created' in image_metadata:
             link_base['properties']['image_timestamp'] = image_metadata['created']