deriva 1.7.1__tar.gz → 1.7.4__tar.gz

{deriva-1.7.1/deriva.egg-info → deriva-1.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.1
+Version: 1.7.4
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division

{deriva-1.7.1 → deriva-1.7.4}/deriva/config/annotation_config.py

@@ -33,7 +33,7 @@ class AttrSpecList(BaseSpecList):
             return None
         new = []
         for item in orig_list:
-            new.append(unicode(item))
+            new.append(item)
         return new
 
     def add_list(self, dictlist):
@@ -85,7 +85,7 @@ class AttrConfig:
         self.toplevel_config = ConfigUtil.find_toplevel_node(self.catalog.getCatalogModel(), schema_name, table_name)
 
     def make_speclist(self, name):
-        d = self.config.get(unicode(name))
+        d = self.config.get(name)
         if d is None:
             d = [dict()]
         return AttrSpecList(self.known_attrs, d)

{deriva-1.7.1 → deriva-1.7.4}/deriva/config/rollback_annotation.py

@@ -1,4 +1,4 @@
-import platform
+import sys
 from deriva.core import get_credential, BaseCLI, DerivaServer, __version__
 
 

{deriva-1.7.1 → deriva-1.7.4}/deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.1"
+__version__ = "1.7.4"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

{deriva-1.7.1 → deriva-1.7.4}/deriva/core/datapath.py

@@ -5,6 +5,7 @@ import copy
 from datetime import date
 import itertools
 import logging
+import time
 import re
 from requests import HTTPError
 import warnings
@@ -695,6 +696,102 @@ class _ResultSet (object):
         logger.debug("Fetched %d entities" % len(self._results_doc))
         return self
 
+def _json_size_approx(data):
+    """Return approximate byte count for minimal JSON encoding of data
+
+    Minimal encoding has no optional whitespace/indentation.
+    """
+    nbytes = 0
+
+    if isinstance(data, (list, tuple)):
+        nbytes += 2
+        for elem in data:
+            nbytes += _json_size_approx(elem) + 1
+    elif isinstance(data, dict):
+        nbytes += 2
+        for k, v in data.items():
+            nbytes += _json_size_approx(k) + _json_size_approx(v) + 2
+    elif isinstance(data, str):
+        nbytes += len(data.encode("utf-8")) + 2
+    else:
+        nbytes += len(str(data))
+
+    return nbytes
+
+def _generate_batches(entities, max_batch_rows=1000, max_batch_bytes=250*1024):
+    """Generate a series of entity batches as slices of the input entities
+
+    """
+    if not isinstance(entities, (list, tuple)):
+        raise TypeError('invalid type %s for entities, list or tuple expected' % (type(entities),))
+
+    if not max_batch_rows:
+        logger.debug("disabling batching due to max_batch_rows=%r" % (max_batch_rows,))
+        return entities
+
+    top = len(entities)
+    lower = 0
+
+    while lower < top:
+        # to ensure progress, always use at least one row per batch regardless of nbytes
+        upper = lower + 1
+        batch_nbytes = _json_size_approx(entities[lower])
+
+        # advance upper position until a batch size limit is reached
+        while (upper - lower) < max_batch_rows:
+            if upper >= top:
+                break
+            batch_nbytes += _json_size_approx(entities[upper])
+            if batch_nbytes > max_batch_bytes:
+                break
+            upper += 1
+
+        # generate one batch and advance for next batch
+        logger.debug("yielding batch of %d/%d entities (%d:%d)" % (upper-lower, top, lower, upper))
+        yield entities[lower:upper]
+        lower = upper
+
+def _request_with_retry(request_func, retry_codes={408, 429, 500, 502, 503, 504}, backoff_factor=4, max_attempts=5):
+    """Perform request func with exponential backoff and retry.
+
+    :param request_func: A function returning a requests.Response object or raising HTTPError
+    :param retry_codes: HTTPError status codes on which to attempt retry
+    :param backoff_factor: Base number of seconds for factor**attempt exponential backoff
+    :param max_attempts: Max number of request attempts.
+
+    Retry will be attempted on HTTPError exceptions which match retry_codes and
+    also on other unknown exceptions, presumed to be transport errors.
+
+    The request_func should do the equivalent of resp.raise_on_status() so that
+    it only returns a response object for successful requests.
+    """
+    attempt = 0
+    last_ex = None
+
+    while attempt < max_attempts:
+        try:
+            if attempt > 0:
+                delay = backoff_factor**(attempt-1)
+                logger.debug("sleeping %d seconds before retry %d..." % (delay, attempt))
+                time.sleep(delay)
+            attempt += 1
+            return request_func()
+        except HTTPError as e:
+            logger.debug(e.response.text)
+            last_ex = e
+            if 400 <= e.response.status_code < 500:
+                last_ex = DataPathException(_http_error_message(e), e)
+            if int(e.response.status_code) not in retry_codes:
+                raise last_ex
+        except Exception as e:
+            logger.debug(e.response.text)
+            last_ex = e
+
+    # early return means we don't get here on successful requests
+    logger.warning("maximum request retry limit %d exceeded" % (max_attempts,))
+    if last_ex is None:
+        raise ValueError('exceeded max_attempts without catching a request exception')
+    raise last_ex
 
 class _TableWrapper (object):
     """Wraps a Table for datapath expressions.
@@ -836,7 +933,7 @@ class _TableWrapper (object):
         """
         return self.path.denormalize(context_name=context_name, heuristic=heuristic, groupkey_name=groupkey_name)
 
-    def insert(self, entities, defaults=set(), nondefaults=set(), add_system_defaults=True, on_conflict_skip=False):
+    def insert(self, entities, defaults=set(), nondefaults=set(), add_system_defaults=True, on_conflict_skip=False, retry_codes={408, 429, 500, 502, 503, 504}, backoff_factor=4, max_attempts=5, max_batch_rows=1000, max_batch_bytes=250*1024):
         """Inserts entities into the table.
 
         :param entities: an iterable collection of entities (i.e., rows) to be inserted into the table.
@@ -844,7 +941,23 @@ class _TableWrapper (object):
         :param nondefaults: optional, set of columns names to override implicit system defaults
         :param add_system_defaults: flag to add system columns to the set of default columns.
         :param on_conflict_skip: flag to skip entities that violate uniqueness constraints.
+        :param retry_codes: set of HTTP status codes for which retry should be considered.
+        :param backoff_factor: number of seconds for base of exponential retry backoff.
+        :param max_attempts: maximum number of requests attempts with retry.
+        :param max_batch_rows: maximum number of rows for one request, or False to disable batching.
+        :param max_batch_bytes: approximate maximum number of bytes for one request.
         :return a collection of newly created entities.
+
+        Retry will only be attempted for idempotent insertion
+        requests, which are when a user-controlled, non-nullable key
+        is present in the table and the key's constituent column(s)
+        are not listed as defaults, and on_conflict_skip=True.
+
+        When performing retries, an exponential backoff delay is
+        introduced after each failed attempt. The delay is
+        backoff_factor**attempt_number seconds for attempts 0 through
+        max_attempts-1.
+
         """
         # empty entities will be accepted but results are therefore an empty entity set
         if not entities:
@@ -879,17 +992,52 @@ class _TableWrapper (object):
         if not hasattr(entities[0], 'keys'):
             raise TypeError('entities[0] does not look like a dictionary -- does not have a "keys()" method')
 
-        try:
-            resp = self._schema._catalog._wrapped_catalog.post(path, json=entities, headers={'Content-Type': 'application/json'})
-            return _ResultSet(self.path.uri, lambda ignore1, ignore2, ignore3: resp.json())
-        except HTTPError as e:
-            logger.debug(e.response.text)
-            if 400 <= e.response.status_code < 500:
-                raise DataPathException(_http_error_message(e), e)
-            else:
-                raise e
+        # perform one batch request in a helper we can hand to retry helper
+        def request_func(batch):
+            return self._schema._catalog._wrapped_catalog.post(path, json=batch, headers={'Content-Type': 'application/json'})
+
+        def _has_user_pkey(table):
+            """Return True if table has at least one primary key other than the system RID key"""
+            for key in table.keys:
+                if { c.name for c in key.unique_columns } != {'RID'}:
+                    if all([ not c.nullok for c in key.unique_columns ]) \
+                       and all([ c.name not in defaults for c in key.unique_columns ]):
+                        return True
+            return False
+
+        # determine whether insert is idempotent and therefore retry safe
+        retry_safe = on_conflict_skip and _has_user_pkey(self._wrapped_table)
+
+        # perform all requests synchronously so the caller can get exceptions
+        results = []
+        for batch in _generate_batches(
+            entities,
+            max_batch_rows=max_batch_rows,
+            max_batch_bytes=max_batch_bytes
+        ):
+            try:
+                if retry_safe:
+                    resp = _request_with_retry(
+                        lambda: request_func(batch),
+                        retry_codes=retry_codes,
+                        backoff_factor=backoff_factor,
+                        max_attempts=max_attempts
+                    )
+                else:
+                    resp = request_func(batch)
+                results.extend(resp.json())
+            except HTTPError as e:
+                logger.debug(e.response.text)
+                if 400 <= e.response.status_code < 500:
+                    raise DataPathException(_http_error_message(e), e)
+                else:
+                    raise e
+
+        result = _ResultSet(self.path.uri, lambda ignore1, ignore2, ignore3: results)
+        return result
+
 
-    def update(self, entities, correlation={'RID'}, targets=None):
+    def update(self, entities, correlation={'RID'}, targets=None, retry_codes={408, 429, 500, 502, 503, 504}, backoff_factor=4, max_attempts=5, max_batch_rows=1000, max_batch_bytes=250*1024):
         """Update entities of a table.
 
         For more information see the ERMrest protocol for the `attributegroup` interface. By default, this method will
@@ -901,7 +1049,17 @@ class _TableWrapper (object):
         :param correlation: an iterable collection of column names used to correlate input set to the set of rows to be
         updated in the catalog. E.g., `{'col name'}` or `{mytable.mycolumn}` will work if you pass a _ColumnWrapper object.
         :param targets: an iterable collection of column names used as the targets of the update operation.
-        :return: a collection of updated entities as returned by the corresponding ERMrest interface.
+        :param retry_codes: set of HTTP status codes for which retry should be considered.
+        :param backoff_factor: number of seconds for base of exponential retry backoff.
+        :param max_attempts: maximum number of requests attempts with retry.
+        :param max_batch_rows: maximum number of rows for one request, or False to disable batching.
+        :param max_batch_bytes: approximate maximum number of bytes for one request.
+        :return a collection of newly created entities.
+
+        When performing retries, an exponential backoff delay is
+        introduced after each failed attempt. The delay is
+        backoff_factor**attempt_number seconds for attempts 0 through
+        max_attempts-1.
         """
         # empty entities will be accepted but results are therefore an empty entity set
         if not entities:
@@ -936,15 +1094,39 @@ class _TableWrapper (object):
             targets=','.join(target_cnames)
         )
 
-        try:
-            resp = self._schema._catalog._wrapped_catalog.put(path, json=entities, headers={'Content-Type': 'application/json'})
-            return _ResultSet(self.path.uri, lambda ignore1, ignore2, ignore3: resp.json())
-        except HTTPError as e:
-            logger.debug(e.response.text)
-            if 400 <= e.response.status_code < 500:
-                raise DataPathException(_http_error_message(e), e)
-            else:
-                raise e
+        # perform one batch request in a helper we can hand to retry helper
+        def request_func(batch):
+            return self._schema._catalog._wrapped_catalog.put(path, json=batch, headers={'Content-Type': 'application/json'})
+
+        # perform all requests synchronously so the caller can get exceptions
+        results = []
+        for batch in _generate_batches(
+            entities,
+            max_batch_rows=max_batch_rows,
+            max_batch_bytes=max_batch_bytes
+        ):
+            try:
+                resp = _request_with_retry(
+                    lambda: request_func(batch),
+                    retry_codes=retry_codes,
+                    backoff_factor=backoff_factor,
+                    max_attempts=max_attempts
+                )
+                results.extend(resp.json())
+            except HTTPError as e:
+                logger.debug(e.response.text)
+                if 400 <= e.response.status_code < 500:
+                    raise DataPathException(_http_error_message(e), e)
+                else:
+                    raise e
+
+        result = _ResultSet(self.path.uri, lambda ignore1, ignore2, ignore3: results)
+        return result
+
+    def delete(self):
+        """Deletes the entity set referenced by the Table.
+        """
+        self.path.delete()
 
 
 class _TableAlias (_TableWrapper):

{deriva-1.7.1 → deriva-1.7.4}/deriva/core/ermrest_catalog.py

@@ -53,11 +53,15 @@ class DerivaServer (DerivaBinding):
         """
         return ErmrestCatalog.connect(self, catalog_id, snaptime)
 
-    def create_ermrest_catalog(self, id=None, owner=None):
+    def create_ermrest_catalog(self, id=None, owner=None, name=None, description=None, is_persistent=None, clone_source=None):
         """Create an ERMrest catalog.
 
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
+        :param is_persistent: Initial (bool) catalog persistence flag if not None
+        :param clone_source: Initial catalog clone_source if not None
 
         The new catalog id will be returned in the response, and used
         in future catalog access. The use of the id parameter
@@ -77,8 +81,17 @@ class DerivaServer (DerivaBinding):
         owner ACL influences which client(s) are allowed to retry
         creation with the same id.
 
+        The name, description, is_persistent, and clone_source
+        parameters are passed through to the catalog creation service
+        to initialize those respective metadata fields of the new
+        catalog's registry entry. See ERMrest documentation for more
+        detail. Authorization failures may occur when attempting to
+        set the is_persistent flag. By default, these fields are not
+        initialized in the catalog creation request, and they instead
+        receive server-assigned defaults.
+
         """
-        return ErmrestCatalog.create(self, id, owner)
+        return ErmrestCatalog.create(self, id, owner, name, description, is_persistent, clone_source)
 
     def connect_ermrest_alias(self, id):
         """Connect to an ERMrest alias and return the alias binding.
@@ -88,12 +101,14 @@ class DerivaServer (DerivaBinding):
         """
         return ErmrestAlias.connect(self, id)
 
-    def create_ermrest_alias(self, id=None, owner=None, alias_target=None):
+    def create_ermrest_alias(self, id=None, owner=None, alias_target=None, name=None, description=None):
         """Create an ERMrest catalog alias.
 
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
         :param alias_target: The initial target catalog id binding desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
 
         The new alias id will be returned in the response, and used
         in future alias access. The use of the id parameter
@@ -118,8 +133,13 @@ class DerivaServer (DerivaBinding):
         influences which client(s) are allowed to retry creation with
         the same id.
 
+        The name and description parameters are passed through to the
+        alias creation service to initialize those respective metadata
+        fields of the new aliase's registry entry. See ERMrest
+        documentation for more detail.
+
         """
-        return ErmrestAlias.create(self, id, owner, alias_target)
+        return ErmrestAlias.create(self, id, owner, alias_target, name, description)
 
 class ErmrestCatalogMutationError(Exception):
     pass
@@ -204,15 +224,22 @@ class ErmrestCatalog(DerivaBinding):
         )
 
     @classmethod
-    def _digest_catalog_args(cls, id, owner):
+    def _digest_catalog_args(cls, id, owner, name=None, description=None, is_persistent=None, clone_source=None):
         rep = dict()
 
-        if isinstance(id, str):
-            rep['id'] = id
-        elif isinstance(id, (type(nochange), type(None))):
-            pass
-        else:
-            raise TypeError('id must be of type str or None or nochange, not %s' % type(id))
+        for v, k, typ in [
+                (id, 'id', str),
+                (name, 'name', str),
+                (description, 'description', str),
+                (is_persistent, 'is_persistent', bool),
+                (clone_source, 'clone_source', str),
+        ]:
+            if isinstance(v, typ):
+                rep[k] = v
+            elif isinstance(v, (type(nochange), type(None))):
+                pass
+            else:
+                raise TypeError('%s must be of type %s or None or nochange, not %s' % (k, typ.__name__, type(v)))
 
         if isinstance(owner, list):
             for e in owner:
@@ -227,12 +254,16 @@ class ErmrestCatalog(DerivaBinding):
         return rep
 
     @classmethod
-    def create(cls, deriva_server, id=None, owner=None):
+    def create(cls, deriva_server, id=None, owner=None, name=None, description=None, is_persistent=None, clone_source=None):
         """Create an ERMrest catalog and return the ERMrest catalog binding.
 
         :param deriva_server: The DerivaServer binding which hosts ermrest.
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
+        :param is_persistent: Initial (bool) catalog persistence flag if not None
+        :param clone_source: Initial catalog clone_source if not None
 
         The new catalog id will be returned in the response, and used
         in future catalog access. The use of the id parameter
@@ -252,9 +283,18 @@ class ErmrestCatalog(DerivaBinding):
         influences which client(s) are allowed to retry creation with
         the same id.
 
+        The name, description, is_persistent, and clone_source
+        parameters are passed through to the catalog creation service
+        to initialize those respective metadata fields of the new
+        catalog's registry entry. See ERMrest documentation for more
+        detail. Authorization failures may occur when attempting to
+        set the is_persistent flag. By default, these fields are not
+        initialized in the catalog creation request, and they instead
+        receive server-assigned defaults.
+
         """
         path = '/ermrest/catalog'
-        r = deriva_server.post(path, json=cls._digest_catalog_args(id, owner))
+        r = deriva_server.post(path, json=cls._digest_catalog_args(id, owner, name, description, is_persistent, clone_source))
         r.raise_for_status()
         return cls.connect(deriva_server, r.json()['id'])
 
@@ -655,7 +695,8 @@ class ErmrestCatalog(DerivaBinding):
                       copy_annotations=True,
                       copy_policy=True,
                       truncate_after=True,
-                      exclude_schemas=None):
+                      exclude_schemas=None,
+                      dst_properties=None):
         """Clone this catalog's content into dest_catalog, creating a new catalog if needed.
 
         :param dst_catalog: Destination catalog or None to request creation of new destination (default).
@@ -664,13 +705,22 @@ class ErmrestCatalog(DerivaBinding):
         :param copy_policy: Copy access-control policies when True (default).
         :param truncate_after: Truncate destination history after cloning when True (default).
         :param exclude_schemas: A list of schema names to exclude from the cloning process.
+        :param dst_properties: A dictionary of custom catalog-creation properties.
 
-        When dest_catalog is provided, attempt an idempotent clone,
+        When dst_catalog is provided, attempt an idempotent clone,
         assuming content MAY be partially cloned already using the
         same parameters. This routine uses a table-level annotation
         "tag:isrd.isi.edu,2018:clone-state" to save progress markers
         which help it restart efficiently if interrupted.
 
+        When dst_catalog is not provided, a new catalog is
+        provisioned. The optional dst_properties can customize
+        metadata properties during this step:
+
+        - name: str
+        - description: str (markdown-formatted)
+        - is_persistent: boolean
+
         Cloning preserves source row RID values for application tables
         so that any RID-based foreign keys are still valid. It is not
         generally advisable to try to merge more than one source into
@@ -692,10 +742,33 @@ class ErmrestCatalog(DerivaBinding):
         session_config["allow_retry_on_all_methods"] = True
 
         if dst_catalog is None:
-            # TODO: refactor with DerivaServer someday
-            server = DerivaBinding(self._scheme, self._server, self._credentials, self._caching, session_config)
-            dst_id = server.post("/ermrest/catalog").json()["id"]
-            dst_catalog = ErmrestCatalog(self._scheme, self._server, dst_id, self._credentials, self._caching, session_config)
+            if dst_properties is not None:
+                if not isinstance(dst_properties, dict):
+                    raise TypeError('dst_properties must be of type dict or None, not %s' % (type(dst_properties),))
+            else:
+                dst_properties = {}
+            kwargs = {
+                "name": dst_properties.get('name', 'Clone of %r' % (self._catalog_id,)),
+                "description": dst_properties.get(
+                    'description',
+                    '''A cloned copy of catalog %r made with ErmrestCatalog.clone_catalog() using the following parameters:
+- `copy_data`: %r
+- `copy_annotations`: %r
+- `copy_policy`: %r
+- `truncate_after`: %r
+- `exclude_schemas`: %r
+''' % (
+    self._catalog_id,
+    copy_data,
+    copy_annotations,
+    copy_policy,
+    truncate_after,
+    exclude_schemas,
+)),
+                "clone_source": dst_properties.get('clone_source', self._catalog_id),
+            }
+            server = self.deriva_server
+            dst_catalog = server.create_ermrest_catalog(**kwargs)
 
         # set top-level config right away and find fatal usage errors...
         if copy_policy:
@@ -1051,8 +1124,8 @@ class ErmrestAlias(DerivaBinding):
         )
 
     @classmethod
-    def _digest_alias_args(cls, id, owner, alias_target):
-        rep = ErmrestCatalog._digest_catalog_args(id, owner)
+    def _digest_alias_args(cls, id, owner, alias_target, name, description):
+        rep = ErmrestCatalog._digest_catalog_args(id, owner, name, description)
 
         if isinstance(alias_target, (str, type(None))):
             rep['alias_target'] = alias_target
@@ -1064,13 +1137,15 @@ class ErmrestAlias(DerivaBinding):
         return rep
 
     @classmethod
-    def create(cls, deriva_server, id=None, owner=None, alias_target=None):
+    def create(cls, deriva_server, id=None, owner=None, alias_target=None, name=None, description=None):
         """Create an ERMrest catalog alias.
 
         :param deriva_server: The DerivaServer binding which hosts ermrest
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
         :param alias_target: The initial target catalog id desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
 
         The new alias id will be returned in the response, and used
         in future alias access. The use of the id parameter
@@ -1095,9 +1170,14 @@ class ErmrestAlias(DerivaBinding):
         influences which client(s) are allowed to retry creation with
         the same id.
 
+        The name and description parameters are passed through to the
+        alias creation service to initialize those respective metadata
+        fields of the new aliase's registry entry. See ERMrest
+        documentation for more detail.
+
         """
         path = '/ermrest/alias'
-        r = deriva_server.post(path, json=cls._digest_alias_args(id, owner, alias_target))
+        r = deriva_server.post(path, json=cls._digest_alias_args(id, owner, alias_target, name, description))
         r.raise_for_status()
         return cls.connect(deriva_server, r.json()['id'])