deriva 1.7.1__py3-none-any.whl → 1.7.3__py3-none-any.whl

deriva/config/annotation_config.py

@@ -33,7 +33,7 @@ class AttrSpecList(BaseSpecList):
             return None
         new = []
         for item in orig_list:
-            new.append(unicode(item))
+            new.append(item)
         return new
 
     def add_list(self, dictlist):
@@ -85,7 +85,7 @@ class AttrConfig:
         self.toplevel_config = ConfigUtil.find_toplevel_node(self.catalog.getCatalogModel(), schema_name, table_name)
 
     def make_speclist(self, name):
-        d = self.config.get(unicode(name))
+        d = self.config.get(name)
         if d is None:
             d = [dict()]
         return AttrSpecList(self.known_attrs, d)

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.1"
+__version__ = "1.7.3"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/core/datapath.py

@@ -5,6 +5,7 @@ import copy
 from datetime import date
 import itertools
 import logging
+import time
 import re
 from requests import HTTPError
 import warnings
@@ -695,6 +696,102 @@ class _ResultSet (object):
         logger.debug("Fetched %d entities" % len(self._results_doc))
         return self
 
+def _json_size_approx(data):
+    """Return approximate byte count for minimal JSON encoding of data
+
+    Minimal encoding has no optional whitespace/indentation.
+    """
+    nbytes = 0
+
+    if isinstance(data, (list, tuple)):
+        nbytes += 2
+        for elem in data:
+            nbytes += _json_size_approx(elem) + 1
+    elif isinstance(data, dict):
+        nbytes += 2
+        for k, v in data.items():
+            nbytes += _json_size_approx(k) + _json_size_approx(v) + 2
+    elif isinstance(data, str):
+        nbytes += len(data.encode("utf-8")) + 2
+    else:
+        nbytes += len(str(data))
+
+    return nbytes
+
+def _generate_batches(entities, max_batch_rows=1000, max_batch_bytes=250*1024):
+    """Generate a series of entity batches as slices of the input entities
+
+    """
+    if not isinstance(entities, (list, tuple)):
+        raise TypeError('invalid type %s for entities, list or tuple expected' % (type(entities),))
+
+    if not max_batch_rows:
+        logger.debug("disabling batching due to max_batch_rows=%r" % (max_batch_rows,))
+        return entities
+
+    top = len(entities)
+    lower = 0
+
+    while lower < top:
+        # to ensure progress, always use at least one row per batch regardless of nbytes
+        upper = lower + 1
+        batch_nbytes = _json_size_approx(entities[lower])
+
+        # advance upper position until a batch size limit is reached
+        while (upper - lower) < max_batch_rows:
+            if upper >= top:
+                break
+            batch_nbytes += _json_size_approx(entities[upper])
+            if batch_nbytes > max_batch_bytes:
+                break
+            upper += 1
+
+        # generate one batch and advance for next batch
+        logger.debug("yielding batch of %d/%d entities (%d:%d)" % (upper-lower, top, lower, upper))
+        yield entities[lower:upper]
+        lower = upper
+
+def _request_with_retry(request_func, retry_codes={408, 429, 500, 502, 503, 504}, backoff_factor=4, max_attempts=5):
+    """Perform request func with exponential backoff and retry.
+
+    :param request_func: A function returning a requests.Response object or raising HTTPError
+    :param retry_codes: HTTPError status codes on which to attempt retry
+    :param backoff_factor: Base number of seconds for factor**attempt exponential backoff
+    :param max_attempts: Max number of request attempts.
+
+    Retry will be attempted on HTTPError exceptions which match retry_codes and
+    also on other unknown exceptions, presumed to be transport errors.
+
+    The request_func should do the equivalent of resp.raise_on_status() so that
+    it only returns a response object for successful requests.
+    """
+    attempt = 0
+    last_ex = None
+
+    while attempt < max_attempts:
+        try:
+            if attempt > 0:
+                delay = backoff_factor**(attempt-1)
+                logger.debug("sleeping %d seconds before retry %d..." % (delay, attempt))
+                time.sleep(delay)
+            attempt += 1
+            return request_func()
+        except HTTPError as e:
+            logger.debug(e.response.text)
+            last_ex = e
+            if 400 <= e.response.status_code < 500:
+                last_ex = DataPathException(_http_error_message(e), e)
+            if int(e.response.status_code) not in retry_codes:
+                raise last_ex
+        except Exception as e:
+            logger.debug(e.response.text)
+            last_ex = e
+
+    # early return means we don't get here on successful requests
+    logger.warning("maximum request retry limit %d exceeded" % (max_attempts,))
+    if last_ex is None:
+        raise ValueError('exceeded max_attempts without catching a request exception')
+    raise last_ex
 
 class _TableWrapper (object):
     """Wraps a Table for datapath expressions.
@@ -836,7 +933,7 @@ class _TableWrapper (object):
         """
         return self.path.denormalize(context_name=context_name, heuristic=heuristic, groupkey_name=groupkey_name)
 
-    def insert(self, entities, defaults=set(), nondefaults=set(), add_system_defaults=True, on_conflict_skip=False):
+    def insert(self, entities, defaults=set(), nondefaults=set(), add_system_defaults=True, on_conflict_skip=False, retry_codes={408, 429, 500, 502, 503, 504}, backoff_factor=4, max_attempts=5, max_batch_rows=1000, max_batch_bytes=250*1024):
         """Inserts entities into the table.
 
         :param entities: an iterable collection of entities (i.e., rows) to be inserted into the table.
@@ -844,7 +941,23 @@ class _TableWrapper (object):
         :param nondefaults: optional, set of columns names to override implicit system defaults
         :param add_system_defaults: flag to add system columns to the set of default columns.
         :param on_conflict_skip: flag to skip entities that violate uniqueness constraints.
+        :param retry_codes: set of HTTP status codes for which retry should be considered.
+        :param backoff_factor: number of seconds for base of exponential retry backoff.
+        :param max_attempts: maximum number of requests attempts with retry.
+        :param max_batch_rows: maximum number of rows for one request, or False to disable batching.
+        :param max_batch_bytes: approximate maximum number of bytes for one request.
         :return a collection of newly created entities.
+
+        Retry will only be attempted for idempotent insertion
+        requests, which are when a user-controlled, non-nullable key
+        is present in the table and the key's constituent column(s)
+        are not listed as defaults, and on_conflict_skip=True.
+
+        When performing retries, an exponential backoff delay is
+        introduced after each failed attempt. The delay is
+        backoff_factor**attempt_number seconds for attempts 0 through
+        max_attempts-1.
+
         """
         # empty entities will be accepted but results are therefore an empty entity set
         if not entities:
@@ -879,17 +992,55 @@ class _TableWrapper (object):
         if not hasattr(entities[0], 'keys'):
             raise TypeError('entities[0] does not look like a dictionary -- does not have a "keys()" method')
 
-        try:
-            resp = self._schema._catalog._wrapped_catalog.post(path, json=entities, headers={'Content-Type': 'application/json'})
-            return _ResultSet(self.path.uri, lambda ignore1, ignore2, ignore3: resp.json())
-        except HTTPError as e:
-            logger.debug(e.response.text)
-            if 400 <= e.response.status_code < 500:
-                raise DataPathException(_http_error_message(e), e)
-            else:
-                raise e
-
-    def update(self, entities, correlation={'RID'}, targets=None):
+        # perform one batch request in a helper we can hand to retry helper
+        def request_func(batch):
+            return self._schema._catalog._wrapped_catalog.post(path, json=batch, headers={'Content-Type': 'application/json'})
+
+        def _has_user_pkey(table):
+            """Return True if table has at least one primary key other than the system RID key"""
+            for key in table.keys:
+                if { c.name for c in key.unique_columns } != {'RID'}:
+                    if all([ not c.nullok for c in key.unique_columns ]) \
+                       and all([ c.name not in defaults for c in key.unique_columns ]):
+                        return True
+            return False
+
+        # determine whether insert is idempotent and therefore retry safe
+        retry_safe = on_conflict_skip and _has_user_pkey(self._wrapped_table)
+
+        # perform all requests in a helper we can hand to _ResultSet
+        def results_func(ignore1, ignore2, ignore3):
+            results = []
+            for batch in _generate_batches(
+                entities,
+                max_batch_rows=max_batch_rows,
+                max_batch_bytes=max_batch_bytes
+            ):
+                try:
+                    if retry_safe:
+                        resp = _request_with_retry(
+                            lambda: request_func(batch),
+                            retry_codes=retry_codes,
+                            backoff_factor=backoff_factor,
+                            max_attempts=max_attempts
+                        )
+                    else:
+                        resp = request_func(batch)
+                    results.extend(resp.json())
+                except HTTPError as e:
+                    logger.debug(e.response.text)
+                    if 400 <= e.response.status_code < 500:
+                        raise DataPathException(_http_error_message(e), e)
+                    else:
+                        raise e
+            return results
+
+        result = _ResultSet(self.path.uri, results_func)
+        result.fetch()
+        return result
+
+
+    def update(self, entities, correlation={'RID'}, targets=None, retry_codes={408, 429, 500, 502, 503, 504}, backoff_factor=4, max_attempts=5, max_batch_rows=1000, max_batch_bytes=250*1024):
         """Update entities of a table.
 
         For more information see the ERMrest protocol for the `attributegroup` interface. By default, this method will
@@ -901,7 +1052,17 @@ class _TableWrapper (object):
         :param correlation: an iterable collection of column names used to correlate input set to the set of rows to be
         updated in the catalog. E.g., `{'col name'}` or `{mytable.mycolumn}` will work if you pass a _ColumnWrapper object.
         :param targets: an iterable collection of column names used as the targets of the update operation.
-        :return: a collection of updated entities as returned by the corresponding ERMrest interface.
+        :param retry_codes: set of HTTP status codes for which retry should be considered.
+        :param backoff_factor: number of seconds for base of exponential retry backoff.
+        :param max_attempts: maximum number of requests attempts with retry.
+        :param max_batch_rows: maximum number of rows for one request, or False to disable batching.
+        :param max_batch_bytes: approximate maximum number of bytes for one request.
+        :return a collection of newly created entities.
+
+        When performing retries, an exponential backoff delay is
+        introduced after each failed attempt. The delay is
+        backoff_factor**attempt_number seconds for attempts 0 through
+        max_attempts-1.
         """
         # empty entities will be accepted but results are therefore an empty entity set
         if not entities:
@@ -936,16 +1097,37 @@ class _TableWrapper (object):
             targets=','.join(target_cnames)
         )
 
-        try:
-            resp = self._schema._catalog._wrapped_catalog.put(path, json=entities, headers={'Content-Type': 'application/json'})
-            return _ResultSet(self.path.uri, lambda ignore1, ignore2, ignore3: resp.json())
-        except HTTPError as e:
-            logger.debug(e.response.text)
-            if 400 <= e.response.status_code < 500:
-                raise DataPathException(_http_error_message(e), e)
-            else:
-                raise e
-
+        # perform one batch request in a helper we can hand to retry helper
+        def request_func(batch):
+            return self._schema._catalog._wrapped_catalog.put(path, json=batch, headers={'Content-Type': 'application/json'})
+
+        # perform all requests in a helper we can hand to _ResultSet
+        def results_func(ignore1, ignore2, ignore3):
+            results = []
+            for batch in _generate_batches(
+                entities,
+                max_batch_rows=max_batch_rows,
+                max_batch_bytes=max_batch_bytes
+            ):
+                try:
+                    resp = _request_with_retry(
+                        lambda: request_func(batch),
+                        retry_codes=retry_codes,
+                        backoff_factor=backoff_factor,
+                        max_attempts=max_attempts
+                    )
+                    results.extend(resp.json())
+                except HTTPError as e:
+                    logger.debug(e.response.text)
+                    if 400 <= e.response.status_code < 500:
+                        raise DataPathException(_http_error_message(e), e)
+                    else:
+                        raise e
+            return results
+
+        result = _ResultSet(self.path.uri, results_func)
+        result.fetch()
+        return result
 
 class _TableAlias (_TableWrapper):
     """Represents a table alias in datapath expressions.

deriva/core/ermrest_catalog.py

@@ -53,11 +53,15 @@ class DerivaServer (DerivaBinding):
         """
         return ErmrestCatalog.connect(self, catalog_id, snaptime)
 
-    def create_ermrest_catalog(self, id=None, owner=None):
+    def create_ermrest_catalog(self, id=None, owner=None, name=None, description=None, is_persistent=None, clone_source=None):
         """Create an ERMrest catalog.
 
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
+        :param is_persistent: Initial (bool) catalog persistence flag if not None
+        :param clone_source: Initial catalog clone_source if not None
 
         The new catalog id will be returned in the response, and used
         in future catalog access. The use of the id parameter
@@ -77,8 +81,17 @@ class DerivaServer (DerivaBinding):
         owner ACL influences which client(s) are allowed to retry
         creation with the same id.
 
+        The name, description, is_persistent, and clone_source
+        parameters are passed through to the catalog creation service
+        to initialize those respective metadata fields of the new
+        catalog's registry entry. See ERMrest documentation for more
+        detail. Authorization failures may occur when attempting to
+        set the is_persistent flag. By default, these fields are not
+        initialized in the catalog creation request, and they instead
+        receive server-assigned defaults.
+
         """
-        return ErmrestCatalog.create(self, id, owner)
+        return ErmrestCatalog.create(self, id, owner, name, description, is_persistent, clone_source)
 
     def connect_ermrest_alias(self, id):
         """Connect to an ERMrest alias and return the alias binding.
@@ -88,12 +101,14 @@ class DerivaServer (DerivaBinding):
         """
         return ErmrestAlias.connect(self, id)
 
-    def create_ermrest_alias(self, id=None, owner=None, alias_target=None):
+    def create_ermrest_alias(self, id=None, owner=None, alias_target=None, name=None, description=None):
         """Create an ERMrest catalog alias.
 
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
         :param alias_target: The initial target catalog id binding desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
 
         The new alias id will be returned in the response, and used
         in future alias access. The use of the id parameter
@@ -118,8 +133,13 @@ class DerivaServer (DerivaBinding):
         influences which client(s) are allowed to retry creation with
         the same id.
 
+        The name and description parameters are passed through to the
+        alias creation service to initialize those respective metadata
+        fields of the new aliase's registry entry. See ERMrest
+        documentation for more detail.
+
         """
-        return ErmrestAlias.create(self, id, owner, alias_target)
+        return ErmrestAlias.create(self, id, owner, alias_target, name, description)
 
 class ErmrestCatalogMutationError(Exception):
     pass
@@ -204,15 +224,22 @@ class ErmrestCatalog(DerivaBinding):
         )
 
     @classmethod
-    def _digest_catalog_args(cls, id, owner):
+    def _digest_catalog_args(cls, id, owner, name=None, description=None, is_persistent=None, clone_source=None):
         rep = dict()
 
-        if isinstance(id, str):
-            rep['id'] = id
-        elif isinstance(id, (type(nochange), type(None))):
-            pass
-        else:
-            raise TypeError('id must be of type str or None or nochange, not %s' % type(id))
+        for v, k, typ in [
+                (id, 'id', str),
+                (name, 'name', str),
+                (description, 'description', str),
+                (is_persistent, 'is_persistent', bool),
+                (clone_source, 'clone_source', str),
+        ]:
+            if isinstance(v, typ):
+                rep[k] = v
+            elif isinstance(v, (type(nochange), type(None))):
+                pass
+            else:
+                raise TypeError('%s must be of type %s or None or nochange, not %s' % (k, typ.__name__, type(v)))
 
         if isinstance(owner, list):
             for e in owner:
@@ -227,12 +254,16 @@ class ErmrestCatalog(DerivaBinding):
         return rep
 
     @classmethod
-    def create(cls, deriva_server, id=None, owner=None):
+    def create(cls, deriva_server, id=None, owner=None, name=None, description=None, is_persistent=None, clone_source=None):
         """Create an ERMrest catalog and return the ERMrest catalog binding.
 
         :param deriva_server: The DerivaServer binding which hosts ermrest.
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
+        :param is_persistent: Initial (bool) catalog persistence flag if not None
+        :param clone_source: Initial catalog clone_source if not None
 
         The new catalog id will be returned in the response, and used
         in future catalog access. The use of the id parameter
@@ -252,9 +283,18 @@ class ErmrestCatalog(DerivaBinding):
         influences which client(s) are allowed to retry creation with
         the same id.
 
+        The name, description, is_persistent, and clone_source
+        parameters are passed through to the catalog creation service
+        to initialize those respective metadata fields of the new
+        catalog's registry entry. See ERMrest documentation for more
+        detail. Authorization failures may occur when attempting to
+        set the is_persistent flag. By default, these fields are not
+        initialized in the catalog creation request, and they instead
+        receive server-assigned defaults.
+
         """
         path = '/ermrest/catalog'
-        r = deriva_server.post(path, json=cls._digest_catalog_args(id, owner))
+        r = deriva_server.post(path, json=cls._digest_catalog_args(id, owner, name, description, is_persistent, clone_source))
         r.raise_for_status()
         return cls.connect(deriva_server, r.json()['id'])
 
@@ -655,7 +695,8 @@ class ErmrestCatalog(DerivaBinding):
                       copy_annotations=True,
                       copy_policy=True,
                       truncate_after=True,
-                      exclude_schemas=None):
+                      exclude_schemas=None,
+                      dst_properties=None):
         """Clone this catalog's content into dest_catalog, creating a new catalog if needed.
 
         :param dst_catalog: Destination catalog or None to request creation of new destination (default).
@@ -664,13 +705,22 @@ class ErmrestCatalog(DerivaBinding):
         :param copy_policy: Copy access-control policies when True (default).
         :param truncate_after: Truncate destination history after cloning when True (default).
         :param exclude_schemas: A list of schema names to exclude from the cloning process.
+        :param dst_properties: A dictionary of custom catalog-creation properties.
 
-        When dest_catalog is provided, attempt an idempotent clone,
+        When dst_catalog is provided, attempt an idempotent clone,
         assuming content MAY be partially cloned already using the
         same parameters. This routine uses a table-level annotation
         "tag:isrd.isi.edu,2018:clone-state" to save progress markers
         which help it restart efficiently if interrupted.
 
+        When dst_catalog is not provided, a new catalog is
+        provisioned. The optional dst_properties can customize
+        metadata properties during this step:
+
+        - name: str
+        - description: str (markdown-formatted)
+        - is_persistent: boolean
+
         Cloning preserves source row RID values for application tables
         so that any RID-based foreign keys are still valid. It is not
         generally advisable to try to merge more than one source into
@@ -692,10 +742,33 @@ class ErmrestCatalog(DerivaBinding):
         session_config["allow_retry_on_all_methods"] = True
 
         if dst_catalog is None:
-            # TODO: refactor with DerivaServer someday
-            server = DerivaBinding(self._scheme, self._server, self._credentials, self._caching, session_config)
-            dst_id = server.post("/ermrest/catalog").json()["id"]
-            dst_catalog = ErmrestCatalog(self._scheme, self._server, dst_id, self._credentials, self._caching, session_config)
+            if dst_properties is not None:
+                if not isinstance(dst_properties, dict):
+                    raise TypeError('dst_properties must be of type dict or None, not %s' % (type(dst_properties),))
+            else:
+                dst_properties = {}
+            kwargs = {
+                "name": dst_properties.get('name', 'Clone of %r' % (self._catalog_id,)),
+                "description": dst_properties.get(
+                    'description',
+                    '''A cloned copy of catalog %r made with ErmrestCatalog.clone_catalog() using the following parameters:
+- `copy_data`: %r
+- `copy_annotations`: %r
+- `copy_policy`: %r
+- `truncate_after`: %r
+- `exclude_schemas`: %r
+''' % (
+    self._catalog_id,
+    copy_data,
+    copy_annotations,
+    copy_policy,
+    truncate_after,
+    exclude_schemas,
+)),
+                "clone_source": dst_properties.get('clone_source', self._catalog_id),
+            }
+            server = self.deriva_server
+            dst_catalog = server.create_ermrest_catalog(**kwargs)
 
         # set top-level config right away and find fatal usage errors...
         if copy_policy:
@@ -1051,8 +1124,8 @@ class ErmrestAlias(DerivaBinding):
         )
 
     @classmethod
-    def _digest_alias_args(cls, id, owner, alias_target):
-        rep = ErmrestCatalog._digest_catalog_args(id, owner)
+    def _digest_alias_args(cls, id, owner, alias_target, name, description):
+        rep = ErmrestCatalog._digest_catalog_args(id, owner, name, description)
 
         if isinstance(alias_target, (str, type(None))):
             rep['alias_target'] = alias_target
@@ -1064,13 +1137,15 @@ class ErmrestAlias(DerivaBinding):
         return rep
 
     @classmethod
-    def create(cls, deriva_server, id=None, owner=None, alias_target=None):
+    def create(cls, deriva_server, id=None, owner=None, alias_target=None, name=None, description=None):
         """Create an ERMrest catalog alias.
 
         :param deriva_server: The DerivaServer binding which hosts ermrest
         :param id: The (str) id desired by the client (default None)
         :param owner: The initial (list of str) ACL desired by the client (default None)
         :param alias_target: The initial target catalog id desired by the client (default None)
+        :param name: Initial (str) catalog name if not None
+        :param description: Initial (str) catalog description if not None
 
         The new alias id will be returned in the response, and used
         in future alias access. The use of the id parameter
@@ -1095,9 +1170,14 @@ class ErmrestAlias(DerivaBinding):
         influences which client(s) are allowed to retry creation with
         the same id.
 
+        The name and description parameters are passed through to the
+        alias creation service to initialize those respective metadata
+        fields of the new aliase's registry entry. See ERMrest
+        documentation for more detail.
+
         """
         path = '/ermrest/alias'
-        r = deriva_server.post(path, json=cls._digest_alias_args(id, owner, alias_target))
+        r = deriva_server.post(path, json=cls._digest_alias_args(id, owner, alias_target, name, description))
         r.raise_for_status()
         return cls.connect(deriva_server, r.json()['id'])
 

deriva/core/ermrest_model.py

@@ -1,7 +1,13 @@
 
+from __future__ import annotations
+
 from collections import OrderedDict
+from collections.abc import Iterable
+from enum import Enum
 import json
 import re
+import base64
+import hashlib
 
 from . import AttrDict, tag, urlquote, stob
 
@@ -17,6 +23,74 @@ class NoChange (object):
 # singletone to use in APIs below
 nochange = NoChange()
 
+def make_id(*components):
+    """Build an identifier that will be OK for ERMrest and Postgres.
+
+    Naively, append as '_'.join(components).
+
+    Fallback to heuristics mixing truncation with short hashes.
+    """
+    # accept lists at top-level for convenience (compound keys, etc.)
+    expanded = []
+    for e in components:
+        if isinstance(e, list):
+            expanded.extend(e)
+        else:
+            expanded.append(e)
+
+    # prefer to use naive name as requested
+    naive_result = '_'.join(expanded)
+    naive_len = len(naive_result.encode('utf8'))
+    if naive_len <= 63:
+        return naive_result
+
+    # we'll need to truncate and hash in some way...
+    def hash(s, nbytes):
+        return base64.urlsafe_b64encode(hashlib.md5(s.encode('utf8')).digest()).decode()[0:nbytes]
+
+    def truncate(s, maxlen):
+        encoded_len = len(s.encode('utf8'))
+        # we need to chop whole (unicode) chars but test encoded byte lengths!
+        for i in range(max(1, len(s) - maxlen), len(s) - 1):
+            result = s[0:-1 * i].rstrip()
+            if len(result.encode('utf8')) <= (maxlen - 2):
+                return result + '..'
+        return s
+
+    naive_hash = hash(naive_result, 5)
+    parts = [
+        (i, expanded[i])
+        for i in range(len(expanded))
+    ]
+
+    # try to find a solution truncating individual fields
+    for maxlen in [15, 12, 9]:
+        parts.sort(key=lambda p: (len(p[1].encode('utf8')), p[0]), reverse=True)
+        for i in range(len(parts)):
+            idx, part = parts[i]
+            if len(part.encode('utf8')) > maxlen:
+                parts[i] = (idx, truncate(part, maxlen))
+            candidate_result = '_'.join([
+                p[1]
+                for p in sorted(parts, key=lambda p: p[0])
+            ] + [naive_hash])
+            if len(candidate_result.encode('utf8')) < 63:
+                return candidate_result
+
+    # fallback to truncating original naive name
+    # try to preserve suffix and trim in middle
+    result = ''.join([
+        truncate(naive_result, len(naive_result)//3),
+        naive_result[-len(naive_result)//3:],
+        '_',
+        naive_hash
+    ])
+    if len(result.encode('utf8')) <= 63:
+        return result
+
+    # last-ditch (e.g. multibyte unicode suffix worst case)
+    return truncate(naive_result, 55) + naive_hash
+
 def presence_annotation(tag_uri):
     """Decorator to establish property getter/setter/deleter for presence annotations.
 
@@ -684,6 +758,15 @@ class KeyedList (list):
         list.append(self, e)
         self.elements[e.name] = e
 
+class FindAssociationResult (object):
+    """Wrapper for results of Table.find_associations()"""
+    def __init__(self, table, self_fkey, other_fkeys):
+        self.table = table
+        self.name = table.name
+        self.schema = table.schema
+        self.self_fkey = self_fkey
+        self.other_fkeys = other_fkeys
+    
 class Table (object):
     """Named table.
     """
@@ -1075,6 +1158,184 @@ class Table (object):
             provide_system
         )
 
+    @classmethod
+    def define_association(
+        cls,
+        associates: Iterable[Key | Table | tuple[str, Key | Table]],
+        metadata: Iterable[Key | Table | dict | tuple[str, bool, Key | Table]] = [],
+        table_name: str | None = None,
+        comment: str | None = None,
+        provide_system: bool = True) -> dict:
+        """Build an association table definition.
+
+        :param associates: the existing Key instances being associated
+        :param metadata: additional metadata fields for impure associations
+        :param table_name: name for the association table or None for default naming
+        :param comment: comment for the association table or None for default comment
+        :param provide_system: add ERMrest system columns when True
+
+        This is a utility function to help build an association table
+        definition. It simplifies the task, but removes some
+        control. For full customization, consider using Table.define()
+        directly instead.
+
+        A normal ("pure") N-ary association is a table with N foreign
+        keys referencing N primary keys in referenced tables, with a
+        composite primary key covering the N foreign keys. These pure
+        association tables manage a set of distinct combinations of
+        the associated foreign key values.
+
+        An "impure" association table adds additional metadata
+        alongside the N foreign keys.
+
+        The "associates" parameter takes an iterable of Key instances
+        from other tables.  The association will be comprised of
+        foreign keys referencing these associates. Optionally, a tuple
+        of (str, Key) can supply a string _base name_ to influence how
+        the foreign key columns and constraint will be named in the
+        new association table. A bare Key instance will get a base
+        name derived from the referenced table name.
+
+        The "metadata" parameter takes an iterable of plain dict
+        column definitions or Key instances. Each dict must be a
+        scalar column definition, such as produced by the
+        Column.define() class method. Key instance will cause
+        corresponding columns and foreign keys to be added to the
+        association table to act as metadata. Optionally, a tuple of
+        (str, bool, Key) can supply a string _base name_ and a boolean
+        _nullok_ property to influence how the foreign key columns and
+        constraint will be constructed and named. A bare Key instance
+        will get a base name derived from the referened table name,
+        and presumed as nullok=False.
+
+        If a Table instance is supplied instead of a Key instance for
+        associates or metadata inputs, an attempt will be made to
+        locate a key based on the RID system column. If this key
+        cannot be found, a KeyError will be raised.
+
+        """
+        associates = list(associates)
+        metadata = list(metadata)
+
+        if len(associates) < 2:
+            raise ValueError('An association table requires at least 2 associates')
+
+        cdefs = []
+        kdefs = []
+        fkdefs = []
+
+        used_names = set()
+
+        def check_basename(basename):
+            if not isinstance(base_name, str):
+                raise TypeError('Base name %r is not of required type str' % (base_name,))
+            if base_name in used_names:
+                raise ValueError('Base name %r is not unique among associates and metadata' % (base_name,))
+            used_names.add(base_name)
+
+        def choose_basename(key):
+            base_name = key.table.name
+            n = 2
+            while base_name in used_names:
+                base_name = '%s%d' % (key.table.name, n)
+                n += 1
+            used_names.add(base_name)
+            return base_name
+
+        def check_key(key):
+            if isinstance(key, Table):
+                return key.key_by_columns(["RID"])
+            return key
+
+        # check and normalize associates into list[(str, Key)] with distinct base names
+        for i in range(len(associates)):
+            if isinstance(associates[i], tuple):
+                base_name, key = associates[i]
+                check_basename(base_name)
+                key = check_key(key)
+                associates[i] = (base_name, key)
+            else:
+                key = check_key(associates[i])
+                base_name = choose_basename(key)
+                associates[i] = (base_name, key)
+
+        # build assoc table name if not provided
+        if table_name is None:
+            table_name = make_id(*[ assoc[1].table.name for assoc in associates ])
+
+        def simplify_type(ctype):
+            if ctype.is_domain and ctype.typename.startswith('ermrest_'):
+                return ctype.base_type
+
+            return ctype
+
+        def cdefs_for_key(base_name, key, nullok=False):
+            return [
+                Column.define(
+                    '%s_%s' % (base_name, col.name) if len(key.unique_columns) > 1 else base_name,
+                    simplify_type(col.type),
+                    nullok=nullok,
+                )
+                for col in key.unique_columns
+            ]
+
+        def fkdef_for_key(base_name, key):
+            return ForeignKey.define(
+                [
+                    '%s_%s' % (base_name, col.name) if len(key.unique_columns) > 1 else base_name
+                    for col in key.unique_columns
+                ],
+                key.table.schema.name,
+                key.table.name,
+                [ col.name for col in key.unique_columns ],
+                on_update='CASCADE',
+                on_delete='CASCADE',
+                constraint_name=make_id(table_name, base_name, 'fkey'),
+            )
+
+        # build core association definition (i.e. the "pure" parts)
+        k_cnames = []
+        for base_name, key in associates:
+            cdefs.extend(cdefs_for_key(base_name, key))
+            fkdefs.append(fkdef_for_key(base_name, key))
+
+            k_cnames.extend([
+                '%s_%s' % (base_name, col.name) if len(key.unique_columns) > 1 else base_name
+                for col in key.unique_columns
+            ])
+
+        kdefs.append(
+            Key.define(
+                k_cnames,
+                constraint_name=make_id(table_name, 'assoc', 'key'),
+            )
+        )
+
+        # check and normalize metadata into list[dict | (str, bool, Key)]
+        for i in range(len(metadata)):
+            if isinstance(metadata[i], tuple):
+                base_name, nullok, key = metadata[i]
+                check_basename(base_name)
+                key = check_key(key)
+                metadata[i] = (base_name, nullok, key)
+            elif isinstance(metadata[i], dict):
+                pass
+            else:
+                key = check_key(metadata[i])
+                base_name = choose_basename(key)
+                metadata[i] = (base_name, False, key)
+
+        # add metadata to definition
+        for md in metadata:
+            if isinstance(md, dict):
+                cdefs.append(md)
+            else:
+                base_name, nullok, key = md
+                cdefs.extend(cdefs_for_key(base_name, key, nullok))
+                fkdefs.append(fkdef_for_key(base_name, key))
+
+        return Table.define(table_name, cdefs, kdefs, fkdefs, comment=comment, provide_system=provide_system)
+
     def prejson(self, prune=True):
         return {
             "schema_name": self.schema.name,
@@ -1348,7 +1609,7 @@ class Table (object):
         if raise_nomatch:
             raise KeyError(from_to_map)
 
-    def is_association(self, min_arity=2, max_arity=2, unqualified=True, pure=True, no_overlap=True):
+    def is_association(self, min_arity=2, max_arity=2, unqualified=True, pure=True, no_overlap=True, return_fkeys=False):
         """Return (truthy) integer arity if self is a matching association, else False.
 
         min_arity: minimum number of associated fkeys (default 2)
@@ -1356,6 +1617,7 @@ class Table (object):
         unqualified: reject qualified associations when True (default True)
         pure: reject impure assocations when True (default True)
         no_overlap: reject overlapping associations when True (default True)
+        return_fkeys: return the set of N associated ForeignKeys if True
 
         The default behavior with no arguments is to test for pure,
         unqualified, non-overlapping, binary assocations.
@@ -1444,9 +1706,43 @@ class Table (object):
             # reject: impure association
             return False
 
-        # return (truthy) arity
-        return len(covered_fkeys)
+        # return (truthy) arity or fkeys
+        if return_fkeys:
+            return covered_fkeys
+        else:
+            return len(covered_fkeys)
+
+    def find_associations(self, min_arity=2, max_arity=2, unqualified=True, pure=True, no_overlap=True) -> Iterable[FindAssociationResult]:
+        """Yield (iterable) Association objects linking to this table and meeting all criteria.
 
+        min_arity: minimum number of associated fkeys (default 2)
+        max_arity: maximum number of associated fkeys (default 2) or None
+        unqualified: reject qualified associations when True (default True)
+        pure: reject impure assocations when True (default True)
+        no_overlap: reject overlapping associations when True (default True)
+
+        See documentation for sibling method Table.is_association(...)
+        for more explanation of these association detection criteria.
+
+        """
+        peer_tables = set()
+        for fkey in self.referenced_by:
+            peer = fkey.table
+            if peer in peer_tables:
+                # check each peer only once
+                continue
+            peer_tables.add(peer)
+            answer = peer.is_association(min_arity=min_arity, max_arity=max_arity, unqualified=unqualified, pure=pure, no_overlap=no_overlap, return_fkeys=True)
+            if answer:
+                answer = set(answer)
+                for fkey in answer:
+                    if fkey.pk_table == self:
+                        answer.remove(fkey)
+                        yield FindAssociationResult(peer, fkey, answer)
+                        # arbitrarily choose first fkey to self
+                        # in case association is back to same table
+                        break
+    
     @presence_annotation(tag.immutable)
     def immutable(self): pass
 
@@ -1495,6 +1791,40 @@ class Table (object):
     @object_annotation(tag.viz_3d_display)
     def viz_3d_display(self): pass
     
+class Quantifier (str, Enum):
+    """Logic quantifiers"""
+    any = 'any'
+    all = 'all'
+
+def find_tables_with_foreign_keys(target_tables: Iterable[Table], quantifier: Quantifier=Quantifier.all) -> set[Table]:
+    """Return set of tables with foreign key references to target tables.
+
+    :param target_tables: an iterable of ermrest_model.Table instances
+    :param quantifier: one of the Quantifiers 'any' or 'all' (default 'all')
+
+    Each returned Table instance will be a table that references the
+    targets according to the selected quantifier. A reference is a
+    direct foreign key in the returned table that refers to a primary
+    key of the target table.
+
+    - quantifier==all: a returned table references ALL targets
+    - quantifier==any: a returned table references AT LEAST ONE target
+
+    For proper function, all target_tables instances MUST come from
+    the same root Model instance hierarchy.
+
+    """
+    candidates = None
+    for table in target_tables:
+        referring = { fkey.table for fkey in table.referenced_by }
+        if candidates is None:
+            candidates = referring
+        elif quantifier == Quantifier.all:
+            candidates.intersection_update(referring)
+        else:
+            candidates.update(referring)
+    return candidates
+
 class Column (object):
     """Named column.
     """
@@ -1696,7 +2026,6 @@ class Column (object):
 
     @object_annotation(tag.column_display)
     def column_display(self): pass
-    
 
 def _constraint_name_parts(constraint, doc):
     # modern systems should have 0 or 1 names here
@@ -1781,10 +2110,29 @@ class Key (object):
         }
 
     @classmethod
-    def define(cls, colnames, constraint_names=[], comment=None, annotations={}):
-        """Build a key definition."""
+    def define(cls, colnames, constraint_names=[], comment=None, annotations={}, constraint_name=None):
+        """Build a key definition.
+
+        :param colnames: List of names of columns participating in the key
+        :param constraint_names: Legacy input [ [ schema_name, constraint_name ] ] (for API backwards-compatibility)
+        :param comment: Comment string
+        :param annotations: Dictionary of { annotation_uri: annotation_value, ... }
+        :param constraint_name: Constraint name string
+
+        The constraint_name kwarg takes a bare constraint name string
+        and acts the same as setting the legacy constraint_names kwarg
+        to: [ [ "placeholder", constraint_name ] ].  This odd syntax
+        is for backwards-compatibility with earlier API versions, and
+        mirrors the structure of constraint names in ERMrest model
+        description outputs. In those outputs, the "placeholder" field
+        contains the schema name of the table containing the
+        constraint.
+
+        """
         if not isinstance(colnames, list):
             raise TypeError('Colnames should be a list.')
+        if constraint_name is not None:
+            constraint_names = [ [ "placeholder", constraint_name ] ]
         return {
             'unique_columns': list(colnames),
             'names': constraint_names,
@@ -1983,9 +2331,41 @@ class ForeignKey (object):
         }
 
     @classmethod
-    def define(cls, fk_colnames, pk_sname, pk_tname, pk_colnames, on_update='NO ACTION', on_delete='NO ACTION', constraint_names=[], comment=None, acls={}, acl_bindings={}, annotations={}):
+    def define(cls, fk_colnames, pk_sname, pk_tname, pk_colnames, on_update='NO ACTION', on_delete='NO ACTION', constraint_names=[], comment=None, acls={}, acl_bindings={}, annotations={}, constraint_name=None):
+        """Define a foreign key.
+
+        :param fk_colnames: List of column names participating in the foreign key
+        :param pk_sname: Schema name string of the referenced primary key
+        :param pk_tname: Table name string of the referenced primary key
+        :param pk_colnames: List of column names participating in the referenced primary key
+        :param on_update: Constraint behavior when referenced primary keys are updated
+        :param on_update: Constraint behavior when referenced primary keys are deleted
+        :param constraint_names: Legacy input [ [ schema_name, constraint_name ] ] (for API backwards-compatibility)
+        :param comment: Comment string
+        :param acls: Dictionary of { acl_name: acl, ... }
+        :param acl_bindings: Dictionary of { binding_name: acl_binding, ... }
+        :param annotations: Dictionary of { annotation_uri: annotation_value, ... }
+        :param constraint_name: Constraint name string
+
+        The contraint behavior values for on_update and on_delete must
+        be one of the following literal strings:
+
+        'NO ACTION', 'RESTRICT', 'CASCADE', 'SET NULL', 'SET DEFAULT'
+
+        The constraint_name kwarg takes a bare constraint name string
+        and acts the same as setting the legacy constraint_names kwarg
+        to: [ [ "placeholder", constraint_name ] ].  This odd syntax
+        is for backwards-compatibility with earlier API versions, and
+        mirrors the structure of constraint names in ERMrest model
+        description outputs. In those outputs, the "placeholder" field
+        contains the schema name of the table containing the
+        constraint.
+
+        """
         if len(fk_colnames) != len(pk_colnames):
             raise ValueError('The fk_colnames and pk_colnames lists must have the same length.')
+        if constraint_name is not None:
+            constraint_names = [ [ "placeholder", constraint_name ], ]
         return {
             'foreign_key_columns': [
                 {

deriva/core/hatrac_cli.py

@@ -185,14 +185,14 @@ class DerivaHatracCLI (BaseCLI):
 
         try:
             acls = self.store.get_acl(self.resource, args.access, args.role)
+            if acls is None:
+                raise DerivaHatracCLIException('No such object or namespace or ACL entry')
             for access in acls:
                 print("%s:" % access)
                 for role in acls.get(access, []):
                     print("  %s" % role)
         except HTTPError as e:
-            if e.response.status_code == requests.codes.not_found:
-                raise ResourceException('No such object or namespace or ACL entry', e)
-            elif e.response.status_code == requests.codes.bad_request:
+            if e.response.status_code == requests.codes.bad_request:
                 raise ResourceException('Invalid ACL name %s' % args.access, e)
             else:
                 raise e
@@ -316,6 +316,8 @@ class DerivaHatracCLI (BaseCLI):
         except HatracHashMismatch as e:
             logging.debug(format_exception(e))
             eprint(_resource_error_message('Checksum verification failed'))
+        except DerivaHatracCLIException as e:
+            eprint(e)
         except RuntimeError as e:
             logging.debug(format_exception(e))
             eprint('Unexpected runtime error occurred')

deriva/core/utils/globus_auth_utils.py

@@ -52,7 +52,7 @@ class GlobusAuthUtil:
         client_id = kwargs.get("client_id")
         client_secret = kwargs.get("client_secret")
         if not (client_id and client_secret):
-            cred_file = kwargs.get("credential_file", CLIENT_CRED_FILE)
+            cred_file = kwargs.get("credential_file", CLIENT_CRED_FILE) or CLIENT_CRED_FILE
             if os.path.isfile(cred_file):
                 creds = read_config(cred_file)
                 if creds:
@@ -60,6 +60,8 @@ class GlobusAuthUtil:
                     if client:
                         client_id = client.get('client_id')
                         client_secret = client.get('client_secret')
+            else:
+                logging.warning("No Globus client credential file found at: %s" % cred_file)
 
         if not (client_id and client_secret):
             logging.warning("Client ID and secret not specified and/or could not be determined.")

deriva/transfer/upload/deriva_upload.py

@@ -233,8 +233,11 @@ class DerivaUpload(object):
         self.credentials = credentials
         self.catalog.set_credentials(self.credentials, host)
         self.store.set_credentials(self.credentials, host)
-        attributes = self.catalog.get_authn_session().json()
-        self.identity = attributes.get("client", self.identity)
+        try:
+            attributes = self.catalog.get_authn_session().json()
+            self.identity = attributes.get("client", self.identity)
+        except Exception as e:
+            logger.warning("Unable to determine user identity: %s" % e)
 
     def setConfig(self, config_file):
         if not config_file:

{deriva-1.7.1.dist-info → deriva-1.7.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.1
+Version: 1.7.3
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division

{deriva-1.7.1.dist-info → deriva-1.7.3.dist-info}/RECORD

@@ -1,23 +1,23 @@
 deriva/__init__.py,sha256=h-QyvMVzDNpT3jyVskcSbUVFXxGCRxieFPrvTveZG9k,64
 deriva/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/config/acl_config.py,sha256=yZ0g_Cwv1MZXajUP2sVbc928XakUDu5Q97w9aK3VCCE,23946
-deriva/config/annotation_config.py,sha256=DQHPn4bKVYUxWPKyHkPfwLbdtOuBWpIyW_Uln-p10K0,11732
+deriva/config/annotation_config.py,sha256=sWcJnRfS5vcb85IdUOd420cmFHRyb4br5OxLCOssbec,11714
 deriva/config/annotation_validate.py,sha256=ZN0jq6As49hfsAmWONUxAStqFG9TeYzvCmRU9TQ-zD8,5228
 deriva/config/base_config.py,sha256=Y5sFotuAWWyo-LZvzbzVY2ZDaC-zhwbMXusgBF1OJYM,20724
 deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIePJnDQgRydwTE,9362
 deriva/config/rollback_annotation.py,sha256=EMVpuaMh2VaXXoHhsr3ldc7g7e92ngszEswdGNEmGFQ,3020
 deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
 deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
-deriva/core/__init__.py,sha256=e2yjmArOhvRrFnmJxgjd_BPYIhiYsTVeYXe3iVQIKaU,4945
+deriva/core/__init__.py,sha256=ZjDR3suQoZV7qrKaA7JdSLjopu6XDHXRC7_yfM6rgmE,4945
 deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
 deriva/core/base_cli.py,sha256=EkLXOTeaFWUbPaYV-eLuLGga1PbkFVWi3Jjo-e_Vb-U,2681
 deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
-deriva/core/datapath.py,sha256=4Q3snZ-rBqQV5x7ZAfU7fWdVcmQYu-8Ma6a7DcOL6zQ,81306
+deriva/core/datapath.py,sha256=HmmV3dZtEe9lHr2b9dYyyVdvzZwy6p07ZUF9ocbFvo0,89248
 deriva/core/deriva_binding.py,sha256=_sA9HGrcVRqT-OhrneMDMOquyVOFOxLq3WzBQhasLIM,12970
 deriva/core/deriva_server.py,sha256=nsW3gwg1sIaHl3BTf-nL41AkSj3dEpcEBlatvjvN8CQ,200
-deriva/core/ermrest_catalog.py,sha256=B8XdzDScxad4PVUxRxT3GLUkV5vSsNnvMA5a76cUdsc,50817
-deriva/core/ermrest_model.py,sha256=opQunJOl4Vm62U5LBbO8w77dz9l11B0DvGDU6Sfe2FQ,85916
-deriva/core/hatrac_cli.py,sha256=l9QmneLRHSMiG_z9S83ea0QGVhTS3Wq1KGPEKEDpecM,14522
+deriva/core/ermrest_catalog.py,sha256=_eqQg16i1aA95R99B7tLZxHWQlYk-rLpN_0zghfNWRc,54991
+deriva/core/ermrest_model.py,sha256=pf8HC_ecm2NiMF0rg6UjbAuTR4ZKMhf5o5JmMYYuV8I,101855
+deriva/core/hatrac_cli.py,sha256=J8Vg2BPMqNrRC3egr0IUml-_Eo3UxuhCvk_dInjbCiw,14554
 deriva/core/hatrac_store.py,sha256=t7gEsZ3SJqt1RefSGc3728PQoVs9E5UWEBNGGbbG6Bw,22582
 deriva/core/polling_ermrest_catalog.py,sha256=KsjiFqPQaHWnJZCVF5i77sdzfubqZHgMBbQ1p8V8D3s,10351
 deriva/core/schemas/app_links.schema.json,sha256=AxrkC2scxomM6N7jyjtdYA73BbZzPrmuqU8PYWe7okI,954
@@ -44,7 +44,7 @@ deriva/core/schemas/visible_foreign_keys.schema.json,sha256=K-oa2qzj5EbmJCEyN6mN
 deriva/core/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/core/utils/__init__.py,sha256=XSbGaWe44hebxYvoh5huFzZkMY6TSKPOCRSjUOvaY70,124
 deriva/core/utils/core_utils.py,sha256=5xUjJ-1QrPFKOiwfSWFDdH8Hm5a-AEhzCvy2GJPjx4A,19500
-deriva/core/utils/globus_auth_utils.py,sha256=M7u6rN1IeQDb3RRN6bM3YIJMitPEP3Usugp-cLgsAr8,55601
+deriva/core/utils/globus_auth_utils.py,sha256=huPzKSr0d2NirpLzyp6CSyR1gIAdlRuP-X0jRo5W_p8,55732
 deriva/core/utils/hash_utils.py,sha256=JqUYVB3jXusCQYX9fkKmweUKBC0WQi8ZI2N8m-uKygQ,2299
 deriva/core/utils/mime_utils.py,sha256=ZT7pMjY2kQWBsgsGC3jY6kjfygdsIyiYW3BKNw_pPyg,1128
 deriva/core/utils/version_utils.py,sha256=HWNUQAZrPXu0oGjhG6gMNm3kWtfa3nR50vfH2uQxBA0,2954
@@ -85,7 +85,7 @@ deriva/transfer/restore/deriva_restore.py,sha256=s0h7cXit2USSdjrIfrj0dr7BJ0rrHHM
 deriva/transfer/restore/deriva_restore_cli.py,sha256=2ViZ1Lyl5ndXPKeJFCHHGnwzkg3DfHhTuRa_bN7eJm8,5603
 deriva/transfer/upload/__init__.py,sha256=4mlc_iUX-v7SpXzlCZmhxQtSiW5JeDGb2FX7bb1E6tY,304
 deriva/transfer/upload/__main__.py,sha256=hqnXtGpRqPthwpO6uvrnf_TQm7McheeyOt960hStSMY,340
-deriva/transfer/upload/deriva_upload.py,sha256=9NJbsPx1FANgGrCr5AP78AjtM41LJsanCJKEW4UH9ws,60376
+deriva/transfer/upload/deriva_upload.py,sha256=GuQb_-PBKR0JstEifXqSOZpF1zTyaPLfM0PCDlu2Qw8,60500
 deriva/transfer/upload/deriva_upload_cli.py,sha256=-Q6xgiYabQziTQcMQdGNDAv-eLxCCHO-BCSo4umbDE4,5082
 deriva/transfer/upload/processors/__init__.py,sha256=sMM5xdJ82UIRdB1lGMKk7ft0BgtjS2oJ0sI4SQSqiIU,2481
 deriva/transfer/upload/processors/archive_processor.py,sha256=ID0lDwDn4vPe5nbxy6m28Ssj_TsZpK4df2xRrM6nJRQ,2015
@@ -98,9 +98,9 @@ deriva/utils/__init__.py,sha256=jv2YF__bseklT3OWEzlqJ5qE24c4aWd5F4r0TTjOrWQ,65
 tests/deriva/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/deriva/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/deriva/core/test_datapath.py,sha256=hC5PqyL9zqNOV4ydY5L4pHdt8r7Or7OgZnX-F52P2nU,37308
-deriva-1.7.1.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-deriva-1.7.1.dist-info/METADATA,sha256=WAYjUe8xgfn45OweE56aX8q67MuxaBcdxdkrSRfZfO4,1623
-deriva-1.7.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deriva-1.7.1.dist-info/entry_points.txt,sha256=72BEmEE4Bes5QhVxUHrl7EvUARrgISWxI2KGa8BbNZ8,786
-deriva-1.7.1.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
-deriva-1.7.1.dist-info/RECORD,,
+deriva-1.7.3.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+deriva-1.7.3.dist-info/METADATA,sha256=u8fofL_SdxidGBInnPPmjbb5qDgZWN6iBFzFwGvsJT0,1623
+deriva-1.7.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+deriva-1.7.3.dist-info/entry_points.txt,sha256=72BEmEE4Bes5QhVxUHrl7EvUARrgISWxI2KGa8BbNZ8,786
+deriva-1.7.3.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
+deriva-1.7.3.dist-info/RECORD,,