deriva 1.7.0__py3-none-any.whl → 1.7.3__py3-none-any.whl

deriva/core/ermrest_model.py

@@ -1,7 +1,13 @@
 
+from __future__ import annotations
+
 from collections import OrderedDict
+from collections.abc import Iterable
+from enum import Enum
 import json
 import re
+import base64
+import hashlib
 
 from . import AttrDict, tag, urlquote, stob
 
@@ -17,6 +23,74 @@ class NoChange (object):
 # singletone to use in APIs below
 nochange = NoChange()
 
+def make_id(*components):
+    """Build an identifier that will be OK for ERMrest and Postgres.
+
+    Naively, append as '_'.join(components).
+
+    Fallback to heuristics mixing truncation with short hashes.
+    """
+    # accept lists at top-level for convenience (compound keys, etc.)
+    expanded = []
+    for e in components:
+        if isinstance(e, list):
+            expanded.extend(e)
+        else:
+            expanded.append(e)
+
+    # prefer to use naive name as requested
+    naive_result = '_'.join(expanded)
+    naive_len = len(naive_result.encode('utf8'))
+    if naive_len <= 63:
+        return naive_result
+
+    # we'll need to truncate and hash in some way...
+    def hash(s, nbytes):
+        return base64.urlsafe_b64encode(hashlib.md5(s.encode('utf8')).digest()).decode()[0:nbytes]
+
+    def truncate(s, maxlen):
+        encoded_len = len(s.encode('utf8'))
+        # we need to chop whole (unicode) chars but test encoded byte lengths!
+        for i in range(max(1, len(s) - maxlen), len(s) - 1):
+            result = s[0:-1 * i].rstrip()
+            if len(result.encode('utf8')) <= (maxlen - 2):
+                return result + '..'
+        return s
+
+    naive_hash = hash(naive_result, 5)
+    parts = [
+        (i, expanded[i])
+        for i in range(len(expanded))
+    ]
+
+    # try to find a solution truncating individual fields
+    for maxlen in [15, 12, 9]:
+        parts.sort(key=lambda p: (len(p[1].encode('utf8')), p[0]), reverse=True)
+        for i in range(len(parts)):
+            idx, part = parts[i]
+            if len(part.encode('utf8')) > maxlen:
+                parts[i] = (idx, truncate(part, maxlen))
+            candidate_result = '_'.join([
+                p[1]
+                for p in sorted(parts, key=lambda p: p[0])
+            ] + [naive_hash])
+            if len(candidate_result.encode('utf8')) < 63:
+                return candidate_result
+
+    # fallback to truncating original naive name
+    # try to preserve suffix and trim in middle
+    result = ''.join([
+        truncate(naive_result, len(naive_result)//3),
+        naive_result[-len(naive_result)//3:],
+        '_',
+        naive_hash
+    ])
+    if len(result.encode('utf8')) <= 63:
+        return result
+
+    # last-ditch (e.g. multibyte unicode suffix worst case)
+    return truncate(naive_result, 55) + naive_hash
+
 def presence_annotation(tag_uri):
     """Decorator to establish property getter/setter/deleter for presence annotations.
 
@@ -684,6 +758,15 @@ class KeyedList (list):
         list.append(self, e)
         self.elements[e.name] = e
 
+class FindAssociationResult (object):
+    """Wrapper for results of Table.find_associations()"""
+    def __init__(self, table, self_fkey, other_fkeys):
+        self.table = table
+        self.name = table.name
+        self.schema = table.schema
+        self.self_fkey = self_fkey
+        self.other_fkeys = other_fkeys
+    
 class Table (object):
     """Named table.
     """
@@ -1075,6 +1158,184 @@ class Table (object):
             provide_system
         )
 
+    @classmethod
+    def define_association(
+        cls,
+        associates: Iterable[Key | Table | tuple[str, Key | Table]],
+        metadata: Iterable[Key | Table | dict | tuple[str, bool, Key | Table]] = [],
+        table_name: str | None = None,
+        comment: str | None = None,
+        provide_system: bool = True) -> dict:
+        """Build an association table definition.
+
+        :param associates: the existing Key instances being associated
+        :param metadata: additional metadata fields for impure associations
+        :param table_name: name for the association table or None for default naming
+        :param comment: comment for the association table or None for default comment
+        :param provide_system: add ERMrest system columns when True
+
+        This is a utility function to help build an association table
+        definition. It simplifies the task, but removes some
+        control. For full customization, consider using Table.define()
+        directly instead.
+
+        A normal ("pure") N-ary association is a table with N foreign
+        keys referencing N primary keys in referenced tables, with a
+        composite primary key covering the N foreign keys. These pure
+        association tables manage a set of distinct combinations of
+        the associated foreign key values.
+
+        An "impure" association table adds additional metadata
+        alongside the N foreign keys.
+
+        The "associates" parameter takes an iterable of Key instances
+        from other tables.  The association will be comprised of
+        foreign keys referencing these associates. Optionally, a tuple
+        of (str, Key) can supply a string _base name_ to influence how
+        the foreign key columns and constraint will be named in the
+        new association table. A bare Key instance will get a base
+        name derived from the referenced table name.
+
+        The "metadata" parameter takes an iterable of plain dict
+        column definitions or Key instances. Each dict must be a
+        scalar column definition, such as produced by the
+        Column.define() class method. Key instance will cause
+        corresponding columns and foreign keys to be added to the
+        association table to act as metadata. Optionally, a tuple of
+        (str, bool, Key) can supply a string _base name_ and a boolean
+        _nullok_ property to influence how the foreign key columns and
+        constraint will be constructed and named. A bare Key instance
+        will get a base name derived from the referened table name,
+        and presumed as nullok=False.
+
+        If a Table instance is supplied instead of a Key instance for
+        associates or metadata inputs, an attempt will be made to
+        locate a key based on the RID system column. If this key
+        cannot be found, a KeyError will be raised.
+
+        """
+        associates = list(associates)
+        metadata = list(metadata)
+
+        if len(associates) < 2:
+            raise ValueError('An association table requires at least 2 associates')
+
+        cdefs = []
+        kdefs = []
+        fkdefs = []
+
+        used_names = set()
+
+        def check_basename(basename):
+            if not isinstance(base_name, str):
+                raise TypeError('Base name %r is not of required type str' % (base_name,))
+            if base_name in used_names:
+                raise ValueError('Base name %r is not unique among associates and metadata' % (base_name,))
+            used_names.add(base_name)
+
+        def choose_basename(key):
+            base_name = key.table.name
+            n = 2
+            while base_name in used_names:
+                base_name = '%s%d' % (key.table.name, n)
+                n += 1
+            used_names.add(base_name)
+            return base_name
+
+        def check_key(key):
+            if isinstance(key, Table):
+                return key.key_by_columns(["RID"])
+            return key
+
+        # check and normalize associates into list[(str, Key)] with distinct base names
+        for i in range(len(associates)):
+            if isinstance(associates[i], tuple):
+                base_name, key = associates[i]
+                check_basename(base_name)
+                key = check_key(key)
+                associates[i] = (base_name, key)
+            else:
+                key = check_key(associates[i])
+                base_name = choose_basename(key)
+                associates[i] = (base_name, key)
+
+        # build assoc table name if not provided
+        if table_name is None:
+            table_name = make_id(*[ assoc[1].table.name for assoc in associates ])
+
+        def simplify_type(ctype):
+            if ctype.is_domain and ctype.typename.startswith('ermrest_'):
+                return ctype.base_type
+
+            return ctype
+
+        def cdefs_for_key(base_name, key, nullok=False):
+            return [
+                Column.define(
+                    '%s_%s' % (base_name, col.name) if len(key.unique_columns) > 1 else base_name,
+                    simplify_type(col.type),
+                    nullok=nullok,
+                )
+                for col in key.unique_columns
+            ]
+
+        def fkdef_for_key(base_name, key):
+            return ForeignKey.define(
+                [
+                    '%s_%s' % (base_name, col.name) if len(key.unique_columns) > 1 else base_name
+                    for col in key.unique_columns
+                ],
+                key.table.schema.name,
+                key.table.name,
+                [ col.name for col in key.unique_columns ],
+                on_update='CASCADE',
+                on_delete='CASCADE',
+                constraint_name=make_id(table_name, base_name, 'fkey'),
+            )
+
+        # build core association definition (i.e. the "pure" parts)
+        k_cnames = []
+        for base_name, key in associates:
+            cdefs.extend(cdefs_for_key(base_name, key))
+            fkdefs.append(fkdef_for_key(base_name, key))
+
+            k_cnames.extend([
+                '%s_%s' % (base_name, col.name) if len(key.unique_columns) > 1 else base_name
+                for col in key.unique_columns
+            ])
+
+        kdefs.append(
+            Key.define(
+                k_cnames,
+                constraint_name=make_id(table_name, 'assoc', 'key'),
+            )
+        )
+
+        # check and normalize metadata into list[dict | (str, bool, Key)]
+        for i in range(len(metadata)):
+            if isinstance(metadata[i], tuple):
+                base_name, nullok, key = metadata[i]
+                check_basename(base_name)
+                key = check_key(key)
+                metadata[i] = (base_name, nullok, key)
+            elif isinstance(metadata[i], dict):
+                pass
+            else:
+                key = check_key(metadata[i])
+                base_name = choose_basename(key)
+                metadata[i] = (base_name, False, key)
+
+        # add metadata to definition
+        for md in metadata:
+            if isinstance(md, dict):
+                cdefs.append(md)
+            else:
+                base_name, nullok, key = md
+                cdefs.extend(cdefs_for_key(base_name, key, nullok))
+                fkdefs.append(fkdef_for_key(base_name, key))
+
+        return Table.define(table_name, cdefs, kdefs, fkdefs, comment=comment, provide_system=provide_system)
+
     def prejson(self, prune=True):
         return {
             "schema_name": self.schema.name,
@@ -1348,7 +1609,7 @@ class Table (object):
         if raise_nomatch:
             raise KeyError(from_to_map)
 
-    def is_association(self, min_arity=2, max_arity=2, unqualified=True, pure=True, no_overlap=True):
+    def is_association(self, min_arity=2, max_arity=2, unqualified=True, pure=True, no_overlap=True, return_fkeys=False):
         """Return (truthy) integer arity if self is a matching association, else False.
 
         min_arity: minimum number of associated fkeys (default 2)
@@ -1356,6 +1617,7 @@ class Table (object):
         unqualified: reject qualified associations when True (default True)
         pure: reject impure assocations when True (default True)
         no_overlap: reject overlapping associations when True (default True)
+        return_fkeys: return the set of N associated ForeignKeys if True
 
         The default behavior with no arguments is to test for pure,
         unqualified, non-overlapping, binary assocations.
@@ -1444,9 +1706,43 @@ class Table (object):
             # reject: impure association
             return False
 
-        # return (truthy) arity
-        return len(covered_fkeys)
+        # return (truthy) arity or fkeys
+        if return_fkeys:
+            return covered_fkeys
+        else:
+            return len(covered_fkeys)
+
+    def find_associations(self, min_arity=2, max_arity=2, unqualified=True, pure=True, no_overlap=True) -> Iterable[FindAssociationResult]:
+        """Yield (iterable) Association objects linking to this table and meeting all criteria.
 
+        min_arity: minimum number of associated fkeys (default 2)
+        max_arity: maximum number of associated fkeys (default 2) or None
+        unqualified: reject qualified associations when True (default True)
+        pure: reject impure assocations when True (default True)
+        no_overlap: reject overlapping associations when True (default True)
+
+        See documentation for sibling method Table.is_association(...)
+        for more explanation of these association detection criteria.
+
+        """
+        peer_tables = set()
+        for fkey in self.referenced_by:
+            peer = fkey.table
+            if peer in peer_tables:
+                # check each peer only once
+                continue
+            peer_tables.add(peer)
+            answer = peer.is_association(min_arity=min_arity, max_arity=max_arity, unqualified=unqualified, pure=pure, no_overlap=no_overlap, return_fkeys=True)
+            if answer:
+                answer = set(answer)
+                for fkey in answer:
+                    if fkey.pk_table == self:
+                        answer.remove(fkey)
+                        yield FindAssociationResult(peer, fkey, answer)
+                        # arbitrarily choose first fkey to self
+                        # in case association is back to same table
+                        break
+    
     @presence_annotation(tag.immutable)
     def immutable(self): pass
 
@@ -1495,6 +1791,40 @@ class Table (object):
     @object_annotation(tag.viz_3d_display)
     def viz_3d_display(self): pass
     
+class Quantifier (str, Enum):
+    """Logic quantifiers"""
+    any = 'any'
+    all = 'all'
+
+def find_tables_with_foreign_keys(target_tables: Iterable[Table], quantifier: Quantifier=Quantifier.all) -> set[Table]:
+    """Return set of tables with foreign key references to target tables.
+
+    :param target_tables: an iterable of ermrest_model.Table instances
+    :param quantifier: one of the Quantifiers 'any' or 'all' (default 'all')
+
+    Each returned Table instance will be a table that references the
+    targets according to the selected quantifier. A reference is a
+    direct foreign key in the returned table that refers to a primary
+    key of the target table.
+
+    - quantifier==all: a returned table references ALL targets
+    - quantifier==any: a returned table references AT LEAST ONE target
+
+    For proper function, all target_tables instances MUST come from
+    the same root Model instance hierarchy.
+
+    """
+    candidates = None
+    for table in target_tables:
+        referring = { fkey.table for fkey in table.referenced_by }
+        if candidates is None:
+            candidates = referring
+        elif quantifier == Quantifier.all:
+            candidates.intersection_update(referring)
+        else:
+            candidates.update(referring)
+    return candidates
+
 class Column (object):
     """Named column.
     """
@@ -1696,7 +2026,6 @@ class Column (object):
 
     @object_annotation(tag.column_display)
     def column_display(self): pass
-    
 
 def _constraint_name_parts(constraint, doc):
     # modern systems should have 0 or 1 names here
@@ -1781,10 +2110,29 @@ class Key (object):
         }
 
     @classmethod
-    def define(cls, colnames, constraint_names=[], comment=None, annotations={}):
-        """Build a key definition."""
+    def define(cls, colnames, constraint_names=[], comment=None, annotations={}, constraint_name=None):
+        """Build a key definition.
+
+        :param colnames: List of names of columns participating in the key
+        :param constraint_names: Legacy input [ [ schema_name, constraint_name ] ] (for API backwards-compatibility)
+        :param comment: Comment string
+        :param annotations: Dictionary of { annotation_uri: annotation_value, ... }
+        :param constraint_name: Constraint name string
+
+        The constraint_name kwarg takes a bare constraint name string
+        and acts the same as setting the legacy constraint_names kwarg
+        to: [ [ "placeholder", constraint_name ] ].  This odd syntax
+        is for backwards-compatibility with earlier API versions, and
+        mirrors the structure of constraint names in ERMrest model
+        description outputs. In those outputs, the "placeholder" field
+        contains the schema name of the table containing the
+        constraint.
+
+        """
         if not isinstance(colnames, list):
             raise TypeError('Colnames should be a list.')
+        if constraint_name is not None:
+            constraint_names = [ [ "placeholder", constraint_name ] ]
         return {
             'unique_columns': list(colnames),
             'names': constraint_names,
@@ -1983,9 +2331,41 @@ class ForeignKey (object):
         }
 
     @classmethod
-    def define(cls, fk_colnames, pk_sname, pk_tname, pk_colnames, on_update='NO ACTION', on_delete='NO ACTION', constraint_names=[], comment=None, acls={}, acl_bindings={}, annotations={}):
+    def define(cls, fk_colnames, pk_sname, pk_tname, pk_colnames, on_update='NO ACTION', on_delete='NO ACTION', constraint_names=[], comment=None, acls={}, acl_bindings={}, annotations={}, constraint_name=None):
+        """Define a foreign key.
+
+        :param fk_colnames: List of column names participating in the foreign key
+        :param pk_sname: Schema name string of the referenced primary key
+        :param pk_tname: Table name string of the referenced primary key
+        :param pk_colnames: List of column names participating in the referenced primary key
+        :param on_update: Constraint behavior when referenced primary keys are updated
+        :param on_update: Constraint behavior when referenced primary keys are deleted
+        :param constraint_names: Legacy input [ [ schema_name, constraint_name ] ] (for API backwards-compatibility)
+        :param comment: Comment string
+        :param acls: Dictionary of { acl_name: acl, ... }
+        :param acl_bindings: Dictionary of { binding_name: acl_binding, ... }
+        :param annotations: Dictionary of { annotation_uri: annotation_value, ... }
+        :param constraint_name: Constraint name string
+
+        The contraint behavior values for on_update and on_delete must
+        be one of the following literal strings:
+
+        'NO ACTION', 'RESTRICT', 'CASCADE', 'SET NULL', 'SET DEFAULT'
+
+        The constraint_name kwarg takes a bare constraint name string
+        and acts the same as setting the legacy constraint_names kwarg
+        to: [ [ "placeholder", constraint_name ] ].  This odd syntax
+        is for backwards-compatibility with earlier API versions, and
+        mirrors the structure of constraint names in ERMrest model
+        description outputs. In those outputs, the "placeholder" field
+        contains the schema name of the table containing the
+        constraint.
+
+        """
         if len(fk_colnames) != len(pk_colnames):
             raise ValueError('The fk_colnames and pk_colnames lists must have the same length.')
+        if constraint_name is not None:
+            constraint_names = [ [ "placeholder", constraint_name ], ]
         return {
             'foreign_key_columns': [
                 {

deriva/core/hatrac_cli.py

@@ -185,14 +185,14 @@ class DerivaHatracCLI (BaseCLI):
 
         try:
             acls = self.store.get_acl(self.resource, args.access, args.role)
+            if acls is None:
+                raise DerivaHatracCLIException('No such object or namespace or ACL entry')
             for access in acls:
                 print("%s:" % access)
                 for role in acls.get(access, []):
                     print("  %s" % role)
         except HTTPError as e:
-            if e.response.status_code == requests.codes.not_found:
-                raise ResourceException('No such object or namespace or ACL entry', e)
-            elif e.response.status_code == requests.codes.bad_request:
+            if e.response.status_code == requests.codes.bad_request:
                 raise ResourceException('Invalid ACL name %s' % args.access, e)
             else:
                 raise e
@@ -316,6 +316,8 @@ class DerivaHatracCLI (BaseCLI):
         except HatracHashMismatch as e:
             logging.debug(format_exception(e))
             eprint(_resource_error_message('Checksum verification failed'))
+        except DerivaHatracCLIException as e:
+            eprint(e)
         except RuntimeError as e:
             logging.debug(format_exception(e))
             eprint('Unexpected runtime error occurred')

deriva/core/utils/globus_auth_utils.py

@@ -52,7 +52,7 @@ class GlobusAuthUtil:
         client_id = kwargs.get("client_id")
         client_secret = kwargs.get("client_secret")
         if not (client_id and client_secret):
-            cred_file = kwargs.get("credential_file", CLIENT_CRED_FILE)
+            cred_file = kwargs.get("credential_file", CLIENT_CRED_FILE) or CLIENT_CRED_FILE
             if os.path.isfile(cred_file):
                 creds = read_config(cred_file)
                 if creds:
@@ -60,6 +60,8 @@ class GlobusAuthUtil:
                     if client:
                         client_id = client.get('client_id')
                         client_secret = client.get('client_secret')
+            else:
+                logging.warning("No Globus client credential file found at: %s" % cred_file)
 
         if not (client_id and client_secret):
             logging.warning("Client ID and secret not specified and/or could not be determined.")

deriva/transfer/__init__.py

@@ -1,9 +1,11 @@
 from deriva.transfer.download.deriva_download import DerivaDownload, GenericDownloader, DerivaDownloadError, \
-    DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError
+    DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, \
+    DerivaDownloadBaggingError
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
 
 from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
-    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError
+    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
+    DerivaUploadAuthenticationError
 from deriva.transfer.upload.deriva_upload_cli import DerivaUploadCLI
 
 from deriva.transfer.backup.deriva_backup import DerivaBackup, DerivaBackupAuthenticationError, \

deriva/transfer/download/__init__.py

@@ -16,3 +16,7 @@ class DerivaDownloadAuthorizationError(Exception):
 
 class DerivaDownloadTimeoutError(Exception):
     pass
+
+
+class DerivaDownloadBaggingError(Exception):
+    pass

deriva/transfer/download/deriva_download.py

@@ -7,6 +7,7 @@ import platform
 import requests
 from requests.exceptions import HTTPError
 from bdbag import bdbag_api as bdb, bdbag_ro as ro, BAG_PROFILE_TAG, BDBAG_RO_PROFILE_ID
+from bdbag.bdbagit import BagValidationError
 from deriva.core import ErmrestCatalog, HatracStore, format_exception, get_credential, format_credential, read_config, \
     stob, Megabyte, __version__ as VERSION
 from deriva.core.utils.version_utils import get_installed_version
@@ -14,7 +15,11 @@ from deriva.transfer.download.processors import find_query_processor, find_trans
 from deriva.transfer.download.processors.base_processor import LOCAL_PATH_KEY, REMOTE_PATHS_KEY, SERVICE_URL_KEY, \
     FILE_SIZE_KEY
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
-    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError
+    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError, \
+    DerivaDownloadBaggingError
+
+
+logger = logging.getLogger(__name__)
 
 
 class DerivaDownload(object):
@@ -43,7 +48,7 @@ class DerivaDownload(object):
         info = "%s v%s [Python %s, %s]" % (
             self.__class__.__name__, get_installed_version(VERSION),
             platform.python_version(), platform.platform(aliased=True))
-        logging.info("Initializing downloader: %s" % info)
+        logger.info("Initializing downloader: %s" % info)
 
         if not self.server:
             raise DerivaDownloadConfigurationError("Server not specified!")
@@ -145,12 +150,12 @@ class DerivaDownload(object):
             try:
                 if not self.credentials:
                     self.set_credentials(get_credential(self.hostname))
-                logging.info("Validating credentials for host: %s" % self.hostname)
+                logger.info("Validating credentials for host: %s" % self.hostname)
                 attributes = self.catalog.get_authn_session().json()
                 identity = attributes["client"]
             except HTTPError as he:
                 if he.response.status_code == 404:
-                    logging.info("No existing login session found for host: %s" % self.hostname)
+                    logger.info("No existing login session found for host: %s" % self.hostname)
             except Exception as e:
                 raise DerivaDownloadAuthenticationError("Unable to validate credentials: %s" % format_exception(e))
         wallet = kwargs.get("wallet", {})
@@ -160,6 +165,7 @@ class DerivaDownload(object):
         bag_archiver = None
         bag_algorithms = None
         bag_idempotent = False
+        bag_strict = True
         bag_config = self.config.get('bag')
         create_bag = True if bag_config else False
         if create_bag:
@@ -171,7 +177,8 @@ class DerivaDownload(object):
             bag_idempotent = stob(bag_config.get('bag_idempotent', False))
             bag_metadata = bag_config.get('bag_metadata', {"Internal-Sender-Identifier":
                                                            "deriva@%s" % self.server_url})
-            bag_ro = create_bag and not bag_idempotent and stob(bag_config.get('bag_ro', "True"))
+            bag_ro = create_bag and not bag_idempotent and stob(bag_config.get('bag_ro', True))
+            bag_strict = stob(bag_config.get('bag_strict', True))
             if create_bag:
                 bdb.ensure_bag_path_exists(bag_path)
                 bag = bdb.make_bag(bag_path, algs=bag_algorithms, metadata=bag_metadata, idempotent=bag_idempotent)
@@ -211,12 +218,13 @@ class DerivaDownload(object):
                                             allow_anonymous=self.allow_anonymous,
                                             timeout=self.timeout)
                 outputs = processor.process()
+                assert outputs is not None
                 if processor.should_abort():
                     raise DerivaDownloadTimeoutError("Timeout (%s seconds) waiting for processor [%s] to complete." %
                                                      (self.timeout_secs, processor_name))
                 self.check_payload_size(outputs)
             except Exception as e:
-                logging.error(format_exception(e))
+                logger.error(format_exception(e))
                 if create_bag:
                     bdb.cleanup_bag(bag_path)
                     if remote_file_manifest and os.path.isfile(remote_file_manifest):
@@ -270,16 +278,27 @@ class DerivaDownload(object):
                              remote_file_manifest=remote_file_manifest
                              if (remote_file_manifest and os.path.getsize(remote_file_manifest) > 0) else None,
                              update=True,
-                             idempotent=bag_idempotent)
+                             idempotent=bag_idempotent,
+                             strict=bag_strict)
+            except BagValidationError as bve:
+                msg = "Unable to validate bag.%s Error: %s" % (
+                    "" if not bag_strict else
+                    " Strict checking has been enabled, which most likely means that this bag "
+                    "is empty (has no payload files or fetch references) and therefore invalid.",
+                    format_exception(bve))
+                logger.error(msg)
+                bdb.cleanup_bag(bag_path)
+                raise DerivaDownloadBaggingError(msg)
             except Exception as e:
-                logging.fatal("Exception while updating bag manifests: %s" % format_exception(e))
+                msg = "Unhandled exception while updating bag manifests: %s" % format_exception(e)
+                logger.error(msg)
                 bdb.cleanup_bag(bag_path)
-                raise
+                raise DerivaDownloadBaggingError(msg)
             finally:
                 if remote_file_manifest and os.path.isfile(remote_file_manifest):
                     os.remove(remote_file_manifest)
 
-            logging.info('Created bag: %s' % bag_path)
+            logger.info('Created bag: %s' % bag_path)
 
             if bag_archiver is not None:
                 try:
@@ -289,8 +308,9 @@ class DerivaDownload(object):
                     bdb.cleanup_bag(bag_path)
                     outputs = {os.path.basename(archive): {LOCAL_PATH_KEY: archive}}
                 except Exception as e:
-                    logging.error("Exception while creating data bag archive: %s" % format_exception(e))
-                    raise
+                    msg = "Exception while creating data bag archive: %s" % format_exception(e)
+                    logger.error(msg)
+                    raise DerivaDownloadBaggingError(msg)
             else:
                 outputs = {os.path.basename(bag_path): {LOCAL_PATH_KEY: bag_path}}
 
@@ -318,7 +338,7 @@ class DerivaDownload(object):
                             (self.timeout_secs, processor_name))
                     self.check_payload_size(outputs)
                 except Exception as e:
-                    logging.error(format_exception(e))
+                    logger.error(format_exception(e))
                     raise
 
         return outputs