deriva 1.7.0__py3-none-any.whl → 1.7.1__py3-none-any.whl

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.0"
+__version__ = "1.7.1"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/core/catalog_cli.py

@@ -55,21 +55,21 @@ class DerivaCatalogCLI (BaseCLI):
 
         # parent arg parser
         self.remove_options(['--config-file', '--credential-file'])
-        self.parser.add_argument("--protocol", choices=["http", "https"], default='https',
+        self.parser.add_argument("-p", "--protocol", choices=["http", "https"], default='https',
                                  help="transport protocol: 'http' or 'https'")
         subparsers = self.parser.add_subparsers(title='sub-commands', dest='subcmd')
 
         # exists parser
         exists_parser = subparsers.add_parser('exists', help="Check if catalog exists.")
-        exists_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        exists_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         exists_parser.set_defaults(func=self.catalog_exists)
 
         # create parser
         create_parser = subparsers.add_parser('create', help="Create a new catalog.")
-        create_parser.add_argument("--id", metavar="<id>", type=str, help="catalog id")
-        create_parser.add_argument("--owner", metavar="<owner> <owner> ...",
+        create_parser.add_argument("--id", metavar="<id>", type=str, help="Catalog ID")
+        create_parser.add_argument("-o", "--owner", metavar="<owner> <owner> ...",
                                    nargs="+", help="List of quoted user or group identifier strings.")
-        create_parser.add_argument("--auto-configure", action="store_true",
+        create_parser.add_argument("-a", "--auto-configure", action="store_true",
                                    help="Configure the new catalog with a set of baseline defaults")
         create_parser.add_argument("--configure-args", metavar="[key=value key=value ...]",
                                    nargs='+', action=KeyValuePairArgs, default={},
@@ -80,11 +80,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # get parser
         get_parser = subparsers.add_parser('get', help="Send a HTTP GET request to the catalog.")
-        get_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        get_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         get_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
-        get_parser.add_argument("--output-file", metavar="<output file path>", help="Path to output file.")
-        get_parser.add_argument("--output-format", choices=["json", "json-stream", "csv"], default="json")
-        get_parser.add_argument("--auto-delete", action="store_true",
+        get_parser.add_argument("-o", "--output-file", metavar="<output file path>", help="Path to output file.")
+        get_parser.add_argument("-f", "--output-format", choices=["json", "json-stream", "csv"], default="json",
+                                help="The output file format. Defaults to 'json'")
+        get_parser.add_argument("-a", "--auto-delete", action="store_true",
                                 help="Automatically delete output file if no results are returned.")
         get_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                 nargs='+', action=KeyValuePairArgs, default={},
@@ -95,10 +96,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # put parser
         put_parser = subparsers.add_parser('put', help="Send a HTTP PUT request to the catalog.")
-        put_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        put_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         put_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
-        put_parser.add_argument("--input-file", metavar="<output file path>", help="Path to input file.")
-        put_parser.add_argument("--input-format", choices=["json", "json-stream", "csv"], default="json")
+        put_parser.add_argument("input-file", metavar="<input file path>",
+                                help="Path to an input file containing the request message body.")
+        put_parser.add_argument("-f", "--input-format", choices=["json", "json-stream", "csv"], default="json",
+                                help="The input file format. Defaults to 'json'")
         put_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                 nargs='+', action=KeyValuePairArgs, default={},
                                 help="Variable length of whitespace-delimited key=value pair arguments used for "
@@ -108,10 +111,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # post parser
         post_parser = subparsers.add_parser('post', help="Send a HTTP POST request to the catalog.")
-        post_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        post_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         post_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
-        post_parser.add_argument("--input-file", metavar="<input file path>", help="Path to input file.")
-        post_parser.add_argument("--input-format", choices=["json", "json-stream", "csv"], default="json")
+        post_parser.add_argument("input-file", metavar="<input file path>",
+                                 help="Path to an input file containing the request message body.")
+        post_parser.add_argument("-f", "--input-format", choices=["json", "json-stream", "csv"], default="json",
+                                 help="The input file format. Defaults to 'json'")
         post_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                  nargs='+', action=KeyValuePairArgs, default={},
                                  help="Variable length of whitespace-delimited key=value pair arguments used for "
@@ -122,7 +127,7 @@ class DerivaCatalogCLI (BaseCLI):
         # delete parser
         del_parser = subparsers.add_parser('delete', help="Send a HTTP DELETE request to the catalog. "
                                                           "Use the 'drop' command to delete the entire catalog.")
-        del_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        del_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         del_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
         del_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                 nargs='+', action=KeyValuePairArgs, default={},
@@ -133,12 +138,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # drop parser
         drop_parser = subparsers.add_parser('drop', help="Delete a catalog.")
-        drop_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        drop_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         drop_parser.set_defaults(func=self.catalog_drop)
 
         # clone parser
         clone_parser = subparsers.add_parser('clone', help="Clone a source catalog to a new destination catalog.")
-        clone_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        clone_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         clone_parser.add_argument("--no-copy-data", action="store_false",
                                   help="Do not copy table contents.")
         clone_parser.add_argument("--no-copy-annotations", action="store_false",
@@ -154,8 +159,8 @@ class DerivaCatalogCLI (BaseCLI):
         # create_alias parser
         create_alias_parser = subparsers.add_parser('create-alias', help="Create a new catalog alias")
         create_alias_parser.add_argument("--id", metavar="<id>", type=str, help="The alias id.")
-        create_alias_parser.add_argument("--alias-target", metavar="<alias>", help="The target catalog id.")
-        create_alias_parser.add_argument("--owner", metavar="<owner> <owner> ...",
+        create_alias_parser.add_argument("-t", "--alias-target", metavar="<alias>", help="The target catalog id.")
+        create_alias_parser.add_argument("-o", "--owner", metavar="<owner> <owner> ...",
                                          nargs="+", help="List of quoted user or group identifier strings.")
         create_alias_parser.set_defaults(func=self.catalog_alias_create)
 
@@ -167,10 +172,10 @@ class DerivaCatalogCLI (BaseCLI):
         # update_alias parser
         update_alias_parser = subparsers.add_parser('update-alias', help="Update an existing catalog alias")
         update_alias_parser.add_argument("--id", metavar="<id>", type=str, help="The alias id.")
-        update_alias_parser.add_argument("--alias-target", metavar="<alias>", nargs='?', default=nochange, const=None,
+        update_alias_parser.add_argument("-t", "--alias-target", metavar="<alias>", nargs='?', default=nochange, const=None,
                                          help="The target catalog id. If specified without a catalog id as an argument "
                                               "value, the existing alias target will be cleared ")
-        update_alias_parser.add_argument("--owner", metavar="<owner> <owner> ...", nargs='+', default=nochange,
+        update_alias_parser.add_argument("-o", "--owner", metavar="<owner> <owner> ...", nargs='+', default=nochange,
                                          help="List of quoted user or group identifier strings.")
         update_alias_parser.set_defaults(func=self.catalog_alias_update)
 
@@ -194,9 +199,29 @@ class DerivaCatalogCLI (BaseCLI):
         self.id = args.id
         self.server = DerivaServer(self.protocol,
                                    args.host,
-                                   credentials=DerivaCatalogCLI._get_credential(self.host,
-                                                                                token=args.token,
-                                                                                oauth2_token=args.oauth2_token))
+                                   credentials=DerivaCatalogCLI._get_credential(
+                                       self.host,
+                                       token=args.token,
+                                       oauth2_token=args.oauth2_token))
+
+    @staticmethod
+    def _decorate_headers(headers, file_format, method="get"):
+
+        header_format_map = {
+            "json": "application/json",
+            "json-stream": "application/x-json-stream",
+            "csv": "text/csv"
+        }
+
+        format_type = header_format_map.get(file_format)
+        if format_type is None:
+            raise UsageException("Unsupported format: %s" % file_format)
+        if str(method).lower() in ["get", "head"]:
+            headers["accept"] = format_type
+        elif str(method).lower() in ["post", "put"]:
+            headers["content-type"] = format_type
+        else:
+            raise UsageException("Unsupported method: %s" % method)
 
     def catalog_exists(self, args):
         """Implements the catalog_exists sub-command.
@@ -250,16 +275,7 @@ class DerivaCatalogCLI (BaseCLI):
         """
         headers = DEFAULT_HEADERS.copy()
         headers.update(args.headers)
-
-        if args.output_format == "json":
-            headers["accept"] = "application/json"
-        elif args.output_format == "json-stream":
-            headers["accept"] = "application/x-json-stream"
-        elif args.output_format == "csv":
-            headers["accept"] = "text/csv"
-        else:
-            raise UsageException("Unsupported output format: %s" % args.output_format)
-
+        self._decorate_headers(headers, args.output_format)
         catalog = self.server.connect_ermrest(args.id)
         try:
             if args.output_file:
@@ -278,13 +294,41 @@ class DerivaCatalogCLI (BaseCLI):
                 os.remove(args.output_file)
             raise
 
-    # TODO: implement PUT at some point
     def catalog_put(self, args):
-        raise NotImplementedError
+        """Implements the catalog_put sub-command.
+        """
+        headers = DEFAULT_HEADERS.copy()
+        headers.update(args.headers)
+        self._decorate_headers(headers, args.input_format, "put")
+        try:
+            catalog = self.server.connect_ermrest(args.id)
+            with open(args.input_file, "rb") as input_file:
+                resp = catalog.put(args.path, data=input_file, headers=headers)
+                if not args.quiet:
+                    pp(resp.json())
+        except HTTPError as e:
+            if e.response.status_code == requests.codes.not_found:
+                raise ResourceException('Catalog not found', e)
+            else:
+                raise e
 
-    # TODO: implement POST at some point
     def catalog_post(self, args):
-        raise NotImplementedError
+        """Implements the catalog_post sub-command.
+        """
+        headers = DEFAULT_HEADERS.copy()
+        headers.update(args.headers)
+        self._decorate_headers(headers, args.input_format, "post")
+        try:
+            catalog = self.server.connect_ermrest(args.id)
+            with open(args.input_file, "rb") as input_file:
+                resp = catalog.post(args.path, data=input_file, headers=headers)
+                if not args.quiet:
+                    pp(resp.json())
+        except HTTPError as e:
+            if e.response.status_code == requests.codes.not_found:
+                raise ResourceException('Catalog not found', e)
+            else:
+                raise e
 
     def catalog_delete(self, args):
         """Implements the catalog_delete sub-command.

deriva/core/datapath.py

@@ -395,7 +395,8 @@ class DataPath (object):
         By default links use inner join semantics on the foreign key / key equality comparison. The `join_type`
         parameter can be used to specify `left`, `right`, or `full` outer join semantics.
 
-        :param right: the right hand table of the link expression
+        :param right: the right hand table of the link expression; if the table or alias name is in use, an incremental
+        number will be used to disambiguate tables instances of the same original name.
         :param on: an equality comparison between key and foreign key columns, a conjunction of such comparisons, or a foreign key object
         :param join_type: the join type of this link which may be 'left', 'right', 'full' outer joins or '' for inner
         join link by default.
@@ -413,7 +414,7 @@ class DataPath (object):
             raise ValueError("'on' must be specified for outer joins")
         if right._schema._catalog != self._root._schema._catalog:
             raise ValueError("'right' is from a different catalog. Cannot link across catalogs.")
-        if isinstance(right, _TableAlias) and right._name in self._table_instances:
+        if isinstance(right, _TableAlias) and right._parent == self:
             raise ValueError("'right' is a table alias that has already been used.")
         else:
             # Generate an unused alias name for the table
@@ -606,6 +607,18 @@ class DataPath (object):
 
         return self
 
+    def denormalize(self, context_name=None, heuristic=None, groupkey_name='RID'):
+        """Denormalizes a path based on a visible-columns annotation 'context' or a heuristic approach.
+
+        This method does not mutate this object. It returns a result set representing the denormalization of the path.
+
+        :param context_name: name of the visible-columns context or if none given, will attempt apply heuristics
+        :param heuristic: heuristic to apply if no context name specified
+        :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+        :return: a results set.
+        """
+        return _datapath_denormalize(self, context_name=context_name, heuristic=heuristic, groupkey_name=groupkey_name)
+
 
 class _ResultSet (object):
     """A set of results for various queries or data manipulations.
@@ -623,6 +636,7 @@ class _ResultSet (object):
         self._fetcher_fn = fetcher_fn
         self._results_doc = None
         self._sort_keys = None
+        self._limit = None
         self.uri = uri
 
     @property
@@ -656,6 +670,19 @@ class _ResultSet (object):
         self._results_doc = None
         return self
 
+    def limit(self, n):
+        """Set a limit on the number of results to be returned.
+
+        :param n: integer or None.
+        :return: self
+        """
+        try:
+            self._limit = None if n is None else int(n)
+            self._results_doc = None
+            return self
+        except ValueError:
+            raise ValueError('limit argument "n" must be an integer or None')
+
     def fetch(self, limit=None, headers=DEFAULT_HEADERS):
         """Fetches the results from the catalog.
 
@@ -663,7 +690,7 @@ class _ResultSet (object):
         :param headers: headers to send in request to server
         :return: self
         """
-        limit = int(limit) if limit else None
+        limit = int(limit) if limit else self._limit
         self._results_doc = self._fetcher_fn(limit, self._sort_keys, headers)
         logger.debug("Fetched %d entities" % len(self._results_doc))
         return self
@@ -797,6 +824,18 @@ class _TableWrapper (object):
         """
         return _AttributeGroup(self, self._query, keys)
 
+    def denormalize(self, context_name=None, heuristic=None, groupkey_name='RID'):
+        """Denormalizes a path based on a visible-columns annotation 'context' or a heuristic approach.
+
+        This method does not mutate this object. It returns a result set representing the denormalization of the path.
+
+        :param context_name: name of the visible-columns context or if none given, will attempt apply heuristics
+        :param heuristic: heuristic to apply if no context name specified
+        :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+        :return: a results set.
+        """
+        return self.path.denormalize(context_name=context_name, heuristic=heuristic, groupkey_name=groupkey_name)
+
     def insert(self, entities, defaults=set(), nondefaults=set(), add_system_defaults=True, on_conflict_skip=False):
         """Inserts entities into the table.
 
@@ -1760,3 +1799,275 @@ class _AttributeGroup (object):
                     bin.maxval = result.get('maxval', bin.maxval)
                     if (bin.minval is None) or (bin.maxval is None):
                         raise ValueError('Automatic determination of binning bounds failed.')
+
+##
+## UTILITIES FOR DENORMALIZATION ##############################################
+##
+
+def _datapath_left_outer_join_by_fkey(path, fk, alias_name=None):
+    """Link a table to the path based on a foreign key reference.
+
+    :param path: a DataPath object
+    :param fk: an ermrest_model.ForeignKey object
+    :param alias_name: an optional 'alias' name to use for the foreign table
+    """
+    assert isinstance(path, DataPath)
+    assert isinstance(fk, _erm.ForeignKey)
+    catalog = path._root._schema._catalog
+
+    # determine 'direction' -- inbound or outbound
+    path_context_table = path.context._base_table._wrapped_table
+    if (path_context_table.schema.name, path_context_table.name) == (fk.table.schema.name, fk.table.name):
+        right = catalog.schemas[fk.pk_table.schema.name].tables[fk.pk_table.name]
+        fkcols = zip(fk.foreign_key_columns, fk.referenced_columns)
+    elif (path_context_table.schema.name, path_context_table.name) == (fk.pk_table.schema.name, fk.pk_table.name):
+        right = catalog.schemas[fk.table.schema.name].tables[fk.table.name]
+        fkcols = zip(fk.referenced_columns, fk.foreign_key_columns)
+    else:
+        raise ValueError('Context table "%s" not referenced by foreign key "%s"' % (path_context_table.name, fk.constraint_name))
+
+    # compose join condition
+    on = None
+    for lcol, rcol in fkcols:
+        lcol = catalog.schemas[lcol.table.schema.name].tables[lcol.table.name].columns[lcol.name]
+        rcol = catalog.schemas[rcol.table.schema.name].tables[rcol.table.name].columns[rcol.name]
+        if on:
+            on = on & (lcol == rcol)
+        else:
+            on = lcol == rcol
+
+    # link
+    path.link(right.alias(alias_name) if alias_name else right, on=on, join_type='left')
+
+
+def _datapath_deserialize_vizcolumn(path, vizcol, sources=None):
+    """Deserializes a visual column specification.
+
+    If the visible column specifies a foreign key path, the datapath object
+    will be changed by linking the foreign keys in the path.
+
+    :param path: a datapath object
+    :param vizcol: a visible column specification
+    :return: the element to be projected from the datapath or None
+    """
+    assert isinstance(path, DataPath)
+    sources = sources if sources else {}
+    context = path.context
+    table = context._wrapped_table
+    model = table.schema.model
+
+    if isinstance(vizcol, str):
+        # column name specification
+        return context.columns[vizcol]
+    elif isinstance(vizcol, list):
+        # constraint specification
+        try:
+            fk = model.fkey(vizcol)
+            _datapath_left_outer_join_by_fkey(path, fk, alias_name='F')
+            return ArrayD(path.context).alias(path.context._name)  # project all attributes
+        except KeyError as e:
+            raise ValueError('Invalid foreign key constraint name: %s. If this is a key constraint name, note that keys are not supported at this time.' % str(e))
+    elif isinstance(vizcol, dict):
+        # resolve visible column
+        while 'sourcekey' in vizcol:
+            temp = sources.get(vizcol['sourcekey'], {})
+            if temp == vizcol:
+                raise ValueError('Visible column self reference for sourcekey "%s"' % vizcol['sourcekey'])
+            vizcol = temp
+        # deserialize source definition
+        source = vizcol.get('source')
+        if not source:
+            # case: none
+            raise ValueError('Could not resolve source definition for visible column')
+        elif isinstance(source, str):
+            # case: column name
+            return context.columns[source]
+        elif isinstance(source, list):
+            # case: path expression
+            # ...validate syntax
+            if not all(isinstance(obj, dict) for obj in source[:-1]):
+                raise ValueError('Source path element must be a foreign key dict')
+            if not isinstance(source[-1], str):
+                raise ValueError('Source path must terminate in a column name string')
+            # link path elements by fkey; and track whether path is outbound only fkeys
+            outbound_only = True
+            for path_elem in source[:-1]:
+                try:
+                    fk = model.fkey(path_elem.get('inbound', path_elem.get('outbound')))
+                    _datapath_left_outer_join_by_fkey(path, fk, alias_name='F')
+                    outbound_only = outbound_only and 'outbound' in path_elem
+                except KeyError as e:
+                    raise ValueError('Invalid foreign key constraint name: %s' % str(e))
+            # return terminating column or entity
+            # ...get terminal name
+            terminal = source[-1]
+            # ...get alias name
+            alias = vizcol.get('markdown_name', vizcol.get('name', path.context._name + '_' + terminal))
+            # ...get aggregate function
+            aggregate = {
+                'min': Min,
+                'max': Max,
+                'cnt': Cnt,
+                'cnd_d': CntD,
+                'array': Array,
+                'array_d': ArrayD
+            }.get(vizcol.get('aggregate'), ArrayD)
+            # ...determine projection mode
+            if vizcol.get('entity', True):
+                # case: whole entities
+                return aggregate(path.context).alias(alias)
+            else:
+                # case: specified attribute value(s)
+                if outbound_only:
+                    # for outbound only paths, we can project a single value
+                    return path.context.columns[terminal].alias(alias)
+                else:
+                    # otherwise, we need to use aggregate the values
+                    return aggregate(path.context.columns[terminal]).alias(alias)
+        else:
+            raise ValueError('Malformed source: %s' % str(source))
+    else:
+        raise ValueError('Malformed visible column: %s' % str(vizcol))
+
+
+def _datapath_contextualize(path, context_name='*', context_body=None, groupkey_name='RID'):
+    """Contextualizes a data path to a named visible columns context.
+
+    :param path: a datapath object
+    :param context_name: name of the context within the path's terminating table's "visible columns" annotations
+    :param context_body: a list of visible column definitions, if given, the `context_name` will be ignored
+    :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+    :return: a 'contextualized' attribute group query object
+    """
+    assert isinstance(path, DataPath)
+    path = copy.deepcopy(path)
+    context = path.context
+    table = context._wrapped_table
+    sources = table.annotations.get(_erm.tag.source_definitions, {}).get('sources')
+    vizcols = context_body if context_body else table.annotations.get(_erm.tag.visible_columns, {}).get(context_name, [])
+    if not vizcols:
+        raise ValueError('Visible columns context "%s" not found for table %s:%s' % (context_name, table.schema.name, table.name))
+    groupkey = context.columns[groupkey_name]
+    projection = []
+
+    for vizcol in vizcols:
+        try:
+            projection.append(_datapath_deserialize_vizcolumn(path, vizcol, sources=sources))
+            path.context = context
+        except ValueError as e:
+            logger.warning(str(e))
+
+    def not_same_as_group_key(x):
+        assert isinstance(groupkey, _ColumnWrapper)
+        if not isinstance(x, _ColumnWrapper):
+            return True
+        return groupkey._wrapped_column != x._wrapped_column
+
+    projection = filter(not_same_as_group_key, projection)  # project groupkey only once
+    query = path.groupby(groupkey).attributes(*projection)
+    return query
+
+
+def _datapath_generate_simple_denormalization(path, include_whole_entities=False):
+    """Generates a denormalized form of the table expressed in a visible columns specification.
+
+    :param path: a datapath object
+    :param include_whole_entities: if a denormalization cannot find a 'name' like terminal, include the whole entity (i.e., all attributes), else return just the 'RID'
+    :return: a generated visible columns specification based on a denormalization heuristic
+    """
+    assert isinstance(path, DataPath)
+    context = path.context
+    table = context._wrapped_table
+
+    fkeys = list(table.foreign_keys)
+    single_column_fkeys = {
+        fkey.foreign_key_columns[0].name: fkey
+        for fkey in table.foreign_keys if len(fkey.foreign_key_columns) == 1
+    }
+
+    def _fkey_to_vizcol(name, fk, inbound=None):
+        # name columns to look for in related tables
+        name_candidates = [
+            'displayname',
+            'preferredname',
+            'fullname',
+            'name',
+            'title',
+            'label'
+        ]
+
+        # determine terminal column
+        terminal = 'RID'
+        for candidate_col in fk.pk_table.columns:
+            if candidate_col.name.lower().replace(' ', '').replace('_', '') in name_candidates:
+                terminal = candidate_col.name
+                break
+
+        # define source path
+        source = [{'outbound': fk.names[0]}, terminal]
+        if inbound:
+            source = [{'inbound': inbound.names[0]}] + source
+
+        # return vizcol spec
+        return {
+            'markdown_name': name,
+            'source': source,
+            'entity': include_whole_entities and terminal == 'RID'
+        }
+
+    # assemble the visible column:
+    #  1. column or single column fkeys
+    #  2. all other (outbound fkey) related tables
+    #  3. all associated tables
+    vizcols = []
+    for col in table.column_definitions:
+        if col.name in single_column_fkeys:
+            fkey = single_column_fkeys[col.name]
+            vizcols.append(_fkey_to_vizcol(col.name, fkey))
+            del single_column_fkeys[col.name]
+            fkeys.remove(fkey)
+        else:
+            vizcols.append(col.name)
+
+    for outbound_fkey in fkeys:
+        vizcols.append(_fkey_to_vizcol(outbound_fkey.constraint_name, outbound_fkey))
+
+    for inbound_fkey in table.referenced_by:
+        if inbound_fkey.table.is_association():
+            vizcols.append(
+                _fkey_to_vizcol(
+                    inbound_fkey.table.name,
+                    inbound_fkey.table.foreign_keys[0] if inbound_fkey != inbound_fkey.table.foreign_keys[0] else inbound_fkey.table.foreign_keys[1],
+                    inbound=inbound_fkey
+                )
+            )
+
+    return vizcols
+
+def simple_denormalization(path):
+    """A simple heuristic denormalization."""
+    return _datapath_generate_simple_denormalization(path)
+
+def simple_denormalization_with_whole_entities(path):
+    """A simple heuristic denormalization with related and associated entities."""
+    return _datapath_generate_simple_denormalization(path, include_whole_entities=True)
+
+def _datapath_denormalize(path, context_name=None, heuristic=None, groupkey_name='RID'):
+    """Denormalizes a path based on annotations or heuristics.
+
+    :param path: a DataPath object
+    :param context_name: name of the visible-columns context or if none given, will attempt apply heuristics
+    :param heuristic: heuristic to apply if no context name specified
+    :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+    """
+    assert isinstance(path, DataPath)
+    assert context_name is None or isinstance(context_name, str)
+    assert isinstance(groupkey_name, str)
+    heuristic = heuristic or simple_denormalization
+    assert callable(heuristic)
+    return _datapath_contextualize(
+        path,
+        context_name=context_name,
+        context_body=None if context_name else heuristic(path),
+        groupkey_name=groupkey_name
+    )

deriva/transfer/__init__.py

@@ -1,9 +1,11 @@
 from deriva.transfer.download.deriva_download import DerivaDownload, GenericDownloader, DerivaDownloadError, \
-    DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError
+    DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, \
+    DerivaDownloadBaggingError
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
 
 from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
-    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError
+    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
+    DerivaUploadAuthenticationError
 from deriva.transfer.upload.deriva_upload_cli import DerivaUploadCLI
 
 from deriva.transfer.backup.deriva_backup import DerivaBackup, DerivaBackupAuthenticationError, \

deriva/transfer/download/__init__.py

@@ -16,3 +16,7 @@ class DerivaDownloadAuthorizationError(Exception):
 
 class DerivaDownloadTimeoutError(Exception):
     pass
+
+
+class DerivaDownloadBaggingError(Exception):
+    pass

deriva/transfer/download/deriva_download.py

@@ -7,6 +7,7 @@ import platform
 import requests
 from requests.exceptions import HTTPError
 from bdbag import bdbag_api as bdb, bdbag_ro as ro, BAG_PROFILE_TAG, BDBAG_RO_PROFILE_ID
+from bdbag.bdbagit import BagValidationError
 from deriva.core import ErmrestCatalog, HatracStore, format_exception, get_credential, format_credential, read_config, \
     stob, Megabyte, __version__ as VERSION
 from deriva.core.utils.version_utils import get_installed_version
@@ -14,7 +15,11 @@ from deriva.transfer.download.processors import find_query_processor, find_trans
 from deriva.transfer.download.processors.base_processor import LOCAL_PATH_KEY, REMOTE_PATHS_KEY, SERVICE_URL_KEY, \
     FILE_SIZE_KEY
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
-    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError
+    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError, \
+    DerivaDownloadBaggingError
+
+
+logger = logging.getLogger(__name__)
 
 
 class DerivaDownload(object):
@@ -43,7 +48,7 @@ class DerivaDownload(object):
         info = "%s v%s [Python %s, %s]" % (
             self.__class__.__name__, get_installed_version(VERSION),
             platform.python_version(), platform.platform(aliased=True))
-        logging.info("Initializing downloader: %s" % info)
+        logger.info("Initializing downloader: %s" % info)
 
         if not self.server:
             raise DerivaDownloadConfigurationError("Server not specified!")
@@ -145,12 +150,12 @@ class DerivaDownload(object):
             try:
                 if not self.credentials:
                     self.set_credentials(get_credential(self.hostname))
-                logging.info("Validating credentials for host: %s" % self.hostname)
+                logger.info("Validating credentials for host: %s" % self.hostname)
                 attributes = self.catalog.get_authn_session().json()
                 identity = attributes["client"]
             except HTTPError as he:
                 if he.response.status_code == 404:
-                    logging.info("No existing login session found for host: %s" % self.hostname)
+                    logger.info("No existing login session found for host: %s" % self.hostname)
             except Exception as e:
                 raise DerivaDownloadAuthenticationError("Unable to validate credentials: %s" % format_exception(e))
         wallet = kwargs.get("wallet", {})
@@ -160,6 +165,7 @@ class DerivaDownload(object):
         bag_archiver = None
         bag_algorithms = None
         bag_idempotent = False
+        bag_strict = True
         bag_config = self.config.get('bag')
         create_bag = True if bag_config else False
         if create_bag:
@@ -171,7 +177,8 @@ class DerivaDownload(object):
             bag_idempotent = stob(bag_config.get('bag_idempotent', False))
             bag_metadata = bag_config.get('bag_metadata', {"Internal-Sender-Identifier":
                                                            "deriva@%s" % self.server_url})
-            bag_ro = create_bag and not bag_idempotent and stob(bag_config.get('bag_ro', "True"))
+            bag_ro = create_bag and not bag_idempotent and stob(bag_config.get('bag_ro', True))
+            bag_strict = stob(bag_config.get('bag_strict', True))
             if create_bag:
                 bdb.ensure_bag_path_exists(bag_path)
                 bag = bdb.make_bag(bag_path, algs=bag_algorithms, metadata=bag_metadata, idempotent=bag_idempotent)
@@ -211,12 +218,13 @@ class DerivaDownload(object):
                                             allow_anonymous=self.allow_anonymous,
                                             timeout=self.timeout)
                 outputs = processor.process()
+                assert outputs is not None
                 if processor.should_abort():
                     raise DerivaDownloadTimeoutError("Timeout (%s seconds) waiting for processor [%s] to complete." %
                                                      (self.timeout_secs, processor_name))
                 self.check_payload_size(outputs)
             except Exception as e:
-                logging.error(format_exception(e))
+                logger.error(format_exception(e))
                 if create_bag:
                     bdb.cleanup_bag(bag_path)
                     if remote_file_manifest and os.path.isfile(remote_file_manifest):
@@ -270,16 +278,27 @@ class DerivaDownload(object):
                              remote_file_manifest=remote_file_manifest
                              if (remote_file_manifest and os.path.getsize(remote_file_manifest) > 0) else None,
                              update=True,
-                             idempotent=bag_idempotent)
+                             idempotent=bag_idempotent,
+                             strict=bag_strict)
+            except BagValidationError as bve:
+                msg = "Unable to validate bag.%s Error: %s" % (
+                    "" if not bag_strict else
+                    " Strict checking has been enabled, which most likely means that this bag "
+                    "is empty (has no payload files or fetch references) and therefore invalid.",
+                    format_exception(bve))
+                logger.error(msg)
+                bdb.cleanup_bag(bag_path)
+                raise DerivaDownloadBaggingError(msg)
             except Exception as e:
-                logging.fatal("Exception while updating bag manifests: %s" % format_exception(e))
+                msg = "Unhandled exception while updating bag manifests: %s" % format_exception(e)
+                logger.error(msg)
                 bdb.cleanup_bag(bag_path)
-                raise
+                raise DerivaDownloadBaggingError(msg)
             finally:
                 if remote_file_manifest and os.path.isfile(remote_file_manifest):
                     os.remove(remote_file_manifest)
 
-            logging.info('Created bag: %s' % bag_path)
+            logger.info('Created bag: %s' % bag_path)
 
             if bag_archiver is not None:
                 try:
@@ -289,8 +308,9 @@ class DerivaDownload(object):
                     bdb.cleanup_bag(bag_path)
                     outputs = {os.path.basename(archive): {LOCAL_PATH_KEY: archive}}
                 except Exception as e:
-                    logging.error("Exception while creating data bag archive: %s" % format_exception(e))
-                    raise
+                    msg = "Exception while creating data bag archive: %s" % format_exception(e)
+                    logger.error(msg)
+                    raise DerivaDownloadBaggingError(msg)
             else:
                 outputs = {os.path.basename(bag_path): {LOCAL_PATH_KEY: bag_path}}
 
@@ -318,7 +338,7 @@ class DerivaDownload(object):
                             (self.timeout_secs, processor_name))
                     self.check_payload_size(outputs)
                 except Exception as e:
-                    logging.error(format_exception(e))
+                    logger.error(format_exception(e))
                     raise
 
         return outputs

deriva/transfer/download/deriva_download_cli.py

@@ -7,7 +7,8 @@ import requests
 from requests.exceptions import HTTPError, ConnectionError
 from deriva.transfer import GenericDownloader
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
-    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError
+    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError, \
+    DerivaDownloadBaggingError
 from deriva.core import BaseCLI, KeyValuePairArgs, format_credential, format_exception, urlparse
 
 
@@ -71,7 +72,7 @@ class DerivaDownloadCLI(BaseCLI):
                     raise DerivaDownloadAuthorizationError(
                         "A requested operation was forbidden. Server responded: %s" % e)
         except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError,
-                DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError) as e:
+                DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError, DerivaDownloadBaggingError) as e:
             sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e))
             if args.debug:
                 traceback.print_exc()

deriva/transfer/download/processors/query/base_query_processor.py

@@ -21,8 +21,8 @@ class BaseQueryProcessor(BaseProcessor):
         self.catalog = kwargs["catalog"]
         self.store = kwargs["store"]
         self.base_path = kwargs["base_path"]
-        self.query = self.parameters["query_path"]
-        if self.envars:
+        self.query = self.parameters.get("query_path", "")
+        if self.query and self.envars:
             self.query = self.query.format(**self.envars)
         self.sub_path = self.parameters.get("output_path")
         self.output_filename = self.parameters.get("output_filename")
@@ -59,6 +59,9 @@ class BaseQueryProcessor(BaseProcessor):
         return self.outputs
 
     def catalogQuery(self, headers=None, as_file=True):
+        if not self.query:
+            return {}
+
         if not headers:
             headers = self.HEADERS.copy()
         else:
@@ -175,8 +178,9 @@ class BaseQueryProcessor(BaseProcessor):
                                                                      envars=self.envars)
 
     def __del__(self):
-        for session in self.sessions.values():
-            session.close()
+        if self.sessions:
+            for session in self.sessions.values():
+                session.close()
 
 
 class CSVQueryProcessor(BaseQueryProcessor):
@@ -232,3 +236,4 @@ class CreateDirProcessor(JSONEnvUpdateProcessor):
         self.create_default_paths()
         make_dirs(self.output_abspath)
 
+        return self.outputs

deriva/transfer/upload/__init__.py

@@ -12,3 +12,7 @@ class DerivaUploadCatalogCreateError (RuntimeError):
 
 class DerivaUploadCatalogUpdateError (RuntimeError):
     pass
+
+
+class DerivaUploadAuthenticationError (RuntimeError):
+    pass

deriva/transfer/upload/deriva_upload.py

@@ -557,6 +557,10 @@ class DerivaUpload(object):
         return None, None, None
 
     def uploadFiles(self, status_callback=None, file_callback=None):
+        if not self.identity:
+            raise DerivaUploadAuthenticationError("Unable to determine user identity for %s. "
+                                                  "Please ensure that you are authenticated successfully." %
+                                                  self.server_url)
         completed = 0
         for group, assets in self.file_list.items():
             if self.cancelled:

deriva/transfer/upload/deriva_upload_cli.py

@@ -3,7 +3,7 @@ import sys
 import json
 import traceback
 from deriva.transfer import DerivaUpload, DerivaUploadError, DerivaUploadConfigurationError, \
-    DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError
+    DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, DerivaUploadAuthenticationError
 from deriva.core import BaseCLI, write_config, format_credential, format_exception, urlparse
 
 
@@ -95,7 +95,7 @@ class DerivaUploadCLI(BaseCLI):
                                    args.dry_run,
                                    args.output_file)
         except (RuntimeError, FileNotFoundError, DerivaUploadError, DerivaUploadConfigurationError,
-                DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError) as e:
+                DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, DerivaUploadAuthenticationError) as e:
             sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e))
             if args.debug:
                 traceback.print_exc()

{deriva-1.7.0.dist-info → deriva-1.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.0
+Version: 1.7.1
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division
@@ -29,7 +29,7 @@ Requires-Dist: certifi
 Requires-Dist: pika
 Requires-Dist: urllib3 <3,>=1.26
 Requires-Dist: portalocker >=1.2.1
-Requires-Dist: bdbag >=1.7.2
+Requires-Dist: bdbag >=1.7.3
 Requires-Dist: globus-sdk <4,>=3
 Requires-Dist: fair-research-login >=0.3.1
 Requires-Dist: fair-identifiers-client >=0.5.1

{deriva-1.7.0.dist-info → deriva-1.7.1.dist-info}/RECORD

@@ -8,11 +8,11 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
 deriva/config/rollback_annotation.py,sha256=EMVpuaMh2VaXXoHhsr3ldc7g7e92ngszEswdGNEmGFQ,3020
 deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
 deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
-deriva/core/__init__.py,sha256=0FEjx_-XAJR5j5-KizIuB71NDiRNC18yF4uHRiRV7Ow,4945
+deriva/core/__init__.py,sha256=e2yjmArOhvRrFnmJxgjd_BPYIhiYsTVeYXe3iVQIKaU,4945
 deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
 deriva/core/base_cli.py,sha256=EkLXOTeaFWUbPaYV-eLuLGga1PbkFVWi3Jjo-e_Vb-U,2681
-deriva/core/catalog_cli.py,sha256=qGZ2s6g0UROJGb_9lHl-xMFIdpDsM2F2IWCmgl9Bn28,21253
-deriva/core/datapath.py,sha256=96wNx6Exm6lfMOPAYwrPXqfMnlp3I_VnweCJWt7jWO8,67548
+deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
+deriva/core/datapath.py,sha256=4Q3snZ-rBqQV5x7ZAfU7fWdVcmQYu-8Ma6a7DcOL6zQ,81306
 deriva/core/deriva_binding.py,sha256=_sA9HGrcVRqT-OhrneMDMOquyVOFOxLq3WzBQhasLIM,12970
 deriva/core/deriva_server.py,sha256=nsW3gwg1sIaHl3BTf-nL41AkSj3dEpcEBlatvjvN8CQ,200
 deriva/core/ermrest_catalog.py,sha256=B8XdzDScxad4PVUxRxT3GLUkV5vSsNnvMA5a76cUdsc,50817
@@ -52,15 +52,15 @@ deriva/core/utils/webauthn_utils.py,sha256=rD0HQZAjUKp4NfqHQG1FhH3x7uKog2et7w7LB
 deriva/seo/__init__.py,sha256=dYn48A7blbeYf40b4T3KVofrQK4u5K5MfxXWfIGloig,54
 deriva/seo/sitemap_builder.py,sha256=Ht_AbodEERDofIoCcd4kPlrl1pVW670WN5dT4cc05LQ,13948
 deriva/seo/sitemap_cli.py,sha256=miCqRfpSj5Dx5BfJGSd8Pi2e4OOQjotDzP_JubukhCM,2654
-deriva/transfer/__init__.py,sha256=BTXIXdI18bOks20aTizoP9hy_iYLCMaKZCtHAdss0uI,1099
+deriva/transfer/__init__.py,sha256=Xbp-s9vmzFbumJZ743PEXE8rI1s6-s234F66C_ioC_E,1172
 deriva/transfer/backup/__init__.py,sha256=vxsZiDLMTJQPybXT89G-07GsUoLhnItTCbLdXcDSyeA,465
 deriva/transfer/backup/__main__.py,sha256=dT12--8C6sKGEtMhsYuy013ebXKpVnBJfhcQNlVtv6Y,361
 deriva/transfer/backup/deriva_backup.py,sha256=IO9Tmzx6jHfUCkP-41nSsAeOFLn9T-0HwQcpRLpM_zs,5228
 deriva/transfer/backup/deriva_backup_cli.py,sha256=T0tvPKWniRinMQt0qG7FI8AoK3GgtlT6EyBZmZCAjL8,2157
-deriva/transfer/download/__init__.py,sha256=eNzozhBmUdthI3H0bM8irxe-qTLm06J0Mnrq34qzh-o,294
+deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w9Xyh88Q,350
 deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
-deriva/transfer/download/deriva_download.py,sha256=CdVRcsL5n6C72j4fgW4ySmUjca5oQMv_ekcj3Pd4AUY,16201
-deriva/transfer/download/deriva_download_cli.py,sha256=m7hylccfhL7kMyba3D3kWq7W4d0P10txbQl6N05U9nE,3878
+deriva/transfer/download/deriva_download.py,sha256=9WHX0iBUsXv3iT0pEy95kpVN-Oh4vc6ywI5tYmJWpfk,17145
+deriva/transfer/download/deriva_download_cli.py,sha256=wN8tyQDv1AIE_aDqjECbmkoEWN050vlEdJyteYbdgSs,3940
 deriva/transfer/download/processors/__init__.py,sha256=evLp36tZn-Z_AMshdfV3JJO8w1es5owsnRN0IFJUwIo,4507
 deriva/transfer/download/processors/base_processor.py,sha256=R6IIHSa_euv4X2Dyhd8fvQAiVYDGJTWMQtPoukHQn-Q,3837
 deriva/transfer/download/processors/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -69,7 +69,7 @@ deriva/transfer/download/processors/postprocess/transfer_post_processor.py,sha25
 deriva/transfer/download/processors/postprocess/url_post_processor.py,sha256=s68iIYqQSZHtbv4y-fCG8pjhApAeMEG6hYcKx2Pvf5Y,2745
 deriva/transfer/download/processors/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/query/bag_fetch_query_processor.py,sha256=tiQtfuy01YgOFFD5b_sP7TGjMnt0Jqcg2gp1KNWqeLE,5645
-deriva/transfer/download/processors/query/base_query_processor.py,sha256=6h-BpqccAONyyiTYWQHT0dR_AT158xbFljQl5ABrgDI,10192
+deriva/transfer/download/processors/query/base_query_processor.py,sha256=oM1pbsbpnPV12h3FLtxo8Sp0OCUQYiRhg22AVqf4N50,10327
 deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=Hg1NbKsaGJh9cB86yIyL7Fm7ywSNVop837Dv8aFXUes,7257
 deriva/transfer/download/processors/transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/transform/base_transform_processor.py,sha256=Ddw5gsNpDANeuLvUaF4utp8psaxOtAzlgXtOg8gb-Pc,4109
@@ -83,10 +83,10 @@ deriva/transfer/restore/__init__.py,sha256=l_VNbzpfjL8aVSASctsqSMbWO6m_9aVhkq5Lq
 deriva/transfer/restore/__main__.py,sha256=iGGSF4_q3gUTLU3pPHYL9PNi9T6XNRHOBeTf1obhOjc,364
 deriva/transfer/restore/deriva_restore.py,sha256=s0h7cXit2USSdjrIfrj0dr7BJ0rrHHMxYr6AxsuoDiI,30298
 deriva/transfer/restore/deriva_restore_cli.py,sha256=2ViZ1Lyl5ndXPKeJFCHHGnwzkg3DfHhTuRa_bN7eJm8,5603
-deriva/transfer/upload/__init__.py,sha256=oq1Jq0bpuRdXnqehmeXDGYPWwhNT_g-yBb7WMVUO87o,239
+deriva/transfer/upload/__init__.py,sha256=4mlc_iUX-v7SpXzlCZmhxQtSiW5JeDGb2FX7bb1E6tY,304
 deriva/transfer/upload/__main__.py,sha256=hqnXtGpRqPthwpO6uvrnf_TQm7McheeyOt960hStSMY,340
-deriva/transfer/upload/deriva_upload.py,sha256=z7uT-thzzmV3HqdgsAqE_8vEm276wgETnnNFqGB5Kdg,60075
-deriva/transfer/upload/deriva_upload_cli.py,sha256=76lBxiWkbveOZcKwYRK-IVZtWexSP2xkKTDowugEiEU,5016
+deriva/transfer/upload/deriva_upload.py,sha256=9NJbsPx1FANgGrCr5AP78AjtM41LJsanCJKEW4UH9ws,60376
+deriva/transfer/upload/deriva_upload_cli.py,sha256=-Q6xgiYabQziTQcMQdGNDAv-eLxCCHO-BCSo4umbDE4,5082
 deriva/transfer/upload/processors/__init__.py,sha256=sMM5xdJ82UIRdB1lGMKk7ft0BgtjS2oJ0sI4SQSqiIU,2481
 deriva/transfer/upload/processors/archive_processor.py,sha256=ID0lDwDn4vPe5nbxy6m28Ssj_TsZpK4df2xRrM6nJRQ,2015
 deriva/transfer/upload/processors/base_processor.py,sha256=Nsr-BP_0TvCtW7sb7s2QbYDEJoijONpQDDdnkVcbmng,666
@@ -97,10 +97,10 @@ deriva/transfer/upload/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 deriva/utils/__init__.py,sha256=jv2YF__bseklT3OWEzlqJ5qE24c4aWd5F4r0TTjOrWQ,65
 tests/deriva/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/deriva/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/deriva/core/test_datapath.py,sha256=KTHs3jf0p7tWKJGZsrbu1aEpQ6AIy_1VYECQBKDaIlo,36023
-deriva-1.7.0.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-deriva-1.7.0.dist-info/METADATA,sha256=cOo4GQhTw5Pi8b62t_Ut00wFYfYFy5VwnAS8GXCmBCs,1623
-deriva-1.7.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deriva-1.7.0.dist-info/entry_points.txt,sha256=72BEmEE4Bes5QhVxUHrl7EvUARrgISWxI2KGa8BbNZ8,786
-deriva-1.7.0.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
-deriva-1.7.0.dist-info/RECORD,,
+tests/deriva/core/test_datapath.py,sha256=hC5PqyL9zqNOV4ydY5L4pHdt8r7Or7OgZnX-F52P2nU,37308
+deriva-1.7.1.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+deriva-1.7.1.dist-info/METADATA,sha256=WAYjUe8xgfn45OweE56aX8q67MuxaBcdxdkrSRfZfO4,1623
+deriva-1.7.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+deriva-1.7.1.dist-info/entry_points.txt,sha256=72BEmEE4Bes5QhVxUHrl7EvUARrgISWxI2KGa8BbNZ8,786
+deriva-1.7.1.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
+deriva-1.7.1.dist-info/RECORD,,

tests/deriva/core/test_datapath.py

@@ -12,7 +12,8 @@ import os
 import unittest
 import sys
 from deriva.core import DerivaServer, get_credential, ermrest_model as _em, __version__
-from deriva.core.datapath import DataPathException, Min, Max, Sum, Avg, Cnt, CntD, Array, ArrayD, Bin
+from deriva.core.datapath import DataPathException, Min, Max, Sum, Avg, Cnt, CntD, Array, ArrayD, Bin, \
+    simple_denormalization_with_whole_entities
 
 try:
     from pandas import DataFrame
@@ -129,11 +130,12 @@ def populate_test_catalog(catalog):
     """Populate the test catalog."""
     paths = catalog.getPathBuilder()
     logger.debug("Inserting project...")
-    logger.debug("Inserting experiment types...")
     proj_table = paths.schemas[SNAME_ISA].tables[TNAME_PROJECT]
+    logger.debug("Inserting investigators...")
     proj_table.insert([
         {"Investigator": TEST_PROJ_INVESTIGATOR, "Num": TEST_PROJ_NUM}
     ])
+    logger.debug("Inserting experiment types...")
     type_table = paths.schemas[SNAME_VOCAB].tables[TNAME_EXPERIMENT_TYPE]
     types = type_table.insert([
         {"Name": "{}".format(name), "Description": "NA"} for name in range(TEST_EXPTYPE_MAX)
@@ -806,6 +808,26 @@ class DatapathTests (unittest.TestCase):
         self.assertEqual(path.context._name, path3.context._name, "Context of composed paths should equal far right-hand path's context")
         self.assertGreater(len(path.Experiment.entities()), 0, "Should have returned results")
 
+    def test_simple_denormalization(self):
+        entities = self.experiment.entities()
+        results = self.experiment.denormalize()
+        self.assertEqual(len(entities), len(results))
+        self.assertNotEqual(entities[0].keys(), results[0].keys())
+        self.assertIn('Type', results[0])
+        self.assertTrue(entities[0]['Type'].startswith('TEST:'))
+        self.assertTrue(results[0]['Type'])
+        self.assertFalse(results[0]['Type'].startswith('TEST:'))
+
+    def test_simple_denormalization_w_entities(self):
+        entities = self.experiment.entities()
+        results = self.experiment.denormalize(heuristic=simple_denormalization_with_whole_entities)
+        self.assertEqual(len(entities), len(results))
+        self.assertLess(len(entities[0].keys()), len(results[0].keys()))
+        self.assertIn('Experiment_Project Investigator_Project_Num_fkey', results[0])
+        self.assertIsInstance(results[0]['Experiment_Project Investigator_Project_Num_fkey'], list)
+        self.assertIsInstance(results[0]['Experiment_Project Investigator_Project_Num_fkey'][0], dict)
+        self.assertIn('RID', results[0]['Experiment_Project Investigator_Project_Num_fkey'][0])
+
 
 if __name__ == '__main__':
     sys.exit(unittest.main())