deriva 1.7.0__tar.gz → 1.7.1__tar.gz

deriva-1.7.1/CHANGELOG.md

@@ -0,0 +1,18 @@
+# CHANGE LOG
+
+## 1.6.5
+
+DataPath feature enhancements and changes:
+* new `denormalize()` method that uses a `visible-columns` "context" or a heuristic function to denormalize a path expression.
+* new `limit()` method as a chaining operation so that clients don't need to explicity `fetch()` results with limits.
+* subtle change to `link()` method for how it determines whether an alias name has been "bound" to the path. Now, it more explicitly checks if an alias object is found in the path's bound table instances. This change also relaxes the alias naming constraint to allow for automatic conflict resolution by appending an incremental numeric disambiguator if the given alias name is already in use. This could result in some `link()` methods now succeeding where they would have failed in the past.
+
+## 1.0.0
+Feature release. 
+* Refactored catalog configuration API.
+* Support for aggregate functions and binning in DataPath API.
+* Various bug fixes.
+* Complete change list [here](https://github.com/informatics-isi-edu/deriva-py/compare/v0.9.0...v1.0.0). 
+
+Change lists for previous releases can be found on the GitHub releases 
+page [here](https://github.com/informatics-isi-edu/deriva-py/releases).

{deriva-1.7.0/deriva.egg-info → deriva-1.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.0
+Version: 1.7.1
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division
@@ -29,7 +29,7 @@ Requires-Dist: certifi
 Requires-Dist: pika
 Requires-Dist: urllib3<3,>=1.26
 Requires-Dist: portalocker>=1.2.1
-Requires-Dist: bdbag>=1.7.2
+Requires-Dist: bdbag>=1.7.3
 Requires-Dist: globus_sdk<4,>=3
 Requires-Dist: fair-research-login>=0.3.1
 Requires-Dist: fair-identifiers-client>=0.5.1

{deriva-1.7.0 → deriva-1.7.1}/deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.0"
+__version__ = "1.7.1"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

{deriva-1.7.0 → deriva-1.7.1}/deriva/core/catalog_cli.py

@@ -55,21 +55,21 @@ class DerivaCatalogCLI (BaseCLI):
 
         # parent arg parser
         self.remove_options(['--config-file', '--credential-file'])
-        self.parser.add_argument("--protocol", choices=["http", "https"], default='https',
+        self.parser.add_argument("-p", "--protocol", choices=["http", "https"], default='https',
                                  help="transport protocol: 'http' or 'https'")
         subparsers = self.parser.add_subparsers(title='sub-commands', dest='subcmd')
 
         # exists parser
         exists_parser = subparsers.add_parser('exists', help="Check if catalog exists.")
-        exists_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        exists_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         exists_parser.set_defaults(func=self.catalog_exists)
 
         # create parser
         create_parser = subparsers.add_parser('create', help="Create a new catalog.")
-        create_parser.add_argument("--id", metavar="<id>", type=str, help="catalog id")
-        create_parser.add_argument("--owner", metavar="<owner> <owner> ...",
+        create_parser.add_argument("--id", metavar="<id>", type=str, help="Catalog ID")
+        create_parser.add_argument("-o", "--owner", metavar="<owner> <owner> ...",
                                    nargs="+", help="List of quoted user or group identifier strings.")
-        create_parser.add_argument("--auto-configure", action="store_true",
+        create_parser.add_argument("-a", "--auto-configure", action="store_true",
                                    help="Configure the new catalog with a set of baseline defaults")
         create_parser.add_argument("--configure-args", metavar="[key=value key=value ...]",
                                    nargs='+', action=KeyValuePairArgs, default={},
@@ -80,11 +80,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # get parser
         get_parser = subparsers.add_parser('get', help="Send a HTTP GET request to the catalog.")
-        get_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        get_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         get_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
-        get_parser.add_argument("--output-file", metavar="<output file path>", help="Path to output file.")
-        get_parser.add_argument("--output-format", choices=["json", "json-stream", "csv"], default="json")
-        get_parser.add_argument("--auto-delete", action="store_true",
+        get_parser.add_argument("-o", "--output-file", metavar="<output file path>", help="Path to output file.")
+        get_parser.add_argument("-f", "--output-format", choices=["json", "json-stream", "csv"], default="json",
+                                help="The output file format. Defaults to 'json'")
+        get_parser.add_argument("-a", "--auto-delete", action="store_true",
                                 help="Automatically delete output file if no results are returned.")
         get_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                 nargs='+', action=KeyValuePairArgs, default={},
@@ -95,10 +96,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # put parser
         put_parser = subparsers.add_parser('put', help="Send a HTTP PUT request to the catalog.")
-        put_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        put_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         put_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
-        put_parser.add_argument("--input-file", metavar="<output file path>", help="Path to input file.")
-        put_parser.add_argument("--input-format", choices=["json", "json-stream", "csv"], default="json")
+        put_parser.add_argument("input-file", metavar="<input file path>",
+                                help="Path to an input file containing the request message body.")
+        put_parser.add_argument("-f", "--input-format", choices=["json", "json-stream", "csv"], default="json",
+                                help="The input file format. Defaults to 'json'")
         put_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                 nargs='+', action=KeyValuePairArgs, default={},
                                 help="Variable length of whitespace-delimited key=value pair arguments used for "
@@ -108,10 +111,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # post parser
         post_parser = subparsers.add_parser('post', help="Send a HTTP POST request to the catalog.")
-        post_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        post_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         post_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
-        post_parser.add_argument("--input-file", metavar="<input file path>", help="Path to input file.")
-        post_parser.add_argument("--input-format", choices=["json", "json-stream", "csv"], default="json")
+        post_parser.add_argument("input-file", metavar="<input file path>",
+                                 help="Path to an input file containing the request message body.")
+        post_parser.add_argument("-f", "--input-format", choices=["json", "json-stream", "csv"], default="json",
+                                 help="The input file format. Defaults to 'json'")
         post_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                  nargs='+', action=KeyValuePairArgs, default={},
                                  help="Variable length of whitespace-delimited key=value pair arguments used for "
@@ -122,7 +127,7 @@ class DerivaCatalogCLI (BaseCLI):
         # delete parser
         del_parser = subparsers.add_parser('delete', help="Send a HTTP DELETE request to the catalog. "
                                                           "Use the 'drop' command to delete the entire catalog.")
-        del_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        del_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         del_parser.add_argument("path", metavar="<request-path>", help="The ERMRest API path.")
         del_parser.add_argument("--headers", metavar="[key=value key=value ...]",
                                 nargs='+', action=KeyValuePairArgs, default={},
@@ -133,12 +138,12 @@ class DerivaCatalogCLI (BaseCLI):
 
         # drop parser
         drop_parser = subparsers.add_parser('drop', help="Delete a catalog.")
-        drop_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        drop_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         drop_parser.set_defaults(func=self.catalog_drop)
 
         # clone parser
         clone_parser = subparsers.add_parser('clone', help="Clone a source catalog to a new destination catalog.")
-        clone_parser.add_argument("id", metavar="<id>", type=str, help="catalog id")
+        clone_parser.add_argument("id", metavar="<id>", type=str, help="Catalog ID")
         clone_parser.add_argument("--no-copy-data", action="store_false",
                                   help="Do not copy table contents.")
         clone_parser.add_argument("--no-copy-annotations", action="store_false",
@@ -154,8 +159,8 @@ class DerivaCatalogCLI (BaseCLI):
         # create_alias parser
         create_alias_parser = subparsers.add_parser('create-alias', help="Create a new catalog alias")
         create_alias_parser.add_argument("--id", metavar="<id>", type=str, help="The alias id.")
-        create_alias_parser.add_argument("--alias-target", metavar="<alias>", help="The target catalog id.")
-        create_alias_parser.add_argument("--owner", metavar="<owner> <owner> ...",
+        create_alias_parser.add_argument("-t", "--alias-target", metavar="<alias>", help="The target catalog id.")
+        create_alias_parser.add_argument("-o", "--owner", metavar="<owner> <owner> ...",
                                          nargs="+", help="List of quoted user or group identifier strings.")
         create_alias_parser.set_defaults(func=self.catalog_alias_create)
 
@@ -167,10 +172,10 @@ class DerivaCatalogCLI (BaseCLI):
         # update_alias parser
         update_alias_parser = subparsers.add_parser('update-alias', help="Update an existing catalog alias")
         update_alias_parser.add_argument("--id", metavar="<id>", type=str, help="The alias id.")
-        update_alias_parser.add_argument("--alias-target", metavar="<alias>", nargs='?', default=nochange, const=None,
+        update_alias_parser.add_argument("-t", "--alias-target", metavar="<alias>", nargs='?', default=nochange, const=None,
                                          help="The target catalog id. If specified without a catalog id as an argument "
                                               "value, the existing alias target will be cleared ")
-        update_alias_parser.add_argument("--owner", metavar="<owner> <owner> ...", nargs='+', default=nochange,
+        update_alias_parser.add_argument("-o", "--owner", metavar="<owner> <owner> ...", nargs='+', default=nochange,
                                          help="List of quoted user or group identifier strings.")
         update_alias_parser.set_defaults(func=self.catalog_alias_update)
 
@@ -194,9 +199,29 @@ class DerivaCatalogCLI (BaseCLI):
         self.id = args.id
         self.server = DerivaServer(self.protocol,
                                    args.host,
-                                   credentials=DerivaCatalogCLI._get_credential(self.host,
-                                                                                token=args.token,
-                                                                                oauth2_token=args.oauth2_token))
+                                   credentials=DerivaCatalogCLI._get_credential(
+                                       self.host,
+                                       token=args.token,
+                                       oauth2_token=args.oauth2_token))
+
+    @staticmethod
+    def _decorate_headers(headers, file_format, method="get"):
+
+        header_format_map = {
+            "json": "application/json",
+            "json-stream": "application/x-json-stream",
+            "csv": "text/csv"
+        }
+
+        format_type = header_format_map.get(file_format)
+        if format_type is None:
+            raise UsageException("Unsupported format: %s" % file_format)
+        if str(method).lower() in ["get", "head"]:
+            headers["accept"] = format_type
+        elif str(method).lower() in ["post", "put"]:
+            headers["content-type"] = format_type
+        else:
+            raise UsageException("Unsupported method: %s" % method)
 
     def catalog_exists(self, args):
         """Implements the catalog_exists sub-command.
@@ -250,16 +275,7 @@ class DerivaCatalogCLI (BaseCLI):
         """
         headers = DEFAULT_HEADERS.copy()
         headers.update(args.headers)
-
-        if args.output_format == "json":
-            headers["accept"] = "application/json"
-        elif args.output_format == "json-stream":
-            headers["accept"] = "application/x-json-stream"
-        elif args.output_format == "csv":
-            headers["accept"] = "text/csv"
-        else:
-            raise UsageException("Unsupported output format: %s" % args.output_format)
-
+        self._decorate_headers(headers, args.output_format)
         catalog = self.server.connect_ermrest(args.id)
         try:
             if args.output_file:
@@ -278,13 +294,41 @@ class DerivaCatalogCLI (BaseCLI):
                 os.remove(args.output_file)
             raise
 
-    # TODO: implement PUT at some point
     def catalog_put(self, args):
-        raise NotImplementedError
+        """Implements the catalog_put sub-command.
+        """
+        headers = DEFAULT_HEADERS.copy()
+        headers.update(args.headers)
+        self._decorate_headers(headers, args.input_format, "put")
+        try:
+            catalog = self.server.connect_ermrest(args.id)
+            with open(args.input_file, "rb") as input_file:
+                resp = catalog.put(args.path, data=input_file, headers=headers)
+                if not args.quiet:
+                    pp(resp.json())
+        except HTTPError as e:
+            if e.response.status_code == requests.codes.not_found:
+                raise ResourceException('Catalog not found', e)
+            else:
+                raise e
 
-    # TODO: implement POST at some point
     def catalog_post(self, args):
-        raise NotImplementedError
+        """Implements the catalog_post sub-command.
+        """
+        headers = DEFAULT_HEADERS.copy()
+        headers.update(args.headers)
+        self._decorate_headers(headers, args.input_format, "post")
+        try:
+            catalog = self.server.connect_ermrest(args.id)
+            with open(args.input_file, "rb") as input_file:
+                resp = catalog.post(args.path, data=input_file, headers=headers)
+                if not args.quiet:
+                    pp(resp.json())
+        except HTTPError as e:
+            if e.response.status_code == requests.codes.not_found:
+                raise ResourceException('Catalog not found', e)
+            else:
+                raise e
 
     def catalog_delete(self, args):
         """Implements the catalog_delete sub-command.

{deriva-1.7.0 → deriva-1.7.1}/deriva/core/datapath.py

@@ -395,7 +395,8 @@ class DataPath (object):
         By default links use inner join semantics on the foreign key / key equality comparison. The `join_type`
         parameter can be used to specify `left`, `right`, or `full` outer join semantics.
 
-        :param right: the right hand table of the link expression
+        :param right: the right hand table of the link expression; if the table or alias name is in use, an incremental
+        number will be used to disambiguate tables instances of the same original name.
         :param on: an equality comparison between key and foreign key columns, a conjunction of such comparisons, or a foreign key object
         :param join_type: the join type of this link which may be 'left', 'right', 'full' outer joins or '' for inner
         join link by default.
@@ -413,7 +414,7 @@ class DataPath (object):
             raise ValueError("'on' must be specified for outer joins")
         if right._schema._catalog != self._root._schema._catalog:
             raise ValueError("'right' is from a different catalog. Cannot link across catalogs.")
-        if isinstance(right, _TableAlias) and right._name in self._table_instances:
+        if isinstance(right, _TableAlias) and right._parent == self:
             raise ValueError("'right' is a table alias that has already been used.")
         else:
             # Generate an unused alias name for the table
@@ -606,6 +607,18 @@ class DataPath (object):
 
         return self
 
+    def denormalize(self, context_name=None, heuristic=None, groupkey_name='RID'):
+        """Denormalizes a path based on a visible-columns annotation 'context' or a heuristic approach.
+
+        This method does not mutate this object. It returns a result set representing the denormalization of the path.
+
+        :param context_name: name of the visible-columns context or if none given, will attempt apply heuristics
+        :param heuristic: heuristic to apply if no context name specified
+        :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+        :return: a results set.
+        """
+        return _datapath_denormalize(self, context_name=context_name, heuristic=heuristic, groupkey_name=groupkey_name)
+
 
 class _ResultSet (object):
     """A set of results for various queries or data manipulations.
@@ -623,6 +636,7 @@ class _ResultSet (object):
         self._fetcher_fn = fetcher_fn
         self._results_doc = None
         self._sort_keys = None
+        self._limit = None
         self.uri = uri
 
     @property
@@ -656,6 +670,19 @@ class _ResultSet (object):
         self._results_doc = None
         return self
 
+    def limit(self, n):
+        """Set a limit on the number of results to be returned.
+
+        :param n: integer or None.
+        :return: self
+        """
+        try:
+            self._limit = None if n is None else int(n)
+            self._results_doc = None
+            return self
+        except ValueError:
+            raise ValueError('limit argument "n" must be an integer or None')
+
     def fetch(self, limit=None, headers=DEFAULT_HEADERS):
         """Fetches the results from the catalog.
 
@@ -663,7 +690,7 @@ class _ResultSet (object):
         :param headers: headers to send in request to server
         :return: self
         """
-        limit = int(limit) if limit else None
+        limit = int(limit) if limit else self._limit
         self._results_doc = self._fetcher_fn(limit, self._sort_keys, headers)
         logger.debug("Fetched %d entities" % len(self._results_doc))
         return self
@@ -797,6 +824,18 @@ class _TableWrapper (object):
         """
         return _AttributeGroup(self, self._query, keys)
 
+    def denormalize(self, context_name=None, heuristic=None, groupkey_name='RID'):
+        """Denormalizes a path based on a visible-columns annotation 'context' or a heuristic approach.
+
+        This method does not mutate this object. It returns a result set representing the denormalization of the path.
+
+        :param context_name: name of the visible-columns context or if none given, will attempt apply heuristics
+        :param heuristic: heuristic to apply if no context name specified
+        :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+        :return: a results set.
+        """
+        return self.path.denormalize(context_name=context_name, heuristic=heuristic, groupkey_name=groupkey_name)
+
     def insert(self, entities, defaults=set(), nondefaults=set(), add_system_defaults=True, on_conflict_skip=False):
         """Inserts entities into the table.
 
@@ -1760,3 +1799,275 @@ class _AttributeGroup (object):
                     bin.maxval = result.get('maxval', bin.maxval)
                     if (bin.minval is None) or (bin.maxval is None):
                         raise ValueError('Automatic determination of binning bounds failed.')
+
+##
+## UTILITIES FOR DENORMALIZATION ##############################################
+##
+
+def _datapath_left_outer_join_by_fkey(path, fk, alias_name=None):
+    """Link a table to the path based on a foreign key reference.
+
+    :param path: a DataPath object
+    :param fk: an ermrest_model.ForeignKey object
+    :param alias_name: an optional 'alias' name to use for the foreign table
+    """
+    assert isinstance(path, DataPath)
+    assert isinstance(fk, _erm.ForeignKey)
+    catalog = path._root._schema._catalog
+
+    # determine 'direction' -- inbound or outbound
+    path_context_table = path.context._base_table._wrapped_table
+    if (path_context_table.schema.name, path_context_table.name) == (fk.table.schema.name, fk.table.name):
+        right = catalog.schemas[fk.pk_table.schema.name].tables[fk.pk_table.name]
+        fkcols = zip(fk.foreign_key_columns, fk.referenced_columns)
+    elif (path_context_table.schema.name, path_context_table.name) == (fk.pk_table.schema.name, fk.pk_table.name):
+        right = catalog.schemas[fk.table.schema.name].tables[fk.table.name]
+        fkcols = zip(fk.referenced_columns, fk.foreign_key_columns)
+    else:
+        raise ValueError('Context table "%s" not referenced by foreign key "%s"' % (path_context_table.name, fk.constraint_name))
+
+    # compose join condition
+    on = None
+    for lcol, rcol in fkcols:
+        lcol = catalog.schemas[lcol.table.schema.name].tables[lcol.table.name].columns[lcol.name]
+        rcol = catalog.schemas[rcol.table.schema.name].tables[rcol.table.name].columns[rcol.name]
+        if on:
+            on = on & (lcol == rcol)
+        else:
+            on = lcol == rcol
+
+    # link
+    path.link(right.alias(alias_name) if alias_name else right, on=on, join_type='left')
+
+
+def _datapath_deserialize_vizcolumn(path, vizcol, sources=None):
+    """Deserializes a visual column specification.
+
+    If the visible column specifies a foreign key path, the datapath object
+    will be changed by linking the foreign keys in the path.
+
+    :param path: a datapath object
+    :param vizcol: a visible column specification
+    :return: the element to be projected from the datapath or None
+    """
+    assert isinstance(path, DataPath)
+    sources = sources if sources else {}
+    context = path.context
+    table = context._wrapped_table
+    model = table.schema.model
+
+    if isinstance(vizcol, str):
+        # column name specification
+        return context.columns[vizcol]
+    elif isinstance(vizcol, list):
+        # constraint specification
+        try:
+            fk = model.fkey(vizcol)
+            _datapath_left_outer_join_by_fkey(path, fk, alias_name='F')
+            return ArrayD(path.context).alias(path.context._name)  # project all attributes
+        except KeyError as e:
+            raise ValueError('Invalid foreign key constraint name: %s. If this is a key constraint name, note that keys are not supported at this time.' % str(e))
+    elif isinstance(vizcol, dict):
+        # resolve visible column
+        while 'sourcekey' in vizcol:
+            temp = sources.get(vizcol['sourcekey'], {})
+            if temp == vizcol:
+                raise ValueError('Visible column self reference for sourcekey "%s"' % vizcol['sourcekey'])
+            vizcol = temp
+        # deserialize source definition
+        source = vizcol.get('source')
+        if not source:
+            # case: none
+            raise ValueError('Could not resolve source definition for visible column')
+        elif isinstance(source, str):
+            # case: column name
+            return context.columns[source]
+        elif isinstance(source, list):
+            # case: path expression
+            # ...validate syntax
+            if not all(isinstance(obj, dict) for obj in source[:-1]):
+                raise ValueError('Source path element must be a foreign key dict')
+            if not isinstance(source[-1], str):
+                raise ValueError('Source path must terminate in a column name string')
+            # link path elements by fkey; and track whether path is outbound only fkeys
+            outbound_only = True
+            for path_elem in source[:-1]:
+                try:
+                    fk = model.fkey(path_elem.get('inbound', path_elem.get('outbound')))
+                    _datapath_left_outer_join_by_fkey(path, fk, alias_name='F')
+                    outbound_only = outbound_only and 'outbound' in path_elem
+                except KeyError as e:
+                    raise ValueError('Invalid foreign key constraint name: %s' % str(e))
+            # return terminating column or entity
+            # ...get terminal name
+            terminal = source[-1]
+            # ...get alias name
+            alias = vizcol.get('markdown_name', vizcol.get('name', path.context._name + '_' + terminal))
+            # ...get aggregate function
+            aggregate = {
+                'min': Min,
+                'max': Max,
+                'cnt': Cnt,
+                'cnd_d': CntD,
+                'array': Array,
+                'array_d': ArrayD
+            }.get(vizcol.get('aggregate'), ArrayD)
+            # ...determine projection mode
+            if vizcol.get('entity', True):
+                # case: whole entities
+                return aggregate(path.context).alias(alias)
+            else:
+                # case: specified attribute value(s)
+                if outbound_only:
+                    # for outbound only paths, we can project a single value
+                    return path.context.columns[terminal].alias(alias)
+                else:
+                    # otherwise, we need to use aggregate the values
+                    return aggregate(path.context.columns[terminal]).alias(alias)
+        else:
+            raise ValueError('Malformed source: %s' % str(source))
+    else:
+        raise ValueError('Malformed visible column: %s' % str(vizcol))
+
+
+def _datapath_contextualize(path, context_name='*', context_body=None, groupkey_name='RID'):
+    """Contextualizes a data path to a named visible columns context.
+
+    :param path: a datapath object
+    :param context_name: name of the context within the path's terminating table's "visible columns" annotations
+    :param context_body: a list of visible column definitions, if given, the `context_name` will be ignored
+    :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+    :return: a 'contextualized' attribute group query object
+    """
+    assert isinstance(path, DataPath)
+    path = copy.deepcopy(path)
+    context = path.context
+    table = context._wrapped_table
+    sources = table.annotations.get(_erm.tag.source_definitions, {}).get('sources')
+    vizcols = context_body if context_body else table.annotations.get(_erm.tag.visible_columns, {}).get(context_name, [])
+    if not vizcols:
+        raise ValueError('Visible columns context "%s" not found for table %s:%s' % (context_name, table.schema.name, table.name))
+    groupkey = context.columns[groupkey_name]
+    projection = []
+
+    for vizcol in vizcols:
+        try:
+            projection.append(_datapath_deserialize_vizcolumn(path, vizcol, sources=sources))
+            path.context = context
+        except ValueError as e:
+            logger.warning(str(e))
+
+    def not_same_as_group_key(x):
+        assert isinstance(groupkey, _ColumnWrapper)
+        if not isinstance(x, _ColumnWrapper):
+            return True
+        return groupkey._wrapped_column != x._wrapped_column
+
+    projection = filter(not_same_as_group_key, projection)  # project groupkey only once
+    query = path.groupby(groupkey).attributes(*projection)
+    return query
+
+
+def _datapath_generate_simple_denormalization(path, include_whole_entities=False):
+    """Generates a denormalized form of the table expressed in a visible columns specification.
+
+    :param path: a datapath object
+    :param include_whole_entities: if a denormalization cannot find a 'name' like terminal, include the whole entity (i.e., all attributes), else return just the 'RID'
+    :return: a generated visible columns specification based on a denormalization heuristic
+    """
+    assert isinstance(path, DataPath)
+    context = path.context
+    table = context._wrapped_table
+
+    fkeys = list(table.foreign_keys)
+    single_column_fkeys = {
+        fkey.foreign_key_columns[0].name: fkey
+        for fkey in table.foreign_keys if len(fkey.foreign_key_columns) == 1
+    }
+
+    def _fkey_to_vizcol(name, fk, inbound=None):
+        # name columns to look for in related tables
+        name_candidates = [
+            'displayname',
+            'preferredname',
+            'fullname',
+            'name',
+            'title',
+            'label'
+        ]
+
+        # determine terminal column
+        terminal = 'RID'
+        for candidate_col in fk.pk_table.columns:
+            if candidate_col.name.lower().replace(' ', '').replace('_', '') in name_candidates:
+                terminal = candidate_col.name
+                break
+
+        # define source path
+        source = [{'outbound': fk.names[0]}, terminal]
+        if inbound:
+            source = [{'inbound': inbound.names[0]}] + source
+
+        # return vizcol spec
+        return {
+            'markdown_name': name,
+            'source': source,
+            'entity': include_whole_entities and terminal == 'RID'
+        }
+
+    # assemble the visible column:
+    #  1. column or single column fkeys
+    #  2. all other (outbound fkey) related tables
+    #  3. all associated tables
+    vizcols = []
+    for col in table.column_definitions:
+        if col.name in single_column_fkeys:
+            fkey = single_column_fkeys[col.name]
+            vizcols.append(_fkey_to_vizcol(col.name, fkey))
+            del single_column_fkeys[col.name]
+            fkeys.remove(fkey)
+        else:
+            vizcols.append(col.name)
+
+    for outbound_fkey in fkeys:
+        vizcols.append(_fkey_to_vizcol(outbound_fkey.constraint_name, outbound_fkey))
+
+    for inbound_fkey in table.referenced_by:
+        if inbound_fkey.table.is_association():
+            vizcols.append(
+                _fkey_to_vizcol(
+                    inbound_fkey.table.name,
+                    inbound_fkey.table.foreign_keys[0] if inbound_fkey != inbound_fkey.table.foreign_keys[0] else inbound_fkey.table.foreign_keys[1],
+                    inbound=inbound_fkey
+                )
+            )
+
+    return vizcols
+
+def simple_denormalization(path):
+    """A simple heuristic denormalization."""
+    return _datapath_generate_simple_denormalization(path)
+
+def simple_denormalization_with_whole_entities(path):
+    """A simple heuristic denormalization with related and associated entities."""
+    return _datapath_generate_simple_denormalization(path, include_whole_entities=True)
+
+def _datapath_denormalize(path, context_name=None, heuristic=None, groupkey_name='RID'):
+    """Denormalizes a path based on annotations or heuristics.
+
+    :param path: a DataPath object
+    :param context_name: name of the visible-columns context or if none given, will attempt apply heuristics
+    :param heuristic: heuristic to apply if no context name specified
+    :param groupkey_name: column name for the group by key of the generated query expression (default: 'RID')
+    """
+    assert isinstance(path, DataPath)
+    assert context_name is None or isinstance(context_name, str)
+    assert isinstance(groupkey_name, str)
+    heuristic = heuristic or simple_denormalization
+    assert callable(heuristic)
+    return _datapath_contextualize(
+        path,
+        context_name=context_name,
+        context_body=None if context_name else heuristic(path),
+        groupkey_name=groupkey_name
+    )

{deriva-1.7.0 → deriva-1.7.1}/deriva/transfer/__init__.py

@@ -1,9 +1,11 @@
 from deriva.transfer.download.deriva_download import DerivaDownload, GenericDownloader, DerivaDownloadError, \
-    DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError
+    DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, \
+    DerivaDownloadBaggingError
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
 
 from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
-    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError
+    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
+    DerivaUploadAuthenticationError
 from deriva.transfer.upload.deriva_upload_cli import DerivaUploadCLI
 
 from deriva.transfer.backup.deriva_backup import DerivaBackup, DerivaBackupAuthenticationError, \

{deriva-1.7.0 → deriva-1.7.1}/deriva/transfer/download/__init__.py

@@ -16,3 +16,7 @@ class DerivaDownloadAuthorizationError(Exception):
 
 class DerivaDownloadTimeoutError(Exception):
     pass
+
+
+class DerivaDownloadBaggingError(Exception):
+    pass