deriva 1.7.7__py3-none-any.whl → 1.7.9__py3-none-any.whl

deriva/config/acl_config.py

@@ -291,7 +291,8 @@ class AclConfig:
         elif group.startswith(self.ROBOT_PREFIX_FORMAT.format(server=self.server)):
             self.validate_webauthn_robot(group)
         else:
-            warnings.warn("Can't determine format of group '{g}'".format(g=group))
+            if self.verbose:
+                warnings.warn("Can't determine format of group '{g}'".format(g=group))
 
     def validate_globus_group(self, group):
         guid = group[len(self.GLOBUS_PREFIX):]

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.7"
+__version__ = "1.7.9"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs
@@ -55,7 +55,7 @@ def get_credential(host,
             scope_map = globus_client.hosts_to_scope_map(hosts=[host], match_scope_tag=match_scope_tag,
                                                          force_refresh=force_scope_lookup,
                                                          warn_on_discovery_failure=True if not creds else False)
-            tokens = globus_client.is_logged_in(exclude_defaults=True)
+            tokens = globus_client.is_logged_in(exclude_defaults=True, hosts_to_scope_map=scope_map)
             if tokens:
                 # 1. look for the explicitly requested scope in the token store, if specified
                 token = globus_client.find_access_token_for_scope(requested_scope, tokens)

deriva/core/catalog_cli.py

@@ -6,7 +6,7 @@ import traceback
 from pprint import pp
 from requests.exceptions import HTTPError, ConnectionError
 from deriva.core import __version__ as VERSION, BaseCLI, KeyValuePairArgs, DerivaServer, DerivaPathError, \
-    get_credential, format_credential, format_exception, DEFAULT_HEADERS
+    get_credential, format_credential, format_exception, read_config, DEFAULT_SESSION_CONFIG, DEFAULT_HEADERS
 from deriva.core.ermrest_model import nochange
 from deriva.core.utils import eprint
 
@@ -191,6 +191,11 @@ class DerivaCatalogCLI (BaseCLI):
         else:
             return get_credential(host_name)
 
+    @staticmethod
+    def _get_session_config():
+        config = read_config()
+        return config.get("session", DEFAULT_SESSION_CONFIG)
+
     def _post_parser_init(self, args):
         """Shared initialization for all sub-commands.
         """
@@ -202,7 +207,8 @@ class DerivaCatalogCLI (BaseCLI):
                                    credentials=DerivaCatalogCLI._get_credential(
                                        self.host,
                                        token=args.token,
-                                       oauth2_token=args.oauth2_token))
+                                       oauth2_token=args.oauth2_token),
+                                   session_config=DerivaCatalogCLI._get_session_config())
 
     @staticmethod
     def _decorate_headers(headers, file_format, method="get"):
@@ -279,10 +285,10 @@ class DerivaCatalogCLI (BaseCLI):
         catalog = self.server.connect_ermrest(args.id)
         try:
             if args.output_file:
-                catalog.getAsFile(args.path,
-                                  destfilename=args.output_file,
-                                  headers=headers,
-                                  delete_if_empty=args.auto_delete)
+                catalog.get_as_file(args.path,
+                                    destfilename=args.output_file,
+                                    headers=headers,
+                                    delete_if_empty=args.auto_delete)
             else:
                 pp(catalog.get(args.path, headers=headers).json())
         except HTTPError as e:

deriva/core/deriva_binding.py

@@ -139,9 +139,10 @@ class DerivaBinding (object):
             server
         )
         self._server_uri = self._base_server_uri
+        self._auth_uri = self._base_server_uri + "/authn/session"
 
-        self.session_config = DEFAULT_SESSION_CONFIG if not session_config else session_config
         self._session = None
+        self.session_config = DEFAULT_SESSION_CONFIG if not session_config else session_config
         self._get_new_session(self.session_config)
 
         self._caching = caching
@@ -160,9 +161,6 @@ class DerivaBinding (object):
         self._close_session()
         self._session = get_new_requests_session(self._server_uri + '/',
                                                  session_config if session_config else self.session_config)
-        # allow loopback requests to bypass SSL cert verification
-        if "https://localhost" in self._server_uri:
-            self._session.verify = False
 
     def _pre_get(self, path, headers):
         self.check_path(path)
@@ -222,19 +220,21 @@ class DerivaBinding (object):
             self._session.headers.update({'Authorization': 'Bearer {token}'.format(token=credentials['bearer-token'])})
         elif 'cookie' in credentials:
             cname, cval = credentials['cookie'].split('=', 1)
-            self._session.cookies.set(cname, cval, domain=server, path='/')
+            # Fix for cookielib domain rewrite to *.local when no "." in hostname. In this case, don't set the domain.
+            # Covers "localhost" and other dev/test scenarios. See "https://github.com/psf/requests/issues/5388"
+            self._session.cookies.set(cname, cval, domain="" if "." not in server else server, path='/')
         elif 'username' in credentials and 'password' in credentials:
             self.post_authn_session(credentials)
 
     def get_authn_session(self):
         headers = { 'deriva-client-context': self.dcctx.encoded() }
-        r = self._session.get(self._base_server_uri + "/authn/session", headers=headers)
+        r = self._session.get(self._auth_uri, headers=headers)
         _response_raise_for_status(r)
         return r
 
     def post_authn_session(self, credentials):
         headers = { 'deriva-client-context': self.dcctx.encoded() }
-        r = self._session.post(self._base_server_uri + "/authn/session", data=credentials, headers=headers)
+        r = self._session.post(self._auth_uri, data=credentials, headers=headers)
         _response_raise_for_status(r)
         return r
 

deriva/core/ermrest_catalog.py

@@ -481,12 +481,26 @@ class ErmrestCatalog(DerivaBinding):
     def getAsFile(self,
                   path,
                   destfilename,
-                  headers=DEFAULT_HEADERS,
-                  callback=None,
-                  delete_if_empty=False,
-                  paged=False,
-                  page_size=DEFAULT_PAGE_SIZE,
-                  page_sort_columns=frozenset(["RID"])):
+                  headers = DEFAULT_HEADERS,
+                  callback = None,
+                  delete_if_empty = False,
+                  paged = False,
+                  page_size = DEFAULT_PAGE_SIZE,
+                  page_sort_columns = frozenset(["RID"])):
+        """
+           Deprecated, call `get_as_file` instead.
+        """
+        self.get_as_file(path, destfilename, headers, callback, delete_if_empty, paged, page_size, page_sort_columns)
+
+    def get_as_file(self,
+                    path,
+                    destfilename,
+                    headers=DEFAULT_HEADERS,
+                    callback=None,
+                    delete_if_empty=False,
+                    paged=False,
+                    page_size=DEFAULT_PAGE_SIZE,
+                    page_sort_columns=frozenset(["RID"])):
         """
            Retrieve catalog data streamed to destination file.
            Caller is responsible to clean up file even on error, when the file may or may not exist.

deriva/core/ermrest_model.py

@@ -188,7 +188,7 @@ def equivalent(doc1, doc2, method=None):
         def canon_cat_acls(d):
             return {
                 k: d.get(k, [])
-                for k in {'owner', 'read', 'write', 'insert', 'update', 'delete'}
+                for k in {'owner', 'enumerate', 'write', 'select', 'insert', 'update', 'delete'}
             }
         return equivalent(canon_cat_acls(doc1), canon_cat_acls(doc2), method='acls')
     elif method == 'foreign_key_acls':

deriva/core/utils/core_utils.py

@@ -48,7 +48,8 @@ DEFAULT_SESSION_CONFIG = {
     "allow_retry_on_all_methods": False,
     "cookie_jar": DEFAULT_COOKIE_JAR_FILE,
     "max_request_size": DEFAULT_MAX_REQUEST_SIZE,
-    "max_chunk_limit": DEFAULT_MAX_CHUNK_LIMIT
+    "max_chunk_limit": DEFAULT_MAX_CHUNK_LIMIT,
+    "bypass_cert_verify_host_list": []
 }
 OAUTH2_SCOPES_KEY = "oauth2_scopes"
 DEFAULT_CONFIG = {
@@ -189,11 +190,15 @@ def get_new_requests_session(url=None, session_config=DEFAULT_SESSION_CONFIG):
                     not session_config.get("allow_retry_on_all_methods", False) else False,
                     raise_on_status=True)
     adapter = TimeoutHTTPAdapter(timeout=session_config.get("timeout", DEFAULT_REQUESTS_TIMEOUT), max_retries=retries)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+
     if url:
-        session.mount(url, adapter)
-    else:
-        session.mount('http://', adapter)
-        session.mount('https://', adapter)
+        # allow whitelisted hosts to bypass SSL cert verification
+        upr = urlparse(url)
+        bypass_cert_verify_host_list = session_config.get("bypass_cert_verify_host_list", [])
+        if upr.scheme == "https" and upr.hostname in bypass_cert_verify_host_list:
+            session.verify = False
 
     return session
 
@@ -301,7 +306,7 @@ def get_oauth_scopes_for_host(host,
                 result = scopes
                 break
     if not result or force_refresh:
-        session = get_new_requests_session(session_config=DEFAULT_SESSION_CONFIG)
+        session = get_new_requests_session(url, session_config=config.get("session", DEFAULT_SESSION_CONFIG))
         try:
             r = session.get(url, headers=DEFAULT_HEADERS)
             r.raise_for_status()

deriva/core/utils/globus_auth_utils.py

@@ -5,7 +5,7 @@ import json
 import logging
 import platform
 import traceback
-import importlib
+import time
 import datetime
 import tzlocal
 import webbrowser
@@ -15,7 +15,8 @@ from bdbag.fetch.auth import keychain as bdbkc
 from deriva.core import __version__ as VERSION, DEFAULT_CONFIG_PATH, DEFAULT_GLOBUS_CREDENTIAL_FILE, urlparse, urljoin,\
     read_config, format_exception, BaseCLI, get_oauth_scopes_for_host, get_new_requests_session
 from deriva.core.utils import eprint
-from globus_sdk import ConfidentialAppAuthClient, AuthClient, AccessTokenAuthorizer, GlobusError, GlobusAPIError
+from globus_sdk import ConfidentialAppAuthClient, AuthClient, AccessTokenAuthorizer, RefreshTokenAuthorizer, \
+    GlobusError, GlobusAPIError, AuthAPIError
 from fair_research_login.client import NativeClient, LoadError, NoSavedTokens, TokensExpired
 
 NATIVE_APP_CLIENT_ID = "8ef15ba9-2b4a-469c-a163-7fd910c9d111"
@@ -586,12 +587,47 @@ class GlobusNativeLogin:
         if not scopes:
             return None
         try:
-            return self.client.load_tokens(scopes)
+            tokens = self.client.load_tokens(list(scopes))
+            return self.refresh_tokens_preemptively(tokens, time_before_expiry=86400)
         except LoadError as e:
             logging.debug("Unable to load or find tokens for specified scopes [%s]: %s" %
                           (scopes, format_exception(e)))
             return None
 
+    def refresh_tokens_preemptively(self, tokens, time_before_expiry=0):
+        needs_refresh = {}
+
+        for rs, ts in {t: ts for t, ts in tokens.items() if bool(ts['refresh_token'])}.items():
+            expiry_threshold = int(time.time()) + time_before_expiry
+            if ts['expires_at_seconds'] < expiry_threshold:
+                needs_refresh.update({rs:ts})
+
+        if not needs_refresh:
+            return tokens
+
+        for rs, ts in needs_refresh.items():
+            authorizer = RefreshTokenAuthorizer(
+                ts['refresh_token'],
+                self.client.client,
+                access_token=ts['access_token'],
+                expires_at=int(time.time()),
+                # setting the expires_at to the current time is vicious hack but necessary since the globus_sdk
+                # client doesn't allow you to refresh an access token unless it has already expired
+            )
+            try:
+                authorizer.ensure_valid_token()
+                ts['access_token'] = authorizer.access_token
+                ts['expires_at_seconds'] = authorizer.expires_at
+            except AuthAPIError as e:
+                if e.message == 'invalid_grant':
+                    logging.warning('Refresh Token expired for resource server: %s', rs)
+
+        self.client.save_tokens(needs_refresh)
+        result = {rs: ts for rs, ts in tokens.items() if rs not in needs_refresh}
+        result.update(needs_refresh)
+
+        return result
+
     def hosts_to_scope_map(self,
                            hosts,
                            match_scope_tag=None,
@@ -662,6 +698,7 @@ class GlobusNativeLogin:
             access_token = token.get("access_token")
             if (token_scope == scope) and (access_token is not None):
                 return access_token
+        return None
 
     def update_bdbag_keychain(self, token=None, host=None, keychain_file=None, allow_redirects=False, delete=False):
         if (token is None) or (host is None):
@@ -699,7 +736,7 @@ class GlobusNativeLogin:
             prefill_named_grant = self.client.app_name + " with requested scopes [%s] " % ", ".join(scopes)
         tokens = self.client.login(no_local_server=no_local_server,
                                    no_browser=no_browser,
-                                   requested_scopes=scopes,
+                                   requested_scopes=list(scopes),
                                    refresh_tokens=refresh_tokens,
                                    prefill_named_grant=prefill_named_grant,
                                    query_params=additional_params,

deriva/transfer/__init__.py

@@ -4,8 +4,8 @@ from deriva.transfer.download.deriva_download import DerivaDownload, GenericDown
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
 from deriva.transfer.download.deriva_export import DerivaExport, DerivaExportCLI
 
-from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
-    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
+from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, UploadState, DerivaUploadError, \
+    DerivaUploadError, DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
     DerivaUploadAuthenticationError
 from deriva.transfer.upload.deriva_upload_cli import DerivaUploadCLI
 

deriva/transfer/download/deriva_export.py

@@ -5,7 +5,6 @@ import traceback
 import requests
 import argparse
 import logging
-import certifi
 import datetime
 from collections.abc import Mapping, Iterable
 from requests.exceptions import HTTPError, ConnectionError, Timeout
@@ -46,6 +45,7 @@ class DerivaExport:
         self.timeout = kwargs.get("timeout")
         self.export_type = kwargs.get("export_type", "bdbag")
         self.base_server_uri = "https://" + self.host
+        self.auth_service_url = self.base_server_uri + "/authn/session"
         self.service_url = self.base_server_uri + EXPORT_SERVICE_PATH % self.export_type
         self.session_config = DEFAULT_SESSION_CONFIG.copy()
         if isinstance(self.timeout, tuple):
@@ -78,16 +78,23 @@ class DerivaExport:
             raise DerivaDownloadAuthenticationError(
                 "The requested service requires authentication and a valid login credential could "
                 "not be found (or was not provided) for the specified host.")
+
         if 'bearer-token' in self.credential:
             self.session.headers.update(
                 {'Authorization': 'Bearer {token}'.format(token=self.credential['bearer-token'])})
         elif 'cookie' in self.credential:
             cname, cval = self.credential['cookie'].split('=', 1)
-            self.session.cookies.set(cname, cval, domain=self.host, path='/')
+            self.session.cookies.set(cname, cval, domain="" if "." not in self.host else self.host, path='/')
+        else:
+            try:
+                r = self.session.post(self.auth_service_url, data=self.credential)
+                r.raise_for_status()
+            except HTTPError as e:
+                raise DerivaDownloadAuthenticationError(
+                    "Exception during POST authentication flow: %s" % format_exception(e))
 
     def validate_authn_session(self):
-        url = self.base_server_uri + "/authn/session"
-        r = self.session.get(url)
+        r = self.session.get(self.auth_service_url)
         if r.status_code == requests.codes.not_found or r.status_code == requests.codes.unauthorized:
             logger.warning("Unable to authenticate. Check for missing or expired credentials.")
         r.raise_for_status()
@@ -125,7 +132,7 @@ class DerivaExport:
 
         filename = parse_content_disposition(content_disposition)
         output_path = os.path.abspath(os.path.join(self.output_dir, filename))
-        with self.session.get(url, stream=True, verify=certifi.where()) as r:
+        with self.session.get(url, stream=True) as r:
             if r.status_code != 200:
                 file_error = "File [%s] transfer failed." % output_path
                 url_error = 'HTTP GET Failed for url: %s' % url

deriva/transfer/download/processors/query/base_query_processor.py

@@ -1,6 +1,5 @@
 import os
 import errno
-import certifi
 import requests
 from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, format_exception, DEFAULT_SESSION_CONFIG
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
@@ -72,13 +71,13 @@ class BaseQueryProcessor(BaseProcessor):
             make_dirs(output_dir)
         try:
             if as_file:
-                return self.catalog.getAsFile(self.query, self.output_abspath,
-                                              headers=headers,
-                                              callback=self.callback,
-                                              delete_if_empty=True,
-                                              paged=self.paged_query,
-                                              page_size=self.paged_query_size,
-                                              page_sort_columns=self.paged_query_sort_columns)
+                return self.catalog.get_as_file(self.query, self.output_abspath,
+                                                headers=headers,
+                                                callback=self.callback,
+                                                delete_if_empty=True,
+                                                paged=self.paged_query,
+                                                page_size=self.paged_query_size,
+                                                page_sort_columns=self.paged_query_sort_columns)
             else:
                 return self.catalog.get(self.query, headers=headers).json()
         except requests.HTTPError as e:
@@ -143,31 +142,31 @@ class BaseQueryProcessor(BaseProcessor):
 
         return url
 
-    def getExternalSession(self, host):
-        sessions = self.sessions
+    def getExternalSession(self, url):
         auth_params = self.kwargs.get("auth_params", dict())
         cookies = auth_params.get("cookies")
         auth_url = auth_params.get("auth_url")
         login_params = auth_params.get("login_params")
         session_config = self.kwargs.get("session_config")
 
-        session = sessions.get(host)
+        host = urlsplit(url).hostname
+        session = self.sessions.get(host)
         if session is not None:
             return session
 
         if not session_config:
             session_config = DEFAULT_SESSION_CONFIG
-        session = get_new_requests_session(session_config=session_config)
+        session = get_new_requests_session(url, session_config=session_config)
 
         if cookies:
             session.cookies.update(cookies)
         if login_params and auth_url:
-            r = session.post(auth_url, data=login_params, verify=certifi.where())
+            r = session.post(auth_url, data=login_params)
             if r.status_code > 203:
                 raise DerivaDownloadError(
                     'GetExternalSession Failed with Status Code: %s\n%s\n' % (r.status_code, r.text))
 
-        sessions[host] = session
+        self.sessions[host] = session
         return session
 
     def create_default_paths(self):

deriva/transfer/download/processors/query/file_download_query_processor.py

@@ -4,7 +4,6 @@ import uuid
 import datetime
 import logging
 import requests
-import certifi
 from bdbag import bdbag_ro as ro
 from deriva.core import urlsplit, format_exception, get_transfer_summary, make_dirs, DEFAULT_CHUNK_SIZE
 from deriva.core.utils.mime_utils import parse_content_disposition
@@ -32,14 +31,13 @@ class FileDownloadQueryProcessor(BaseQueryProcessor):
         return self.outputs
 
     def getExternalFile(self, url, output_path, headers=None):
-        host = urlsplit(url).netloc
         if output_path:
             if not headers:
                 headers = self.HEADERS.copy()
             else:
                 headers.update(self.HEADERS)
-            session = self.getExternalSession(host)
-            with session.get(url, headers=headers, stream=True, verify=certifi.where()) as r:
+            session = self.getExternalSession(url)
+            with session.get(url, headers=headers, stream=True) as r:
                 if r.status_code != 200:
                     file_error = "File [%s] transfer failed." % output_path
                     url_error = 'HTTP GET Failed for url: %s' % url
@@ -59,6 +57,7 @@ class FileDownloadQueryProcessor(BaseQueryProcessor):
                     length = int(r.headers.get('Content-Length'))
                     content_type = r.headers.get("Content-Type")
                     return output_path, length, content_type
+        return None
 
     def downloadFiles(self, input_manifest):
         logging.info("Attempting to download file(s) based on the results of query: %s" % self.query)

deriva/transfer/upload/deriva_upload.py

@@ -157,8 +157,12 @@ class DerivaUpload(object):
 
         # determine identity
         if self.credentials:
-            attributes = self.catalog.get_authn_session().json()
-            self.identity = attributes.get("client", self.identity)
+            try:
+                attributes = self.catalog.get_authn_session().json()
+                self.identity = attributes.get("client", self.identity)
+            except Exception as e:
+                # not a big deal since the credential token being used could be expired
+                logger.debug("Unable to determine user identity from existing credential (may be expired): %s" % e)
 
         # init dcctx cid to a default
         self.set_dcctx_cid(self.dcctx_cid)
@@ -342,7 +346,7 @@ class DerivaUpload(object):
         return '%s:%s' % (urlquote(schema_name), urlquote(table_name))
 
     @staticmethod
-    def interpolateDict(src, dst, allowNone=False):
+    def interpolateDict(src, dst, allow_none_column_list=[]):
         if not (isinstance(src, dict) and isinstance(dst, dict)):
             raise ValueError("Invalid input parameter type(s): (src = %s, dst = %s), expected (dict, dict)" % (
                 type(src).__name__, type(dst).__name__))
@@ -356,26 +360,27 @@ class DerivaUpload(object):
         for k, v in dst.items():
             try:
                 value = v.format(**src)
+                if k in src:
+                    v_type = type(src[k])
+                    value = v_type(value)
             except KeyError:
                 value = v
                 if value:
                     if value.startswith('{') and value.endswith('}'):
                         value = None
             dst.update({k: value})
-        # remove all None valued entries in the dest, if disallowed
-        if not allowNone:
-            empty = [k for k, v in dst.items() if v is None]
-            for k in empty:
-                del dst[k]
+        # remove all None valued entries in the dest, if column is not explicitly allowed
+        dst = {k: v for k, v in dst.items() if v is not None or k in allow_none_column_list}
 
         return dst
 
     @staticmethod
-    def pruneDict(src, dst, stringify=True):
+    def pruneDict(src, dst, allow_none_column_list=[]):
         dst = dst.copy()
         for k in dst.keys():
             value = src.get(k)
-            dst[k] = str(value) if (stringify and value is not None) else value
+            if value is not None or (value is None and k in allow_none_column_list):
+                dst[k] = value
         return dst
 
     def getCurrentConfigFilePath(self):
@@ -712,7 +717,8 @@ class DerivaUpload(object):
             self.metadata["URI"] = versioned_uri
         else:
             self.metadata["URI"] = versioned_uri.rsplit(":")[0]
-        self.metadata["URI_urlencoded"] = urlquote(self.metadata["URI"])
+        safe_overrides = asset_mapping.get("url_encoding_safe_overrides", {}).get("URI", "")
+        self.metadata["URI_urlencoded"] = urlquote(self.metadata["URI"], safe=safe_overrides)
 
         # 7. Check for an existing record and create a new one if necessary
         if not record:
@@ -720,7 +726,8 @@ class DerivaUpload(object):
 
         # 8. Update an existing record, if necessary
         column_map = asset_mapping.get("column_map", {})
-        updated_record = self.interpolateDict(self.metadata, column_map)
+        allow_none_col_list = asset_mapping.get("allow_empty_columns_on_update", [])
+        updated_record = self.interpolateDict(self.metadata, column_map, allow_none_col_list)
         if updated_record != record:
             record_update_template = asset_mapping.get("record_update_template")
             require_record_update_template = stob(asset_mapping.get("require_record_update_template", False))
@@ -778,6 +785,7 @@ class DerivaUpload(object):
         """
         record = None
         column_map = asset_mapping.get("column_map", {})
+        allow_none_col_list = asset_mapping.get("allow_empty_columns_on_update", [])
         rqt = asset_mapping['record_query_template']
         try:
             path = rqt.format(**self.metadata)
@@ -787,14 +795,14 @@ class DerivaUpload(object):
         if result:
             record = result[0]
             self._updateFileMetadata(record, no_overwrite=True)
-            return self.pruneDict(record, column_map), record
+            return self.pruneDict(record, column_map, allow_none_col_list), record
         else:
             row = self.interpolateDict(self.metadata, column_map)
             result = self._catalogRecordCreate(self.metadata['target_table'], row)
             if result:
                 record = result[0]
                 self._updateFileMetadata(record)
-            return self.interpolateDict(self.metadata, column_map, allowNone=True), record
+            return self.interpolateDict(self.metadata, column_map, allow_none_column_list=allow_none_col_list), record
 
     def _urlEncodeMetadata(self, safe_overrides=None):
         urlencoded = dict()
@@ -886,12 +894,14 @@ class DerivaUpload(object):
     def _getFileHatracMetadata(self, asset_mapping):
         try:
             hatrac_templates = asset_mapping["hatrac_templates"]
+            # convert None values to empty strings for URI and content-disposition template replacement
+            metadata = {k: ('' if v is None else v) for k, v in self.metadata.items()}
             # URI is required
-            self.metadata["URI"] = hatrac_templates["hatrac_uri"].format(**self.metadata)
+            self.metadata["URI"] = hatrac_templates["hatrac_uri"].format(**metadata)
             # overridden content-disposition is optional
             content_disposition = hatrac_templates.get("content-disposition")
             if content_disposition:
-                filename = content_disposition.format(**self.metadata)
+                filename = content_disposition.format(**metadata)
             else:
                 filename = urlparse(self.metadata["URI"]).path.rsplit("/", 1)[-1]
 

{deriva-1.7.7.dist-info → deriva-1.7.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: deriva
-Version: 1.7.7
+Version: 1.7.9
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division
@@ -25,14 +25,26 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging
 Requires-Dist: requests
-Requires-Dist: certifi
 Requires-Dist: pika
-Requires-Dist: urllib3 <3,>=1.26
-Requires-Dist: portalocker >=1.2.1
-Requires-Dist: bdbag >=1.7.3
-Requires-Dist: globus-sdk <4,>=3
-Requires-Dist: fair-research-login >=0.3.1
-Requires-Dist: fair-identifiers-client >=0.5.1
-Requires-Dist: jsonschema >=3.1
+Requires-Dist: urllib3<3,>=1.26.20
+Requires-Dist: portalocker>=1.2.1
+Requires-Dist: bdbag>=1.7.5
+Requires-Dist: globus_sdk<4,>=3
+Requires-Dist: fair-research-login>=0.3.1
+Requires-Dist: fair-identifiers-client>=0.5.1
+Requires-Dist: jsonschema>=3.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: maintainer
+Dynamic: maintainer-email
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 For further information, visit the project [homepage](https://github.com/informatics-isi-edu/deriva-py).

{deriva-1.7.7.dist-info → deriva-1.7.9.dist-info}/RECORD

@@ -1,6 +1,6 @@
 deriva/__init__.py,sha256=h-QyvMVzDNpT3jyVskcSbUVFXxGCRxieFPrvTveZG9k,64
 deriva/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deriva/config/acl_config.py,sha256=yZ0g_Cwv1MZXajUP2sVbc928XakUDu5Q97w9aK3VCCE,23946
+deriva/config/acl_config.py,sha256=JK6OKJ9OEaloY2vFZzD9leA3p4yeAtc9QEVhTYEipc0,23979
 deriva/config/annotation_config.py,sha256=sWcJnRfS5vcb85IdUOd420cmFHRyb4br5OxLCOssbec,11714
 deriva/config/annotation_validate.py,sha256=ZN0jq6As49hfsAmWONUxAStqFG9TeYzvCmRU9TQ-zD8,5228
 deriva/config/base_config.py,sha256=Y5sFotuAWWyo-LZvzbzVY2ZDaC-zhwbMXusgBF1OJYM,20724
@@ -8,15 +8,15 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
 deriva/config/rollback_annotation.py,sha256=vqrIcen-KZX8LDpu2OVNivzIHpQoQgWkZAChZJctvtk,3015
 deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
 deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
-deriva/core/__init__.py,sha256=D7aya8r3BTbewaSUmhbb4DB-Ltwe2aXV10CMfPo13rI,4945
+deriva/core/__init__.py,sha256=NKiBln9VluAuOX25MSfa-xDpYY86NwJY1civQ_Kg5QY,4975
 deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
 deriva/core/base_cli.py,sha256=78Ilf3_f2xREQb3IIj6q0jwWAiXSObZszG0JURs36lA,2902
-deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
+deriva/core/catalog_cli.py,sha256=CwfTf7C81SpU1J_aPsWiIbPOBAyekkIh384KUivq5H8,23550
 deriva/core/datapath.py,sha256=w8LvPAd_DuknKxHc_YS6NUHhTY6XpCSkfa0m_xQUdZE,89068
-deriva/core/deriva_binding.py,sha256=_sA9HGrcVRqT-OhrneMDMOquyVOFOxLq3WzBQhasLIM,12970
+deriva/core/deriva_binding.py,sha256=6yGXIbnayDtb9LU-JVK43zk02-aQwshrKiaLi1pru-8,13086
 deriva/core/deriva_server.py,sha256=nsW3gwg1sIaHl3BTf-nL41AkSj3dEpcEBlatvjvN8CQ,200
-deriva/core/ermrest_catalog.py,sha256=_eqQg16i1aA95R99B7tLZxHWQlYk-rLpN_0zghfNWRc,54991
-deriva/core/ermrest_model.py,sha256=NdvrMLkmNTY4ncodzojpyV1XjgSlod7x-SXyUu65Z0Q,124921
+deriva/core/ermrest_catalog.py,sha256=2mShQgo35DIQ-0UOEqu2VgCDTE3sUrQSWVDm1jW_tQ8,55547
+deriva/core/ermrest_model.py,sha256=oydtO6IU54ihjWQXcq0nfCyqAFJSW2FI9Z_0FjU7Uk0,124936
 deriva/core/hatrac_cli.py,sha256=l9QmneLRHSMiG_z9S83ea0QGVhTS3Wq1KGPEKEDpecM,14522
 deriva/core/hatrac_store.py,sha256=NcVuO4h4hswbEAct8tTKZ1pNtXBn74Nn9TelKN_jr8Q,22323
 deriva/core/mmo.py,sha256=dcB8akgsqbYMi22ClbVpOKVL6so8FjSDjSb6gP4_jFo,17852
@@ -44,8 +44,8 @@ deriva/core/schemas/visible_columns.schema.json,sha256=-JKqvhfKT-5btKBtzKn3p6EgY
 deriva/core/schemas/visible_foreign_keys.schema.json,sha256=K-oa2qzj5EbmJCEyN6mN3vubblHENFxxbZEeQfFV_QQ,3364
 deriva/core/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/core/utils/__init__.py,sha256=XSbGaWe44hebxYvoh5huFzZkMY6TSKPOCRSjUOvaY70,124
-deriva/core/utils/core_utils.py,sha256=5xUjJ-1QrPFKOiwfSWFDdH8Hm5a-AEhzCvy2GJPjx4A,19500
-deriva/core/utils/globus_auth_utils.py,sha256=huPzKSr0d2NirpLzyp6CSyR1gIAdlRuP-X0jRo5W_p8,55732
+deriva/core/utils/core_utils.py,sha256=bMPVBG0W5-OAPcY7rXYlIfPAofmN1We6h1uyzXYnR9s,19821
+deriva/core/utils/globus_auth_utils.py,sha256=x5Dh4PlAMIKTm4b8nKUeuCMSFn7NMgj_NqtmnS_FM0I,57366
 deriva/core/utils/hash_utils.py,sha256=JqUYVB3jXusCQYX9fkKmweUKBC0WQi8ZI2N8m-uKygQ,2299
 deriva/core/utils/mime_utils.py,sha256=ZT7pMjY2kQWBsgsGC3jY6kjfygdsIyiYW3BKNw_pPyg,1128
 deriva/core/utils/version_utils.py,sha256=HWNUQAZrPXu0oGjhG6gMNm3kWtfa3nR50vfH2uQxBA0,2954
@@ -53,7 +53,7 @@ deriva/core/utils/webauthn_utils.py,sha256=rD0HQZAjUKp4NfqHQG1FhH3x7uKog2et7w7LB
 deriva/seo/__init__.py,sha256=dYn48A7blbeYf40b4T3KVofrQK4u5K5MfxXWfIGloig,54
 deriva/seo/sitemap_builder.py,sha256=Ht_AbodEERDofIoCcd4kPlrl1pVW670WN5dT4cc05LQ,13948
 deriva/seo/sitemap_cli.py,sha256=miCqRfpSj5Dx5BfJGSd8Pi2e4OOQjotDzP_JubukhCM,2654
-deriva/transfer/__init__.py,sha256=3a01U6e68kBEnFUp0dRMFLRDj2p50iIJF9RZaG3TXVI,1253
+deriva/transfer/__init__.py,sha256=VRxmElD5SIOGzrB1biUU8bP3_iXPMvJuau-9alU5oy0,1266
 deriva/transfer/backup/__init__.py,sha256=vxsZiDLMTJQPybXT89G-07GsUoLhnItTCbLdXcDSyeA,465
 deriva/transfer/backup/__main__.py,sha256=dT12--8C6sKGEtMhsYuy013ebXKpVnBJfhcQNlVtv6Y,361
 deriva/transfer/backup/deriva_backup.py,sha256=IO9Tmzx6jHfUCkP-41nSsAeOFLn9T-0HwQcpRLpM_zs,5228
@@ -62,7 +62,7 @@ deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w
 deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
 deriva/transfer/download/deriva_download.py,sha256=ulFrHHEDj3oJA2pAo7MGvGyDF9rA3l8yCUoK3FMvHEk,17100
 deriva/transfer/download/deriva_download_cli.py,sha256=wN8tyQDv1AIE_aDqjECbmkoEWN050vlEdJyteYbdgSs,3940
-deriva/transfer/download/deriva_export.py,sha256=R6f08eJSt1iBeDNNQhThN3W1l4UwE0xzy2DjXbqNskE,13085
+deriva/transfer/download/deriva_export.py,sha256=zvFVUUmtQfx8cDa35YiQXjZzjWXW0tpNolDwEC9q0JU,13449
 deriva/transfer/download/processors/__init__.py,sha256=evLp36tZn-Z_AMshdfV3JJO8w1es5owsnRN0IFJUwIo,4507
 deriva/transfer/download/processors/base_processor.py,sha256=R6IIHSa_euv4X2Dyhd8fvQAiVYDGJTWMQtPoukHQn-Q,3837
 deriva/transfer/download/processors/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,8 +71,8 @@ deriva/transfer/download/processors/postprocess/transfer_post_processor.py,sha25
 deriva/transfer/download/processors/postprocess/url_post_processor.py,sha256=s68iIYqQSZHtbv4y-fCG8pjhApAeMEG6hYcKx2Pvf5Y,2745
 deriva/transfer/download/processors/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/query/bag_fetch_query_processor.py,sha256=tiQtfuy01YgOFFD5b_sP7TGjMnt0Jqcg2gp1KNWqeLE,5645
-deriva/transfer/download/processors/query/base_query_processor.py,sha256=0tGxTJKCEqm7ecdOcfd8szA94muc2jbYwbqYvTGwAac,10465
-deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=Hg1NbKsaGJh9cB86yIyL7Fm7ywSNVop837Dv8aFXUes,7257
+deriva/transfer/download/processors/query/base_query_processor.py,sha256=LlABWacU_hCySg1dhBJnWeauay0JSGW_f2103D3nU8Y,10459
+deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=_oxaf0r6kxxS8UMc_D96YDeGGumlUICbNAzzbU5PTvw,7201
 deriva/transfer/download/processors/transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/transform/base_transform_processor.py,sha256=Ddw5gsNpDANeuLvUaF4utp8psaxOtAzlgXtOg8gb-Pc,4109
 deriva/transfer/download/processors/transform/column_transform_processor.py,sha256=X98XSwpGrw0xhSlQ4i7_EPMvqU0Zz5aZp9Dam2AgYRU,2627
@@ -87,7 +87,7 @@ deriva/transfer/restore/deriva_restore.py,sha256=s0h7cXit2USSdjrIfrj0dr7BJ0rrHHM
 deriva/transfer/restore/deriva_restore_cli.py,sha256=2ViZ1Lyl5ndXPKeJFCHHGnwzkg3DfHhTuRa_bN7eJm8,5603
 deriva/transfer/upload/__init__.py,sha256=4mlc_iUX-v7SpXzlCZmhxQtSiW5JeDGb2FX7bb1E6tY,304
 deriva/transfer/upload/__main__.py,sha256=hqnXtGpRqPthwpO6uvrnf_TQm7McheeyOt960hStSMY,340
-deriva/transfer/upload/deriva_upload.py,sha256=_kndWxoAHb9vKJERCC9tjExp2EWISrOzhBXnSEbFIKI,60766
+deriva/transfer/upload/deriva_upload.py,sha256=CFWhvsMl9Ndw6QJYvQoOsErwQuF1wOpgqGju8G0QgTQ,61708
 deriva/transfer/upload/deriva_upload_cli.py,sha256=-Q6xgiYabQziTQcMQdGNDAv-eLxCCHO-BCSo4umbDE4,5082
 deriva/transfer/upload/processors/__init__.py,sha256=sMM5xdJ82UIRdB1lGMKk7ft0BgtjS2oJ0sI4SQSqiIU,2481
 deriva/transfer/upload/processors/archive_processor.py,sha256=ID0lDwDn4vPe5nbxy6m28Ssj_TsZpK4df2xRrM6nJRQ,2015
@@ -97,6 +97,7 @@ deriva/transfer/upload/processors/metadata_update_processor.py,sha256=Hgu5huZf7Z
 deriva/transfer/upload/processors/rename_processor.py,sha256=UQ-JQuQgyYCGT-fU9kHA53kPdQ20kt-2Bb486od7B14,2423
 deriva/transfer/upload/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/utils/__init__.py,sha256=jv2YF__bseklT3OWEzlqJ5qE24c4aWd5F4r0TTjOrWQ,65
+deriva-1.7.9.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
 tests/deriva/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/deriva/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/deriva/core/test_datapath.py,sha256=0vUlQ48KYkDW5L7hzwyGAETa_40y0JmZDalk0W0fCq4,38722
@@ -108,9 +109,8 @@ tests/deriva/core/mmo/test_mmo_find.py,sha256=PcUN76sik68B3XKg0G3wHVpKcPEld_6Rtb
 tests/deriva/core/mmo/test_mmo_prune.py,sha256=4pYtYL8g1BgadlewNPVpVA5lT_gV6SPTDYf04ZKzBTA,6851
 tests/deriva/core/mmo/test_mmo_rename.py,sha256=4oSR1G3Od701Ss3AnolI1Z7CbMxKuQF2uSr2_IcoR6s,8512
 tests/deriva/core/mmo/test_mmo_replace.py,sha256=w-66LWyiQ_ajC7Ipmhc4kAKwIloPdQELeUPsvelTdX8,8439
-deriva-1.7.7.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-deriva-1.7.7.dist-info/METADATA,sha256=V1dOlM-ESgLHfPMX_EOZ-kI6jvouKBIIzBcwwZ_bnnM,1623
-deriva-1.7.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deriva-1.7.7.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
-deriva-1.7.7.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
-deriva-1.7.7.dist-info/RECORD,,
+deriva-1.7.9.dist-info/METADATA,sha256=QXs8Ca5GdL-Sw7SY41j0tYR5maXv16epbvoIoH6SnNU,1890
+deriva-1.7.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva-1.7.9.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
+deriva-1.7.9.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
+deriva-1.7.9.dist-info/RECORD,,

{deriva-1.7.7.dist-info → deriva-1.7.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any