deriva 1.7.7__py3-none-any.whl → 1.7.8__py3-none-any.whl

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.7"
+__version__ = "1.7.8"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/transfer/__init__.py

@@ -4,8 +4,8 @@ from deriva.transfer.download.deriva_download import DerivaDownload, GenericDown
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
 from deriva.transfer.download.deriva_export import DerivaExport, DerivaExportCLI
 
-from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
-    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
+from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, UploadState, DerivaUploadError, \
+    DerivaUploadError, DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
     DerivaUploadAuthenticationError
 from deriva.transfer.upload.deriva_upload_cli import DerivaUploadCLI
 

deriva/transfer/download/deriva_export.py

@@ -5,7 +5,6 @@ import traceback
 import requests
 import argparse
 import logging
-import certifi
 import datetime
 from collections.abc import Mapping, Iterable
 from requests.exceptions import HTTPError, ConnectionError, Timeout
@@ -125,7 +124,7 @@ class DerivaExport:
 
         filename = parse_content_disposition(content_disposition)
         output_path = os.path.abspath(os.path.join(self.output_dir, filename))
-        with self.session.get(url, stream=True, verify=certifi.where()) as r:
+        with self.session.get(url, stream=True) as r:
             if r.status_code != 200:
                 file_error = "File [%s] transfer failed." % output_path
                 url_error = 'HTTP GET Failed for url: %s' % url

deriva/transfer/download/processors/query/base_query_processor.py

@@ -1,6 +1,5 @@
 import os
 import errno
-import certifi
 import requests
 from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, format_exception, DEFAULT_SESSION_CONFIG
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
@@ -162,7 +161,7 @@ class BaseQueryProcessor(BaseProcessor):
         if cookies:
             session.cookies.update(cookies)
         if login_params and auth_url:
-            r = session.post(auth_url, data=login_params, verify=certifi.where())
+            r = session.post(auth_url, data=login_params)
             if r.status_code > 203:
                 raise DerivaDownloadError(
                     'GetExternalSession Failed with Status Code: %s\n%s\n' % (r.status_code, r.text))

deriva/transfer/download/processors/query/file_download_query_processor.py

@@ -4,7 +4,6 @@ import uuid
 import datetime
 import logging
 import requests
-import certifi
 from bdbag import bdbag_ro as ro
 from deriva.core import urlsplit, format_exception, get_transfer_summary, make_dirs, DEFAULT_CHUNK_SIZE
 from deriva.core.utils.mime_utils import parse_content_disposition
@@ -39,7 +38,7 @@ class FileDownloadQueryProcessor(BaseQueryProcessor):
             else:
                 headers.update(self.HEADERS)
             session = self.getExternalSession(host)
-            with session.get(url, headers=headers, stream=True, verify=certifi.where()) as r:
+            with session.get(url, headers=headers, stream=True) as r:
                 if r.status_code != 200:
                     file_error = "File [%s] transfer failed." % output_path
                     url_error = 'HTTP GET Failed for url: %s' % url

deriva/transfer/upload/deriva_upload.py

@@ -157,8 +157,12 @@ class DerivaUpload(object):
 
         # determine identity
         if self.credentials:
-            attributes = self.catalog.get_authn_session().json()
-            self.identity = attributes.get("client", self.identity)
+            try:
+                attributes = self.catalog.get_authn_session().json()
+                self.identity = attributes.get("client", self.identity)
+            except Exception as e:
+                # not a big deal since the credential token being used could be expired
+                logger.debug("Unable to determine user identity from existing credential (may be expired): %s" % e)
 
         # init dcctx cid to a default
         self.set_dcctx_cid(self.dcctx_cid)
@@ -342,7 +346,7 @@ class DerivaUpload(object):
         return '%s:%s' % (urlquote(schema_name), urlquote(table_name))
 
     @staticmethod
-    def interpolateDict(src, dst, allowNone=False):
+    def interpolateDict(src, dst, allow_none=False, allow_none_column_list=[]):
         if not (isinstance(src, dict) and isinstance(dst, dict)):
             raise ValueError("Invalid input parameter type(s): (src = %s, dst = %s), expected (dict, dict)" % (
                 type(src).__name__, type(dst).__name__))
@@ -362,10 +366,10 @@ class DerivaUpload(object):
                     if value.startswith('{') and value.endswith('}'):
                         value = None
             dst.update({k: value})
-        # remove all None valued entries in the dest, if disallowed
-        if not allowNone:
-            empty = [k for k, v in dst.items() if v is None]
-            for k in empty:
+        # remove all None valued entries in the dest, if globally disallowed or the column is not explicitly allowed
+        empty = [k for k, v in dst.items() if v is None]
+        for k in empty:
+            if not allow_none or (allow_none and k not in allow_none_column_list):
                 del dst[k]
 
         return dst
@@ -720,7 +724,9 @@ class DerivaUpload(object):
 
         # 8. Update an existing record, if necessary
         column_map = asset_mapping.get("column_map", {})
-        updated_record = self.interpolateDict(self.metadata, column_map)
+        allow_none_col_list = asset_mapping.get("allow_empty_columns_on_update", [])
+        allow_none = True if allow_none_col_list else False
+        updated_record = self.interpolateDict(self.metadata, column_map, allow_none, allow_none_col_list)
         if updated_record != record:
             record_update_template = asset_mapping.get("record_update_template")
             require_record_update_template = stob(asset_mapping.get("require_record_update_template", False))
@@ -794,7 +800,7 @@ class DerivaUpload(object):
             if result:
                 record = result[0]
                 self._updateFileMetadata(record)
-            return self.interpolateDict(self.metadata, column_map, allowNone=True), record
+            return self.interpolateDict(self.metadata, column_map, allow_none=True), record
 
     def _urlEncodeMetadata(self, safe_overrides=None):
         urlencoded = dict()

{deriva-1.7.7.dist-info → deriva-1.7.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.7
+Version: 1.7.8
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division
@@ -25,7 +25,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging
 Requires-Dist: requests
-Requires-Dist: certifi
 Requires-Dist: pika
 Requires-Dist: urllib3 <3,>=1.26
 Requires-Dist: portalocker >=1.2.1

{deriva-1.7.7.dist-info → deriva-1.7.8.dist-info}/RECORD

@@ -8,7 +8,7 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
 deriva/config/rollback_annotation.py,sha256=vqrIcen-KZX8LDpu2OVNivzIHpQoQgWkZAChZJctvtk,3015
 deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
 deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
-deriva/core/__init__.py,sha256=D7aya8r3BTbewaSUmhbb4DB-Ltwe2aXV10CMfPo13rI,4945
+deriva/core/__init__.py,sha256=ijQ1wwJffdlUmiyyLSKYYj1aNWZAZfw8U_j1FMO8lBk,4945
 deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
 deriva/core/base_cli.py,sha256=78Ilf3_f2xREQb3IIj6q0jwWAiXSObZszG0JURs36lA,2902
 deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
@@ -53,7 +53,7 @@ deriva/core/utils/webauthn_utils.py,sha256=rD0HQZAjUKp4NfqHQG1FhH3x7uKog2et7w7LB
 deriva/seo/__init__.py,sha256=dYn48A7blbeYf40b4T3KVofrQK4u5K5MfxXWfIGloig,54
 deriva/seo/sitemap_builder.py,sha256=Ht_AbodEERDofIoCcd4kPlrl1pVW670WN5dT4cc05LQ,13948
 deriva/seo/sitemap_cli.py,sha256=miCqRfpSj5Dx5BfJGSd8Pi2e4OOQjotDzP_JubukhCM,2654
-deriva/transfer/__init__.py,sha256=3a01U6e68kBEnFUp0dRMFLRDj2p50iIJF9RZaG3TXVI,1253
+deriva/transfer/__init__.py,sha256=VRxmElD5SIOGzrB1biUU8bP3_iXPMvJuau-9alU5oy0,1266
 deriva/transfer/backup/__init__.py,sha256=vxsZiDLMTJQPybXT89G-07GsUoLhnItTCbLdXcDSyeA,465
 deriva/transfer/backup/__main__.py,sha256=dT12--8C6sKGEtMhsYuy013ebXKpVnBJfhcQNlVtv6Y,361
 deriva/transfer/backup/deriva_backup.py,sha256=IO9Tmzx6jHfUCkP-41nSsAeOFLn9T-0HwQcpRLpM_zs,5228
@@ -62,7 +62,7 @@ deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w
 deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
 deriva/transfer/download/deriva_download.py,sha256=ulFrHHEDj3oJA2pAo7MGvGyDF9rA3l8yCUoK3FMvHEk,17100
 deriva/transfer/download/deriva_download_cli.py,sha256=wN8tyQDv1AIE_aDqjECbmkoEWN050vlEdJyteYbdgSs,3940
-deriva/transfer/download/deriva_export.py,sha256=R6f08eJSt1iBeDNNQhThN3W1l4UwE0xzy2DjXbqNskE,13085
+deriva/transfer/download/deriva_export.py,sha256=unL3PNgmvZ64ruGjzRigx45LQstvySfDgi0-2lOxo_w,13046
 deriva/transfer/download/processors/__init__.py,sha256=evLp36tZn-Z_AMshdfV3JJO8w1es5owsnRN0IFJUwIo,4507
 deriva/transfer/download/processors/base_processor.py,sha256=R6IIHSa_euv4X2Dyhd8fvQAiVYDGJTWMQtPoukHQn-Q,3837
 deriva/transfer/download/processors/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,8 +71,8 @@ deriva/transfer/download/processors/postprocess/transfer_post_processor.py,sha25
 deriva/transfer/download/processors/postprocess/url_post_processor.py,sha256=s68iIYqQSZHtbv4y-fCG8pjhApAeMEG6hYcKx2Pvf5Y,2745
 deriva/transfer/download/processors/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/query/bag_fetch_query_processor.py,sha256=tiQtfuy01YgOFFD5b_sP7TGjMnt0Jqcg2gp1KNWqeLE,5645
-deriva/transfer/download/processors/query/base_query_processor.py,sha256=0tGxTJKCEqm7ecdOcfd8szA94muc2jbYwbqYvTGwAac,10465
-deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=Hg1NbKsaGJh9cB86yIyL7Fm7ywSNVop837Dv8aFXUes,7257
+deriva/transfer/download/processors/query/base_query_processor.py,sha256=CLP18cFdh7fXJDoizHHHrB0CFQehClu2vOm-DPaoY0c,10426
+deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=NRuwqr8LAO-I8Jmliu7RfVSb15YeAXs3eiZM09W3Dlk,7218
 deriva/transfer/download/processors/transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/transform/base_transform_processor.py,sha256=Ddw5gsNpDANeuLvUaF4utp8psaxOtAzlgXtOg8gb-Pc,4109
 deriva/transfer/download/processors/transform/column_transform_processor.py,sha256=X98XSwpGrw0xhSlQ4i7_EPMvqU0Zz5aZp9Dam2AgYRU,2627
@@ -87,7 +87,7 @@ deriva/transfer/restore/deriva_restore.py,sha256=s0h7cXit2USSdjrIfrj0dr7BJ0rrHHM
 deriva/transfer/restore/deriva_restore_cli.py,sha256=2ViZ1Lyl5ndXPKeJFCHHGnwzkg3DfHhTuRa_bN7eJm8,5603
 deriva/transfer/upload/__init__.py,sha256=4mlc_iUX-v7SpXzlCZmhxQtSiW5JeDGb2FX7bb1E6tY,304
 deriva/transfer/upload/__main__.py,sha256=hqnXtGpRqPthwpO6uvrnf_TQm7McheeyOt960hStSMY,340
-deriva/transfer/upload/deriva_upload.py,sha256=_kndWxoAHb9vKJERCC9tjExp2EWISrOzhBXnSEbFIKI,60766
+deriva/transfer/upload/deriva_upload.py,sha256=aFSO_BY1TM-5cO-uCDQZoXTRmIZVVUivrNK5a5ACp9o,61335
 deriva/transfer/upload/deriva_upload_cli.py,sha256=-Q6xgiYabQziTQcMQdGNDAv-eLxCCHO-BCSo4umbDE4,5082
 deriva/transfer/upload/processors/__init__.py,sha256=sMM5xdJ82UIRdB1lGMKk7ft0BgtjS2oJ0sI4SQSqiIU,2481
 deriva/transfer/upload/processors/archive_processor.py,sha256=ID0lDwDn4vPe5nbxy6m28Ssj_TsZpK4df2xRrM6nJRQ,2015
@@ -108,9 +108,9 @@ tests/deriva/core/mmo/test_mmo_find.py,sha256=PcUN76sik68B3XKg0G3wHVpKcPEld_6Rtb
 tests/deriva/core/mmo/test_mmo_prune.py,sha256=4pYtYL8g1BgadlewNPVpVA5lT_gV6SPTDYf04ZKzBTA,6851
 tests/deriva/core/mmo/test_mmo_rename.py,sha256=4oSR1G3Od701Ss3AnolI1Z7CbMxKuQF2uSr2_IcoR6s,8512
 tests/deriva/core/mmo/test_mmo_replace.py,sha256=w-66LWyiQ_ajC7Ipmhc4kAKwIloPdQELeUPsvelTdX8,8439
-deriva-1.7.7.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-deriva-1.7.7.dist-info/METADATA,sha256=V1dOlM-ESgLHfPMX_EOZ-kI6jvouKBIIzBcwwZ_bnnM,1623
-deriva-1.7.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deriva-1.7.7.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
-deriva-1.7.7.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
-deriva-1.7.7.dist-info/RECORD,,
+deriva-1.7.8.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+deriva-1.7.8.dist-info/METADATA,sha256=DfdN4u_GvxraP96KY24vO5hu-GDmdRAw7nCeVAPMJp0,1599
+deriva-1.7.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+deriva-1.7.8.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
+deriva-1.7.8.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
+deriva-1.7.8.dist-info/RECORD,,