deriva 1.7.6__tar.gz → 1.7.8__tar.gz

{deriva-1.7.6/deriva.egg-info → deriva-1.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.6
+Version: 1.7.8
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division
@@ -25,7 +25,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging
 Requires-Dist: requests
-Requires-Dist: certifi
 Requires-Dist: pika
 Requires-Dist: urllib3<3,>=1.26
 Requires-Dist: portalocker>=1.2.1

{deriva-1.7.6 → deriva-1.7.8}/deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.6"
+__version__ = "1.7.8"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

{deriva-1.7.6 → deriva-1.7.8}/deriva/core/base_cli.py

@@ -53,6 +53,11 @@ class BaseCLI(object):
 
         return args
 
+    # Function to convert comma-separated CLI input into a tuple
+    @staticmethod
+    def parse_tuple(value):
+        # Split the input by commas and convert to a tuple
+        return tuple(map(float, value.split(',')))
 
 class KeyValuePairArgs(argparse.Action):
     def __init__(self, option_strings, dest, nargs=None, **kwargs):

{deriva-1.7.6 → deriva-1.7.8}/deriva/core/hatrac_store.py

@@ -184,6 +184,7 @@ class HatracStore(DerivaBinding):
 
         headers = headers.copy()
 
+        file_opened = False
         if hasattr(data, 'read') and hasattr(data, 'seek'):
             data.seek(0, os.SEEK_END)
             file_size = data.tell()
@@ -192,6 +193,7 @@ class HatracStore(DerivaBinding):
         else:
             file_size = os.path.getsize(data)
             f = open(data, 'rb')
+            file_opened = True
 
         if not (md5 or sha256):
             md5 = hu.compute_hashes(f, hashes=['md5'])['md5'][1]
@@ -208,7 +210,8 @@ class HatracStore(DerivaBinding):
                 if (md5 and r.headers.get('Content-MD5') == md5 or
                         sha256 and r.headers.get('Content-SHA256') == sha256):
                     # object already has same content so skip upload
-                    f.close()
+                    if file_opened:
+                        f.close()
                     return r.headers.get('Content-Location')
                 elif not allow_versioning:
                     raise NotModified("The data cannot be uploaded because content already exists for this object "
@@ -232,6 +235,8 @@ class HatracStore(DerivaBinding):
         url = '%s%s' % (url.rstrip("/") if url.endswith("/") else url,
                         "" if not parents else "?parents=%s" % str(parents).lower())
         r = self._session.put(url, data=f, headers=headers)
+        if file_opened:
+            f.close()
         self._response_raise_for_status(r)
         loc = r.text.strip() or r.url
         if loc.startswith(self._server_uri):

{deriva-1.7.6 → deriva-1.7.8}/deriva/transfer/__init__.py

@@ -4,8 +4,8 @@ from deriva.transfer.download.deriva_download import DerivaDownload, GenericDown
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
 from deriva.transfer.download.deriva_export import DerivaExport, DerivaExportCLI
 
-from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
-    DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
+from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, UploadState, DerivaUploadError, \
+    DerivaUploadError, DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
     DerivaUploadAuthenticationError
 from deriva.transfer.upload.deriva_upload_cli import DerivaUploadCLI
 

{deriva-1.7.6 → deriva-1.7.8}/deriva/transfer/download/deriva_download.py

@@ -8,8 +8,8 @@ import requests
 from requests.exceptions import HTTPError
 from bdbag import bdbag_api as bdb, bdbag_ro as ro, BAG_PROFILE_TAG, BDBAG_RO_PROFILE_ID
 from bdbag.bdbagit import BagValidationError
-from deriva.core import ErmrestCatalog, HatracStore, format_exception, get_credential, format_credential, read_config, \
-    stob, Megabyte, __version__ as VERSION
+from deriva.core import DerivaServer, ErmrestCatalog, HatracStore, format_exception, get_credential, \
+                         format_credential, read_config, stob, Megabyte, __version__ as VERSION
 from deriva.core.utils.version_utils import get_installed_version
 from deriva.transfer.download.processors import find_query_processor, find_transform_processor, find_post_processor
 from deriva.transfer.download.processors.base_processor import LOCAL_PATH_KEY, REMOTE_PATHS_KEY, SERVICE_URL_KEY, \
@@ -76,14 +76,9 @@ class DerivaDownload(object):
                                                  password=password)
 
         # catalog and file store initialization
-        if self.catalog:
-            del self.catalog
-        self.catalog = ErmrestCatalog(
-            protocol, self.hostname, catalog_id, self.credentials, session_config=session_config)
-        if self.store:
-            del self.store
-        self.store = HatracStore(
-            protocol, self.hostname, self.credentials, session_config=session_config)
+        server = DerivaServer(protocol, self.hostname, credentials=self.credentials, session_config=session_config)
+        self.catalog = server.connect_ermrest(catalog_id)
+        self.store = HatracStore(protocol, self.hostname, self.credentials, session_config=session_config)
 
         # init dcctx cid
         self.set_dcctx_cid(kwargs.get("dcctx_cid", "api/" + self.__class__.__name__))

{deriva-1.7.6 → deriva-1.7.8}/deriva/transfer/download/deriva_export.py

@@ -5,7 +5,6 @@ import traceback
 import requests
 import argparse
 import logging
-import certifi
 import datetime
 from collections.abc import Mapping, Iterable
 from requests.exceptions import HTTPError, ConnectionError, Timeout
@@ -29,7 +28,8 @@ Client tool for interacting with DERIVA Export service.
 :param envars (dict): A dictionary of variables used for template substitution. Optional.
 :param output_dir (str): The directory where exported data will be stored (default: "."). Optional.
 :param defer_download (bool): Whether to defer the actual data download. Optional.
-:param timeout (int): Timeout value for export operations. Optional.
+:param timeout (tuple) OR (float): Timeout value as a tuple of floats in seconds for (connect,read) export operations. 
+        If a single float value is passed, it will apply to both connect and read operations. Optional.
 :param export_type (str): The type of export to perform (default: "bdbag"). Optional.
 
 :return: The full path to the downloaded file. If "defer_download" is True, the URL(s) where the export can be downloaded.
@@ -47,10 +47,20 @@ class DerivaExport:
         self.base_server_uri = "https://" + self.host
         self.service_url = self.base_server_uri + EXPORT_SERVICE_PATH % self.export_type
         self.session_config = DEFAULT_SESSION_CONFIG.copy()
-        if self.timeout is not None:
-            self.session_config["timeout"] = self.timeout
+        if isinstance(self.timeout, tuple):
+            if len(self.timeout) == 2:
+                self.session_config["timeout"] = self.timeout
+            else:
+                self.session_config["timeout"] = float(self.timeout[0])
+        elif self.timeout is not None:
+            try:
+                self.session_config["timeout"] = float(self.timeout)
+            except ValueError:
+                logger.warning("Unparseable timeout value: %r. Defaults will be used: %r." %
+                               (self.timeout, self.session_config["timeout"]))
         self.session = get_new_requests_session(self.service_url, self.session_config)
         self.dcctx = DerivaClientContext()
+        self.dcctx['cid'] = kwargs.get("dcctx_cid", "api/" + self.__class__.__name__)
         self.session.headers.update({'deriva-client-context': self.dcctx.encoded()})
 
         # credential initialization
@@ -114,7 +124,7 @@ class DerivaExport:
 
         filename = parse_content_disposition(content_disposition)
         output_path = os.path.abspath(os.path.join(self.output_dir, filename))
-        with self.session.get(url, stream=True, verify=certifi.where()) as r:
+        with self.session.get(url, stream=True) as r:
             if r.status_code != 200:
                 file_error = "File [%s] transfer failed." % output_path
                 url_error = 'HTTP GET Failed for url: %s' % url
@@ -190,8 +200,10 @@ class DerivaExportCLI(BaseCLI):
         BaseCLI.__init__(self, description, epilog, **kwargs)
         self.parser.add_argument("--defer-download", action="store_true",
                                  help="Do not download exported file(s). Default: False")
-        self.parser.add_argument("--timeout", metavar="<seconds>",
-                                 help="Total number of seconds elapsed before the download is aborted.")
+        self.parser.add_argument("--timeout", metavar="<connect,read>", type=BaseCLI.parse_tuple,
+                                 help="Timeout value(s) in seconds (int or float) for connect and read operations. "
+                                      "Separate using commas. If a single value is provided it will be used for both "
+                                      "connect and read timeouts.")
         self.parser.add_argument("--export-type", choices=["bdbag", "file"], default="bdbag",
                                  help="Export type: {bdbag|file}. Default is bdbag.",)
         self.parser.add_argument("--output-dir", metavar="<output dir>", default=".",
@@ -212,7 +224,7 @@ class DerivaExportCLI(BaseCLI):
             sys.stderr.write("\n")
 
         try:
-            exporter = DerivaExport(**vars(args))
+            exporter = DerivaExport(**vars(args), dcctx_cid="cli/" + self.__class__.__name__)
             exporter.export()
         except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError,
                 DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError) as e:

{deriva-1.7.6 → deriva-1.7.8}/deriva/transfer/download/processors/query/base_query_processor.py

@@ -1,8 +1,7 @@
 import os
 import errno
-import certifi
 import requests
-from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, DEFAULT_SESSION_CONFIG
+from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, format_exception, DEFAULT_SESSION_CONFIG
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
     DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError
 from deriva.transfer.download.processors.base_processor import BaseProcessor, \
@@ -83,12 +82,12 @@ class BaseQueryProcessor(BaseProcessor):
                 return self.catalog.get(self.query, headers=headers).json()
         except requests.HTTPError as e:
             if e.response.status_code == 401:
-                raise DerivaDownloadAuthenticationError(e)
+                raise DerivaDownloadAuthenticationError(format_exception(e))
             if e.response.status_code == 403:
-                raise DerivaDownloadAuthorizationError(e)
+                raise DerivaDownloadAuthorizationError(format_exception(e))
             if as_file:
                 os.remove(self.output_abspath)
-            raise DerivaDownloadError("Error executing catalog query: %s" % e)
+            raise DerivaDownloadError("Error executing catalog query: %s" % format_exception(e))
         except Exception:
             if as_file:
                 os.remove(self.output_abspath)
@@ -162,7 +161,7 @@ class BaseQueryProcessor(BaseProcessor):
         if cookies:
             session.cookies.update(cookies)
         if login_params and auth_url:
-            r = session.post(auth_url, data=login_params, verify=certifi.where())
+            r = session.post(auth_url, data=login_params)
             if r.status_code > 203:
                 raise DerivaDownloadError(
                     'GetExternalSession Failed with Status Code: %s\n%s\n' % (r.status_code, r.text))

{deriva-1.7.6 → deriva-1.7.8}/deriva/transfer/download/processors/query/file_download_query_processor.py

@@ -4,7 +4,6 @@ import uuid
 import datetime
 import logging
 import requests
-import certifi
 from bdbag import bdbag_ro as ro
 from deriva.core import urlsplit, format_exception, get_transfer_summary, make_dirs, DEFAULT_CHUNK_SIZE
 from deriva.core.utils.mime_utils import parse_content_disposition
@@ -39,7 +38,7 @@ class FileDownloadQueryProcessor(BaseQueryProcessor):
             else:
                 headers.update(self.HEADERS)
             session = self.getExternalSession(host)
-            with session.get(url, headers=headers, stream=True, verify=certifi.where()) as r:
+            with session.get(url, headers=headers, stream=True) as r:
                 if r.status_code != 200:
                     file_error = "File [%s] transfer failed." % output_path
                     url_error = 'HTTP GET Failed for url: %s' % url

{deriva-1.7.6 → deriva-1.7.8}/deriva/transfer/upload/deriva_upload.py

@@ -157,8 +157,12 @@ class DerivaUpload(object):
 
         # determine identity
         if self.credentials:
-            attributes = self.catalog.get_authn_session().json()
-            self.identity = attributes.get("client", self.identity)
+            try:
+                attributes = self.catalog.get_authn_session().json()
+                self.identity = attributes.get("client", self.identity)
+            except Exception as e:
+                # not a big deal since the credential token being used could be expired
+                logger.debug("Unable to determine user identity from existing credential (may be expired): %s" % e)
 
         # init dcctx cid to a default
         self.set_dcctx_cid(self.dcctx_cid)
@@ -342,7 +346,7 @@ class DerivaUpload(object):
         return '%s:%s' % (urlquote(schema_name), urlquote(table_name))
 
     @staticmethod
-    def interpolateDict(src, dst, allowNone=False):
+    def interpolateDict(src, dst, allow_none=False, allow_none_column_list=[]):
         if not (isinstance(src, dict) and isinstance(dst, dict)):
             raise ValueError("Invalid input parameter type(s): (src = %s, dst = %s), expected (dict, dict)" % (
                 type(src).__name__, type(dst).__name__))
@@ -362,10 +366,10 @@ class DerivaUpload(object):
                     if value.startswith('{') and value.endswith('}'):
                         value = None
             dst.update({k: value})
-        # remove all None valued entries in the dest, if disallowed
-        if not allowNone:
-            empty = [k for k, v in dst.items() if v is None]
-            for k in empty:
+        # remove all None valued entries in the dest, if globally disallowed or the column is not explicitly allowed
+        empty = [k for k, v in dst.items() if v is None]
+        for k in empty:
+            if not allow_none or (allow_none and k not in allow_none_column_list):
                 del dst[k]
 
         return dst
@@ -720,7 +724,9 @@ class DerivaUpload(object):
 
         # 8. Update an existing record, if necessary
         column_map = asset_mapping.get("column_map", {})
-        updated_record = self.interpolateDict(self.metadata, column_map)
+        allow_none_col_list = asset_mapping.get("allow_empty_columns_on_update", [])
+        allow_none = True if allow_none_col_list else False
+        updated_record = self.interpolateDict(self.metadata, column_map, allow_none, allow_none_col_list)
         if updated_record != record:
             record_update_template = asset_mapping.get("record_update_template")
             require_record_update_template = stob(asset_mapping.get("require_record_update_template", False))
@@ -794,7 +800,7 @@ class DerivaUpload(object):
             if result:
                 record = result[0]
                 self._updateFileMetadata(record)
-            return self.interpolateDict(self.metadata, column_map, allowNone=True), record
+            return self.interpolateDict(self.metadata, column_map, allow_none=True), record
 
     def _urlEncodeMetadata(self, safe_overrides=None):
         urlencoded = dict()

{deriva-1.7.6 → deriva-1.7.8/deriva.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.6
+Version: 1.7.8
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division
@@ -25,7 +25,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging
 Requires-Dist: requests
-Requires-Dist: certifi
 Requires-Dist: pika
 Requires-Dist: urllib3<3,>=1.26
 Requires-Dist: portalocker>=1.2.1

{deriva-1.7.6 → deriva-1.7.8}/deriva.egg-info/requires.txt

@@ -1,6 +1,5 @@
 packaging
 requests
-certifi
 pika
 urllib3<3,>=1.26
 portalocker>=1.2.1

{deriva-1.7.6 → deriva-1.7.8}/setup.py

@@ -64,7 +64,6 @@ setup(
     install_requires=[
         'packaging',
         'requests',
-        'certifi',
         'pika',
         'urllib3>=1.26,<3',
         'portalocker>=1.2.1',