deriva 1.7.6__py3-none-any.whl → 1.7.7__py3-none-any.whl

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.6"
+__version__ = "1.7.7"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/core/base_cli.py

@@ -53,6 +53,11 @@ class BaseCLI(object):
 
         return args
 
+    # Function to convert comma-separated CLI input into a tuple
+    @staticmethod
+    def parse_tuple(value):
+        # Split the input by commas and convert to a tuple
+        return tuple(map(float, value.split(',')))
 
 class KeyValuePairArgs(argparse.Action):
     def __init__(self, option_strings, dest, nargs=None, **kwargs):

deriva/core/hatrac_store.py

@@ -184,6 +184,7 @@ class HatracStore(DerivaBinding):
 
         headers = headers.copy()
 
+        file_opened = False
         if hasattr(data, 'read') and hasattr(data, 'seek'):
             data.seek(0, os.SEEK_END)
             file_size = data.tell()
@@ -192,6 +193,7 @@ class HatracStore(DerivaBinding):
         else:
             file_size = os.path.getsize(data)
             f = open(data, 'rb')
+            file_opened = True
 
         if not (md5 or sha256):
             md5 = hu.compute_hashes(f, hashes=['md5'])['md5'][1]
@@ -208,7 +210,8 @@ class HatracStore(DerivaBinding):
                 if (md5 and r.headers.get('Content-MD5') == md5 or
                         sha256 and r.headers.get('Content-SHA256') == sha256):
                     # object already has same content so skip upload
-                    f.close()
+                    if file_opened:
+                        f.close()
                     return r.headers.get('Content-Location')
                 elif not allow_versioning:
                     raise NotModified("The data cannot be uploaded because content already exists for this object "
@@ -232,6 +235,8 @@ class HatracStore(DerivaBinding):
         url = '%s%s' % (url.rstrip("/") if url.endswith("/") else url,
                         "" if not parents else "?parents=%s" % str(parents).lower())
         r = self._session.put(url, data=f, headers=headers)
+        if file_opened:
+            f.close()
         self._response_raise_for_status(r)
         loc = r.text.strip() or r.url
         if loc.startswith(self._server_uri):

deriva/transfer/download/deriva_download.py

@@ -8,8 +8,8 @@ import requests
 from requests.exceptions import HTTPError
 from bdbag import bdbag_api as bdb, bdbag_ro as ro, BAG_PROFILE_TAG, BDBAG_RO_PROFILE_ID
 from bdbag.bdbagit import BagValidationError
-from deriva.core import ErmrestCatalog, HatracStore, format_exception, get_credential, format_credential, read_config, \
-    stob, Megabyte, __version__ as VERSION
+from deriva.core import DerivaServer, ErmrestCatalog, HatracStore, format_exception, get_credential, \
+                         format_credential, read_config, stob, Megabyte, __version__ as VERSION
 from deriva.core.utils.version_utils import get_installed_version
 from deriva.transfer.download.processors import find_query_processor, find_transform_processor, find_post_processor
 from deriva.transfer.download.processors.base_processor import LOCAL_PATH_KEY, REMOTE_PATHS_KEY, SERVICE_URL_KEY, \
@@ -76,14 +76,9 @@ class DerivaDownload(object):
                                                  password=password)
 
         # catalog and file store initialization
-        if self.catalog:
-            del self.catalog
-        self.catalog = ErmrestCatalog(
-            protocol, self.hostname, catalog_id, self.credentials, session_config=session_config)
-        if self.store:
-            del self.store
-        self.store = HatracStore(
-            protocol, self.hostname, self.credentials, session_config=session_config)
+        server = DerivaServer(protocol, self.hostname, credentials=self.credentials, session_config=session_config)
+        self.catalog = server.connect_ermrest(catalog_id)
+        self.store = HatracStore(protocol, self.hostname, self.credentials, session_config=session_config)
 
         # init dcctx cid
         self.set_dcctx_cid(kwargs.get("dcctx_cid", "api/" + self.__class__.__name__))

deriva/transfer/download/deriva_export.py

@@ -29,7 +29,8 @@ Client tool for interacting with DERIVA Export service.
 :param envars (dict): A dictionary of variables used for template substitution. Optional.
 :param output_dir (str): The directory where exported data will be stored (default: "."). Optional.
 :param defer_download (bool): Whether to defer the actual data download. Optional.
-:param timeout (int): Timeout value for export operations. Optional.
+:param timeout (tuple) OR (float): Timeout value as a tuple of floats in seconds for (connect,read) export operations. 
+        If a single float value is passed, it will apply to both connect and read operations. Optional.
 :param export_type (str): The type of export to perform (default: "bdbag"). Optional.
 
 :return: The full path to the downloaded file. If "defer_download" is True, the URL(s) where the export can be downloaded.
@@ -47,10 +48,20 @@ class DerivaExport:
         self.base_server_uri = "https://" + self.host
         self.service_url = self.base_server_uri + EXPORT_SERVICE_PATH % self.export_type
         self.session_config = DEFAULT_SESSION_CONFIG.copy()
-        if self.timeout is not None:
-            self.session_config["timeout"] = self.timeout
+        if isinstance(self.timeout, tuple):
+            if len(self.timeout) == 2:
+                self.session_config["timeout"] = self.timeout
+            else:
+                self.session_config["timeout"] = float(self.timeout[0])
+        elif self.timeout is not None:
+            try:
+                self.session_config["timeout"] = float(self.timeout)
+            except ValueError:
+                logger.warning("Unparseable timeout value: %r. Defaults will be used: %r." %
+                               (self.timeout, self.session_config["timeout"]))
         self.session = get_new_requests_session(self.service_url, self.session_config)
         self.dcctx = DerivaClientContext()
+        self.dcctx['cid'] = kwargs.get("dcctx_cid", "api/" + self.__class__.__name__)
         self.session.headers.update({'deriva-client-context': self.dcctx.encoded()})
 
         # credential initialization
@@ -190,8 +201,10 @@ class DerivaExportCLI(BaseCLI):
         BaseCLI.__init__(self, description, epilog, **kwargs)
         self.parser.add_argument("--defer-download", action="store_true",
                                  help="Do not download exported file(s). Default: False")
-        self.parser.add_argument("--timeout", metavar="<seconds>",
-                                 help="Total number of seconds elapsed before the download is aborted.")
+        self.parser.add_argument("--timeout", metavar="<connect,read>", type=BaseCLI.parse_tuple,
+                                 help="Timeout value(s) in seconds (int or float) for connect and read operations. "
+                                      "Separate using commas. If a single value is provided it will be used for both "
+                                      "connect and read timeouts.")
         self.parser.add_argument("--export-type", choices=["bdbag", "file"], default="bdbag",
                                  help="Export type: {bdbag|file}. Default is bdbag.",)
         self.parser.add_argument("--output-dir", metavar="<output dir>", default=".",
@@ -212,7 +225,7 @@ class DerivaExportCLI(BaseCLI):
             sys.stderr.write("\n")
 
         try:
-            exporter = DerivaExport(**vars(args))
+            exporter = DerivaExport(**vars(args), dcctx_cid="cli/" + self.__class__.__name__)
             exporter.export()
         except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError,
                 DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError) as e:

deriva/transfer/download/processors/query/base_query_processor.py

@@ -2,7 +2,7 @@ import os
 import errno
 import certifi
 import requests
-from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, DEFAULT_SESSION_CONFIG
+from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, format_exception, DEFAULT_SESSION_CONFIG
 from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
     DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError
 from deriva.transfer.download.processors.base_processor import BaseProcessor, \
@@ -83,12 +83,12 @@ class BaseQueryProcessor(BaseProcessor):
                 return self.catalog.get(self.query, headers=headers).json()
         except requests.HTTPError as e:
             if e.response.status_code == 401:
-                raise DerivaDownloadAuthenticationError(e)
+                raise DerivaDownloadAuthenticationError(format_exception(e))
             if e.response.status_code == 403:
-                raise DerivaDownloadAuthorizationError(e)
+                raise DerivaDownloadAuthorizationError(format_exception(e))
             if as_file:
                 os.remove(self.output_abspath)
-            raise DerivaDownloadError("Error executing catalog query: %s" % e)
+            raise DerivaDownloadError("Error executing catalog query: %s" % format_exception(e))
         except Exception:
             if as_file:
                 os.remove(self.output_abspath)

{deriva-1.7.6.dist-info → deriva-1.7.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.6
+Version: 1.7.7
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division

{deriva-1.7.6.dist-info → deriva-1.7.7.dist-info}/RECORD

@@ -8,9 +8,9 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
 deriva/config/rollback_annotation.py,sha256=vqrIcen-KZX8LDpu2OVNivzIHpQoQgWkZAChZJctvtk,3015
 deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
 deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
-deriva/core/__init__.py,sha256=aeGZj0oHsMpa48MGoOyfk693jk-xnWq3o8TcznkudM0,4945
+deriva/core/__init__.py,sha256=D7aya8r3BTbewaSUmhbb4DB-Ltwe2aXV10CMfPo13rI,4945
 deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
-deriva/core/base_cli.py,sha256=EkLXOTeaFWUbPaYV-eLuLGga1PbkFVWi3Jjo-e_Vb-U,2681
+deriva/core/base_cli.py,sha256=78Ilf3_f2xREQb3IIj6q0jwWAiXSObZszG0JURs36lA,2902
 deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
 deriva/core/datapath.py,sha256=w8LvPAd_DuknKxHc_YS6NUHhTY6XpCSkfa0m_xQUdZE,89068
 deriva/core/deriva_binding.py,sha256=_sA9HGrcVRqT-OhrneMDMOquyVOFOxLq3WzBQhasLIM,12970
@@ -18,7 +18,7 @@ deriva/core/deriva_server.py,sha256=nsW3gwg1sIaHl3BTf-nL41AkSj3dEpcEBlatvjvN8CQ,
 deriva/core/ermrest_catalog.py,sha256=_eqQg16i1aA95R99B7tLZxHWQlYk-rLpN_0zghfNWRc,54991
 deriva/core/ermrest_model.py,sha256=NdvrMLkmNTY4ncodzojpyV1XjgSlod7x-SXyUu65Z0Q,124921
 deriva/core/hatrac_cli.py,sha256=l9QmneLRHSMiG_z9S83ea0QGVhTS3Wq1KGPEKEDpecM,14522
-deriva/core/hatrac_store.py,sha256=pRupabYZzXvf1iQTYmv0AME8n7KCkTCMfrojhl_JOTs,22178
+deriva/core/hatrac_store.py,sha256=NcVuO4h4hswbEAct8tTKZ1pNtXBn74Nn9TelKN_jr8Q,22323
 deriva/core/mmo.py,sha256=dcB8akgsqbYMi22ClbVpOKVL6so8FjSDjSb6gP4_jFo,17852
 deriva/core/polling_ermrest_catalog.py,sha256=KsjiFqPQaHWnJZCVF5i77sdzfubqZHgMBbQ1p8V8D3s,10351
 deriva/core/schemas/app_links.schema.json,sha256=AxrkC2scxomM6N7jyjtdYA73BbZzPrmuqU8PYWe7okI,954
@@ -60,9 +60,9 @@ deriva/transfer/backup/deriva_backup.py,sha256=IO9Tmzx6jHfUCkP-41nSsAeOFLn9T-0Hw
 deriva/transfer/backup/deriva_backup_cli.py,sha256=T0tvPKWniRinMQt0qG7FI8AoK3GgtlT6EyBZmZCAjL8,2157
 deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w9Xyh88Q,350
 deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
-deriva/transfer/download/deriva_download.py,sha256=9WHX0iBUsXv3iT0pEy95kpVN-Oh4vc6ywI5tYmJWpfk,17145
+deriva/transfer/download/deriva_download.py,sha256=ulFrHHEDj3oJA2pAo7MGvGyDF9rA3l8yCUoK3FMvHEk,17100
 deriva/transfer/download/deriva_download_cli.py,sha256=wN8tyQDv1AIE_aDqjECbmkoEWN050vlEdJyteYbdgSs,3940
-deriva/transfer/download/deriva_export.py,sha256=lymAfCrD0Ol2kVgx7g0UAGez755D3ak8tMEneCHzwUQ,12102
+deriva/transfer/download/deriva_export.py,sha256=R6f08eJSt1iBeDNNQhThN3W1l4UwE0xzy2DjXbqNskE,13085
 deriva/transfer/download/processors/__init__.py,sha256=evLp36tZn-Z_AMshdfV3JJO8w1es5owsnRN0IFJUwIo,4507
 deriva/transfer/download/processors/base_processor.py,sha256=R6IIHSa_euv4X2Dyhd8fvQAiVYDGJTWMQtPoukHQn-Q,3837
 deriva/transfer/download/processors/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,7 +71,7 @@ deriva/transfer/download/processors/postprocess/transfer_post_processor.py,sha25
 deriva/transfer/download/processors/postprocess/url_post_processor.py,sha256=s68iIYqQSZHtbv4y-fCG8pjhApAeMEG6hYcKx2Pvf5Y,2745
 deriva/transfer/download/processors/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/query/bag_fetch_query_processor.py,sha256=tiQtfuy01YgOFFD5b_sP7TGjMnt0Jqcg2gp1KNWqeLE,5645
-deriva/transfer/download/processors/query/base_query_processor.py,sha256=DGvYM4Ykwo5OYq2D-HKB2bYPuqhJMKdMQ8GGw7OCMz0,10393
+deriva/transfer/download/processors/query/base_query_processor.py,sha256=0tGxTJKCEqm7ecdOcfd8szA94muc2jbYwbqYvTGwAac,10465
 deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=Hg1NbKsaGJh9cB86yIyL7Fm7ywSNVop837Dv8aFXUes,7257
 deriva/transfer/download/processors/transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva/transfer/download/processors/transform/base_transform_processor.py,sha256=Ddw5gsNpDANeuLvUaF4utp8psaxOtAzlgXtOg8gb-Pc,4109
@@ -108,9 +108,9 @@ tests/deriva/core/mmo/test_mmo_find.py,sha256=PcUN76sik68B3XKg0G3wHVpKcPEld_6Rtb
 tests/deriva/core/mmo/test_mmo_prune.py,sha256=4pYtYL8g1BgadlewNPVpVA5lT_gV6SPTDYf04ZKzBTA,6851
 tests/deriva/core/mmo/test_mmo_rename.py,sha256=4oSR1G3Od701Ss3AnolI1Z7CbMxKuQF2uSr2_IcoR6s,8512
 tests/deriva/core/mmo/test_mmo_replace.py,sha256=w-66LWyiQ_ajC7Ipmhc4kAKwIloPdQELeUPsvelTdX8,8439
-deriva-1.7.6.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-deriva-1.7.6.dist-info/METADATA,sha256=qEPrF-sL3FER0i9i_RzniT-nGZeNMcA8j1tkm5pcDmE,1623
-deriva-1.7.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deriva-1.7.6.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
-deriva-1.7.6.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
-deriva-1.7.6.dist-info/RECORD,,
+deriva-1.7.7.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+deriva-1.7.7.dist-info/METADATA,sha256=V1dOlM-ESgLHfPMX_EOZ-kI6jvouKBIIzBcwwZ_bnnM,1623
+deriva-1.7.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+deriva-1.7.7.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
+deriva-1.7.7.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
+deriva-1.7.7.dist-info/RECORD,,