deriva 1.7.5__py3-none-any.whl → 1.7.6__py3-none-any.whl

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.5"
+__version__ = "1.7.6"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/transfer/__init__.py

@@ -2,6 +2,7 @@ from deriva.transfer.download.deriva_download import DerivaDownload, GenericDown
     DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, \
     DerivaDownloadBaggingError
 from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
+from deriva.transfer.download.deriva_export import DerivaExport, DerivaExportCLI
 
 from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
     DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \

deriva/transfer/download/deriva_export.py

@@ -0,0 +1,241 @@
+import os
+import sys
+import json
+import traceback
+import requests
+import argparse
+import logging
+import certifi
+import datetime
+from collections.abc import Mapping, Iterable
+from requests.exceptions import HTTPError, ConnectionError, Timeout
+from deriva.core.deriva_binding import DerivaClientContext
+from deriva.core.utils.mime_utils import parse_content_disposition
+from deriva.core import BaseCLI, KeyValuePairArgs, get_new_requests_session, get_transfer_summary, get_credential, \
+                         format_credential, format_exception, urlsplit, DEFAULT_SESSION_CONFIG, DEFAULT_CHUNK_SIZE
+from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
+    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError
+
+logger = logging.getLogger(__name__)
+
+EXPORT_SERVICE_PATH = "/deriva/export/%s"
+
+"""
+Client tool for interacting with DERIVA Export service.
+
+:param host (str): The host server for the export operation.
+:param config_file (str): Path to an export configuration file.
+:param credential (dict): Authentication credential (returned from get_credential()) for the export process. Optional.
+:param envars (dict): A dictionary of variables used for template substitution. Optional.
+:param output_dir (str): The directory where exported data will be stored (default: "."). Optional.
+:param defer_download (bool): Whether to defer the actual data download. Optional.
+:param timeout (int): Timeout value for export operations. Optional.
+:param export_type (str): The type of export to perform (default: "bdbag"). Optional.
+
+:return: The full path to the downloaded file. If "defer_download" is True, the URL(s) where the export can be downloaded.
+"""
+class DerivaExport:
+    def __init__(self, **kwargs):
+        self.host = kwargs.get("host")
+        self.config_file = kwargs.get("config_file")
+        self.envars = kwargs.get("envars", dict())
+        self.credential = kwargs.get("credential")
+        self.output_dir = kwargs.get("output_dir", ".")
+        self.defer_download = kwargs.get("defer_download")
+        self.timeout = kwargs.get("timeout")
+        self.export_type = kwargs.get("export_type", "bdbag")
+        self.base_server_uri = "https://" + self.host
+        self.service_url = self.base_server_uri + EXPORT_SERVICE_PATH % self.export_type
+        self.session_config = DEFAULT_SESSION_CONFIG.copy()
+        if self.timeout is not None:
+            self.session_config["timeout"] = self.timeout
+        self.session = get_new_requests_session(self.service_url, self.session_config)
+        self.dcctx = DerivaClientContext()
+        self.session.headers.update({'deriva-client-context': self.dcctx.encoded()})
+
+        # credential initialization
+        if self.credential is None:
+            token = kwargs.get("token")
+            oauth2_token = kwargs.get("oauth2_token")
+            credential_file = kwargs.get("credential_file")
+            if token or oauth2_token:
+                self.credential = format_credential(token=token, oauth2_token=oauth2_token)
+            else:
+                self.credential = get_credential(self.host, credential_file)
+
+        if self.credential is None:
+            raise DerivaDownloadAuthenticationError(
+                "The requested service requires authentication and a valid login credential could "
+                "not be found (or was not provided) for the specified host.")
+        if 'bearer-token' in self.credential:
+            self.session.headers.update(
+                {'Authorization': 'Bearer {token}'.format(token=self.credential['bearer-token'])})
+        elif 'cookie' in self.credential:
+            cname, cval = self.credential['cookie'].split('=', 1)
+            self.session.cookies.set(cname, cval, domain=self.host, path='/')
+
+    def validate_authn_session(self):
+        url = self.base_server_uri + "/authn/session"
+        r = self.session.get(url)
+        if r.status_code == requests.codes.not_found or r.status_code == requests.codes.unauthorized:
+            logger.warning("Unable to authenticate. Check for missing or expired credentials.")
+        r.raise_for_status()
+        return r.json()
+
+    def recursive_format(self, d, **kwargs):
+        """
+        Recursively apply str.format to all string-based values in a dictionary.
+        Supports nested dictionaries and lists.
+
+        :param d: Dictionary or iterable containing values to be formatted
+        :param kwargs: Formatting arguments
+        :return: New dictionary or iterable with formatted strings
+        """
+        if isinstance(d, Mapping):
+            return {k: self.recursive_format(v, **kwargs) for k, v in d.items()}
+        elif isinstance(d, str):
+            return d.format(**kwargs)
+        elif isinstance(d, Iterable) and not isinstance(d, (str, bytes)):
+            return type(d)(self.recursive_format(v, **kwargs) for v in d)
+        else:
+            return d
+
+
+    def retrieve_file(self, url):
+        content_disposition = None
+        try:
+            head = self.session.head(url)
+            if head.ok:
+                content_disposition = head.headers.get("Content-Disposition") if head.ok else None
+            if not content_disposition:
+                raise DerivaDownloadError("HEAD response missing Content-Disposition header.")
+        except requests.HTTPError as e:
+            raise DerivaDownloadError("HEAD request for [%s] failed: %s" % (url, e))
+
+        filename = parse_content_disposition(content_disposition)
+        output_path = os.path.abspath(os.path.join(self.output_dir, filename))
+        with self.session.get(url, stream=True, verify=certifi.where()) as r:
+            if r.status_code != 200:
+                file_error = "File [%s] transfer failed." % output_path
+                url_error = 'HTTP GET Failed for url: %s' % url
+                host_error = "Host %s responded:\n\n%s" % (urlsplit(url).netloc, r.text)
+                raise DerivaDownloadError('%s\n\n%s\n%s' % (file_error, url_error, host_error))
+            else:
+                total = 0
+                start = datetime.datetime.now()
+                logging.debug("Transferring file %s to %s" % (url, output_path))
+                with open(output_path, 'wb') as data_file:
+                    for chunk in r.iter_content(chunk_size=DEFAULT_CHUNK_SIZE):
+                        data_file.write(chunk)
+                        total += len(chunk)
+                elapsed = datetime.datetime.now() - start
+                summary = get_transfer_summary(total, elapsed)
+                logging.info("File [%s] transfer successful. %s" % (output_path, summary))
+                return output_path
+
+    def export(self):
+        try:
+            auth = self.validate_authn_session()
+            logger.debug("Authenticated session established. Session attributes: %s" % auth)
+
+            try:
+                logger.info("Processing export config file: %s" % self.config_file)
+                with open(self.config_file, encoding='utf-8') as cf:
+                    config = json.loads(cf.read())
+                    env = config.get("env", {})
+                    env = self.recursive_format(env, **self.envars)
+                    config.update({"env": env})
+            except Exception as e:
+                raise DerivaDownloadConfigurationError("Error processing export config file: %s" % format_exception(e))
+
+            logger.info("Requesting %s export at: %s" % (self.export_type, self.service_url))
+            response = self.session.post(self.service_url, json=config)
+            response.raise_for_status()
+            result_urls = response.text.split('\n')
+            logger.info("Export successful. Service responded with URL list: %s" % result_urls)
+            if not self.defer_download:
+                if self.export_type == "bdbag":
+                    result_url = result_urls[1] if len(result_urls) > 1 else result_urls[0]
+                    logger.info("Downloading exported bag content from %s to directory: %s" %
+                                (result_url, os.path.abspath(self.output_dir)))
+                    return self.retrieve_file(result_url)
+                elif self.export_type == "file":
+                    for result_url in result_urls:
+                        self.retrieve_file(result_url)
+                        logger.info("Downloading exported file content from %s to directory: %s" %
+                                    (result_url, os.path.abspath(self.output_dir)))
+                else:
+                    pass
+            else:
+                return result_urls
+        except ConnectionError as e:
+            raise DerivaDownloadError("Connection error occurred. %s" % format_exception(e))
+        except Timeout as e:
+            raise DerivaDownloadTimeoutError("Connection timeout occurred. %s" % format_exception(e))
+        except HTTPError as e:
+            if e.response.status_code == requests.codes.unauthorized:
+                raise DerivaDownloadAuthenticationError(
+                    "The requested service requires authentication and a valid login session could "
+                    "not be found for the specified host. Server responded: %s" % format_exception(e))
+            elif e.response.status_code == requests.codes.forbidden:
+                raise DerivaDownloadAuthorizationError(
+                    "A requested operation was forbidden. Server responded: %s" % format_exception(e))
+            else:
+                raise DerivaDownloadError(format_exception(e))
+
+
+class DerivaExportCLI(BaseCLI):
+    def __init__(self, description, epilog, **kwargs):
+
+        BaseCLI.__init__(self, description, epilog, **kwargs)
+        self.parser.add_argument("--defer-download", action="store_true",
+                                 help="Do not download exported file(s). Default: False")
+        self.parser.add_argument("--timeout", metavar="<seconds>",
+                                 help="Total number of seconds elapsed before the download is aborted.")
+        self.parser.add_argument("--export-type", choices=["bdbag", "file"], default="bdbag",
+                                 help="Export type: {bdbag|file}. Default is bdbag.",)
+        self.parser.add_argument("--output-dir", metavar="<output dir>", default=".",
+                                 help="Path to an output directory. Default is current directory.")
+        self.parser.add_argument("envars", metavar="[key=value key=value ...]",
+                                 nargs=argparse.REMAINDER, action=KeyValuePairArgs, default={},
+                                 help="Variable length of whitespace-delimited key=value pair arguments used for "
+                                      "string interpolation in specific parts of the configuration file. "
+                                      "For example: key1=value1 key2=value2")
+
+    def main(self):
+        try:
+            args = self.parse_cli()
+        except ValueError as e:
+            sys.stderr.write(str(e))
+            return 2
+        if not args.quiet:
+            sys.stderr.write("\n")
+
+        try:
+            exporter = DerivaExport(**vars(args))
+            exporter.export()
+        except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError,
+                DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError) as e:
+            sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e))
+            if args.debug:
+                traceback.print_exc()
+            return 1
+        except:
+            sys.stderr.write("An unexpected error occurred.")
+            traceback.print_exc()
+            return 1
+        finally:
+            if not args.quiet:
+                sys.stderr.write("\n\n")
+        return 0
+
+DESC = "Deriva Export Service Download Utility - CLI"
+INFO = "For more information see: https://github.com/informatics-isi-edu/deriva-py"
+
+def main():
+    cli = DerivaExportCLI(DESC, INFO, hostname_required=True, config_file_required=True)
+    return cli.main()
+
+
+if __name__ == '__main__':
+    sys.exit(main())

{deriva-1.7.5.dist-info → deriva-1.7.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deriva
-Version: 1.7.5
+Version: 1.7.6
 Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
 Home-page: https://github.com/informatics-isi-edu/deriva-py
 Author: USC Information Sciences Institute, Informatics Systems Research Division

{deriva-1.7.5.dist-info → deriva-1.7.6.dist-info}/RECORD

@@ -8,7 +8,7 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
 deriva/config/rollback_annotation.py,sha256=vqrIcen-KZX8LDpu2OVNivzIHpQoQgWkZAChZJctvtk,3015
 deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
 deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
-deriva/core/__init__.py,sha256=IT6J4hMqwaL8v4DRAsPa8FVUOfVCOWY_FTw0b8UthzQ,4945
+deriva/core/__init__.py,sha256=aeGZj0oHsMpa48MGoOyfk693jk-xnWq3o8TcznkudM0,4945
 deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
 deriva/core/base_cli.py,sha256=EkLXOTeaFWUbPaYV-eLuLGga1PbkFVWi3Jjo-e_Vb-U,2681
 deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
@@ -53,7 +53,7 @@ deriva/core/utils/webauthn_utils.py,sha256=rD0HQZAjUKp4NfqHQG1FhH3x7uKog2et7w7LB
 deriva/seo/__init__.py,sha256=dYn48A7blbeYf40b4T3KVofrQK4u5K5MfxXWfIGloig,54
 deriva/seo/sitemap_builder.py,sha256=Ht_AbodEERDofIoCcd4kPlrl1pVW670WN5dT4cc05LQ,13948
 deriva/seo/sitemap_cli.py,sha256=miCqRfpSj5Dx5BfJGSd8Pi2e4OOQjotDzP_JubukhCM,2654
-deriva/transfer/__init__.py,sha256=Xbp-s9vmzFbumJZ743PEXE8rI1s6-s234F66C_ioC_E,1172
+deriva/transfer/__init__.py,sha256=3a01U6e68kBEnFUp0dRMFLRDj2p50iIJF9RZaG3TXVI,1253
 deriva/transfer/backup/__init__.py,sha256=vxsZiDLMTJQPybXT89G-07GsUoLhnItTCbLdXcDSyeA,465
 deriva/transfer/backup/__main__.py,sha256=dT12--8C6sKGEtMhsYuy013ebXKpVnBJfhcQNlVtv6Y,361
 deriva/transfer/backup/deriva_backup.py,sha256=IO9Tmzx6jHfUCkP-41nSsAeOFLn9T-0HwQcpRLpM_zs,5228
@@ -62,6 +62,7 @@ deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w
 deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
 deriva/transfer/download/deriva_download.py,sha256=9WHX0iBUsXv3iT0pEy95kpVN-Oh4vc6ywI5tYmJWpfk,17145
 deriva/transfer/download/deriva_download_cli.py,sha256=wN8tyQDv1AIE_aDqjECbmkoEWN050vlEdJyteYbdgSs,3940
+deriva/transfer/download/deriva_export.py,sha256=lymAfCrD0Ol2kVgx7g0UAGez755D3ak8tMEneCHzwUQ,12102
 deriva/transfer/download/processors/__init__.py,sha256=evLp36tZn-Z_AMshdfV3JJO8w1es5owsnRN0IFJUwIo,4507
 deriva/transfer/download/processors/base_processor.py,sha256=R6IIHSa_euv4X2Dyhd8fvQAiVYDGJTWMQtPoukHQn-Q,3837
 deriva/transfer/download/processors/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -107,9 +108,9 @@ tests/deriva/core/mmo/test_mmo_find.py,sha256=PcUN76sik68B3XKg0G3wHVpKcPEld_6Rtb
 tests/deriva/core/mmo/test_mmo_prune.py,sha256=4pYtYL8g1BgadlewNPVpVA5lT_gV6SPTDYf04ZKzBTA,6851
 tests/deriva/core/mmo/test_mmo_rename.py,sha256=4oSR1G3Od701Ss3AnolI1Z7CbMxKuQF2uSr2_IcoR6s,8512
 tests/deriva/core/mmo/test_mmo_replace.py,sha256=w-66LWyiQ_ajC7Ipmhc4kAKwIloPdQELeUPsvelTdX8,8439
-deriva-1.7.5.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-deriva-1.7.5.dist-info/METADATA,sha256=10B2Mj3_umQBki2v847VPJZx-PfF6f1STkv7RkcFMD4,1623
-deriva-1.7.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deriva-1.7.5.dist-info/entry_points.txt,sha256=72BEmEE4Bes5QhVxUHrl7EvUARrgISWxI2KGa8BbNZ8,786
-deriva-1.7.5.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
-deriva-1.7.5.dist-info/RECORD,,
+deriva-1.7.6.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+deriva-1.7.6.dist-info/METADATA,sha256=qEPrF-sL3FER0i9i_RzniT-nGZeNMcA8j1tkm5pcDmE,1623
+deriva-1.7.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+deriva-1.7.6.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
+deriva-1.7.6.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
+deriva-1.7.6.dist-info/RECORD,,

{deriva-1.7.5.dist-info → deriva-1.7.6.dist-info}/entry_points.txt

@@ -7,6 +7,7 @@ deriva-annotation-validate = deriva.config.annotation_validate:main
 deriva-backup-cli = deriva.transfer.backup.__main__:main
 deriva-catalog-cli = deriva.core.catalog_cli:main
 deriva-download-cli = deriva.transfer.download.__main__:main
+deriva-export-cli = deriva.transfer.download.deriva_export:main
 deriva-globus-auth-utils = deriva.core.utils.globus_auth_utils:main
 deriva-hatrac-cli = deriva.core.hatrac_cli:main
 deriva-restore-cli = deriva.transfer.restore.__main__:main