rda-python-globus 1.0.2__tar.gz → 1.0.11__tar.gz

{rda_python_globus-1.0.2/src/rda_python_globus.egg-info → rda_python_globus-1.0.11}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: rda_python_globus
-Version: 1.0.2
-Summary: Tools for managing Globus transfers supporting the NSF NCAR Research Data Archive
+Version: 1.0.11
+Summary: Tools for managing Globus transfers supporting the NSF NCAR Geoscience Data Exchange (GDEX) project.
 Author-email: Thomas Cram <tcram@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-globus
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 5 - Production/Stable
-Requires-Python: >=3.7
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: globus_sdk
-Requires-Dist: click
+Requires-Dist: globus_sdk<4.0.0
+Requires-Dist: click>=8.3.0
 Requires-Dist: pyyaml
 Requires-Dist: six
 Dynamic: license-file

{rda_python_globus-1.0.2 → rda_python_globus-1.0.11}/pyproject.toml

@@ -6,13 +6,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_globus"
-version = "1.0.2"
+version = "1.0.11"
 authors = [
   { name="Thomas Cram", email="tcram@ucar.edu" },
 ]
-description = "Tools for managing Globus transfers supporting the NSF NCAR Research Data Archive"
+description = "Tools for managing Globus transfers supporting the NSF NCAR Geoscience Data Exchange (GDEX) project."
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.11"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
@@ -20,8 +20,8 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
 ]
 dependencies = [
-    "globus_sdk",
-    "click",
+    "globus_sdk<4.0.0",
+    "click>=8.3.0",
     "pyyaml",
     "six",
 ]

{rda_python_globus-1.0.2 → rda_python_globus-1.0.11}/src/rda_python_globus/file_management.py

@@ -31,10 +31,10 @@ def add_batch_to_delete_data(batch, delete_data):
 \b
 === Examples ===
 \b
-1. Create a directory on the RDA Quasar endpoint:
+1. Create a directory on the GDEX Quasar endpoint:
 \b
    $ dsglobus mkdir \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --path /d999009/new_directory
 '''
 )
@@ -63,24 +63,24 @@ def mkdir_command(
 \b
 === Examples ===
 \b
-1. Rename a single file on the RDA Quasar endpoint:
+1. Rename a single file on the GDEX Quasar endpoint:
 \b
    $ dsglobus rename \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --old-path /d999009/old_file.txt \\
        --new-path /d999009/new_file.txt
 \b
-2. Rename a directory on the RDA Quasar endpoint:
+2. Rename a directory on the GDEX Quasar endpoint:
 \b
    $ dsglobus rename \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --old-path /d999009/old_directory \\
        --new-path /d999009/new_directory
 \b
-3. Rename a batch of files/directories on the RDA Quasar endpoint:
+3. Rename a batch of files/directories on the GDEX Quasar endpoint:
 \b
    $ dsglobus rename \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --batch /path/to/batch.json
 \b
    The batch file should contain a JSON array of file paths to rename.
@@ -105,7 +105,7 @@ def mkdir_command(
 4. The batch files can also be read from stdin using '-':
 \b
    $ dsglobus rename \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --batch -
    [
        {
@@ -189,25 +189,25 @@ def rename_command(
 \b
 === Examples ===
 \b
-1. Delete a single file on the RDA Quasar endpoint:
+1. Delete a single file on the GDEX Quasar endpoint:
 \b
    $ dsglobus delete \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --target-file /d999009/file.txt
 
 \b
-2. Delete a directory on the RDA Quasar endpoint.  --recursive is required
+2. Delete a directory on the GDEX Quasar endpoint.  --recursive is required
    to delete a directory and its contents:
 \b
    $ dsglobus delete \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --target-file /d999009/dir \\
        --recursive
 \b
-3. Delete a batch of files/directories on the RDA Quasar endpoint:
+3. Delete a batch of files/directories on the GDEX Quasar endpoint:
 \b
    $ dsglobus delete \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --batch /path/to/batch.json \\
        --recursive
 \b
@@ -225,7 +225,7 @@ def rename_command(
 4. The batch files can also be read from stdin using '-':
 \b
    $ dsglobus delete \\
-       --endpoint rda-quasar \\
+       --endpoint gdex-quasar \\
        --recursive \\
        --batch -
    [

{rda_python_globus-1.0.2 → rda_python_globus-1.0.11}/src/rda_python_globus/lib/__init__.py

@@ -8,7 +8,7 @@ import six
 import click
 
 from .auth import token_storage_adapter, auth_client, transfer_client
-from .config import ENDPOINT_ALIASES, LOGPATH, LOGFILE
+from .config import ENDPOINT_ALIASES, LOGPATH, LOGFILE, TACC_GLOBUS_ENDPOINT, TACC_BASE_PATH
 
 def common_options(f):
     # any shared/common options for all commands
@@ -239,4 +239,6 @@ __all__ = (
     "transfer_client",
     "ENDPOINT_ALIASES",
     "CustomEpilog",
+    "TACC_GLOBUS_ENDPOINT",
+    "TACC_BASE_PATH",
 )

rda_python_globus-1.0.11/src/rda_python_globus/lib/auth.py

@@ -0,0 +1,57 @@
+import globus_sdk
+from globus_sdk.tokenstorage import JSONTokenStorage
+from .config import (
+    QUASAR_CLIENT_ID,
+    TACC_CLIENT_ID,
+    CLIENT_TOKEN_CONFIG,
+    TACC_TOKEN_CONFIG,
+)
+
+AUTH_RESOURCE_SERVER = "auth.globus.org"
+AUTH_SCOPES = ["openid", "profile"]
+TRANSFER_RESOURCE_SERVER = "transfer.api.globus.org"
+TRANSFFER_SCOPES = "urn:globus:auth:scope:transfer.api.globus.org:all"
+
+def token_storage_adapter(namespace="DEFAULT"):
+    """ Return a JSONTokenStorage instance for the specified namespace. If an instance already exists, return the existing instance. """
+    if namespace == "tacc":
+        json_config = TACC_TOKEN_CONFIG
+    else:
+        json_config = CLIENT_TOKEN_CONFIG
+    
+    if not hasattr(token_storage_adapter, "_instance"):
+        token_storage_adapter._instance = JSONTokenStorage(json_config, namespace=namespace)
+    return token_storage_adapter._instance
+
+def internal_auth_client(client_id=QUASAR_CLIENT_ID):
+    """ Return a NativeAppAuthClient instance for the specified client ID. """
+    return globus_sdk.NativeAppAuthClient(client_id, app_name="dsglobus")
+
+def auth_client():
+    authorizer = globus_sdk.ClientCredentialsAuthorizer(internal_auth_client(), AUTH_SCOPES)
+    return globus_sdk.AuthClient(authorizer=authorizer, app_name="dsglobus")
+
+def transfer_client(namespace="DEFAULT"):
+    """ Return a TransferClient instance for the specified namespace. """
+
+    if namespace == 'tacc':
+        client_id = TACC_CLIENT_ID
+    else:
+        client_id = QUASAR_CLIENT_ID
+
+    auth_client = internal_auth_client(client_id=client_id)
+
+    storage_adapter = token_storage_adapter(namespace)
+    token_data = storage_adapter.get_token_data(TRANSFER_RESOURCE_SERVER)
+    access_token = token_data.access_token
+    refresh_token = token_data.refresh_token
+    access_token_expires = token_data.expires_at_seconds
+
+    authorizer = globus_sdk.RefreshTokenAuthorizer(
+        refresh_token,
+        auth_client,
+        access_token=access_token,
+        expires_at=int(access_token_expires),
+        on_refresh=storage_adapter.store_token_response(auth_client.oauth2_refresh_token(refresh_token)),
+    )
+    return globus_sdk.TransferClient(authorizer=authorizer, app_name="dsglobus")

rda_python_globus-1.0.11/src/rda_python_globus/lib/config.py

@@ -0,0 +1,55 @@
+import os
+
+""" Globus Client IDs """
+QUASAR_CLIENT_ID = "05c2f58b-c667-4fc4-94fb-546e1cd8f41f"
+TACC_CLIENT_ID = "3320e8d9-030c-41d0-94f0-2357ae2e3e4d"
+
+""" Token storage configuration """
+CLIENT_TOKEN_CONFIG = '/glade/u/home/gdexdata/globus/globus_gdex_quasar_tokens.json'
+TACC_TOKEN_CONFIG = '/glade/u/home/gdexdata/globus/globus_tacc_transfer_tokens.json'
+
+""" Log file path and name """
+GDEX_BASE_PATH = '/glade/campaign/collections/gdex'
+SCRATCH_PATH = '/lustre/desc1/scratch/tcram'
+LOGPATH = os.path.join(SCRATCH_PATH, 'logs/globus')
+LOGFILE = 'dsglobus-app.log'
+
+""" Endpoint IDs """
+RDA_DATASET_ENDPOINT = 'b6b5d5e8-eb14-4f6b-8928-c02429d67998'
+RDA_DSRQST_ENDPOINT = 'e6cd9f43-935c-42e3-8d19-764d03241719'
+RDA_STRATUS_ENDPOINT = 'be4aa6a8-9e35-11eb-8a8e-d70d98a40c8d'
+RDA_GLADE_ENDPOINT = '7f0acd80-dfb2-4412-b7b5-ebc970bedf24'
+RDA_QUASAR_ENDPOINT = 'e50caa88-feae-11ea-81a2-0e2f230cc907'
+RDA_QUASAR_DR_ENDPOINT = '4c42c32c-feaf-11ea-81a2-0e2f230cc907'
+GLOBUS_CGD_ENDPOINT_ID = '11651c26-80c2-4dac-a236-7755530731ac'
+
+GDEX_DATASET_ENDPOINT = 'c4e40965-a024-43d7-bef4-6010f3731b61'
+GDEX_DSRQST_ENDPOINT = 'e6cd9f43-935c-42e3-8d19-764d03241719'
+GDEX_STRATUS_ENDPOINT = RDA_STRATUS_ENDPOINT  # same as RDA Stratus
+GDEX_OS_ENDPOINT = '558ad782-80dd-4656-a64a-2245f38a7c9e' # GDEX Boreas S3 guest collection
+GDEX_GLADE_ENDPOINT = '039e1667-8a6c-4cbd-8e26-1f86c72f6e89'
+GDEX_QUASAR_ENDPOINT = RDA_QUASAR_ENDPOINT  # same as RDA Quasar
+GDEX_QUASAR_DR_ENDPOINT = RDA_QUASAR_DR_ENDPOINT  # same as RDA Quasar DR
+
+TACC_GLOBUS_ENDPOINT = '57c4032a-2b50-47f0-adf8-13fff3a7d77d'
+TACC_BASE_PATH = '/scoutfs/projects/ASC26015'
+
+""" Endpoint aliases """
+ENDPOINT_ALIASES = {
+    "rda-glade": RDA_GLADE_ENDPOINT,
+    "rda-quasar": RDA_QUASAR_ENDPOINT,
+    "rda-quasar-drdata": RDA_QUASAR_DR_ENDPOINT,
+    "rda-dataset": RDA_DATASET_ENDPOINT,
+    "rda-dsrqst": RDA_DSRQST_ENDPOINT,
+    "rda-stratus": RDA_STRATUS_ENDPOINT,
+    "gdex-glade": GDEX_GLADE_ENDPOINT,
+    "gdex-quasar": GDEX_QUASAR_ENDPOINT,
+    "gdex-quasar-drdata": GDEX_QUASAR_DR_ENDPOINT,
+    "gdex-dataset": GDEX_DATASET_ENDPOINT,
+    "gdex-dsrqst": GDEX_DSRQST_ENDPOINT,
+    "gdex-stratus": GDEX_STRATUS_ENDPOINT,
+    "gdex-os": GDEX_OS_ENDPOINT,
+    "gdex-boreas": GDEX_OS_ENDPOINT,
+    "cgd": GLOBUS_CGD_ENDPOINT_ID,
+    "tacc": TACC_GLOBUS_ENDPOINT
+}

{rda_python_globus-1.0.2 → rda_python_globus-1.0.11}/src/rda_python_globus/list.py

@@ -1,5 +1,4 @@
 import click
-from globus_sdk import TransferData, GlobusAPIError, NetworkError
 
 from .lib import (
     common_options,
@@ -7,6 +6,7 @@ from .lib import (
     path_options,
     print_table, 
     transfer_client,
+    TACC_GLOBUS_ENDPOINT
 )
 
 import logging
@@ -80,7 +80,11 @@ def ls_command(
 			("Filename", cleaned_item_name),
 	]
 
-    tc = transfer_client()
+    if endpoint == TACC_GLOBUS_ENDPOINT:
+        tc = transfer_client(namespace="tacc")
+    else:
+        tc = transfer_client()
+
     ls_response = tc.operation_ls(endpoint, **ls_params)
     print_table(ls_response, fields)
 	

{rda_python_globus-1.0.2 → rda_python_globus-1.0.11}/src/rda_python_globus/transfer.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import json
 import typing as t
@@ -12,12 +13,14 @@ from .lib import (
     transfer_client,
     process_json_stream,
     validate_endpoint,
+    TACC_BASE_PATH,
+    TACC_GLOBUS_ENDPOINT,
 )
 
 import logging
 logger = logging.getLogger(__name__)
 
-def add_batch_to_transfer_data(batch, transfer_data):
+def add_batch_to_transfer_data(batch, transfer_data, destination_endpoint):
     """ Add batch of files to transfer data object. """
 
     batch_json = process_json_stream(batch)
@@ -30,7 +33,10 @@ def add_batch_to_transfer_data(batch, transfer_data):
 
     for i in range(len(files)):
         source_file = files[i]['source_file']
-        dest_file = files[i]['destination_file']		
+        dest_file = files[i]['destination_file']
+        if destination_endpoint == TACC_GLOBUS_ENDPOINT:
+            # prepend TACC base path to destination file
+            dest_file = os.path.join(TACC_BASE_PATH, dest_file.lstrip('/'))
         transfer_data.add_item(source_file, dest_file)
 
     return transfer_data
@@ -40,20 +46,21 @@ def add_batch_to_transfer_data(batch, transfer_data):
     help="Submit a Globus transfer task.",
     epilog='''
 \b
-=== Valid RDA endpoint names ===
-- rda-glade
-- rda-quasar
-- rda-quasar-drdata
+=== Valid GDEX endpoint names ===
+- gdex-glade
+- gdex-quasar
+- gdex-quasar-drdata
+- tacc # TACC transfer endpoint, for transfers to/from TACC only
 
 \b
 === Examples ===
 \b
-1. Transfer a single file from GLADE to the NCAR Quasar tape system:
+1. Transfer a single file from GLADE to the Quasar tape system:
 
 \b
    $ dsglobus transfer \\
-       --source-endpoint rda-glade \\
-       --destination-endpoint rda-quasar \\
+       --source-endpoint gdex-glade \\
+       --destination-endpoint gdex-quasar \\
        --source-file /data/d999009/file.txt \\
        --destination-file /d999009/file.txt	  			 
 
@@ -141,7 +148,10 @@ def transfer_command(
     if source_file is None and destination_file is None and batch is None:
         raise click.UsageError('--source-file and --destination-file, or --batch is required.')
 
-    tc = transfer_client()
+    if source_endpoint == TACC_GLOBUS_ENDPOINT or destination_endpoint == TACC_GLOBUS_ENDPOINT:
+        tc = transfer_client(namespace="tacc")
+    else:
+        tc = transfer_client()
 		
     transfer_data = TransferData(
         transfer_client=tc,
@@ -152,10 +162,13 @@ def transfer_command(
     )
 
     if batch:
-        transfer_data = add_batch_to_transfer_data(batch, transfer_data)
+        transfer_data = add_batch_to_transfer_data(batch, transfer_data, destination_endpoint)
     else:
         if source_file is None or destination_file is None:
             raise click.UsageError('--source-file and --destination-file are required is --batch is not used.')
+        if destination_endpoint == TACC_GLOBUS_ENDPOINT:
+            # prepend TACC base path to destination file
+            destination_file = os.path.join(TACC_BASE_PATH, destination_file.lstrip('/'))
         transfer_data.add_item(source_file, destination_file)
 		
     if dry_run:

{rda_python_globus-1.0.2 → rda_python_globus-1.0.11/src/rda_python_globus.egg-info}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: rda_python_globus
-Version: 1.0.2
-Summary: Tools for managing Globus transfers supporting the NSF NCAR Research Data Archive
+Version: 1.0.11
+Summary: Tools for managing Globus transfers supporting the NSF NCAR Geoscience Data Exchange (GDEX) project.
 Author-email: Thomas Cram <tcram@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-globus
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 5 - Production/Stable
-Requires-Python: >=3.7
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: globus_sdk
-Requires-Dist: click
+Requires-Dist: globus_sdk<4.0.0
+Requires-Dist: click>=8.3.0
 Requires-Dist: pyyaml
 Requires-Dist: six
 Dynamic: license-file

rda_python_globus-1.0.11/src/rda_python_globus.egg-info/requires.txt

@@ -0,0 +1,4 @@
+globus_sdk<4.0.0
+click>=8.3.0
+pyyaml
+six

rda_python_globus-1.0.2/src/rda_python_globus/lib/auth.py

@@ -1,40 +0,0 @@
-import globus_sdk
-from globus_sdk.tokenstorage import SimpleJSONFileAdapter
-from .config import (
-    CLIENT_ID,
-    CLIENT_TOKEN_CONFIG,
-)
-
-AUTH_RESOURCE_SERVER = "auth.globus.org"
-AUTH_SCOPES = ["openid", "profile"]
-TRANSFER_RESOURCE_SERVER = "transfer.api.globus.org"
-TRANSFFER_SCOPES = "urn:globus:auth:scope:transfer.api.globus.org:all"
-
-def token_storage_adapter():
-    if not hasattr(token_storage_adapter, "_instance"):
-        token_storage_adapter._instance = SimpleJSONFileAdapter(CLIENT_TOKEN_CONFIG)
-    return token_storage_adapter._instance
-
-def internal_auth_client():
-    return globus_sdk.NativeAppAuthClient(CLIENT_ID, app_name="dsglobus")
-
-def auth_client():
-    authorizer = globus_sdk.ClientCredentialsAuthorizer(internal_auth_client(), AUTH_SCOPES)
-    return globus_sdk.AuthClient(authorizer=authorizer, app_name="dsglobus")
-
-def transfer_client():
-    storage_adapter = token_storage_adapter()
-    token_data = storage_adapter.get_token_data(TRANSFER_RESOURCE_SERVER)
-
-    access_token = token_data["access_token"]
-    refresh_token = token_data["refresh_token"]
-    access_token_expires = token_data["expires_at_seconds"]
-    authorizer = globus_sdk.RefreshTokenAuthorizer(
-        refresh_token,
-        internal_auth_client(),
-        access_token=access_token,
-        expires_at=int(access_token_expires),
-        on_refresh=storage_adapter.on_refresh,
-    )
-
-    return globus_sdk.TransferClient(authorizer=authorizer, app_name="dsglobus")

rda_python_globus-1.0.2/src/rda_python_globus/lib/config.py

@@ -1,32 +0,0 @@
-import os
-
-""" RDA Quasar Client ID """
-CLIENT_ID = "05c2f58b-c667-4fc4-94fb-546e1cd8f41f"
-
-""" Token storage configuration """
-CLIENT_TOKEN_CONFIG = '/glade/u/home/rdadata/lib/python/globus_rda_quasar_tokens.json'
-
-""" Log file path and name """
-RDA_BASE_PATH = '/glade/campaign/collections/rda'
-LOGPATH = os.path.join(RDA_BASE_PATH, 'work/tcram/logs/globus')
-LOGFILE = 'dsglobus-app.log'
-
-""" Endpoint IDs """
-RDA_DATASET_ENDPOINT = 'b6b5d5e8-eb14-4f6b-8928-c02429d67998'
-RDA_DSRQST_ENDPOINT = 'e1e2997e-d794-4868-838e-d4b8d5590853'
-RDA_STRATUS_ENDPOINT = 'be4aa6a8-9e35-11eb-8a8e-d70d98a40c8d'
-RDA_GLADE_ENDPOINT = '7f0acd80-dfb2-4412-b7b5-ebc970bedf24'
-RDA_QUASAR_ENDPOINT = 'e50caa88-feae-11ea-81a2-0e2f230cc907'
-RDA_QUASAR_DR_ENDPOINT = '4c42c32c-feaf-11ea-81a2-0e2f230cc907'
-GLOBUS_CGD_ENDPOINT_ID = '11651c26-80c2-4dac-a236-7755530731ac'
-
-""" Endpoint aliases """
-ENDPOINT_ALIASES = {
-    "rda-glade": RDA_GLADE_ENDPOINT,
-    "rda-quasar": RDA_QUASAR_ENDPOINT,
-    "rda-quasar-drdata": RDA_QUASAR_DR_ENDPOINT,
-    "rda-dataset": RDA_DATASET_ENDPOINT,
-    "rda-dsrqst": RDA_DSRQST_ENDPOINT,
-    "rda-stratus": RDA_STRATUS_ENDPOINT,
-    "cgd": GLOBUS_CGD_ENDPOINT_ID,
-}

rda_python_globus-1.0.2/src/rda_python_globus.egg-info/requires.txt

@@ -1,4 +0,0 @@
-globus_sdk
-click
-pyyaml
-six