dxpy 0.399.1__tar.gz → 0.400.1__tar.gz

{dxpy-0.399.1 → dxpy-0.400.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.399.1
+Version: 0.400.1
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/bindings/dxproject.py

@@ -367,7 +367,7 @@ class DXProject(DXContainer):
                allowed_executables=None, unset_allowed_executables=None,
                database_ui_view_only=None, external_upload_restricted=None,
                database_results_restricted=None, unset_database_results_restricted=None,
-               https_app_isolated_browsing=None, **kwargs):
+               https_app_isolated_browsing=None, https_app_isolated_browsing_options=None, **kwargs):
         """
         :param name: If provided, the new project name
         :type name: string
@@ -398,6 +398,8 @@ class DXProject(DXContainer):
         :param https_app_isolated_browsing: Whether all https access to jobs in this project should be wrapped in Isolated Browsing.
                                             If true, httpsApp-enabled executables must have httpsApp.shared_access set to 'NONE' to run in this project.
         :type https_app_isolated_browsing: boolean
+        :param https_app_isolated_browsing_options: Options for Isolated Browsing. See https://documentation.dnanexus.com/developer/api/data-containers/projects#api-method-project-xxxx-update for a list of supported keys.
+        :type https_app_isolated_browsing_options: dict
 
         Updates the project with the new fields. All fields are
         optional. Fields that are not provided are not changed.
@@ -435,6 +437,8 @@ class DXProject(DXContainer):
             update_hash["databaseResultsRestricted"] = None
         if https_app_isolated_browsing is not None:
             update_hash["httpsAppIsolatedBrowsing"] = https_app_isolated_browsing
+        if https_app_isolated_browsing_options is not None:
+            update_hash["httpsAppIsolatedBrowsingOptions"] = https_app_isolated_browsing_options
         dxpy.api.project_update(self._dxid, update_hash, **kwargs)
 
     def invite(self, invitee, level, send_email=True, **kwargs):

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/cli/parsers.py

@@ -464,6 +464,11 @@ def get_update_project_args(args):
         input_params['externalUploadRestricted'] = args.external_upload_restricted == 'true'
     if args.https_app_isolated_browsing is not None:
         input_params['httpsAppIsolatedBrowsing'] = args.https_app_isolated_browsing == 'true'
+    if args.https_app_isolated_browsing_options is not None:
+        try:
+            input_params['httpsAppIsolatedBrowsingOptions'] = json.loads(args.https_app_isolated_browsing_options)
+        except ValueError as e:
+            raise DXParserError('Error while parsing the value for --https-app-isolated-browsing-options') from e
     return input_params
 
 def process_phi_param(args):

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/exceptions.py

@@ -79,6 +79,9 @@ class PermissionDenied(DXAPIError):
 class SpendingLimitExceeded(DXAPIError):
     ''' Raised when the spending limit has been reached for the account that would be billed for this action. '''
 
+class OrgExpired(DXAPIError):
+    ''' Raised when the organization that would be billed for this action has expired. '''
+
 class ResourceNotFound(DXAPIError):
     ''' Raised when a specified entity or resource could not be found. '''
 

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/app_asset_projects_ids_prod.json

@@ -5,5 +5,6 @@
     "aws:me-south-1": "project-GvGXBYV3pK4v9X7F2jv9zz94",
     "aws:us-east-1": "project-B6JG897KGbkGb6Z7pQ9Q02jG",
     "azure:westeurope": "project-FGZZFkjBPJqBXq1X84pq4VJf",
-    "azure:westus": "project-F3vk1q09FX8Jxxp20pq6z8P9"
+    "azure:westus": "project-F3vk1q09FX8Jxxp20pq6z8P9",
+    "oci:us-ashburn-1": "project-J1q0ZJV6VQg98Q4Bz36JbQKJ"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/app_asset_projects_ids_staging.json

@@ -5,5 +5,6 @@
     "aws:me-south-1": "project-Gv8PVXk32gZ83kfp0pZ1v3V2",
     "aws:us-east-1": "project-G0zG4B808VFFYVz84FQkvF1K",
     "azure:westeurope": "project-G0zG4BQBq1XQby2F4FqQpPXv",
-    "azure:westus": "project-G0zG4B09ZJ5gx1Vz4Fb8Q9XX"
+    "azure:westus": "project-G0zG4B09ZJ5gx1Vz4Fb8Q9XX",
+    "oci:us-ashburn-1": "project-J1q0ZJV6z2pJXY2Qqv8xvX5G"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/awscli_assets.json

@@ -5,5 +5,6 @@
   "aws:me-south-1": "record-Gyv70B932Vy4bg78PFvbB4Q7",
   "aws:us-east-1": "record-Gyv6Q9j0PPgJQJ72226Gz7j7",
   "azure:westeurope": "record-Gyv73f0BKJkvjx5FPJq6F22b",
-  "azure:westus": "record-Gyv77Vj9zv30Pfx8V890xQ6Q"
+  "azure:westus": "record-Gyv77Vj9zv30Pfx8V890xQ6Q",
+  "oci:us-ashburn-1": "record-J30Z62V6ZVX41GJBgkxbkFzp"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/awscli_assets.staging.json

@@ -5,5 +5,6 @@
   "aws:me-south-1": "record-Gyqp5F13fKQ44GfVP0VkKZ0z",
   "aws:us-east-1": "record-GyqkB3j08Gb14Pf0P8x5Xq5Q",
   "azure:westeurope": "record-Gyqp8g8B3BVZ4GfVP0VkKZ15",
-  "azure:westus": "record-GyqpG7Q9Z3044GfVP0VkKZ1J"
+  "azure:westus": "record-GyqpG7Q9Z3044GfVP0VkKZ1J",
+  "oci:us-ashburn-1": "record-J30Xb1k6p6JxZFPzfpBfY6Qq"
 }

dxpy-0.400.1/dxpy/nextflow/default_nextflow_instance_types.json

@@ -0,0 +1,10 @@
+{
+  "aws:ap-southeast-2": "mem2_ssd1_v2_x4",
+  "aws:eu-central-1": "mem2_ssd1_v2_x4",
+  "aws:us-east-1": "mem2_ssd1_v2_x4",
+  "aws:me-south-1": "mem2_ssd1_v2_x4",
+  "azure:westeurope": "azure:mem2_ssd1_x4",
+  "azure:westus": "azure:mem2_ssd1_x4",
+  "aws:eu-west-2-g": "mem2_ssd1_v2_x4",
+  "oci:us-ashburn-1": "oci:mem2_ssd1_v3i_x4"
+}

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/nextaur_assets.json

@@ -5,5 +5,6 @@
     "aws:me-south-1": "record-GzyP85V306B9xkQ4Pv1yf7Kz",
     "aws:us-east-1": "record-GzyP6580KGjqV4gp6QXFGBzX",
     "azure:westeurope": "record-GzyP9z0BzJx8zjf6bpvY6Yqv",
-    "azure:westus": "record-GzyP9vQ9zggPyfkj5GKKYVG5"
+    "azure:westus": "record-GzyP9vQ9zggPyfkj5GKKYVG5",
+    "oci:us-ashburn-1": "record-J30Z6396B086xYKfqQj5gG1j"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/nextaur_assets.staging.json

@@ -5,5 +5,6 @@
     "aws:me-south-1": "record-GzyJ2GV311jBJYzx2XpgZXQ0",
     "aws:us-east-1": "record-GzyJ08002j9B36YfkbQQxpkf",
     "azure:westeurope": "record-GzyJ4B8B095jZ8F4BF5p2Ggz",
-    "azure:westus": "record-GzyJ3q89X1QBpvfvgx9bZzgP"
+    "azure:westus": "record-GzyJ3q89X1QBpvfvgx9bZzgP",
+    "oci:us-ashburn-1": "record-J30XbBV6j7q0jq6q1Jfz5650"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/nextflow_assets.json

@@ -5,5 +5,6 @@
     "aws:me-south-1": "record-GzyP8593PgfZjYXPJP9FFqV4",
     "aws:us-east-1": "record-GzyP6X00F3gq8p88gP7PQFkx",
     "azure:westeurope": "record-GzyPB98BB2b867v2KK1BXx8p",
-    "azure:westus": "record-GzyPB6j9b5Yy81G8XV89jkZX"
+    "azure:westus": "record-GzyPB6j9b5Yy81G8XV89jkZX",
+    "oci:us-ashburn-1": "record-J30Z63V67qG7Y9ZfxjPpp90G"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/nextflow_assets.staging.json

@@ -5,5 +5,6 @@
     "aws:me-south-1": "record-GzyJ2Y93z3b2Q4202Jpx1j78",
     "aws:us-east-1": "record-GzyJ19j08Ffppvfvgx9bZzfz",
     "azure:westeurope": "record-GzyJ4QjByJz29GJg3K3Gv2yB",
-    "azure:westus": "record-GzyJ45Q9xQPB36YfkbQQxpp8"
+    "azure:westus": "record-GzyJ45Q9xQPB36YfkbQQxpp8",
+    "oci:us-ashburn-1": "record-J30Xbb16kfQfgJqjXGf7561F"
 }

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/nextflow/nextflow_utils.py

@@ -149,15 +149,14 @@ def get_regional_options(region, resources_dir, profile, cache_docker, nextflow_
 
 
 def get_instance_type(region):
-    instance_type = {
-        "aws:ap-southeast-2": "mem2_ssd1_v2_x4",
-        "aws:eu-central-1": "mem2_ssd1_v2_x4",
-        "aws:us-east-1": "mem2_ssd1_v2_x4",
-        "aws:me-south-1": "mem2_ssd1_v2_x4",
-        "azure:westeurope": "azure:mem2_ssd1_x4",
-        "azure:westus": "azure:mem2_ssd1_x4",
-        "aws:eu-west-2-g": "mem2_ssd1_v2_x4"
-    }.get(region)
+    json_file_path = path.join(path.dirname(dxpy.__file__), 'nextflow', 'default_nextflow_instance_types.json')
+    try:
+        with open(json_file_path, 'r') as f:
+            instance_type_mapping = json.load(f)
+    except FileNotFoundError:
+        raise dxpy.exceptions.ResourceNotFound(f"Instance types file not found at {json_file_path}.")
+
+    instance_type = instance_type_mapping.get(region)
     if not instance_type:
         raise dxpy.exceptions.ResourceNotFound("Instance type is not specified for region {}.".format(region))
     return instance_type
@@ -184,6 +183,7 @@ def get_nextflow_assets(region):
         with open(nextaur_assets, 'r') as nextaur_f, open(nextflow_assets, 'r') as nextflow_f, open(awscli_assets, 'r') as awscli_f:
             return json.load(nextaur_f)[region], json.load(nextflow_f)[region], json.load(awscli_f)[region]
 
+
 def get_nested(args, arg_path):
     """
     :param args: extra args from command input

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/scripts/dx.py

@@ -5416,6 +5416,7 @@ parser_update_project.add_argument('--https-app-isolated-browsing',
                                    help="Whether all https access to jobs in this project should be wrapped in Isolated Browsing."
                                         " If true, httpsApp-enabled executables must have httpsApp.shared_access set to 'NONE' to run in this project.",
                                    choices=["true", "false"])
+parser_update_project.add_argument('--https-app-isolated-browsing-options', help="A JSON string with options for Isolated Browsing. See https://documentation.dnanexus.com/developer/api/data-containers/projects#api-method-project-xxxx-update for a list of supported keys.", type=str)
 allowed_executables_group = parser_update_project.add_mutually_exclusive_group()
 allowed_executables_group.add_argument('--allowed-executables', help='Executable ID(s) this project is allowed to run.  This operation overrides any existing list of executables.', type=str, nargs="+")
 allowed_executables_group.add_argument('--unset-allowed-executables', help='Removes any restriction to run executables as set by --allowed-executables', action='store_true')

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy/templating/templates/nextflow/src/nextflow.sh

@@ -104,7 +104,7 @@ main() {
     $nextflow_pipeline_params)"
 
   NEXTFLOW_CMD_ENV+=("${applet_runtime_inputs[@]}")
-  
+
   AWS_ENV="$HOME/.dx-aws.env"
 
   get_nextflow_environment "${NEXTFLOW_CMD_ENV[@]}"
@@ -112,10 +112,10 @@ main() {
 
   # Login to AWS, if configured
   aws_login
-  aws_relogin_loop & AWS_RELOGIN_PID=$!
+  refresh_web_identity_token_loop & TOKEN_REFRESH_PID=$!
 
   set_vars_session_and_cache
-  
+
   if [[ $preserve_cache == true ]]; then
     set_job_properties_cache
     check_cache_db_storage_limit
@@ -168,7 +168,7 @@ main() {
   wait $NXF_EXEC_PID
   ret=$?
 
-  kill "$AWS_RELOGIN_PID"
+  kill "$TOKEN_REFRESH_PID" 2>/dev/null || true
   exit $ret
 }
 
@@ -243,7 +243,7 @@ nf_task_entry() {
   [[ -f $CREDENTIALS ]] && docker_registry_login  || echo "no docker credential available"
   dx download "$DX_WORKSPACE_ID:/.dx-aws.env" -o $AWS_ENV -f --no-progress 2>/dev/null || true
   aws_login
-  aws_relogin_loop & AWS_RELOGIN_PID=$!
+  refresh_web_identity_token_loop & TOKEN_REFRESH_PID=$!
   # capture the exit code
   trap nf_task_exit EXIT
 
@@ -256,12 +256,13 @@ nf_task_entry() {
   set +e
   bash .command.run > >(tee .command.log) 2>&1
   export exit_code=$?
-  kill "$AWS_RELOGIN_PID"
   dx set_properties ${DX_JOB_ID} nextflow_exit_code=$exit_code
   set -e
 }
 
 nf_task_exit() {
+  kill "$TOKEN_REFRESH_PID" 2>/dev/null || true
+
   if [ -f .command.log ]; then
     if [[ $USING_S3_WORKDIR == true ]]; then
       aws s3 cp .command.log "${cmd_log_file}"
@@ -336,34 +337,56 @@ aws_login() {
   if [ -f "$AWS_ENV" ]; then
     source $AWS_ENV
     detect_if_using_s3_workdir
-  
-    # aws env file example values:
-    # "iamRoleArnToAssume", "jobTokenAudience", "jobTokenSubjectClaims", "awsRegion"
-    roleSessionName="dnanexus_${DX_JOB_ID}"
-    job_id_token=$(dx-jobutil-get-identity-token --aud ${jobTokenAudience} --subject_claims ${jobTokenSubjectClaims})
-    output=$(aws sts assume-role-with-web-identity --role-arn $iamRoleArnToAssume --role-session-name $roleSessionName --web-identity-token $job_id_token --duration-seconds 3600)
-    mkdir -p /home/dnanexus/.aws/
-
-    cat <<EOF > /home/dnanexus/.aws/credentials
-[default]
-aws_access_key_id = $(echo "$output" | jq -r '.Credentials.AccessKeyId')
-aws_secret_access_key = $(echo "$output" | jq -r '.Credentials.SecretAccessKey')
-aws_session_token = $(echo "$output" | jq -r '.Credentials.SessionToken')
-EOF
-    cat <<EOF > /home/dnanexus/.aws/config
-[default]
-region = $awsRegion
-EOF
-    echo "Successfully authenticated to AWS - $(aws sts get-caller-identity)"
+
+    local web_identity_token_file="/tmp/aws_web_identity_token"
+
+    dx-jobutil-get-identity-token --aud ${jobTokenAudience} --subject_claims ${jobTokenSubjectClaims} > ${web_identity_token_file}
+
+    export AWS_REGION="$awsRegion"
+    export AWS_ROLE_ARN="$iamRoleArnToAssume"
+    export AWS_ROLE_SESSION_NAME="dnanexus_${DX_JOB_ID}"
+    export AWS_WEB_IDENTITY_TOKEN_FILE="$web_identity_token_file"
+
+    # Clean up any old AWS config files to avoid conflicts. The SDK will now ignore ~/.aws/config and ~/.aws/credentials because the environment variables take precedence.
+    rm -rf /home/dnanexus/.aws/
+
+    # This explicit check is added to ensure that existing pytest tests for invalid credentials still fail correctly.
+    # In the new model, the AWS SDK would normally handle this lazily, but the tests expect an immediate failure.
+    aws sts assume-role-with-web-identity --role-arn "$iamRoleArnToAssume" \
+      --role-session-name "dnanexus_${DX_JOB_ID}" \
+      --web-identity-token "$(cat "${web_identity_token_file}")" > /dev/null
+
+    echo "Successfully configured AWS with Web Identity Token File."
+    # Optional sanity check; SDK/CLI will now do STS AssumeRoleWithWebIdentity on demand
+    aws sts get-caller-identity >/dev/null 2>&1 || echo "Note: initial STS call deferred to Nextflow/AWS SDK."
   fi
 }
 
-aws_relogin_loop() {
+refresh_web_identity_token_loop() {
+  # The DNAnexus job identity token expires in ~5 minutes.
+  # Refresh the OIDC token file proactively so the AWS SDK v2 can re-assume the role on demand.
+
   while true; do
-    sleep 3300 # relogin every 55 minutes, first login is done separately, so we wait before the login
-      if [ -f "$AWS_ENV" ]; then
-        aws_login
-      fi
+    sleep 240 # 4 minutes
+
+    if [ -f "$AWS_ENV" ]; then
+      source "$AWS_ENV"
+
+      local tmp_token_file="${AWS_WEB_IDENTITY_TOKEN_FILE}.tmp"
+      local attempt=0
+
+      while [ "$attempt" -le 3 ]; do
+        if dx-jobutil-get-identity-token --aud "${jobTokenAudience}" --subject_claims "${jobTokenSubjectClaims}" > "$tmp_token_file"; then
+          mv -f "$tmp_token_file" "$AWS_WEB_IDENTITY_TOKEN_FILE"
+
+          break
+        else
+          echo "WARNING: AWS token refresh failed (attempt $((attempt+1))/3)" >&2
+          sleep 5 # wait 5s before retry
+        fi
+        attempt=$((attempt+1))
+      done
+    fi
   done
 }
 
@@ -553,8 +576,8 @@ get_resume_session_id() {
 
   valid_id_pattern='^\{?[A-Z0-9a-z]{8}-[A-Z0-9a-z]{4}-[A-Z0-9a-z]{4}-[A-Z0-9a-z]{4}-[A-Z0-9a-z]{12}\}?$'
   [[ "$PREV_JOB_SESSION_ID" =~ $valid_id_pattern ]] ||
-    dx-jobutil-report-error "Invalid resume value. Please provide either \"true\", \"last\", or \"sessionID\". 
-    If a sessionID was provided, Nextflow cached content could not be found under $DX_CACHEDIR/$PREV_JOB_SESSION_ID/. 
+    dx-jobutil-report-error "Invalid resume value. Please provide either \"true\", \"last\", or \"sessionID\".
+    If a sessionID was provided, Nextflow cached content could not be found under $DX_CACHEDIR/$PREV_JOB_SESSION_ID/.
     Please provide the exact sessionID for \"resume\" or run without resume."
 
   NXF_UUID=$PREV_JOB_SESSION_ID
@@ -596,8 +619,8 @@ check_no_concurrent_job_same_cache() {
   )
 
   [[ -n $FIRST_RESUMED_JOB && $DX_JOB_ID == $FIRST_RESUMED_JOB ]] ||
-    dx-jobutil-report-error "There is at least one other non-terminal state job with the same sessionID $NXF_UUID. 
-    Please wait until all other jobs sharing the same sessionID to enter their terminal state and rerun, 
+    dx-jobutil-report-error "There is at least one other non-terminal state job with the same sessionID $NXF_UUID.
+    Please wait until all other jobs sharing the same sessionID to enter their terminal state and rerun,
     or run without preserve_cache set to true."
 }
 
@@ -605,12 +628,12 @@ restore_cache_and_set_resume_cmd() {
   # download latest cache.tar from $DX_CACHEDIR/$PREV_JOB_SESSION_ID/
   PREV_JOB_CACHE_FILE=$(
     dx api system findDataObjects \
-      '{"visibility": "either", 
+      '{"visibility": "either",
         "name":"cache.tar",
         "scope": {
-          "project": "'$DX_PROJECT_CONTEXT_ID'", 
-          "folder": "/.nextflow_cache_db/'$NXF_UUID'", 
-          "recurse": false}, 
+          "project": "'$DX_PROJECT_CONTEXT_ID'",
+          "folder": "/.nextflow_cache_db/'$NXF_UUID'",
+          "recurse": false},
         "describe": true}' 2>/dev/null |
       jq -r '.results | sort_by(.describe.created)[-1] | .id // empty'
   )

dxpy-0.400.1/dxpy/toolkit_version.py

@@ -0,0 +1 @@
+version = '0.400.1'

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.399.1
+Version: 0.400.1
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.399.1 → dxpy-0.400.1}/dxpy.egg-info/SOURCES.txt

@@ -111,6 +111,7 @@ dxpy/nextflow/app_asset_projects_ids_staging.json
 dxpy/nextflow/awscli_assets.json
 dxpy/nextflow/awscli_assets.staging.json
 dxpy/nextflow/collect_images.py
+dxpy/nextflow/default_nextflow_instance_types.json
 dxpy/nextflow/nextaur_assets.json
 dxpy/nextflow/nextaur_assets.staging.json
 dxpy/nextflow/nextflow_assets.json

{dxpy-0.399.1 → dxpy-0.400.1}/test/test_dxclient.py

@@ -11112,10 +11112,11 @@ class TestDXCp(DXTestCase):
         fname1 = self.gen_uniq_fname()
         create_file_in_project(fname1, self.proj_id1)
 
-        # The file {proj_id1}:/{f} exists, however, {proj_id1}/{f} does not
-        expected_err_msg = "ResolutionError: The specified folder could not be found in {p}".format(p=self.project)
+        # The file {proj_id1}:/{f} exists, however, {proj_id1}/non_existing_folder/{f} does not
+        non_existing_folder_name = "/non_existing_folder"
+        expected_err_msg = "Folder \"{fn}\" does not exist in project \"{p1}\"".format(p1=self.proj_id1, fn=non_existing_folder_name)
         with self.assertSubprocessFailure(stderr_regexp=expected_err_msg, exit_code=3):
-            run("dx cp {p1}/{f} {p2}:/".format(p1=self.proj_id1, f=fname1, p2=self.proj_id2))
+            run("dx cp {p1}:{fn}/{f} {p2}:/".format(p1=self.proj_id1, fn=non_existing_folder_name , f=fname1, p2=self.proj_id2))
 
         with self.assertSubprocessFailure(stderr_regexp="The destination folder does not exist",
                                           exit_code=3):

dxpy-0.399.1/dxpy/toolkit_version.py

@@ -1 +0,0 @@
-version = '0.399.1'