dsub 0.5.2__tar.gz → 0.5.4__tar.gz

{dsub-0.5.2/dsub.egg-info → dsub-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsub
-Version: 0.5.2
+Version: 0.5.4
 Summary: A command-line tool that makes it easy to submit and run batch scripts in the cloud
 Home-page: https://github.com/DataBiosphere/dsub
 Author: Verily
@@ -108,7 +108,7 @@ While not used directly by `dsub` for the `google-batch` provider, you are likel
 Cloud SDK](https://cloud.google.com/sdk/).
 
 If you will be using the `local` provider for faster job development,
-you *will* need to install the Google Cloud SDK, which uses `gsutil` to ensure
+you *will* need to install the Google Cloud SDK, which uses `gcloud storage` to ensure
 file operation semantics consistent with the Google `dsub` providers.
 
 1. [Install the Google Cloud SDK](https://cloud.google.com/sdk/)
@@ -234,10 +234,10 @@ The steps for getting started differ slightly as indicated in the steps below:
 
     The dsub logs and output files will be written to a bucket. Create a
     bucket using the [storage browser](https://console.cloud.google.com/storage/browser?project=)
-    or run the command-line utility [gsutil](https://cloud.google.com/storage/docs/gsutil),
+    or run the command-line utility [gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage),
     included in the Cloud SDK.
 
-        gsutil mb gs://my-bucket
+        gcloud storage buckets create gs://my-bucket
 
     Change `my-bucket` to a unique name that follows the
     [bucket-naming conventions](https://cloud.google.com/storage/docs/bucket-naming).
@@ -267,7 +267,7 @@ The steps for getting started differ slightly as indicated in the steps below:
 
 1. View the output file.
 
-        gsutil cat gs://my-bucket/output/out.txt
+        gcloud storage cat gs://my-bucket/output/out.txt
 
 ## Backend providers
 
@@ -403,9 +403,8 @@ by:
 
 To upload the files to Google Cloud Storage, you can use the
 [storage browser](https://console.cloud.google.com/storage/browser?project=) or
-[gsutil](https://cloud.google.com/storage/docs/gsutil). You can also run on data
-that’s public or shared with your service account, an email address that you
-can find in the [Google Cloud Console](https://console.cloud.google.com).
+[gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage). 
+You can also run on data that’s public or shared with your service account, an email address that you can find in the [Google Cloud Console](https://console.cloud.google.com).
 
 #### Files
 
@@ -780,7 +779,9 @@ of the service account will be `sa-name@project-id.iam.gserviceaccount.com`.
 
 2. Grant IAM access on buckets, etc. to the service account.
 
-        gsutil iam ch serviceAccount:sa-name@project-id.iam.gserviceaccount.com:roles/storage.objectAdmin gs://bucket-name
+        gcloud storage buckets add-iam-policy-binding gs://bucket-name \
+            --member=serviceAccount:sa-name@project-id.iam.gserviceaccount.com \
+            --role=roles/storage.objectAdmin
 
 3. Update your `dsub` command to include `--service-account`
 

{dsub-0.5.2 → dsub-0.5.4}/README.md

@@ -56,7 +56,7 @@ While not used directly by `dsub` for the `google-batch` provider, you are likel
 Cloud SDK](https://cloud.google.com/sdk/).
 
 If you will be using the `local` provider for faster job development,
-you *will* need to install the Google Cloud SDK, which uses `gsutil` to ensure
+you *will* need to install the Google Cloud SDK, which uses `gcloud storage` to ensure
 file operation semantics consistent with the Google `dsub` providers.
 
 1. [Install the Google Cloud SDK](https://cloud.google.com/sdk/)
@@ -182,10 +182,10 @@ The steps for getting started differ slightly as indicated in the steps below:
 
     The dsub logs and output files will be written to a bucket. Create a
     bucket using the [storage browser](https://console.cloud.google.com/storage/browser?project=)
-    or run the command-line utility [gsutil](https://cloud.google.com/storage/docs/gsutil),
+    or run the command-line utility [gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage),
     included in the Cloud SDK.
 
-        gsutil mb gs://my-bucket
+        gcloud storage buckets create gs://my-bucket
 
     Change `my-bucket` to a unique name that follows the
     [bucket-naming conventions](https://cloud.google.com/storage/docs/bucket-naming).
@@ -215,7 +215,7 @@ The steps for getting started differ slightly as indicated in the steps below:
 
 1. View the output file.
 
-        gsutil cat gs://my-bucket/output/out.txt
+        gcloud storage cat gs://my-bucket/output/out.txt
 
 ## Backend providers
 
@@ -351,9 +351,8 @@ by:
 
 To upload the files to Google Cloud Storage, you can use the
 [storage browser](https://console.cloud.google.com/storage/browser?project=) or
-[gsutil](https://cloud.google.com/storage/docs/gsutil). You can also run on data
-that’s public or shared with your service account, an email address that you
-can find in the [Google Cloud Console](https://console.cloud.google.com).
+[gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage). 
+You can also run on data that’s public or shared with your service account, an email address that you can find in the [Google Cloud Console](https://console.cloud.google.com).
 
 #### Files
 
@@ -728,7 +727,9 @@ of the service account will be `sa-name@project-id.iam.gserviceaccount.com`.
 
 2. Grant IAM access on buckets, etc. to the service account.
 
-        gsutil iam ch serviceAccount:sa-name@project-id.iam.gserviceaccount.com:roles/storage.objectAdmin gs://bucket-name
+        gcloud storage buckets add-iam-policy-binding gs://bucket-name \
+            --member=serviceAccount:sa-name@project-id.iam.gserviceaccount.com \
+            --role=roles/storage.objectAdmin
 
 3. Update your `dsub` command to include `--service-account`
 

{dsub-0.5.2 → dsub-0.5.4}/dsub/_dsub_version.py

@@ -27,5 +27,5 @@ A typical release sequence will be versioned as:
 """
 
 
-DSUB_VERSION = '0.5.2'
+DSUB_VERSION = '0.5.4'
 

{dsub-0.5.2 → dsub-0.5.4}/dsub/lib/param_util.py

@@ -827,25 +827,22 @@ def directory_fmt(directory):
 
   Multiple files copy, works as intended in all cases:
       $ touch a.txt b.txt
-      $ gsutil cp ./*.txt gs://mybucket/text_dest
-      $ gsutil ls gs://mybucket/text_dest/
-            0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/a.txt
-            0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/b.txt
-      TOTAL: 2 objects, 0 bytes (0 B)
+      $ gcloud storage cp ./*.txt gs://mybucket/text_dest
+      $ gcloud storage ls gs://mybucket/text_dest/
+      gs://mybucket/text_dest/a.txt
+      gs://mybucket/text_dest/b.txt
 
   Single file copy fails to copy into a directory:
       $ touch 1.bam
-      $ gsutil cp ./*.bam gs://mybucket/bad_dest
-      $ gsutil ls gs://mybucket/bad_dest
-               0  2017-07-19T21:46:16Z  gs://mybucket/bad_dest
-      TOTAL: 1 objects, 0 bytes (0 B)
+      $ gcloud storage cp ./*.bam gs://mybucket/bad_dest
+      $ gcloud storage ls gs://mybucket/bad_dest
+      gs://mybucket/bad_dest
 
   Adding a trailing forward slash fixes this:
       $ touch my.sam
-      $ gsutil cp ./*.sam gs://mybucket/good_folder
-      $ gsutil ls gs://mybucket/good_folder
-               0  2017-07-19T21:46:16Z  gs://mybucket/good_folder/my.sam
-      TOTAL: 1 objects, 0 bytes (0 B)
+      $ gcloud storage cp ./*.sam gs://mybucket/good_folder
+      $ gcloud storage ls gs://mybucket/good_folder
+      gs://mybucket/good_folder/my.sam
 
   Args:
     directory (str): a uri without an blob or file basename.

{dsub-0.5.2 → dsub-0.5.4}/dsub/lib/providers_util.py

@@ -20,7 +20,7 @@ from . import job_model
 from .._dsub_version import DSUB_VERSION
 
 _LOCALIZE_COMMAND_MAP = {
-    job_model.P_GCS: 'gsutil -m rsync -r',
+    job_model.P_GCS: 'gcloud storage rsync -r',
     job_model.P_LOCAL: 'rsync -r',
 }
 

{dsub-0.5.2 → dsub-0.5.4}/dsub/providers/google_batch.py

@@ -145,11 +145,11 @@ _LOG_CP = textwrap.dedent("""
       "${{LOGGING_DIR}}/stderr.txt" \
       "{user_action}"
 
-  gsutil_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
   STDOUT_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
   STDERR_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
   LOG_PID=$!
 
   wait "${{STDOUT_PID}}"
@@ -168,7 +168,7 @@ _FINAL_LOGGING_CMD = textwrap.dedent("""\
   touch "${{LOGGING_DIR}}/.stop_logging"
 
   {log_msg_fn}
-  {gsutil_cp_fn}
+  {gcloud_cp_fn}
 
   {log_cp}
 """)
@@ -182,14 +182,14 @@ _CONTINUOUS_LOGGING_CMD = textwrap.dedent("""\
   readonly LOGGING_DIR="{logging_dir}"
 
   {log_msg_fn}
-  {gsutil_cp_fn}
+  {gcloud_cp_fn}
 
   # Make sure the logging work directory exists
   mkdir -p "${{LOGGING_DIR}}"
 
   # Prep the log filter script
   echo "${{{log_filter_var}}}" \
-    | python -c '{python_decode_script}' \
+    | python3 -c '{python_decode_script}' \
     > "{log_filter_script_path}"
   chmod a+x "{log_filter_script_path}"
 
@@ -609,7 +609,7 @@ class GoogleBatchJobProvider(google_utils.GoogleJobProviderBase):
 
     continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(
         log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        gcloud_cp_fn=google_utils.GCLOUD_CP_FN,
         log_filter_var=_LOG_FILTER_VAR,
         log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
         python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
@@ -623,7 +623,7 @@ class GoogleBatchJobProvider(google_utils.GoogleJobProviderBase):
 
     logging_cmd = _FINAL_LOGGING_CMD.format(
         log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        gcloud_cp_fn=google_utils.GCLOUD_CP_FN,
         log_filter_var=_LOG_FILTER_VAR,
         log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
         python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
@@ -729,8 +729,8 @@ class GoogleBatchJobProvider(google_utils.GoogleJobProviderBase):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.LOCALIZATION_LOOP,
                 ),
             ],
@@ -778,8 +778,8 @@ class GoogleBatchJobProvider(google_utils.GoogleJobProviderBase):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.DELOCALIZATION_LOOP,
                 ),
             ],

{dsub-0.5.2 → dsub-0.5.4}/dsub/providers/google_utils.py

@@ -68,9 +68,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 # pylint: enable=g-complex-comprehension
 
 
-# Action steps that interact with GCS need gsutil and Python.
+# Action steps that interact with GCS need gcloud and Python.
 # Use the 'slim' variant of the cloud-sdk image as it is much smaller.
-CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:294.0.0-slim'
+CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:499.0.0-slim'
 
 # Name of the data disk
 DATA_DISK_NAME = 'datadisk'
@@ -94,10 +94,10 @@ LOG_MSG_FN = textwrap.dedent("""\
   }
 """)
 
-# Define a bash function for "gsutil cp" to be used by the logging,
+# Define a bash function for "gcloud storage cp" to be used by the logging,
 # localization, and delocalization actions.
-GSUTIL_CP_FN = textwrap.dedent("""\
-  function gsutil_cp() {
+GCLOUD_CP_FN = textwrap.dedent("""\
+  function gcloud_cp() {
     local src="${1}"
     local dst="${2}"
     local content_type="${3}"
@@ -105,33 +105,33 @@ GSUTIL_CP_FN = textwrap.dedent("""\
 
     local headers=""
     if [[ -n "${content_type}" ]]; then
-      headers="-h Content-Type:${content_type}"
+      headers="--content-type=${content_type}"
     fi
 
     local user_project_flag=""
     if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
+      user_project_flag="--billing-project=${user_project_name}"
     fi
 
     local attempt
     for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
-      if gsutil ${headers} ${user_project_flag} -mq cp "${src}" "${dst}"; then
+      log_info "gcloud storage cp --no-user-output-enabled ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
+      if gcloud storage cp --no-user-output-enabled ${headers} ${user_project_flag} "${src}" "${dst}"; then
         return
       fi
       if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+        log_warning "Sleeping 10s before the next attempt of failed gcloud command"
+        log_warning "gcloud storage cp --no-user-output-enabled ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
         sleep 10s
       fi
     done
 
-    log_error "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+    log_error "gcloud storage cp --no-user-output-enabled ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
     exit 1
   }
 """)
 
-LOG_CP_FN = GSUTIL_CP_FN + textwrap.dedent("""\
+LOG_CP_FN = GCLOUD_CP_FN + textwrap.dedent("""\
 
   function log_cp() {
     local src="${1}"
@@ -144,43 +144,43 @@ LOG_CP_FN = GSUTIL_CP_FN + textwrap.dedent("""\
       return
     fi
 
-    # Copy the log files to a local temporary location so that our "gsutil cp" is never
+    # Copy the log files to a local temporary location so that our "gcloud storage cp" is never
     # executed on a file that is changing.
 
     local tmp_path="${tmp}/$(basename ${src})"
     cp "${src}" "${tmp_path}"
 
-    gsutil_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
+    gcloud_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
   }
 """)
 
-# Define a bash function for "gsutil rsync" to be used by the logging,
+# Define a bash function for "gcloud rsync" to be used by the logging,
 # localization, and delocalization actions.
-GSUTIL_RSYNC_FN = textwrap.dedent("""\
-  function gsutil_rsync() {
+GCLOUD_RSYNC_FN = textwrap.dedent("""\
+  function gcloud_rsync() {
     local src="${1}"
     local dst="${2}"
     local user_project_name="${3}"
 
     local user_project_flag=""
     if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
+      user_project_flag="--billing-project=${user_project_name}"
     fi
 
     local attempt
     for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
-      if gsutil ${user_project_flag} -mq rsync -r "${src}" "${dst}"; then
+      log_info "gcloud storage rsync -r --no-user-output-enabled ${user_project_flag} \"${src}\" \"${dst}\""
+      if gcloud storage rsync -r --no-user-output-enabled ${user_project_flag} "${src}" "${dst}"; then
         return
       fi
       if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+        log_warning "Sleeping 10s before the next attempt of failed gcloud command"
+        log_warning "gcloud storage rsync -r --no-user-output-enabled ${user_project_flag} \"${src}\" \"${dst}\""
         sleep 10s
       fi
     done
 
-    log_error "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+    log_error "gcloud storage rsync -r --no-user-output-enabled ${user_project_flag} \"${src}\" \"${dst}\""
     exit 1
   }
 """)
@@ -198,9 +198,9 @@ LOCALIZATION_LOOP = textwrap.dedent("""\
 
     log_info "Localizing ${!INPUT_VAR}"
     if [[ "${!INPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
+      gcloud_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
     else
-      gsutil_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
+      gcloud_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
     fi
   done
 """)
@@ -218,9 +218,9 @@ DELOCALIZATION_LOOP = textwrap.dedent("""\
 
     log_info "Delocalizing ${!OUTPUT_VAR}"
     if [[ "${!OUTPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
+      gcloud_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
     else
-      gsutil_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
+      gcloud_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
     fi
   done
 """)
@@ -274,7 +274,7 @@ PREPARE_CMD = textwrap.dedent("""\
   {mk_runtime_dirs}
 
   echo "${{{script_var}}}" \
-    | python -c '{python_decode_script}' \
+    | python3 -c '{python_decode_script}' \
     > "{script_path}"
   chmod a+x "{script_path}"
 

{dsub-0.5.2 → dsub-0.5.4}/dsub/providers/google_v2_base.py

@@ -529,8 +529,8 @@ class GoogleV2JobProviderBase(google_utils.GoogleJobProviderBase):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.LOCALIZATION_LOOP,
                 ),
             ],
@@ -566,8 +566,8 @@ class GoogleV2JobProviderBase(google_utils.GoogleJobProviderBase):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.DELOCALIZATION_LOOP,
                 ),
             ],

{dsub-0.5.2 → dsub-0.5.4}/dsub/providers/local.py

@@ -712,9 +712,9 @@ class LocalJobProvider(base.JobProvider):
     elif logging_path.file_provider == job_model.P_GCS:
       mkdir_cmd = ''
       if user_project:
-        cp_cmd = 'gsutil -u {} -mq cp'.format(user_project)
+        cp_cmd = 'gcloud storage cp --billing-project={}'.format(user_project)
       else:
-        cp_cmd = 'gsutil -mq cp'
+        cp_cmd = 'gcloud storage cp'
     else:
       assert False
 
@@ -773,7 +773,7 @@ class LocalJobProvider(base.JobProvider):
     return '\n'.join(provider_commands)
 
   def _get_input_target_path(self, local_file_path):
-    """Returns a directory or file path to be the target for "gsutil cp".
+    """Returns a directory or file path to be the target for "gcloud storage cp".
 
     If the filename contains a wildcard, then the target path must
     be a directory in order to ensure consistency whether the source pattern
@@ -784,7 +784,7 @@ class LocalJobProvider(base.JobProvider):
       local_file_path: A full path terminating in a file or a file wildcard.
 
     Returns:
-      The path to use as the "gsutil cp" target.
+      The path to use as the "gcloud storage cp" target.
     """
 
     path, filename = os.path.split(local_file_path)
@@ -806,21 +806,34 @@ class LocalJobProvider(base.JobProvider):
 
       commands.append('mkdir -p "%s"' % os.path.dirname(local_file_path))
 
-      if i.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
-        # The semantics that we expect here are implemented consistently in
-        # "gsutil cp", and are a bit different than "cp" when it comes to
-        # wildcard handling, so use it for both local and GCS:
-        #
-        # - `cp path/* dest/` will error if "path" has subdirectories.
-        # - `cp "path/*" "dest/"` will fail (it expects wildcard expansion
-        #   to come from shell).
+      if i.file_provider == job_model.P_GCS:
+        # `cp path/* dest/` will error if "path" has subdirectories, and
+        # `cp "path/*" "dest/"` will fail (it expects wildcard expansion
+        # to come from shell), so use "gcloud storage cp" instead.
         if user_project:
-          command = 'gsutil -u %s -mq cp "%s" "%s"' % (
+          command = 'gcloud storage cp --billing-project=%s "%s" "%s"' % (
               user_project, source_file_path, dest_file_path)
         else:
-          command = 'gsutil -mq cp "%s" "%s"' % (source_file_path,
+          command = 'gcloud storage cp "%s" "%s"' % (source_file_path,
                                                  dest_file_path)
         commands.append(command)
+      elif i.file_provider == job_model.P_LOCAL:
+        # "gcloud storage cp" does not support local-to-local copies, so use
+        # rsync (without "-r", since this path is non-recursive) instead.
+        # It matches the wildcard semantics we need: like the old "gsutil
+        # cp", it silently skips subdirectories rather than erroring like
+        # plain "cp" does. rsync does not expand a quoted wildcard itself,
+        # so for a wildcard filename, only the directory portion is quoted,
+        # leaving the wildcard for the shell to expand. A non-wildcard
+        # filename is quoted as a whole so that spaces in it are preserved
+        # as a single rsync argument.
+        source_dir, source_filename = os.path.split(source_file_path)
+        if '*' in source_filename:
+          command = 'rsync "%s"/%s "%s"' % (source_dir, source_filename,
+                                             dest_file_path)
+        else:
+          command = 'rsync "%s" "%s"' % (source_file_path, dest_file_path)
+        commands.append(command)
 
     return '\n'.join(commands)
 
@@ -865,13 +878,25 @@ class LocalJobProvider(base.JobProvider):
       if o.file_provider == job_model.P_LOCAL:
         commands.append('mkdir -p "%s"' % dest_path)
 
-      # Use gsutil even for local files (explained in _localize_inputs_command).
-      if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
+      if o.file_provider == job_model.P_GCS:
         if user_project:
-          command = 'gsutil -u %s -mq cp "%s" "%s"' % (user_project, local_path,
+          command = 'gcloud storage cp --billing-project=%s "%s" "%s"' % (user_project, local_path,
                                                        dest_path)
         else:
-          command = 'gsutil -mq cp "%s" "%s"' % (local_path, dest_path)
+          command = 'gcloud storage cp "%s" "%s"' % (local_path, dest_path)
+        commands.append(command)
+      elif o.file_provider == job_model.P_LOCAL:
+        # "gcloud storage cp" does not support local-to-local copies (see
+        # _localize_inputs_command), so use rsync instead. As there, only
+        # quote the directory portion for a wildcard filename so the shell
+        # can expand it; a non-wildcard filename is quoted as a whole so
+        # that spaces in it are preserved as a single rsync argument.
+        local_dir, local_filename = os.path.split(local_path)
+        if '*' in local_filename:
+          command = 'rsync "%s"/%s "%s"' % (local_dir, local_filename,
+                                             dest_path)
+        else:
+          command = 'rsync "%s" "%s"' % (local_path, dest_path)
         commands.append(command)
 
     return '\n'.join(commands)

{dsub-0.5.2 → dsub-0.5.4/dsub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsub
-Version: 0.5.2
+Version: 0.5.4
 Summary: A command-line tool that makes it easy to submit and run batch scripts in the cloud
 Home-page: https://github.com/DataBiosphere/dsub
 Author: Verily
@@ -108,7 +108,7 @@ While not used directly by `dsub` for the `google-batch` provider, you are likel
 Cloud SDK](https://cloud.google.com/sdk/).
 
 If you will be using the `local` provider for faster job development,
-you *will* need to install the Google Cloud SDK, which uses `gsutil` to ensure
+you *will* need to install the Google Cloud SDK, which uses `gcloud storage` to ensure
 file operation semantics consistent with the Google `dsub` providers.
 
 1. [Install the Google Cloud SDK](https://cloud.google.com/sdk/)
@@ -234,10 +234,10 @@ The steps for getting started differ slightly as indicated in the steps below:
 
     The dsub logs and output files will be written to a bucket. Create a
     bucket using the [storage browser](https://console.cloud.google.com/storage/browser?project=)
-    or run the command-line utility [gsutil](https://cloud.google.com/storage/docs/gsutil),
+    or run the command-line utility [gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage),
     included in the Cloud SDK.
 
-        gsutil mb gs://my-bucket
+        gcloud storage buckets create gs://my-bucket
 
     Change `my-bucket` to a unique name that follows the
     [bucket-naming conventions](https://cloud.google.com/storage/docs/bucket-naming).
@@ -267,7 +267,7 @@ The steps for getting started differ slightly as indicated in the steps below:
 
 1. View the output file.
 
-        gsutil cat gs://my-bucket/output/out.txt
+        gcloud storage cat gs://my-bucket/output/out.txt
 
 ## Backend providers
 
@@ -403,9 +403,8 @@ by:
 
 To upload the files to Google Cloud Storage, you can use the
 [storage browser](https://console.cloud.google.com/storage/browser?project=) or
-[gsutil](https://cloud.google.com/storage/docs/gsutil). You can also run on data
-that’s public or shared with your service account, an email address that you
-can find in the [Google Cloud Console](https://console.cloud.google.com).
+[gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage). 
+You can also run on data that’s public or shared with your service account, an email address that you can find in the [Google Cloud Console](https://console.cloud.google.com).
 
 #### Files
 
@@ -780,7 +779,9 @@ of the service account will be `sa-name@project-id.iam.gserviceaccount.com`.
 
 2. Grant IAM access on buckets, etc. to the service account.
 
-        gsutil iam ch serviceAccount:sa-name@project-id.iam.gserviceaccount.com:roles/storage.objectAdmin gs://bucket-name
+        gcloud storage buckets add-iam-policy-binding gs://bucket-name \
+            --member=serviceAccount:sa-name@project-id.iam.gserviceaccount.com \
+            --role=roles/storage.objectAdmin
 
 3. Update your `dsub` command to include `--service-account`
 

{dsub-0.5.2 → dsub-0.5.4}/test/integration/e2e_after.py

@@ -56,7 +56,7 @@ next_job = test.run_dsub([
 
 print('\nChecking output...')
 
-RESULT = test_util.gsutil_cat(TEST_FILE_PATH_2)
+RESULT = test_util.gcloud_cat(TEST_FILE_PATH_2)
 if 'hello world' not in RESULT:
   print('Output file does not match expected', file=sys.stderr)
   sys.exit(1)

{dsub-0.5.2 → dsub-0.5.4}/test/integration/e2e_env_list.py

@@ -73,7 +73,7 @@ VAR4=VAL4
 VAR5=VAL5
 """.lstrip()
 
-RESULT = test_util.gsutil_cat(test.STDOUT_LOG)
+RESULT = test_util.gcloud_cat(test.STDOUT_LOG)
 if not test_util.diff(RESULT_EXPECTED, RESULT):
   print('Output file does not match expected')
   sys.exit(1)

{dsub-0.5.2 → dsub-0.5.4}/test/integration/e2e_io_tasks.py

@@ -104,7 +104,7 @@ for i in range(TASKS_COUNT):
       '--output OUTPUT_PATH')
   OUTPUT_FILE = '%s/%s.md5' % (OUTPUT_PATH[:-len('/*.md5')],
                                os.path.basename(INPUT_BAM))
-  RESULT = test_util.gsutil_cat(OUTPUT_FILE)
+  RESULT = test_util.gcloud_cat(OUTPUT_FILE)
 
   if not test_util.diff(RESULT_EXPECTED.strip(), RESULT.strip()):
     print('Output file does not match expected')
@@ -119,7 +119,7 @@ for i in range(TASKS_COUNT):
 RESULT_EXPECTED = POPULATION_MD5
 for i in range(TASKS_COUNT):
   OUTPUT_FILE = '%s/TASK_%s.md5' % (test.OUTPUTS, (i + 1))
-  RESULT = test_util.gsutil_cat(OUTPUT_FILE)
+  RESULT = test_util.gcloud_cat(OUTPUT_FILE)
 
   if not test_util.diff(RESULT_EXPECTED.strip(), RESULT.strip()):
     print('Output file does not match expected')

{dsub-0.5.2 → dsub-0.5.4}/test/integration/test_setup_e2e.py

@@ -100,9 +100,9 @@ else:
 print("  Bucket detected as: %s" % DSUB_BUCKET)
 
 print("  Checking if bucket exists")
-if not test_util.gsutil_ls_check("gs://%s" % DSUB_BUCKET):
+if not test_util.gcloud_ls_check("gs://%s" % DSUB_BUCKET):
   print("Bucket does not exist: %s" % DSUB_BUCKET, file=sys.stderr)
-  print("Create the bucket with \"gsutil mb\".", file=sys.stderr)
+  print("Create the bucket with \"gcloud storage buckets create\".", file=sys.stderr)
   sys.exit(1)
 
 # Set standard LOGGING, INPUTS, and OUTPUTS values
@@ -130,10 +130,10 @@ print("Input path: %s" % INPUTS)
 print("Output path: %s" % OUTPUTS)
 
 print("  Checking if remote test files already exists")
-if test_util.gsutil_ls_check("%s/**" % TEST_GCS_ROOT):
+if test_util.gcloud_ls_check("%s/**" % TEST_GCS_ROOT):
   print("Test files exist: %s" % TEST_GCS_ROOT, file=sys.stderr)
   print("Remove contents:", file=sys.stderr)
-  print("  gsutil -m rm %s/**" % TEST_GCS_ROOT, file=sys.stderr)
+  print("  gcloud storage rm %s/**" % TEST_GCS_ROOT, file=sys.stderr)
   sys.exit(1)
 
 if TASKS_FILE:

{dsub-0.5.2 → dsub-0.5.4}/test/integration/test_util.py

@@ -29,13 +29,13 @@ def to_string(stdoutbytes):
   return stdoutbytes.decode(encoding)
 
 
-def gsutil_ls_check(path):
-  return not subprocess.call('gsutil ls "%s" 2>/dev/null' % path, shell=True)
+def gcloud_ls_check(path):
+  return not subprocess.call('gcloud storage ls "%s" 2>/dev/null' % path, shell=True)
 
 
-def gsutil_cat(path):
+def gcloud_cat(path):
   return to_string(
-      subprocess.check_output('gsutil cat "%s"' % path, shell=True))
+      subprocess.check_output('gcloud storage cat "%s"' % path, shell=True))
 
 
 def diff(str1, str2):