s3p 3.5.4 → 3.6.1

package/package.json

@@ -5,24 +5,26 @@
   },
   "bugs": "https:/github.com/generalui/s3p/issues",
   "dependencies": {
-    "@art-suite/cli": "^1.3.11",
+    "@art-suite/cli": "2.2",
     "@aws-sdk/client-s3": "^3.436.0",
+    "@aws-sdk/lib-storage": "^3.635.0",
     "art-class-system": "^1.11.13",
     "art-communication-status": "^1.6.0",
-    "art-standard-lib": "^1.70.12",
+    "art-standard-lib": "^1.73.3",
     "caffeine-script-runtime": "^1.14.0",
+    "glob": "^11.0.0",
     "neptune-namespaces-runtime": "^3.2.9",
     "shell-escape": "^0.2.0"
   },
   "description": "S3P is a CLI and library that is 5x to 50x faster than aws-cli for bulk S3 operations including: summarize, compare, copy and sync.",
   "devDependencies": {
+    "@art-suite/standard-package-builder": "^1.2.6",
     "art-build-configurator": "^1.29.3",
     "art-testbench": "^3.2.2",
-    "caffeine-script": "^0.74.0",
+    "caffeine-script": "^0.75.1",
     "jest": "^29.3.1",
     "mock-fs": "^5.0.0",
-    "neptune-namespaces": "^4.2.9",
-    "npm-watch": "^0.11.0"
+    "neptune-namespaces": "^4.2.9"
   },
   "homepage": "https://github.com/generalui/s3p",
   "jest": {
@@ -30,6 +32,10 @@
       "build/**/*.js",
       "!build/test/**/*.js"
     ],
+    "coveragePathIgnorePatterns": [
+      "/index.js$",
+      "/namespace.js$"
+    ],
     "testEnvironment": "node",
     "testPathIgnorePatterns": [
       "/node_modules/",
@@ -44,14 +50,15 @@
     "url": "https://github.com/generalui/s3p.git"
   },
   "scripts": {
-    "build": "caf -m -c source -o build\nnpm run nn",
-    "build-clean": "npm run clean; npm run build",
+    "build": "rsync -av  --exclude='*.caf' ./source/ ./build/\ncaf -m -c source -o build\nnpm run nn",
+    "build-clean": "# BUILD CLEAN START\n#   - building into: build-temp\n#\nrm -rf ./__build-old__\nrsync -av  --exclude='*.caf' ./source/ ./build-temp/\ncaf -m -c source -o build-temp\nnpm run nn\n\nnn build-temp/*\n\n# BUILD CLEAN FINIALIZING\n#  - replace old build in:   build\n#  - with new build from:    build-temp\n#\nmv build __build-old__\nmv build-temp build\nrm -rf ./__build-old__",
+    "build-watch": "nodemon -e js,ts,jsx,tsx,caf,coffee,json -i \"build/**\" -x npm run build",
     "clean": "rm -rf build/*",
     "nn": "nn build/* ",
-    "test": "npm run nn\njest",
-    "watch": "npm-watch"
+    "test": "jest",
+    "test-watch": "jest --watch"
   },
-  "version": "3.5.4",
+  "version": "3.6.1",
   "watch": {
     "build": {
       "extensions": [

package/source/S3Parallel/Lib/LibMisc.caf

@@ -1,22 +1,89 @@
 import &StandardImport, {} &path
 exec = &util.promisify &child_process.exec
 
-createS3Url: (bucket, folder, key) ->
-    if folder
-      path.join folder, key
-    else
-      "" s3://#{bucket}/#{key}
-
-shellExec: (command) ->
-  exec command
-  .then ({stdout}) -> stdout
-
-humanByteSize: (bytes, decimals = 2) ->
-  switch
-  when bytes < 1024     then "" #{bytes/1024**0 | 0}_B
-  when bytes < 1024**2  then "" #{(bytes/1024**1).toFixed decimals}kB
-  when bytes < 1024**3  then "" #{(bytes/1024**2).toFixed decimals}mB
-  when bytes < 1024**4  then "" #{(bytes/1024**3).toFixed decimals}gB
-  when bytes < 1024**5  then "" #{(bytes/1024**4).toFixed decimals}tB
-  when bytes < 1024**6  then "" #{(bytes/1024**5).toFixed decimals}pB
-  else                       "" #{(bytes/1024**6).toFixed decimals}eB
+{}
+  createS3Url: (bucket, folder, key) ->
+      if folder
+        path.join folder, key
+      else
+        "" s3://#{bucket}/#{key}
+
+  shellExec: (command) ->
+    exec command
+    .then ({stdout}) -> stdout
+
+  humanByteSize: (bytes, decimals = 2) ->
+    switch
+    when bytes < 1024     then "" #{bytes/1024**0 | 0}_B
+    when bytes < 1024**2  then "" #{(bytes/1024**1).toFixed decimals}kB
+    when bytes < 1024**3  then "" #{(bytes/1024**2).toFixed decimals}mB
+    when bytes < 1024**4  then "" #{(bytes/1024**3).toFixed decimals}gB
+    when bytes < 1024**5  then "" #{(bytes/1024**4).toFixed decimals}tB
+    when bytes < 1024**6  then "" #{(bytes/1024**5).toFixed decimals}pB
+    else                       "" #{(bytes/1024**6).toFixed decimals}eB
+
+  s3UrlPattern = ///i ^s3://([^\/]+)\/?(.*)$
+
+  # https://mybucket.s3.amazonaws.com/prefix
+  # https://mybucket.s3-us-west-2.amazonaws.com/prefix
+  httpUrlPattern =
+    ///i
+      ^
+      https?://
+
+      ([^.]+) # bucket
+
+      \.s3
+
+      ([-.]([-a-z0-9]+))? # region
+
+      \.amazonaws\.com
+
+      (\/(.+))? # prefix
+      $
+
+  # https://s3-us-west-2.amazonaws.com/mybucket/prefix
+  regionFirstHttpUrlPattern  =
+    ///i
+      ^
+      https?://s3[-.]
+
+      ([-a-z0-9]+) # region
+
+      \.amazonaws\.com
+
+      \/([^/]+) # bucket
+
+      (\/(.+))? # prefix
+      $
+
+  parseS3Url = (input) ->
+    if isString input
+      input = input.trim()
+
+      log parseS3Url: {} input
+
+      switch
+      when match = input.match s3UrlPattern
+        [_, bucket, prefix] = match
+
+      when match = input.match httpUrlPattern
+        [_, bucket, _, region, _, prefix] = match
+
+      when match = input.match regionFirstHttpUrlPattern
+        [_, region, bucket, _, prefix] = match
+
+      else
+        folder = input
+
+      {} bucket, prefix, region, folder
+    else {}
+
+  processArgsIntoOptions = (args = []) ->
+    [fromStr, toStr] = args
+    if fromStr
+      {bucket, prefix, region, folder} = parseS3Url fromStr
+    if toStr
+      {bucket: toBucket, prefix: toPrefix, region: toRegion, folder: tofolder} = parseS3Url toStr
+
+    merge {} bucket, prefix, region, folder, toBucket, toPrefix, toRegion, tofolder

package/source/S3Parallel/Lib/LocalFileLister.caf

@@ -0,0 +1,99 @@
+import &StandardImport, &glob, {} &path, fs: &fs.promises
+
+class LocalFileLister extends BaseClass
+
+  constructor: ({fromFolder, _allFiles, _getFileStat}) ->
+    @_fromFolder =
+      # ensure has trailing slash
+      path.join fromFolder, "aaa"
+      .replace /aaa$/, ''
+
+    @_custom_getFileStat = _getFileStat
+
+    @_fileStats = {}
+
+    @_allFilesPromise =
+      if _allFiles
+        Promise.resolve _allFiles
+      else
+        glob
+          path.join fromFolder, "**"
+          nodir: true
+
+        .then ({found}) -> found
+
+      .then (files) ->
+        array file in files.sort()
+          file = file.slice fromFolder.length + 1, file.length
+
+  @getter files: -> @_allFilesPromise
+  @getter :fromFolder
+
+  ## list
+    IN:
+      limit: maximum number of files to return
+      startAfter: if present, returns files strictly greater than startAfter
+      stopAt: if present, returns files up to and including stopAt
+
+    OUT:
+      []
+        Key:
+        LastModified:
+        Size
+        Owner
+  list: ({limit=1000, startAfter, stopAt}) ->
+    if present startAfter then @_findStartIndex startAfter
+    else Promise.deepAll {} @files, startIndex: 0
+    .then ({files, startIndex}) ->
+      out = []
+      i = startIndex
+      tilIndex = min files.length, startIndex + limit
+      stopAt ?= peek files
+      while i < tilIndex && files[i] <= stopAt
+        out.push @_getFileStat files[i]
+        i++
+
+      Promise.all out
+
+  #################################################
+  # PRIVATE
+  #################################################
+  ##
+    IN: file name
+    OUT: []
+      Key: file
+      LastModified: stats.mtime
+      Size: stats.size
+      Owner:
+        DisplayName: stats.uid.toString()
+        ID: stats.uid.toString()
+  _getFileStat: (file) ->
+    if @_custom_getFileStat
+      @_custom_getFileStat file
+    else
+      if f = @_fileStats[file]
+        Promise.resolve f
+      else
+        filePath = path.join @fromFolder, file
+        fs.stat path.join @fromFolder, file
+        .then (stats) -> @_fileStats[file] =
+          Key: file
+          LastModified: stats.mtime
+          Size: stats.size
+          Owner:
+            DisplayName: stats.uid.toString()
+            ID: stats.uid.toString()
+
+  _findStartIndex: (startAfter) ->
+    @files
+    .then (files) ->
+      j = 0
+      k = files.length
+      while j < k
+        m = (j + k) / 2 | 0
+        pivot = files[m]
+        switch
+        when startAfter == pivot then k = j = m
+        when startAfter <  pivot then k = m
+        else                          j = m + 1
+      {} files, startIndex: j

package/source/S3Parallel/Lib/S3.caf

@@ -1,22 +1,14 @@
-import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, {} &shellEscape, &util.promisify, &fs, &path
+import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, &@aws-sdk/lib-storage, {} &shellEscape, &util.promisify, &path
 escape = (single) -> shellEscape [] single
 
+fs = merge
+  {createReadStream} = &fs
+  {copyFile} = &fs.promises
+
 # &@aws-sdk/client-s3.config.setPromisesDependency &bluebird
 
 class S3 extends BaseClass
 
-  @awsSdkS3 = new S3Client merge region: process.env.AWS_REGION # useAccelerateEndpoint: true
-  # https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html
-
-  @classGetter
-    s3: ->
-      listBuckets:    (options = {}) -> Promise.resolve @awsSdkS3.send new ListBucketsCommand options
-      listObjectsV2:  (options = {}) -> Promise.resolve @awsSdkS3.send new ListObjectsV2Command options
-      getObject:      (options = {}) -> Promise.resolve @awsSdkS3.send new GetObjectCommand options
-      copyObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new CopyObjectCommand options
-      deleteObject:   (options = {}) -> Promise.resolve @awsSdkS3.send new DeleteObjectCommand options
-      headObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new HeadObjectCommand options
-
   @commonCopyOptionsSdkKeys =
     :ACL
     :CacheControl
@@ -30,12 +22,41 @@ class S3 extends BaseClass
 
   @commonSdkCopyOptionsByLowerCamelCaseKeys = object k from @commonCopyOptionsSdkKeys with-key lowerCamelCase k
 
-  @listBuckets: =>
+  ## constructor
+    IN: options:
+      region: string, default: process.env.AWS_REGION
+      useAccelerateEndpoint: boolean, default: false
+
+  constructor: (options = []) ->
+    @awsSdkS3 = new S3Client merge
+      region: process.env.AWS_REGION
+      {region, useAccelerateEndpoint} = options
+
+  @getter
+    s3: ->
+      listBuckets:    (options = {}) -> Promise.resolve @awsSdkS3.send new ListBucketsCommand options
+      listObjectsV2:  (options = {}) -> Promise.resolve @awsSdkS3.send new ListObjectsV2Command options
+      getObject:      (options = {}) -> Promise.resolve @awsSdkS3.send new GetObjectCommand options
+      copyObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new CopyObjectCommand options
+      deleteObject:   (options = {}) -> Promise.resolve @awsSdkS3.send new DeleteObjectCommand options
+      headObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new HeadObjectCommand options
+
+
+  listBuckets: ->
     @s3.listBuckets()
     .then ({Buckets}) ->
       object {Name, CreationDate} from Buckets with-key Name with CreationDate
 
-  @list: ({bucket, prefix, limit=1000, fetchOwner, startAfter}) =>
+  ## list
+    OUT:
+      []
+        Key:
+        LastModified:
+        Size
+        ETag
+        StorageClass
+        Owner
+  list: ({bucket, prefix, limit=1000, fetchOwner, startAfter, stopAt}) ->
     startTime = currentSecond()
     @s3.listObjectsV2
       Bucket:     bucket
@@ -43,6 +64,7 @@ class S3 extends BaseClass
       MaxKeys:    limit
       StartAfter: startAfter
       FetchOwner: fetchOwner
+
     .tapCatch (error) ->
       log.error S3.list-error: {} bucket, prefix, startAfter, limit, error
 
@@ -56,14 +78,12 @@ class S3 extends BaseClass
       else if duration > 60
         log.warn S3.list-slow: {} bucket, prefix, startAfter, limit, duration, results: merge results, Contents: "Array #{results.Contents.length}"
 
-      results.Contents
+      if present stopAt
+        array item in-array results.Contents when item.Key <= stopAt
+      else
+        results.Contents
 
-  ##
-    TODO:
-      For large files (>5GB), we need to just use the CLI:
-        aws s3 cp s3://resbio-fastq-drop-internal/180316_NB501104_0354_AHM2L2BGX5/Undetermined/Undetermined_S0_L001_R1_001.fastq.gz s3://genui-dynamodb-migration-scratch/large-test.gz
-        We can add --quiet (probably)
-      Further, that should use acceleration, if enabled - still need to test that.
+  ## copy
     IN:
       options:
         size:       Bytes (optional) if provided and if bigger than the max size allowed by copyObject, largeCopy is used
@@ -73,25 +93,28 @@ class S3 extends BaseClass
         key:      <String>
         fromKey:  from-key (default: key)
         toKey:    to-key (default: key)
+        fromFolder: string
 
         toFolder:   local folder
     OUT:
       Promise.then -> see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#copyObject-property > Callback > Parameters
         {} CopyObjectResult, ...
-  @copy: (options) =>
+  copy: (options) ->
     if options extract largeCopyThreshold, scratchState
       copyScratchState = scratchState.copyScratchState ?= {}
 
     @_normalizeCopyOptions(options) extract
-      fromBucket, toBucket, fromKey, toKey, size
-      toFolder
-      pretend
-      verbose
+      fromBucket, fromFolder, fromKey
+      toBucket, toFolder, toKey
+      pretend, verbose
+      size
 
     if size >= largeCopyThreshold
       @largeCopy {}
         commonCopyOptions: @extractCommonCopyOptions options
-        fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose
+        fromBucket, fromFolder, fromKey
+        toBucket, toFolder, toKey
+        pretend, verbose
 
     else
       copyOptions = merge
@@ -116,14 +139,30 @@ class S3 extends BaseClass
       if pretend
         timeout 1 -> pretend: true
 
+      else if fromFolder
+        fromFilePath = path.join fromFolder, fromKey
+        if toBucket
+          new Upload
+            client: @s3
+            params:
+              Bucket: toBucket
+              Key:    toKey
+              Body:   fs.createReadStream fromFilePath
+
+          .done()
+
+        else
+          createParentDirs toLocalFile, copyScratchState
+          .then -> fs.copyFile fromFilePath, toLocalFile
+
       else if toFolder
         if /\/$/.test(toKey) && size == 0 # this is a folder
-          createParentDirs toLocalFile + "ignored-file-name"
-        else
-          createWriteStreamSafe
-            toLocalFile
+          createParentDirs
+            path.join toLocalFile, "ignored-file-name"
             copyScratchState
 
+        else
+          createWriteStreamSafe toLocalFile, copyScratchState
           .then (writeStream) -> new Promise (resolve, reject) ->
             @s3.getObject copyOptions
             .then ({Body}) ->
@@ -135,12 +174,12 @@ class S3 extends BaseClass
       else
         @s3.copyObject copyOptions
 
-  @delete: (options) =>
+  delete: (options) ->
     @s3.deleteObject
       Bucket: options.bucket
       Key:    options.key
 
-  @largeCopy: (options) =>
+  largeCopy: (options) ->
     options extract commonCopyOptions
     @_normalizeCopyOptions(options) extract
       fromBucket,
@@ -170,13 +209,13 @@ class S3 extends BaseClass
     else
       shellExec command
 
-  # get object's metdata
-  @headObject: ({bucket, key}) =>
+  # get object's metadata
+  headObject: ({bucket, key}) ->
     @s3.headObject
       Bucket: bucket
       Key: key
 
-  @_normalizeCopyOptions: (options) ->
+  _normalizeCopyOptions: (options) ->
     options extract
       bucket, key
       fromBucket = bucket
@@ -192,15 +231,15 @@ class S3 extends BaseClass
       throw new Error "" Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default
     merge options, {} fromBucket, fromKey, toBucket, toKey
 
-  @_getCommonCopyOptionsForSdk: (options) =>
-    object sdkKey, lowerCamelCasedKey from @commonSdkCopyOptionsByLowerCamelCaseKeys with-key sdkKey
+  _getCommonCopyOptionsForSdk: (options) ->
+    object sdkKey, lowerCamelCasedKey from S3.commonSdkCopyOptionsByLowerCamelCaseKeys with-key sdkKey
       options[lowerCamelCasedKey]
 
-  @extractCommonCopyOptions: (options) =>
-    object v, k from @commonSdkCopyOptionsByLowerCamelCaseKeys
+  extractCommonCopyOptions: (options) ->
+    object v, k from S3.commonSdkCopyOptionsByLowerCamelCaseKeys
       options[k]
 
-  @shouldSyncObjects: (options) ->
+  shouldSyncObjects: (options) ->
     @_normalizeCopyOptions(options) extract fromBucket, toBucket, fromKey, toKey
     Promise.then ->
       if options.size < 1024**2 || !options.size
@@ -216,7 +255,7 @@ class S3 extends BaseClass
     OUT:
       copied: Bool
       status: if not copied, explains why
-  @syncObject: (options) =>
+  syncObject: (options) ->
     @shouldSyncObjects options
     .then (shouldSync) ->
       if shouldSync

package/source/S3Parallel/S3Comprehensions.caf

@@ -1,4 +1,4 @@
-import &path, &ArtStandardLib, &ArtClassSystem, &Lib, {} &colors
+import &path, &ArtStandardLib, &ArtClassSystem, &Lib
 
 class S3Comprehensions
 
@@ -210,7 +210,7 @@ class S3Comprehensions
           While it returns true, it will be called once/second/parallel-branch until it returns false.
 
         COMPREHENSION
-        with: (Item) -> ignored - alais of `map`
+        with: (Item) -> ignored - alias of `map`
         when: (Item) -> T/F - if true, pass it on to `map`, else skip
         returning:    if present, return this value instead of stats
         into:         <ALIAS: returning>
@@ -234,6 +234,7 @@ class S3Comprehensions
       startAfter
       stopAt
       filter
+      fromFolder
 
       # compare-to options
       compare
@@ -243,7 +244,7 @@ class S3Comprehensions
       # limits
       limit = 1000
       maxListRequests
-      listConcurrency = 100 # max open reqeusts
+      listConcurrency = 100 # max open requests
 
       # actions
       returnValue
@@ -257,10 +258,16 @@ class S3Comprehensions
       throttle
       stats
 
+      # s3 options
+      region
+      useAccelerateEndpoint
+
       # internal use only
-      s3 = &Lib/S3
+      s3
       pwp = new PromiseWorkerPool listConcurrency # internal use only
 
+    s3 ?= new &Lib/S3 {} region, useAccelerateEndpoint: !!useAccelerateEndpoint
+
     if compare && map && toBucket
       throw new Error "cannot use both `compare` and `map` - use mapList instead"
 
@@ -303,7 +310,7 @@ class S3Comprehensions
     ## applyF
       Apply the custom map or mapList functions
       IN:
-        items:        list of S3 items in the source bucekt as returned by the AwsSdk s3 list call
+        items:        list of S3 items in the source bucket as returned by the AwsSdk s3 list call
         compareItems: (optional, if compare is set), list of items in the toBucket
         compareStartAfter
         compareStopAt
@@ -373,8 +380,8 @@ class S3Comprehensions
         maxOutstanding = Math.max maxOutstanding, outstanding += 2
 
         Promise.all []
-          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter
-          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter: middleKey
+          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter, stopAt: middleKey
+          pwp.queue -> s3.list {} bucket, limit, fetchOwner, stopAt, startAfter: middleKey
 
         .then ([rawLeftItems, rawRightItems]) ->
           rawLeftCount  = rawLeftItems?.length

package/source/S3Parallel/S3P.caf

@@ -1,4 +1,5 @@
-import &path, &ArtStandardLib, &ArtClassSystem, &Lib, {} &colors
+import &path, &ArtStandardLib, &ArtClassSystem, &Lib
+colors = terminalColors
 
 S3C = &S3Comprehensions
 
@@ -304,7 +305,7 @@ class S3P
   @_copyWrapper: (options, eachFunction) =>
     options = S3C.normalizeOptions options
     options extract
-      s3 = &Lib/S3
+      s3
       toKey
       stats
       pretend
@@ -316,6 +317,11 @@ class S3P
       largeCopyPwp
       largeCopyThreshold
 
+      # s3 options
+      region
+      useAccelerateEndpoint
+    s3 ?= new &Lib/S3 {} region, useAccelerateEndpoint
+
     stats ?= {}
     stats.copiedBytes = 0
     stats.copiedBytesPerSecond = 0

package/source/S3Parallel/S3PCli.caf

@@ -1,5 +1,15 @@
 import &StandardImport, &Lib
 
+standardFromArgs =
+  description:  "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix"
+  argument:     "[bucket/prefix]"
+  argsToOptions: processArgsIntoOptions
+
+standardFromToArgs =
+  description:  "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix"
+  argument:     "[from [to]]"
+  argsToOptions: processArgsIntoOptions
+
 allCommandOptions =
   quiet:                      "" no output
   verbose:                    "" extra output
@@ -12,6 +22,7 @@ allCommandOptions =
   start-after:  :key          "" Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used.
   stop-at:      :key          "" Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used.
   fetch-owner:  "" If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.
+  region:       "" The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.
 
   pattern:
     "" string OR js:/^any-javascript-regexp/i
@@ -70,7 +81,7 @@ writeOptions =
       OUTPOSTS
 
 toBucketOptions =
-  to-bucket:    argument: :bucket-name  required: true description:     "" The target bucket. It can be the same bucket.
+  to-bucket:    argument: :bucket-name description:     "" The target bucket. It can be the same bucket.
   to-prefix:    :key-prefix         "" If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix.
   add-prefix:   :key-prefix         "" The source key is prepended with this string for the target bucket.
   to-key:       '"js:(key) => key"' "" Provide an arbitrary JavaScript function for re-keying keys.
@@ -106,12 +117,14 @@ cliCommands =
     options:      merge allCommandOptions, advancedOptionsForAll,
       long: "" output item size and date as well as key
       raw:  "" output one S3 item per line as JSON with all properties return from listObjects
+    args: standardFromArgs
 
   summarize:
     description:  "" Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.
     options:      merge allCommandOptions,
                   summarize-folders: "" show count and size of each folder
                   advancedOptionsForAll
+    args: standardFromArgs
     examples:
       bucket: :my-bucket
       "" get a detailed summary of item counts and sizes in my-bucket
@@ -122,6 +135,7 @@ cliCommands =
   compare:
     description:  "" Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.
     options:      merge allCommandOptions, toBucketOptions, advancedOptionsForAll
+    args:         standardFromToArgs
     examples:
       bucket: :my-bucket to-bucket: :my-to-bucket
       "" Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes.
@@ -133,6 +147,7 @@ cliCommands =
 
         NOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.
     options:      merge allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions
+    args:         standardFromToArgs
     examples:
       bucket: :my-bucket to-bucket: :my-to-bucket
       "" Copy everything from my-bucket to my-to-bucket
@@ -165,6 +180,7 @@ cliCommands =
     description:  "" Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.
     options:      merge allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy,
                   overwrite: "" If set, sync will overwrite existing files with different file sizes.
+    args:         standardFromToArgs
 
     examples:
       bucket: :my-bucket to-bucket: :my-to-bucket
@@ -175,6 +191,7 @@ cliCommands =
     options:      merge allCommandOptions, advancedOptionsForAll,
                   map:        :function "" This gets called for each item found. A javascript function of the form (item) => ...
                   map-list:   :function "" This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ...
+    args: standardFromArgs
     examples: []
       bucket: :my-bucket
       map:    "" "js:(item) => console.log(item)"
@@ -204,6 +221,7 @@ cliCommands =
         ""
           If present, this function will be applied to produce the final result after the last call to reduce.
           Form: (finalReduceResult) -> ...
+    args: standardFromArgs
 
     examples:
       bucket: :my-bucket