s3p 3.7.0 → 3.7.2

package/LICENSE.md

@@ -1,5 +1,5 @@
 ISC License (ISC)
-Copyright 2020, GenUI
+Copyright 2026, Shane Brinkman-Davis Delamore
 
 Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
 

package/README.md

@@ -169,4 +169,4 @@ Output:
 
 S3P was originally developed in 2020 by me, Shane Delamore, while I was working at [GenUI.com](https://www.genui.com/) in conjunction with a project for [Resolution Bioscience, Inc.](http://www.resolutionbio.com/). It was open source with ResBio's permission.
 
-Since then I've been maintaining and improving S3P in my free time. Recently (in 2026) I've decided to fork it over to my own account and continue maintaining it from here. Nothing else has changed, though I'm not migrating any past issues or pull requests.
+Since then I've been maintaining and improving S3P in my free time. Recently (in 2026) I've decided to fork it over to my own account and continue maintaining it from here. As part of that, the ISC license copyright has been updated to me, Shane Brinkman-Davis Delamore. Nothing else has changed, though I'm not migrating any past issues or pull requests.

package/build/S3Parallel/Lib/S3.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["merge", "BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "present", "encodeURIComponent", "path", "timeout", "Upload", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "process", "S3Client"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), require('@aws-sdk/lib-storage'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, path: require('path')}], (merge, BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, present, encodeURIComponent, path, timeout, Upload, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, process, S3Client) => {let escape, fs, createReadStream, copyFile, S3; escape = function(single) {return shellEscape([single]);}; fs = merge(({createReadStream} = require('fs'), {createReadStream}), ({copyFile} = require('fs').promises, {copyFile})); return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = {}) {let endpoint, region, useAccelerateEndpoint, forcePathStyle, temp; super(...arguments); endpoint = ((temp = options.endpoint) != null ? temp : process.env.S3_ENDPOINT); this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, endpoint ? {endpoint, forcePathStyle: true} : undefined, ({region, useAccelerateEndpoint, forcePathStyle} = options, {region, useAccelerateEndpoint, forcePathStyle})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter, stopAt}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration, from, into, to, i, temp; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return present(stopAt) ? (from = results.Contents, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= stopAt) {into.push(item);}; temp = i++;}; return temp;})()) : undefined, into) : results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose, size, copyOptions, toLocalFile, fromFilePath, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; fromFolder = temp1.fromFolder; fromKey = temp1.fromKey; toBucket = temp1.toBucket; toFolder = temp1.toFolder; toKey = temp1.toKey; pretend = temp1.pretend; verbose = temp1.verbose; size = temp1.size; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : fromFolder ? (fromFilePath = path.join(fromFolder, fromKey), toBucket ? (new Upload({client: this.s3, params: {Bucket: toBucket, Key: toKey, Body: fs.createReadStream(fromFilePath)}})).done() : createParentDirs(toLocalFile, copyScratchState).then(() => fs.copyFile(fromFilePath, toLocalFile))) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(path.join(toLocalFile, "ignored-file-name"), copyScratchState) : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["merge", "BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "present", "encodeURIComponent", "path", "timeout", "Upload", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "process", "S3Client", "NodeHttpHandler"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), require('@aws-sdk/lib-storage'), require('@smithy/node-http-handler'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, path: require('path')}], (merge, BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, present, encodeURIComponent, path, timeout, Upload, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, process, S3Client, NodeHttpHandler) => {let escape, http, https, fs, createReadStream, copyFile, S3; escape = function(single) {return shellEscape([single]);}; http = require('http'); https = require('https'); fs = merge(({createReadStream} = require('fs'), {createReadStream}), ({copyFile} = require('fs').promises, {copyFile})); return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = {}) {let endpoint, maxSockets, region, useAccelerateEndpoint, forcePathStyle, temp; super(...arguments); endpoint = ((temp = options.endpoint) != null ? temp : process.env.S3_ENDPOINT); maxSockets = options.maxSockets; if (options.verbose) {log(`s3p: S3-client maxSockets: ${Caf.toString(maxSockets != null ? maxSockets : "SDK-default (50)")}`);}; this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, endpoint ? {endpoint, forcePathStyle: true} : undefined, maxSockets ? {requestHandler: new NodeHttpHandler({httpAgent: new (http.Agent)({maxSockets, keepAlive: true}), httpsAgent: new (https.Agent)({maxSockets, keepAlive: true})})} : undefined, ({region, useAccelerateEndpoint, forcePathStyle} = options, {region, useAccelerateEndpoint, forcePathStyle})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter, stopAt}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration, from, into, to, i, temp; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return present(stopAt) ? (from = results.Contents, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= stopAt) {into.push(item);}; temp = i++;}; return temp;})()) : undefined, into) : results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose, size, copyOptions, toLocalFile, fromFilePath, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; fromFolder = temp1.fromFolder; fromKey = temp1.fromKey; toBucket = temp1.toBucket; toFolder = temp1.toFolder; toKey = temp1.toKey; pretend = temp1.pretend; verbose = temp1.verbose; size = temp1.size; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : fromFolder ? (fromFilePath = path.join(fromFolder, fromKey), toBucket ? (new Upload({client: this.s3, params: {Bucket: toBucket, Key: toKey, Body: fs.createReadStream(fromFilePath)}})).done() : createParentDirs(toLocalFile, copyScratchState).then(() => fs.copyFile(fromFilePath, toLocalFile))) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(path.join(toLocalFile, "ignored-file-name"), copyScratchState) : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
 //# sourceMappingURL=S3.js.map

package/build/S3Parallel/Lib/S3.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/Lib/S3.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,sfAAO,8BAAiB,qBAAS,sBAAU,+BAAqB,kCAA0B,sCAAc,sCAAiB,6cACzH,SAAM,QAAG,CAAC,SAAW,oBAAe,YAEpC,KAAK,MACH,EAAC,oBAAoB,oCACrB,EAAC,YAAY,gFAIE,sBAsBF,CAAC,uGACZ,WAAW,4CAAmB,0BAE9B,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACL,WAAQ,CACN,UAAU,gBAAgB,mBAC/B,EAAC,QAAQ,uBAAuB,kBAAkB,gaAEtD,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAWhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,YAAY,yBAC1D,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,oDACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEjH,QAAQ,UAAM,QACK,8HAAsB,YAAY,iFAEtD,6CAkBN,QAAM,CAAC,2MACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,wPAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,YAAY,SACxB,UAAU,UAAU,OACpB,SAAS,YAET,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,aAAU,CAChB,eAAe,UAAU,YAAY,UAClC,WACD,KAAI,OAAM,CACR,QAAQ,SACR,SACE,QAAQ,UACR,KAAQ,OACR,MAAQ,oBAAoB,2BAKhC,iBAAiB,aAAa,uBACzB,MAAI,YAAY,cAAc,iBAE/B,WACH,YAAW,UAAU,SAAQ,KAC9B,iBACE,UAAU,aAAa,sBACvB,oBAGF,sBAAsB,aAAa,uBAC7B,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
+  "mappings": "AAAA;;AAAA,ygBAAO,8BAAiB,qBAAS,sBAAU,+BAAqB,iCAAuB,uCAA+B,sCAAc,sCAAiB,2eACrJ,SAAM,QAAG,CAAC,SAAW,oBAAe,YAEpC,OAAO,iBACP,QAAQ,kBAER,KAAK,MACH,EAAC,oBAAoB,oCACrB,EAAC,YAAY,gFAIE,sBA0BF,CAAC,mHACZ,WAAW,4CAAmB,0BAC9B,aAAa,wBAE0D,6HAEvE,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACL,WAAQ,CACN,UAAU,gBAAgB,mBAC5B,aAAU,CACX,gBAAgB,IAAI,gBAAe,CACjC,WAAY,IAAI,cAAe,YAAY,WAAW,QACtD,YAAY,IAAI,eAAe,YAAY,WAAW,uBAC1D,EAAC,QAAQ,uBAAuB,kBAAkB,gaAEtD,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAWhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,YAAY,yBAC1D,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,oDACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEjH,QAAQ,UAAM,QACK,8HAAsB,YAAY,iFAEtD,6CAkBN,QAAM,CAAC,2MACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,wPAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,YAAY,SACxB,UAAU,UAAU,OACpB,SAAS,YAET,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,aAAU,CAChB,eAAe,UAAU,YAAY,UAClC,WACD,KAAI,OAAM,CACR,QAAQ,SACR,SACE,QAAQ,UACR,KAAQ,OACR,MAAQ,oBAAoB,2BAKhC,iBAAiB,aAAa,uBACzB,MAAI,YAAY,cAAc,iBAE/B,WACH,YAAW,UAAU,SAAQ,KAC9B,iBACE,UAAU,aAAa,sBACvB,oBAGF,sBAAsB,aAAa,uBAC7B,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
 }

package/build/S3Parallel/S3Comprehensions.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, fromFolder, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, endpoint, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; fromFolder = options.fromFolder; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; endpoint = options.endpoint; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter, stopAt: middleKey})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, stopAt, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, fromFolder, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, endpoint, maxSockets, verbose, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; fromFolder = options.fromFolder; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; endpoint = options.endpoint; maxSockets = options.maxSockets; verbose = options.verbose; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); maxSockets != null ? maxSockets : maxSockets = listConcurrency; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, maxSockets, verbose, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter, stopAt: middleKey})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, stopAt, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
 //# sourceMappingURL=S3Comprehensions.js.map

package/build/S3Parallel/S3Comprehensions.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3Comprehensions.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,yQAEhD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,sfACN,UAAU,sBAAkB,kBAC5B,sFAEkB,qVAiBR,+HAEU,6aAqBZ,IAAI,kBAAkB,kBAE9B,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,yBAAyB,yBAEhE,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,QAAQ,cACvE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,QAAQ,YAAY,oBAEnE,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
+  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,yQAEhD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,2gBACN,UAAU,sBAAkB,kBAC5B,sFAEkB,qVAiBR,+HAEU,yeAuBZ,IAAI,kBAAkB,kBAE9B,kCAAU,aAAI,iBACd,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,YAAY,SAAS,yBAAyB,yBAErF,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,QAAQ,cACvE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,QAAQ,YAAY,oBAEnE,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
 }

package/build/S3Parallel/S3P.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["terminalColors", "merge", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "PromiseWorkerPool", "Promise", "objectWithout", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (terminalColors, merge, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, PromiseWorkerPool, Promise, objectWithout, present, abs, round) => {let colors, S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; colors = terminalColors; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let keys; return S3C.each(merge(options, {returnValue: keys = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = keys, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key; ({Key} = from[i]); keys.push(Key); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, groupBy, summary; summarizeFolders = options.summarizeFolders; groupBy = options.groupBy; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}, groups: groupBy && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: (item) => {let Size, Key, floorSize, logSize, group, groupKey, groupUnit, g, out, bucket, folder, temp, temp1, base, into, i1, temp2, base1, from, into1, to, i2, temp3, temp4, temp5; ({Size, Key} = item); floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (groupBy) {groupKey = groupBy(item); bucket = ((temp2 = (base1 = summary.groups)[groupKey]) != null ? temp2 : base1[groupKey] = {items: 0, size: 0}); bucket.items++; bucket.size += Size;}; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp6; subFolder = from[i2]; folder = ((temp6 = folder[subFolder]) != null ? temp6 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp3 = Size >= summary.maxSize) != null ? temp3 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp4 = Size <= summary.minSize) != null ? temp4 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp5 = summary.minSize) != null ? temp5 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.delete = (options) => {let bucket, confirmDeleteItemsFromBucket, dryrun, pretend, verbose, stats, deleteConcurrency, deletePwp, region, endpoint, useAccelerateEndpoint, s3, temp, temp1; bucket = options.bucket; confirmDeleteItemsFromBucket = options.confirmDeleteItemsFromBucket; dryrun = options.dryrun; pretend = (undefined !== (temp = options.pretend)) ? temp : dryrun; verbose = options.verbose; stats = options.stats; deleteConcurrency = (undefined !== (temp1 = options.deleteConcurrency)) ? temp1 : 500; deletePwp = options.deletePwp; region = options.region; endpoint = options.endpoint; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 = options.s3; if (!(pretend || confirmDeleteItemsFromBucket === bucket)) {throw new Error(`confirm-delete-items-from-bucket (${Caf.toString(confirmDeleteItemsFromBucket)}) must exactly match bucket (${Caf.toString(bucket)}) (hint: use --dryrun or --pretend to skip this check)`);}; deletePwp != null ? deletePwp : deletePwp = new PromiseWorkerPool(deleteConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.deletedFiles = 0; stats.deletedBytes = 0; return S3C.eachPromises(merge(options, stats, {map: ({Key, Size}) => deletePwp.queue(() => pretend ? Promise.then(() => {if (verbose) {log(`${Caf.toString(pretend ? "PRETEND " : "")}delete s3://${Caf.toString(bucket)}/${Caf.toString(Key)} # ${Caf.toString(humanByteSize(Size))}`);}; stats.deletedFiles++; return stats.deletedBytes += Size;}) : s3.delete({bucket, key: Key}).then(() => {if (verbose) {log(`delete s3://${Caf.toString(bucket)}/${Caf.toString(Key)} # ${Caf.toString(humanByteSize(Size))}`);}; stats.deletedFiles++; return stats.deletedBytes += Size;}))})).then((info) => {return {finalStats: S3P.getStatsWithHumanByteSizes(merge(info, stats))};});}; this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint, endpoint; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; endpoint = options.endpoint; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp, temp1; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp = stats.keyDoesNotExistFiles) != null ? temp : 0)) + 1, stats.keyDoesNotExistBytes = (((temp1 = stats.keyDoesNotExistBytes) != null ? temp1 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["terminalColors", "merge", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "PromiseWorkerPool", "Promise", "objectWithout", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (terminalColors, merge, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, PromiseWorkerPool, Promise, objectWithout, present, abs, round) => {let colors, S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; colors = terminalColors; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let keys; return S3C.each(merge(options, {returnValue: keys = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = keys, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key; ({Key} = from[i]); keys.push(Key); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, groupBy, summary; summarizeFolders = options.summarizeFolders; groupBy = options.groupBy; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}, groups: groupBy && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: (item) => {let Size, Key, floorSize, logSize, group, groupKey, groupUnit, g, out, bucket, folder, temp, temp1, base, into, i1, temp2, base1, from, into1, to, i2, temp3, temp4, temp5; ({Size, Key} = item); floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (groupBy) {groupKey = groupBy(item); bucket = ((temp2 = (base1 = summary.groups)[groupKey]) != null ? temp2 : base1[groupKey] = {items: 0, size: 0}); bucket.items++; bucket.size += Size;}; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp6; subFolder = from[i2]; folder = ((temp6 = folder[subFolder]) != null ? temp6 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp3 = Size >= summary.maxSize) != null ? temp3 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp4 = Size <= summary.minSize) != null ? temp4 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp5 = summary.minSize) != null ? temp5 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.delete = (options) => {let bucket, confirmDeleteItemsFromBucket, dryrun, pretend, verbose, stats, deleteConcurrency, deletePwp, region, endpoint, useAccelerateEndpoint, maxSockets, listConcurrency, s3, temp, temp1, temp2; bucket = options.bucket; confirmDeleteItemsFromBucket = options.confirmDeleteItemsFromBucket; dryrun = options.dryrun; pretend = (undefined !== (temp = options.pretend)) ? temp : dryrun; verbose = options.verbose; stats = options.stats; deleteConcurrency = (undefined !== (temp1 = options.deleteConcurrency)) ? temp1 : 500; deletePwp = options.deletePwp; region = options.region; endpoint = options.endpoint; useAccelerateEndpoint = options.useAccelerateEndpoint; maxSockets = options.maxSockets; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; s3 = options.s3; if (!(pretend || confirmDeleteItemsFromBucket === bucket)) {throw new Error(`confirm-delete-items-from-bucket (${Caf.toString(confirmDeleteItemsFromBucket)}) must exactly match bucket (${Caf.toString(bucket)}) (hint: use --dryrun or --pretend to skip this check)`);}; deletePwp != null ? deletePwp : deletePwp = new PromiseWorkerPool(deleteConcurrency); maxSockets != null ? maxSockets : maxSockets = deleteConcurrency + listConcurrency; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint, maxSockets, verbose}); stats != null ? stats : stats = {}; stats.deletedFiles = 0; stats.deletedBytes = 0; return S3C.eachPromises(merge(options, stats, {map: ({Key, Size}) => deletePwp.queue(() => pretend ? Promise.then(() => {if (verbose) {log(`${Caf.toString(pretend ? "PRETEND " : "")}delete s3://${Caf.toString(bucket)}/${Caf.toString(Key)} # ${Caf.toString(humanByteSize(Size))}`);}; stats.deletedFiles++; return stats.deletedBytes += Size;}) : s3.delete({bucket, key: Key}).then(() => {if (verbose) {log(`delete s3://${Caf.toString(bucket)}/${Caf.toString(Key)} # ${Caf.toString(humanByteSize(Size))}`);}; stats.deletedFiles++; return stats.deletedBytes += Size;}))})).then((info) => {return {finalStats: S3P.getStatsWithHumanByteSizes(merge(info, stats))};});}; this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint, endpoint, maxSockets, listConcurrency, temp; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; endpoint = options.endpoint; maxSockets = options.maxSockets; listConcurrency = (undefined !== (temp = options.listConcurrency)) ? temp : 100; maxSockets != null ? maxSockets : maxSockets = copyConcurrency + listConcurrency; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint, maxSockets, verbose}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp1, temp2; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp1 = stats.keyDoesNotExistFiles) != null ? temp1 : 0)) + 1, stats.keyDoesNotExistBytes = (((temp2 = stats.keyDoesNotExistBytes) != null ? temp2 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
 //# sourceMappingURL=S3P.js.map

package/build/S3Parallel/S3P.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3P.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,2RAAO,iBAAO,6BAAiB,6BAAiB,4SAChD,SAAS,gBAET,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,aAAa,WACb,SAAS,CAAC,oCAAM,eAAsB,UAAO,qGAAU,UAAU,kFAEzD,CAAC,qDAAW,mBACtB,qDACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,wBACT,QAAQ,uBAEV,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,sLACJ,EAAC,MAAM,OAAO,OACd,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,UACD,WAAW,QAAQ,OACnB,4BAAS,gBAAe,6BAAS,mBAAI,OAAO,GAAG,MAAM,KACrD,gBACM,eAAS,QAEjB,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,gCAcZ,CAAC,gLAAW,SACnB,0KAIY,4IAGU,kKAKtB,KAAO,YAAW,iCAAgC,UAChD,MAAM,IAAI,wDAC4B,0EACP,oEAGjC,gCAAS,YAAI,IAAI,kBAAkB,oBACnC,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,yBAEvC,wBAAK,YACA,qBAAgB,GAChB,qBAAgB,UAErB,iBAAiB,MAAM,SACrB,QACA,KAAK,CAAC,CAAC,KAAK,UACV,gBAAe,MACV,UACD,aAAY,OAAG,IACsF,8JACnG,6BACK,sBAAiB,UAExB,WAAa,QAAQ,KAAK,WACrB,OAAG,IACwD,oHAC9D,6BACK,sBAAiB,iBAE1B,CAAC,UAAQ,QACb,YAAY,+BAA+B,MAAM,MAAM,2BAEpD,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,mMACvB,UAAU,qBAAqB,eAC/B,mdAiBA,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,yBAEvC,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,4BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,uDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
+  "mappings": "AAAA;;AAAA,2RAAO,iBAAO,6BAAiB,6BAAiB,4SAChD,SAAS,gBAET,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,aAAa,WACb,SAAS,CAAC,oCAAM,eAAsB,UAAO,qGAAU,UAAU,kFAEzD,CAAC,qDAAW,mBACtB,qDACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,wBACT,QAAQ,uBAEV,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,sLACJ,EAAC,MAAM,OAAO,OACd,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,UACD,WAAW,QAAQ,OACnB,4BAAS,gBAAe,6BAAS,mBAAI,OAAO,GAAG,MAAM,KACrD,gBACM,eAAS,QAEjB,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,gCAcZ,CAAC,oNAAW,SACnB,0KAIY,4IAGU,gQAIF,sBAGpB,KAAO,YAAW,iCAAgC,UAChD,MAAM,IAAI,wDAC4B,0EACP,oEAGjC,gCAAS,YAAI,IAAI,kBAAkB,oBACnC,kCAAU,aAAI,oBAAoB,iBAClC,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,uBAAuB,YAAY,WAE1E,wBAAK,YACA,qBAAgB,GAChB,qBAAgB,UAErB,iBAAiB,MAAM,SACrB,QACA,KAAK,CAAC,CAAC,KAAK,UACV,gBAAe,MACV,UACD,aAAY,OAAG,IACsF,8JACnG,6BACK,sBAAiB,UAExB,WAAa,QAAQ,KAAK,WACrB,OAAG,IACwD,oHAC9D,6BACK,sBAAiB,iBAE1B,CAAC,UAAQ,QACb,YAAY,+BAA+B,MAAM,MAAM,2BAEpD,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,sOACvB,UAAU,qBAAqB,eAC/B,gkBAkBoB,KACpB,kCAAU,aAAI,kBAAkB,iBAChC,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,uBAAuB,YAAY,WAE1E,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,6BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,yDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
 }

package/build/S3Parallel/S3PCli.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["processArgsIntoOptions", "merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (processArgsIntoOptions, merge, lowerCamelCase, String, JSON) => {let standardFromArgs, standardFromToArgs, allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; standardFromArgs = {description: "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[bucket/prefix]", argsToOptions: processArgsIntoOptions}; standardFromToArgs = {description: "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[from [to]]", argsToOptions: processArgsIntoOptions}; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", region: "The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"}), args: standardFromArgs}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder", "group-by": ['"js:(item) => String"', "Provide a JS function that returns a string for each item. Summarize will track {items, size} per unique string and return them as a 'groups' map. Replaces the default size-histogram grouping."]}, advancedOptionsForAll), args: standardFromArgs, examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte", {bucket: "my-bucket", "group-by": '"js:({Key}) => Key.split(\'/\')[0]"'}, "get item/size totals grouped by the first path segment"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, delete: {description: "Delete all matching files from a bucket. Uses s3.listObjectsV2 and s3.deleteObject.\n\nSAFETY: --confirm-delete-items-from-bucket MUST be provided and MUST exactly match --bucket. Without this, s3p refuses to proceed.", options: merge(allCommandOptions, advancedOptionsForAll, {"confirm-delete-items-from-bucket": ["bucket-name", "REQUIRED. Must exactly match --bucket. Guards against accidental deletion."], dryrun: "Will not modify anything. List everything that would be deleted.", pretend: "alias for --dryrun", "delete-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous delete operations."}}), args: standardFromArgs, examples: [{bucket: "my-bucket", "confirm-delete-items-from-bucket": "my-bucket", prefix: "old/"}, "Delete every key under 'old/' from my-bucket.", {bucket: "my-bucket", "confirm-delete-items-from-bucket": "my-bucket", dryrun: true}, "Dry-run: list what would be deleted but delete nothing."]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n  s3p uses the same creds as the aws-cli. Learn more:\n  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => `   ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
+Caf.defMod(module, () => {return Caf.importInvoke(["processArgsIntoOptions", "merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (processArgsIntoOptions, merge, lowerCamelCase, String, JSON) => {let standardFromArgs, standardFromToArgs, allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; standardFromArgs = {description: "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[bucket/prefix]", argsToOptions: processArgsIntoOptions}; standardFromToArgs = {description: "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[from [to]]", argsToOptions: processArgsIntoOptions}; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", region: "The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}, "max-sockets": {advanced: true, argument: "number", description: "Maximum concurrent HTTP connections to S3. Defaults to match the operation's concurrency options: list-concurrency for list-only commands, copy-concurrency + list-concurrency for copy/sync, delete-concurrency + list-concurrency for delete."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"}), args: standardFromArgs}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder", "group-by": ['"js:(item) => String"', "Provide a JS function that returns a string for each item. Summarize will track {items, size} per unique string and return them as a 'groups' map. Replaces the default size-histogram grouping."]}, advancedOptionsForAll), args: standardFromArgs, examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte", {bucket: "my-bucket", "group-by": '"js:({Key}) => Key.split(\'/\')[0]"'}, "get item/size totals grouped by the first path segment"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, delete: {description: "Delete all matching files from a bucket. Uses s3.listObjectsV2 and s3.deleteObject.\n\nSAFETY: --confirm-delete-items-from-bucket MUST be provided and MUST exactly match --bucket. Without this, s3p refuses to proceed.", options: merge(allCommandOptions, advancedOptionsForAll, {"confirm-delete-items-from-bucket": ["bucket-name", "REQUIRED. Must exactly match --bucket. Guards against accidental deletion."], dryrun: "Will not modify anything. List everything that would be deleted.", pretend: "alias for --dryrun", "delete-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous delete operations."}}), args: standardFromArgs, examples: [{bucket: "my-bucket", "confirm-delete-items-from-bucket": "my-bucket", prefix: "old/"}, "Delete every key under 'old/' from my-bucket.", {bucket: "my-bucket", "confirm-delete-items-from-bucket": "my-bucket", dryrun: true}, "Dry-run: list what would be deleted but delete nothing."]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n  s3p uses the same creds as the aws-cli. Learn more:\n  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => `   ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
 //# sourceMappingURL=S3PCli.js.map

package/build/S3Parallel/S3PCli.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3PCli.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,oIAAO,6BAAiB,8PAExB,oBACE,aAAc,8LACd,UAAc,mBACd,eAAe,yBAEjB,sBACE,aAAc,wMACd,UAAc,eACd,eAAe,yBAEjB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAChB,QAAgB,4FAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAa,aAAmB,kDACxD,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,8FAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,qFACV,MAAM,mBAER,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,sCACrB,aACE,yBACE,sMAEJ,wBACd,MAAM,kBACN,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,+CAEF,QAAQ,aAAW,YAAY,wCAC7B,4DAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAC3B,MAAc,oBAEd,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,SACE,aACK,6NAIL,SAAS,MAAM,mBAAmB,wBAChC,qCACE,eACE,+EAEJ,QAA4B,oEAC5B,SAA4B,sBAC5B,uBAA0B,UAAU,MAAK,UAAU,OAAK,aAAe,wDACzE,MAAM,kBACN,YACE,QAAQ,aAAW,oCAAkC,aAAW,QAAQ,SACtE,kDAEF,QAAQ,aAAW,oCAAkC,aAAW,QAAQ,OACtE,6DAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,MAAM,kBACN,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAGN,MAAM,kBAEN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
+  "mappings": "AAAA;;AAAA,oIAAO,6BAAiB,8PAExB,oBACE,aAAc,8LACd,UAAc,mBACd,eAAe,yBAEjB,sBACE,aAAc,wMACd,UAAc,eACd,eAAe,yBAEjB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAChB,QAAgB,4FAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAa,aAAmB,kDACxD,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,6FAC1E,gBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,qPAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,qFACV,MAAM,mBAER,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,sCACrB,aACE,yBACE,sMAEJ,wBACd,MAAM,kBACN,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,+CAEF,QAAQ,aAAW,YAAY,wCAC7B,4DAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAC3B,MAAc,oBAEd,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,SACE,aACK,6NAIL,SAAS,MAAM,mBAAmB,wBAChC,qCACE,eACE,+EAEJ,QAA4B,oEAC5B,SAA4B,sBAC5B,uBAA0B,UAAU,MAAK,UAAU,OAAK,aAAe,wDACzE,MAAM,kBACN,YACE,QAAQ,aAAW,oCAAkC,aAAW,QAAQ,SACtE,kDAEF,QAAQ,aAAW,oCAAkC,aAAW,QAAQ,OACtE,6DAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,MAAM,kBACN,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAGN,MAAM,kBAEN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
 }

package/package.json

@@ -1,5 +1,5 @@
 {
-  "author": "Shane Delamore",
+  "author": "Shane Brinkman-Davis Delamore",
   "bin": {
     "s3p": "bin/s3p"
   },
@@ -8,6 +8,7 @@
     "@art-suite/cli": "2.2",
     "@aws-sdk/client-s3": "^3.436.0",
     "@aws-sdk/lib-storage": "^3.635.0",
+    "@smithy/node-http-handler": "^4.5.2",
     "art-class-system": "^1.11.13",
     "art-communication-status": "^1.6.0",
     "art-standard-lib": "^1.73.3",
@@ -58,7 +59,7 @@
     "test": "NODE_OPTIONS=--experimental-vm-modules jest",
     "test-watch": "jest --watch"
   },
-  "version": "3.7.0",
+  "version": "3.7.2",
   "watch": {
     "build": {
       "extensions": [

package/plans/cross-region-cross-account-copy.md

@@ -0,0 +1,104 @@
+# Plan: Cross-Region & Cross-Account Copy Support
+
+Status: **proposed** (not started)
+Origin: customer report — "s3p couldn't reliably handle the cross-region, cross-account S3 transfer with the separate credentials and regional configuration required for our setup."
+
+## Problem
+
+The report is architecturally accurate. Three independent design decisions each block the scenario:
+
+1. **One `S3Client` for everything.** `S3.caf` constructs a single client (one region from `--region`/`AWS_REGION`, one optional `S3_ENDPOINT`), and `S3P._copyWrapper` uses that one instance to list the source, head-check the destination, and copy. But `ListObjectsV2` must hit the *source* bucket's regional endpoint while `CopyObject` must be sent to the *destination* bucket's region — a single-region client gets `301 PermanentRedirect` on whichever side doesn't match. `followRegionRedirects` is not set (and would cost an extra round trip per call at s3p volumes anyway).
+2. **No way to express two credential sets.** Credentials are implicit (default provider chain only — no `--profile`, no per-side options). Server-side `CopyObject` requires a *single* principal with read on source and write on destination; "separate credentials per account" cannot be expressed at all.
+3. **Large objects take a different path.** `largeCopy` shells out to `aws s3 cp`, which resolves credentials/region independently from the Node SDK with no flags passed through. Small objects succeed while large ones fail (or copy under a different identity) — the likely source of "couldn't *reliably* handle."
+
+Secondary: cross-account copies without `--acl bucket-owner-full-control` leave objects owned by the source account (unless the destination bucket has Object Ownership "bucket owner enforced").
+
+## Current-code facts (verified 2026-07)
+
+- `S3.caf:32-39` — constructor builds one `S3Client`; region from option or `process.env.AWS_REGION`; endpoint from `S3_ENDPOINT`.
+- `S3P.caf:389` (and `S3P.caf:256`, `S3Comprehensions.caf:270`) — single `S3` instance created and shared for the whole operation.
+- `S3.caf:181` — small-object copy is server-side `CopyObjectCommand`.
+- `S3.caf:188-216` — `largeCopy` shells out to `aws s3 cp`; no `--profile`/`--region` passed.
+- `LibMisc.caf:87-89` — `parseS3Url` already parses `toRegion` from destination URLs, but **nothing consumes it**.
+- `S3.caf:12-23` — `ACL` and other common copy options are already passed through to the SDK.
+- `docker-compose.yml` — minio-based integration test infra exists.
+
+### Pre-existing bug to fix first
+
+`S3.caf:152` passes `client: @s3` to `lib-storage`'s `Upload`, but `@s3` is the getter's plain command-wrapper object, not the `S3Client`. `Upload` calls `client.send(...)`, so this should be `@awsSdkS3`. Affects the existing local→S3 large-upload path; fix and add a test before building on this code.
+
+## Correct SDK v3 patterns (reference)
+
+**Cross-region, one credential set** — server-side copy works; send `CopyObjectCommand` to a client configured for the *destination* region. `CopySource` may reference a bucket in any region. Listing needs a *source*-region client, so two clients are required even in the single-credential case.
+
+**Objects > 5 GB** — `CopyObject` hard-fails at 5 GB. Use server-side multipart: `CreateMultipartUpload` → parallel `UploadPartCopyCommand` (byte-range per part, zero data through the machine) → `CompleteMultipartUpload`, all against the destination-region client. This replaces the `aws s3 cp` shell-out.
+
+**Separate credentials per side** — server-side copy is impossible (one request = one signer). Stream through the machine:
+
+```js
+const { Body, ContentType, CacheControl, Metadata } = await fromClient.send(
+  new GetObjectCommand({ Bucket: srcBucket, Key: srcKey }))
+await new Upload({
+  client: toClient,
+  params: { Bucket: destBucket, Key: destKey, Body, ContentType, CacheControl, Metadata },
+  queueSize: 4, partSize: 16 * 1024 * 1024,  // bounds memory per transfer
+}).done()
+```
+
+**Region discovery** — `HeadBucketCommand` returns `BucketRegion` in current SDK versions; on 301/403 the region is still in the `x-amz-bucket-region` response header. One probe per side at startup.
+
+**Profile credentials** — `fromIni({ profile })` from `@aws-sdk/credential-providers`.
+
+## Plan
+
+### Phase 1 — Per-side clients (plumbing)
+
+- Extend the `S3` class constructor to accept `{region, endpoint, credentials}` explicitly.
+- `S3P._copyWrapper` constructs `fromS3` and `toS3`:
+  - `fromS3`: listing (`S3Comprehensions`), `GetObject` reads.
+  - `toS3`: `CopyObject`, `headObject`/`shouldSyncObjects` comparisons, uploads.
+- When no per-side options are given, both names point at one shared instance — existing behavior and test suite unchanged.
+- Region resolution order per side: explicit flag → region parsed from the s3/https URL (wire up the already-parsed `toRegion`) → `HeadBucket` probe → `AWS_REGION`. Resolving regions correctly here fixes the 301-redirect failure mode for all users, not just cross-account ones.
+
+### Phase 2 — CLI surface
+
+- New options: `--from-profile` / `--to-profile`, `--from-region` / `--to-region`, `--from-endpoint` / `--to-endpoint`.
+- Existing `--region` / `S3_ENDPOINT` remain as defaults applied to both sides.
+- Add `@aws-sdk/credential-providers` dependency for `fromIni`.
+- Mirror all of this in the programmatic API options.
+
+### Phase 3 — Copy strategy selection
+
+Branch in `S3.copy` on whether the two sides share credentials:
+
+- **Shared credentials** (default): server-side `CopyObject` via `toS3` (today's path, now region-correct). For objects ≥ `largeCopyThreshold`: SDK multipart `UploadPartCopy` fanned out over the existing `PromiseWorkerPool`. Delete the `aws s3 cp` shell-out — removes the CLI dependency, the split credential resolution, and the small/large reliability divergence.
+- **Distinct credentials**: streamed `GetObject(fromS3)` → `Upload(toS3)`. s3p already streams S3→disk and disk→S3 in this same function; this composes the two halves. Metadata (`ContentType`, `CacheControl`, `Metadata`, etc.) must be carried explicitly — server-side copy preserves it automatically, the stream path does not.
+- Detection: explicit and dumb — any `--to-profile`/to-credential option set ⇒ stream mode. Override with `--copy-mode server|stream` for odd cases (e.g. same creds but VPC/egress reasons to force one mode).
+
+### Phase 4 — Guardrails & semantics
+
+- Stream mode moves every byte through the machine: give it its own concurrency cap (separate, much lower default than `copyConcurrency`, which is sized for cheap server-side calls) to bound memory and NAT bandwidth.
+- Cross-account server-side copies: warn (or default) `--acl bucket-owner-full-control` when the two sides' identities differ; document the Object Ownership interaction.
+- Progress/stats: stream mode can report real byte progress per part; keep stats shape identical across modes.
+
+### Phase 5 — Tests & docs
+
+- Extend the minio setup to two containers with different keys — simulates two accounts *and* two endpoints end-to-end (list from A, stream to B, verify metadata preservation, verify large-object multipart both modes).
+- Unit tests: region resolution order, strategy selection, `toRegion` URL wiring.
+- README: new options, cross-account recipe, ownership/ACL note, migration note for anyone who relied on the `aws` CLI shell-out.
+
+## Explicitly out of scope
+
+- The parallel listing engine (`S3Comprehensions` splitting logic) — untouched; this is all in the thin client layer around it.
+- Assumed-role orchestration (users can point a profile at a role via standard AWS config).
+- Anything beyond two sides (no multi-destination fan-out).
+
+## Open questions
+
+- `largeCopyThreshold` default once shell-out is gone — keep as multipart cutoff, or align to the 5 GB `CopyObject` hard limit with multipart only above it? (Multipart below 5 GB can still be faster via parallel `UploadPartCopy`.)
+- Should `--to-region` alone (same creds) also be accepted as a hint even though HeadBucket auto-detect makes it optional? (Cheap yes.)
+- Stream-mode default concurrency — needs empirical tuning; start conservative (e.g. 8–16) and expose the flag.
+
+## Effort estimate
+
+Phases 1–2 are careful threading, roughly a day or two. Phase 3 is the substantive new code but leans on `lib-storage` for the hard parts. Phases 4–5 round it out. Ballpark: a focused week including tests.

package/plans/socket-pool-sizing.md

@@ -0,0 +1,59 @@
+# Plan: HTTP Socket Pool Sizing
+
+Status: **implemented** (branch `fix/socket-pool-sizing`)
+Origin: runtime warning —
+
+```
+@smithy/node-http-handler:WARN - socket usage at capacity=50 and 450 additional requests are enqueued.
+See https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/node-configuring-maxsockets.html
+or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler config.
+```
+
+## Problem
+
+The AWS SDK v3's `NodeHttpHandler` defaults to an https agent with `maxSockets: 50`. s3p's worker pools are sized far larger — `copyConcurrency = 500`, `deleteConcurrency = 500`, `listConcurrency = 100` (`S3Comprehensions.caf:24-27`, `S3P.caf:244`) — and `S3.caf` never configures a `requestHandler`. The numbers in the warning map exactly: 50 requests on the wire, the other 450 copy workers queued inside the HTTP handler.
+
+Consequences:
+
+- **Concurrency flags are an illusion.** Effective concurrency is capped at 50 regardless of `--copy-concurrency`; 90% of the pool just relocates the queue from s3p's `PromiseWorkerPool` into the SDK.
+- **Distorted timings.** Socket-queue wait counts toward request duration, so the `S3.list-slow` check (`S3.caf:84`, duration > 60s) can fire on socket-starved requests rather than slow S3 responses. Any timeout tuning is similarly polluted.
+- **The warning's own suggestion is a trap.** Raising `socketAcquisitionWarningTimeout` silences the message without touching the bottleneck.
+
+## Fix
+
+Configure the request handler in the `S3` constructor (`S3.caf:32`):
+
+```js
+import { NodeHttpHandler } from '@smithy/node-http-handler'  // already a transitive dep of client-s3
+import { Agent } from 'https'
+
+new S3Client({
+  region,
+  requestHandler: new NodeHttpHandler({
+    httpsAgent: new Agent({ keepAlive: true, maxSockets }),
+  }),
+})
+```
+
+- `maxSockets` becomes an `S3` constructor option, threaded from the callers:
+  - copy/sync paths: default `≈ copyConcurrency + listConcurrency` (list and copy run simultaneously against the same client).
+  - delete path: default from `deleteConcurrency`.
+- Expose a CLI flag (`--max-sockets`) for override; document that it normally follows the concurrency flags automatically.
+
+## Guardrails
+
+- **File descriptors:** decided not to clamp — modern macOS/Linux defaults are far above 600 (this dev machine: 1,048,576), and Node handles hundreds of sockets trivially. Users on constrained systems can lower `--max-sockets`.
+- **S3 rate limits become reachable:** at a true 500-way COPY against one prefix, S3's ~3,500 mutating-requests/sec/prefix limit is hittable, surfacing `503 SlowDown`. The SDK's adaptive retry handles it, but note in docs that some users' current stability is an accident of the 50-socket throttle — raising it may surface retries they've never seen.
+
+## Testing
+
+- Unit: constructor passes `maxSockets` through to the agent; defaults derive from concurrency options.
+- Integration (minio): run a copy with `copyConcurrency > 50` and assert no `@smithy/node-http-handler` capacity warning is emitted; sanity-check throughput scales past the 50-socket ceiling.
+
+## Interaction with other plans
+
+Independent of, and shippable before, `cross-region-cross-account-copy.md`. If that plan lands, note that its stream mode holds sockets for the full transfer duration on both sides, so the `fromS3` and `toS3` clients need independently sized pools.
+
+## Effort estimate
+
+Small — a few hours including tests. The only real decision is default sizing vs. ulimit clamping behavior.

package/source/S3Parallel/Lib/S3.caf

@@ -1,6 +1,9 @@
-import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, &@aws-sdk/lib-storage, {} &shellEscape, &util.promisify, &path
+import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, &@aws-sdk/lib-storage, &@smithy/node-http-handler, {} &shellEscape, &util.promisify, &path
 escape = (single) -> shellEscape [] single
 
+http = &http
+https = &https
+
 fs = merge
   {createReadStream} = &fs
   {copyFile} = &fs.promises
@@ -28,14 +31,25 @@ class S3 extends BaseClass
       useAccelerateEndpoint: boolean, default: false
       endpoint: string, default: process.env.S3_ENDPOINT
       forcePathStyle: boolean, default: true if endpoint is set
+      maxSockets: number, default: undefined (SDK default: 50)
+        Max concurrent HTTP connections. Size this to match the number of
+        concurrent SDK calls or requests queue inside the HTTP handler.
+      verbose: boolean, log client configuration on creation
 
   constructor: (options = {}) ->
     endpoint = options.endpoint ? process.env.S3_ENDPOINT
+    maxSockets = options.maxSockets
+
+    log "s3p: S3-client maxSockets: #{maxSockets ? 'SDK-default (50)'}" if options.verbose
 
     @awsSdkS3 = new S3Client merge
       region: process.env.AWS_REGION
       if endpoint
         {} endpoint, forcePathStyle: true
+      if maxSockets
+        requestHandler: new NodeHttpHandler
+          httpAgent:  new http.Agent  {} maxSockets, keepAlive: true
+          httpsAgent: new https.Agent {} maxSockets, keepAlive: true
       {region, useAccelerateEndpoint, forcePathStyle} = options
 
   @getter

package/source/S3Parallel/S3Comprehensions.caf

@@ -262,12 +262,15 @@ class S3Comprehensions
       region
       useAccelerateEndpoint
       endpoint
+      maxSockets
+      verbose
 
       # internal use only
       s3
       pwp = new PromiseWorkerPool listConcurrency # internal use only
 
-    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint: !!useAccelerateEndpoint
+    maxSockets ?= listConcurrency
+    s3 ?= new &Lib/S3 {} region, endpoint, maxSockets, verbose, useAccelerateEndpoint: !!useAccelerateEndpoint
 
     if compare && map && toBucket
       throw new Error "cannot use both `compare` and `map` - use mapList instead"

package/source/S3Parallel/S3P.caf

@@ -244,6 +244,8 @@ class S3P
       deleteConcurrency = 500
       deletePwp
       region, endpoint, useAccelerateEndpoint
+      maxSockets
+      listConcurrency = 100
       s3
 
     unless pretend || confirmDeleteItemsFromBucket == bucket
@@ -253,7 +255,8 @@ class S3P
         (hint: use --dryrun or --pretend to skip this check)
 
     deletePwp ?= new PromiseWorkerPool deleteConcurrency
-    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint
+    maxSockets ?= deleteConcurrency + listConcurrency
+    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint, maxSockets, verbose
 
     stats ?= {}
     stats.deletedFiles = 0
@@ -386,7 +389,10 @@ class S3P
       region
       useAccelerateEndpoint
       endpoint
-    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint
+      maxSockets
+      listConcurrency = 100
+    maxSockets ?= copyConcurrency + listConcurrency
+    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint, maxSockets, verbose
 
     stats ?= {}
     stats.copiedBytes = 0

package/source/S3Parallel/S3PCli.caf

@@ -93,6 +93,7 @@ toFolderOptions = # to-bucket copied here to REMOVE the required, since one
 advancedOptionsForAll =
   list-concurrency:        advanced: true argument: :100     description: ""Maximum number of simultaneous list operations
   max-list-requests:       advanced: true argument: :number  description: "" Not set by default; If set, will stop when hit. Use to limit how many requests get used.
+  max-sockets:             advanced: true argument: :number  description: "" Maximum concurrent HTTP connections to S3. Defaults to match the operation's concurrency options: list-concurrency for list-only commands, copy-concurrency + list-concurrency for copy/sync, delete-concurrency + list-concurrency for delete.
 
 advancedOptionsForCopy = merge advancedOptionsForAll,
   copy-concurrency:        advanced: true argument: :500       description: "" Maximum number of simultaneous small-copies