s3p 3.5.4 → 3.6.0

package/CHANGELOG.md

@@ -1,3 +1,84 @@
+# 3.6.0 - 2024-08-14
+- NEW: can specify from and to without -- args; NEW: can specify --region
+
+# 3.5.4
+- patch/deps: deps version bump, in particular, AWS-SDKv3 version bump
+
+# 3.5.3
+- patch/fix: FIX: copying "directories" from S3 to-local-folder: they are just keys that in "/" and have size-0;
+
+# patch/tooling: fix CI
+- patch/doc: updated README
+- patch/refactor: no longer checkin in build files
+
+# 3.5.1
+- patch/improvement: now "NoSuchKey: The specified key does not exist" are logged but are not fatal
+
+# 3.5.0
+- patch/tooling: initial conversion to AWS SDK v3 - ListObjectsV2 is almost 2x faster!
+
+# 3.4.10
+- patch/tooling: updated external deps
+
+# 3.4.9
+- patch/tooling: a couple other files not needed in npm
+
+# 3.4.8
+- patch/tooling: don't build tests into npm
+
+# 3.4.7
+- patch/fix: fixed edge-case for getBisectKey
+
+# 3.4.6
+- no longer need to shell out for names with spaces - duh
+- added encodeURIComponent to CopySource as per the API doc
+
+# 3.4.5
+- corrected example in readme
+
+# 3.4.4
+- npm audit fix
+- update node to v16 for testing
+
+# 3.4.3
+- patch/fix: fix issue #67 - --pattern wasn't actually working
+
+# 3.4.2
+- patch/fix: fix issue #73 - --to-prefix "" was being ignored
+- patch/deps: updated deps
+- patch/deps: updated deps
+- patch/test: fixed tests
+- patch/doc: added a little dock around getBisectKey
+
+# 3.4.1
+- patch/doc: added filter by owner examples
+
+# 3.4.0
+- patch/improvement: made --fetch-owner optional as it incurrs at 10% overhead in bucket-listing
+
+# 3.3.4
+- patch/fix: fixed issue #54
+- set FetchOwner true in s3.listObjectsV2
+
+# 3.3.3
+- patch/deps: updated deps
+- patch/doc: readme
+- patch/doc: improved examples in changelog
+- patch/doc: fixed readme link
+- patch/tooling: build
+
+# 3.3.1
+- patch/doc: improved CLI doc
+
+# 3.3.0
+- minor/feature: updated deps - art-suite/cli now does exit(1) on errors
+
+# 3.2.2
+- patch/deps: updated deps
+
+# 3.2.1
+- patch/fix: tested (manually) and fixed the new common-copy-comands; should be working
+
 # 3.2
 
 Added common copy options:

package/build/S3Parallel/Lib/LibMisc.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["path"], [global, require('../StandardImport'), {path: require('path')}], (path) => {let exec; exec = require('util').promisify(require('child_process').exec); return {createS3Url: function(bucket, folder, key) {return folder ? path.join(folder, key) : `s3://${Caf.toString(bucket)}/${Caf.toString(key)}`;}, shellExec: function(command) {return exec(command).then(({stdout}) => stdout);}, humanByteSize: function(bytes, decimals = 2) {return (() => {switch (false) {case !(bytes < 1024): return `${Caf.toString(bytes / Caf.pow(1024, 0) | 0)}_B`; case !(bytes < Caf.pow(1024, 2)): return `${Caf.toString((bytes / Caf.pow(1024, 1)).toFixed(decimals))}kB`; case !(bytes < Caf.pow(1024, 3)): return `${Caf.toString((bytes / Caf.pow(1024, 2)).toFixed(decimals))}mB`; case !(bytes < Caf.pow(1024, 4)): return `${Caf.toString((bytes / Caf.pow(1024, 3)).toFixed(decimals))}gB`; case !(bytes < Caf.pow(1024, 5)): return `${Caf.toString((bytes / Caf.pow(1024, 4)).toFixed(decimals))}tB`; case !(bytes < Caf.pow(1024, 6)): return `${Caf.toString((bytes / Caf.pow(1024, 5)).toFixed(decimals))}pB`; default: return `${Caf.toString((bytes / Caf.pow(1024, 6)).toFixed(decimals))}eB`;};})();}};});});
+Caf.defMod(module, () => {return Caf.importInvoke(["path", "isString", "log", "merge"], [global, require('../StandardImport'), {path: require('path')}], (path, isString, log, merge) => {let exec, s3UrlPattern, httpUrlPattern, regionFirstHttpUrlPattern, parseS3Url, processArgsIntoOptions; exec = require('util').promisify(require('child_process').exec); return {createS3Url: function(bucket, folder, key) {return folder ? path.join(folder, key) : `s3://${Caf.toString(bucket)}/${Caf.toString(key)}`;}, shellExec: function(command) {return exec(command).then(({stdout}) => stdout);}, humanByteSize: function(bytes, decimals = 2) {return (() => {switch (false) {case !(bytes < 1024): return `${Caf.toString(bytes / Caf.pow(1024, 0) | 0)}_B`; case !(bytes < Caf.pow(1024, 2)): return `${Caf.toString((bytes / Caf.pow(1024, 1)).toFixed(decimals))}kB`; case !(bytes < Caf.pow(1024, 3)): return `${Caf.toString((bytes / Caf.pow(1024, 2)).toFixed(decimals))}mB`; case !(bytes < Caf.pow(1024, 4)): return `${Caf.toString((bytes / Caf.pow(1024, 3)).toFixed(decimals))}gB`; case !(bytes < Caf.pow(1024, 5)): return `${Caf.toString((bytes / Caf.pow(1024, 4)).toFixed(decimals))}tB`; case !(bytes < Caf.pow(1024, 6)): return `${Caf.toString((bytes / Caf.pow(1024, 5)).toFixed(decimals))}pB`; default: return `${Caf.toString((bytes / Caf.pow(1024, 6)).toFixed(decimals))}eB`;};})();}, s3UrlPattern: s3UrlPattern = /^s3:\/\/([^\/]+)\/?(.*)$/i, httpUrlPattern: httpUrlPattern = /^https?:\/\/([^.]+)\.s3([-.]([-a-z0-9]+))?\.amazonaws\.com(\/(.+))?$/i, regionFirstHttpUrlPattern: regionFirstHttpUrlPattern = /^https?:\/\/s3[-.]([-a-z0-9]+)\.amazonaws\.com\/([^\/]+)(\/(.+))?$/i, parseS3Url: parseS3Url = function(input) {let match, _, bucket, prefix, region, folder; return isString(input) ? (input = input.trim(), log({parseS3Url: {input}}), (() => {switch (false) {case !(match = input.match(s3UrlPattern)): return ([_, bucket, prefix] = match, [_, bucket, prefix]); case !(match = input.match(httpUrlPattern)): return ([_, bucket, _, region, _, prefix] = match, [_, bucket, _, region, _, prefix]); case !(match = input.match(regionFirstHttpUrlPattern)): return ([_, region, bucket, _, prefix] = match, [_, region, bucket, _, prefix]); default: return folder = input;};})(), {bucket, prefix, region, folder}) : {};}, processArgsIntoOptions: processArgsIntoOptions = function(args = []) {let fromStr, toStr, bucket, prefix, region, folder, toBucket, toPrefix, toRegion, tofolder; ([fromStr, toStr] = args); if (fromStr) {({bucket, prefix, region, folder} = parseS3Url(fromStr));}; if (toStr) {({bucket: toBucket, prefix: toPrefix, region: toRegion, folder: tofolder} = parseS3Url(toStr));}; return merge({bucket, prefix, region, folder, toBucket, toPrefix, toRegion, tofolder});}};});});
 //# sourceMappingURL=LibMisc.js.map

package/build/S3Parallel/Lib/LibMisc.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/Lib/LibMisc.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,qEAAO,+BAAoB,8CAC3B,OAAO,0BAAgB,wCAEvB,qBAAa,CAAC,QAAQ,QAAQ,MAAO,OAC9B,SACD,UAAU,QAAQ,OAChB,qBACQ,wBAAU,UAE1B,mBAAW,CAAC,UAAW,OACrB,KAAK,cACC,CAAC,CAAC,YAAY,WAEtB,uBAAe,CAAC,OAAO,WAAW,IAAK,OACrC,wBAAM,MACD,SAAQ,OAAc,uBAAU,gBAAC,MAAM,KAAI,QAAI,MAC/C,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,yBAC7B,uBAAM,SAAK,QAAC,MAAM,YAAW"
+  "mappings": "AAAA;;AAAA,iGAAO,+BAAoB,iKAC3B,OAAO,0BAAgB,wCAGrB,qBAAa,CAAC,QAAQ,QAAQ,MAAO,OAC9B,SACD,UAAU,QAAQ,OAChB,qBACQ,wBAAU,UAE1B,mBAAW,CAAC,UAAW,OACrB,KAAK,cACC,CAAC,CAAC,YAAY,WAEtB,uBAAe,CAAC,OAAO,WAAW,IAAK,OACrC,wBAAM,MACD,SAAQ,OAAc,uBAAU,gBAAC,MAAM,KAAI,QAAI,MAC/C,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,gBAAW,MAC9D,SAAK,QAAG,MAAM,KAAQ,uBAAM,SAAK,QAAC,MAAM,YAAW,yBAC7B,uBAAM,SAAK,QAAC,MAAM,YAAW,yBAE1D,6BAAe,6BAIf,iCACE,yEAgBF,uDACE,uEAaF,yBAAU,QAAG,CAAC,sDAAS,OAClB,SAAS,SAAK,CACf,QAAQ,cAER,KAAI,aAAe,UAEnB,wBAAM,MACD,SAAQ,YAAY,gBAAY,OACnC,EAAC,GAAG,QAAQ,UAAU,6BAAK,MAExB,SAAQ,YAAY,kBAAc,OACrC,EAAC,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,2CAAK,MAEtC,SAAQ,YAAY,6BAAyB,OAChD,EAAC,GAAG,QAAQ,QAAQ,GAAG,UAAU,iDAE/B,OACF,SAAS,eAER,QAAQ,QAAQ,QAAQ,WACxB,MAEP,iDAAsB,QAAG,CAAC,wGACxB,EAAC,SAAS,SAAS,OACnB,IAAG,UACD,EAAC,QAAQ,QAAQ,QAAQ,UAAU,WAAW,aAChD,IAAG,QACD,UAAS,kBAAkB,kBAAkB,kBAAkB,YAAY,WAAW,kBAExF,OAAS,QAAQ,QAAQ,QAAQ,QAAQ,UAAU,UAAU,UAAU"
 }

package/build/S3Parallel/Lib/S3.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["BaseClass", "shellEscape", "S3Client", "merge", "process", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "lowerCamelCase", "currentSecond", "log", "Array", "Error", "encodeURIComponent", "path", "timeout", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "present"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, fs: require('fs'), path: require('path')}], (BaseClass, shellEscape, S3Client, merge, process, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, lowerCamelCase, currentSecond, log, Array, Error, encodeURIComponent, path, timeout, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, present) => {let escape, S3; escape = function(single) {return shellEscape([single]);}; return S3 = Caf.defClass(class S3 extends BaseClass {}, function(S3, classSuper, instanceSuper) {this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION})); this.classGetter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.listBuckets = () => this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name)); this.list = ({bucket, prefix, limit = 1000, fetchOwner, startAfter}) => {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return results.Contents;});}; this.copy = (options) => {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, toBucket, fromKey, toKey, size, toFolder, pretend, verbose, copyOptions, toLocalFile, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; toBucket = temp1.toBucket; fromKey = temp1.fromKey; toKey = temp1.toKey; size = temp1.size; toFolder = temp1.toFolder; pretend = temp1.pretend; verbose = temp1.verbose; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(toLocalFile + "ignored-file-name") : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.delete = (options) => this.s3.deleteObject({Bucket: options.bucket, Key: options.key}); this.largeCopy = (options) => {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.headObject = ({bucket, key}) => this.s3.headObject({Bucket: bucket, Key: key}); this._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this._getCommonCopyOptionsForSdk = (options) => Caf.object(this.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey); this.extractCommonCopyOptions = (options) => Caf.object(this.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]); this.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.syncObject = (options) => this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "merge", "encodeURIComponent", "path", "timeout", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "present", "S3Client", "process"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, fs: require('fs'), path: require('path')}], (BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, merge, encodeURIComponent, path, timeout, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, present, S3Client, process) => {let escape, S3; escape = function(single) {return shellEscape([single]);}; return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = []) {let region, useAccelerateEndpoint; super(...arguments); this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, ({region, useAccelerateEndpoint} = options, {region, useAccelerateEndpoint})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, toBucket, fromKey, toKey, size, toFolder, pretend, verbose, copyOptions, toLocalFile, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; toBucket = temp1.toBucket; fromKey = temp1.fromKey; toKey = temp1.toKey; size = temp1.size; toFolder = temp1.toFolder; pretend = temp1.pretend; verbose = temp1.verbose; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(toLocalFile + "ignored-file-name") : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
 //# sourceMappingURL=S3.js.map

package/build/S3Parallel/Lib/S3.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/Lib/S3.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,4eAAO,8BAAiB,qBAAS,sBAAU,gCAAwB,sCAAc,sCAAiB,mBAAK,qaACvG,SAAM,QAAG,CAAC,SAAW,oBAAe,sDAInB,uDAEf,gBAAY,IAAI,SAAS,MAAK,CAAC,QAAQ,2BAGvC,iBAAY,CACV,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,iBAE3F,iCACE,OACA,gBACA,sBACA,mBACA,mBACA,eACA,WACA,gBACA,iBAEF,2DAA0D,wDAAmC,eAAe,wBAE5G,MACE,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,wEAEzD,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,gCAC/C,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBACJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,2BACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEpH,mCAsBG,CAAC,oLACN,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,yNAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,UAAU,UAAU,SAAS,OAAO,SAAS,YAEzD,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,WACH,YAAW,UAAU,SAAQ,KAC9B,iBAAiB,cAAc,uBAE/B,sBACE,aACA,uBAEI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,+BAEZ,CAAC,YACR,qBAAgB,CACd,QAAQ,gBACR,KAAQ,gCAEA,CAAC,iIAAW,oBACtB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,8BAGD,CAAC,CAAC,QAAQ,SACrB,oBACE,QAAQ,QACR,KAAK,oCAET,QAAwB,CAAC,kGAAW,SAClC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,8CAErB,CAAC,uBACU,+EACrC,QAAQ,0GAEe,CAAC,uBACT,yDACf,QAAQ,8BAEZ,QAAoB,CAAC,0DAAW,OAC9B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,oCASrC,CAAC,YACZ,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
+  "mappings": "AAAA;;AAAA,4eAAO,8BAAiB,qBAAS,sBAAU,gCAAwB,sCAAc,sCAAiB,mBAAK,qaACvG,SAAM,QAAG,CAAC,SAAW,oBAAe,sDAInB,sBAoBF,CAAC,uEACZ,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACR,EAAC,QAAQ,yBAAyB,gZAEtC,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAEhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,6BAC9C,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,2BACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEpH,6CAsBJ,QAAM,CAAC,iLACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,yNAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,UAAU,UAAU,SAAS,OAAO,SAAS,YAEzD,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,WACH,YAAW,UAAU,SAAQ,KAC9B,iBAAiB,cAAc,uBAE/B,sBACE,aACA,uBAEI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
 }

package/build/S3Parallel/S3Comprehensions.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3, temp4; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; s3 = (undefined !== (temp3 = options.s3)) ? temp3 : require('./Lib/S3'); pwp = (undefined !== (temp4 = options.pwp)) ? temp4 : new PromiseWorkerPool(listConcurrency); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp5, temp6, temp7, temp8; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp5 = i++;}; return temp5;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp6 = i1++;}; return temp6;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp7 = i2++;}; return temp7;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp8 = i3++;}; return temp8;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp5, temp6; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp5 = i++;}; return temp5;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp6 = i1++;}; return temp6;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
 //# sourceMappingURL=S3Comprehensions.js.map

package/build/S3Parallel/S3Comprehensions.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3Comprehensions.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,wcACN,UAAU,sBAAkB,kBAC5B,sFAEkB,oTAgBR,+HAEU,6SAeb,2EACC,IAAI,kBAAkB,kBAE9B,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,eACnD,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,oBAE3D,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
+  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,geACN,UAAU,sBAAkB,kBAC5B,sFAEkB,oTAgBR,+HAEU,gZAoBZ,IAAI,kBAAkB,kBAE9B,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAAyB,yBAEtD,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,eACnD,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,oBAE3D,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
 }

package/build/S3Parallel/S3P.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["merge", "colors", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "objectWithout", "Promise", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (merge, colors, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, objectWithout, Promise, present, abs, round) => {let S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let list; return S3C.each(merge(options, {returning: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, summary; summarizeFolders = options.summarizeFolders; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: ({Size, Key}) => {let floorSize, logSize, group, groupKey, groupUnit, g, out, folder, temp, temp1, base, into, i1, from, into1, to, i2, temp2, temp3, temp4; floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp5; subFolder = from[i2]; folder = ((temp5 = folder[subFolder]) != null ? temp5 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp2 = Size >= summary.maxSize) != null ? temp2 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp3 = Size <= summary.minSize) != null ? temp3 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp4 = summary.minSize) != null ? temp4 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, temp; options = S3C.normalizeOptions(options); s3 = (undefined !== (temp = options.s3)) ? temp : require('./Lib/S3'); toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp1, temp2; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp1 = stats.keyDoesNotExistFiles) != null ? temp1 : 0)) + 1, stats.keyDoesNotExistBytes = (((temp2 = stats.keyDoesNotExistBytes) != null ? temp2 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["merge", "colors", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "objectWithout", "Promise", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (merge, colors, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, objectWithout, Promise, present, abs, round) => {let S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let list; return S3C.each(merge(options, {returning: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, summary; summarizeFolders = options.summarizeFolders; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: ({Size, Key}) => {let floorSize, logSize, group, groupKey, groupUnit, g, out, folder, temp, temp1, base, into, i1, from, into1, to, i2, temp2, temp3, temp4; floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp5; subFolder = from[i2]; folder = ((temp5 = folder[subFolder]) != null ? temp5 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp2 = Size >= summary.maxSize) != null ? temp2 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp3 = Size <= summary.minSize) != null ? temp3 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp4 = summary.minSize) != null ? temp4 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp, temp1; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp = stats.keyDoesNotExistFiles) != null ? temp : 0)) + 1, stats.keyDoesNotExistBytes = (((temp1 = stats.keyDoesNotExistBytes) != null ? temp1 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
 //# sourceMappingURL=S3P.js.map

package/build/S3Parallel/S3P.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3P.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8PAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,WAAW,WACX,SAAS,CAAC,oCAAM,eAAiB,UAAO,uLAEhC,CAAC,4CAAW,mBACtB,0BACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,gCAEX,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,CAAC,MAAM,qJACX,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,8BAEd,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,gKACvB,UAAU,qBAAqB,sCAC/B,sBACO,+WAYP,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,6BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,yDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
+  "mappings": "AAAA;;AAAA,8PAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,WAAW,WACX,SAAS,CAAC,oCAAM,eAAiB,UAAO,uLAEhC,CAAC,4CAAW,mBACtB,0BACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,gCAEX,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,CAAC,MAAM,qJACX,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,8BAEd,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,yLACvB,UAAU,qBAAqB,eAC/B,sbAgBA,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAE7B,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,4BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,uDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
 }

package/build/S3Parallel/S3PCli.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (merge, lowerCamelCase, String, JSON) => {let allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", required: true, description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"})}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder"}, advancedOptionsForAll), examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n  s3p uses the same creds as the aws-cli. Learn more:\n  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => `   ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
+Caf.defMod(module, () => {return Caf.importInvoke(["processArgsIntoOptions", "merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (processArgsIntoOptions, merge, lowerCamelCase, String, JSON) => {let standardFromArgs, standardFromToArgs, allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; standardFromArgs = {description: "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[bucket/prefix]", argsToOptions: processArgsIntoOptions}; standardFromToArgs = {description: "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[from [to]]", argsToOptions: processArgsIntoOptions}; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", region: "The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"}), args: standardFromArgs}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder"}, advancedOptionsForAll), args: standardFromArgs, examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n  s3p uses the same creds as the aws-cli. Learn more:\n  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => `   ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
 //# sourceMappingURL=S3PCli.js.map

package/build/S3Parallel/S3PCli.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3PCli.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,0GAAO,6BAAiB,gMAExB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAc,UAAU,MAAK,aAAmB,kDACxE,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,8FAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,sFAEZ,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,uCACrB,wBACd,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,gDAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAE3B,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAIN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
+  "mappings": "AAAA;;AAAA,oIAAO,6BAAiB,8PAExB,oBACE,aAAc,8LACd,UAAc,mBACd,eAAe,yBAEjB,sBACE,aAAc,wMACd,UAAc,eACd,eAAe,yBAEjB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAChB,QAAgB,4FAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAa,aAAmB,kDACxD,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,8FAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,qFACV,MAAM,mBAER,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,uCACrB,wBACd,MAAM,kBACN,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,gDAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAC3B,MAAc,oBAEd,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,MAAM,kBACN,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAGN,MAAM,kBAEN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
 }

package/build/S3Parallel/S3PCliCommands.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["merge", "console", "JSON", "formatDate", "pad", "humanByteSize"], [global, require('./StandardImport'), require('./Lib')], (merge, console, JSON, formatDate, pad, humanByteSize) => {return {version: function() {return require('../../package').version;}, ls: function(options) {let items, onItem, output, fromCli, verbose, raw, longForm; items = []; onItem = options.onItem; output = options.output; fromCli = options.fromCli; verbose = options.verbose; raw = options.raw; longForm = options.long; output != null ? output : output = (v) => console.log(v); onItem != null ? onItem : onItem = fromCli ? (item) => {let LastModified, Size, Key; LastModified = item.LastModified; Size = item.Size; Key = item.Key; return output((() => {switch (false) {case !raw: return JSON.stringify(item); case !longForm: return `${Caf.toString(formatDate(LastModified, "yyyy-mm-dd HH:MM:ss"))} ${Caf.toString(pad(humanByteSize(Size), 10, " ", true))} ${Caf.toString(Key)}`; default: return Key;};})());} : (item) => items.push(raw ? item : item.Key); return require('./S3Comprehensions').each(merge({quiet: true}, options, {mapList: (l) => {let from, into, to, i; from = l; into = from; if (from != null) {to = from.length; i = 0; while (i < to) {let item; item = from[i]; onItem(item); i++;};}; into; return null;}})).then((info) => fromCli ? verbose ? info : null : verbose ? merge(info, {items}) : items);}, listBuckets: require('./Lib/S3').listBuckets, summarize: require('./S3P').summarize, compare: require('./S3P').compare, cp: require('./S3P').copy, sync: require('./S3P').sync, each: require('./S3Comprehensions').each, map: require('./S3Comprehensions').map};});});
+Caf.defMod(module, () => {return Caf.importInvoke(["mergeInto", "processArgsIntoOptions", "merge", "console", "JSON", "formatDate", "pad", "humanByteSize"], [global, require('./StandardImport'), require('./Lib')], (mergeInto, processArgsIntoOptions, merge, console, JSON, formatDate, pad, humanByteSize) => {let wrapCommandWithArgsProcessing; wrapCommandWithArgsProcessing = function(command) {return (options, args) => {mergeInto(options, processArgsIntoOptions(args)); return command(options);};}; return {version: function() {return require('../../package').version;}, ls: wrapCommandWithArgsProcessing(function(options, args) {let items, onItem, output, fromCli, verbose, raw, longForm; items = []; onItem = options.onItem; output = options.output; fromCli = options.fromCli; verbose = options.verbose; raw = options.raw; longForm = options.long; output != null ? output : output = (v) => console.log(v); onItem != null ? onItem : onItem = fromCli ? (item) => {let LastModified, Size, Key; LastModified = item.LastModified; Size = item.Size; Key = item.Key; return output((() => {switch (false) {case !raw: return JSON.stringify(item); case !longForm: return `${Caf.toString(formatDate(LastModified, "yyyy-mm-dd HH:MM:ss"))} ${Caf.toString(pad(humanByteSize(Size), 10, " ", true))} ${Caf.toString(Key)}`; default: return Key;};})());} : (item) => items.push(raw ? item : item.Key); return require('./S3Comprehensions').each(merge({quiet: true}, options, {mapList: (l) => {let from, into, to, i; from = l; into = from; if (from != null) {to = from.length; i = 0; while (i < to) {let item; item = from[i]; onItem(item); i++;};}; into; return null;}})).then((info) => fromCli ? verbose ? info : null : verbose ? merge(info, {items}) : items);}), listBuckets: wrapCommandWithArgsProcessing(function() {return (new (require('./Lib/S3'))()).listBuckets();}), summarize: wrapCommandWithArgsProcessing(require('./S3P').summarize), compare: wrapCommandWithArgsProcessing(require('./S3P').compare), cp: wrapCommandWithArgsProcessing(require('./S3P').copy), sync: wrapCommandWithArgsProcessing(require('./S3P').sync), each: wrapCommandWithArgsProcessing(require('./S3Comprehensions').each), map: wrapCommandWithArgsProcessing(require('./S3Comprehensions').map)};});});
 //# sourceMappingURL=S3PCliCommands.js.map

package/build/S3Parallel/S3PCliCommands.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3PCliCommands.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,+HAAO,6BAAiB,8EAExB,QACE,qBAAY,2CAEZ,YAAI,CAAC,sEACH,qBACA,2IACA,0BAAM,SAAI,CAAC,MAAM,YAAY,IAC7B,0BAAM,SACD,UACD,CAAC,uCAAQ,eACP,4DACA,OACE,wBAAM,MACD,KAAU,sBAAe,OAAI,MAC7B,UAAc,uBAAK,WAAW,cAAc,wCAAyB,IAAI,cAAc,OAAO,IAAI,KAAK,uBAAQ,iBAC/G,uBAET,CAAC,SAAS,WAAc,MAAS,OAAU,kBAE/C,mCAAuB,OACrB,OAAO,OACP,UACA,SAAS,CAAC,8BAAK,OACQ,sGACnB,OAAO,6BACT,eACE,CAAC,SACF,UAAgB,UAAa,OAAU,OAClC,UAAa,MAAM,OAAS,UAAW,UAEnD,aAAc,iCACd,WAAc,4BACd,SAAc,0BACd,IAAc,uBACd,MAAc,uBACd,MAAc,oCACd,KAAc"
+  "mappings": "AAAA;;AAAA,sKAAO,6BAAiB,oJAExB,gCAA6B,QAAG,CAAC,UAAW,OAC1C,CAAC,SAAS,UACR,UAAU,SAAS,uBAAuB,eAC1C,QAAQ,sBAGV,qBAAY,2CAEZ,IAAI,8BAA6B,QAAC,CAAC,SAAS,mEAC1C,qBACA,2IACA,0BAAM,SAAI,CAAC,MAAM,YAAY,IAC7B,0BAAM,SACD,UACD,CAAC,uCAAQ,eACP,4DACA,OACE,wBAAM,MACD,KAAU,sBAAe,OAAI,MAC7B,UAAc,uBAAK,WAAW,cAAc,wCAAyB,IAAI,cAAc,OAAO,IAAI,KAAK,uBAAQ,iBAC/G,uBAET,CAAC,SAAS,WAAc,MAAS,OAAU,kBAE/C,mCAAuB,OACrB,OAAO,OACP,UACA,SAAS,CAAC,8BAAK,OACQ,sGACnB,OAAO,6BACT,eACE,CAAC,SACF,UAAgB,UAAa,OAAU,OAClC,UAAa,MAAM,OAAS,UAAW,WAEnD,aAAc,8BAA6B,YAAI,YAAI,2CACnD,WAAc,8BAA8B,6BAC5C,SAAc,8BAA8B,2BAC5C,IAAc,8BAA8B,wBAC5C,MAAc,8BAA8B,wBAC5C,MAAc,8BAA8B,qCAC5C,KAAc,8BAA8B"
 }

package/package.json

@@ -5,24 +5,24 @@
   },
   "bugs": "https:/github.com/generalui/s3p/issues",
   "dependencies": {
-    "@art-suite/cli": "^1.3.11",
+    "@art-suite/cli": "2.2",
     "@aws-sdk/client-s3": "^3.436.0",
     "art-class-system": "^1.11.13",
     "art-communication-status": "^1.6.0",
-    "art-standard-lib": "^1.70.12",
+    "art-standard-lib": "^1.73.3",
     "caffeine-script-runtime": "^1.14.0",
     "neptune-namespaces-runtime": "^3.2.9",
     "shell-escape": "^0.2.0"
   },
   "description": "S3P is a CLI and library that is 5x to 50x faster than aws-cli for bulk S3 operations including: summarize, compare, copy and sync.",
   "devDependencies": {
+    "@art-suite/standard-package-builder": "^1.2.6",
     "art-build-configurator": "^1.29.3",
     "art-testbench": "^3.2.2",
     "caffeine-script": "^0.74.0",
     "jest": "^29.3.1",
     "mock-fs": "^5.0.0",
-    "neptune-namespaces": "^4.2.9",
-    "npm-watch": "^0.11.0"
+    "neptune-namespaces": "^4.2.9"
   },
   "homepage": "https://github.com/generalui/s3p",
   "jest": {
@@ -30,6 +30,10 @@
       "build/**/*.js",
       "!build/test/**/*.js"
     ],
+    "coveragePathIgnorePatterns": [
+      "/index.js$",
+      "/namespace.js$"
+    ],
     "testEnvironment": "node",
     "testPathIgnorePatterns": [
       "/node_modules/",
@@ -44,14 +48,15 @@
     "url": "https://github.com/generalui/s3p.git"
   },
   "scripts": {
-    "build": "caf -m -c source -o build\nnpm run nn",
-    "build-clean": "npm run clean; npm run build",
+    "build": "rsync -av  --exclude='*.caf' ./source/ ./build/\ncaf -m -c source -o build\nnpm run nn",
+    "build-clean": "# BUILD CLEAN START\n#   - building into: build-temp\n#\nrm -rf ./__build-old__\nrsync -av  --exclude='*.caf' ./source/ ./build-temp/\ncaf -m -c source -o build-temp\nnpm run nn\n\nnn build-temp/*\n\n# BUILD CLEAN FINIALIZING\n#  - replace old build in:   build\n#  - with new build from:    build-temp\n#\nmv build __build-old__\nmv build-temp build\nrm -rf ./__build-old__",
+    "build-watch": "nodemon -e js,ts,jsx,tsx,caf,coffee,json -i \"build/**\" -x npm run build",
     "clean": "rm -rf build/*",
     "nn": "nn build/* ",
-    "test": "npm run nn\njest",
-    "watch": "npm-watch"
+    "test": "jest",
+    "test-watch": "jest --watch"
   },
-  "version": "3.5.4",
+  "version": "3.6.0",
   "watch": {
     "build": {
       "extensions": [

package/source/S3Parallel/Lib/LibMisc.caf

@@ -1,22 +1,89 @@
 import &StandardImport, {} &path
 exec = &util.promisify &child_process.exec
 
-createS3Url: (bucket, folder, key) ->
-    if folder
-      path.join folder, key
-    else
-      "" s3://#{bucket}/#{key}
-
-shellExec: (command) ->
-  exec command
-  .then ({stdout}) -> stdout
-
-humanByteSize: (bytes, decimals = 2) ->
-  switch
-  when bytes < 1024     then "" #{bytes/1024**0 | 0}_B
-  when bytes < 1024**2  then "" #{(bytes/1024**1).toFixed decimals}kB
-  when bytes < 1024**3  then "" #{(bytes/1024**2).toFixed decimals}mB
-  when bytes < 1024**4  then "" #{(bytes/1024**3).toFixed decimals}gB
-  when bytes < 1024**5  then "" #{(bytes/1024**4).toFixed decimals}tB
-  when bytes < 1024**6  then "" #{(bytes/1024**5).toFixed decimals}pB
-  else                       "" #{(bytes/1024**6).toFixed decimals}eB
+{}
+  createS3Url: (bucket, folder, key) ->
+      if folder
+        path.join folder, key
+      else
+        "" s3://#{bucket}/#{key}
+
+  shellExec: (command) ->
+    exec command
+    .then ({stdout}) -> stdout
+
+  humanByteSize: (bytes, decimals = 2) ->
+    switch
+    when bytes < 1024     then "" #{bytes/1024**0 | 0}_B
+    when bytes < 1024**2  then "" #{(bytes/1024**1).toFixed decimals}kB
+    when bytes < 1024**3  then "" #{(bytes/1024**2).toFixed decimals}mB
+    when bytes < 1024**4  then "" #{(bytes/1024**3).toFixed decimals}gB
+    when bytes < 1024**5  then "" #{(bytes/1024**4).toFixed decimals}tB
+    when bytes < 1024**6  then "" #{(bytes/1024**5).toFixed decimals}pB
+    else                       "" #{(bytes/1024**6).toFixed decimals}eB
+
+  s3UrlPattern = ///i ^s3://([^\/]+)\/?(.*)$
+
+  # https://mybucket.s3.amazonaws.com/prefix
+  # https://mybucket.s3-us-west-2.amazonaws.com/prefix
+  httpUrlPattern =
+    ///i
+      ^
+      https?://
+
+      ([^.]+) # bucket
+
+      \.s3
+
+      ([-.]([-a-z0-9]+))? # region
+
+      \.amazonaws\.com
+
+      (\/(.+))? # prefix
+      $
+
+  # https://s3-us-west-2.amazonaws.com/mybucket/prefix
+  regionFirstHttpUrlPattern  =
+    ///i
+      ^
+      https?://s3[-.]
+
+      ([-a-z0-9]+) # region
+
+      \.amazonaws\.com
+
+      \/([^/]+) # bucket
+
+      (\/(.+))? # prefix
+      $
+
+  parseS3Url = (input) ->
+    if isString input
+      input = input.trim()
+
+      log parseS3Url: {} input
+
+      switch
+      when match = input.match s3UrlPattern
+        [_, bucket, prefix] = match
+
+      when match = input.match httpUrlPattern
+        [_, bucket, _, region, _, prefix] = match
+
+      when match = input.match regionFirstHttpUrlPattern
+        [_, region, bucket, _, prefix] = match
+
+      else
+        folder = input
+
+      {} bucket, prefix, region, folder
+    else {}
+
+  processArgsIntoOptions = (args = []) ->
+    [fromStr, toStr] = args
+    if fromStr
+      {bucket, prefix, region, folder} = parseS3Url fromStr
+    if toStr
+      {bucket: toBucket, prefix: toPrefix, region: toRegion, folder: tofolder} = parseS3Url toStr
+
+    merge {} bucket, prefix, region, folder, toBucket, toPrefix, toRegion, tofolder

package/source/S3Parallel/Lib/S3.caf

@@ -5,18 +5,6 @@ escape = (single) -> shellEscape [] single
 
 class S3 extends BaseClass
 
-  @awsSdkS3 = new S3Client merge region: process.env.AWS_REGION # useAccelerateEndpoint: true
-  # https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html
-
-  @classGetter
-    s3: ->
-      listBuckets:    (options = {}) -> Promise.resolve @awsSdkS3.send new ListBucketsCommand options
-      listObjectsV2:  (options = {}) -> Promise.resolve @awsSdkS3.send new ListObjectsV2Command options
-      getObject:      (options = {}) -> Promise.resolve @awsSdkS3.send new GetObjectCommand options
-      copyObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new CopyObjectCommand options
-      deleteObject:   (options = {}) -> Promise.resolve @awsSdkS3.send new DeleteObjectCommand options
-      headObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new HeadObjectCommand options
-
   @commonCopyOptionsSdkKeys =
     :ACL
     :CacheControl
@@ -30,12 +18,32 @@ class S3 extends BaseClass
 
   @commonSdkCopyOptionsByLowerCamelCaseKeys = object k from @commonCopyOptionsSdkKeys with-key lowerCamelCase k
 
-  @listBuckets: =>
+  ## constructor
+    IN: options:
+      region: string, default: process.env.AWS_REGION
+      useAccelerateEndpoint: boolean, default: false
+
+  constructor: (options = []) ->
+    @awsSdkS3 = new S3Client merge
+      region: process.env.AWS_REGION
+      {region, useAccelerateEndpoint} = options
+
+  @getter
+    s3: ->
+      listBuckets:    (options = {}) -> Promise.resolve @awsSdkS3.send new ListBucketsCommand options
+      listObjectsV2:  (options = {}) -> Promise.resolve @awsSdkS3.send new ListObjectsV2Command options
+      getObject:      (options = {}) -> Promise.resolve @awsSdkS3.send new GetObjectCommand options
+      copyObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new CopyObjectCommand options
+      deleteObject:   (options = {}) -> Promise.resolve @awsSdkS3.send new DeleteObjectCommand options
+      headObject:     (options = {}) -> Promise.resolve @awsSdkS3.send new HeadObjectCommand options
+
+
+  listBuckets: ->
     @s3.listBuckets()
     .then ({Buckets}) ->
       object {Name, CreationDate} from Buckets with-key Name with CreationDate
 
-  @list: ({bucket, prefix, limit=1000, fetchOwner, startAfter}) =>
+  list: ({bucket, prefix, limit=1000, fetchOwner, startAfter}) ->
     startTime = currentSecond()
     @s3.listObjectsV2
       Bucket:     bucket
@@ -43,6 +51,7 @@ class S3 extends BaseClass
       MaxKeys:    limit
       StartAfter: startAfter
       FetchOwner: fetchOwner
+
     .tapCatch (error) ->
       log.error S3.list-error: {} bucket, prefix, startAfter, limit, error
 
@@ -78,7 +87,7 @@ class S3 extends BaseClass
     OUT:
       Promise.then -> see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#copyObject-property > Callback > Parameters
         {} CopyObjectResult, ...
-  @copy: (options) =>
+  copy: (options) ->
     if options extract largeCopyThreshold, scratchState
       copyScratchState = scratchState.copyScratchState ?= {}
 
@@ -135,12 +144,12 @@ class S3 extends BaseClass
       else
         @s3.copyObject copyOptions
 
-  @delete: (options) =>
+  delete: (options) ->
     @s3.deleteObject
       Bucket: options.bucket
       Key:    options.key
 
-  @largeCopy: (options) =>
+  largeCopy: (options) ->
     options extract commonCopyOptions
     @_normalizeCopyOptions(options) extract
       fromBucket,
@@ -171,12 +180,12 @@ class S3 extends BaseClass
       shellExec command
 
   # get object's metdata
-  @headObject: ({bucket, key}) =>
+  headObject: ({bucket, key}) ->
     @s3.headObject
       Bucket: bucket
       Key: key
 
-  @_normalizeCopyOptions: (options) ->
+  _normalizeCopyOptions: (options) ->
     options extract
       bucket, key
       fromBucket = bucket
@@ -192,15 +201,15 @@ class S3 extends BaseClass
       throw new Error "" Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default
     merge options, {} fromBucket, fromKey, toBucket, toKey
 
-  @_getCommonCopyOptionsForSdk: (options) =>
-    object sdkKey, lowerCamelCasedKey from @commonSdkCopyOptionsByLowerCamelCaseKeys with-key sdkKey
+  _getCommonCopyOptionsForSdk: (options) ->
+    object sdkKey, lowerCamelCasedKey from S3.commonSdkCopyOptionsByLowerCamelCaseKeys with-key sdkKey
       options[lowerCamelCasedKey]
 
-  @extractCommonCopyOptions: (options) =>
-    object v, k from @commonSdkCopyOptionsByLowerCamelCaseKeys
+  extractCommonCopyOptions: (options) ->
+    object v, k from S3.commonSdkCopyOptionsByLowerCamelCaseKeys
       options[k]
 
-  @shouldSyncObjects: (options) ->
+  shouldSyncObjects: (options) ->
     @_normalizeCopyOptions(options) extract fromBucket, toBucket, fromKey, toKey
     Promise.then ->
       if options.size < 1024**2 || !options.size
@@ -216,7 +225,7 @@ class S3 extends BaseClass
     OUT:
       copied: Bool
       status: if not copied, explains why
-  @syncObject: (options) =>
+  syncObject: (options) ->
     @shouldSyncObjects options
     .then (shouldSync) ->
       if shouldSync

package/source/S3Parallel/S3Comprehensions.caf

@@ -257,10 +257,16 @@ class S3Comprehensions
       throttle
       stats
 
+      # s3 options
+      region
+      useAccelerateEndpoint
+
       # internal use only
-      s3 = &Lib/S3
+      s3
       pwp = new PromiseWorkerPool listConcurrency # internal use only
 
+    s3 ?= new &Lib/S3 {} region, useAccelerateEndpoint: !!useAccelerateEndpoint
+
     if compare && map && toBucket
       throw new Error "cannot use both `compare` and `map` - use mapList instead"
 
@@ -303,7 +309,7 @@ class S3Comprehensions
     ## applyF
       Apply the custom map or mapList functions
       IN:
-        items:        list of S3 items in the source bucekt as returned by the AwsSdk s3 list call
+        items:        list of S3 items in the source bucket as returned by the AwsSdk s3 list call
         compareItems: (optional, if compare is set), list of items in the toBucket
         compareStartAfter
         compareStopAt

package/source/S3Parallel/S3P.caf

@@ -304,7 +304,7 @@ class S3P
   @_copyWrapper: (options, eachFunction) =>
     options = S3C.normalizeOptions options
     options extract
-      s3 = &Lib/S3
+      s3
       toKey
       stats
       pretend
@@ -316,6 +316,11 @@ class S3P
       largeCopyPwp
       largeCopyThreshold
 
+      # s3 options
+      region
+      useAccelerateEndpoint
+    s3 ?= new &Lib/S3 {} region, useAccelerateEndpoint
+
     stats ?= {}
     stats.copiedBytes = 0
     stats.copiedBytesPerSecond = 0

package/source/S3Parallel/S3PCli.caf

@@ -1,5 +1,15 @@
 import &StandardImport, &Lib
 
+standardFromArgs =
+  description:  "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix"
+  argument:     "[bucket/prefix]"
+  argsToOptions: processArgsIntoOptions
+
+standardFromToArgs =
+  description:  "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix"
+  argument:     "[from [to]]"
+  argsToOptions: processArgsIntoOptions
+
 allCommandOptions =
   quiet:                      "" no output
   verbose:                    "" extra output
@@ -12,6 +22,7 @@ allCommandOptions =
   start-after:  :key          "" Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used.
   stop-at:      :key          "" Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used.
   fetch-owner:  "" If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.
+  region:       "" The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.
 
   pattern:
     "" string OR js:/^any-javascript-regexp/i
@@ -70,7 +81,7 @@ writeOptions =
       OUTPOSTS
 
 toBucketOptions =
-  to-bucket:    argument: :bucket-name  required: true description:     "" The target bucket. It can be the same bucket.
+  to-bucket:    argument: :bucket-name description:     "" The target bucket. It can be the same bucket.
   to-prefix:    :key-prefix         "" If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix.
   add-prefix:   :key-prefix         "" The source key is prepended with this string for the target bucket.
   to-key:       '"js:(key) => key"' "" Provide an arbitrary JavaScript function for re-keying keys.
@@ -106,12 +117,14 @@ cliCommands =
     options:      merge allCommandOptions, advancedOptionsForAll,
       long: "" output item size and date as well as key
       raw:  "" output one S3 item per line as JSON with all properties return from listObjects
+    args: standardFromArgs
 
   summarize:
     description:  "" Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.
     options:      merge allCommandOptions,
                   summarize-folders: "" show count and size of each folder
                   advancedOptionsForAll
+    args: standardFromArgs
     examples:
       bucket: :my-bucket
       "" get a detailed summary of item counts and sizes in my-bucket
@@ -122,6 +135,7 @@ cliCommands =
   compare:
     description:  "" Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.
     options:      merge allCommandOptions, toBucketOptions, advancedOptionsForAll
+    args:         standardFromToArgs
     examples:
       bucket: :my-bucket to-bucket: :my-to-bucket
       "" Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes.
@@ -133,6 +147,7 @@ cliCommands =
 
         NOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.
     options:      merge allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions
+    args:         standardFromToArgs
     examples:
       bucket: :my-bucket to-bucket: :my-to-bucket
       "" Copy everything from my-bucket to my-to-bucket
@@ -165,6 +180,7 @@ cliCommands =
     description:  "" Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.
     options:      merge allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy,
                   overwrite: "" If set, sync will overwrite existing files with different file sizes.
+    args:         standardFromToArgs
 
     examples:
       bucket: :my-bucket to-bucket: :my-to-bucket
@@ -175,6 +191,7 @@ cliCommands =
     options:      merge allCommandOptions, advancedOptionsForAll,
                   map:        :function "" This gets called for each item found. A javascript function of the form (item) => ...
                   map-list:   :function "" This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ...
+    args: standardFromArgs
     examples: []
       bucket: :my-bucket
       map:    "" "js:(item) => console.log(item)"
@@ -204,6 +221,7 @@ cliCommands =
         ""
           If present, this function will be applied to produce the final result after the last call to reduce.
           Form: (finalReduceResult) -> ...
+    args: standardFromArgs
 
     examples:
       bucket: :my-bucket

package/source/S3Parallel/S3PCliCommands.caf

@@ -1,9 +1,14 @@
 import &StandardImport, &Lib
 
+wrapCommandWithArgsProcessing = (command) ->
+  (options, args) ->
+    mergeInto options, processArgsIntoOptions args
+    command options
+
 {}
   version: -> &package.version
 
-  ls: (options) ->
+  ls: wrapCommandWithArgsProcessing (options, args) ->
     items = []
     options extract onItem, output, fromCli, verbose, raw, long as longForm
     output ?= (v) -> console.log v
@@ -30,10 +35,10 @@ import &StandardImport, &Lib
       if fromCli then if verbose then info else null
       else if verbose then merge info, {} items else items
 
-  listBuckets:  &lib/S3.listBuckets
-  summarize:    &S3P.summarize
-  compare:      &S3P.compare
-  cp:           &S3P.copy
-  sync:         &S3P.sync
-  each:         &S3Comprehensions.each
-  map:          &S3Comprehensions.map
+  listBuckets:  wrapCommandWithArgsProcessing -> new &lib/S3().listBuckets()
+  summarize:    wrapCommandWithArgsProcessing &S3P.summarize
+  compare:      wrapCommandWithArgsProcessing &S3P.compare
+  cp:           wrapCommandWithArgsProcessing &S3P.copy
+  sync:         wrapCommandWithArgsProcessing &S3P.sync
+  each:         wrapCommandWithArgsProcessing &S3Comprehensions.each
+  map:          wrapCommandWithArgsProcessing &S3Comprehensions.map