s3p 3.6.0 → 3.7.0

package/README.md

@@ -151,7 +151,7 @@ require("s3p")
     bucket: "foo",
     quiet: true,
   })
-  .then(out => console.log(out));
+  .then((out) => console.log(out));
 ```
 
 Output:
@@ -167,6 +167,6 @@ Output:
 
 # Developed
 
-S3P was originally developed by [GenUI.com](https://www.genui.com/) in conjunction with [Resolution Bioscience, Inc.](http://www.resolutionbio.com/)
+S3P was originally developed in 2020 by me, Shane Delamore, while I was working at [GenUI.com](https://www.genui.com/) in conjunction with a project for [Resolution Bioscience, Inc.](http://www.resolutionbio.com/). It was open source with ResBio's permission.
 
-GenUI is a technology commercialization software consultancy based in Seattle. We accelerate software roadmaps. Please feel free to [contact GenUI](https://www.genui.com/contact) and tell us about your project. We'd love to hear from you.
+Since then I've been maintaining and improving S3P in my free time. Recently (in 2026) I've decided to fork it over to my own account and continue maintaining it from here. Nothing else has changed, though I'm not migrating any past issues or pull requests.

package/build/S3Parallel/Lib/LocalFileLister.js

@@ -0,0 +1,4 @@
+"use strict"
+let Caf = require('caffeine-script-runtime');
+Caf.defMod(module, () => {return Caf.importInvoke(["BaseClass", "present", "Promise", "min", "peek", "path", "fs", "glob"], [global, require('../StandardImport'), require('glob'), {path: require('path'), fs: require('fs').promises}], (BaseClass, present, Promise, min, peek, path, fs, glob) => {let LocalFileLister; return LocalFileLister = Caf.defClass(class LocalFileLister extends BaseClass {constructor({fromFolder, _allFiles, _getFileStat}) {super(...arguments); this._fromFolder = path.join(fromFolder, "aaa").replace(/aaa$/, ""); this._custom_getFileStat = _getFileStat; this._fileStats = {}; this._allFilesPromise = (_allFiles ? Promise.resolve(_allFiles) : glob(path.join(fromFolder, "**"), {nodir: true}).then(({found}) => found)).then((files) => Caf.array(files.sort(), (file) => file = file.slice(fromFolder.length + 1, file.length)));};}, function(LocalFileLister, classSuper, instanceSuper) {this.getter({files: function() {return this._allFilesPromise;}}); this.getter("fromFolder"); this.prototype.list = function({limit = 1000, startAfter, stopAt}) {return (present(startAfter) ? this._findStartIndex(startAfter) : Promise.deepAll({files: this.files, startIndex: 0})).then(({files, startIndex}) => {let out, i, tilIndex; out = []; i = startIndex; tilIndex = min(files.length, startIndex + limit); stopAt != null ? stopAt : stopAt = peek(files); while (i < tilIndex && files[i] <= stopAt) {out.push(this._getFileStat(files[i])); i++;}; return Promise.all(out);});}; this.prototype._getFileStat = function(file) {let f, filePath; return this._custom_getFileStat ? this._custom_getFileStat(file) : (f = this._fileStats[file]) ? Promise.resolve(f) : (filePath = path.join(this.fromFolder, file), fs.stat(path.join(this.fromFolder, file)).then((stats) => this._fileStats[file] = {Key: file, LastModified: stats.mtime, Size: stats.size, Owner: {DisplayName: stats.uid.toString(), ID: stats.uid.toString()}}));}; this.prototype._findStartIndex = function(startAfter) {return this.files.then((files) => {let j, k; j = 0; k = files.length; while (j < k) {let m, pivot; m = (j + k) / 2 | 0; pivot = files[m]; switch (false) {case !(startAfter === pivot): k = j = m; break; case !(startAfter < pivot): k = m; break; default: j = m + 1;};}; return {files, startIndex: j};});};});});});
+//# sourceMappingURL=LocalFileLister.js.map

package/build/S3Parallel/Lib/LocalFileLister.js.map

@@ -0,0 +1,10 @@
+{
+  "version": 3,
+  "file": "LocalFileLister.js",
+  "sourceRoot": "../../..",
+  "sources": [
+    "source/S3Parallel/Lib/LocalFileLister.caf"
+  ],
+  "names": [],
+  "mappings": "AAAA;;AAAA,qIAAO,8BAAiB,kBAAU,uBAAO,IAAI,4GAE7C,oEAA8B,sBAEf,CAAC,CAAC,YAAY,WAAW,qCACpC,mBAEE,UAAU,YAAY,eACb,QAAQ,KAEnB,2BAAuB,cAEvB,sBAEA,wBACE,CAAG,YACD,gBAAgB,aAEhB,KACE,UAAU,YAAY,QACtB,OAAO,YAEH,CAAC,CAAC,WAAW,aAEf,CAAC,oBACS,wBACZ,OAAO,WAAW,oBAAoB,GAAG,0EAEjD,aAAQ,mBAAU,kCAClB,YAAQ,qCAcR,QAAM,CAAC,CAAC,cAAY,YAAY,UAAW,OACzC,CAAG,QAAQ,cAAgB,qBAAiB,cACvC,iBAAmB,mBAAQ,YAAY,UACtC,CAAC,CAAC,OAAO,uCACb,UACA,IAAI,YACJ,WAAW,IAAI,cAAc,aAAa,QAC1C,0BAAM,SAAI,KAAK,QACf,OAAO,IAAG,YAAY,MAAM,MAAM,SAChC,SAAS,kBAAc,MAAM,MAC7B,cAEF,YAAY,yCAchB,QAAc,CAAC,wBAAQ,OAClB,2BACD,yBAAqB,QAElB,KAAI,gBAAY,SACjB,gBAAgB,KACd,CACF,WAAW,UAAU,iBAAa,OAClC,QAAQ,UAAU,iBAAa,YACzB,CAAC,UAAqB,gBAAC,QAAK,CAChC,KAAK,MACL,cAAc,aACd,MAAM,YACN,QACE,aAAa,sBACb,IAAI,6DAEd,QAAiB,CAAC,aAAc,OAC9B,gBACM,CAAC,qBACL,IAAI,GACJ,IAAI,cACJ,OAAM,IAAI,kBACR,IAAW,UAAG,IAAI,GAClB,QAAQ,MAAM,IACd,gBAAM,MACD,gBAAc,QAAW,IAAI,IAAI,UAAC,MAClC,cAAc,QAAW,IAAI,mBACJ,IAAI,IAAI,eACrC,OAAO,YAAY"
+}

package/build/S3Parallel/Lib/S3.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "merge", "encodeURIComponent", "path", "timeout", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "present", "S3Client", "process"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, fs: require('fs'), path: require('path')}], (BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, merge, encodeURIComponent, path, timeout, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, present, S3Client, process) => {let escape, S3; escape = function(single) {return shellEscape([single]);}; return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = []) {let region, useAccelerateEndpoint; super(...arguments); this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, ({region, useAccelerateEndpoint} = options, {region, useAccelerateEndpoint})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, toBucket, fromKey, toKey, size, toFolder, pretend, verbose, copyOptions, toLocalFile, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; toBucket = temp1.toBucket; fromKey = temp1.fromKey; toKey = temp1.toKey; size = temp1.size; toFolder = temp1.toFolder; pretend = temp1.pretend; verbose = temp1.verbose; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(toLocalFile + "ignored-file-name") : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["merge", "BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "present", "encodeURIComponent", "path", "timeout", "Upload", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "process", "S3Client"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), require('@aws-sdk/lib-storage'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, path: require('path')}], (merge, BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, present, encodeURIComponent, path, timeout, Upload, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, process, S3Client) => {let escape, fs, createReadStream, copyFile, S3; escape = function(single) {return shellEscape([single]);}; fs = merge(({createReadStream} = require('fs'), {createReadStream}), ({copyFile} = require('fs').promises, {copyFile})); return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = {}) {let endpoint, region, useAccelerateEndpoint, forcePathStyle, temp; super(...arguments); endpoint = ((temp = options.endpoint) != null ? temp : process.env.S3_ENDPOINT); this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, endpoint ? {endpoint, forcePathStyle: true} : undefined, ({region, useAccelerateEndpoint, forcePathStyle} = options, {region, useAccelerateEndpoint, forcePathStyle})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter, stopAt}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration, from, into, to, i, temp; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return present(stopAt) ? (from = results.Contents, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= stopAt) {into.push(item);}; temp = i++;}; return temp;})()) : undefined, into) : results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose, size, copyOptions, toLocalFile, fromFilePath, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; fromFolder = temp1.fromFolder; fromKey = temp1.fromKey; toBucket = temp1.toBucket; toFolder = temp1.toFolder; toKey = temp1.toKey; pretend = temp1.pretend; verbose = temp1.verbose; size = temp1.size; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : fromFolder ? (fromFilePath = path.join(fromFolder, fromKey), toBucket ? (new Upload({client: this.s3, params: {Bucket: toBucket, Key: toKey, Body: fs.createReadStream(fromFilePath)}})).done() : createParentDirs(toLocalFile, copyScratchState).then(() => fs.copyFile(fromFilePath, toLocalFile))) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(path.join(toLocalFile, "ignored-file-name"), copyScratchState) : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
 //# sourceMappingURL=S3.js.map

package/build/S3Parallel/Lib/S3.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/Lib/S3.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,4eAAO,8BAAiB,qBAAS,sBAAU,gCAAwB,sCAAc,sCAAiB,mBAAK,qaACvG,SAAM,QAAG,CAAC,SAAW,oBAAe,sDAInB,sBAoBF,CAAC,uEACZ,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACR,EAAC,QAAQ,yBAAyB,gZAEtC,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAEhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,6BAC9C,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,2BACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEpH,6CAsBJ,QAAM,CAAC,iLACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,yNAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,UAAU,UAAU,SAAS,OAAO,SAAS,YAEzD,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,WACH,YAAW,UAAU,SAAQ,KAC9B,iBAAiB,cAAc,uBAE/B,sBACE,aACA,uBAEI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
+  "mappings": "AAAA;;AAAA,sfAAO,8BAAiB,qBAAS,sBAAU,+BAAqB,kCAA0B,sCAAc,sCAAiB,6cACzH,SAAM,QAAG,CAAC,SAAW,oBAAe,YAEpC,KAAK,MACH,EAAC,oBAAoB,oCACrB,EAAC,YAAY,gFAIE,sBAsBF,CAAC,uGACZ,WAAW,4CAAmB,0BAE9B,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACL,WAAQ,CACN,UAAU,gBAAgB,mBAC/B,EAAC,QAAQ,uBAAuB,kBAAkB,gaAEtD,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAWhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,YAAY,yBAC1D,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,oDACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEjH,QAAQ,UAAM,QACK,8HAAsB,YAAY,iFAEtD,6CAkBN,QAAM,CAAC,2MACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,wPAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,YAAY,SACxB,UAAU,UAAU,OACpB,SAAS,YAET,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,aAAU,CAChB,eAAe,UAAU,YAAY,UAClC,WACD,KAAI,OAAM,CACR,QAAQ,SACR,SACE,QAAQ,UACR,KAAQ,OACR,MAAQ,oBAAoB,2BAKhC,iBAAiB,aAAa,uBACzB,MAAI,YAAY,cAAc,iBAE/B,WACH,YAAW,UAAU,SAAQ,KAC9B,iBACE,UAAU,aAAa,sBACvB,oBAGF,sBAAsB,aAAa,uBAC7B,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
 }

package/build/S3Parallel/Lib/index.js

@@ -6,6 +6,7 @@
 .addModules({
   FsEasy:            require('./FsEasy'),
   LibMisc:           require('./LibMisc'),
+  LocalFileLister:   require('./LocalFileLister'),
   PromiseWorkerPool: require('./PromiseWorkerPool'),
   S3:                require('./S3'),
   S3Keys:            require('./S3Keys')

package/build/S3Parallel/S3Comprehensions.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, fromFolder, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, endpoint, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; fromFolder = options.fromFolder; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; endpoint = options.endpoint; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter, stopAt: middleKey})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, stopAt, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
 //# sourceMappingURL=S3Comprehensions.js.map

package/build/S3Parallel/S3Comprehensions.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3Comprehensions.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,geACN,UAAU,sBAAkB,kBAC5B,sFAEkB,oTAgBR,+HAEU,gZAoBZ,IAAI,kBAAkB,kBAE9B,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAAyB,yBAEtD,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,eACnD,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,oBAE3D,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
+  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,yQAEhD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,sfACN,UAAU,sBAAkB,kBAC5B,sFAEkB,qVAiBR,+HAEU,6aAqBZ,IAAI,kBAAkB,kBAE9B,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,yBAAyB,yBAEhE,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,QAAQ,cACvE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,QAAQ,YAAY,oBAEnE,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
 }

package/build/S3Parallel/S3P.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["merge", "colors", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "objectWithout", "Promise", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (merge, colors, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, objectWithout, Promise, present, abs, round) => {let S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let list; return S3C.each(merge(options, {returning: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, summary; summarizeFolders = options.summarizeFolders; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: ({Size, Key}) => {let floorSize, logSize, group, groupKey, groupUnit, g, out, folder, temp, temp1, base, into, i1, from, into1, to, i2, temp2, temp3, temp4; floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp5; subFolder = from[i2]; folder = ((temp5 = folder[subFolder]) != null ? temp5 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp2 = Size >= summary.maxSize) != null ? temp2 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp3 = Size <= summary.minSize) != null ? temp3 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp4 = summary.minSize) != null ? temp4 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp, temp1; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp = stats.keyDoesNotExistFiles) != null ? temp : 0)) + 1, stats.keyDoesNotExistBytes = (((temp1 = stats.keyDoesNotExistBytes) != null ? temp1 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["terminalColors", "merge", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "PromiseWorkerPool", "Promise", "objectWithout", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (terminalColors, merge, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, PromiseWorkerPool, Promise, objectWithout, present, abs, round) => {let colors, S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; colors = terminalColors; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let keys; return S3C.each(merge(options, {returnValue: keys = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = keys, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key; ({Key} = from[i]); keys.push(Key); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, groupBy, summary; summarizeFolders = options.summarizeFolders; groupBy = options.groupBy; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}, groups: groupBy && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: (item) => {let Size, Key, floorSize, logSize, group, groupKey, groupUnit, g, out, bucket, folder, temp, temp1, base, into, i1, temp2, base1, from, into1, to, i2, temp3, temp4, temp5; ({Size, Key} = item); floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (groupBy) {groupKey = groupBy(item); bucket = ((temp2 = (base1 = summary.groups)[groupKey]) != null ? temp2 : base1[groupKey] = {items: 0, size: 0}); bucket.items++; bucket.size += Size;}; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp6; subFolder = from[i2]; folder = ((temp6 = folder[subFolder]) != null ? temp6 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp3 = Size >= summary.maxSize) != null ? temp3 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp4 = Size <= summary.minSize) != null ? temp4 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp5 = summary.minSize) != null ? temp5 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.delete = (options) => {let bucket, confirmDeleteItemsFromBucket, dryrun, pretend, verbose, stats, deleteConcurrency, deletePwp, region, endpoint, useAccelerateEndpoint, s3, temp, temp1; bucket = options.bucket; confirmDeleteItemsFromBucket = options.confirmDeleteItemsFromBucket; dryrun = options.dryrun; pretend = (undefined !== (temp = options.pretend)) ? temp : dryrun; verbose = options.verbose; stats = options.stats; deleteConcurrency = (undefined !== (temp1 = options.deleteConcurrency)) ? temp1 : 500; deletePwp = options.deletePwp; region = options.region; endpoint = options.endpoint; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 = options.s3; if (!(pretend || confirmDeleteItemsFromBucket === bucket)) {throw new Error(`confirm-delete-items-from-bucket (${Caf.toString(confirmDeleteItemsFromBucket)}) must exactly match bucket (${Caf.toString(bucket)}) (hint: use --dryrun or --pretend to skip this check)`);}; deletePwp != null ? deletePwp : deletePwp = new PromiseWorkerPool(deleteConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.deletedFiles = 0; stats.deletedBytes = 0; return S3C.eachPromises(merge(options, stats, {map: ({Key, Size}) => deletePwp.queue(() => pretend ? Promise.then(() => {if (verbose) {log(`${Caf.toString(pretend ? "PRETEND " : "")}delete s3://${Caf.toString(bucket)}/${Caf.toString(Key)} # ${Caf.toString(humanByteSize(Size))}`);}; stats.deletedFiles++; return stats.deletedBytes += Size;}) : s3.delete({bucket, key: Key}).then(() => {if (verbose) {log(`delete s3://${Caf.toString(bucket)}/${Caf.toString(Key)} # ${Caf.toString(humanByteSize(Size))}`);}; stats.deletedFiles++; return stats.deletedBytes += Size;}))})).then((info) => {return {finalStats: S3P.getStatsWithHumanByteSizes(merge(info, stats))};});}; this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint, endpoint; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; endpoint = options.endpoint; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, endpoint, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp, temp1; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp = stats.keyDoesNotExistFiles) != null ? temp : 0)) + 1, stats.keyDoesNotExistBytes = (((temp1 = stats.keyDoesNotExistBytes) != null ? temp1 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
 //# sourceMappingURL=S3P.js.map

package/build/S3Parallel/S3P.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3P.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8PAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,WAAW,WACX,SAAS,CAAC,oCAAM,eAAiB,UAAO,uLAEhC,CAAC,4CAAW,mBACtB,0BACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,gCAEX,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,CAAC,MAAM,qJACX,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,8BAEd,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,yLACvB,UAAU,qBAAqB,eAC/B,sbAgBA,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAE7B,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,4BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,uDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
+  "mappings": "AAAA;;AAAA,2RAAO,iBAAO,6BAAiB,6BAAiB,4SAChD,SAAS,gBAET,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,aAAa,WACb,SAAS,CAAC,oCAAM,eAAsB,UAAO,qGAAU,UAAU,kFAEzD,CAAC,qDAAW,mBACtB,qDACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,wBACT,QAAQ,uBAEV,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,sLACJ,EAAC,MAAM,OAAO,OACd,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,UACD,WAAW,QAAQ,OACnB,4BAAS,gBAAe,6BAAS,mBAAI,OAAO,GAAG,MAAM,KACrD,gBACM,eAAS,QAEjB,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,gCAcZ,CAAC,gLAAW,SACnB,0KAIY,4IAGU,kKAKtB,KAAO,YAAW,iCAAgC,UAChD,MAAM,IAAI,wDAC4B,0EACP,oEAGjC,gCAAS,YAAI,IAAI,kBAAkB,oBACnC,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,yBAEvC,wBAAK,YACA,qBAAgB,GAChB,qBAAgB,UAErB,iBAAiB,MAAM,SACrB,QACA,KAAK,CAAC,CAAC,KAAK,UACV,gBAAe,MACV,UACD,aAAY,OAAG,IACsF,8JACnG,6BACK,sBAAiB,UAExB,WAAa,QAAQ,KAAK,WACrB,OAAG,IACwD,oHAC9D,6BACK,sBAAiB,iBAE1B,CAAC,UAAQ,QACb,YAAY,+BAA+B,MAAM,MAAM,2BAEpD,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,mMACvB,UAAU,qBAAqB,eAC/B,mdAiBA,kBAAE,KAAI,IAAI,uBAAW,QAAQ,UAAU,yBAEvC,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,4BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,uDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
 }

package/build/S3Parallel/S3PCli.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["processArgsIntoOptions", "merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (processArgsIntoOptions, merge, lowerCamelCase, String, JSON) => {let standardFromArgs, standardFromToArgs, allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; standardFromArgs = {description: "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[bucket/prefix]", argsToOptions: processArgsIntoOptions}; standardFromToArgs = {description: "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[from [to]]", argsToOptions: processArgsIntoOptions}; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", region: "The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"}), args: standardFromArgs}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder"}, advancedOptionsForAll), args: standardFromArgs, examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n  s3p uses the same creds as the aws-cli. Learn more:\n  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => `   ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
+Caf.defMod(module, () => {return Caf.importInvoke(["processArgsIntoOptions", "merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (processArgsIntoOptions, merge, lowerCamelCase, String, JSON) => {let standardFromArgs, standardFromToArgs, allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; standardFromArgs = {description: "Specify the bucket and optionally a prefix. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[bucket/prefix]", argsToOptions: processArgsIntoOptions}; standardFromToArgs = {description: "Specify where 'from' and where 'to' do the operation. Supported forms: local/file/path, s3://bucket/prefix, https://s3.region.amazonaws.com/bucket/prefix and https://bucket.s3.amazonaws.com/prefix", argument: "[from [to]]", argsToOptions: processArgsIntoOptions}; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", region: "The AWS region of the bucket. Will use AWS_REGION environment variable if not specified.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"}), args: standardFromArgs}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder", "group-by": ['"js:(item) => String"', "Provide a JS function that returns a string for each item. Summarize will track {items, size} per unique string and return them as a 'groups' map. Replaces the default size-histogram grouping."]}, advancedOptionsForAll), args: standardFromArgs, examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte", {bucket: "my-bucket", "group-by": '"js:({Key}) => Key.split(\'/\')[0]"'}, "get item/size totals grouped by the first path segment"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), args: standardFromToArgs, examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, delete: {description: "Delete all matching files from a bucket. Uses s3.listObjectsV2 and s3.deleteObject.\n\nSAFETY: --confirm-delete-items-from-bucket MUST be provided and MUST exactly match --bucket. Without this, s3p refuses to proceed.", options: merge(allCommandOptions, advancedOptionsForAll, {"confirm-delete-items-from-bucket": ["bucket-name", "REQUIRED. Must exactly match --bucket. Guards against accidental deletion."], dryrun: "Will not modify anything. List everything that would be deleted.", pretend: "alias for --dryrun", "delete-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous delete operations."}}), args: standardFromArgs, examples: [{bucket: "my-bucket", "confirm-delete-items-from-bucket": "my-bucket", prefix: "old/"}, "Delete every key under 'old/' from my-bucket.", {bucket: "my-bucket", "confirm-delete-items-from-bucket": "my-bucket", dryrun: true}, "Dry-run: list what would be deleted but delete nothing."]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), args: standardFromArgs, examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n  s3p uses the same creds as the aws-cli. Learn more:\n  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => `   ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
 //# sourceMappingURL=S3PCli.js.map

package/build/S3Parallel/S3PCli.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3PCli.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,oIAAO,6BAAiB,8PAExB,oBACE,aAAc,8LACd,UAAc,mBACd,eAAe,yBAEjB,sBACE,aAAc,wMACd,UAAc,eACd,eAAe,yBAEjB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAChB,QAAgB,4FAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAa,aAAmB,kDACxD,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,8FAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,qFACV,MAAM,mBAER,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,uCACrB,wBACd,MAAM,kBACN,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,gDAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAC3B,MAAc,oBAEd,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,MAAM,kBACN,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAGN,MAAM,kBAEN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
+  "mappings": "AAAA;;AAAA,oIAAO,6BAAiB,8PAExB,oBACE,aAAc,8LACd,UAAc,mBACd,eAAe,yBAEjB,sBACE,aAAc,wMACd,UAAc,eACd,eAAe,yBAEjB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAChB,QAAgB,4FAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAa,aAAmB,kDACxD,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,8FAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,qFACV,MAAM,mBAER,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,sCACrB,aACE,yBACE,sMAEJ,wBACd,MAAM,kBACN,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,+CAEF,QAAQ,aAAW,YAAY,wCAC7B,4DAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,MAAc,oBACd,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAC3B,MAAc,oBAEd,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,SACE,aACK,6NAIL,SAAS,MAAM,mBAAmB,wBAChC,qCACE,eACE,+EAEJ,QAA4B,oEAC5B,SAA4B,sBAC5B,uBAA0B,UAAU,MAAK,UAAU,OAAK,aAAe,wDACzE,MAAM,kBACN,YACE,QAAQ,aAAW,oCAAkC,aAAW,QAAQ,SACtE,kDAEF,QAAQ,aAAW,oCAAkC,aAAW,QAAQ,OACtE,6DAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,MAAM,kBACN,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAGN,MAAM,kBAEN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
 }

package/build/S3Parallel/S3PCliCommands.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["mergeInto", "processArgsIntoOptions", "merge", "console", "JSON", "formatDate", "pad", "humanByteSize"], [global, require('./StandardImport'), require('./Lib')], (mergeInto, processArgsIntoOptions, merge, console, JSON, formatDate, pad, humanByteSize) => {let wrapCommandWithArgsProcessing; wrapCommandWithArgsProcessing = function(command) {return (options, args) => {mergeInto(options, processArgsIntoOptions(args)); return command(options);};}; return {version: function() {return require('../../package').version;}, ls: wrapCommandWithArgsProcessing(function(options, args) {let items, onItem, output, fromCli, verbose, raw, longForm; items = []; onItem = options.onItem; output = options.output; fromCli = options.fromCli; verbose = options.verbose; raw = options.raw; longForm = options.long; output != null ? output : output = (v) => console.log(v); onItem != null ? onItem : onItem = fromCli ? (item) => {let LastModified, Size, Key; LastModified = item.LastModified; Size = item.Size; Key = item.Key; return output((() => {switch (false) {case !raw: return JSON.stringify(item); case !longForm: return `${Caf.toString(formatDate(LastModified, "yyyy-mm-dd HH:MM:ss"))} ${Caf.toString(pad(humanByteSize(Size), 10, " ", true))} ${Caf.toString(Key)}`; default: return Key;};})());} : (item) => items.push(raw ? item : item.Key); return require('./S3Comprehensions').each(merge({quiet: true}, options, {mapList: (l) => {let from, into, to, i; from = l; into = from; if (from != null) {to = from.length; i = 0; while (i < to) {let item; item = from[i]; onItem(item); i++;};}; into; return null;}})).then((info) => fromCli ? verbose ? info : null : verbose ? merge(info, {items}) : items);}), listBuckets: wrapCommandWithArgsProcessing(function() {return (new (require('./Lib/S3'))()).listBuckets();}), summarize: wrapCommandWithArgsProcessing(require('./S3P').summarize), compare: wrapCommandWithArgsProcessing(require('./S3P').compare), cp: wrapCommandWithArgsProcessing(require('./S3P').copy), sync: wrapCommandWithArgsProcessing(require('./S3P').sync), each: wrapCommandWithArgsProcessing(require('./S3Comprehensions').each), map: wrapCommandWithArgsProcessing(require('./S3Comprehensions').map)};});});
+Caf.defMod(module, () => {return Caf.importInvoke(["mergeInto", "processArgsIntoOptions", "merge", "console", "JSON", "formatDate", "pad", "humanByteSize"], [global, require('./StandardImport'), require('./Lib')], (mergeInto, processArgsIntoOptions, merge, console, JSON, formatDate, pad, humanByteSize) => {let wrapCommandWithArgsProcessing; wrapCommandWithArgsProcessing = function(command) {return (options, args) => {mergeInto(options, processArgsIntoOptions(args)); return command(options);};}; return {version: function() {return require('../../package').version;}, ls: wrapCommandWithArgsProcessing(function(options, args) {let items, onItem, output, fromCli, verbose, raw, longForm; items = []; onItem = options.onItem; output = options.output; fromCli = options.fromCli; verbose = options.verbose; raw = options.raw; longForm = options.long; output != null ? output : output = (v) => console.log(v); onItem != null ? onItem : onItem = fromCli ? (item) => {let LastModified, Size, Key; LastModified = item.LastModified; Size = item.Size; Key = item.Key; return output((() => {switch (false) {case !raw: return JSON.stringify(item); case !longForm: return `${Caf.toString(formatDate(LastModified, "yyyy-mm-dd HH:MM:ss"))} ${Caf.toString(pad(humanByteSize(Size), 10, " ", true))} ${Caf.toString(Key)}`; default: return Key;};})());} : (item) => items.push(raw ? item : item.Key); return require('./S3Comprehensions').each(merge({quiet: true}, options, {mapList: (l) => {let from, into, to, i; from = l; into = from; if (from != null) {to = from.length; i = 0; while (i < to) {let item; item = from[i]; onItem(item); i++;};}; into; return null;}})).then((info) => fromCli ? verbose ? info : null : verbose ? merge(info, {items}) : items);}), listBuckets: wrapCommandWithArgsProcessing(function() {return (new (require('./Lib/S3'))()).listBuckets();}), summarize: wrapCommandWithArgsProcessing(require('./S3P').summarize), compare: wrapCommandWithArgsProcessing(require('./S3P').compare), cp: wrapCommandWithArgsProcessing(require('./S3P').copy), sync: wrapCommandWithArgsProcessing(require('./S3P').sync), delete: wrapCommandWithArgsProcessing(require('./S3P').delete), each: wrapCommandWithArgsProcessing(require('./S3Comprehensions').each), map: wrapCommandWithArgsProcessing(require('./S3Comprehensions').map)};});});
 //# sourceMappingURL=S3PCliCommands.js.map

package/build/S3Parallel/S3PCliCommands.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3PCliCommands.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,sKAAO,6BAAiB,oJAExB,gCAA6B,QAAG,CAAC,UAAW,OAC1C,CAAC,SAAS,UACR,UAAU,SAAS,uBAAuB,eAC1C,QAAQ,sBAGV,qBAAY,2CAEZ,IAAI,8BAA6B,QAAC,CAAC,SAAS,mEAC1C,qBACA,2IACA,0BAAM,SAAI,CAAC,MAAM,YAAY,IAC7B,0BAAM,SACD,UACD,CAAC,uCAAQ,eACP,4DACA,OACE,wBAAM,MACD,KAAU,sBAAe,OAAI,MAC7B,UAAc,uBAAK,WAAW,cAAc,wCAAyB,IAAI,cAAc,OAAO,IAAI,KAAK,uBAAQ,iBAC/G,uBAET,CAAC,SAAS,WAAc,MAAS,OAAU,kBAE/C,mCAAuB,OACrB,OAAO,OACP,UACA,SAAS,CAAC,8BAAK,OACQ,sGACnB,OAAO,6BACT,eACE,CAAC,SACF,UAAgB,UAAa,OAAU,OAClC,UAAa,MAAM,OAAS,UAAW,WAEnD,aAAc,8BAA6B,YAAI,YAAI,2CACnD,WAAc,8BAA8B,6BAC5C,SAAc,8BAA8B,2BAC5C,IAAc,8BAA8B,wBAC5C,MAAc,8BAA8B,wBAC5C,MAAc,8BAA8B,qCAC5C,KAAc,8BAA8B"
+  "mappings": "AAAA;;AAAA,sKAAO,6BAAiB,oJAExB,gCAA6B,QAAG,CAAC,UAAW,OAC1C,CAAC,SAAS,UACR,UAAU,SAAS,uBAAuB,eAC1C,QAAQ,sBAGV,qBAAY,2CAEZ,IAAI,8BAA6B,QAAC,CAAC,SAAS,mEAC1C,qBACA,2IACA,0BAAM,SAAI,CAAC,MAAM,YAAY,IAC7B,0BAAM,SACD,UACD,CAAC,uCAAQ,eACP,4DACA,OACE,wBAAM,MACD,KAAU,sBAAe,OAAI,MAC7B,UAAc,uBAAK,WAAW,cAAc,wCAAyB,IAAI,cAAc,OAAO,IAAI,KAAK,uBAAQ,iBAC/G,uBAET,CAAC,SAAS,WAAc,MAAS,OAAU,kBAE/C,mCAAuB,OACrB,OAAO,OACP,UACA,SAAS,CAAC,8BAAK,OACQ,sGACnB,OAAO,6BACT,eACE,CAAC,SACF,UAAgB,UAAa,OAAU,OAClC,UAAa,MAAM,OAAS,UAAW,WAEnD,aAAc,8BAA6B,YAAI,YAAI,2CACnD,WAAc,8BAA8B,6BAC5C,SAAc,8BAA8B,2BAC5C,IAAc,8BAA8B,wBAC5C,MAAc,8BAA8B,wBAC5C,QAAc,8BAA8B,0BAC5C,MAAc,8BAA8B,qCAC5C,KAAc,8BAA8B"
 }

package/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  s3:
+    image: minio/minio
+    volumes:
+      - minio-data:/data
+    entrypoint: sh
+    command: -c 'mkdir -p /data/s3p-test && /usr/bin/minio server --console-address=0.0.0.0:9002 /data'
+    ports:
+      - '9000:9000'
+      - '9002:9002'
+    environment:
+      MINIO_ROOT_USER: testAccessKey
+      MINIO_ROOT_PASSWORD: testSecretKey
+
+volumes:
+  minio-data:

package/package.json

@@ -1,16 +1,18 @@
 {
-  "author": "GenUI LLC",
+  "author": "Shane Delamore",
   "bin": {
     "s3p": "bin/s3p"
   },
-  "bugs": "https:/github.com/generalui/s3p/issues",
+  "bugs": "https:/github.com/shanebdavis/s3p/issues",
   "dependencies": {
     "@art-suite/cli": "2.2",
     "@aws-sdk/client-s3": "^3.436.0",
+    "@aws-sdk/lib-storage": "^3.635.0",
     "art-class-system": "^1.11.13",
     "art-communication-status": "^1.6.0",
     "art-standard-lib": "^1.73.3",
     "caffeine-script-runtime": "^1.14.0",
+    "glob": "^13.0.6",
     "neptune-namespaces-runtime": "^3.2.9",
     "shell-escape": "^0.2.0"
   },
@@ -19,7 +21,7 @@
     "@art-suite/standard-package-builder": "^1.2.6",
     "art-build-configurator": "^1.29.3",
     "art-testbench": "^3.2.2",
-    "caffeine-script": "^0.74.0",
+    "caffeine-script": "^0.75.1",
     "jest": "^29.3.1",
     "mock-fs": "^5.0.0",
     "neptune-namespaces": "^4.2.9"
@@ -45,7 +47,7 @@
   "name": "s3p",
   "repository": {
     "type": "git",
-    "url": "https://github.com/generalui/s3p.git"
+    "url": "https://github.com/shanebdavis/s3p"
   },
   "scripts": {
     "build": "rsync -av  --exclude='*.caf' ./source/ ./build/\ncaf -m -c source -o build\nnpm run nn",
@@ -53,10 +55,10 @@
     "build-watch": "nodemon -e js,ts,jsx,tsx,caf,coffee,json -i \"build/**\" -x npm run build",
     "clean": "rm -rf build/*",
     "nn": "nn build/* ",
-    "test": "jest",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest",
     "test-watch": "jest --watch"
   },
-  "version": "3.6.0",
+  "version": "3.7.0",
   "watch": {
     "build": {
       "extensions": [

package/source/S3Parallel/Lib/LocalFileLister.caf

@@ -0,0 +1,99 @@
+import &StandardImport, &glob, {} &path, fs: &fs.promises
+
+class LocalFileLister extends BaseClass
+
+  constructor: ({fromFolder, _allFiles, _getFileStat}) ->
+    @_fromFolder =
+      # ensure has trailing slash
+      path.join fromFolder, "aaa"
+      .replace /aaa$/, ''
+
+    @_custom_getFileStat = _getFileStat
+
+    @_fileStats = {}
+
+    @_allFilesPromise =
+      if _allFiles
+        Promise.resolve _allFiles
+      else
+        glob
+          path.join fromFolder, "**"
+          nodir: true
+
+        .then ({found}) -> found
+
+      .then (files) ->
+        array file in files.sort()
+          file = file.slice fromFolder.length + 1, file.length
+
+  @getter files: -> @_allFilesPromise
+  @getter :fromFolder
+
+  ## list
+    IN:
+      limit: maximum number of files to return
+      startAfter: if present, returns files strictly greater than startAfter
+      stopAt: if present, returns files up to and including stopAt
+
+    OUT:
+      []
+        Key:
+        LastModified:
+        Size
+        Owner
+  list: ({limit=1000, startAfter, stopAt}) ->
+    if present startAfter then @_findStartIndex startAfter
+    else Promise.deepAll {} @files, startIndex: 0
+    .then ({files, startIndex}) ->
+      out = []
+      i = startIndex
+      tilIndex = min files.length, startIndex + limit
+      stopAt ?= peek files
+      while i < tilIndex && files[i] <= stopAt
+        out.push @_getFileStat files[i]
+        i++
+
+      Promise.all out
+
+  #################################################
+  # PRIVATE
+  #################################################
+  ##
+    IN: file name
+    OUT: []
+      Key: file
+      LastModified: stats.mtime
+      Size: stats.size
+      Owner:
+        DisplayName: stats.uid.toString()
+        ID: stats.uid.toString()
+  _getFileStat: (file) ->
+    if @_custom_getFileStat
+      @_custom_getFileStat file
+    else
+      if f = @_fileStats[file]
+        Promise.resolve f
+      else
+        filePath = path.join @fromFolder, file
+        fs.stat path.join @fromFolder, file
+        .then (stats) -> @_fileStats[file] =
+          Key: file
+          LastModified: stats.mtime
+          Size: stats.size
+          Owner:
+            DisplayName: stats.uid.toString()
+            ID: stats.uid.toString()
+
+  _findStartIndex: (startAfter) ->
+    @files
+    .then (files) ->
+      j = 0
+      k = files.length
+      while j < k
+        m = (j + k) / 2 | 0
+        pivot = files[m]
+        switch
+        when startAfter == pivot then k = j = m
+        when startAfter <  pivot then k = m
+        else                          j = m + 1
+      {} files, startIndex: j

package/source/S3Parallel/Lib/S3.caf

@@ -1,6 +1,10 @@
-import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, {} &shellEscape, &util.promisify, &fs, &path
+import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, &@aws-sdk/lib-storage, {} &shellEscape, &util.promisify, &path
 escape = (single) -> shellEscape [] single
 
+fs = merge
+  {createReadStream} = &fs
+  {copyFile} = &fs.promises
+
 # &@aws-sdk/client-s3.config.setPromisesDependency &bluebird
 
 class S3 extends BaseClass
@@ -22,11 +26,17 @@ class S3 extends BaseClass
     IN: options:
       region: string, default: process.env.AWS_REGION
       useAccelerateEndpoint: boolean, default: false
+      endpoint: string, default: process.env.S3_ENDPOINT
+      forcePathStyle: boolean, default: true if endpoint is set
+
+  constructor: (options = {}) ->
+    endpoint = options.endpoint ? process.env.S3_ENDPOINT
 
-  constructor: (options = []) ->
     @awsSdkS3 = new S3Client merge
       region: process.env.AWS_REGION
-      {region, useAccelerateEndpoint} = options
+      if endpoint
+        {} endpoint, forcePathStyle: true
+      {region, useAccelerateEndpoint, forcePathStyle} = options
 
   @getter
     s3: ->
@@ -43,7 +53,16 @@ class S3 extends BaseClass
     .then ({Buckets}) ->
       object {Name, CreationDate} from Buckets with-key Name with CreationDate
 
-  list: ({bucket, prefix, limit=1000, fetchOwner, startAfter}) ->
+  ## list
+    OUT:
+      []
+        Key:
+        LastModified:
+        Size
+        ETag
+        StorageClass
+        Owner
+  list: ({bucket, prefix, limit=1000, fetchOwner, startAfter, stopAt}) ->
     startTime = currentSecond()
     @s3.listObjectsV2
       Bucket:     bucket
@@ -65,14 +84,12 @@ class S3 extends BaseClass
       else if duration > 60
         log.warn S3.list-slow: {} bucket, prefix, startAfter, limit, duration, results: merge results, Contents: "Array #{results.Contents.length}"
 
-      results.Contents
+      if present stopAt
+        array item in-array results.Contents when item.Key <= stopAt
+      else
+        results.Contents
 
-  ##
-    TODO:
-      For large files (>5GB), we need to just use the CLI:
-        aws s3 cp s3://resbio-fastq-drop-internal/180316_NB501104_0354_AHM2L2BGX5/Undetermined/Undetermined_S0_L001_R1_001.fastq.gz s3://genui-dynamodb-migration-scratch/large-test.gz
-        We can add --quiet (probably)
-      Further, that should use acceleration, if enabled - still need to test that.
+  ## copy
     IN:
       options:
         size:       Bytes (optional) if provided and if bigger than the max size allowed by copyObject, largeCopy is used
@@ -82,6 +99,7 @@ class S3 extends BaseClass
         key:      <String>
         fromKey:  from-key (default: key)
         toKey:    to-key (default: key)
+        fromFolder: string
 
         toFolder:   local folder
     OUT:
@@ -92,15 +110,17 @@ class S3 extends BaseClass
       copyScratchState = scratchState.copyScratchState ?= {}
 
     @_normalizeCopyOptions(options) extract
-      fromBucket, toBucket, fromKey, toKey, size
-      toFolder
-      pretend
-      verbose
+      fromBucket, fromFolder, fromKey
+      toBucket, toFolder, toKey
+      pretend, verbose
+      size
 
     if size >= largeCopyThreshold
       @largeCopy {}
         commonCopyOptions: @extractCommonCopyOptions options
-        fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose
+        fromBucket, fromFolder, fromKey
+        toBucket, toFolder, toKey
+        pretend, verbose
 
     else
       copyOptions = merge
@@ -125,14 +145,30 @@ class S3 extends BaseClass
       if pretend
         timeout 1 -> pretend: true
 
+      else if fromFolder
+        fromFilePath = path.join fromFolder, fromKey
+        if toBucket
+          new Upload
+            client: @s3
+            params:
+              Bucket: toBucket
+              Key:    toKey
+              Body:   fs.createReadStream fromFilePath
+
+          .done()
+
+        else
+          createParentDirs toLocalFile, copyScratchState
+          .then -> fs.copyFile fromFilePath, toLocalFile
+
       else if toFolder
         if /\/$/.test(toKey) && size == 0 # this is a folder
-          createParentDirs toLocalFile + "ignored-file-name"
-        else
-          createWriteStreamSafe
-            toLocalFile
+          createParentDirs
+            path.join toLocalFile, "ignored-file-name"
             copyScratchState
 
+        else
+          createWriteStreamSafe toLocalFile, copyScratchState
           .then (writeStream) -> new Promise (resolve, reject) ->
             @s3.getObject copyOptions
             .then ({Body}) ->
@@ -179,7 +215,7 @@ class S3 extends BaseClass
     else
       shellExec command
 
-  # get object's metdata
+  # get object's metadata
   headObject: ({bucket, key}) ->
     @s3.headObject
       Bucket: bucket

package/source/S3Parallel/S3Comprehensions.caf

@@ -1,4 +1,4 @@
-import &path, &ArtStandardLib, &ArtClassSystem, &Lib, {} &colors
+import &path, &ArtStandardLib, &ArtClassSystem, &Lib
 
 class S3Comprehensions
 
@@ -210,7 +210,7 @@ class S3Comprehensions
           While it returns true, it will be called once/second/parallel-branch until it returns false.
 
         COMPREHENSION
-        with: (Item) -> ignored - alais of `map`
+        with: (Item) -> ignored - alias of `map`
         when: (Item) -> T/F - if true, pass it on to `map`, else skip
         returning:    if present, return this value instead of stats
         into:         <ALIAS: returning>
@@ -234,6 +234,7 @@ class S3Comprehensions
       startAfter
       stopAt
       filter
+      fromFolder
 
       # compare-to options
       compare
@@ -243,7 +244,7 @@ class S3Comprehensions
       # limits
       limit = 1000
       maxListRequests
-      listConcurrency = 100 # max open reqeusts
+      listConcurrency = 100 # max open requests
 
       # actions
       returnValue
@@ -260,12 +261,13 @@ class S3Comprehensions
       # s3 options
       region
       useAccelerateEndpoint
+      endpoint
 
       # internal use only
       s3
       pwp = new PromiseWorkerPool listConcurrency # internal use only
 
-    s3 ?= new &Lib/S3 {} region, useAccelerateEndpoint: !!useAccelerateEndpoint
+    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint: !!useAccelerateEndpoint
 
     if compare && map && toBucket
       throw new Error "cannot use both `compare` and `map` - use mapList instead"
@@ -379,8 +381,8 @@ class S3Comprehensions
         maxOutstanding = Math.max maxOutstanding, outstanding += 2
 
         Promise.all []
-          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter
-          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter: middleKey
+          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter, stopAt: middleKey
+          pwp.queue -> s3.list {} bucket, limit, fetchOwner, stopAt, startAfter: middleKey
 
         .then ([rawLeftItems, rawRightItems]) ->
           rawLeftCount  = rawLeftItems?.length

package/source/S3Parallel/S3P.caf

@@ -1,4 +1,5 @@
-import &path, &ArtStandardLib, &ArtClassSystem, &Lib, {} &colors
+import &path, &ArtStandardLib, &ArtClassSystem, &Lib
+colors = terminalColors
 
 S3C = &S3Comprehensions
 
@@ -35,11 +36,11 @@ class S3P
   ###########################################################
   @list: (options) =>
     S3C.each merge options,
-      returning: list = []
-      mapList: (l) -> array from-array l into list
+      returnValue: keys = []
+      mapList: (l) -> each {Key} from-array l into keys with keys.push Key
 
   @summarize: (options) =>
-    options extract summarizeFolders
+    options extract summarizeFolders, groupBy
     summary =
       size: 0
       maxSize: null
@@ -48,6 +49,7 @@ class S3P
       minSizeKey: null
       sizeHistogram: {}
       folders: summarizeFolders && {}
+      groups: groupBy && {}
 
     S3C.each merge options,
       getProgress: ->
@@ -57,7 +59,8 @@ class S3P
           "" maxSize: #{} colors.green "" #{humanByteSize summary.maxSize}
         .join ' '
 
-      map: ({Size, Key}) ->
+      map: (item) ->
+        {Size, Key} = item
         floorSize = humanByteSize
           Math.pow 2, 1 + logSize = (Math.log(Size) / Math.log(2)) | 0
           0
@@ -72,6 +75,12 @@ class S3P
         g.size += Size
         g[floorSize] = (g[floorSize] | 0) + 1
 
+        if groupBy
+          groupKey = groupBy item
+          bucket = summary.groups[groupKey] ?= items: 0, size: 0
+          bucket.items++
+          bucket.size += Size
+
         if summarizeFolders
           folder = summary.folders
           each subFolder in-array
@@ -212,6 +221,63 @@ class S3P
     @_copyWrapper options, (updatedOptions) =>
       S3C.eachPromises updatedOptions
 
+  ##
+    IN: options:
+      bucket:                         <String> required - bucket to delete FROM
+      confirmDeleteItemsFromBucket:   <String> required - must === bucket (safety)
+      prefix, filter, pattern, etc.   standard S3Comprehensions.each options
+      pretend:                        <Boolean> if true, lists but does NOT delete
+      deleteConcurrency:              <Number>  [500] max in-flight delete calls
+    OUT:
+      finalStats:
+        deletedFiles
+        deletedBytes
+        ...
+  @delete: (options) =>
+    options extract
+      bucket
+      confirmDeleteItemsFromBucket
+      dryrun
+      pretend = dryrun
+      verbose
+      stats
+      deleteConcurrency = 500
+      deletePwp
+      region, endpoint, useAccelerateEndpoint
+      s3
+
+    unless pretend || confirmDeleteItemsFromBucket == bucket
+      throw new Error ""
+        confirm-delete-items-from-bucket (#{confirmDeleteItemsFromBucket})
+        must exactly match bucket (#{bucket})
+        (hint: use --dryrun or --pretend to skip this check)
+
+    deletePwp ?= new PromiseWorkerPool deleteConcurrency
+    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint
+
+    stats ?= {}
+    stats.deletedFiles = 0
+    stats.deletedBytes = 0
+
+    S3C.eachPromises merge options,
+      stats
+      map: ({Key, Size}) ->
+        deletePwp.queue ->
+          if pretend
+            Promise.then ->
+              log "#{if pretend then 'PRETEND ' else ''}delete s3://#{bucket}/#{Key} # #{humanByteSize Size}" if verbose
+              stats.deletedFiles++
+              stats.deletedBytes += Size
+          else
+            s3.delete {} bucket, key: Key
+            .then ->
+              log "delete s3://#{bucket}/#{Key} # #{humanByteSize Size}" if verbose
+              stats.deletedFiles++
+              stats.deletedBytes += Size
+
+    .then (info) ->
+      finalStats: S3P.getStatsWithHumanByteSizes merge info, stats
+
   @sync: (options) =>
     @_copyWrapper options, (options2) ->
       options2 extract
@@ -319,7 +385,8 @@ class S3P
       # s3 options
       region
       useAccelerateEndpoint
-    s3 ?= new &Lib/S3 {} region, useAccelerateEndpoint
+      endpoint
+    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint
 
     stats ?= {}
     stats.copiedBytes = 0

package/source/S3Parallel/S3PCli.caf

@@ -123,6 +123,10 @@ cliCommands =
     description:  "" Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.
     options:      merge allCommandOptions,
                   summarize-folders: "" show count and size of each folder
+                  group-by: []
+                    '"js:(item) => String"'
+                    ""
+                      Provide a JS function that returns a string for each item. Summarize will track {items, size} per unique string and return them as a 'groups' map. Replaces the default size-histogram grouping.
                   advancedOptionsForAll
     args: standardFromArgs
     examples:
@@ -132,6 +136,9 @@ cliCommands =
       bucket: :my-bucket filter: "" "js:({Size}) => Size > 1024*1024"
       "" summarize all files larger than 1 Megabyte
 
+      bucket: :my-bucket group-by: "" "js:({Key}) => Key.split('/')[0]"
+      "" get item/size totals grouped by the first path segment
+
   compare:
     description:  "" Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.
     options:      merge allCommandOptions, toBucketOptions, advancedOptionsForAll
@@ -186,6 +193,28 @@ cliCommands =
       bucket: :my-bucket to-bucket: :my-to-bucket
       "" Copy everything from my-bucket to my-to-bucket
 
+  delete:
+    description:
+      """
+        Delete all matching files from a bucket. Uses s3.listObjectsV2 and s3.deleteObject.
+
+        SAFETY: --confirm-delete-items-from-bucket MUST be provided and MUST exactly match --bucket. Without this, s3p refuses to proceed.
+    options: merge allCommandOptions, advancedOptionsForAll,
+      confirm-delete-items-from-bucket: []
+        :bucket-name
+        ""
+          REQUIRED. Must exactly match --bucket. Guards against accidental deletion.
+      dryrun:                   "" Will not modify anything. List everything that would be deleted.
+      pretend:                  "" alias for --dryrun
+      delete-concurrency:       advanced: true argument: :500 description: "" Maximum number of simultaneous delete operations.
+    args: standardFromArgs
+    examples:
+      bucket: :my-bucket confirm-delete-items-from-bucket: :my-bucket prefix: :old/
+      "" Delete every key under 'old/' from my-bucket.
+
+      bucket: :my-bucket confirm-delete-items-from-bucket: :my-bucket dryrun: true
+      "" Dry-run: list what would be deleted but delete nothing.
+
   each:
     description:  "" Create your own iteration. Specify a --map or --map-list option.
     options:      merge allCommandOptions, advancedOptionsForAll,

package/source/S3Parallel/S3PCliCommands.caf

@@ -40,5 +40,6 @@ wrapCommandWithArgsProcessing = (command) ->
   compare:      wrapCommandWithArgsProcessing &S3P.compare
   cp:           wrapCommandWithArgsProcessing &S3P.copy
   sync:         wrapCommandWithArgsProcessing &S3P.sync
+  delete:       wrapCommandWithArgsProcessing &S3P.delete
   each:         wrapCommandWithArgsProcessing &S3Comprehensions.each
   map:          wrapCommandWithArgsProcessing &S3Comprehensions.map