s3p 3.6.0 → 3.6.1

package/build/S3Parallel/Lib/LocalFileLister.js

@@ -0,0 +1,4 @@
+"use strict"
+let Caf = require('caffeine-script-runtime');
+Caf.defMod(module, () => {return Caf.importInvoke(["BaseClass", "present", "Promise", "min", "peek", "path", "fs", "glob"], [global, require('../StandardImport'), require('glob'), {path: require('path'), fs: require('fs').promises}], (BaseClass, present, Promise, min, peek, path, fs, glob) => {let LocalFileLister; return LocalFileLister = Caf.defClass(class LocalFileLister extends BaseClass {constructor({fromFolder, _allFiles, _getFileStat}) {super(...arguments); this._fromFolder = path.join(fromFolder, "aaa").replace(/aaa$/, ""); this._custom_getFileStat = _getFileStat; this._fileStats = {}; this._allFilesPromise = (_allFiles ? Promise.resolve(_allFiles) : glob(path.join(fromFolder, "**"), {nodir: true}).then(({found}) => found)).then((files) => Caf.array(files.sort(), (file) => file = file.slice(fromFolder.length + 1, file.length)));};}, function(LocalFileLister, classSuper, instanceSuper) {this.getter({files: function() {return this._allFilesPromise;}}); this.getter("fromFolder"); this.prototype.list = function({limit = 1000, startAfter, stopAt}) {return (present(startAfter) ? this._findStartIndex(startAfter) : Promise.deepAll({files: this.files, startIndex: 0})).then(({files, startIndex}) => {let out, i, tilIndex; out = []; i = startIndex; tilIndex = min(files.length, startIndex + limit); stopAt != null ? stopAt : stopAt = peek(files); while (i < tilIndex && files[i] <= stopAt) {out.push(this._getFileStat(files[i])); i++;}; return Promise.all(out);});}; this.prototype._getFileStat = function(file) {let f, filePath; return this._custom_getFileStat ? this._custom_getFileStat(file) : (f = this._fileStats[file]) ? Promise.resolve(f) : (filePath = path.join(this.fromFolder, file), fs.stat(path.join(this.fromFolder, file)).then((stats) => this._fileStats[file] = {Key: file, LastModified: stats.mtime, Size: stats.size, Owner: {DisplayName: stats.uid.toString(), ID: stats.uid.toString()}}));}; this.prototype._findStartIndex = function(startAfter) {return this.files.then((files) => {let j, k; j = 0; k = files.length; while (j < k) {let m, pivot; m = (j + k) / 2 | 0; pivot = files[m]; switch (false) {case !(startAfter === pivot): k = j = m; break; case !(startAfter < pivot): k = m; break; default: j = m + 1;};}; return {files, startIndex: j};});};});});});
+//# sourceMappingURL=LocalFileLister.js.map

package/build/S3Parallel/Lib/LocalFileLister.js.map

@@ -0,0 +1,10 @@
+{
+  "version": 3,
+  "file": "LocalFileLister.js",
+  "sourceRoot": "../../..",
+  "sources": [
+    "source/S3Parallel/Lib/LocalFileLister.caf"
+  ],
+  "names": [],
+  "mappings": "AAAA;;AAAA,qIAAO,8BAAiB,kBAAU,uBAAO,IAAI,4GAE7C,oEAA8B,sBAEf,CAAC,CAAC,YAAY,WAAW,qCACpC,mBAEE,UAAU,YAAY,eACb,QAAQ,KAEnB,2BAAuB,cAEvB,sBAEA,wBACE,CAAG,YACD,gBAAgB,aAEhB,KACE,UAAU,YAAY,QACtB,OAAO,YAEH,CAAC,CAAC,WAAW,aAEf,CAAC,oBACS,wBACZ,OAAO,WAAW,oBAAoB,GAAG,0EAEjD,aAAQ,mBAAU,kCAClB,YAAQ,qCAcR,QAAM,CAAC,CAAC,cAAY,YAAY,UAAW,OACzC,CAAG,QAAQ,cAAgB,qBAAiB,cACvC,iBAAmB,mBAAQ,YAAY,UACtC,CAAC,CAAC,OAAO,uCACb,UACA,IAAI,YACJ,WAAW,IAAI,cAAc,aAAa,QAC1C,0BAAM,SAAI,KAAK,QACf,OAAO,IAAG,YAAY,MAAM,MAAM,SAChC,SAAS,kBAAc,MAAM,MAC7B,cAEF,YAAY,yCAchB,QAAc,CAAC,wBAAQ,OAClB,2BACD,yBAAqB,QAElB,KAAI,gBAAY,SACjB,gBAAgB,KACd,CACF,WAAW,UAAU,iBAAa,OAClC,QAAQ,UAAU,iBAAa,YACzB,CAAC,UAAqB,gBAAC,QAAK,CAChC,KAAK,MACL,cAAc,aACd,MAAM,YACN,QACE,aAAa,sBACb,IAAI,6DAEd,QAAiB,CAAC,aAAc,OAC9B,gBACM,CAAC,qBACL,IAAI,GACJ,IAAI,cACJ,OAAM,IAAI,kBACR,IAAW,UAAG,IAAI,GAClB,QAAQ,MAAM,IACd,gBAAM,MACD,gBAAc,QAAW,IAAI,IAAI,UAAC,MAClC,cAAc,QAAW,IAAI,mBACJ,IAAI,IAAI,eACrC,OAAO,YAAY"
+}

package/build/S3Parallel/Lib/S3.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "merge", "encodeURIComponent", "path", "timeout", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "present", "S3Client", "process"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, fs: require('fs'), path: require('path')}], (BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, merge, encodeURIComponent, path, timeout, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, present, S3Client, process) => {let escape, S3; escape = function(single) {return shellEscape([single]);}; return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = []) {let region, useAccelerateEndpoint; super(...arguments); this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, ({region, useAccelerateEndpoint} = options, {region, useAccelerateEndpoint})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, toBucket, fromKey, toKey, size, toFolder, pretend, verbose, copyOptions, toLocalFile, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; toBucket = temp1.toBucket; fromKey = temp1.fromKey; toKey = temp1.toKey; size = temp1.size; toFolder = temp1.toFolder; pretend = temp1.pretend; verbose = temp1.verbose; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(toLocalFile + "ignored-file-name") : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["merge", "BaseClass", "shellEscape", "lowerCamelCase", "Promise", "ListBucketsCommand", "ListObjectsV2Command", "GetObjectCommand", "CopyObjectCommand", "DeleteObjectCommand", "HeadObjectCommand", "currentSecond", "log", "Array", "Error", "present", "encodeURIComponent", "path", "timeout", "Upload", "createParentDirs", "createWriteStreamSafe", "compactFlatten", "createS3Url", "shellExec", "dashCase", "isFunction", "S3Client", "process"], [global, require('../StandardImport'), require('./FsEasy'), require('./LibMisc'), require('@aws-sdk/client-s3'), require('@aws-sdk/lib-storage'), {shellEscape: require('shell-escape'), promisify: require('util').promisify, path: require('path')}], (merge, BaseClass, shellEscape, lowerCamelCase, Promise, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, CopyObjectCommand, DeleteObjectCommand, HeadObjectCommand, currentSecond, log, Array, Error, present, encodeURIComponent, path, timeout, Upload, createParentDirs, createWriteStreamSafe, compactFlatten, createS3Url, shellExec, dashCase, isFunction, S3Client, process) => {let escape, fs, createReadStream, copyFile, S3; escape = function(single) {return shellEscape([single]);}; fs = merge(({createReadStream} = require('fs'), {createReadStream}), ({copyFile} = require('fs').promises, {copyFile})); return S3 = Caf.defClass(class S3 extends BaseClass {constructor(options = []) {let region, useAccelerateEndpoint; super(...arguments); this.awsSdkS3 = new S3Client(merge({region: process.env.AWS_REGION}, ({region, useAccelerateEndpoint} = options, {region, useAccelerateEndpoint})));};}, function(S3, classSuper, instanceSuper) {this.commonCopyOptionsSdkKeys = ["ACL", "CacheControl", "ContentDisposition", "ContentEncoding", "ContentLanguage", "ContentType", "Expires", "RequestPayer", "StorageClass"]; this.commonSdkCopyOptionsByLowerCamelCaseKeys = Caf.object(this.commonCopyOptionsSdkKeys, null, null, null, (k) => lowerCamelCase(k)); this.getter({s3: function() {return {listBuckets: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListBucketsCommand(options))), listObjectsV2: (options = {}) => Promise.resolve(this.awsSdkS3.send(new ListObjectsV2Command(options))), getObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new GetObjectCommand(options))), copyObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new CopyObjectCommand(options))), deleteObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new DeleteObjectCommand(options))), headObject: (options = {}) => Promise.resolve(this.awsSdkS3.send(new HeadObjectCommand(options)))};}}); this.prototype.listBuckets = function() {return this.s3.listBuckets().then(({Buckets}) => Caf.object(Buckets, ({Name, CreationDate}) => CreationDate, null, null, ({Name, CreationDate}) => Name));}; this.prototype.list = function({bucket, prefix, limit = 1000, fetchOwner, startAfter, stopAt}) {let startTime; startTime = currentSecond(); return this.s3.listObjectsV2({Bucket: bucket, Prefix: prefix, MaxKeys: limit, StartAfter: startAfter, FetchOwner: fetchOwner}).tapCatch((error) => log.error({"S3.list-error": {bucket, prefix, startAfter, limit, error}})).then((results) => {let duration, from, into, to, i, temp; duration = currentSecond() - startTime; if (!(results.Contents != null)) {results.Contents = [];}; if (!(Caf.is(results.Contents, Array))) {log.warn({"S3.list-no-Contents": {bucket, prefix, startAfter, limit, duration, results}}); throw new Error("S3.list: Contents is not an array");} else {if (duration > 60) {log.warn({"S3.list-slow": {bucket, prefix, startAfter, limit, duration, results: merge(results, {Contents: `Array ${Caf.toString(results.Contents.length)}`})}});};}; return present(stopAt) ? (from = results.Contents, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= stopAt) {into.push(item);}; temp = i++;}; return temp;})()) : undefined, into) : results.Contents;});}; this.prototype.copy = function(options) {let largeCopyThreshold, scratchState, copyScratchState, fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose, size, copyOptions, toLocalFile, fromFilePath, temp, temp1; if ((largeCopyThreshold = options.largeCopyThreshold, scratchState = options.scratchState)) {copyScratchState = ((temp = scratchState.copyScratchState) != null ? temp : scratchState.copyScratchState = {});}; temp1 = this._normalizeCopyOptions(options); fromBucket = temp1.fromBucket; fromFolder = temp1.fromFolder; fromKey = temp1.fromKey; toBucket = temp1.toBucket; toFolder = temp1.toFolder; toKey = temp1.toKey; pretend = temp1.pretend; verbose = temp1.verbose; size = temp1.size; return (size >= largeCopyThreshold) ? this.largeCopy({commonCopyOptions: this.extractCommonCopyOptions(options), fromBucket, fromFolder, fromKey, toBucket, toFolder, toKey, pretend, verbose}) : (copyOptions = merge(toFolder ? {Bucket: fromBucket, Key: fromKey} : {CopySource: encodeURIComponent(`${Caf.toString(fromBucket)}/${Caf.toString(fromKey)}`), Bucket: toBucket, Key: toKey}, this._getCommonCopyOptionsForSdk(options)), toLocalFile = toFolder ? path.join(toFolder, toKey) : undefined, verbose ? log.unquoted(merge({copyObject: copyOptions, toLocalFile})) : undefined, pretend ? timeout(1, () => {return {pretend: true};}) : fromFolder ? (fromFilePath = path.join(fromFolder, fromKey), toBucket ? (new Upload({client: this.s3, params: {Bucket: toBucket, Key: toKey, Body: fs.createReadStream(fromFilePath)}})).done() : createParentDirs(toLocalFile, copyScratchState).then(() => fs.copyFile(fromFilePath, toLocalFile))) : toFolder ? (/\/$/.test(toKey) && size === 0) ? createParentDirs(path.join(toLocalFile, "ignored-file-name"), copyScratchState) : createWriteStreamSafe(toLocalFile, copyScratchState).then((writeStream) => new Promise((resolve, reject) => this.s3.getObject(copyOptions).then(({Body}) => Body.pipe(writeStream).on("finish", resolve).on("error", reject)))) : this.s3.copyObject(copyOptions));}; this.prototype.delete = function(options) {return this.s3.deleteObject({Bucket: options.bucket, Key: options.key});}; this.prototype.largeCopy = function(options) {let commonCopyOptions, fromBucket, fromFolder, toBucket, toFolder, fromKey, toKey, pretend, verbose, command, temp; commonCopyOptions = options.commonCopyOptions; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; fromFolder = temp.fromFolder; toBucket = temp.toBucket; toFolder = temp.toFolder; fromKey = temp.fromKey; toKey = temp.toKey; pretend = temp.pretend; verbose = temp.verbose; command = compactFlatten([`aws s3 cp ${Caf.toString(escape(createS3Url(fromBucket, fromFolder, fromKey)))} ${Caf.toString(escape(createS3Url(toBucket, toFolder, toKey)))}`, Caf.array(commonCopyOptions, (v, key) => `--${Caf.toString(dashCase(key))} ${Caf.toString((() => {switch (v) {case true: return ""; default: return escape(v);};})())}`, (v, key) => v)]).join(" "); if (verbose) {log.unquoted({largeCopy: command});}; return pretend ? timeout(1, () => {return {pretend: true};}) : shellExec(command);}; this.prototype.headObject = function({bucket, key}) {return this.s3.headObject({Bucket: bucket, Key: key});}; this.prototype._normalizeCopyOptions = function(options) {let bucket, key, fromBucket, toBucket, fromKey, toKey, size, temp, temp1, temp2, temp3; bucket = options.bucket; key = options.key; fromBucket = (undefined !== (temp = options.fromBucket)) ? temp : bucket; toBucket = (undefined !== (temp1 = options.toBucket)) ? temp1 : bucket; fromKey = (undefined !== (temp2 = options.fromKey)) ? temp2 : key; toKey = (undefined !== (temp3 = options.toKey)) ? temp3 : key; size = options.size; if (isFunction(toKey)) {toKey = toKey(fromKey, fromBucket, toBucket, options.size);}; if (!(present(fromBucket) && present(toBucket) && present(fromKey) && present(toKey))) {throw new Error("Missing one of: fromBucket, toBucket, fromKey, toKey or bucket or key as a default");}; return merge(options, {fromBucket, fromKey, toBucket, toKey});}; this.prototype._getCommonCopyOptionsForSdk = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (sdkKey, lowerCamelCasedKey) => options[lowerCamelCasedKey], null, null, (sdkKey, lowerCamelCasedKey) => sdkKey);}; this.prototype.extractCommonCopyOptions = function(options) {return Caf.object(S3.commonSdkCopyOptionsByLowerCamelCaseKeys, (v, k) => options[k]);}; this.prototype.shouldSyncObjects = function(options) {let fromBucket, toBucket, fromKey, toKey, temp; temp = this._normalizeCopyOptions(options); fromBucket = temp.fromBucket; toBucket = temp.toBucket; fromKey = temp.fromKey; toKey = temp.toKey; return Promise.then(() => (options.size < Caf.pow(1024, 2) || !options.size) ? true : this.headObject(options).then(({ContentLength}) => ContentLength !== options.size));}; this.prototype.syncObject = function(options) {return this.shouldSyncObjects(options).then((shouldSync) => shouldSync ? this.copyObject(options).then((result) => merge(result, {copied: true})) : {copied: false});};});});});
 //# sourceMappingURL=S3.js.map

package/build/S3Parallel/Lib/S3.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/Lib/S3.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,4eAAO,8BAAiB,qBAAS,sBAAU,gCAAwB,sCAAc,sCAAiB,mBAAK,qaACvG,SAAM,QAAG,CAAC,SAAW,oBAAe,sDAInB,sBAoBF,CAAC,uEACZ,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACR,EAAC,QAAQ,yBAAyB,gZAEtC,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAEhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,6BAC9C,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,2BACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEpH,6CAsBJ,QAAM,CAAC,iLACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,yNAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,UAAU,UAAU,SAAS,OAAO,SAAS,YAEzD,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,WACH,YAAW,UAAU,SAAQ,KAC9B,iBAAiB,cAAc,uBAE/B,sBACE,aACA,uBAEI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
+  "mappings": "AAAA;;AAAA,sfAAO,8BAAiB,qBAAS,sBAAU,+BAAqB,kCAA0B,sCAAc,sCAAiB,6cACzH,SAAM,QAAG,CAAC,SAAW,oBAAe,YAEpC,KAAK,MACH,EAAC,oBAAoB,oCACrB,EAAC,YAAY,gFAIE,sBAoBF,CAAC,uEACZ,gBAAY,IAAI,SAAS,MAAK,CAC5B,QAAQ,yBACR,EAAC,QAAQ,yBAAyB,gZAEtC,YAAO,CACL,gBAAM,QACJ,aAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,mBAAmB,YACxF,eAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,qBAAqB,YAC1F,WAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,iBAAiB,YACtF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,YACvF,cAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,oBAAoB,YACzF,YAAgB,CAAC,iBAAiB,gBAAgB,mBAAe,IAAI,kBAAkB,8CAG3F,YAAe,OACb,2BACM,CAAC,CAAC,wBAC2B,mCAA2B,oFAWhE,QAAM,CAAC,CAAC,QAAQ,QAAQ,cAAY,YAAY,YAAY,yBAC1D,YAAY,wBACZ,uBACE,QAAY,QACZ,QAAY,QACZ,SAAY,OACZ,YAAY,YACZ,YAAY,sBAEJ,CAAC,UACT,WAAU,kBAAkB,QAAQ,QAAQ,YAAY,OAAO,eAE3D,CAAC,oDACL,WAAW,kBAAkB,WAC7B,MAAI,4BAA8B,yBAClC,KAAO,0BAAoB,UACzB,UAAS,wBAAwB,QAAQ,QAAQ,YAAY,OAAO,UAAU,YAC9E,MAAM,IAAI,MAAM,6CAEb,IAAG,WAAW,KACjB,SAAQ,CAAC,iBAAiB,QAAQ,QAAQ,YAAY,OAAO,UAAU,SAAS,MAAM,UAAS,gCAAmB,4CAEjH,QAAQ,UAAM,QACK,8HAAsB,YAAY,iFAEtD,6CAkBN,QAAM,CAAC,2MACL,0BAAG,mEACD,4BAA+B,gDAAiB,+CAElD,2BAAuB,wPAMpB,SAAQ,sBACT,eAAU,CACR,mBAAmB,8BAA0B,UAC7C,YAAY,YAAY,SACxB,UAAU,UAAU,OACpB,SAAS,YAET,CACF,cAAc,MACT,WAAQ,CACT,QAAY,YACZ,KAAY,WAEV,CACF,YAAY,mCAAwB,4BAAc,aAClD,QAAY,UACZ,KAAY,QAEd,iCAA6B,WAE/B,cAAiB,WAAc,UAAU,UAAU,oBAEhD,UACD,aAAa,OACX,YAAY,aACZ,4BAED,UACD,QAAQ,UAAK,iBAAS,WAEhB,aAAU,CAChB,eAAe,UAAU,YAAY,UAClC,WACD,KAAI,OAAM,CACR,QAAQ,SACR,SACE,QAAQ,UACR,KAAQ,OACR,MAAQ,oBAAoB,2BAKhC,iBAAiB,aAAa,uBACzB,MAAI,YAAY,cAAc,iBAE/B,WACH,YAAW,UAAU,SAAQ,KAC9B,iBACE,UAAU,aAAa,sBACvB,oBAGF,sBAAsB,aAAa,uBAC7B,CAAC,gBAAgB,IAAI,QAAQ,CAAC,SAAS,WAC3C,kBAAc,kBACR,CAAC,CAAC,UACN,UACM,gBACF,UAAQ,YACR,SAAQ,aAGlB,mBAAe,yCAErB,QAAQ,CAAC,UAAW,OAClB,qBAAgB,CACd,QAAQ,gBACR,KAAQ,4CAEZ,QAAW,CAAC,8HAAW,oBACrB,kCACA,2BAAuB,sNAMvB,UACE,gBAAc,0BAGN,OAAO,YAAY,YAAY,YAAY,2BAC3C,OAAO,YAAY,UAAU,UAAU,sBAC3B,+BAAwB,kBACjC,SAAS,sBACd,gBAAO,IAAC,KACH,MAAU,oBACV,cAAO,oCAEZ,MAER,IAAG,UACD,cAAa,WAAW,oBAEvB,UACD,QAAQ,UAAK,iBAAS,WAEtB,UAAU,wCAGd,QAAY,CAAC,CAAC,QAAQ,OAAQ,OAC5B,oBACE,QAAQ,QACR,KAAK,gDAET,QAAuB,CAAC,kGAAW,SACjC,qGAEe,wEACF,sEACD,+DACF,0BAGV,IAAG,WAAW,SACZ,QAAQ,MAAM,SAAS,YAAY,UAAU,iBAE/C,KAAO,CAAmB,uBAAI,QAAQ,aAAa,QAAQ,YAAY,QAAQ,UAC7E,MAAM,IAAI,MAAQ,gGACpB,MAAM,UAAY,YAAY,SAAS,UAAU,wDAEnD,QAA6B,CAAC,UAAW,kBACA,6EACrC,QAAQ,sHAEZ,QAA0B,CAAC,UAAW,kBACnB,uDACf,QAAQ,0CAEZ,QAAmB,CAAC,0DAAW,OAC7B,2BAAuB,qHACvB,aAAY,MACP,CAAY,uBAAG,MAAM,OAAM,gBAC5B,OAEA,gBAAY,cACN,CAAC,CAAC,mBAAmB,kBAAiB,8CASlD,QAAY,CAAC,UAAW,OACtB,uBAAmB,cACb,CAAC,eACF,aACD,gBAAY,cACN,CAAC,WAAW,MAAM,SAAQ,QAAQ,UAErC,CAAG,QAAQ"
 }

package/build/S3Parallel/Lib/index.js

@@ -6,6 +6,7 @@
 .addModules({
   FsEasy:            require('./FsEasy'),
   LibMisc:           require('./LibMisc'),
+  LocalFileLister:   require('./LocalFileLister'),
   PromiseWorkerPool: require('./PromiseWorkerPool'),
   S3:                require('./S3'),
   S3Keys:            require('./S3Keys')

package/build/S3Parallel/S3Comprehensions.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "compactFlatten", "Error", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (PromiseWorkerPool, getLastKeyWithPrefix, compactFlatten, Error, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn || pattern) {if (compactFlatten([whenFn, filter, pattern]).length > 1) {throw new Error("only use one: when, filter or pattern");}; filter = whenFn != null ? whenFn : (({Key}) => Key.match(pattern));}; if (withFn) {if (compactFlatten([withFn, map, mapList]).length > 1) {throw new Error("only use one: with, map or mapList");}; map = withFn;}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, fromFolder, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, region, useAccelerateEndpoint, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; fromFolder = options.fromFolder; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 = options.s3; pwp = (undefined !== (temp3 = options.pwp)) ? temp3 : new PromiseWorkerPool(listConcurrency); s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint: !!useAccelerateEndpoint}); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp4, temp5, temp6, temp7; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp6 = i2++;}; return temp6;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp7 = i3++;}; return temp7;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START:  ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))}  middleKey: ${Caf.toString(debugKey(middleKey))}  stopAt: ${Caf.toString(debugKey(stopAt))}  usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter, stopAt: middleKey})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, stopAt, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp4, temp5; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp4 = i++;}; return temp4;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp5 = i1++;}; return temp5;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n  INPUTS:\n    startAfter:       ${Caf.toString(debugKey(startAfter))}\n    middleKey:        ${Caf.toString(debugKey(middleKey))}\n    stopAt:           ${Caf.toString(debugKey(stopAt))}\n    usePrefixBisect:  ${Caf.toString(usePrefixBisect)}\n  RESULTS: (before recursion)\n    overlap:          ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n    lastLeftKey:      ${Caf.toString(debugKey(lastLeftKey))}\n    lastRightKey:     ${Caf.toString(debugKey(lastRightKey))}\n    counts:           applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n  PLAN:\n    recurse:              ${Caf.toString(recurse)}\n    leftStartAfter:       ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n    leftStopAt:           ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n    leftUsePrefixBisect:  ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n    rightStartAfter:      ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n    rightStopAt:          ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
 //# sourceMappingURL=S3Comprehensions.js.map

package/build/S3Parallel/S3Comprehensions.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3Comprehensions.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,geACN,UAAU,sBAAkB,kBAC5B,sFAEkB,oTAgBR,+HAEU,gZAoBZ,IAAI,kBAAkB,kBAE9B,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAAyB,yBAEtD,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,eACnD,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,oBAE3D,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
+  "mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,yQAEhD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,UAAU,UAAO,IAC0C,gBAAgB,QAAQ,QAAQ,mBAAmB,gEAC/G,SAAS,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEzC,IAAG,SAAM,IACiD,gBAAgB,QAAQ,KAAK,mBAAmB,6DACxG,MAAM,UAER,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,4eACN,UAAU,sBAAkB,kBAC5B,sFAEkB,qVAiBR,+HAEU,gZAoBZ,IAAI,kBAAkB,kBAE9B,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAAyB,yBAEtD,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,QAAQ,cACvE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,QAAQ,YAAY,oBAEnE,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
 }

package/build/S3Parallel/S3P.js

@@ -1,4 +1,4 @@
 "use strict"
 let Caf = require('caffeine-script-runtime');
-Caf.defMod(module, () => {return Caf.importInvoke(["merge", "colors", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "objectWithout", "Promise", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (merge, colors, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, objectWithout, Promise, present, abs, round) => {let S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let list; return S3C.each(merge(options, {returning: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, summary; summarizeFolders = options.summarizeFolders; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: ({Size, Key}) => {let floorSize, logSize, group, groupKey, groupUnit, g, out, folder, temp, temp1, base, into, i1, from, into1, to, i2, temp2, temp3, temp4; floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp5; subFolder = from[i2]; folder = ((temp5 = folder[subFolder]) != null ? temp5 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp2 = Size >= summary.maxSize) != null ? temp2 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp3 = Size <= summary.minSize) != null ? temp3 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp4 = summary.minSize) != null ? temp4 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp, temp1; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp = stats.keyDoesNotExistFiles) != null ? temp : 0)) + 1, stats.keyDoesNotExistBytes = (((temp1 = stats.keyDoesNotExistBytes) != null ? temp1 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
+Caf.defMod(module, () => {return Caf.importInvoke(["terminalColors", "merge", "humanByteSize", "Math", "dirname", "Number", "Object", "Error", "compactFlatten", "objectDiff", "log", "createS3Url", "objectWithout", "Promise", "present", "abs", "round"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib')], (terminalColors, merge, humanByteSize, Math, dirname, Number, Object, Error, compactFlatten, objectDiff, log, createS3Url, objectWithout, Promise, present, abs, round) => {let colors, S3C, itemsByKey, summarizeHistogramGroups, summarizeHistogramGroupUnits, S3P; colors = terminalColors; S3C = require('./S3Comprehensions'); itemsByKey = function(itemList, toKey) {let from, into, to, i, temp; return (from = itemList, into = {}, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let Key, Size; ({Key, Size} = from[i]); into[(toKey != null) ? toKey(Key) : Key] = Size; temp = i++;}; return temp;})()) : undefined, into);}; summarizeHistogramGroups = ["bytes", "kilobytes", "megabytes", "gigabytes", "terabytes", "petabytes", "exabytes", "zetabytes"]; summarizeHistogramGroupUnits = ["_B", "kB", "mB", "gB", "tB", "pB", "eB", "zB"]; return S3P = Caf.defClass(class S3P extends Object {}, function(S3P, classSuper, instanceSuper) {this.list = (options) => {let list; return S3C.each(merge(options, {returning: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}}));}; this.summarize = (options) => {let summarizeFolders, summary; summarizeFolders = options.summarizeFolders; summary = {size: 0, maxSize: null, minSize: null, maxSizeKey: null, minSizeKey: null, sizeHistogram: {}, folders: summarizeFolders && {}}; return S3C.each(merge(options, {getProgress: () => ([`totalSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.size))}`))}`, `minSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.minSize))}`))}`, `maxSize: ${Caf.toString(colors.green(`${Caf.toString(humanByteSize(summary.maxSize))}`))}`]).join(" "), map: ({Size, Key}) => {let floorSize, logSize, group, groupKey, groupUnit, g, out, folder, temp, temp1, base, into, i1, from, into1, to, i2, temp2, temp3, temp4; floorSize = humanByteSize(Math.pow(2, 1 + (logSize = Math.log(Size) / Math.log(2) | 0)), 0); group = ((temp = summarizeHistogramGroups[groupKey = (logSize + 1) / 10 | 0]) != null ? temp : "big"); groupUnit = summarizeHistogramGroupUnits[groupKey]; g = ((temp1 = (base = summary.sizeHistogram)[group]) != null ? temp1 : base[group] = (into = out = {items: 0, size: 0}, i1 = 0, (() => {while (i1 < 10) {let i; i = i1; out[(1 << i) + groupUnit] = 0; i1++;};})(), into)); g.items += 1; g.size += Size; g[floorSize] = (g[floorSize] | 0) + 1; if (summarizeFolders) {folder = summary.folders; from = dirname(Key).split("/"); into1 = from; if (from != null) {to = from.length; i2 = 0; while (i2 < to) {let subFolder, temp5; subFolder = from[i2]; folder = ((temp5 = folder[subFolder]) != null ? temp5 : folder[subFolder] = {size: 0, files: 0}); folder.size += Size; folder.files++; i2++;};}; into1;}; summary.size += Size; if (((temp2 = Size >= summary.maxSize) != null ? temp2 : Size)) {summary.maxSize = Size; summary.maxSizeKey = Key;}; if (((temp3 = Size <= summary.minSize) != null ? temp3 : Size)) {summary.minSize = Size; summary.minSizeKey = Key;}; return summary.minSize = Math.min(((temp4 = summary.minSize) != null ? temp4 : Size), Size);}})).then((stats) => {let humanize, temp, from, into, temp2; summary.averageSize = summary.size / (((temp = stats.matchingItems) != null ? temp : stats.items)) | 0; if (summarizeFolders) {humanize = (folder) => {let from1, into1, temp1; if (Caf.is(folder.size, Number)) {folder.humanSize = humanByteSize(folder.size);}; return (from1 = folder, into1 = from1, (from1 != null) ? (() => {for (let k in from1) {let subFolder; subFolder = from1[k]; temp1 = (Caf.is(subFolder, Object)) ? humanize(subFolder) : undefined;}; return temp1;})() : undefined, into1);}; humanize(summary.folders);}; return merge(stats, summary, {human: (from = summary, into = {}, (from != null) ? (() => {for (let k1 in from) {let v, k; v = from[k1]; k = k1; temp2 = /size$/i.test(k) ? into[k1] = humanByteSize(v) : undefined;}; return temp2;})() : undefined, into)});});}; this.compare = (options) => {let logToDelete, logToCopy, logToReplace, verbose, toKey, bucket, fromFolder, toBucket, toFolder, stats, counts, bytes; options = S3C.normalizeOptions(options); logToDelete = options.logToDelete; logToCopy = options.logToCopy; logToReplace = options.logToReplace; verbose = options.verbose; toKey = options.toKey; bucket = options.bucket; fromFolder = options.fromFolder; toBucket = options.toBucket; toFolder = options.toFolder; if (!toBucket) {throw new Error("toBucket required");}; return S3C.each(merge([options, {returning: stats = {counts: counts = {needToCopy: 0, needToReplace: 0, needToDelete: 0, missingInTarget: 0, different: 0, replaceSmaller: 0, replaceBigger: 0, same: 0}, bytes: bytes = {needToCopy: 0, needToDelete: 0, needToReplace: 0, needToReplaceWith: 0, same: 0}}}], {getProgress: () => compactFlatten([(counts.same > 0) ? `same: ${Caf.toString(colors.green(`${Caf.toString(counts.same)}(${Caf.toString(humanByteSize(bytes.same))})`))}` : undefined, (counts.needToCopy > 0) ? `toCopy: ${Caf.toString(colors.green(`${Caf.toString(counts.needToCopy)}(${Caf.toString(humanByteSize(bytes.needToCopy))})`))}` : undefined, (counts.needToReplace > 0) ? `toReplace: ${Caf.toString(colors.green(`${Caf.toString(counts.needToReplace)}(${Caf.toString(humanByteSize(bytes.needToReplace))} with ${Caf.toString(humanByteSize(bytes.needToReplaceWith))})`))}` : undefined, (counts.needToDelete > 0) ? `toDelete: ${Caf.toString(colors.green(`${Caf.toString(counts.needToDelete)}(${Caf.toString(humanByteSize(bytes.needToDelete))})`))}` : undefined]).join(" "), compare: true, mapList: (sourceItems, targetItems) => {let reverseKeyMap; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); return objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToCopy || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(sourceValue))}`);}; bytes.needToCopy += sourceValue; counts.needToCopy++; return counts.missingInTarget++;}, (key, targetValue) => {if (logToDelete || verbose) {log(`rm ${Caf.toString(createS3Url(toBucket, toFolder, key))} # ${Caf.toString(humanByteSize(targetValue))}`);}; bytes.needToDelete += targetValue; return counts.needToDelete++;}, (key, sourceValue, targetValue) => {let fromKey, temp; fromKey = ((temp = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp : key); if (logToReplace || verbose) {log(`aws s3 cp ${Caf.toString(createS3Url(bucket, fromFolder, fromKey))} ${Caf.toString(createS3Url(toBucket, toFolder, key))} # replace ${Caf.toString(humanByteSize(targetValue))} with ${Caf.toString(humanByteSize(sourceValue))}`);}; counts.different++; counts.needToReplace++; bytes.needToReplace += targetValue; bytes.needToReplaceWith += sourceValue; return (sourceValue > targetValue) ? counts.replaceSmaller++ : counts.replaceBigger++;}, (key, value) => {bytes.same += value; return counts.same++;});}})).then((stats) => {let cleanStats; cleanStats = (s) => Caf.object(s, (v) => (Caf.is(v, Object)) ? cleanStats(v) : v, (v) => v !== 0); return cleanStats(stats);});}; this.copy = (options) => this._copyWrapper(options, (updatedOptions) => S3C.eachPromises(updatedOptions)); this.sync = (options) => this._copyWrapper(options, (options2) => {let copyFile, stats, overwrite, dryrun, pretend, toKey, toBucket, temp; copyFile = options2.map; stats = options2.stats; overwrite = options2.overwrite; dryrun = options2.dryrun; pretend = (undefined !== (temp = options2.pretend)) ? temp : dryrun; toKey = options2.toKey; toBucket = options2.toBucket; stats.toDeleteFiles = 0; stats.toDeleteBytes = 0; stats.toReplaceFiles = 0; stats.toReplaceBytes = 0; stats.toReplaceWithBytes = 0; stats.replacedFiles = 0; stats.replacedBytes = 0; stats.unchangedFiles = 0; stats.unchangedBytes = 0; return S3C.each(merge(objectWithout(options2, "map"), {compare: true, getProgress: (duration) => options2.getProgress(duration) + ` same: ${Caf.toString(colors.green(`${Caf.toString(stats.unchangedFiles)}(${Caf.toString(humanByteSize(stats.unchangedBytes))})`))} toDelete: ${Caf.toString(colors.green(`${Caf.toString(stats.toDeleteFiles)}(${Caf.toString(humanByteSize(stats.toDeleteBytes))})`))}`, mapList: (sourceItems, targetItems) => {let copyPromises, reverseKeyMap; copyPromises = []; reverseKeyMap = toKey && Caf.object(sourceItems, ({Key}) => Key, null, null, ({Key}) => toKey(Key)); objectDiff(itemsByKey(sourceItems, toKey), itemsByKey(targetItems), (key, sourceValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return copyPromises.push(copyFile({Key: fromKey, Size: sourceValue}));}, (key, targetValue) => {log(`aws s3 rm s3://${Caf.toString(toBucket)}/${Caf.toString(key)} # you must do this. size: ${Caf.toString(humanByteSize(targetValue))}`); stats.toDeleteBytes += targetValue; return stats.toDeleteFiles++;}, (key, sourceValue, targetValue) => {let fromKey, temp1; fromKey = ((temp1 = Caf.exists(reverseKeyMap) && reverseKeyMap[key]) != null ? temp1 : key); return overwrite ? (log(`# overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)}`), copyPromises.push(copyFile({Key: fromKey, Size: sourceValue})), stats.replacedFiles++, stats.replacedBytes += targetValue) : (log(`# NOT overwriting s3://${Caf.toString(toBucket)}/${Caf.toString(key)} - replacing targetSize: ${Caf.toString(targetValue)} with sourceSize ${Caf.toString(sourceValue)} (use overwrite: true to overwrite)`), stats.toReplaceFiles++, stats.toReplaceBytes += targetValue, stats.toReplaceWithBytes = sourceValue);}, (key, value) => {stats.unchangedBytes += value; return stats.unchangedFiles++;}); return Promise.all(copyPromises);}}));}); this._copyWrapper = (options, eachFunction) => {let s3, toKey, stats, pretend, verbose, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, region, useAccelerateEndpoint; options = S3C.normalizeOptions(options); s3 = options.s3; toKey = options.toKey; stats = options.stats; pretend = options.pretend; verbose = options.verbose; copyConcurrency = options.copyConcurrency; largeCopyConcurrency = options.largeCopyConcurrency; maxQueueSize = options.maxQueueSize; copyPwp = options.copyPwp; largeCopyPwp = options.largeCopyPwp; largeCopyThreshold = options.largeCopyThreshold; region = options.region; useAccelerateEndpoint = options.useAccelerateEndpoint; s3 != null ? s3 : s3 = new (require('./Lib/S3'))({region, useAccelerateEndpoint}); stats != null ? stats : stats = {}; stats.copiedBytes = 0; stats.copiedBytesPerSecond = 0; stats.copyingBytesInFlight = 0; stats.copyingBytesStarted = 0; stats.copyingFilesStarted = 0; stats.copiedFiles = 0; if (!(present(options.toBucket) || present(options.toFolder))) {throw new Error("toBucket or toFolder required");}; return eachFunction(merge(options, {stats, throttle: () => copyPwp.queueSize + largeCopyPwp.queueSize >= maxQueueSize, getProgress: (duration) => compactFlatten(["copied", `${Caf.toString(colors.green(`${Caf.toString(stats.copiedFiles)}/${Caf.toString(stats.copyingFilesStarted)} ${Caf.toString(humanByteSize(stats.copiedBytes))}/${Caf.toString(humanByteSize(stats.copyingBytesStarted))}`))}`, colors.blue(`${Caf.toString(humanByteSize(stats.copiedBytesPerSecond = stats.copiedBytes / duration))}/s`), `inFlight: ${Caf.toString(colors.green(humanByteSize(stats.copyingBytesInFlight)))}`, `copyWorkers: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.activeWorkers)} + ${Caf.toString(largeCopyPwp.activeWorkers)}`))}`, (0 < copyPwp.queueSize + largeCopyPwp.queueSize) ? `copyQueue: ${Caf.toString(colors.green(`${Caf.toString(copyPwp.queueSize)} + ${Caf.toString(largeCopyPwp.queueSize)}`))}` : undefined, pretend ? colors.yellow("PRETENDING") : undefined]).join(" "), map: ({Key: key, Size}) => {stats.copyingFilesStarted++; stats.copyingBytesStarted += Size; return ((Size < largeCopyThreshold) ? copyPwp : largeCopyPwp).queue(() => {stats.copyingBytesInFlight += Size; options = merge(s3.extractCommonCopyOptions(options), {pretend, verbose, bucket: options.bucket, toBucket: options.toBucket, toFolder: options.toFolder, key, toKey, largeCopyThreshold, size: Size}); return s3.copy(options).then(() => {stats.copyingBytesInFlight -= Size; stats.copiedFiles++; return stats.copiedBytes += Size;}).catch((error) => {let temp, temp1; return /NoSuchKey: The specified key does not exist/.test(error.stack) ? (verbose ? log(`skipping because "NoSuchKey: The specified key does not exist": ${Caf.toString(key)} (${Caf.toString(Size)} bytes)`) : undefined, stats.keyDoesNotExistFiles = (((temp = stats.keyDoesNotExistFiles) != null ? temp : 0)) + 1, stats.keyDoesNotExistBytes = (((temp1 = stats.keyDoesNotExistBytes) != null ? temp1 : 0)) + Size) : (() => {throw error;})();});});}})).then((stats) => {delete stats.copyingBytesInFlight; delete stats.copyingFilesStarted; delete stats.copyingBytesStarted; stats.copiedBytesPerSecond = stats.copiedBytes / stats.duration; return {finalStats: this.getStatsWithHumanByteSizes(stats)};});}; this.getStatsWithHumanByteSizes = function(stats) {return merge(Caf.object(stats, (v) => (Caf.is(v, Number)) ? (abs(v) < 100) ? round(v, .01) : round(v) : v), {human: Caf.object(stats, (stat, key) => humanByteSize(stat), (stat, key) => /byte|size/i.test(key))});};});});});
 //# sourceMappingURL=S3P.js.map

package/build/S3Parallel/S3P.js.map

@@ -6,5 +6,5 @@
     "source/S3Parallel/S3P.caf"
   ],
   "names": [],
-  "mappings": "AAAA;;AAAA,8PAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,WAAW,WACX,SAAS,CAAC,oCAAM,eAAiB,UAAO,uLAEhC,CAAC,4CAAW,mBACtB,0BACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,gCAEX,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,CAAC,MAAM,qJACX,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,8BAEd,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,yLACvB,UAAU,qBAAqB,eAC/B,sbAgBA,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAE7B,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,4BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,uDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
+  "mappings": "AAAA;;AAAA,sQAAO,iBAAO,6BAAiB,6BAAiB,yRAChD,SAAS,gBAET,MAAM,+BAEN,aAAU,QAAG,CAAC,UAAU,qCAAS,eACH,qIACxB,CAAG,iBACD,MAAM,OACH,OACP,4DAEJ,4BACE,SACA,aACA,aACA,aACA,aACA,aACA,YACA,cAEF,gCACE,MACA,MACA,MACA,MACA,MACA,MACA,MACA,oHAMO,CAAC,uBAAW,OACjB,SAAS,MAAM,UACb,WAAW,WACX,SAAS,CAAC,oCAAM,eAAiB,UAAO,uLAEhC,CAAC,4CAAW,mBACtB,0BACA,WACE,MAAM,GACN,SAAS,MACT,SAAS,MACT,YAAY,MACZ,YAAY,MACZ,mBACA,SAAS,gCAEX,SAAS,MAAM,UACb,mBAKO,8RAAC,MAER,KAAK,CAAC,CAAC,MAAM,qJACX,YAAY,cACV,SAAS,GAAG,IAAI,WAAW,SAAS,QAAQ,SAAS,KAAM,KAC3D,IAEF,QAAQ,kCAAyB,WAAa,WAAU,KAAG,KAAM,qBAAK,QACtE,YAAY,6BAA6B,WACzC,sBAAI,uBAAsB,0BAAM,sBACX,OAAM,OAAO,GAAE,MAAM,oDACnC,IAAE,MAAK,KAAK,aAAa,wBAE/B,WAAU,GACV,UAAS,MACT,EAAC,aAAc,GAAE,aAAa,KAAK,GAEpC,IAAG,mBACD,SAAS,wBAEL,QAAQ,WACD,8HAET,mBAAe,OAAC,8BAAU,qBAAI,MAAM,GAAG,OAAO,KACxC,eAAS,MACf,mCAEG,gBAAS,MAChB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,OAEvB,IAAG,UAAI,QAAI,mCAAkB,QACpB,kBAAW,MACX,qBAAc,cAEhB,kBAAW,SAChB,6CAAkB,OAClB,gBAEA,CAAC,kDACE,sBAA2B,eAAI,gDAAsB,gBAAe,GAC3E,IAAG,mBACD,WAAW,CAAC,qCAAU,IAC4B,oBAAe,2EACtC,oHAAY,mBAAa,WAChD,SAAS,mEAEb,SAAS,2BAEX,MACE,OACA,UAEE,eAA6B,2GAAkC,cAAc,4GAE3E,CAAC,qIACT,UAAU,qBAAqB,wBAC/B,+PAMA,KAAO,WACL,MAAM,IAAI,MAAM,+BAElB,SAAS,OACP,UACA,WAAW,QAAK,CACd,kBACE,YAAoB,GACpB,eAAoB,GACpB,cAAoB,GACpB,iBAAoB,GACpB,WAAoB,GACpB,gBAAoB,GACpB,eAAoB,GACpB,MAAoB,IACtB,gBACE,YAAoB,GACpB,cAAoB,GACpB,eAAoB,GACpB,mBAAoB,GACpB,MAAoB,QAExB,mBACE,gBACK,eAAc,KAAgB,sBAAa,aAAY,gBAAM,6BAAe,cAAc,iCAC1F,qBAAoB,KAAU,wBAAe,aAAY,gBAAM,mCAAqB,cAAc,uCAClG,wBAAuB,KAAO,2BAAkB,aAAY,gBAAM,sCAAwB,cAAc,2CAA4B,cAAc,8CAClJ,uBAAsB,KAAQ,0BAAiB,aAAY,gBAAM,qCAAuB,cAAc,+CACrG,MAER,SAAS,MACT,SAAS,CAAC,aAAa,oCAErB,gBAAgB,oBAA0B,wBAAoC,gDAE9E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,oCACJ,UAAU,oDAAe,uBAAO,UACoG,aAAa,4LAC5I,oBAAe,aAAa,4BAAqB,4BAGxD,CAAC,KAAK,iBAAe,IAC+D,eAAe,2HAC5F,sBAAiB,oBAAa,yBAGrC,CAAC,KAAK,aAAa,oCACjB,UAAU,oDAAe,uBAAO,UAC8I,gBAAgB,qPAC9L,oBACA,wBACK,uBAAgB,aAChB,2BAAoB,oBACtB,eAAc,eACC,0BACA,0BAGpB,CAAC,KAAK,WACC,cAAS,cACd,2BAKF,CAAC,2BACL,aAAa,CAAC,iBACA,UACP,WAAK,WACN,WAAW,KACR,2BACT,WAAW,yBAcR,CAAC,YACN,kBAAc,SAAS,CAAC,mBACtB,iBAAiB,8BAEd,CAAC,YACN,kBAAc,SAAS,CAAC,sFAAY,WAClC,6JAKY,8DAIP,sBAAiB,GACjB,sBAAiB,GACjB,uBAAkB,GAClB,uBAAkB,GAClB,2BAAsB,GACtB,sBAAiB,GACjB,sBAAiB,GAEjB,uBAAkB,GAClB,uBAAkB,UAEvB,SAAS,MACP,cAAc,UAAU,SACxB,SAAS,MACT,aAAa,CAAC,aACZ,qBAAqB,YACrB,uBACc,aAAY,gBAAM,sCAAwB,cAAc,sDACtD,aAAY,gBAAM,qCAAuB,cAAc,8BAEzE,SAAS,CAAC,aAAa,kDACrB,mBAEA,gBAAgB,oBAAyB,wBAAoC,yCAE7E,WACE,WAAW,aAAa,QACxB,WAAW,cAEX,CAAC,KAAK,qCACJ,UAAU,qDAAe,wBAAO,aAChC,kBAAkB,UAAY,KAAK,SAAS,MAAM,kBAGpD,CAAC,KAAK,iBACJ,IAAG,+BAAmB,0BAAY,+CAAiC,cAAc,kBAC5E,uBAAkB,oBACvB,yBAGF,CAAC,KAAK,aAAa,qCACjB,UAAU,qDAAe,wBAAO,aAC7B,YAAS,CACV,uCAA0B,0BAAY,6CAA+B,6CAA+B,iBACpG,kBAAkB,UAAY,KAAK,SAAS,MAAM,gBAClD,uBACK,uBAAkB,eACrB,CACF,2CAA8B,0BAAY,6CAA+B,6CAA+B,oDACxG,wBACK,wBAAmB,aACnB,2BAAsB,gBAG/B,CAAC,KAAK,WACC,wBAAmB,cACxB,kCAEJ,YAAY,2CAoBL,CAAC,SAAS,yLACvB,UAAU,qBAAqB,eAC/B,sbAgBA,kBAAE,KAAI,IAAI,uBAAW,QAAQ,yBAE7B,wBAAK,YACA,oBAAe,GACf,6BAAwB,GACxB,6BAAwB,GACxB,4BAAuB,GACvB,4BAAuB,GACvB,oBAAe,QAEoC,SAAQ,qBAAqB,QAAQ,gFAC7F,aAAa,MAAM,UACjB,OACA,gBAA8B,oBAAG,0BAA0B,cAC3D,aAAa,CAAC,aACZ,gBACI,0BACK,aAAY,gBAAM,mCAAqB,2CAA6B,cAAc,oCAAqB,cAAc,mCAC5H,YAAW,gBAAM,cAAmB,6BAAwB,oBAAoB,2CAC/D,aAAa,cAAc,8DACxB,aAAY,gBAAM,yCAA2B,mCAC9D,KAAI,oBAAoB,0BAAsB,2BAC7B,aAAY,gBAAM,qCAAuB,2CAC/B,yDAC1B,MAER,KAAK,CAAC,MAAM,KAAK,WACf,6BACK,6BAAwB,aAE7B,CAAG,QAAO,sBAAwB,UAAa,oBACzC,OACC,8BAAyB,MAC9B,UAAU,MACR,4BAA4B,WAE1B,SACA,SACA,wBACA,4BACA,4BACA,KACA,OACA,oBACA,MAAM,eAEV,QAAQ,cACH,OACE,8BAAyB,MAC9B,4BACK,qBAAgB,cAEhB,CAAC,4BAAS,OACZ,mDAAmD,eAAW,CAC5D,UAAa,oFAAyE,sBAAQ,6BAC5F,6BACH,uDAA6B,MAC3B,GACC,6BACH,yDAA6B,MAC3B,QACD,cAAM,2BAEb,CAAC,WAAS,OACP,mCACA,kCACA,2BACF,6BAAwB,oBAAoB,wBACjD,YAAY,gCAA4B,gDAE5C,QAA6B,CAAC,QAAS,OACrC,iBACc,cACP,WAAK,WACH,KAAI,KAAK,OACV,MAAM,GAAG,OAET,MAAM,KAEL,KACP,kBAA2B,sBACzB,cAAc"
 }

package/package.json

@@ -7,10 +7,12 @@
   "dependencies": {
     "@art-suite/cli": "2.2",
     "@aws-sdk/client-s3": "^3.436.0",
+    "@aws-sdk/lib-storage": "^3.635.0",
     "art-class-system": "^1.11.13",
     "art-communication-status": "^1.6.0",
     "art-standard-lib": "^1.73.3",
     "caffeine-script-runtime": "^1.14.0",
+    "glob": "^11.0.0",
     "neptune-namespaces-runtime": "^3.2.9",
     "shell-escape": "^0.2.0"
   },
@@ -19,7 +21,7 @@
     "@art-suite/standard-package-builder": "^1.2.6",
     "art-build-configurator": "^1.29.3",
     "art-testbench": "^3.2.2",
-    "caffeine-script": "^0.74.0",
+    "caffeine-script": "^0.75.1",
     "jest": "^29.3.1",
     "mock-fs": "^5.0.0",
     "neptune-namespaces": "^4.2.9"
@@ -56,7 +58,7 @@
     "test": "jest",
     "test-watch": "jest --watch"
   },
-  "version": "3.6.0",
+  "version": "3.6.1",
   "watch": {
     "build": {
       "extensions": [

package/source/S3Parallel/Lib/LocalFileLister.caf

@@ -0,0 +1,99 @@
+import &StandardImport, &glob, {} &path, fs: &fs.promises
+
+class LocalFileLister extends BaseClass
+
+  constructor: ({fromFolder, _allFiles, _getFileStat}) ->
+    @_fromFolder =
+      # ensure has trailing slash
+      path.join fromFolder, "aaa"
+      .replace /aaa$/, ''
+
+    @_custom_getFileStat = _getFileStat
+
+    @_fileStats = {}
+
+    @_allFilesPromise =
+      if _allFiles
+        Promise.resolve _allFiles
+      else
+        glob
+          path.join fromFolder, "**"
+          nodir: true
+
+        .then ({found}) -> found
+
+      .then (files) ->
+        array file in files.sort()
+          file = file.slice fromFolder.length + 1, file.length
+
+  @getter files: -> @_allFilesPromise
+  @getter :fromFolder
+
+  ## list
+    IN:
+      limit: maximum number of files to return
+      startAfter: if present, returns files strictly greater than startAfter
+      stopAt: if present, returns files up to and including stopAt
+
+    OUT:
+      []
+        Key:
+        LastModified:
+        Size
+        Owner
+  list: ({limit=1000, startAfter, stopAt}) ->
+    if present startAfter then @_findStartIndex startAfter
+    else Promise.deepAll {} @files, startIndex: 0
+    .then ({files, startIndex}) ->
+      out = []
+      i = startIndex
+      tilIndex = min files.length, startIndex + limit
+      stopAt ?= peek files
+      while i < tilIndex && files[i] <= stopAt
+        out.push @_getFileStat files[i]
+        i++
+
+      Promise.all out
+
+  #################################################
+  # PRIVATE
+  #################################################
+  ##
+    IN: file name
+    OUT: []
+      Key: file
+      LastModified: stats.mtime
+      Size: stats.size
+      Owner:
+        DisplayName: stats.uid.toString()
+        ID: stats.uid.toString()
+  _getFileStat: (file) ->
+    if @_custom_getFileStat
+      @_custom_getFileStat file
+    else
+      if f = @_fileStats[file]
+        Promise.resolve f
+      else
+        filePath = path.join @fromFolder, file
+        fs.stat path.join @fromFolder, file
+        .then (stats) -> @_fileStats[file] =
+          Key: file
+          LastModified: stats.mtime
+          Size: stats.size
+          Owner:
+            DisplayName: stats.uid.toString()
+            ID: stats.uid.toString()
+
+  _findStartIndex: (startAfter) ->
+    @files
+    .then (files) ->
+      j = 0
+      k = files.length
+      while j < k
+        m = (j + k) / 2 | 0
+        pivot = files[m]
+        switch
+        when startAfter == pivot then k = j = m
+        when startAfter <  pivot then k = m
+        else                          j = m + 1
+      {} files, startIndex: j

package/source/S3Parallel/Lib/S3.caf

@@ -1,6 +1,10 @@
-import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, {} &shellEscape, &util.promisify, &fs, &path
+import &StandardImport, &FsEasy, &LibMisc, &@aws-sdk/client-s3, &@aws-sdk/lib-storage, {} &shellEscape, &util.promisify, &path
 escape = (single) -> shellEscape [] single
 
+fs = merge
+  {createReadStream} = &fs
+  {copyFile} = &fs.promises
+
 # &@aws-sdk/client-s3.config.setPromisesDependency &bluebird
 
 class S3 extends BaseClass
@@ -43,7 +47,16 @@ class S3 extends BaseClass
     .then ({Buckets}) ->
       object {Name, CreationDate} from Buckets with-key Name with CreationDate
 
-  list: ({bucket, prefix, limit=1000, fetchOwner, startAfter}) ->
+  ## list
+    OUT:
+      []
+        Key:
+        LastModified:
+        Size
+        ETag
+        StorageClass
+        Owner
+  list: ({bucket, prefix, limit=1000, fetchOwner, startAfter, stopAt}) ->
     startTime = currentSecond()
     @s3.listObjectsV2
       Bucket:     bucket
@@ -65,14 +78,12 @@ class S3 extends BaseClass
       else if duration > 60
         log.warn S3.list-slow: {} bucket, prefix, startAfter, limit, duration, results: merge results, Contents: "Array #{results.Contents.length}"
 
-      results.Contents
+      if present stopAt
+        array item in-array results.Contents when item.Key <= stopAt
+      else
+        results.Contents
 
-  ##
-    TODO:
-      For large files (>5GB), we need to just use the CLI:
-        aws s3 cp s3://resbio-fastq-drop-internal/180316_NB501104_0354_AHM2L2BGX5/Undetermined/Undetermined_S0_L001_R1_001.fastq.gz s3://genui-dynamodb-migration-scratch/large-test.gz
-        We can add --quiet (probably)
-      Further, that should use acceleration, if enabled - still need to test that.
+  ## copy
     IN:
       options:
         size:       Bytes (optional) if provided and if bigger than the max size allowed by copyObject, largeCopy is used
@@ -82,6 +93,7 @@ class S3 extends BaseClass
         key:      <String>
         fromKey:  from-key (default: key)
         toKey:    to-key (default: key)
+        fromFolder: string
 
         toFolder:   local folder
     OUT:
@@ -92,15 +104,17 @@ class S3 extends BaseClass
       copyScratchState = scratchState.copyScratchState ?= {}
 
     @_normalizeCopyOptions(options) extract
-      fromBucket, toBucket, fromKey, toKey, size
-      toFolder
-      pretend
-      verbose
+      fromBucket, fromFolder, fromKey
+      toBucket, toFolder, toKey
+      pretend, verbose
+      size
 
     if size >= largeCopyThreshold
       @largeCopy {}
         commonCopyOptions: @extractCommonCopyOptions options
-        fromBucket, toBucket, toFolder, fromKey, toKey, pretend, verbose
+        fromBucket, fromFolder, fromKey
+        toBucket, toFolder, toKey
+        pretend, verbose
 
     else
       copyOptions = merge
@@ -125,14 +139,30 @@ class S3 extends BaseClass
       if pretend
         timeout 1 -> pretend: true
 
+      else if fromFolder
+        fromFilePath = path.join fromFolder, fromKey
+        if toBucket
+          new Upload
+            client: @s3
+            params:
+              Bucket: toBucket
+              Key:    toKey
+              Body:   fs.createReadStream fromFilePath
+
+          .done()
+
+        else
+          createParentDirs toLocalFile, copyScratchState
+          .then -> fs.copyFile fromFilePath, toLocalFile
+
       else if toFolder
         if /\/$/.test(toKey) && size == 0 # this is a folder
-          createParentDirs toLocalFile + "ignored-file-name"
-        else
-          createWriteStreamSafe
-            toLocalFile
+          createParentDirs
+            path.join toLocalFile, "ignored-file-name"
             copyScratchState
 
+        else
+          createWriteStreamSafe toLocalFile, copyScratchState
           .then (writeStream) -> new Promise (resolve, reject) ->
             @s3.getObject copyOptions
             .then ({Body}) ->
@@ -179,7 +209,7 @@ class S3 extends BaseClass
     else
       shellExec command
 
-  # get object's metdata
+  # get object's metadata
   headObject: ({bucket, key}) ->
     @s3.headObject
       Bucket: bucket

package/source/S3Parallel/S3Comprehensions.caf

@@ -1,4 +1,4 @@
-import &path, &ArtStandardLib, &ArtClassSystem, &Lib, {} &colors
+import &path, &ArtStandardLib, &ArtClassSystem, &Lib
 
 class S3Comprehensions
 
@@ -210,7 +210,7 @@ class S3Comprehensions
           While it returns true, it will be called once/second/parallel-branch until it returns false.
 
         COMPREHENSION
-        with: (Item) -> ignored - alais of `map`
+        with: (Item) -> ignored - alias of `map`
         when: (Item) -> T/F - if true, pass it on to `map`, else skip
         returning:    if present, return this value instead of stats
         into:         <ALIAS: returning>
@@ -234,6 +234,7 @@ class S3Comprehensions
       startAfter
       stopAt
       filter
+      fromFolder
 
       # compare-to options
       compare
@@ -243,7 +244,7 @@ class S3Comprehensions
       # limits
       limit = 1000
       maxListRequests
-      listConcurrency = 100 # max open reqeusts
+      listConcurrency = 100 # max open requests
 
       # actions
       returnValue
@@ -379,8 +380,8 @@ class S3Comprehensions
         maxOutstanding = Math.max maxOutstanding, outstanding += 2
 
         Promise.all []
-          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter
-          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter: middleKey
+          pwp.queue -> s3.list {} bucket, limit, fetchOwner, startAfter, stopAt: middleKey
+          pwp.queue -> s3.list {} bucket, limit, fetchOwner, stopAt, startAfter: middleKey
 
         .then ([rawLeftItems, rawRightItems]) ->
           rawLeftCount  = rawLeftItems?.length

package/source/S3Parallel/S3P.caf

@@ -1,4 +1,5 @@
-import &path, &ArtStandardLib, &ArtClassSystem, &Lib, {} &colors
+import &path, &ArtStandardLib, &ArtClassSystem, &Lib
+colors = terminalColors
 
 S3C = &S3Comprehensions