s3p 3.4.1 → 3.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/S3Parallel/Lib/S3Keys.js.map +1 -1
- package/build/S3Parallel/S3Comprehensions.js +1 -1
- package/build/S3Parallel/S3Comprehensions.js.map +1 -1
- package/build/S3Parallel/S3PCli.js +1 -1
- package/build/S3Parallel/S3PCli.js.map +1 -1
- package/build/test/Lib/FsEasy.test.js +1 -1
- package/build/test/Lib/FsEasy.test.js.map +1 -1
- package/build/test/Lib/S3Keys.test.js +1 -1
- package/build/test/Lib/S3Keys.test.js.map +1 -1
- package/package.json +2 -2
- package/source/S3Parallel/Lib/S3Keys.caf +36 -11
- package/source/S3Parallel/S3Comprehensions.caf +2 -2
- package/source/S3Parallel/S3PCli.caf +2 -2
- package/source/test/Lib/FsEasy.test.caf +2 -0
- package/source/test/Lib/S3Keys.test.caf +24 -24
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
"source/S3Parallel/Lib/S3Keys.caf"
|
|
7
7
|
],
|
|
8
8
|
"names": [],
|
|
9
|
-
"mappings": "AAAA;;AAAA,iHAAO,mPAEP,QACE,uCACI,qGAEJ,6BAAe,MACf,2BAAc,KAAK,oBACnB,+BAAgB,kBAA0C,2BAAG,IAAI,IAEjE,mCAAe,QAAG,CAAC,YAAc,OAAG,YAAe,0BAA0B,aAAe,KAE5F,qCAAgB,QAAG,CAAC,KAAK,kBACvB,IACE,UAAU,GAAG,kBACN,uBACN,oBAAQ,QAAG,MAAQ,UAAU,GAAG,IAAI,
|
|
9
|
+
"mappings": "AAAA;;AAAA,iHAAO,mPAEP,QACE,uCACI,qGAEJ,6BAAe,MACf,2BAAc,KAAK,oBACnB,+BAAgB,kBAA0C,2BAAG,IAAI,IAEjE,mCAAe,QAAG,CAAC,YAAc,OAAG,YAAe,0BAA0B,aAAe,KAE5F,qCAAgB,QAAG,CAAC,KAAK,kBACvB,IACE,UAAU,GAAG,kBACN,uBACN,oBAAQ,QAAG,MAAQ,UAAU,GAAG,IAAI,6BA6BzC,6BAAY,QAAG,CAAC,YAAY,QAAQ,8FAAiB,OAAG,cAAa,UAAM,CACzE,IAAI,GACJ,kBAAkB,MAClB,eAAM,WAAW,OAAM,OAAO,KAC5B,IAAG,WAAW,OAAM,MAClB,kBAAkB,KACpB,YAEF,aAAgB,OAAM,qBAAuB,IAAO,gBAAgB,WAAW,KAC/E,aAAa,gBAAgB,OAAO,KAEjC,CAAU,aAAG,KAAK,aAAa,KAChC,cAAM,IAAI,MAAM,yCAAyC,kBAAoB,YAAY,QAAQ,GAAG,YAAY,YAAY,uCAE9H,YACE,wBAAM,MACD,iBACD,KAAI,iBAAuC,0BAAG,IAAI,MAC/C,OAAM,iBAAiB,YAAY,CAAG,2BAAsB,kBAAkB,IAAO,OAAC,OAE3F,qBAAqB,MAAG,MAErB,CAAU,aAAG,MAAK,aAAU,OAC/B,iBAAiB,GAAG,IAAI,KACtB,kBAAqF,sEAAG,IAAI,aAE5F,OACF,iBAAiB,GAAG,KAClB,kBAA2C,4BAAG,IAAI,WAExD,CAAO,CAAU,cAAI,aAAa,aAAa,UAC7C,cAAM,IAAI,8BAAiB,+BAAiB,8BAAgB,sDAE9D,0BAGF,uBAAS,QAAG,CAAC,MAAO,OACf,cAAc,OAAG,iBACZ,UAEH,OAEP,iBAAM,QAAG,CAAC,MAAO,OACf,IAAI,KAAK,KAAK,UAAU,aAAa,QASvC,6CAAoB,QAAG,CAAC,SAAU,OAC7B,iBAAgB,gBACjB,SAAS,mBAAmB,eAAe,8BAE/C,qBAAQ,QAAG,CAAC,KAAK,YAAY,+BAAQ,OACnC,CAAG,eAAI,CACL,OAAO,GACP,IAAI,YAAY,aAAa,GAC7B,eAAM,IAAI,OAAM,cAAe,YAE/B,MACE,wBAAM,MACD,SAAO,KAAE,OACZ,MAAI,MAED,KAAI,YAAS,OAChB,UAAU,UAAU,GAAG,IAAI,uBACrB,6BAAe,YAAY,gBAE/B,OACF,UAAU,aAEX,YACD,OAAO,OAEJ,OAEF,iBAAI,KAAK"
|
|
10
10
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
"use strict"
|
|
2
2
|
let Caf = require('caffeine-script-runtime');
|
|
3
|
-
Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "Error", "compactFlatten", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (PromiseWorkerPool, getLastKeyWithPrefix, Error, compactFlatten, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn) {if (filter) {throw new Error("only use one: when or filter");}; filter = whenFn;}; if (withFn || pattern) {if (compactFlatten([withFn, map, pattern, mapList]).length > 1) {throw new Error("only use one: with, map, pattern or mapList");}; map = withFn != null ? withFn : (({Key}) => Key.match(pattern));}; toKey != null ? toKey : toKey = (toPrefix || addPrefix) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, toPrefix ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3, temp4; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; s3 = (undefined !== (temp3 = options.s3)) ? temp3 : require('./Lib/S3'); pwp = (undefined !== (temp4 = options.pwp)) ? temp4 : new PromiseWorkerPool(listConcurrency); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp5, temp6, temp7, temp8; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp5 = i++;}; return temp5;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp6 = i1++;}; return temp6;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp7 = i2++;}; return temp7;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp8 = i3++;}; return temp8;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START: ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))} middleKey: ${Caf.toString(debugKey(middleKey))} stopAt: ${Caf.toString(debugKey(stopAt))} usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp5, temp6; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp5 = i++;}; return temp5;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp6 = i1++;}; return temp6;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n INPUTS:\n startAfter: ${Caf.toString(debugKey(startAfter))}\n middleKey: ${Caf.toString(debugKey(middleKey))}\n stopAt: ${Caf.toString(debugKey(stopAt))}\n usePrefixBisect: ${Caf.toString(usePrefixBisect)}\n RESULTS: (before recursion)\n overlap: ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n lastLeftKey: ${Caf.toString(debugKey(lastLeftKey))}\n lastRightKey: ${Caf.toString(debugKey(lastRightKey))}\n counts: applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n PLAN:\n recurse: ${Caf.toString(recurse)}\n leftStartAfter: ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n leftStopAt: ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n leftUsePrefixBisect: ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n rightStartAfter: ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n rightStopAt: ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
|
|
3
|
+
Caf.defMod(module, () => {return Caf.importInvoke(["PromiseWorkerPool", "getLastKeyWithPrefix", "Error", "compactFlatten", "log", "merge", "objectWithout", "Function", "Promise", "currentSecond", "interval", "durationString", "timeout", "getBisectKey", "pad", "debugKey", "Math", "peek", "formattedInspect"], [global, require('path'), require('art-standard-lib'), require('art-class-system'), require('./Lib'), {colors: require('colors')}], (PromiseWorkerPool, getLastKeyWithPrefix, Error, compactFlatten, log, merge, objectWithout, Function, Promise, currentSecond, interval, durationString, timeout, getBisectKey, pad, debugKey, Math, peek, formattedInspect) => {let S3Comprehensions; return S3Comprehensions = Caf.defClass(class S3Comprehensions extends Object {}, function(S3Comprehensions, classSuper, instanceSuper) {let filterNone; this.normalizeOptions = (options) => {let dryrun, quiet, toPrefix, addPrefix, prefix, pattern, pretend, toKey, map, mapList, returnValue, filter, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold, withFn, whenFn, returningV, prefixStopAt, r, e, temp, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; dryrun = options.dryrun; quiet = options.quiet; toPrefix = options.toPrefix; addPrefix = options.addPrefix; prefix = (undefined !== (temp = options.prefix)) ? temp : ""; pattern = options.pattern; pretend = (undefined !== (temp1 = options.pretend)) ? temp1 : dryrun; toKey = options.toKey; map = options.map; mapList = options.mapList; returnValue = options.returnValue; filter = options.filter; startAfter = (undefined !== (temp2 = options.startAfter)) ? temp2 : ""; stopAt = options.stopAt; copyConcurrency = (undefined !== (temp3 = options.copyConcurrency)) ? temp3 : 500; largeCopyConcurrency = (undefined !== (temp4 = options.largeCopyConcurrency)) ? temp4 : 75; maxQueueSize = (undefined !== (temp5 = options.maxQueueSize)) ? temp5 : copyConcurrency * 100; copyPwp = (undefined !== (temp6 = options.copyPwp)) ? temp6 : new PromiseWorkerPool(copyConcurrency); largeCopyPwp = (undefined !== (temp7 = options.largeCopyPwp)) ? temp7 : new PromiseWorkerPool(largeCopyConcurrency); largeCopyThreshold = (undefined !== (temp8 = options.largeCopyThreshold)) ? temp8 : 100 * Caf.pow(1024, 2); withFn = options.with; whenFn = options.when; returningV = ((temp9 = options.returning) != null ? temp9 : options.into); if (!(dryrun || toPrefix || withFn || pattern || returningV || whenFn || prefix != null || !(startAfter != null) || !(stopAt != null))) {return options;}; try {startAfter = `${Caf.toString(startAfter)}`; if (stopAt) {stopAt = `${Caf.toString(stopAt)}`;}; prefix = `${Caf.toString(prefix)}`; if (startAfter < prefix) {startAfter = prefix;}; prefixStopAt = getLastKeyWithPrefix(prefix != null ? prefix : ""); if (!stopAt || stopAt > prefixStopAt) {stopAt = prefixStopAt;}; if (whenFn) {if (filter) {throw new Error("only use one: when or filter");}; filter = whenFn;}; if (withFn || pattern) {if (compactFlatten([withFn, map, pattern, mapList]).length > 1) {throw new Error("only use one: with, map, pattern or mapList");}; map = withFn != null ? withFn : (({Key}) => Key.match(pattern));}; toKey != null ? toKey : toKey = (toPrefix != null || addPrefix != null) ? ((compactFlatten([toPrefix, addPrefix, toKey]).length > 1) ? (() => {throw new Error("only use one: addPrefix, toPrefix, toKey");})() : undefined, (toPrefix != null) ? prefix ? (r = RegExp(`^${Caf.toString(prefix)}`), (key) => key.replace(r, toPrefix)) : (key) => toPrefix + key : (key) => addPrefix + key) : (key) => key;} catch (error) {e = error; log.unquoted({options}); throw e;}; return merge(objectWithout(options, "dryrun", "toPrefix", "withFn", "pattern", "into", "returning", "whenFn", "prefix", "addPrefix"), {originalOptions: ((temp10 = options.originalOptions) != null ? temp10 : options), scratchState: {}, quiet, pretend, toKey, filter, map, mapList, returnValue, startAfter, stopAt, copyConcurrency, largeCopyConcurrency, maxQueueSize, copyPwp, largeCopyPwp, largeCopyThreshold});}; filterNone = function() {return true;}; this.eachPromises = (options) => {let failed, map, filter, temp; options = this.normalizeOptions(options); failed = undefined; map = options.map; filter = (undefined !== (temp = options.filter)) ? temp : filterNone; if (!(Caf.is(map, Function))) {throw new Error("Expecting options.map");}; return this.each(merge(options, {mapList: (items) => {let from, into, to, i, temp1; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filterNone(item)) {into.push(Promise.then(() => map(item)).catch((error) => {log({Key: item.Key, error}); return (failed != null ? failed : failed = []).push(item.Key);}));}; temp1 = i++;}; return temp1;})()) : undefined, into));}})).then((result) => (failed != null) ? merge(result, {failed}) : result);}; this.map = (options) => {let map, _reduce, result, temp, temp1; map != null ? map : map = ((temp = options.map) != null ? temp : ((a) => a)); _reduce = ((temp1 = options.reduce) != null ? temp1 : ((a, b) => compactFlatten([a, b]))); result = undefined; return this.each(merge(objectWithout(options, "map"), {mapList: (items) => {let from, into, to, i, temp2; return Promise.all((from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; into.push(Promise.then(() => map(item)).then((m) => result = (result === undefined) ? m : _reduce(result, m))); temp2 = i++;}; return temp2;})()) : undefined, into));}})).catch((error) => error.found || (() => {throw error;})()).then(() => (result === undefined) ? options.default : options.finally ? options.finally(result) : result);}; this.find = (options) => {let withFn, temp; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); return this.each(merge(options, {map: (item) => (() => {throw {found: withFn(item)};})()})).catch((error) => error.found || (() => {throw error;})());}; this.array = (options) => {let withFn, apply, store, intoArray, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = ((temp1 = options.with) != null ? temp1 : options.map)) != null ? temp : ((a) => a)); apply = (item) => withFn(item, item.Key); store = (v) => intoArray.push(v); return this.each(merge(options, {into: intoArray = ((temp2 = options.into) != null ? temp2 : []), with: (item) => Promise.resolve(withFn(item, item.Key)).then(store)}));}; this.object = (options) => {let withFn, withKey, store, intoObject, temp, temp1, temp2; options = this.normalizeOptions(options); withFn = ((temp = options.map) != null ? temp : ((a) => a)); withKey = ((temp1 = options.withKey) != null ? temp1 : (({Key}) => Key)); store = ([k, v]) => intoObject[k] = v; return this.each(merge(options, {into: intoObject = ((temp2 = options.into) != null ? temp2 : {}), with: (item) => Promise.all([withKey(item, item.Key), withFn(item, item.Key)]).then(store)}));}; this.each = (options) => {let quiet, showProgress, debug, fetchOwner, bucket, startAfter, stopAt, filter, compare, toBucket, toKey, limit, maxListRequests, listConcurrency, returnValue, map, mapList, aggressive, getProgress, verboseProgress, throttle, stats, s3, pwp, _reduce, itemsFound, requestsUsed, maxOutstanding, outstanding, startTime, matchingItems, report, progressReporter, applyF, throttled, waitForThrottle, eachRecursive, temp, temp1, temp2, temp3, temp4; options = this.normalizeOptions(options); quiet = options.quiet; showProgress = (undefined !== (temp = options.showProgress)) ? temp : !quiet; debug = options.debug; fetchOwner = options.fetchOwner; bucket = options.bucket; startAfter = options.startAfter; stopAt = options.stopAt; filter = options.filter; compare = options.compare; toBucket = options.toBucket; toKey = options.toKey; limit = (undefined !== (temp1 = options.limit)) ? temp1 : 1000; maxListRequests = options.maxListRequests; listConcurrency = (undefined !== (temp2 = options.listConcurrency)) ? temp2 : 100; returnValue = options.returnValue; map = options.map; mapList = options.mapList; aggressive = options.aggressive; getProgress = options.getProgress; verboseProgress = options.verboseProgress; throttle = options.throttle; stats = options.stats; s3 = (undefined !== (temp3 = options.s3)) ? temp3 : require('./Lib/S3'); pwp = (undefined !== (temp4 = options.pwp)) ? temp4 : new PromiseWorkerPool(listConcurrency); if (compare && map && toBucket) {throw new Error("cannot use both `compare` and `map` - use mapList instead");}; _reduce = options.reduce; itemsFound = requestsUsed = maxOutstanding = outstanding = 0; startTime = currentSecond(); matchingItems = filter ? 0 : undefined; if (showProgress) {report = (message) => {let duration, itemsPerSecond, efficiency; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; efficiency = itemsFound / (requestsUsed * limit) * 100 | 0; return log("s3p: " + compactFlatten([`d: ${Caf.toString(durationString(duration, 2))}`, `items: ${Caf.toString(itemsFound)}`, `items/s: ${Caf.toString(itemsPerSecond | 0)}`, `listRequests: ${Caf.toString(requestsUsed)}`, verboseProgress ? `efficiency: ${Caf.toString(efficiency)}%` : undefined, verboseProgress ? `outstanding: ${Caf.toString(outstanding)}` : undefined, matchingItems ? `matches: ${Caf.toString(matchingItems)}` : undefined, (throttled > 0) ? "listWorkers: throttled" : (pwp.activeWorkers > 0) ? `listWorkers: ${Caf.toString(pwp.activeWorkers)}` : undefined, (pwp.queueSize > 0) ? `listQueue: ${Caf.toString(pwp.queueSize)}` : undefined, Caf.isF(getProgress) && getProgress(duration) || null, message]).join(", "));}; progressReporter = interval(1000, report);}; applyF = (items, compareItems, compareStartAfter, compareStopAt) => {itemsFound += items.length; return Promise.then(() => {let filteredItems, from, into, to, i, from1, into1, to1, i1, from2, into2, to2, i2, from3, into3, to3, i3, temp5, temp6, temp7, temp8; return (() => {switch (false) {case !mapList: return filter ? (filteredItems = (from = items, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (filter(item)) {into.push(item);}; temp5 = i++;}; return temp5;})()) : undefined, into), matchingItems += filteredItems.length, mapList(filteredItems, (from1 = compareItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (filter(item)) {into1.push(item);}; temp6 = i1++;}; return temp6;})()) : undefined, into1), compareStartAfter, compareStopAt)) : mapList(items, compareItems, compareStartAfter, compareStopAt); case !(map && filter): return (from2 = items, into2 = from2, (from2 != null) ? (to2 = from2.length, i2 = 0, (() => {while (i2 < to2) {let item; item = from2[i2]; if (filter(item)) {matchingItems++; map(item);}; temp7 = i2++;}; return temp7;})()) : undefined, into2); case !map: return (from3 = items, into3 = from3, (from3 != null) ? (to3 = from3.length, i3 = 0, (() => {while (i3 < to3) {let item; item = from3[i3]; map(item); temp8 = i3++;}; return temp8;})()) : undefined, into3);};})();}).then(() => items.length);}; throttled = 0; waitForThrottle = () => Promise.then(() => (Caf.isF(throttle) && throttle()) ? (throttled++, timeout(1000).then(() => {throttled--; return waitForThrottle();})) : undefined); eachRecursive = (startAfter, stopAt, usePrefixBisect = false, debugContext) => {let middleKey, rawLeftCount, rawRightCount, applyPromise; if (requestsUsed >= maxListRequests || startAfter >= stopAt) {return Promise.resolve(0);}; middleKey = getBisectKey(startAfter, stopAt, usePrefixBisect); if (showProgress === "verbose") {report();}; debug && log(`debug: START: ${Caf.toString(pad(debugContext != null ? debugContext : "root", 10))} startAfter: ${Caf.toString(debugKey(startAfter))} middleKey: ${Caf.toString(debugKey(middleKey))} stopAt: ${Caf.toString(debugKey(stopAt))} usePrefixBisect: ${Caf.toString(usePrefixBisect)}`); rawLeftCount = rawRightCount = null; applyPromise = null; return waitForThrottle().then(() => {requestsUsed += 2; maxOutstanding = Math.max(maxOutstanding, outstanding += 2); return Promise.all([pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter})), pwp.queue(() => s3.list({bucket, limit, fetchOwner, startAfter: middleKey}))]).then(([rawLeftItems, rawRightItems]) => {let from, into, to, i, from1, into1, to1, i1, temp5, temp6; rawLeftCount = Caf.exists(rawLeftItems) && rawLeftItems.length; rawRightCount = Caf.exists(rawRightItems) && rawRightItems.length; return [(from = rawLeftItems, into = [], (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let item; item = from[i]; if (item.Key <= middleKey) {into.push(item);}; temp5 = i++;}; return temp5;})()) : undefined, into), (from1 = rawRightItems, into1 = [], (from1 != null) ? (to1 = from1.length, i1 = 0, (() => {while (i1 < to1) {let item; item = from1[i1]; if (item.Key <= stopAt) {into1.push(item);}; temp6 = i1++;}; return temp6;})()) : undefined, into1)];});}).finally(() => outstanding -= 2).then(([leftItems, rightItems]) => {let leftCompareOptions, rightCompareOptions; return Promise.all([leftItems, rightItems, compare && toBucket && this._compareList(leftCompareOptions = {limit, pwp, startAfter: toKey(startAfter), bucket: toBucket, stopAt: toKey((leftItems.length === limit) ? peek(leftItems).Key : middleKey)}), compare && toBucket && this._compareList(rightCompareOptions = {limit, pwp, bucket: toBucket, startAfter: toKey(middleKey), stopAt: toKey((rightItems.length === limit) ? peek(rightItems).Key : stopAt)}), leftCompareOptions, rightCompareOptions]);}).then(([leftItems, rightItems, compareLeftItems, compareRightItems, leftCompareOptions, rightCompareOptions]) => {let base, base1; applyPromise = Promise.all([applyF(leftItems, compareLeftItems, Caf.exists(leftCompareOptions) && leftCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt), applyF(rightItems, compareRightItems, Caf.exists(rightCompareOptions) && rightCompareOptions.startAfter, Caf.exists(leftCompareOptions) && leftCompareOptions.stopAt)]); return [leftItems.length, rightItems.length, Caf.exists(base = peek(leftItems)) && base.Key, Caf.exists(base1 = peek(rightItems)) && base1.Key];}).then(([leftCount, rightCount, lastLeftKey, lastRightKey]) => {let recurseLeft, recurseRight, leftStartAfter, leftStopAt, rightStartAfter, rightStopAt, leftUsePrefixBisect, newMiddleKey, recurse; recurseLeft = leftCount >= limit; recurseRight = rightCount >= limit; leftStartAfter = lastLeftKey; leftStopAt = middleKey; rightStartAfter = lastRightKey; rightStopAt = stopAt; leftUsePrefixBisect = rightCount === 0; if (aggressive && leftCount === 0 && recurseRight) {recurseLeft = true; leftUsePrefixBisect = true; leftStartAfter = lastRightKey; leftStopAt = newMiddleKey = getBisectKey(lastRightKey, rightStopAt); rightStartAfter = newMiddleKey;}; recurse = recurseLeft ? recurseRight ? "both" : "left" : "right"; if (debug === "verbose") {log(`-------------------------------------------------------------\ndebug:\n INPUTS:\n startAfter: ${Caf.toString(debugKey(startAfter))}\n middleKey: ${Caf.toString(debugKey(middleKey))}\n stopAt: ${Caf.toString(debugKey(stopAt))}\n usePrefixBisect: ${Caf.toString(usePrefixBisect)}\n RESULTS: (before recursion)\n overlap: ${Caf.toString((lastLeftKey < middleKey) ? "no" : (lastLeftKey === lastRightKey) ? "full" : "partial")}\n lastLeftKey: ${Caf.toString(debugKey(lastLeftKey))}\n lastRightKey: ${Caf.toString(debugKey(lastRightKey))}\n counts: applied: [${Caf.toString(leftCount)}, ${Caf.toString(rightCount)}] raw: [${Caf.toString(rawLeftCount)}, ${Caf.toString(rawRightCount)}]\n PLAN:\n recurse: ${Caf.toString(recurse)}\n leftStartAfter: ${Caf.toString(recurseLeft && debugKey(leftStartAfter))}\n leftStopAt: ${Caf.toString(recurseLeft && debugKey(leftStopAt))}\n leftUsePrefixBisect: ${Caf.toString(recurseLeft && leftUsePrefixBisect)}\n rightStartAfter: ${Caf.toString(recurseRight && debugKey(rightStartAfter))}\n rightStopAt: ${Caf.toString(recurseRight && debugKey(rightStopAt))}`);}; return Promise.all([applyPromise.finally(() => applyPromise = null), recurseLeft ? eachRecursive(leftStartAfter, leftStopAt, leftUsePrefixBisect, recurseRight ? "recurse-BL" : "recurse-L").then((c) => c + leftCount) : leftCount, recurseRight ? eachRecursive(rightStartAfter, rightStopAt, false, recurseLeft ? "recurse-BR" : "recurse-R").then((c) => c + rightCount) : rightCount]).then(([leftCount, rightCount]) => leftCount + rightCount);}).tapCatch((error) => (Caf.is(error, Error)) ? log.error({eachRecursive: {startAfter, stopAt, usePrefixBisect, error}}) : undefined);}; return eachRecursive(startAfter, stopAt).finally(() => Caf.exists(progressReporter) && progressReporter.stop()).then((count) => {let duration, itemsPerSecond, requestsPerSecond, averageItemsPerRequest, info, e; if (!quiet) {Caf.isF(report) && report("DONE");}; duration = currentSecond() - startTime; itemsPerSecond = itemsFound / duration; if (itemsPerSecond > 10) {itemsPerSecond = itemsPerSecond | 0;}; requestsPerSecond = verboseProgress && requestsUsed / duration; if (requestsPerSecond > 10) {requestsPerSecond = requestsPerSecond | 0;}; averageItemsPerRequest = verboseProgress && itemsFound / requestsUsed; if (averageItemsPerRequest > 10) {averageItemsPerRequest = averageItemsPerRequest | 0;}; info = merge({duration, matchingItems, items: itemsFound, itemsPerSecond, requests: requestsUsed, requestsPerSecond, maxOutstanding: verboseProgress && maxOutstanding, averageItemsPerRequest}); return (requestsUsed > maxListRequests) ? (e = Error("S3Comprehensions.each maxListRequestsReached:\n" + formattedInspect(info)), e.info = info, (() => {throw e;})()) : ((returnValue != null && showProgress) ? log.unquoted({finalStats: info}) : undefined, returnValue != null ? returnValue : merge({dryrun: options.pretend}, stats, info));});}; this._compareList = (options) => {let list; return this.each(merge(options, {returnValue: list = [], mapList: (l) => {let from, into, to, i, temp; return (from = l, into = list, (from != null) ? (to = from.length, i = 0, (() => {while (i < to) {let v; v = from[i]; into.push(v); temp = i++;}; return temp;})()) : undefined, into);}, quiet: true}));};});});});
|
|
4
4
|
//# sourceMappingURL=S3Comprehensions.js.map
|
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
"source/S3Parallel/S3Comprehensions.caf"
|
|
7
7
|
],
|
|
8
8
|
"names": [],
|
|
9
|
-
"mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,SAAM,IAC2C,4DAClD,SAAS,UAEX,IAAG,UAAU,UAAO,IAC+C,gBAAgB,QAAQ,KAAK,SAAS,mBAAmB,sEAC1H,MAAM,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEtC,wBAAK,
|
|
9
|
+
"mappings": "AAAA;;AAAA,8TAAO,iBAAO,6BAAiB,6BAAiB,mBAAS,mRAEzD,gLAEqB,CAAC,0WAAW,SAC7B,6JAMW,6FAIC,6MAMG,2GAEW,6FACA,4EACA,kBAAkB,mEAClB,IAAI,kBAAkB,0FACtB,IAAI,kBAAkB,2GACtB,MAAG,QAAG,MAAQ,IAExC,SAAS,cACT,SAAS,cACT,aAAa,+CAAoB,oBAEX,CAAM,UAAI,YAAY,UAAU,WAAW,cAAc,UAAU,oBAAY,yBAAgB,0CAGnH,6BAAgB,mBACQ,+CACxB,yBAAY,WACZ,IAAG,aAAa,SACd,aAAa,UAEf,eAAe,qBAAqB,0BAAS,KAC7C,KAAI,UAAU,SAAS,eACrB,SAAS,gBAEX,IAAG,SAAM,IAC2C,4DAClD,SAAS,UAEX,IAAG,UAAU,UAAO,IAC+C,gBAAgB,QAAQ,KAAK,SAAS,mBAAmB,sEAC1H,MAAM,0BAAM,CAAG,CAAC,CAAC,SAAS,UAAU,aAEtC,wBAAK,QACH,CAAG,oBAAa,qBAAU,CACsC,iBAAgB,UAAU,WAAW,iBAAiB,2FACpH,CAAG,oBACE,SAAM,CACP,IAAI,wBAAO,YACX,CAAC,QAAQ,YAAY,GAAG,aAExB,CAAC,QAAQ,WAAW,MAEtB,CAAC,QAAQ,YAAY,OACpB,CAAC,QAAQ,MAAG,0BAGnB,cAAgB,WAChB,MAAM,YAER,MACE,cAAc,SAAS,UAAQ,YAAU,UAAQ,WAAS,QAAM,aAAW,UAAQ,UAAQ,eAEzF,iBAAiB,uDAA0B,UAC3C,kBACA,OACA,SACA,OACA,QACA,KACA,SACA,aACA,YACA,QACA,iBACA,sBACA,cACA,SACA,cACA,wBAEN,yBAAgB,mCAGD,CAAC,4CACd,UAAU,sBAAkB,UAC5B,SAAS,iBACT,uEAA8B,iBAEiB,aAAO,gEAEtD,UAAM,MAAM,UACV,SAAS,CAAC,yCAAS,OACjB,oBAAgC,mHAAW,WAAW,kBACpD,aAAY,MAAI,IAAI,aACb,CAAC,WACN,IAAG,CAAI,eAAU,gBACjB,2BAAM,kBACA,iFACR,CAAC,WACL,CAAG,kBACD,MAAM,SAAW,WACd,sBAEH,CAAC,oDACL,oBAAG,MAAI,wCAAc,CAAC,MAAM,KAC5B,UAAU,4CAAc,CAAG,CAAC,GAAG,MAC7B,gBAAkB,GAAG,OAEvB,SAAS,kBAET,UAAM,MACJ,cAAc,SAAS,SACvB,SAAS,CAAC,yCAAS,OACjB,oBAAgC,yHAC9B,aAAY,MAAI,IAAI,YACd,CAAC,MACL,SAAY,YAAU,aACpB,IAEA,QAAQ,QAAQ,uEAEnB,CAAC,UAAU,eAAe,cAAM,iBAClC,MACA,YAAU,aACX,kBACM,kBACN,gBAAgB,UACb,uBAYF,CAAC,+BACN,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,YAC/B,UAAM,MAAM,UAAS,KAAK,CAAC,SAAS,eAAM,OAAO,OAAO,qBACjD,CAAC,UAAU,eAAe,cAAM,4BAGjC,CAAC,sEACP,UAAU,sBAAkB,UAC5B,SAAS,SAAY,0CAAG,gCAAc,CAAC,MAAM,KAC7C,QAAQ,CAAC,SAAS,OAAO,MAAM,WAC/B,QAAQ,CAAC,MAAM,eAAe,WAC9B,UAAM,MACJ,UACA,MAAM,YAAY,+CAClB,MAAM,CAAC,SACL,gBAAgB,OAAO,MAAM,gBACvB,2BAGH,CAAC,yEACR,UAAU,sBAAkB,UAC5B,SAAU,wCAAc,CAAC,MAAM,KAC/B,UAAU,8CAAkB,CAAC,CAAC,SAAS,OACvC,QAAQ,CAAC,CAAC,GAAG,OAAiB,WAAC,KAAK,UACpC,UAAM,MACJ,UACA,MAAU,aAAa,+CACvB,MAAM,CAAC,SACL,aACE,QAAQ,MAAM,WACd,OAAO,MAAM,iBACT,yBAuCL,CAAC,wcACN,UAAU,sBAAkB,kBAC5B,sFAEkB,oTAgBR,+HAEU,6SAeb,2EACC,IAAI,kBAAkB,kBAE9B,IAAU,WAAI,OAAO,WACnB,MAAM,IAAI,MAAM,gEAElB,UAAU,gBAGV,aACA,eACA,iBACA,cAAc,GACd,YAAY,iBACZ,gBAAmB,SAAY,IAAO,WAItC,IAAG,eACD,SAAS,CAAC,uDACR,WAAW,kBAAkB,WAC7B,iBAAkB,aAAa,UAC/B,aAAkD,sCAAG,MAAM,UAE3D,IAAI,UACF,gBAAc,mBACJ,eAAe,UAAU,8BACrB,wCACE,iBAAiB,oCACZ,iBAChB,kBAAkC,4BAAiB,4BACnD,kBAAkC,6BAAkB,4BACpD,gBAAkC,yBAAc,8BAChD,aAAY,KAAwB,2BAC/B,qBAAoB,KAAS,6BAAkB,kCACpD,iBAAgB,KAAkB,2BAAgB,8BACrD,oCAAa,aAAa,MAC1B,eACI,UAEV,mBAAmB,SAAS,MAAM,WAgBpC,SAAS,CAAC,OAAO,cAAc,mBAAmB,mBAChD,cAAc,qBACd,aAAY,8IAAG,OACb,wBAAM,MACD,SAAO,OACP,SAAM,CACP,wBAAoC,mHAAW,OAAO,iFACtD,iBAAiB,sBAEjB,QACE,wBACoB,mIAAkB,OAAO,oFAC7C,mBACA,kBAGF,QACE,OACA,cACA,mBACA,gBAAa,MAEd,QAAO,SAAa,gBAAmB,+HAA0C,OAAO,8FAAI,MAC5F,KAAoB,gBAAmB,2HAAW,IAAI,6EACxD,MAAI,iBAEX,YAAY,GACZ,kBAAe,MACb,aAAY,MACV,SAAG,2BAAW,CACZ,aACA,QAAQ,WACH,OACH,oBACA,oCAIR,gBAAgB,CAAC,YAAY,QAAQ,kBAAkB,OAAO,4EAAgB,IACnC,gBAAI,mBAAmB,cAAc,sCAE9E,YAAY,aAAa,YAAY,QAAQ,sBAEjC,iBAAgB,wBAC5B,SAAS,IAAG,+BAAqB,IAAI,sCAAe,QAAO,iCAAkB,SAAS,yCAA0B,SAAS,qCAAsB,SAAS,2CAA4B,qBAEpL,eAAe,gBAAgB,MAC/B,eAAe,aAEf,uBAKK,OACH,gBAAgB,GAChB,iBAAiB,SAAS,gBAAgB,eAAe,WAEzD,aACE,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,eACnD,UAAS,MAAI,SAAW,QAAQ,OAAO,YAAY,YAAY,oBAE3D,CAAC,CAAC,cAAc,gFACpB,0BAAgB,sCAChB,2BAAgB,wDAEM,0HAAmB,YAAY,8FAC/B,oIAAmB,YAAY,mGAEjD,MAAI,eAAe,QAKrB,CAAC,CAAC,WAAW,8DAAe,OAChC,aACE,WACA,YAEO,WAAI,YAAY,kBAAc,qBAAkB,CACrD,OAAO,KACP,YAAY,MAAM,aAClB,QAAQ,UACR,QAAQ,MAAS,sBAAoB,SAAW,KAAK,iBAAoB,cAEpE,WAAI,YAAY,kBAAc,sBAAmB,CACtD,OAAO,KACP,QAAQ,UACR,YAAY,MAAM,YAClB,QAAQ,MAAS,uBAAqB,SAAW,KAAK,kBAAqB,WAE7E,oBACA,8BASE,CAAC,CAAC,WAAW,YAAY,kBAAkB,mBAAmB,oBAAoB,2CAEtF,eAAe,aACb,OAAO,WAAa,6BAAoB,iEAAgC,mDACxE,OAAO,YAAa,8BAAoB,mEAAiC,6DAGzE,kBACA,qCACA,KAAK,4CACL,KAAK,mCAKH,CAAC,CAAC,WAAW,YAAY,aAAa,wJAC1C,cAAkB,aAAa,OAC/B,eAAkB,cAAc,OAEhC,iBAAkB,aAClB,aAAkB,WAClB,kBAAkB,cAClB,cAAkB,QAElB,sBAAsB,eAAc,GAQpC,IAAa,cAAI,cAAa,KAAK,eACjC,cAAc,MACd,sBAAsB,MACtB,iBAAiB,cACjB,aAAa,eAAe,aAAa,cAAc,cACvD,kBAAkB,gBAEpB,UAAa,cAAoB,eAAkB,SAAW,SAAW,SACzE,IAAG,UAAS,YACV,IAAG,wHAKyB,SAAS,oDACT,SAAS,mDACT,SAAS,gDACT,uFAEG,eAAc,aAAe,OAAY,iBAAe,gBAAkB,SAAW,kDACxF,SAAS,qDACT,SAAS,gEACC,4BAAc,mCAAqB,+BAAiB,oEAE1D,oDACA,eAAe,SAAS,4DACxB,eAAe,SAAS,wDACxB,eAAe,gEACf,gBAAgB,SAAS,6DACzB,gBAAgB,SAAS,2BAE3D,aACE,qBACQ,MAAI,eAAe,OAExB,cACD,cAAc,gBAAkB,YAAc,qBAAyB,eAAkB,eAAiB,kBACpG,CAAC,MAAM,IAAI,aACd,WAEF,eACD,cAAc,iBAAkB,aAAc,OAAyB,cAAiB,eAAiB,kBACnG,CAAC,MAAM,IAAI,cACd,kBAED,CAAC,CAAC,WAAW,gBAAgB,YAAY,wBAEvC,CAAC,UAAoF,eAAS,0GAE1G,cACE,YACA,gBAEM,iBAAI,mDACN,CAAC,6FAAS,KACO,6CACrB,WAA0B,kBAAkB,WAC5C,iBAA0B,aAAa,cACS,iBAAiB,4CACjE,oBAA0B,mBAAmB,eAAe,cACT,oBAAoB,kDACvE,yBAA0B,mBAAmB,aAAa,kBACF,yBAAyB,4DACjF,OAAO,OACL,UACA,eACA,OAAoB,YACpB,gBACA,UAAoB,cACpB,mBACA,gBAAoB,mBAAmB,gBACvC,iCAEC,gBAAe,mBAAe,CAC/B,IAAI,MAAM,oDAAoD,iBAAiB,QAC9E,SAAQ,MACT,cAAM,UAEJ,CACF,CAAG,uBAAgB,gBACjB,cAAa,YAAY,oBAE3B,oCACE,MAAK,CACL,QAAQ,kBACR,OACA,iCAEO,CAAC,uBAAW,OACzB,UAAM,MAAM,UACV,aAAa,WACb,SAAS,CAAC,oCAAM,eAAiB,UAAO,iKACxC,OAAO"
|
|
10
10
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
"use strict"
|
|
2
2
|
let Caf = require('caffeine-script-runtime');
|
|
3
|
-
Caf.defMod(module, () => {return Caf.importInvoke(["merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (merge, lowerCamelCase, String, JSON) => {let allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefifx, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefifx, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", required: true, description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"})}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder"}, advancedOptionsForAll), examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n s3p uses the same creds as the aws-cli. Learn more:\n https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => ` ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
|
|
3
|
+
Caf.defMod(module, () => {return Caf.importInvoke(["merge", "lowerCamelCase", "String", "JSON"], [global, require('./StandardImport'), require('./Lib')], (merge, lowerCamelCase, String, JSON) => {let allCommandOptions, writeOptions, toBucketOptions, toFolderOptions, advancedOptionsForAll, advancedOptionsForCopy, cliCommands; allCommandOptions = {quiet: "no output", verbose: "extra output", "api-example": {advanced: true, description: "Output an example of how to invoke the S3P API. Start by constructing the S3P command you want to use with the appropriate command-line options. Then append this option and execute the command. Instead of doing the actual S3P command, S3P will output the JavaScript for invoking this command using the S3P API."}, bucket: {argument: "bucket-name", description: "The source bucket", required: true}, prefix: ["key", "Only iterate over keys with this prefix. If 'startAfter' or 'stopAt' are also specified, the set-intersection of the two will be used."], "start-after": ["key", "Start iteratating after this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "stop-at": ["key", "Iterate up to, and including, this key. If 'prefix' is also specified, the set-intersection of the two will be used."], "fetch-owner": "If you want to filter on the Owner property of items, you'll need to add this option to fetch that property. Note: this adds about 10% overhead for listing items.", pattern: ["string OR js:/^any-javascript-regexp/i", "Source keys must contain the string, OR source keys must match the JavaScript regexp. Note: This won't speed up listing. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern. It may, however, speed up copying by reducing the total number of files copied."], filter: ['"js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"', "Filter which items will be processed. Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter. It may, however, speed up copying by reducing the total number of files copied. \n\nNotes: \n- This won't speed up listing. \n- If you want to access the Owner property, you need to add the --fetch-owner option. \n- For details on the objects passed to filter, see the Contents property returned from the AWS S3 SDK's listObjectsV2 command: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property"]}; writeOptions = {dryrun: {description: "Will not modify anything. For sync/copy commands, do everything except actually copy files."}, pretend: {description: "alias for 'dryrun'"}, acl: {argument: "string", description: "The canned ACL to apply to the object. Possible values include: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read and bucket-owner-full-control"}, "cache-control": {argument: "string", description: "Specifies caching behavior along the request/reply chain."}, "content-disposition": {argument: "string", description: "Specifies presentational information for the object."}, "content-encoding": {argument: "string", description: "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type referenced by the Content-Type header field."}, "content-language": {argument: "string", description: "The language the content is in."}, "content-type": {argument: "string", description: "A standard MIME type describing the format of the object data."}, expires: {argument: "date-time", description: "The date and time at which the object is no longer cacheable. e.g. 'js:new Date' or 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' or 123456789"}, "request-payer": {argument: "string", description: "Confirms that the requester knows that they will be charged for the request. Bucket owners need not specify this parameter in their requests."}, "storage-class": {argument: "string", description: "By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and high availability. Depending on performance needs, you can specify a different Storage Class. Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. Possible values include: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE and OUTPOSTS"}}; toBucketOptions = {"to-bucket": {argument: "bucket-name", required: true, description: "The target bucket. It can be the same bucket."}, "to-prefix": ["key-prefix", "If 'prefix' is specified, the target key will REPLACE its source prefix with toPrefix Otherwise, this is the same as add-prefix."], "add-prefix": ["key-prefix", "The source key is prepended with this string for the target bucket."], "to-key": ['"js:(key) => key"', "Provide an arbitrary JavaScript function for re-keying keys."]}; toFolderOptions = {"to-bucket": ["bucket-name", "The target bucket. It can be the same bucket. (to-bucket OR to-folder required)"], "to-folder": ["to-folder", "Path to a folder in the local file system. (to-bucket OR to-folder required)"]}; advancedOptionsForAll = {"list-concurrency": {advanced: true, argument: "100", description: "Maximum number of simultaneous list operations"}, "max-list-requests": {advanced: true, argument: "number", description: "Not set by default; If set, will stop when hit. Use to limit how many requests get used."}}; advancedOptionsForCopy = merge(advancedOptionsForAll, {"copy-concurrency": {advanced: true, argument: "500", description: "Maximum number of simultaneous small-copies"}, "large-copy-concurrency": {advanced: true, argument: "75", description: "Maximum number of simultaneous large-copies"}, "max-queue-size": {advanced: true, argument: "50000", description: "Maximum number of files that can be queued for copying before list-reading is throttled."}, "large-copy-threshold": {advanced: true, argument: "104857600", description: "Files larger than this byte-size will use the large-copy strategy, which is currently a shell-exec of 'aws s3 cp'. Currently this must be set <= 5368709120 (5 gigabytes). This is s3.copyObject's max supported size, so S3P must shell-exec aws-cli for larger files. 100 megabytes, the default, has been tested to be a good selection for maximum performance."}}); cliCommands = {version: {description: "Show s3p's version."}, "list-buckets": {description: "List all your S3 buckets."}, ls: {description: "List all matching files. Uses s3.listObjectsV2.", options: merge(allCommandOptions, advancedOptionsForAll, {long: "output item size and date as well as key", raw: "output one S3 item per line as JSON with all properties return from listObjects"})}, summarize: {description: "Scan all items in one bucket and produce a summary of all the items. Uses s3.listObjectsV2.", options: merge(allCommandOptions, {"summarize-folders": "show count and size of each folder"}, advancedOptionsForAll), examples: [{bucket: "my-bucket"}, "get a detailed summary of item counts and sizes in my-bucket", {bucket: "my-bucket", filter: '"js:({Size}) => Size > 1024*1024"'}, "summarize all files larger than 1 Megabyte"]}, compare: {description: "Compare two buckets and produce a summary of their differences. Uses s3.listObjectsV2.", options: merge(allCommandOptions, toBucketOptions, advancedOptionsForAll), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Compare items from my-bucket with my-to-bucket. Shows how many items exist in both, only one, or are difference sizes."]}, cp: {description: "Copy all files from one bucket to another bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.\n\nNOTE: This overwrites existing files in the target bucket. Try the 'sync' command for smarter copies when some of the files have already been copied.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, toFolderOptions), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket", {bucket: "my-bucket", "to-folder": "my/local/folder"}, "Copy everything from my-bucket to ./my/local/folder/*", {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/". The copied items will have the same keys as source items.', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-prefix": "2020-04-14-backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and REPLACES prefixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14-backup/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "add-prefix": "backup/"}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" and ADDS prefixes. Example: "2020-04-14/foo.jpg" is copied to "backup/2020-04-14/foo.jpg"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", "to-key": '"js:(key) => key + \'-old\'"'}, 'Copy everything from my-bucket to my-to-bucket with CUSTOM function that adds suffixes. Example: "2020-04-14/foo.jpg" is copied to "2020-04-14/foo.jpg-old"', {bucket: "my-bucket", "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Key}) => /foo/.test(Key)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" that also contains the string \'foo\'.', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => /john/i.test(Owner.DisplayName)"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner who\'s displayname contains \'john\' (case ignored).', {bucket: "my-bucket", "fetch-owner": true, "to-bucket": "my-to-bucket", prefix: "2020-04-14/", filter: '"js:({Owner}) => Owner.ID === \'abc123\'"'}, 'Copy everything from my-bucket to my-to-bucket with the prefix "2020-04-14/" with an owner\'s ID is \'abc123\'.']}, sync: {description: "Only copy files which do not exist in the target bucket. Uses s3.listObjectsV2, s3.copyObject and shell-exec 'aws s3 cp'.", options: merge(allCommandOptions, toBucketOptions, writeOptions, advancedOptionsForCopy, {overwrite: "If set, sync will overwrite existing files with different file sizes."}), examples: [{bucket: "my-bucket", "to-bucket": "my-to-bucket"}, "Copy everything from my-bucket to my-to-bucket"]}, each: {description: "Create your own iteration. Specify a --map or --map-list option.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found. A javascript function of the form (item) => ..."], "map-list": ["function", "This gets called with an array of items (length between 1 and 1000). A javascript function of the form (itemList) => ..."]}), examples: [{bucket: "my-bucket", map: '"js:(item) => console.log(item)"'}, "Log every item found."]}, map: {description: "Map and reduce over the results of listBucket. Though 'map' and 'reduce' have default values, you'll likely want to override at least one of them. Further, you may wish to add a 'finally' function.", options: merge(allCommandOptions, advancedOptionsForAll, {map: ["function", "This gets called for each item found.\nForm: ({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => ...\nDefault: (a) => a\nNote: If you want to access the Owner property, you need to add the --fetch-owner option."], reduce: ["function", "Merge the two results of previous `map` or `reduce` calls into one.\nForm: (previousA, previousB) -> ...\nDefault: (a, b) => require('art-standard-lib').compactFlatten([a, b])"], default: ["any", "The default value to return if no items were found."], finally: ["function", "If present, this function will be applied to produce the final result after the last call to reduce. Form: (finalReduceResult) -> ..."]}), examples: [{bucket: "my-bucket", map: '"js:({Size}) => 1"', reduce: '"js:(a, b) => a + b"'}, "total count", {bucket: "my-bucket", map: '"js:({Size}) => Size"', reduce: '"js:(a, b) => a + b"'}, "total file size", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"'}, "newest item details", {bucket: "my-bucket", reduce: '"js:(a, b) => a.LastModified > b.lastModified ? a : b"', finally: '"js:({Key}) => Key"'}, "key of the newest item"]}}; return {main: function(options) {return require('@art-suite/cli').start(merge(options, {description: "S3 summarize, compare, copy, sync and more with massively parallel power.\n\nConfigure AWS credentials with environment variables:\n s3p uses the same creds as the aws-cli. Learn more:\n https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html\n\nSource: https://github.com/generalui/s3p", commands: Caf.object(cliCommands, (props, name) => {let commaneName; commaneName = lowerCamelCase(name); return merge(props, {run: (options) => options.apiExample ? `require('s3p').${Caf.toString(commaneName)}({\n${Caf.toString(Caf.array(options, (value, key) => ` ${Caf.toString(key)}: ` + ((Caf.is(value, String)) ? JSON.stringify(value) : `${Caf.toString(value)}`), (value, key) => key !== "apiExample").join(",\n"))}\n})\n// > Promise` : require('./S3PCliCommands')[commaneName](merge(options, {fromCli: true}))});})}));}};});});
|
|
4
4
|
//# sourceMappingURL=S3PCli.js.map
|
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
"source/S3Parallel/S3PCli.caf"
|
|
7
7
|
],
|
|
8
8
|
"names": [],
|
|
9
|
-
"mappings": "AAAA;;AAAA,0GAAO,6BAAiB,gMAExB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAEhB,UACI,0CACA,
|
|
9
|
+
"mappings": "AAAA;;AAAA,0GAAO,6BAAiB,gMAExB,qBACE,OAA8B,aAC9B,SAA8B,gBAC9B,gBAA4B,UAAU,MAAK,aAAe,2TAC1D,SACE,UAAc,eACd,aAAgB,qBAChB,UAAc,OAChB,SAAc,OAAgB,2IAC9B,gBAAc,OAAgB,mHAC9B,YAAc,OAAgB,yHAC9B,eAAgB,sKAEhB,UACI,0CACA,iVAMJ,SACI,0FACA,0lBASN,gBACE,SACE,aAAe,gGACjB,UAAS,aAAe,uBACxB,MAAK,UAAU,UAAQ,aACnB,gMAYJ,kBAAsB,UAAU,UAAQ,aAAe,8DACvD,wBAAsB,UAAU,UAAQ,aAAe,yDACvD,qBAAsB,UAAU,UAAQ,aAAe,6LACvD,qBAAsB,UAAU,UAAQ,aAAe,oCACvD,iBAAsB,UAAU,UAAQ,aAAe,mEACvD,UAAsB,UAAU,aAAW,aAAe,+IAC1D,kBAAsB,UAAU,UAAQ,aAAe,kJACvD,kBAAsB,UAAU,UAAQ,aACpC,qbAYN,mBACE,cAAc,UAAU,eAAc,UAAU,MAAK,aAAmB,kDACxE,cAAc,cAAsB,qIACpC,eAAc,cAAsB,wEACpC,WAAc,qBAAsB,kEAEtC,mBACE,cAAc,eAAsB,oFACpC,cAAc,aAAsB,kFAEtC,yBACE,qBAAyB,UAAU,MAAK,UAAU,OAAS,aAAe,mDAC1E,sBAAyB,UAAU,MAAK,UAAU,UAAS,aAAe,8FAE5E,yBAAyB,MAAM,wBAC7B,qBAAyB,UAAU,MAAK,UAAU,OAAW,aAAe,gDAC5E,2BAAyB,UAAU,MAAK,UAAU,MAAW,aAAe,gDAC5E,mBAAyB,UAAU,MAAK,UAAU,SAAW,aAAe,6FAC5E,yBAAyB,UAAU,MAAK,UAAU,aAAW,aACzD,0WAMN,cAAW,CACT,UACE,aAAgB,wBAElB,iBACE,aAAgB,8BAElB,KACE,aAAgB,mDAChB,SAAc,MAAM,mBAAmB,wBACrC,MAAQ,4CACR,KAAQ,sFAEZ,YACE,aAAgB,+FAChB,SAAc,MAAM,oBACN,qBAAqB,uCACrB,wBACd,YACE,QAAQ,cACN,iEAEF,QAAQ,aAAW,QAAU,sCAC3B,gDAEN,UACE,aAAgB,0FAChB,SAAc,MAAM,mBAAmB,iBAAiB,wBACxD,YACE,QAAQ,aAAW,aAAW,iBAC5B,4HAEN,KACE,aACK,+QAIL,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,wBAAwB,kBAC9F,YACE,QAAQ,aAAW,aAAW,iBAC5B,mDAEF,QAAQ,aAAW,aAAW,oBAC5B,0DAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,gBAClD,4IAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,aAAW,uBAC1E,+KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,cAAY,YAC3E,2KAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,UAAU,iCACzE,gKAEF,QAAQ,aAAW,aAAW,gBAAc,QAAQ,eAAa,QAAU,oCACzE,wHAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,sDAC3F,oJAEF,QAAQ,aAAW,eAAa,MAAK,aAAW,gBAAc,QAAQ,eAAa,QAAU,8CAC3F,qHAEN,OACE,aAAgB,6HAChB,SAAc,MAAM,mBAAmB,iBAAiB,cAAc,yBACxD,WAAa,2EAE3B,YACE,QAAQ,aAAW,aAAW,iBAC5B,oDAEN,OACE,aAAgB,oEAChB,SAAc,MAAM,mBAAmB,wBACzB,MAAY,YAAY,0FACxB,aAAY,YAAY,+HACtC,YACE,QAAQ,aACR,KAAU,qCACR,2BAEN,MACE,aAAgB,yMAChB,SAAS,MAAM,mBAAmB,wBAChC,MACE,YACG,gPAML,SACE,YACG,oLAKL,UAAS,OAAO,wDAChB,UACE,YACE,4IAIN,YACE,QAAQ,aACR,KAAU,sBACV,QAAU,yBACR,gBAEF,QAAQ,aACR,KAAU,yBACV,QAAU,yBACR,oBAEF,QAAQ,aACR,QAAU,2DACR,wBAEF,QAAQ,aACR,QAAU,0DACV,SAAW,wBACT,qCAER,cAAM,CAAC,UAAW,OAChB,gCACO,MAAM,UACX,aACK,qTASL,qBAC0B,gDACtB,cAAc,eAAe,cAC7B,MAAM,QAAO,KAAK,CAAC,YACd,qBAAkB,+BAEA,gCAEf,UAAoB,yBAElB,8BAAE,CAAG,eAAS,WAAY,eAAe,SACpC,gBAAK,sDACN,8BAIV,4BAAgB,aAAa,MAAM,UAAS,SAAS"
|
|
10
10
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
"use strict"
|
|
2
2
|
let Caf = require('caffeine-script-runtime');
|
|
3
|
-
Caf.defMod(module, () => {return Caf.importInvoke(["describe", "
|
|
3
|
+
Caf.defMod(module, () => {return Caf.importInvoke(["describe", "test"], [global, require('../StandardImport'), {mockFs: require('mock-fs')}], (describe, test) => {return describe({createParentDirs: function() {return test("no-op", () => {});}});});});
|
|
4
4
|
//# sourceMappingURL=FsEasy.test.js.map
|
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
"source/test/Lib/FsEasy.test.caf"
|
|
7
7
|
],
|
|
8
8
|
"names": [],
|
|
9
|
-
"mappings": "AAAA;;AAAA,
|
|
9
|
+
"mappings": "AAAA;;AAAA,iFAAO,+BAAoB,mDAE3B,gBAAQ,CACN,8BAAoB,OAClB,KAAK"
|
|
10
10
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
"use strict"
|
|
2
2
|
let Caf = require('caffeine-script-runtime');
|
|
3
|
-
Caf.defMod(module, () => {return Caf.importInvoke(["describe", "test", "assert", "getUpToNextSlash", "debugKey", "repeat", "lastKeyChar", "getBisectKey", "middleKeyChar", "getLastKeyWithPrefix"], [global, require('../StandardImport')], (describe, test, assert, getUpToNextSlash, debugKey, repeat, lastKeyChar, getBisectKey, middleKeyChar, getLastKeyWithPrefix) => {return describe({getUpToNextSlash: function() {test(":this/is/ok 0 -> :this/ ", () => assert.eq("this/", getUpToNextSlash("this/is/ok", 0))); test(":this/is/ok 4 -> :this/ ", () => assert.eq("this/", getUpToNextSlash("this/is/ok", 4))); test(":this/is/ok 5 -> :this/is/", () => assert.eq("this/is/", getUpToNextSlash("this/is/ok", 5))); test(":this/is/ok 6 -> :this/is/", () => assert.eq("this/is/", getUpToNextSlash("this/is/ok", 6))); test(":this/is/ok 7 -> :this/is/", () => assert.eq("this/is/", getUpToNextSlash("this/is/ok", 7))); test("undefined if no /", () => assert.eq(undefined, getUpToNextSlash("this", 0))); test("undefined if > last /", () => assert.eq(undefined, getUpToNextSlash("this/is/okthere", 10))); return test("undefined if > length", () => assert.eq(undefined, getUpToNextSlash("this/is/ok", 100)));}, debugKey: function() {test("debugKey :a", () => assert.match(debugKey("a"), /^a +$/)); test("debugKey ''", () => assert.match(debugKey(""), "'' ")); return test("debugKey :ab~~~~~~~~~~~", () => assert.eq(debugKey("ab" + repeat(lastKeyChar, 10)), "ab(~*10) "));}, getBisectKey: {nulls_and_errors: function() {test("matching returns null", () => assert.notPresent(getBisectKey("a", "a"))); test("invalid char in first param", () => assert.rejects(() => getBisectKey("\n", "a"))); test("invalid char in second param", () => assert.rejects(() => getBisectKey("b", "😀"))); return test("reverse-order returns null", () => assert.notPresent(getBisectKey("b", "a")));}, basic: function() {test(":alpha :alphadude -> :b", () => assert.eq("alphaB", getBisectKey("alpha", "alphadude"))); test('"alpha" "alpha me" -> "alpha "', () => assert.eq("alpha ", getBisectKey("alpha", "alpha me"))); test('"alpha" "alpha " -> "alpha "', () => assert.eq("alpha ", getBisectKey("alpha", "alpha "))); test(":a :c -> :b", () => assert.eq("b", getBisectKey("a", "c"))); test(":alphabet :c -> :b", () => assert.eq("b", getBisectKey("alphabet", "c"))); test(":aa :ac -> :ab", () => assert.eq(`aa${Caf.toString(middleKeyChar)}`, getBisectKey("aa", "ab"))); return test("with spaces", () => assert.eq("My `", getBisectKey("My Other thing.bar", "My report.foo")));}, bisectPrefix: {dirAware: function() {return Caf.each2([["alpha/beta", "z", getLastKeyWithPrefix("alpha/")], ["alpha/beta/gamma", "alpha/boomTown", getLastKeyWithPrefix("alpha/beta/")], ["alpha/beta", "alpha/foo", "alpha/d"], ["run2/bcl-sync-status/39e8278706c47520c03865b84b06921a", "u", getLastKeyWithPrefix("run2/")]], ([a, b, out]) => test(`${Caf.toString(a)} ${Caf.toString(b)} -> ${Caf.toString(out.slice(0, 25))}${Caf.toString((out.length > 25) ? "..." : undefined)}`, () => assert.eq(out, getBisectKey(a, b, true))));}}, regressions: function() {test(":alpha :T -> notPresent", () => assert.notPresent(getBisectKey("alpha", "T"))); test(`:alpha :alphb -> :alpha${Caf.toString(middleKeyChar)}`, () => assert.eq(`alpha${Caf.toString(middleKeyChar)}`, getBisectKey("alpha", "alphb"))); return test(":./.git/objects/17/3d216b08fff7c71eeb7bf1c7d5e8aef789461e :./.git/obk -> :./.git/objq", () => assert.eq("./.git/objq", getBisectKey("./.git/objects/17/3d216b08fff7c71eeb7bf1c7d5e8aef789461e", "./.git/obk")));}}});});});
|
|
3
|
+
Caf.defMod(module, () => {return Caf.importInvoke(["describe", "test", "assert", "getUpToNextSlash", "debugKey", "repeat", "lastKeyChar", "getBisectKey", "middleKeyChar", "getLastKeyWithPrefix"], [global, require('../StandardImport')], (describe, test, assert, getUpToNextSlash, debugKey, repeat, lastKeyChar, getBisectKey, middleKeyChar, getLastKeyWithPrefix) => {return describe({getUpToNextSlash: function() {test(":this/is/ok 0 -> :this/ ", () => assert.eq("this/", getUpToNextSlash("this/is/ok", 0))); test(":this/is/ok 4 -> :this/ ", () => assert.eq("this/", getUpToNextSlash("this/is/ok", 4))); test(":this/is/ok 5 -> :this/is/", () => assert.eq("this/is/", getUpToNextSlash("this/is/ok", 5))); test(":this/is/ok 6 -> :this/is/", () => assert.eq("this/is/", getUpToNextSlash("this/is/ok", 6))); test(":this/is/ok 7 -> :this/is/", () => assert.eq("this/is/", getUpToNextSlash("this/is/ok", 7))); test("undefined if no /", () => assert.eq(undefined, getUpToNextSlash("this", 0))); test("undefined if > last /", () => assert.eq(undefined, getUpToNextSlash("this/is/okthere", 10))); return test("undefined if > length", () => assert.eq(undefined, getUpToNextSlash("this/is/ok", 100)));}, debugKey: function() {test("debugKey :a", () => assert.match(debugKey("a"), /^a +$/)); test("debugKey ''", () => assert.match(debugKey(""), "'' ")); return test("debugKey :ab~~~~~~~~~~~", () => assert.eq(debugKey("ab" + repeat(lastKeyChar, 10)), "ab(~*10) "));}, getBisectKey: {nulls_and_errors: function() {test("matching returns null", () => assert.notPresent(getBisectKey("a", "a"))); test("invalid char in first param", () => assert.rejects(() => getBisectKey("\n", "a"))); test("invalid char in second param", () => assert.rejects(() => getBisectKey("b", "😀"))); return test("reverse-order returns null", () => assert.notPresent(getBisectKey("b", "a")));}, basic: function() {test(":alpha :alphadude -> :b", () => assert.eq("alphaB", getBisectKey("alpha", "alphadude"))); test('"alpha" "alpha me" -> "alpha "', () => assert.eq("alpha ", getBisectKey("alpha", "alpha me"))); test('"alpha" "alpha " -> "alpha "', () => assert.eq("alpha ", getBisectKey("alpha", "alpha "))); test(":a :c -> :b", () => assert.eq("b", getBisectKey("a", "c"))); test(":alphabet :c -> :b", () => assert.eq("b", getBisectKey("alphabet", "c"))); test(":aa :ac -> :ab", () => assert.eq(`aa${Caf.toString(middleKeyChar)}`, getBisectKey("aa", "ab"))); return test("with spaces", () => assert.eq("My `", getBisectKey("My Other thing.bar", "My report.foo")));}, bisectPrefix: {dirAware: function() {return Caf.each2([["alpha/beta", "z", getLastKeyWithPrefix("alpha/")], ["alpha/beta/gamma", "alpha/boomTown", getLastKeyWithPrefix("alpha/beta/")], ["alpha/beta", "alpha/foo", "alpha/d"], ["run2/bcl-sync-status/39e8278706c47520c03865b84b06921a", "u", getLastKeyWithPrefix("run2/")]], ([a, b, out]) => test(`${Caf.toString(a)} ${Caf.toString(b)} -> ${Caf.toString(out.slice(0, 25))}${Caf.toString((out.length > 25) ? "..." : undefined)}`, () => assert.eq(out, getBisectKey(a, b, true))));}, basic: function() {test(":a :c -> :b~...", () => assert.eq("b", getBisectKey("a", "c", true))); test('"alpha" "alpha me" -> "alpha "', () => assert.eq("alpha ", getBisectKey("alpha", "alpha me", true))); test(":alpha :alphadude -> :alphaB", () => assert.eq("alphaB", getBisectKey("alpha", "alphadude", true))); test(":alphabet :c -> :alph~...", () => assert.eq("b", getBisectKey("alphabet", "c", true))); return test(":alphabet :alpi -> :alpha~...", () => assert.eq("alpho", getBisectKey("alphabet", "alpi", true)));}}, regressions: function() {test(":alpha :T -> notPresent", () => assert.notPresent(getBisectKey("alpha", "T"))); test(`:alpha :alphb -> :alpha${Caf.toString(middleKeyChar)}`, () => assert.eq(`alpha${Caf.toString(middleKeyChar)}`, getBisectKey("alpha", "alphb"))); return test(":./.git/objects/17/3d216b08fff7c71eeb7bf1c7d5e8aef789461e :./.git/obk -> :./.git/objq", () => assert.eq("./.git/objq", getBisectKey("./.git/objects/17/3d216b08fff7c71eeb7bf1c7d5e8aef789461e", "./.git/obk")));}}});});});
|
|
4
4
|
//# sourceMappingURL=S3Keys.test.js.map
|
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
"source/test/Lib/S3Keys.test.caf"
|
|
7
7
|
],
|
|
8
8
|
"names": [],
|
|
9
|
-
"mappings": "AAAA;;AAAA,6MAAO,gKAEP,gBAAQ,CACN,8BACE,KAAK,oCAAgC,UAAU,SAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,SAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,YAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,YAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,YAAW,iBAAiB,cAAY,MACvF,KAAK,2BAAuB,UAAU,WAAW,iBAAiB,QAAQ,MAC1E,KAAK,+BAA2B,UAAU,WAAW,iBAAiB,mBAAmB,cACzF,KAAK,+BAA2B,UAAU,WAAW,iBAAiB,cAAc,UAEtF,sBACE,KAAK,qBACH,aACE,SAAS,MACT,WAEJ,KAAK,qBACH,aACE,SAAS,KACT,iCAEJ,KAAK,iCACH,UACE,SAAS,OAAM,OAAO,aAAa,MACnC,4BAEN,eAEE,8BACE,KACI,+BACC,kBAAkB,aAAa,KAAG,QAEvC,KACI,qCACC,eAAc,MAAI,aAAa,MAAK,QAEzC,KACI,sCACC,eAAc,MAAI,aAAa,KAAG,gBAEvC,KACI,oCACC,kBAAkB,aAAa,KAAG,UAEzC,mBACE,KACI,iCACC,UAAU,UAAQ,aAAa,SAAO,gBAE3C,KACI,wCACC,UAAU,UAAS,aAAa,SAAO,eAE5C,KACI,sCACC,UAAU,UAAS,aAAa,SAAO,aAE5C,KACI,qBACC,UAAU,KAAG,aAAa,KAAG,QAElC,KACI,4BACC,UAAU,KAAG,aAAa,YAAU,QAEzC,KACI,wBACC,4BAAe,kBAAgB,aAAa,MAAI,gBAErD,KACI,qBACC,UACC,QACF,aACE,sBACA,sBAER,eACE,sBAAY,iBAEU,EACZ,cAAY,KAAG,qBAAqB,aACpC,oBAAkB,kBAAgB,qBAAqB,kBACvD,cAAY,aAAW,aACvB,yDAAuD,KAAG,qBAAqB,6BACrF,KAAI,gBAAI,mBAAK,sBAAQ,UAAU,GAAG,oBAAe,cAAa,kCAC5D,UAAU,KAAK,aAAa,GAAG,GAAG,
|
|
9
|
+
"mappings": "AAAA;;AAAA,6MAAO,gKAEP,gBAAQ,CACN,8BACE,KAAK,oCAAgC,UAAU,SAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,SAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,YAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,YAAW,iBAAiB,cAAY,MACvF,KAAK,oCAAgC,UAAU,YAAW,iBAAiB,cAAY,MACvF,KAAK,2BAAuB,UAAU,WAAW,iBAAiB,QAAQ,MAC1E,KAAK,+BAA2B,UAAU,WAAW,iBAAiB,mBAAmB,cACzF,KAAK,+BAA2B,UAAU,WAAW,iBAAiB,cAAc,UAEtF,sBACE,KAAK,qBACH,aACE,SAAS,MACT,WAEJ,KAAK,qBACH,aACE,SAAS,KACT,iCAEJ,KAAK,iCACH,UACE,SAAS,OAAM,OAAO,aAAa,MACnC,4BAEN,eAEE,8BACE,KACI,+BACC,kBAAkB,aAAa,KAAG,QAEvC,KACI,qCACC,eAAc,MAAI,aAAa,MAAK,QAEzC,KACI,sCACC,eAAc,MAAI,aAAa,KAAG,gBAEvC,KACI,oCACC,kBAAkB,aAAa,KAAG,UAEzC,mBACE,KACI,iCACC,UAAU,UAAQ,aAAa,SAAO,gBAE3C,KACI,wCACC,UAAU,UAAS,aAAa,SAAO,eAE5C,KACI,sCACC,UAAU,UAAS,aAAa,SAAO,aAE5C,KACI,qBACC,UAAU,KAAG,aAAa,KAAG,QAElC,KACI,4BACC,UAAU,KAAG,aAAa,YAAU,QAEzC,KACI,wBACC,4BAAe,kBAAgB,aAAa,MAAI,gBAErD,KACI,qBACC,UACC,QACF,aACE,sBACA,sBAER,eACE,sBAAY,iBAEU,EACZ,cAAY,KAAG,qBAAqB,aACpC,oBAAkB,kBAAgB,qBAAqB,kBACvD,cAAY,aAAW,aACvB,yDAAuD,KAAG,qBAAqB,6BACrF,KAAI,gBAAI,mBAAK,sBAAQ,UAAU,GAAG,oBAAe,cAAa,kCAC5D,UAAU,KAAK,aAAa,GAAG,GAAG,YAExC,mBACE,KACI,yBACC,UAAU,KAAG,aAAa,KAAG,KAAG,SAErC,KACI,wCACC,UAAU,UAAS,aAAa,SAAO,YAAW,SAEvD,KACI,sCACC,UAAU,UAAQ,aAAa,SAAO,aAAW,SAEtD,KACI,mCACC,UACD,KACA,aAAa,YAAU,KAAG,gBAE9B,KACI,uCACC,UACD,SACA,aAAa,YAAU,QAAM,YAErC,yBACE,KACI,iCACC,kBAAkB,aAAa,SAAO,QAE3C,4CAC8B,wBACzB,+BAAkB,kBAAgB,aAAa,SAAO,mBAE3D,KACI,+FACC,UAAU,eAAa,aAAa,4DAA0D"
|
|
10
10
|
}
|
package/package.json
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"art-build-configurator": "^1.29.3",
|
|
19
19
|
"art-testbench": "^3.2.2",
|
|
20
20
|
"caffeine-script": "^0.74.0",
|
|
21
|
-
"jest": "^
|
|
21
|
+
"jest": "^29.3.1",
|
|
22
22
|
"mock-fs": "^5.0.0",
|
|
23
23
|
"neptune-namespaces": "^4.2.9",
|
|
24
24
|
"npm-watch": "^0.11.0"
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"test": "npm run nn\njest",
|
|
51
51
|
"watch": "npm-watch"
|
|
52
52
|
},
|
|
53
|
-
"version": "3.4.
|
|
53
|
+
"version": "3.4.2",
|
|
54
54
|
"watch": {
|
|
55
55
|
"build": {
|
|
56
56
|
"extensions": [
|
|
@@ -16,17 +16,33 @@ import &StandardImport
|
|
|
16
16
|
.match /(^[^\/]*)\//
|
|
17
17
|
if found = m?[0] then key.slice 0, i + found.length
|
|
18
18
|
|
|
19
|
-
##
|
|
19
|
+
## getBisectKey
|
|
20
20
|
IN:
|
|
21
|
-
startAfter:
|
|
22
|
-
stopAt:
|
|
23
|
-
|
|
21
|
+
startAfter: <String>
|
|
22
|
+
stopAt: <String>
|
|
23
|
+
bisectPrefix: boolean [default: false]
|
|
24
|
+
if false: find the key alphanumerically between startAfter and stopAT
|
|
25
|
+
if true: chop off part of startKey and then increment it alpha numerically such that
|
|
26
|
+
the returned key is still > startKey
|
|
27
|
+
This can be an effective way to bisect directory structures:
|
|
28
|
+
|
|
29
|
+
startAfter: /this/is/a/nice/directory/structure
|
|
30
|
+
bisectedKey: /this/is/b
|
|
31
|
+
|
|
32
|
+
Note: This feature is not used by default; it was tested to see if it would enhance performance
|
|
33
|
+
on existing workloads. It did not. However, other directory structures may perform better
|
|
34
|
+
using bisectPrefix.
|
|
35
|
+
|
|
36
|
+
Note: it occurs to me that there might be a bug because bisectKey could be after stopAt.
|
|
37
|
+
|
|
38
|
+
OUT: bisectedKey:
|
|
24
39
|
if stopAt <= startAfter,
|
|
25
40
|
NULL
|
|
26
41
|
else
|
|
27
|
-
<String>
|
|
28
|
-
|
|
29
|
-
|
|
42
|
+
bisectedKey <String>
|
|
43
|
+
Guarantees:
|
|
44
|
+
startAfter < bisectedKey
|
|
45
|
+
bisectedKey < stopAt
|
|
30
46
|
getBisectKey = (startAfter, stopAt, bisectPrefix) -> if startAfter < stopAt
|
|
31
47
|
i = 0
|
|
32
48
|
lastCommonSlash = null
|
|
@@ -47,16 +63,13 @@ import &StandardImport
|
|
|
47
63
|
i < prefixBisectI = (i + startAfter.length) / 2 | 0
|
|
48
64
|
&& key = getUpToNextSlash startAfter, if lastCommonSlash? then lastCommonSlash + 1 else 0
|
|
49
65
|
|
|
50
|
-
# log bisectPrefix: startAfter, stopAt, i, charIndex1, charIndex2, prefixBisectI, nextKeyChar
|
|
51
66
|
getLastKeyWithPrefix key
|
|
52
67
|
|
|
53
68
|
when charIndex1 + 1 == charIndex2
|
|
54
|
-
# log one-different: startAfter, stopAt, i, charIndex1, charIndex2
|
|
55
69
|
startAfter.slice 0, i + 1
|
|
56
70
|
+ supportedKeyChars[(supportedKeyChars.length - 1 + getKeyCharIndex startAfter[i + 1]) / 2 | 0]
|
|
57
71
|
|
|
58
72
|
else
|
|
59
|
-
# log more-than-one-different: startAfter, stopAt, i, charIndex1, charIndex2
|
|
60
73
|
startAfter.slice 0, i
|
|
61
74
|
+ supportedKeyChars[(charIndex1 + charIndex2) / 2 | 0]
|
|
62
75
|
|
|
@@ -69,11 +82,19 @@ import &StandardImport
|
|
|
69
82
|
escapeKey = (key) ->
|
|
70
83
|
if /[\s()]/.test key
|
|
71
84
|
"" "#{key}"
|
|
85
|
+
|
|
72
86
|
else key
|
|
73
87
|
|
|
74
88
|
padKey = (key) ->
|
|
75
89
|
pad key, 20 * Math.ceil key.length / 20
|
|
76
90
|
|
|
91
|
+
## getLastKeyWithPrefix
|
|
92
|
+
IN: prefix: string
|
|
93
|
+
OUT:
|
|
94
|
+
returned-key: string
|
|
95
|
+
returned-key has the same prefix as the one passed in
|
|
96
|
+
any possible other-key > returned-key will not have the same prefix
|
|
97
|
+
as the one passed in
|
|
77
98
|
getLastKeyWithPrefix = (prefix) ->
|
|
78
99
|
if prefix.length < maxKeyLength
|
|
79
100
|
prefix + lastKeyChar.repeat maxKeyLength - prefix.length
|
|
@@ -84,17 +105,21 @@ import &StandardImport
|
|
|
84
105
|
i = lastIndex = key.length - 1
|
|
85
106
|
while key[i] == lastKeyChar do i--
|
|
86
107
|
|
|
87
|
-
key =
|
|
108
|
+
key =
|
|
109
|
+
switch
|
|
88
110
|
when key == ''
|
|
89
111
|
"''"
|
|
112
|
+
|
|
90
113
|
when i < lastIndex
|
|
91
114
|
escapeKey key.slice 0, i + 1
|
|
92
115
|
+ "(#{lastKeyChar}*#{lastIndex - i})"
|
|
116
|
+
|
|
93
117
|
else
|
|
94
118
|
escapeKey key
|
|
95
119
|
|
|
96
120
|
if shouldPad
|
|
97
121
|
padKey key
|
|
122
|
+
|
|
98
123
|
else key
|
|
99
124
|
|
|
100
125
|
else "(#{'' + key})"
|
|
@@ -54,9 +54,9 @@ class S3Comprehensions
|
|
|
54
54
|
map = withFn ? ({Key}) -> Key.match pattern
|
|
55
55
|
|
|
56
56
|
toKey ?=
|
|
57
|
-
if toPrefix or addPrefix
|
|
57
|
+
if toPrefix? or addPrefix?
|
|
58
58
|
throw new Error "only use one: addPrefix, toPrefix, toKey" if compactFlatten([toPrefix, addPrefix, toKey]).length > 1
|
|
59
|
-
if toPrefix
|
|
59
|
+
if toPrefix?
|
|
60
60
|
if prefix
|
|
61
61
|
r = /// ^#{prefix}
|
|
62
62
|
(key) -> key.replace r, toPrefix
|
|
@@ -18,14 +18,14 @@ allCommandOptions =
|
|
|
18
18
|
""
|
|
19
19
|
Source keys must contain the string, OR source keys must match the JavaScript regexp.
|
|
20
20
|
Note: This won't speed up listing.
|
|
21
|
-
Every key matching the specified
|
|
21
|
+
Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided pattern.
|
|
22
22
|
It may, however, speed up copying by reducing the total number of files copied.
|
|
23
23
|
|
|
24
24
|
filter:
|
|
25
25
|
"" "js:({Key, Size, LastModified, ETag, StorageClass, Owner: {DisplayName, ID}}) => true"
|
|
26
26
|
""
|
|
27
27
|
Filter which items will be processed.
|
|
28
|
-
Every key matching the specified
|
|
28
|
+
Every key matching the specified prefix, start-after and stop-at clause will ready from S3 and be tested with the provided filter.
|
|
29
29
|
It may, however, speed up copying by reducing the total number of files copied.
|
|
30
30
|
\n\nNotes:
|
|
31
31
|
\n- This won't speed up listing.
|
|
@@ -90,30 +90,30 @@ describe
|
|
|
90
90
|
test "#{a} #{b} -> #{out.slice(0, 25)}#{"..." if out.length > 25}" ->
|
|
91
91
|
assert.eq out, getBisectKey a, b, true
|
|
92
92
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
93
|
+
basic: ->
|
|
94
|
+
test
|
|
95
|
+
"" :a :c -> :b~...
|
|
96
|
+
-> assert.eq :b getBisectKey :a :c true
|
|
97
|
+
|
|
98
|
+
test
|
|
99
|
+
"" "alpha" "alpha me" -> "alpha "
|
|
100
|
+
-> assert.eq "alpha " getBisectKey :alpha "alpha me" true
|
|
101
|
+
|
|
102
|
+
test
|
|
103
|
+
"" :alpha :alphadude -> :alphaB
|
|
104
|
+
-> assert.eq :alphaB getBisectKey :alpha :alphadude true
|
|
105
|
+
|
|
106
|
+
test
|
|
107
|
+
"" :alphabet :c -> :alph~...
|
|
108
|
+
-> assert.eq
|
|
109
|
+
:b
|
|
110
|
+
getBisectKey :alphabet :c true
|
|
111
|
+
|
|
112
|
+
test
|
|
113
|
+
"" :alphabet :alpi -> :alpha~...
|
|
114
|
+
-> assert.eq
|
|
115
|
+
:alpho
|
|
116
|
+
getBisectKey :alphabet :alpi true
|
|
117
117
|
|
|
118
118
|
regressions: ->
|
|
119
119
|
test
|