s3db.js 2.0.0 → 2.1.1

package/build/cache/s3-cache.class.js

@@ -39,6 +39,7 @@ exports.S3Cache = void 0;
 const zlib_1 = __importDefault(require("zlib"));
 const path = __importStar(require("path"));
 const lodash_1 = require("lodash");
+const sha256_1 = __importDefault(require("crypto-js/sha256"));
 const serializers_type_1 = __importDefault(require("./serializers.type"));
 const json_serializer_1 = require("./json.serializer");
 const avro_serializer_1 = require("./avro.serializer");
@@ -60,11 +61,8 @@ class S3Cache {
         if (filename.length === 0)
             filename = `empty`;
         if (hashed) {
-            filename = Buffer.from(filename)
-                .toString("base64")
-                .split("")
-                .reverse()
-                .join("");
+            filename = (0, sha256_1.default)(filename);
+            // filename = Buffer.from(filename).toString("base64").split("").reverse().join("");
         }
         if (additionalPrefix.length > 0) {
             filename = additionalPrefix + filename;

package/build/cache/s3-resource-cache.class.js

@@ -25,10 +25,11 @@ class S3ResourceCache extends s3_cache_class_1.default {
         this.resource = resource;
     }
     getKey({ action = "list", params }) {
-        return super.getKey({
+        const key = super.getKey({
             params,
             additionalPrefix: `resource=${this.resource.name}/action=${action}|`,
         });
+        return key;
     }
     put({ action = "list", params, data, }) {
         const _super = Object.create(null, {

package/build/s3-client.class.js

@@ -300,6 +300,72 @@ class S3Client extends events_1.default {
             return keys;
         });
     }
+    getContinuationTokenAfterOffset({ prefix, offset = 1000, }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (offset === 0)
+                return null;
+            let truncated = true;
+            let continuationToken;
+            let skipped = 0;
+            while (truncated) {
+                let maxKeys = offset < 1000
+                    ? offset
+                    : offset - skipped > 1000
+                        ? 1000
+                        : offset - skipped;
+                const options = {
+                    prefix,
+                    maxKeys,
+                    continuationToken,
+                };
+                const res = yield this.listObjects(options);
+                if (res.Contents) {
+                    skipped += res.Contents.length;
+                }
+                truncated = res.IsTruncated || false;
+                continuationToken = res.NextContinuationToken;
+                if (skipped >= offset) {
+                    break;
+                }
+            }
+            return continuationToken;
+        });
+    }
+    getKeysPage({ prefix, offset = 0, amount = 100, } = {}) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let keys = [];
+            let truncated = true;
+            let continuationToken;
+            if (offset > 0) {
+                continuationToken = yield this.getContinuationTokenAfterOffset({
+                    prefix,
+                    offset,
+                });
+            }
+            while (truncated) {
+                const options = {
+                    prefix,
+                    continuationToken,
+                };
+                const res = yield this.listObjects(options);
+                if (res.Contents) {
+                    keys = keys.concat(res.Contents.map((x) => x.Key));
+                }
+                truncated = res.IsTruncated || false;
+                continuationToken = res.NextContinuationToken;
+                if (keys.length > amount) {
+                    keys = keys.splice(0, amount);
+                    break;
+                }
+            }
+            if (this.keyPrefix) {
+                keys = keys
+                    .map((x) => x.replace(this.keyPrefix, ""))
+                    .map((x) => (x.startsWith("/") ? x.replace(`/`, "") : x));
+            }
+            return keys;
+        });
+    }
 }
 exports.S3Client = S3Client;
 exports.default = S3Client;

package/build/s3-resource.class.js

@@ -416,7 +416,10 @@ class S3Resource extends events_1.default {
     getMany(ids) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.s3Cache) {
-                const cached = yield this.s3Cache.get({ action: "getMany" });
+                const cached = yield this.s3Cache.get({
+                    action: "getMany",
+                    params: { ids: ids.sort() },
+                });
                 if (cached)
                     return cached;
             }
@@ -429,7 +432,11 @@ class S3Resource extends events_1.default {
                 return data;
             }));
             if (this.s3Cache)
-                yield this.s3Cache.put({ action: "getMany", data: results });
+                yield this.s3Cache.put({
+                    action: "getMany",
+                    params: { ids: ids.sort() },
+                    data: results,
+                });
             this.emit("getMany", ids.length);
             return results;
         });
@@ -467,6 +474,32 @@ class S3Resource extends events_1.default {
             return results;
         });
     }
+    page({ offset = 0, size = 100 }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.s3Cache) {
+                const cached = yield this.s3Cache.get({
+                    action: "page",
+                    params: { offset, size },
+                });
+                if (cached)
+                    return cached;
+            }
+            const keys = yield this.s3Client.getKeysPage({
+                amount: size,
+                offset: offset,
+                prefix: `resource=${this.name}`,
+            });
+            const ids = keys.map((x) => x.replace(`resource=${this.name}/id=`, ""));
+            const data = yield this.getMany(ids);
+            if (this.s3Cache)
+                yield this.s3Cache.put({
+                    action: "page",
+                    params: { offset, size },
+                    data,
+                });
+            return data;
+        });
+    }
     readable() {
         const stream = new resource_ids_read_stream_class_1.ResourceIdsReadStream({ resource: this });
         const transformer = new resource_ids_transformer_class_1.ResourceIdsToDataTransformer({ resource: this });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "description": "Use AWS S3 as a cheap document database.",
   "main": "build/index.js",
   "author": "forattini-dev",