s3db.js 1.0.0

package/build/s3-client.class.js

@@ -0,0 +1,274 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = __importStar(require("path"));
+const lodash_1 = require("lodash");
+const nanoid_1 = require("nanoid");
+const events_1 = __importDefault(require("events"));
+const aws_sdk_1 = require("aws-sdk");
+const promise_pool_1 = __importDefault(require("@supercharge/promise-pool"));
+const errors_1 = require("./errors");
+class S3Client extends events_1.default {
+    constructor({ connectionString, parallelism = 10, AwsS3, }) {
+        super();
+        this.id = (0, nanoid_1.nanoid)(7);
+        const uri = new URL(connectionString);
+        this.bucket = uri.hostname;
+        this.parallelism = parallelism;
+        let [, ...subpath] = uri.pathname.split("/");
+        this.keyPrefix = [...(subpath || [])].join("/");
+        this.client =
+            AwsS3 ||
+                new aws_sdk_1.S3({
+                    credentials: new aws_sdk_1.Credentials({
+                        accessKeyId: uri.username,
+                        secretAccessKey: uri.password,
+                    }),
+                });
+    }
+    /**
+     *
+     * @param param0
+     * @returns
+     */
+    getObject({ key }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const options = {
+                    Bucket: this.bucket,
+                    Key: path.join(this.keyPrefix, key),
+                };
+                const response = yield this.client.getObject(options).promise();
+                this.emit("request", "getObject", options);
+                return response;
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    if (error.name === "NoSuchKey") {
+                        return Promise.reject(new errors_1.ClientNoSuchKey({ bucket: this.bucket, key }));
+                    }
+                }
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     *
+     * @param param0
+     * @returns
+     */
+    putObject({ key, metadata, contentType, body, contentEncoding, }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const options = {
+                    Bucket: this.bucket,
+                    Key: path.join(this.keyPrefix, key),
+                    Metadata: Object.assign({}, metadata),
+                    Body: body,
+                    ContentType: contentType,
+                    ContentEncoding: contentEncoding,
+                };
+                const response = yield this.client.putObject(options).promise();
+                this.emit("request", "putObject", options);
+                return response;
+            }
+            catch (error) {
+                this.emit("error", error);
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Proxy to AWS S3's headObject
+     * @param {Object} param
+     * @param {string} param.key
+     * @returns
+     */
+    headObject({ key }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const options = {
+                    Bucket: this.bucket,
+                    Key: path.join(this.keyPrefix, key),
+                };
+                const response = yield this.client.headObject(options).promise();
+                this.emit("request", "headObject", options);
+                return response;
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    if (error.name === "NoSuchKey" || error.name === "NotFound") {
+                        return Promise.reject(new errors_1.ClientNoSuchKey({ bucket: this.bucket, key }));
+                    }
+                }
+                this.emit("error", error);
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Proxy to AWS S3's deleteObject
+     * @param {Object} param
+     * @param {string} param.key
+     * @returns
+     */
+    deleteObject(key) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const options = {
+                    Bucket: this.bucket,
+                    Key: path.join(this.keyPrefix, key),
+                };
+                const response = yield this.client.deleteObject(options).promise();
+                this.emit("request", "deleteObject", options);
+                return response;
+            }
+            catch (error) {
+                this.emit("error", error);
+                if (error instanceof Error) {
+                    if (error.name === "NoSuchKey") {
+                        return Promise.reject(new errors_1.ClientNoSuchKey({ bucket: this.bucket, key }));
+                    }
+                }
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Proxy to AWS S3's deleteObjects
+     * @param {Object} param
+     * @param {string} param.keys
+     * @returns
+     */
+    deleteObjects(keys) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const packages = (0, lodash_1.chunk)(keys, 1000);
+            const { results, errors } = yield promise_pool_1.default.for(packages)
+                .withConcurrency(this.parallelism)
+                .process((keys) => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    const options = {
+                        Bucket: this.bucket,
+                        Delete: {
+                            Objects: keys.map((key) => ({
+                                Key: path.join(this.keyPrefix, key),
+                            })),
+                        },
+                    };
+                    const response = yield this.client.deleteObjects(options).promise();
+                    this.emit("request", "deleteObjects", options);
+                    return response;
+                }
+                catch (error) {
+                    this.emit("error", error);
+                    return Promise.reject(error);
+                }
+            }));
+            return {
+                deleted: results,
+                notFound: errors,
+            };
+        });
+    }
+    /**
+     *
+     * @param param0
+     * @returns
+     */
+    listObjects({ prefix, maxKeys = 1000, continuationToken, }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const options = {
+                    Bucket: this.bucket,
+                    MaxKeys: maxKeys,
+                    ContinuationToken: continuationToken,
+                    Prefix: path.join(this.keyPrefix, prefix || ""),
+                };
+                const response = yield this.client.listObjectsV2(options).promise();
+                this.emit("request", "listObjectsV2", options);
+                return response;
+            }
+            catch (error) {
+                this.emit("error", error);
+                return Promise.reject(error);
+            }
+        });
+    }
+    count({ prefix } = {}) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.emit("request", "count", { prefix });
+            let count = 0;
+            let truncated = true;
+            let continuationToken;
+            while (truncated) {
+                const options = {
+                    prefix,
+                    continuationToken,
+                };
+                const res = yield this.listObjects(options);
+                count += res.KeyCount || 0;
+                truncated = res.IsTruncated || false;
+                continuationToken = res.NextContinuationToken;
+            }
+            return count;
+        });
+    }
+    getAllKeys({ prefix } = {}) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.emit("request", "getAllKeys", { prefix });
+            let keys = [];
+            let truncated = true;
+            let continuationToken;
+            while (truncated) {
+                const options = {
+                    prefix,
+                    continuationToken,
+                };
+                const res = yield this.listObjects(options);
+                if (res.Contents) {
+                    keys = keys.concat(res.Contents.map((x) => x.Key));
+                }
+                truncated = res.IsTruncated || false;
+                continuationToken = res.NextContinuationToken;
+            }
+            return keys
+                .map((x) => x.replace(this.keyPrefix, ""))
+                .map((x) => (x.startsWith("/") ? x.replace(`/`, "") : x));
+        });
+    }
+}
+exports.default = S3Client;

package/build/s3db-config.interface.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/build/s3db.class.js

@@ -0,0 +1,185 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const flat_1 = require("flat");
+const lodash_1 = require("lodash");
+const events_1 = __importDefault(require("events"));
+const resource_class_1 = __importDefault(require("./resource.class"));
+const s3_client_class_1 = __importDefault(require("./s3-client.class"));
+const validator_1 = require("./validator");
+const errors_1 = require("./errors");
+class S3db extends events_1.default {
+    /**
+     * Constructor
+     */
+    constructor(options) {
+        super();
+        this.keyPrefix = "";
+        this.bucket = "s3db";
+        this.cache = false;
+        this.version = "1";
+        this.resources = {};
+        this.options = options;
+        this.parallelism = parseInt(options.parallelism + "") || 10;
+        this.plugins = options.plugins || [];
+        this.cache = options.cache;
+        this.passphrase = options.passphrase || "";
+        this.validatorInstance = (0, validator_1.ValidatorFactory)({
+            passphrase: options === null || options === void 0 ? void 0 : options.passphrase,
+        });
+        this.client = new s3_client_class_1.default({
+            connectionString: options.uri,
+            parallelism: this.parallelism,
+        });
+        this.bucket = this.client.bucket;
+        this.keyPrefix = this.client.keyPrefix;
+        this.startPlugins();
+    }
+    /**
+     * Remotely setups s3db file.
+     */
+    connect() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let metadata = null;
+            try {
+                metadata = yield this.getMetadataFile();
+            }
+            catch (error) {
+                if (error instanceof errors_1.S3dbMissingMetadata) {
+                    metadata = this.blankMetadataStructure();
+                    yield this.uploadMetadataFile();
+                }
+                else {
+                    this.emit("error", error);
+                    return Promise.reject(error);
+                }
+            }
+            for (const resource of Object.entries(metadata.resources)) {
+                const [name, definition] = resource;
+                this.resources[name] = new resource_class_1.default({
+                    name,
+                    s3db: this,
+                    s3Client: this.client,
+                    schema: definition.schema,
+                    options: definition.options,
+                    validatorInstance: this.validatorInstance,
+                });
+            }
+            this.emit("connected", new Date());
+        });
+    }
+    startPlugins() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.plugins && !(0, lodash_1.isEmpty)(this.plugins)) {
+                const startProms = this.plugins.map((plugin) => plugin.setup(this));
+                yield Promise.all(startProms);
+                this.plugins.map((plugin) => plugin.start());
+            }
+        });
+    }
+    /**
+     * Downloads current metadata.
+     * If there isnt any file, creates an empty metadata.
+     * @returns MetadataInterface
+     */
+    getMetadataFile() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const request = yield this.client.getObject({ key: `s3db.json` });
+                const metadata = JSON.parse(String(request === null || request === void 0 ? void 0 : request.Body));
+                return this.unserializeMetadata(metadata);
+            }
+            catch (error) {
+                if (error instanceof errors_1.ClientNoSuchKey) {
+                    return Promise.reject(new errors_1.S3dbMissingMetadata({ bucket: this.bucket, cause: error }));
+                }
+                else {
+                    return Promise.reject(error);
+                }
+            }
+        });
+    }
+    unserializeMetadata(metadata) {
+        const file = Object.assign({}, metadata);
+        if ((0, lodash_1.isEmpty)(file.resources))
+            return file;
+        for (const [name, structure] of Object.entries(file.resources)) {
+            for (const [attr, value] of Object.entries(structure.schema)) {
+                file.resources[name].schema[attr] = JSON.parse(value);
+            }
+        }
+        return file;
+    }
+    uploadMetadataFile() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const file = {
+                version: this.version,
+                resources: Object.entries(this.resources).reduce((acc, definition) => {
+                    const [name, resource] = definition;
+                    acc[name] = resource.export();
+                    return acc;
+                }, {}),
+            };
+            yield this.client.putObject({
+                key: `s3db.json`,
+                body: JSON.stringify(file, null, 2),
+            });
+        });
+    }
+    /**
+     * Generates empty metadata structure.
+     * @returns MetadataInterface
+     */
+    blankMetadataStructure() {
+        return {
+            version: `1`,
+            resources: {},
+        };
+    }
+    /**
+     * Generates a new resorce with its translators and validatos.
+     * @param {Object} param
+     * @param {string} param.resourceName
+     * @param {Object} param.attributes
+     * @param {Object} param.options
+     */
+    createResource({ resourceName, attributes, options = {}, }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const schema = (0, flat_1.flatten)(attributes, { safe: true });
+            const resource = new resource_class_1.default({
+                schema,
+                s3db: this,
+                name: resourceName,
+                s3Client: this.client,
+                validatorInstance: this.validatorInstance,
+                options: Object.assign({ autoDecrypt: true, cache: this.cache }, options),
+            });
+            this.resources[resourceName] = resource;
+            yield this.uploadMetadataFile();
+            return resource;
+        });
+    }
+    /**
+     * Looper
+     * @param {string} resourceName
+     * @returns
+     */
+    resource(resourceName) {
+        if (!this.resources[resourceName]) {
+            return Promise.reject(`resource ${resourceName} does not exist`);
+        }
+        return this.resources[resourceName];
+    }
+}
+exports.default = S3db;

package/build/stream/resource-ids-read-stream.class.js

@@ -0,0 +1,100 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = __importStar(require("path"));
+const lodash_1 = require("lodash");
+const node_stream_1 = require("node:stream");
+const promise_pool_1 = require("@supercharge/promise-pool");
+class ResourceIdsReadStream extends node_stream_1.Readable {
+    constructor({ resource }) {
+        super({
+            objectMode: true,
+            highWaterMark: resource.s3Client.parallelism * 3,
+        });
+        this.resource = resource;
+        this.pagesCount = 0;
+        this.content = [];
+        this.finishedReadingResource = false;
+        this.loading = this.getItems();
+    }
+    _read(size) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.content.length === 0) {
+                if (this.loading) {
+                    yield this.loading;
+                }
+                else if (this.finishedReadingResource) {
+                    this.push(null);
+                    return;
+                }
+            }
+            const data = this.content.shift();
+            this.push(data);
+        });
+    }
+    getItems({ continuationToken = null, } = {}) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.emit("page", this.pagesCount++);
+            const res = yield this.resource.s3Client.listObjects({
+                prefix: `resource=${this.resource.name}`,
+                continuationToken,
+            });
+            if (res.Contents) {
+                const contents = (0, lodash_1.chunk)(res.Contents, this.resource.s3Client.parallelism);
+                yield promise_pool_1.PromisePool.for(contents)
+                    .withConcurrency(5)
+                    .handleError((error, content) => __awaiter(this, void 0, void 0, function* () {
+                    this.emit("error", error, content);
+                }))
+                    .process((pkg) => {
+                    const ids = pkg.map((obj) => {
+                        return (obj.Key || "").replace(path.join(this.resource.s3Client.keyPrefix, `resource=${this.resource.name}`, "id="), "");
+                    });
+                    this.content.push(ids);
+                    ids.forEach((id) => this.emit("id", this.resource.name, id));
+                });
+            }
+            this.finishedReadingResource = !res.IsTruncated;
+            if (res.NextContinuationToken) {
+                this.loading = this.getItems({
+                    continuationToken: res.NextContinuationToken,
+                });
+            }
+            else {
+                this.loading = null;
+            }
+        });
+    }
+}
+exports.default = ResourceIdsReadStream;

package/build/stream/resource-ids-transformer.class.js

@@ -0,0 +1,40 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const lodash_1 = require("lodash");
+const promise_pool_1 = require("@supercharge/promise-pool");
+const node_stream_1 = require("node:stream");
+class ResourceIdsToDataTransformer extends node_stream_1.Transform {
+    constructor({ resource }) {
+        super({ objectMode: true, highWaterMark: resource.s3Client.parallelism * 2 });
+        this.resource = resource;
+    }
+    _transform(chunk, encoding, callback) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!(0, lodash_1.isArray)(chunk))
+                this.push(null);
+            this.emit("page", chunk);
+            yield promise_pool_1.PromisePool.for(chunk)
+                .withConcurrency(this.resource.s3Client.parallelism)
+                .handleError((error, content) => __awaiter(this, void 0, void 0, function* () {
+                this.emit("error", error, content);
+            }))
+                .process((id) => __awaiter(this, void 0, void 0, function* () {
+                this.emit("id", id);
+                const data = yield this.resource.getById(id);
+                this.push(data);
+                return data;
+            }));
+            callback(null);
+        });
+    }
+}
+exports.default = ResourceIdsToDataTransformer;

package/build/stream/resource-write-stream.class.js

@@ -0,0 +1,76 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const lodash_1 = require("lodash");
+const node_stream_1 = require("node:stream");
+class ResourceWriteStream extends node_stream_1.Writable {
+    constructor({ resource }) {
+        super({ objectMode: true, highWaterMark: resource.s3Client.parallelism * 2 });
+        this.resource = resource;
+        this.contents = [];
+        this.running = null;
+        this.receivedFinalMessage = false;
+    }
+    _write(chunk, encoding, callback) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.running)
+                yield this.running;
+            if (!(0, lodash_1.isEmpty)(chunk)) {
+                this.contents.push(chunk);
+            }
+            else {
+                this.receivedFinalMessage = true;
+            }
+            this.running = this.writeOrWait();
+            return callback(null);
+        });
+    }
+    _writev(chunks, callback) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.running)
+                yield this.running;
+            if (!(0, lodash_1.isEmpty)(chunks)) {
+                for (const obj of chunks.map((c) => c.chunk)) {
+                    this.contents.push(obj);
+                }
+            }
+            else {
+                this.receivedFinalMessage = true;
+            }
+            this.running = this.writeOrWait();
+            return callback(null);
+        });
+    }
+    writeOrWait() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.receivedFinalMessage) {
+                const data = this.contents.splice(0, this.contents.length - 1);
+                yield this.resource.bulkInsert(data);
+                this.emit("end");
+                return;
+            }
+            if (this.contents.length < this.resource.s3Client.parallelism)
+                return;
+            const objs = this.contents.splice(0, this.resource.s3Client.parallelism);
+            objs.forEach((obj) => this.emit("id", obj.id));
+            yield this.resource.bulkInsert(objs);
+            objs.forEach((obj) => this.emit("data", obj));
+        });
+    }
+    _final(callback) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.receivedFinalMessage = true;
+            yield this.writeOrWait();
+            callback(null);
+        });
+    }
+}
+exports.default = ResourceWriteStream;