s3db.js 1.0.0

package/src/s3-client.class.ts

@@ -0,0 +1,297 @@
+import * as path from "path";
+import { chunk } from "lodash";
+import { nanoid } from "nanoid";
+import { Stream } from "stream";
+import EventEmitter from "events";
+import { S3, Credentials } from "aws-sdk";
+import PromisePool from "@supercharge/promise-pool";
+
+import { ClientNoSuchKey } from "./errors";
+
+export default class S3Client extends EventEmitter {
+  id: string;
+  client: S3;
+  bucket: string;
+  keyPrefix: string;
+  parallelism: number;
+
+  constructor({
+    connectionString,
+    parallelism = 10,
+    AwsS3,
+  }: {
+    connectionString: string;
+    parallelism?: number;
+    AwsS3?: S3;
+  }) {
+    super();
+    this.id = nanoid(7);
+
+    const uri = new URL(connectionString);
+    this.bucket = uri.hostname;
+    this.parallelism = parallelism;
+    
+    let [, ...subpath] = uri.pathname.split("/");
+    this.keyPrefix = [...(subpath || [])].join("/");
+
+    this.client =
+      AwsS3 ||
+      new S3({
+        credentials: new Credentials({
+          accessKeyId: uri.username,
+          secretAccessKey: uri.password,
+        }),
+      });
+  }
+
+  /**
+   *
+   * @param param0
+   * @returns
+   */
+  async getObject({ key }: { key: string }) {
+    try {
+      const options = {
+        Bucket: this.bucket,
+        Key: path.join(this.keyPrefix, key),
+      };
+
+      const response = await this.client.getObject(options).promise();
+      this.emit("request", "getObject", options);
+
+      return response;
+    } catch (error: unknown) {
+      if (error instanceof Error) {
+        if (error.name === "NoSuchKey") {
+          return Promise.reject(
+            new ClientNoSuchKey({ bucket: this.bucket, key })
+          );
+        }
+      }
+
+      return Promise.reject(error);
+    }
+  }
+
+  /**
+   *
+   * @param param0
+   * @returns
+   */
+  async putObject({
+    key,
+    metadata,
+    contentType,
+    body,
+    contentEncoding,
+  }: {
+    key: string;
+    metadata?: object;
+    contentType?: string;
+    body: string | Stream | Uint8Array;
+    contentEncoding?: string | null | undefined;
+  }) {
+    try {
+      const options: any = {
+        Bucket: this.bucket,
+        Key: path.join(this.keyPrefix, key),
+        Metadata: { ...metadata },
+        Body: body,
+        ContentType: contentType,
+        ContentEncoding: contentEncoding,
+      };
+
+      const response = await this.client.putObject(options).promise();
+      this.emit("request", "putObject", options);
+
+      return response;
+    } catch (error) {
+      this.emit("error", error);
+      return Promise.reject(error);
+    }
+  }
+
+  /**
+   * Proxy to AWS S3's headObject
+   * @param {Object} param
+   * @param {string} param.key
+   * @returns
+   */
+  async headObject({ key }: { key: string }) {
+    try {
+      const options: any = {
+        Bucket: this.bucket,
+        Key: path.join(this.keyPrefix, key),
+      };
+
+      const response = await this.client.headObject(options).promise();
+      this.emit("request", "headObject", options);
+
+      return response;
+    } catch (error: unknown) {
+      if (error instanceof Error) {
+        if (error.name === "NoSuchKey" || error.name === "NotFound") {
+          return Promise.reject(
+            new ClientNoSuchKey({ bucket: this.bucket, key })
+          );
+        }
+      }
+
+      this.emit("error", error);
+      return Promise.reject(error);
+    }
+  }
+
+  /**
+   * Proxy to AWS S3's deleteObject
+   * @param {Object} param
+   * @param {string} param.key
+   * @returns
+   */
+  async deleteObject(key: string) {
+    try {
+      const options: any = {
+        Bucket: this.bucket,
+        Key: path.join(this.keyPrefix, key),
+      };
+
+      const response = await this.client.deleteObject(options).promise();
+      this.emit("request", "deleteObject", options);
+
+      return response;
+    } catch (error: unknown) {
+      this.emit("error", error);
+
+      if (error instanceof Error) {
+        if (error.name === "NoSuchKey") {
+          return Promise.reject(
+            new ClientNoSuchKey({ bucket: this.bucket, key })
+          );
+        }
+      }
+
+      return Promise.reject(error);
+    }
+  }
+
+  /**
+   * Proxy to AWS S3's deleteObjects
+   * @param {Object} param
+   * @param {string} param.keys
+   * @returns
+   */
+  async deleteObjects(keys: string[]) {
+    const packages = chunk(keys, 1000);
+
+    const { results, errors } = await PromisePool.for(packages)
+      .withConcurrency(this.parallelism)
+      .process(async (keys: string[]) => {
+        try {
+          const options = {
+            Bucket: this.bucket,
+            Delete: {
+              Objects: keys.map((key) => ({
+                Key: path.join(this.keyPrefix, key),
+              })),
+            },
+          };
+
+          const response = await this.client.deleteObjects(options).promise();
+          this.emit("request", "deleteObjects", options);
+
+          return response;
+        } catch (error: unknown) {
+          this.emit("error", error);
+          return Promise.reject(error);
+        }
+      });
+
+    return {
+      deleted: results,
+      notFound: errors,
+    };
+  }
+
+  /**
+   *
+   * @param param0
+   * @returns
+   */
+  async listObjects({
+    prefix,
+    maxKeys = 1000,
+    continuationToken,
+  }: {
+    prefix?: string;
+    maxKeys?: number;
+    continuationToken: any;
+  }): Promise<S3.ListObjectsV2Output> {
+    try {
+      const options = {
+        Bucket: this.bucket,
+        MaxKeys: maxKeys,
+        ContinuationToken: continuationToken,
+        Prefix: path.join(this.keyPrefix, prefix || ""),
+      };
+
+      const response = await this.client.listObjectsV2(options).promise();
+      this.emit("request", "listObjectsV2", options);
+
+      return response;
+    } catch (error: unknown) {
+      this.emit("error", error);
+      return Promise.reject(error);
+    }
+  }
+
+  async count({ prefix }: { prefix?: string } = {}) {
+    this.emit("request", "count", { prefix });
+
+    let count = 0;
+    let truncated = true;
+    let continuationToken;
+
+    while (truncated) {
+      const options = {
+        prefix,
+        continuationToken,
+      };
+
+      const res: S3.ListObjectsV2Output = await this.listObjects(options);
+
+      count += res.KeyCount || 0;
+      truncated = res.IsTruncated || false;
+      continuationToken = res.NextContinuationToken;
+    }
+
+    return count;
+  }
+
+  async getAllKeys({ prefix }: { prefix?: string } = {}) {
+    this.emit("request", "getAllKeys", { prefix });
+
+    let keys: any[] = [];
+    let truncated = true;
+    let continuationToken;
+
+    while (truncated) {
+      const options: any = {
+        prefix,
+        continuationToken,
+      };
+
+      const res = await this.listObjects(options);
+
+      if (res.Contents) {
+        keys = keys.concat(res.Contents.map((x) => x.Key));
+      }
+
+      truncated = res.IsTruncated || false;
+      continuationToken = res.NextContinuationToken;
+    }
+
+    return keys
+      .map((x) => x.replace(this.keyPrefix, ""))
+      .map((x) => (x.startsWith("/") ? x.replace(`/`, "") : x));
+  }
+}

package/src/s3db-config.interface.ts

@@ -0,0 +1,9 @@
+import PluginInterface from "./plugin.interface";
+
+export default interface S3dbConfigInterface {
+  uri: string;
+  cache?: boolean;
+  parallelism?: number;
+  plugins?: PluginInterface[];
+  passphrase?: string | undefined;
+}

package/src/s3db.class.ts

@@ -0,0 +1,215 @@
+import { flatten } from "flat";
+import { isEmpty } from "lodash";
+import EventEmitter from "events";
+
+import Resource from "./resource.class";
+import S3Client from "./s3-client.class";
+import { ValidatorFactory } from "./validator";
+import PluginInterface from "./plugin.interface";
+import S3dbConfigInterface from "./s3db-config.interface";
+import MetadataInterface from "./metadata.interface";
+import { S3dbMissingMetadata, ClientNoSuchKey } from "./errors";
+import { MetadataResourceInterface } from "./resource.interface";
+
+export default class S3db extends EventEmitter {
+  options: S3dbConfigInterface;
+  client: S3Client;
+  keyPrefix: string = "";
+  bucket: string = "s3db";
+  version: string;
+  validatorInstance: any;
+  parallelism: number;
+  resources: any;
+  passphrase: string;
+  plugins: PluginInterface[];
+  cache: boolean | undefined = false;
+
+  /**
+   * Constructor
+   */
+  constructor(options: S3dbConfigInterface) {
+    super();
+
+    this.version = "1";
+    this.resources = {};
+    this.options = options;
+    this.parallelism = parseInt(options.parallelism + "") || 10;
+    this.plugins = options.plugins || [];
+    this.cache = options.cache;
+    this.passphrase = options.passphrase || ""
+
+    this.validatorInstance = ValidatorFactory({
+      passphrase: options?.passphrase,
+    });
+
+    this.client = new S3Client({
+      connectionString: options.uri,
+      parallelism: this.parallelism,
+    });
+
+    this.bucket = this.client.bucket;
+    this.keyPrefix = this.client.keyPrefix;
+
+    this.startPlugins();
+  }
+
+  /**
+   * Remotely setups s3db file.
+   */
+  async connect(): Promise<void> {
+    let metadata = null;
+
+    try {
+      metadata = await this.getMetadataFile();
+    } catch (error) {
+      if (error instanceof S3dbMissingMetadata) {
+        metadata = this.blankMetadataStructure();
+        await this.uploadMetadataFile();
+      } else {
+        this.emit("error", error);
+        return Promise.reject(error);
+      }
+    }
+
+    for (const resource of Object.entries(metadata.resources)) {
+      const [name, definition]: [string, any] = resource;
+
+      this.resources[name] = new Resource({
+        name,
+        s3db: this,
+        s3Client: this.client,
+        schema: definition.schema,
+        options: definition.options,
+        validatorInstance: this.validatorInstance,
+      });
+    }
+
+    this.emit("connected", new Date());
+  }
+
+  async startPlugins() {
+    if (this.plugins && !isEmpty(this.plugins)) {
+      const startProms = this.plugins.map((plugin) => plugin.setup(this));
+      await Promise.all(startProms);
+      this.plugins.map((plugin) => plugin.start());
+    }
+  }
+
+  /**
+   * Downloads current metadata.
+   * If there isnt any file, creates an empty metadata.
+   * @returns MetadataInterface
+   */
+  private async getMetadataFile() {
+    try {
+      const request = await this.client.getObject({ key: `s3db.json` });
+      const metadata = JSON.parse(String(request?.Body));
+      return this.unserializeMetadata(metadata);
+    } catch (error: unknown) {
+      if (error instanceof ClientNoSuchKey) {
+        return Promise.reject(
+          new S3dbMissingMetadata({ bucket: this.bucket, cause: error })
+        );
+      } else {
+        return Promise.reject(error);
+      }
+    }
+  }
+
+  private unserializeMetadata(metadata: any) {
+    const file = { ...metadata };
+    if (isEmpty(file.resources)) return file;
+
+    for (const [name, structure] of Object.entries(
+      file.resources as MetadataResourceInterface[]
+    )) {
+      for (const [attr, value] of Object.entries(structure.schema)) {
+        file.resources[name].schema[attr] = JSON.parse(value as any);
+      }
+    }
+
+    return file;
+  }
+
+  async uploadMetadataFile() {
+    const file = {
+      version: this.version,
+      resources: Object.entries(this.resources).reduce(
+        (acc: any, definition) => {
+          const [name, resource] = definition;
+          acc[name] = (resource as Resource).export();
+          return acc;
+        },
+        {}
+      ),
+    };
+
+    await this.client.putObject({
+      key: `s3db.json`,
+      body: JSON.stringify(file, null, 2),
+    });
+  }
+
+  /**
+   * Generates empty metadata structure.
+   * @returns MetadataInterface
+   */
+  private blankMetadataStructure(): MetadataInterface {
+    return {
+      version: `1`,
+      resources: {},
+    };
+  }
+
+  /**
+   * Generates a new resorce with its translators and validatos.
+   * @param {Object} param
+   * @param {string} param.resourceName
+   * @param {Object} param.attributes
+   * @param {Object} param.options
+   */
+  async createResource({
+    resourceName,
+    attributes,
+    options = {},
+  }: {
+    resourceName: string;
+    attributes: any;
+    options?: any;
+  }) {
+    const schema: any = flatten(attributes, { safe: true });
+
+    const resource = new Resource({
+      schema,
+      s3db: this,
+      name: resourceName,
+      s3Client: this.client,
+      validatorInstance: this.validatorInstance,
+
+      options: {
+        autoDecrypt: true,
+        cache: this.cache,
+        ...options,
+      },
+    });
+
+    this.resources[resourceName] = resource;
+
+    await this.uploadMetadataFile();
+
+    return resource;
+  }
+
+  /**
+   * Looper
+   * @param {string} resourceName
+   * @returns
+   */
+  resource(resourceName: string): Resource | any {
+    if (!this.resources[resourceName]) {
+      return Promise.reject(`resource ${resourceName} does not exist`);
+    }
+
+    return this.resources[resourceName];
+  }
+}

package/src/stream/resource-ids-read-stream.class.ts

@@ -0,0 +1,90 @@
+import * as path from "path";
+import { S3 } from "aws-sdk";
+import { chunk } from "lodash";
+import { Readable } from "node:stream";
+import { PromisePool } from "@supercharge/promise-pool";
+
+import Resource from "../resource.class";
+
+export default class ResourceIdsReadStream extends Readable {
+  resource: Resource;
+  finishedReadingResource: boolean;
+  content: any[];
+  loading: Promise<void> | null;
+  pagesCount: number;
+
+  constructor({ resource }: { resource: Resource }) {
+    super({
+      objectMode: true,
+      highWaterMark: resource.s3Client.parallelism * 3,
+    });
+
+    this.resource = resource;
+    this.pagesCount = 0;
+    this.content = [];
+    this.finishedReadingResource = false;
+    this.loading = this.getItems();
+  }
+
+  async _read(size: number): Promise<void> {
+    if (this.content.length === 0) {
+      if (this.loading) {
+        await this.loading;
+      } else if (this.finishedReadingResource) {
+        this.push(null);
+        return;
+      }
+    }
+
+    const data = this.content.shift();
+    this.push(data);
+  }
+
+  async getItems({
+    continuationToken = null,
+  }: {
+    continuationToken?: string | null;
+  } = {}) {
+    this.emit("page", this.pagesCount++);
+
+    const res: S3.ListObjectsV2Output = await this.resource.s3Client.listObjects({
+      prefix: `resource=${this.resource.name}`,
+      continuationToken,
+    });
+
+    if (res.Contents) {
+      const contents = chunk(res.Contents, this.resource.s3Client.parallelism);
+
+      await PromisePool.for(contents)
+        .withConcurrency(5)
+        .handleError(async (error, content) => {
+          this.emit("error", error, content);
+        })
+        .process((pkg: any[]) => {
+          const ids = pkg.map((obj) => {
+            return (obj.Key || "").replace(
+              path.join(
+                this.resource.s3Client.keyPrefix,
+                `resource=${this.resource.name}`,
+                "id="
+              ),
+              ""
+            );
+          });
+
+          this.content.push(ids);
+          ids.forEach((id: string) => this.emit("id", this.resource.name, id));
+        });
+    }
+
+    this.finishedReadingResource = !res.IsTruncated;
+
+    if (res.NextContinuationToken) {
+      this.loading = this.getItems({
+        continuationToken: res.NextContinuationToken,
+      });
+    } else {
+      this.loading = null;
+    }
+  }
+}

package/src/stream/resource-ids-transformer.class.ts

@@ -0,0 +1,38 @@
+import { isArray } from "lodash";
+import { PromisePool } from "@supercharge/promise-pool";
+import { Transform, TransformCallback } from "node:stream";
+
+import Resource from "../resource.class";
+
+export default class ResourceIdsToDataTransformer extends Transform {
+  resource: Resource;
+
+  constructor({ resource }: { resource: Resource }) {
+    super({ objectMode: true, highWaterMark: resource.s3Client.parallelism * 2 });
+
+    this.resource = resource;
+  }
+
+  async _transform(
+    chunk: any,
+    encoding: BufferEncoding,
+    callback: TransformCallback
+  ): Promise<void> {
+    if (!isArray(chunk)) this.push(null);
+    this.emit("page", chunk);
+
+    await PromisePool.for(chunk)
+      .withConcurrency(this.resource.s3Client.parallelism)
+      .handleError(async (error, content) => {
+        this.emit("error", error, content);
+      })
+      .process(async (id: any) => {
+        this.emit("id",  id);
+        const data = await this.resource.getById(id);
+        this.push(data);
+        return data;
+      });
+
+    callback(null);
+  }
+}