s3db.js 1.0.0

package/src/cache/s3-cache.class.ts

@@ -0,0 +1,155 @@
+import zlib from "zlib";
+import * as path from "path";
+import { isString } from "lodash";
+
+import S3Client from "../s3-client.class";
+import Serializers from "./serializers.type";
+import { JsonSerializer } from "./json.serializer";
+import { AvroSerializer } from "./avro.serializer";
+
+export default class S3Cache {
+  serializers: any;
+  s3Client: S3Client;
+  compressData: boolean;
+  serializer: Serializers;
+
+  constructor({
+    s3Client,
+    compressData = true,
+    serializer = Serializers.json,
+  }: {
+    s3Client: S3Client;
+    compressData?: boolean;
+    serializer?: Serializers;
+  }) {
+    this.s3Client = s3Client;
+    this.serializer = serializer;
+    this.compressData = compressData;
+
+    this.serializers = {
+      [Serializers.json]: JsonSerializer,
+      [Serializers.avro]: AvroSerializer,
+    };
+  }
+
+  getKey({
+    params,
+    hashed = true,
+    additionalPrefix = "",
+  }: {
+    params?: any;
+    hashed?: boolean;
+    additionalPrefix?: string;
+  }) {
+    let filename: any = Object.keys(params || {})
+      .sort()
+      .map((x) => `${x}:${params[x]}`)
+      .join("|");
+
+    if (filename.length === 0) filename = `empty`;
+
+    if (additionalPrefix.length > 0) {
+      filename = additionalPrefix + `|` + filename;
+    }
+
+    if (hashed) {
+      filename = Buffer.from(filename)
+        .toString("base64")
+        .split("")
+        .reverse()
+        .join("");
+    }
+
+    filename = filename + "." + this.serializer;
+    if (this.compressData) filename += ".gz";
+
+    return path.join("cache", filename);
+  }
+
+  async _put({ key, data }: { key: string; data: any }) {
+    const lengthRaw = isString(data)
+      ? data.length
+      : JSON.stringify(data).length;
+
+    let body: string | Uint8Array = this.serialize({ data });
+    const lengthSerialized = body.length;
+
+    if (this.compressData) {
+      body = zlib.gzipSync(body);
+    }
+
+    const metadata = {
+      compressor: "zlib",
+      "client-id": this.s3Client.id,
+      serializer: String(this.serializer),
+      compressed: String(this.compressData),
+      "length-raw": String(lengthRaw),
+      "length-serialized": String(lengthSerialized),
+      "length-compressed": String(body.length),
+    };
+
+    return this.s3Client.putObject({
+      key,
+      body,
+      metadata,
+      contentEncoding: this.compressData ? "gzip" : null,
+      contentType: this.compressData
+        ? "application/gzip"
+        : `application/${this.serializer}`,
+    });
+  }
+
+  async _get({ key }: { key: string }) {
+    try {
+      const res = await this.s3Client.getObject({ key });
+
+      if (!res.Body) return "";
+      let content = res.Body;
+
+      if (res.Metadata) {
+        const { serializer, compressor, compressed } = res.Metadata;
+
+        if (["true", true].includes(compressed)) {
+          if (compressor === `zlib`) {
+            content = zlib.unzipSync(content as Buffer);
+          }
+        }
+
+        const { data } = this.serializers[serializer].unserialize(content);
+        return data;
+      }
+
+      return this.unserialize(content);
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.name !== "ClientNoSuchKey") {
+          return Promise.reject(error);
+        }
+      }
+    }
+
+    return null;
+  }
+
+  async _delete({ key }: { key: string }) {
+    try {
+      await this.s3Client.deleteObject(key);
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.name !== "ClientNoSuchKey") {
+          return Promise.reject(error);
+        }
+      }
+    }
+
+    return true;
+  }
+
+  serialize(data: any) {
+    return this.serializers[this.serializer].serialize(data);
+  }
+
+  unserialize(data: any) {
+    return this.serializers[this.serializer].unserialize(data);
+  }
+}

package/src/cache/s3-resource-cache.class.ts

@@ -0,0 +1,75 @@
+import S3Cache from "./s3-cache.class";
+import Resource from "../resource.class";
+import Serializers from "./serializers.type";
+
+export default class S3ResourceCache extends S3Cache {
+  resource: Resource;
+
+  constructor({
+    resource,
+    compressData = true,
+    serializer = Serializers.json,
+  }: {
+    resource: Resource;
+    compressData?: boolean;
+    serializer?: Serializers;
+  }) {
+    super({
+      s3Client: resource.s3Client,
+      compressData: compressData,
+      serializer: serializer,
+    });
+
+    this.resource = resource;
+  }
+
+  getKey({ action = "list", params }: { action?: string; params?: any }) {
+    return super.getKey({
+      params,
+      additionalPrefix: `${this.resource.name}|${action}`,
+    });
+  }
+
+  async put({
+    action = "list",
+    params,
+    data,
+  }: {
+    action?: string;
+    params?: any;
+    data: any;
+  }) {
+    return super._put({
+      data,
+      key: this.getKey({ action, params }),
+    });
+  }
+
+  async get({ action = "list", params }: { action?: string; params?: any }) {
+    return super._get({
+      key: this.getKey({ action, params }),
+    });
+  }
+
+  async delete({ action = "list", params }: { action?: string; params: any }) {
+    const key = this.getKey({ action, params });
+
+    return super._delete({
+      key: this.getKey({ action, params }),
+    });
+  }
+
+  async purge() {
+    let keys = await this.s3Client.getAllKeys({ prefix: "cache" });
+
+    const key = Buffer.from(this.resource.name)
+      .toString("base64")
+      .split("")
+      .reverse()
+      .join("");
+
+    keys = keys.filter((k) => k.includes(key));
+
+    await this.s3Client.deleteObjects(keys);
+  }
+}

package/src/cache/serializers.type.ts

@@ -0,0 +1,8 @@
+export const Serializers = {
+  json: "json",
+  avro: "avro",
+} as const
+
+export type Serializers = typeof Serializers[keyof typeof Serializers]
+
+export default Serializers

package/src/errors.ts

@@ -0,0 +1,96 @@
+// Errors interfaces
+export interface S3Error {
+  name: string;
+  message: string;
+  cause?: Error;
+}
+
+export interface S3dbError {
+  name: string;
+  message: string;
+  cause?: Error;
+}
+
+export class BaseError extends Error {
+  bucket: any;
+  thrownAt: Date;
+  cause: Error | undefined;
+
+  constructor({ bucket, message, cause }: { bucket: string; message: string, cause?: Error }) {
+    super(message);
+
+    if (typeof Error.captureStackTrace === 'function') {
+      Error.captureStackTrace(this, this.constructor);
+    } else { 
+      this.stack = (new Error(message)).stack; 
+    }
+
+    super.name = this.constructor.name;
+    this.name = this.constructor.name;
+    this.cause = cause
+    this.thrownAt = new Date();
+  }
+
+  toJson() {
+    return { ...this };
+  }
+
+  toString() {
+    return `${this.name} | ${this.message}`;
+  }
+}
+
+// AWS S3 errors
+export abstract class BaseS3Error extends BaseError implements S3Error {
+  constructor({ bucket, message }: { bucket: string; message: string }) {
+    super({ bucket, message });
+  }
+}
+
+export class ClientNoSuchKey extends BaseS3Error {
+  key: string;
+  constructor({ bucket, key }: { bucket: string; key: string }) {
+    super({ bucket, message: `Key does not exists [s3://${bucket}/${key}]` });
+    this.key = key;
+  }
+}
+
+// Our errors
+export abstract class BaseS3dbError extends BaseError implements S3dbError {
+  constructor({ bucket, message, cause }: { bucket: string; message: string, cause?: Error }) {
+    super({ bucket, message, cause });
+  }
+}
+
+export class S3dbMissingMetadata extends BaseS3dbError {
+  constructor({ bucket, cause }: { bucket: string, cause?: Error }) {
+    super({ bucket, cause, message: `Missing metadata for bucket [s3://${bucket}]` });
+  }
+}
+
+export class S3dbInvalidResource extends BaseS3dbError {
+  resourceName: any;
+  attributes: any;
+  validation: any;
+
+  constructor({
+    bucket,
+    resourceName,
+    attributes,
+    validation,
+  }: {
+    bucket: string;
+    resourceName: string;
+    attributes: string;
+    validation: any[];
+  }) {
+    super({
+      bucket,
+      message: `Resource is not valid. Name=${resourceName} [s3://${bucket}].\n${JSON.stringify(validation, null, 2)}`,
+    });
+
+    this.resourceName = resourceName;
+    this.attributes = attributes;
+    this.validation = validation;
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+import database from './s3db.class'
+
+export const S3db = database
+export default database

package/src/metadata.interface.ts

@@ -0,0 +1,4 @@
+export default interface MetadataInterface {
+  version: string;
+  resources: any;
+}

package/src/plugin.interface.ts

@@ -0,0 +1,4 @@
+export default interface PluginInterface {
+  setup: Function
+  start: Function
+}