s3db.js 1.0.0

package/build/validator.js

@@ -0,0 +1,37 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidatorFactory = exports.CustomValidator = void 0;
+const crypto_js_1 = __importDefault(require("crypto-js"));
+const fastest_validator_1 = __importDefault(require("fastest-validator"));
+class CustomValidator extends fastest_validator_1.default {
+    constructor(options, passphrase) {
+        super(options);
+        this.passphrase = passphrase;
+    }
+}
+exports.CustomValidator = CustomValidator;
+function ValidatorFactory({ passphrase }) {
+    let options = {
+        useNewCustomCheckerFunction: true,
+        defaults: {
+            object: {
+                strict: "remove",
+            },
+        },
+    };
+    const validator = new CustomValidator(options, passphrase);
+    validator.alias("secret", {
+        type: "string",
+        custom: (v) => {
+            if (!validator.passphrase)
+                throw new Error("No passphrase defined.");
+            const ciphertext = crypto_js_1.default.AES.encrypt(String(v), validator.passphrase);
+            return ciphertext.toString();
+        },
+    });
+    return validator;
+}
+exports.ValidatorFactory = ValidatorFactory;

package/examples/1-bulk-insert.js

@@ -0,0 +1,64 @@
+const { ENV, CostsPlugin, S3db } = require("./concerns");
+
+const { nanoid } = require("nanoid");
+const Fakerator = require("fakerator");
+const ProgressBar = require("progress");
+
+const TOTAL = 10
+
+async function main() {
+  const fake = Fakerator();
+
+  const s3db = new S3db({
+    uri: ENV.CONNECTION_STRING,
+    passphrase: ENV.PASSPRHASE,
+    parallelism: ENV.PARALLELISM,
+    plugins: [CostsPlugin],
+  });
+
+  console.log(`creating ${TOTAL} leads.`);
+  console.log(`parallelism of ${ENV.PARALLELISM} requests.\n`);
+
+  await s3db.connect();
+
+  const barItem = new ProgressBar(
+    "bulk-writing  :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds) [:requests requests]",
+    {
+      width: 30,
+      total: TOTAL,
+      incomplete: " ",
+    }
+  );
+
+  if (!s3db.resources.leads) {
+    await s3db.createResource({
+      resourceName: "leads",
+      attributes: {
+        name: "string",
+        email: "string",
+        token: "secret",
+      },
+    });
+  }
+
+  s3db.on("inserted", () =>
+    barItem.tick({ requests: s3db.client.costs.requests.total })
+  );
+
+  console.time("bulk-writing");
+
+  await s3db.resource("leads").bulkInsert(
+    new Array(TOTAL).fill(0).map((v, k) => ({
+      id: k,
+      name: fake.names.name(),
+      email: fake.internet.email(),
+      token: nanoid(),
+    }))
+  );
+
+  console.timeEnd("bulk-writing");
+  process.stdout.write("\n\n");
+  console.log("Total cost:", s3db.client.costs.total.toFixed(4), "USD");
+}
+
+main();

package/examples/2-read-stream.js

@@ -0,0 +1,61 @@
+const { ENV, CostsPlugin, S3db } = require("./concerns");
+
+const Multiprogress = require("multi-progress");
+
+async function main() {
+  const s3db = new S3db({
+    uri: ENV.CONNECTION_STRING,
+    passphrase: ENV.PASSPRHASE,
+    parallelism: ENV.PARALLELISM,
+    plugins: [CostsPlugin],
+  });
+  
+  await s3db.connect();
+  const total = await s3db.resource("leads").count();
+
+  console.log(`reading ${total} leads.`);
+  console.log(`parallelism of ${ENV.PARALLELISM} requests.\n`);
+  
+  const multi = new Multiprogress(process.stdout);
+  const options = {
+    total,
+    width: 30,
+    incomplete: " ",
+  };
+
+  const barPages = multi.newBar(
+    "reading-pages   :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    {
+      ...options,
+      total: 1,
+    }
+  );
+
+  const barIds = multi.newBar(
+    "reading-ids     :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    options
+  );
+
+  const barData = multi.newBar(
+    "reading-data    :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    options
+  );
+
+  const stream = s3db.resource("leads").readable();
+
+  console.time("reading");
+
+  stream.on("page", () => barPages.tick());
+  stream.on("id", () => barIds.tick());
+  stream.on("data", () => barData.tick());
+  stream.on("error", (err) => console.error(err));
+
+  stream.on("end", () => {
+    process.stdout.write("\n");
+    console.timeEnd("reading");
+    process.stdout.write("\n\n");
+    console.log("Total cost:", s3db.client.costs.total.toFixed(4), "USD");
+  });
+}
+
+main();

package/examples/3-read-stream-to-csv.js

@@ -0,0 +1,57 @@
+const { ENV, S3db } = require("./concerns");
+
+const fs = require("fs");
+const ProgressBar = require("progress");
+const { Transform } = require("stream");
+
+async function main() {
+  const s3db = new S3db({
+    uri: ENV.CONNECTION_STRING,
+    passphrase: ENV.PASSPRHASE,
+    parallelism: ENV.PARALLELISM,
+  });
+
+  await s3db.connect();
+  const total = await s3db.resource("leads").count();
+
+  console.log(`reading ${total} leads.`);
+  console.log(`parallelism of ${ENV.PARALLELISM} requests.\n`);
+
+  const barData = new ProgressBar(
+    "reading-data  :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    {
+      total,
+      width: 30,
+      incomplete: " ",
+    }
+  );
+
+  const filename = __dirname + "/tmp/leads." + Date.now() + ".csv";
+
+  const stream = await s3db.resource("leads").readable();
+  const streamWrite = fs.createWriteStream(filename);
+
+  const transformer = new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback) {
+      this.push(
+        [chunk.id, chunk.email, chunk.name, chunk.token].join(";") + "\n"
+      );
+      callback();
+    },
+  });
+
+  console.time("reading-data");
+  stream.on("data", () => barData.tick());
+
+  stream.on("end", () => {
+    console.timeEnd("reading-data");
+    process.stdout.write("\n");
+    const { size } = fs.statSync(filename);
+    console.log(`\nTotal size: ${(size / (1024 * 1000)).toFixed(2)} Mb`);
+  });
+
+  stream.pipe(transformer).pipe(streamWrite);
+}
+
+main();

package/examples/4-read-stream-to-zip.js

@@ -0,0 +1,56 @@
+const { ENV, S3db } = require("./concerns");
+
+const fs = require("fs");
+const zlib = require("node:zlib");
+const ProgressBar = require("progress");
+const { Transform } = require("node:stream");
+const { pipeline } = require("node:stream/promises");
+
+async function main() {
+  const s3db = new S3db({
+    uri: ENV.CONNECTION_STRING,
+    passphrase: ENV.PASSPRHASE,
+    parallelism: ENV.PARALLELISM,
+  });
+
+  await s3db.connect();
+  const total = await s3db.resource("leads").count();
+
+  console.log(`reading ${total} leads.`);
+  console.log(`parallelism of ${ENV.PARALLELISM} requests.\n`);
+
+  const barData = new ProgressBar(
+    "reading-data  :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    {
+      total,
+      width: 30,
+      incomplete: " ",
+    }
+  );
+
+  const filename = __dirname + "/tmp/leads." + Date.now() + ".csv.gzip";
+  const stream = await s3db.resource("leads").readable();
+  const streamWrite = fs.createWriteStream(filename);
+
+  const transformer = new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback) {
+      this.push([chunk.id, chunk.name, chunk.token].join(";") + "\n");
+      callback();
+    },
+  });
+
+  console.time("reading-data");
+
+  stream.on("data", () => barData.tick());
+  stream.on("end", () => {
+    console.timeEnd("reading-data");
+    process.stdout.write("\n");
+    const { size } = fs.statSync(filename);
+    console.log(`\nTotal zip size: ${(size / (1024 * 1000)).toFixed(2)} Mb`);
+  });
+
+  pipeline(stream, transformer, zlib.createGzip(), streamWrite);
+}
+
+main();

package/examples/5-write-stream.js

@@ -0,0 +1,98 @@
+const { ENV, S3db, CostsPlugin } = require("./concerns");
+
+const Multiprogress = require("multi-progress");
+const { pipeline } = require("stream");
+
+async function main() {
+  const s3db = new S3db({
+    uri: ENV.CONNECTION_STRING,
+    passphrase: ENV.PASSPRHASE,
+    parallelism: ENV.PARALLELISM,
+    plugins: [CostsPlugin],
+  });
+
+  await s3db.connect();
+
+  if (!s3db.resources.copyLeads) {
+    await s3db.createResource({
+      resourceName: "copy-leads",
+      attributes: {
+        name: "string",
+        email: "string",
+        token: "secret",
+      },
+    });
+  }
+
+  const total = await s3db.resource("leads").count();
+
+  console.log(`reading ${total} leads.`);
+  console.log(`parallelism of ${ENV.PARALLELISM} requests.\n`);
+
+  const multi = new Multiprogress(process.stdout);
+  const options = {
+    total,
+    width: 30,
+    incomplete: " ",
+  };
+
+  const requestsBar = multi.newBar(
+    "requests        :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    {
+      ...options,
+      total: 1,
+    }
+  );
+
+  const readPages = multi.newBar(
+    "reading-pages   :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    {
+      ...options,
+      total: 1,
+    }
+  );
+
+  const readIds = multi.newBar(
+    "reading-ids     :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    options
+  );
+
+  const readData = multi.newBar(
+    "reading-data    :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    options
+  );
+
+  const writeIds = multi.newBar(
+    "writing-ids     :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    options
+  );
+
+  const writeData = multi.newBar(
+    "writing-data    :current/:total (:percent)  [:bar]  :rate/bps  :etas (:elapseds)",
+    options
+  );
+
+  const readStream = s3db.resource("leads").readable();
+  const writeStream = s3db.resource("copy-leads").writable();
+
+  console.time("copying-data");
+  s3db.client.on("request", () => requestsBar.tick());
+
+  readStream.on("page", () => readPages.tick());
+  readStream.on("id", () => readIds.tick());
+  readStream.on("data", () => readData.tick());
+
+  writeStream.on("id", () => writeIds.tick());
+  writeStream.on("data", () => writeData.tick());
+
+  writeStream.on("end", () => {
+    process.stdout.write("\n");
+    console.timeEnd("copying-data");
+    process.stdout.write("\n\n");
+    console.log("Total cost:", s3db.client.costs.total.toFixed(4), "USD");
+  });
+
+  pipeline(readStream, writeStream, (err) => console.error(err));
+}
+
+main();

package/examples/6-jwt-tokens.js

@@ -0,0 +1,124 @@
+const { ENV, S3db } = require("./concerns");
+
+const jwt = require("jsonwebtoken");
+const { nanoid } = require("nanoid");
+const Fakerator = require("fakerator");
+const sha256 = require("crypto-js/sha256");
+const { take, shuffle } = require("lodash");
+
+const fake = Fakerator();
+
+const userFactory = () => {
+  const scopes = ["admin", "guest", "users:read", "tokens:read"];
+  const email = fake.internet.email();
+
+  return {
+    id: email,
+    email,
+    name: fake.names.name(),
+    password: nanoid(),
+    scopes: take(shuffle(scopes), fake.random.number(scopes.length)),
+  };
+};
+
+const Token = {
+  createToken: async (s3db, email, password) => {
+    const user = await s3db.resource("users").getById(email);
+
+    if (user.password !== password) {
+      console.log({ user, email, password });
+      throw new Error("invalid-user");
+    }
+
+    const data = {
+      name: user.name,
+      email: user.email,
+      scopes: user.scopes,
+      email_verified: true,
+    };
+
+    const token = jwt.sign(data, ENV.PASSPRHASE, {
+      expiresIn: "2s",
+      subject: "test",
+      issuer: "s3db.js",
+      audience: "default",
+    });
+
+    const decoded = jwt.decode(token, ENV.PASSPRHASE);
+
+    await s3db.resource("tokens").insert({
+      id: sha256(token).toString(),
+      ...decoded,
+    });
+
+    return token;
+  },
+
+  validateToken: async (client, token) => {
+    const tokenId = sha256(token).toString();
+
+    try {
+      const decoded = jwt.decode(token, ENV.PASSPRHASE);
+      const savedToken = await client.resource("tokens").getById(tokenId);
+
+      return [null, { decoded, savedToken }];
+    } catch (error) {
+      return [error];
+    }
+  },
+};
+
+async function main() {
+  const s3db = new S3db({
+    uri: ENV.CONNECTION_STRING + Date.now(),
+    passphrase: ENV.PASSPRHASE,
+    parallelism: ENV.PARALLELISM,
+  });
+
+  await s3db.connect();
+
+  await s3db.createResource({
+    resourceName: "users",
+    attributes: {
+      name: "string",
+      email: "string",
+      password: "secret",
+      scopes: "array|items:string",
+    },
+  });
+
+  await s3db.createResource({
+    resourceName: "tokens",
+    attributes: {
+      iss: ["string", "url"],
+      sub: "string",
+      aud: "string",
+      exp: "number",
+      email: "email",
+      name: "string",
+      email_verified: "boolean",
+      scopes: "array|items:string",
+    },
+  });
+
+  const users = new Array(5).fill(0).map(userFactory);
+  await s3db.resource("users").bulkInsert(users);
+
+  let tokens = [];
+  process.stdout.write("Created tokens: ");
+  for (const user of users) {
+    const token = await Token.createToken(s3db, user.email, user.password);
+    tokens.push(token);
+    process.stdout.write(".");
+  }
+
+  process.stdout.write("\nValidated tokens: ");
+  for (const token of tokens) {
+    const [error, data] = await Token.validateToken(s3db, token);
+    if (!error) {
+      process.stdout.write(".");
+    }
+  }
+}
+
+main();

package/examples/concerns/index.js

@@ -0,0 +1,64 @@
+require("dotenv").config({ path: `${process.cwd()}/../.env` });
+
+const { bucket, accessKeyId, secretAccessKey } = process.env;
+
+module.exports = {
+  ENV: {
+    PARALLELISM: 250,
+    PASSPRHASE: 'super-secret-leaked-fluffy-passphrase',
+    CONNECTION_STRING:
+      `s3://${accessKeyId}:${secretAccessKey}@${bucket}/databases/examples-` +
+      new Date().toISOString().substring(0, 10),
+  },
+
+  S3db: require("../../build").S3db,
+
+  CostsPlugin: {
+    async setup (s3db) {
+      this.client = s3db.client
+      
+      this.client.costs = {
+        total: 0,
+        
+        prices: {
+          put: 0.000005,
+          post: 0.000005,
+          copy: 0.000005,
+          list: 0.000005,
+          get: 0.0000004,
+          select: 0.0000004,
+          delete: 0.0000004,
+        },
+
+        requests: {
+          total: 0,
+          put: 0,
+          post: 0,
+          copy: 0,
+          list: 0,
+          get: 0,
+          select: 0,
+          delete: 0,
+        },
+      };
+    },
+
+    start () {
+      const addRequest = (req) => {
+        this.client.costs.requests[req]++;
+        this.client.costs.total += this.client.costs.prices[req];
+      };
+  
+      this.client.on("request", (name) => {
+        this.client.costs.requests.total++;
+  
+        if (name === "getObject") addRequest("get");
+        else if (name === "putObject") addRequest("put");
+        else if (name === "headObject") addRequest("get");
+        else if (name === "deleteObject") addRequest("delete");
+        else if (name === "deleteObjects") addRequest("delete");
+        else if (name === "listObjectsV2") addRequest("list");
+      });
+    }
+  },
+};

package/jest.config.ts

@@ -0,0 +1,10 @@
+import type { Config } from "@jest/types";
+
+const config: Config.InitialOptions = {
+  verbose: true,
+  transform: {
+    "^.+\\.tsx?$": "ts-jest",
+  },
+};
+
+export default config;

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "s3db.js",
+  "version": "1.0.0",
+  "description": "Use AWS S3 as a database",
+  "main": "build/index.js",
+  "scripts": {
+    "build": "rimraf ./build && tsc",
+    "test": "jest --coverage",
+    "test:cache": "jest --coverage ./tests/cache.spec.ts",
+    "example:1": "cd examples; node 1-bulk-insert.js",
+    "example:2": "cd examples; node 2-read-stream.js",
+    "example:3": "cd examples; node 3-read-stream-to-csv.js",
+    "example:4": "cd examples; node 4-read-stream-to-zip.js",
+    "example:5": "cd examples; node 5-write-stream.js",
+    "example:6": "cd examples; node 6-jwt-tokens.js",
+    "coverage": "npx http-server ./coverage/lcov-report"
+  },
+  "author": "filipeforattini",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/crypto-js": "^4.1.1",
+    "@types/flat": "^5.0.2",
+    "@types/jest": "^29.2.3",
+    "@types/lodash": "^4.14.190",
+    "@types/node": "^18.11.9",
+    "@types/pako": "^2.0.0",
+    "@types/progress": "^2.0.5",
+    "dotenv": "^16.0.3",
+    "esm": "^3.2.25",
+    "fakerator": "^0.3.6",
+    "jest": "^29.3.1",
+    "jsonwebtoken": "^8.5.1",
+    "multi-progress": "^4.0.0",
+    "progress": "^2.0.3",
+    "rimraf": "^3.0.2",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.3"
+  },
+  "dependencies": {
+    "@supercharge/promise-pool": "^2.3.2",
+    "avsc": "^5.7.7",
+    "aws-sdk": "^2.1261.0",
+    "crypto-js": "^4.1.1",
+    "fastest-validator": "^1.15.0",
+    "flat": "^5.0.2",
+    "lodash": "^4.17.21",
+    "nanoid": "3.3.4",
+    "ts-mixer": "^6.0.2"
+  }
+}

package/src/cache/avro.serializer.ts

@@ -0,0 +1,12 @@
+import avro from "avsc";
+
+export const CacheAvroSchema = avro.Type.forSchema({
+  name: "Cache",
+  type: "record",
+  fields: [{ name: "data", type: ["string"] }],
+});
+
+export const AvroSerializer = {
+  serialize: (data: any) => String(CacheAvroSchema.toBuffer(data)),
+  unserialize: (data: any) => CacheAvroSchema.fromBuffer(Buffer.from(data)),
+}

package/src/cache/json.serializer.ts

@@ -0,0 +1,4 @@
+export const JsonSerializer = {
+  serialize: (data: any) => JSON.stringify(data),
+  unserialize: (data: any) => JSON.parse(data),
+}