s3db.js 6.1.0 → 7.0.0

package/src/stream/index.js

@@ -0,0 +1,16 @@
+export * from "./resource-reader.class.js"
+export * from "./resource-writer.class.js"
+export * from "./resource-ids-reader.class.js"
+export * from "./resource-ids-page-reader.class.js"
+
+export function streamToString(stream) {
+  return new Promise((resolve, reject) => {
+    if (!stream) {
+      return reject(new Error('streamToString: stream is undefined'));
+    }
+    const chunks = [];
+    stream.on('data', (chunk) => chunks.push(chunk));
+    stream.on('error', reject);
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+  });
+}

package/src/stream/resource-ids-page-reader.class.js

@@ -0,0 +1,10 @@
+import ResourceIdsReader from "./resource-ids-reader.class.js";
+
+export class ResourceIdsPageReader extends ResourceIdsReader {
+  enqueue(ids) {
+    this.controller.enqueue(ids)
+    this.emit("page", ids);
+  }
+}
+
+export default ResourceIdsPageReader

package/src/stream/resource-ids-reader.class.js

@@ -0,0 +1,63 @@
+import EventEmitter from "events";
+import { ReadableStream } from "node:stream/web";
+
+export class ResourceIdsReader extends EventEmitter {
+  constructor({ resource }) {
+    super()
+
+    this.resource = resource;
+    this.client = resource.client;
+
+    this.stream = new ReadableStream({
+      highWaterMark: this.client.parallelism * 3,
+      start: this._start.bind(this),
+      pull: this._pull.bind(this),
+      cancel: this._cancel.bind(this),
+    });
+  }
+
+  build () {
+    return this.stream.getReader();
+  }
+
+  async _start(controller) {
+    this.controller = controller;
+    this.continuationToken = null;
+    this.closeNextIteration = false;
+  }
+
+  async _pull(controller) {
+    if (this.closeNextIteration) {
+      controller.close();
+      return;
+    }
+
+    const response = await this.client.listObjects({
+      prefix: `resource=${this.resource.name}`,
+      continuationToken: this.continuationToken,
+    });
+
+    const keys = response?.Contents
+      .map((x) => x.Key)
+      .map((x) => x.replace(this.client.config.keyPrefix, ""))
+      .map((x) => (x.startsWith("/") ? x.replace(`/`, "") : x))
+      .map((x) => x.replace(`resource=${this.resource.name}/id=`, ""))
+
+    this.continuationToken = response.NextContinuationToken;
+    this.enqueue(keys);
+
+    if (!response.IsTruncated) this.closeNextIteration = true;
+  }
+
+  enqueue(ids) {
+    ids.forEach((key) => {
+      this.controller.enqueue(key)
+      this.emit("id", key);
+    });
+  }
+
+  _cancel(reason) {
+  }
+}
+
+export default ResourceIdsReader

package/src/stream/resource-reader.class.js

@@ -0,0 +1,81 @@
+import EventEmitter from "events";
+import { Transform } from "stream";
+import { PromisePool } from "@supercharge/promise-pool";
+
+import { ResourceIdsPageReader } from "./resource-ids-page-reader.class.js"
+import tryFn from "../concerns/try-fn.js";
+
+export class ResourceReader extends EventEmitter {
+  constructor({ resource, batchSize = 10, concurrency = 5 }) {
+    super()
+
+    if (!resource) {
+      throw new Error("Resource is required for ResourceReader");
+    }
+
+    this.resource = resource;
+    this.client = resource.client;
+    this.batchSize = batchSize;
+    this.concurrency = concurrency;
+    
+    this.input = new ResourceIdsPageReader({ resource: this.resource });
+
+    // Create a Node.js Transform stream instead of Web Stream
+    this.transform = new Transform({
+      objectMode: true,
+      transform: this._transform.bind(this)
+    });
+
+    // Set up event forwarding
+    this.input.on('data', (chunk) => {
+      this.transform.write(chunk);
+    });
+
+    this.input.on('end', () => {
+      this.transform.end();
+    });
+
+    this.input.on('error', (error) => {
+      this.emit('error', error);
+    });
+
+    // Forward transform events
+    this.transform.on('data', (data) => {
+      this.emit('data', data);
+    });
+
+    this.transform.on('end', () => {
+      this.emit('end');
+    });
+
+    this.transform.on('error', (error) => {
+      this.emit('error', error);
+    });
+  }
+
+  build() {
+    return this;
+  }
+
+  async _transform(chunk, encoding, callback) {
+    const [ok, err] = await tryFn(async () => {
+      await PromisePool.for(chunk)
+        .withConcurrency(this.concurrency)
+        .handleError(async (error, content) => {
+          this.emit("error", error, content);
+        })
+        .process(async (id) => {
+          const data = await this.resource.get(id);
+          this.push(data);
+          return data;
+        });
+    });
+    callback(err);
+  }
+
+  resume() {
+    this.input.resume();
+  }
+}
+
+export default ResourceReader;

package/src/stream/resource-writer.class.js

@@ -0,0 +1,92 @@
+import EventEmitter from "events";
+import { Writable } from 'stream';
+import { PromisePool } from '@supercharge/promise-pool';
+import tryFn from "../concerns/try-fn.js";
+
+export class ResourceWriter extends EventEmitter {
+  constructor({ resource, batchSize = 10, concurrency = 5 }) {
+    super()
+
+    this.resource = resource;
+    this.client = resource.client;
+    this.batchSize = batchSize;
+    this.concurrency = concurrency;
+    this.buffer = [];
+    this.writing = false;
+
+    // Create a Node.js Writable stream instead of Web Stream
+    this.writable = new Writable({
+      objectMode: true,
+      write: this._write.bind(this)
+    });
+
+    // Set up event forwarding
+    this.writable.on('finish', () => {
+      this.emit('finish');
+    });
+
+    this.writable.on('error', (error) => {
+      this.emit('error', error);
+    });
+  }
+
+  build() {
+    return this;
+  }
+
+  write(chunk) {
+    this.buffer.push(chunk);
+    this._maybeWrite().catch(error => {
+      this.emit('error', error);
+    });
+    return true;
+  }
+
+  end() {
+    this.ended = true;
+    this._maybeWrite().catch(error => {
+      this.emit('error', error);
+    });
+  }
+
+  async _maybeWrite() {
+    if (this.writing) return;
+    if (this.buffer.length === 0 && !this.ended) return;
+    this.writing = true;
+    while (this.buffer.length > 0) {
+      const batch = this.buffer.splice(0, this.batchSize);
+      const [ok, err] = await tryFn(async () => {
+        await PromisePool.for(batch)
+          .withConcurrency(this.concurrency)
+          .handleError(async (error, content) => {
+            this.emit("error", error, content);
+          })
+          .process(async (item) => {
+            const [ok, err, result] = await tryFn(async () => {
+              const res = await this.resource.insert(item);
+              return res;
+            });
+            if (!ok) {
+              this.emit('error', err, item);
+              return null;
+            }
+            return result;
+          });
+      });
+      if (!ok) {
+        this.emit('error', err);
+      }
+    }
+    this.writing = false;
+    if (this.ended) {
+      this.writable.emit('finish');
+    }
+  }
+
+  async _write(chunk, encoding, callback) {
+    // Not used, as we handle batching in write/end
+    callback();
+  }
+}
+
+export default ResourceWriter;

package/src/validator.class.js

@@ -0,0 +1,97 @@
+import { merge, isString } from "lodash-es";
+import FastestValidator from "fastest-validator";
+
+import { encrypt } from "./concerns/crypto.js";
+import tryFn, { tryFnSync } from "./concerns/try-fn.js";
+import { ValidationError } from "./errors.js";
+
+async function secretHandler (actual, errors, schema) {
+  if (!this.passphrase) {
+    errors.push(new ValidationError("Missing configuration for secrets encryption.", {
+      actual,
+      type: "encryptionKeyMissing",
+      suggestion: "Provide a passphrase for secret encryption."
+    }));
+    return actual;
+  }
+
+  const [ok, err, res] = await tryFn(() => encrypt(String(actual), this.passphrase));
+  if (ok) return res;
+  errors.push(new ValidationError("Problem encrypting secret.", {
+    actual,
+    type: "encryptionProblem",
+    error: err,
+    suggestion: "Check the passphrase and input value."
+  }));
+  return actual;
+}
+
+async function jsonHandler (actual, errors, schema) {
+  if (isString(actual)) return actual;
+  const [ok, err, json] = tryFnSync(() => JSON.stringify(actual));
+  if (!ok) throw new ValidationError("Failed to stringify JSON", { original: err, input: actual });
+  return json;
+}
+
+export class Validator extends FastestValidator {
+  constructor({ options, passphrase, autoEncrypt = true } = {}) {
+    super(merge({}, {
+      useNewCustomCheckerFunction: true,
+
+      messages: {
+        encryptionKeyMissing: "Missing configuration for secrets encryption.",
+        encryptionProblem: "Problem encrypting secret. Actual: {actual}. Error: {error}",
+      },
+
+      defaults: {
+        string: {
+          trim: true,
+        },
+        object: {
+          strict: "remove",
+        },
+        number: {
+          convert: true,
+        }
+      },
+    }, options))
+
+    this.passphrase = passphrase;
+    this.autoEncrypt = autoEncrypt;
+
+    this.alias('secret', {
+      type: "string",
+      custom: this.autoEncrypt ? secretHandler : undefined,
+      messages: {
+        string: "The '{field}' field must be a string.",
+        stringMin: "This secret '{field}' field length must be at least {expected} long.",
+      },
+    })
+
+    this.alias('secretAny', { 
+      type: "any" ,
+      custom: this.autoEncrypt ? secretHandler : undefined,
+    })
+
+    this.alias('secretNumber', { 
+      type: "number",
+      custom: this.autoEncrypt ? secretHandler : undefined,
+    })
+
+    this.alias('json', {
+      type: "any",
+      custom: this.autoEncrypt ? jsonHandler : undefined,
+    })
+  }
+}
+
+export const ValidatorManager = new Proxy(Validator, {
+  instance: null,
+
+  construct(target, args) {
+    if (!this.instance) this.instance = new target(...args);
+    return this.instance;
+  }
+})
+
+export default Validator;