s3-test-server 2.0.0 → 2.1.0

package/README.md

@@ -2,18 +2,12 @@
 
 A lightweight Amazon S3 emulator for local development and testing.
 
-Zero runtime dependencies, a single small package, and wire-level
-compatibility with both old and new S3 clients:
-
-- **AWS SDK for JavaScript v3**, including flexible checksums
-  (CRC32, CRC32C, CRC64NVME, SHA1, SHA256), `aws-chunked` streaming uploads
-  with trailers, and full-object / composite multipart checksums
-- **AWS SDK for JavaScript v2**, SigV4 and legacy SigV2 (`signatureVersion: 's3'`)
-- **Any other S3 client** that signs with Signature V2 or V4, header-based or
-  presigned, path-style or virtual-host-style
-
-Data is stored on the filesystem (a temp directory by default), so it survives
-restarts when you point it at a directory of your own.
+Zero runtime dependencies and wire-level compatibility with old and new S3
+clients: AWS SDK for JavaScript v3 (flexible checksums, `aws-chunked`
+streaming, multipart), v2 (SigV4 and legacy SigV2), and any other client that
+signs with Signature V2/V4, header-based or presigned, path-style or
+virtual-host-style. Objects are stored on the filesystem and persist across
+restarts when you point `directory` at a folder of your own.
 
 ## Quick start
 
@@ -314,6 +308,33 @@ server.on('event', (notification) => {
 Emitted for `ObjectCreated:Put`, `ObjectCreated:Copy`, `ObjectCreated:Post`,
 `ObjectCreated:CompleteMultipartUpload` and `ObjectRemoved:Delete`.
 
+## Test helpers
+
+Three helpers make assertions and fixtures easy without building an S3
+client; they talk to the store directly:
+
+```js
+// seed fixtures (bucket is created if missing; no events are emitted)
+await server.putObjects('reports', [
+  { key: 'public/a.csv', body: 'a,b,c', contentType: 'text/csv' },
+  { key: 'internal/b.csv', body: Buffer.from('d,e,f') },
+]);
+
+// inspect bucket contents in assertions
+const all = await server.getObjects('reports');
+const csvs = await server.getObjects('reports', {
+  prefix: 'public/',
+  bodies: true, // each summary gets a `body` Buffer
+});
+
+// wait for background uploads from the code under test;
+// multipart uploads only count once completed
+const files = await server.waitForObjects('reports', 2, { timeout: 5000 });
+```
+
+After `run()` resolves, `server.hostPort` (`"localhost:4568"`) and
+`server.endpoint` (`"http://localhost:4568"`) are also set on the instance.
+
 ## Using as middleware
 
 ```js
@@ -332,6 +353,9 @@ http.createServer(s3.callback()).listen(4568);
 - `callback()` / `getMiddleware()` returns a request handler
 - `configureBuckets()` provisions `options.configureBuckets` without the API
 - `reset()` deletes all buckets and configurations
+- `getObjects(bucket, { prefix, bodies })` lists completed objects
+- `waitForObjects(bucket, count, { prefix, timeout })` awaits uploads
+- `putObjects(bucket, objects)` seeds fixtures without the HTTP API
 
 TypeScript definitions ship with the package (`lib/index.d.ts`).
 

package/lib/index.d.ts

@@ -11,6 +11,12 @@ declare class S3TestServer extends EventEmitter {
   /** The underlying HTTP(S) server, available after `run()` resolves. */
   httpServer: HttpServer | null;
 
+  /** `host:port` the server listens on; set once `run()` resolves. */
+  hostPort: string | null;
+
+  /** Full endpoint URL (e.g. `http://localhost:4568`); set once `run()` resolves. */
+  endpoint: string | null;
+
   /** Starts the server. */
   run(): Promise<S3TestServer.RunAddress>;
   run(
@@ -32,6 +38,34 @@ declare class S3TestServer extends EventEmitter {
   /** Deletes all buckets, objects and configurations in the store. */
   reset(): Promise<void>;
 
+  /**
+   * Lists the completed objects in a bucket, sorted by key. With
+   * `bodies: true`, each summary also carries its content as a Buffer.
+   */
+  getObjects(
+    bucket: string,
+    options?: { prefix?: string; bodies?: boolean }
+  ): Promise<S3TestServer.ObjectSummary[]>;
+
+  /**
+   * Resolves with the bucket's objects once at least `count` exist
+   * (optionally under `prefix`); rejects after `timeout` ms (default 10000).
+   */
+  waitForObjects(
+    bucket: string,
+    count: number,
+    options?: { prefix?: string; timeout?: number }
+  ): Promise<S3TestServer.ObjectSummary[]>;
+
+  /**
+   * Seeds objects directly into the store (the bucket is created when
+   * missing). No S3 event notifications are emitted for seeded objects.
+   */
+  putObjects(
+    bucket: string,
+    objects: S3TestServer.SeedObject[]
+  ): Promise<S3TestServer.ObjectSummary[]>;
+
   on(
     event: 'event',
     listener: (notification: S3TestServer.S3EventNotification) => void
@@ -139,11 +173,33 @@ declare namespace S3TestServer {
       >
     >;
 
+  interface ObjectSummary {
+    key: string;
+    size: number;
+    /** Hex ETag without quotes. */
+    etag: string;
+    lastModified: Date;
+    contentType?: string;
+    /** User metadata (`x-amz-meta-*`), with the prefix stripped. */
+    metadata: Record<string, string>;
+    /** Present when requested with `bodies: true`. */
+    body?: Buffer;
+  }
+
+  interface SeedObject {
+    key: string;
+    body: string | Buffer;
+    contentType?: string;
+    metadata?: Record<string, string>;
+  }
+
   interface RunAddress {
     /** Bound address, as reported by `net.Server#address()`. */
     address: string;
     family: string;
     port: number;
+    /** `host:port` the server listens on. */
+    hostPort: string;
     /** `"http"` or `"https"`. */
     protocol: string;
     /** Convenience endpoint URL, e.g. `"http://localhost:4568"`. */
@@ -198,6 +254,8 @@ declare namespace S3TestServer {
 
   export {
     S3TestServer,
+    ObjectSummary,
+    SeedObject,
     CannedBucketAcl,
     BucketConfiguration,
     S3TestServerOptions,

package/lib/index.js

@@ -52,6 +52,8 @@ class S3TestServer extends EventEmitter {
     };
     this.httpServer = null;
     this.store = null;
+    this.hostPort = null;
+    this.endpoint = null;
     this._middleware = null;
     this._sequencer = 0;
   }
@@ -110,13 +112,14 @@ class S3TestServer extends EventEmitter {
       });
       const address = this.httpServer.address();
       const protocol = this.options.key && this.options.cert ? 'https' : 'http';
-      this.logger.info(
-        `S3 test server listening on ${protocol}://${this.options.address}:${address.port}`
-      );
+      this.hostPort = `${this.options.address}:${address.port}`;
+      this.endpoint = `${protocol}://${this.hostPort}`;
+      this.logger.info(`S3 test server listening on ${this.endpoint}`);
       return {
         ...address,
         protocol,
-        endpoint: `${protocol}://${this.options.address}:${address.port}`,
+        hostPort: this.hostPort,
+        endpoint: this.endpoint,
       };
     })();
     if (typeof callback === 'function') {
@@ -217,6 +220,143 @@ class S3TestServer extends EventEmitter {
     await this.store.reset();
   }
 
+  // ---------------------------------------------------------------- helpers
+
+  _toSummary(meta) {
+    return {
+      key: meta.key,
+      size: meta.size,
+      etag: meta.etag,
+      lastModified: new Date(meta.lastModified),
+      contentType: meta.contentType,
+      metadata: { ...(meta.userMetadata || {}) },
+    };
+  }
+
+  /**
+   * Lists the completed objects in a bucket (multipart parts in flight are
+   * never included), sorted by key. With `bodies: true`, each entry also
+   * carries its content as a Buffer.
+   */
+  async getObjects(bucket, { prefix = '', bodies = false } = {}) {
+    this._ensureStore();
+    if (!(await this.store.getBucket(bucket))) {
+      throw new S3Error('NoSuchBucket', 'The specified bucket does not exist', {
+        BucketName: bucket,
+      });
+    }
+    const metas = (await this.store.listObjects(bucket)).filter(
+      (meta) => !prefix || meta.key.startsWith(prefix)
+    );
+    const summaries = metas.map((meta) => this._toSummary(meta));
+    if (bodies) {
+      for (const summary of summaries) {
+        summary.body = await this._readObject(bucket, summary.key);
+      }
+    }
+    return summaries;
+  }
+
+  _readObject(bucket, key) {
+    return new Promise((resolve, reject) => {
+      const chunks = [];
+      this.store
+        .createObjectReadStream(bucket, key)
+        .on('data', (chunk) => chunks.push(chunk))
+        .on('end', () => resolve(Buffer.concat(chunks)))
+        .on('error', reject);
+    });
+  }
+
+  /**
+   * Resolves with the bucket's objects once at least `count` of them exist
+   * (optionally under `prefix`). Useful when the code under test uploads in
+   * the background. A bucket that does not exist yet simply counts as empty.
+   */
+  async waitForObjects(bucket, count, { prefix = '', timeout = 10000 } = {}) {
+    this._ensureStore();
+    const matching = async () => {
+      if (!(await this.store.getBucket(bucket))) return null;
+      const metas = (await this.store.listObjects(bucket)).filter(
+        (meta) => !prefix || meta.key.startsWith(prefix)
+      );
+      return metas.length >= count
+        ? metas.map((meta) => this._toSummary(meta))
+        : null;
+    };
+
+    const found = await matching();
+    if (found) return found;
+
+    return new Promise((resolve, reject) => {
+      let interval;
+      let timer;
+      const check = async () => {
+        const objects = await matching().catch(() => null);
+        if (objects) {
+          cleanup();
+          resolve(objects);
+        }
+      };
+      const cleanup = () => {
+        clearInterval(interval);
+        clearTimeout(timer);
+        this.removeListener('event', check);
+      };
+      this.on('event', check);
+      // Poll as well, so out-of-band writes (e.g. putObjects) are seen too.
+      interval = setInterval(check, 150);
+      timer = setTimeout(() => {
+        cleanup();
+        reject(
+          new Error(
+            `Timed out after ${timeout}ms waiting for ${count} object(s) in "${bucket}"` +
+              (prefix ? ` under "${prefix}"` : '')
+          )
+        );
+      }, timeout);
+    });
+  }
+
+  /**
+   * Seeds objects directly into the store, bypassing the HTTP API. The
+   * bucket is created when missing. Each entry is
+   * `{ key, body, contentType?, metadata? }` with a string or Buffer body.
+   * No S3 event notifications are emitted for seeded objects.
+   */
+  async putObjects(bucket, objects) {
+    this._ensureStore();
+    await this.store.initialize();
+    const { validateBucketName } = require('./utils');
+    validateBucketName(bucket);
+    await this.store.createBucket(bucket);
+    const { Readable } = require('stream');
+    const results = [];
+    for (const object of objects) {
+      if (!object || typeof object.key !== 'string') {
+        throw new TypeError('putObjects entries need a string `key`');
+      }
+      const body = Buffer.isBuffer(object.body)
+        ? object.body
+        : Buffer.from(object.body ?? '');
+      const staged = await this.store.stageObject(
+        bucket,
+        Readable.from([body]),
+        ['CRC64NVME']
+      );
+      const meta = await staged.commit(object.key, {
+        contentType: object.contentType || 'binary/octet-stream',
+        userMetadata: { ...(object.metadata || {}) },
+        tagging: {},
+        checksums: staged.digests.checksums,
+        checksumAlgorithm: 'CRC64NVME',
+        checksumType: 'FULL_OBJECT',
+      });
+      results.push(this._toSummary(meta));
+    }
+    return results;
+  }
+
   _emitEvent(eventName, bucket, objectMeta) {
     if (!objectMeta) return;
     this._sequencer += 1;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-test-server",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "A modern, lightweight Amazon S3 emulator for local development and testing. Compatible with AWS SDK v2 and v3, Signature V2/V4, path-style and virtual-host-style addressing, streaming (aws-chunked) uploads and flexible checksums.",
   "license": "MIT",
   "author": "Israel Alagbe <israelalagbe53@gmail.com>",