s3-test-server 0.0.0-stage → 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Israel Alagbe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,232 @@
-# Temporary Holding Version
+# AWS S3 Local Emulator Test Server
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A lightweight Amazon S3 emulator for local development and testing.
+
+Zero runtime dependencies, a single small package, and wire-level
+compatibility with both old and new S3 clients:
+
+- **AWS SDK for JavaScript v3**, including flexible checksums
+  (CRC32, CRC32C, CRC64NVME, SHA1, SHA256), `aws-chunked` streaming uploads
+  with trailers, and full-object / composite multipart checksums
+- **AWS SDK for JavaScript v2**, SigV4 and legacy SigV2 (`signatureVersion: 's3'`)
+- **Any other S3 client** that signs with Signature V2 or V4, header-based or
+  presigned, path-style or virtual-host-style
+
+Data is stored on the filesystem (a temp directory by default), so it survives
+restarts when you point it at a directory of your own.
+
+## Quick start
+
+```bash
+npm install --save-dev s3-test-server
+```
+
+```js
+const S3TestServer = require('s3-test-server');
+
+const server = new S3TestServer({
+  port: 4568,
+  configureBuckets: [{ name: 'my-bucket' }],
+});
+
+const { endpoint } = await server.run(); // http://localhost:4568
+// ... run your tests ...
+await server.close();
+```
+
+Point any S3 client at it:
+
+```js
+import { S3Client } from '@aws-sdk/client-s3';
+
+const client = new S3Client({
+  endpoint: 'http://localhost:4568',
+  region: 'us-east-1',
+  credentials: { accessKeyId: 'S3TEST', secretAccessKey: 'S3TEST' },
+  forcePathStyle: true,
+});
+```
+
+Or from the command line:
+
+```bash
+npx s3-test-server -d ./s3-data -p 4568 --configure-bucket my-bucket
+```
+
+Run `s3-test-server --help` for all flags.
+
+## Why not just mock?
+
+Mocking the S3 client tests your assumptions about S3, not S3 itself. Stubs
+skip request signing, checksum validation, multipart ETag rules, pagination
+contracts and error shapes, so code that passes mocked tests can still break
+against the real service, and every SDK upgrade silently invalidates the
+stubs.
+
+Before, with a mocked client, nothing below the stub actually runs:
+
+```js
+jest.mock('@aws-sdk/client-s3');
+S3Client.prototype.send = jest.fn().mockResolvedValue({ ETag: '"abc123"' });
+
+test('uploads a report', async () => {
+  await uploadReport('march.csv', csv);
+  expect(S3Client.prototype.send).toHaveBeenCalled(); // proves nothing about S3
+});
+```
+
+After, the same test against `s3-test-server`. Your production code runs
+unmodified and every byte goes over HTTP (`aws-s3.local` is any hostname you
+map to 127.0.0.1):
+
+```js
+const server = new S3TestServer({
+  address: 'aws-s3.local',
+  configureBuckets: [{ name: 'reports' }],
+});
+
+beforeAll(() => server.run()); // http://aws-s3.local:4568
+afterAll(() => server.close());
+
+test('uploads a report and returns its URL', async () => {
+  const url = await uploadReport('march.csv', csv); // real code, real wire
+
+  // the full object URL is real, not invented by a stub
+  expect(url).toBe('http://aws-s3.local:4568/reports/march.csv');
+
+  // and it resolves: read the object back like any HTTP resource
+  const saved = await fetch(url);
+  expect(await saved.text()).toBe(csv.toString());
+});
+```
+
+If `uploadReport` signs incorrectly, streams a corrupt body or mishandles
+multipart thresholds, this test fails, exactly as it would against AWS.
+
+## Options
+
+All options are optional. `new S3TestServer()` starts an HTTP server on
+`localhost:4568` backed by a temporary directory.
+
+### Server
+
+| Option         | Type               | Default       | Description                                       |
+| -------------- | ------------------ | ------------- | ------------------------------------------------- |
+| `address`      | `string`           | `'localhost'` | Hostname or IP to bind to.                        |
+| `port`         | `number`           | `4568`        | Port to listen on. Use `0` for an ephemeral port. |
+| `key` / `cert` | `string \| Buffer` | none          | TLS key and certificate. Provide both for HTTPS.  |
+| `silent`       | `boolean`          | `false`       | Suppress log output.                              |
+
+### Storage
+
+| Option             | Type                    | Default  | Description                                                                                |
+| ------------------ | ----------------------- | -------- | ------------------------------------------------------------------------------------------ |
+| `directory`        | `string`                | temp dir | Where buckets and objects are stored. Data persists across restarts.                       |
+| `resetOnStart`     | `boolean`               | `false`  | Clear the data directory on every start, so a persistent `directory` begins empty.         |
+| `resetOnClose`     | `boolean`               | `false`  | Delete all bucket data when `close()` is called.                                           |
+| `configureBuckets` | `BucketConfiguration[]` | `[]`     | Buckets to create at startup, each with optional raw XML configs (CORS, website, tagging). |
+
+### Credentials and addressing
+
+| Option                      | Type      | Default           | Description                                                                             |
+| --------------------------- | --------- | ----------------- | --------------------------------------------------------------------------------------- |
+| `accessKeyId`               | `string`  | `'S3TEST'`        | Access key clients must sign with.                                                      |
+| `secretAccessKey`           | `string`  | `'S3TEST'`        | Matching secret key.                                                                    |
+| `region`                    | `string`  | `'us-east-1'`     | Region reported in responses and event records.                                         |
+| `vhostBuckets`              | `boolean` | `true`            | Virtual-host-style addressing (`bucket.localhost`, `bucket.s3.<region>.amazonaws.com`). |
+| `serviceEndpoint`           | `string`  | `'amazonaws.com'` | Root domain recognized for virtual-host-style access.                                   |
+| `allowMismatchedSignatures` | `boolean` | `false`           | Accept well-formed but incorrect signatures.                                            |
+
+Anonymous (unsigned) requests are accepted, as on a public bucket, which is
+handy for plain `fetch`/`curl` in tests. Signed requests are verified against
+`accessKeyId`/`secretAccessKey`.
+
+## Supported operations
+
+**Service / buckets**: ListBuckets, Create/Delete/HeadBucket,
+GetBucketLocation, ListObjects (v1), ListObjectsV2, ListObjectVersions
+(unversioned mapping), Get/Put/DeleteBucketCors, Get/Put/DeleteBucketWebsite,
+Get/Put/DeleteBucketTagging, GetBucketVersioning, GetBucketEncryption,
+GetBucketNotification/RequestPayment/Logging/Accelerate, ACLs (dummy),
+ListMultipartUploads.
+
+**Objects**: Put/Get/Head/DeleteObject, DeleteObjects, CopyObject
+(COPY/REPLACE directives, copy conditions), POST form uploads
+(`multipart/form-data`, `${filename}` substitution, `success_action_*`),
+Get/Put/DeleteObjectTagging, GetObjectAttributes, object ACLs (dummy),
+Range requests, conditional reads (`If-Match`, `If-None-Match`,
+`If-(Un)Modified-Since`) and **conditional writes** (`If-None-Match: *`,
+`If-Match`), `response-*` header overrides, `Content-MD5` and
+`x-amz-checksum-*` validation (headers or `aws-chunked` trailers).
+
+**Multipart**: Create/Abort/CompleteMultipartUpload, UploadPart,
+UploadPartCopy (with `x-amz-copy-source-range`), ListParts, correct
+multipart ETags (`md5-of-md5s-N`), composite and full-object checksums.
+
+**Static website hosting**: index/error documents, folder redirects,
+`x-amz-website-redirect-location`, `RedirectAllRequestsTo`; served for
+anonymous GET/HEAD requests on buckets with a website configuration.
+
+**CORS**: preflight (`OPTIONS`) evaluation and response headers, driven by
+the bucket's CORS configuration.
+
+Not implemented: bucket versioning (`PutBucketVersioning` with `Enabled`
+returns `NotImplemented`), object locking, lifecycle execution, bucket
+policies/IAM. Per-chunk signatures inside `aws-chunked` bodies are not
+re-verified (the seed signature is).
+
+## Events
+
+The server is an `EventEmitter` and emits S3-style notifications:
+
+```js
+server.on('event', (notification) => {
+  const [record] = notification.Records;
+  console.log(record.eventName, record.s3.object.key);
+});
+```
+
+Emitted for `ObjectCreated:Put`, `ObjectCreated:Copy`, `ObjectCreated:Post`,
+`ObjectCreated:CompleteMultipartUpload` and `ObjectRemoved:Delete`.
+
+## Using as middleware
+
+```js
+const http = require('http');
+const S3TestServer = require('s3-test-server');
+
+const s3 = new S3TestServer({ silent: true });
+http.createServer(s3.callback()).listen(4568);
+// also works as an Express/Connect handler
+```
+
+## Programmatic API
+
+- `run([callback])` returns `Promise<{ address, port, protocol, endpoint }>`
+- `close([callback])` returns `Promise<void>`
+- `callback()` / `getMiddleware()` returns a request handler
+- `configureBuckets()` provisions `options.configureBuckets` without the API
+- `reset()` deletes all buckets and configurations
+
+TypeScript definitions ship with the package (`lib/index.d.ts`).
+
+## HTTPS
+
+```js
+const fs = require('fs');
+const server = new S3TestServer({
+  key: fs.readFileSync('./key.pem'),
+  cert: fs.readFileSync('./cert.pem'),
+});
+```
+
+With a self-signed certificate, configure your client to accept it (for the
+AWS SDKs, a custom HTTPS agent with `rejectUnauthorized: false`).
+
+## Requirements
+
+Node.js >= 20.
+
+## License
+
+MIT

package/bin/s3-test-server.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict';
+
+require('../lib/cli').main();

package/lib/checksums.js

@@ -0,0 +1,176 @@
+'use strict';
+
+const crypto = require('crypto');
+
+// CRC tables are built lazily. Implemented locally (rather than via
+// zlib.crc32) so the package runs on all Node.js 20.x releases.
+
+function makeCrc32Table(poly) {
+  const table = new Uint32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? (c >>> 1) ^ poly : c >>> 1;
+    }
+    table[n] = c >>> 0;
+  }
+  return table;
+}
+
+let crc32Table;
+let crc32cTable;
+let crc64Table;
+
+function makeCrc64Table(poly) {
+  const table = new BigUint64Array(256);
+  for (let n = 0n; n < 256n; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1n ? (c >> 1n) ^ poly : c >> 1n;
+    }
+    table[Number(n)] = c;
+  }
+  return table;
+}
+
+class Crc32 {
+  constructor(castagnoli = false) {
+    if (castagnoli) {
+      crc32cTable = crc32cTable || makeCrc32Table(0x82f63b78);
+      this.table = crc32cTable;
+    } else {
+      crc32Table = crc32Table || makeCrc32Table(0xedb88320);
+      this.table = crc32Table;
+    }
+    this.crc = 0xffffffff;
+  }
+
+  update(buf) {
+    let crc = this.crc;
+    const table = this.table;
+    for (let i = 0; i < buf.length; i++) {
+      crc = table[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8);
+    }
+    this.crc = crc >>> 0;
+  }
+
+  digest() {
+    const out = Buffer.alloc(4);
+    out.writeUInt32BE((this.crc ^ 0xffffffff) >>> 0);
+    return out;
+  }
+}
+
+// CRC-64/NVME: reflected polynomial of 0xAD93D23594C93659.
+const CRC64_POLY = 0x9a6c9329ac4bc9b5n;
+const CRC64_MASK = 0xffffffffffffffffn;
+
+class Crc64Nvme {
+  constructor() {
+    crc64Table = crc64Table || makeCrc64Table(CRC64_POLY);
+    this.crc = CRC64_MASK;
+  }
+
+  update(buf) {
+    let crc = this.crc;
+    const table = crc64Table;
+    for (let i = 0; i < buf.length; i++) {
+      crc = table[Number((crc ^ BigInt(buf[i])) & 0xffn)] ^ (crc >> 8n);
+    }
+    this.crc = crc;
+  }
+
+  digest() {
+    const out = Buffer.alloc(8);
+    out.writeBigUInt64BE((this.crc ^ CRC64_MASK) & CRC64_MASK);
+    return out;
+  }
+}
+
+class NodeHash {
+  constructor(algorithm) {
+    this.hash = crypto.createHash(algorithm);
+  }
+
+  update(buf) {
+    this.hash.update(buf);
+  }
+
+  digest() {
+    return this.hash.digest();
+  }
+}
+
+const CHECKSUM_ALGORITHMS = ['CRC32', 'CRC32C', 'CRC64NVME', 'SHA1', 'SHA256'];
+
+function createChecksummer(algorithm) {
+  switch (algorithm) {
+    case 'CRC32':
+      return new Crc32(false);
+    case 'CRC32C':
+      return new Crc32(true);
+    case 'CRC64NVME':
+      return new Crc64Nvme();
+    case 'SHA1':
+      return new NodeHash('sha1');
+    case 'SHA256':
+      return new NodeHash('sha256');
+    default:
+      throw new Error(`Unsupported checksum algorithm: ${algorithm}`);
+  }
+}
+
+/**
+ * Computes MD5 + SHA256 (always) and any requested flexible checksums over a
+ * stream of buffers.
+ */
+class DigestCalculator {
+  constructor(algorithms = []) {
+    this.md5 = crypto.createHash('md5');
+    this.sha256 = crypto.createHash('sha256');
+    this.size = 0;
+    this.checksummers = {};
+    for (const algorithm of algorithms) {
+      if (!this.checksummers[algorithm]) {
+        this.checksummers[algorithm] = createChecksummer(algorithm);
+      }
+    }
+  }
+
+  update(buf) {
+    this.size += buf.length;
+    this.md5.update(buf);
+    this.sha256.update(buf);
+    for (const checksummer of Object.values(this.checksummers)) {
+      checksummer.update(buf);
+    }
+  }
+
+  digest() {
+    const md5 = this.md5.digest();
+    const checksums = {};
+    for (const [algorithm, checksummer] of Object.entries(this.checksummers)) {
+      checksums[algorithm] = checksummer.digest().toString('base64');
+    }
+    return {
+      size: this.size,
+      md5Hex: md5.toString('hex'),
+      md5Base64: md5.toString('base64'),
+      sha256Hex: this.sha256.digest('hex'),
+      checksums,
+    };
+  }
+}
+
+function checksumOf(algorithm, buf) {
+  const checksummer = createChecksummer(algorithm);
+  checksummer.update(buf);
+  return checksummer.digest().toString('base64');
+}
+
+module.exports = {
+  CHECKSUM_ALGORITHMS,
+  DigestCalculator,
+  createChecksummer,
+  checksumOf,
+};

package/lib/chunked.js

@@ -0,0 +1,118 @@
+'use strict';
+
+const { Transform } = require('stream');
+
+const MAX_HEADER_LENGTH = 64 * 1024;
+
+/**
+ * Decodes the `aws-chunked` content encoding used by SigV4 streaming uploads:
+ *
+ *   <hex-size>[;chunk-signature=...]\r\n<data>\r\n ... 0[;chunk-signature=...]\r\n
+ *   [trailer-name:value\r\n ...]\r\n
+ *
+ * Emits the raw payload bytes and collects any trailing checksum headers in
+ * `this.trailers` (lowercased names), available once the stream ends.
+ * Chunk signatures are not themselves verified; the seed signature in the
+ * Authorization header has already been checked by then.
+ */
+class AwsChunkedDecoder extends Transform {
+  constructor() {
+    super();
+    this.buffer = Buffer.alloc(0);
+    this.state = 'size';
+    this.remaining = 0;
+    this.trailers = {};
+  }
+
+  _transform(chunk, _encoding, callback) {
+    this.buffer =
+      this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk]);
+    try {
+      this._consume();
+      callback();
+    } catch (err) {
+      callback(err);
+    }
+  }
+
+  _flush(callback) {
+    if (
+      this.state === 'data' ||
+      (this.state === 'size' && this.buffer.length > 0)
+    ) {
+      callback(new Error('Truncated aws-chunked payload'));
+      return;
+    }
+    callback();
+  }
+
+  _readLine() {
+    const idx = this.buffer.indexOf('\r\n');
+    if (idx === -1) {
+      if (this.buffer.length > MAX_HEADER_LENGTH) {
+        throw new Error('aws-chunked header line too long');
+      }
+      return null;
+    }
+    const line = this.buffer.subarray(0, idx).toString('utf8');
+    this.buffer = this.buffer.subarray(idx + 2);
+    return line;
+  }
+
+  _consume() {
+    for (;;) {
+      if (this.state === 'done') {
+        this.buffer = Buffer.alloc(0);
+        return;
+      }
+      if (this.state === 'size') {
+        const line = this._readLine();
+        if (line === null) return;
+        if (line === '') continue; // tolerate stray blank lines between chunks
+        const sizeToken = line.split(';')[0].trim();
+        if (!/^[0-9a-fA-F]+$/.test(sizeToken)) {
+          throw new Error(
+            `Invalid aws-chunked size line: ${JSON.stringify(line)}`
+          );
+        }
+        this.remaining = parseInt(sizeToken, 16);
+        this.state = this.remaining === 0 ? 'trailer' : 'data';
+      } else if (this.state === 'data') {
+        if (this.buffer.length === 0) return;
+        const take = Math.min(this.remaining, this.buffer.length);
+        this.push(this.buffer.subarray(0, take));
+        this.buffer = this.buffer.subarray(take);
+        this.remaining -= take;
+        if (this.remaining === 0) this.state = 'data-crlf';
+      } else if (this.state === 'data-crlf') {
+        if (this.buffer.length < 2) return;
+        // Expect \r\n after the chunk body; be lenient if it is missing.
+        if (this.buffer[0] === 0x0d && this.buffer[1] === 0x0a) {
+          this.buffer = this.buffer.subarray(2);
+        }
+        this.state = 'size';
+      } else if (this.state === 'trailer') {
+        const line = this._readLine();
+        if (line === null) {
+          // No trailing CRLF after the zero chunk is also accepted.
+          if (this.buffer.length === 0) this.state = 'done';
+          return;
+        }
+        if (line === '') {
+          this.state = 'done';
+          continue;
+        }
+        const idx = line.indexOf(':');
+        if (idx > 0) {
+          const name = line.slice(0, idx).trim().toLowerCase();
+          const value = line.slice(idx + 1).trim();
+          if (name !== 'x-amz-trailer-signature') {
+            this.trailers[name] = value;
+          }
+        }
+      }
+    }
+  }
+}
+
+module.exports = { AwsChunkedDecoder };

package/lib/cli.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const S3TestServer = require('./index');
+
+const HELP = `Usage: s3-test-server [options]
+
+A lightweight Amazon S3 emulator for development and testing.
+
+Options:
+  -a, --address <host>             Hostname or IP to bind to (default: localhost)
+  -p, --port <n>                   Port of the server (default: 4568)
+  -d, --directory <path>           Data directory (default: a temporary directory)
+      --access-key-id <id>         Access key clients must sign with (default: S3TEST)
+      --secret-access-key <key>    Matching secret access key (default: S3TEST)
+      --region <region>            Region reported by the server (default: us-east-1)
+      --key <path>                 Path to a private key file for HTTPS
+      --cert <path>                Path to a certificate file for HTTPS
+      --service-endpoint <host>    Root domain for virtual-host-style access
+                                   (default: amazonaws.com)
+      --no-vhost-buckets           Disable virtual-host-style bucket addressing
+      --allow-mismatched-signatures  Accept well-formed but incorrect signatures
+      --reset-on-start             Clear the data directory when the server starts
+      --reset-on-close             Delete all bucket data when the server closes
+      --configure-bucket <name[,config,...]>
+                                   Create a bucket at startup; optional paths to
+                                   CORS/website/tagging XML files (repeatable)
+  -s, --silent                     Suppress log output
+  -v, --version                    Print the version and exit
+  -h, --help                       Show this help
+
+Example:
+  s3-test-server -d ./s3-data -p 9000 \\
+    --configure-bucket "website,./cors.xml,./website.xml"
+`;
+
+function parseArgs(argv) {
+  const options = { configureBuckets: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    const next = () => {
+      i += 1;
+      if (i >= argv.length) {
+        console.error(`Missing value for ${arg}`);
+        process.exit(1);
+      }
+      return argv[i];
+    };
+    switch (arg) {
+      case '-a':
+      case '--address':
+        options.address = next();
+        break;
+      case '-p':
+      case '--port':
+        options.port = parseInt(next(), 10);
+        break;
+      case '-d':
+      case '--directory':
+        options.directory = path.resolve(next());
+        break;
+      case '--access-key-id':
+        options.accessKeyId = next();
+        break;
+      case '--secret-access-key':
+        options.secretAccessKey = next();
+        break;
+      case '--region':
+        options.region = next();
+        break;
+      case '--key':
+        options.key = fs.readFileSync(next());
+        break;
+      case '--cert':
+        options.cert = fs.readFileSync(next());
+        break;
+      case '--service-endpoint':
+        options.serviceEndpoint = next();
+        break;
+      case '--no-vhost-buckets':
+        options.vhostBuckets = false;
+        break;
+      case '--allow-mismatched-signatures':
+        options.allowMismatchedSignatures = true;
+        break;
+      case '--reset-on-start':
+        options.resetOnStart = true;
+        break;
+      case '--reset-on-close':
+        options.resetOnClose = true;
+        break;
+      case '--configure-bucket': {
+        const [name, ...configPaths] = next().split(',');
+        options.configureBuckets.push({
+          name: name.trim(),
+          configs: configPaths.map((configPath) =>
+            fs.readFileSync(path.resolve(configPath.trim()))
+          ),
+        });
+        break;
+      }
+      case '-s':
+      case '--silent':
+        options.silent = true;
+        break;
+      case '-v':
+      case '--version':
+        console.log(require('../package.json').version);
+        process.exit(0);
+        break;
+      case '-h':
+      case '--help':
+        console.log(HELP);
+        process.exit(0);
+        break;
+      default:
+        console.error(`Unknown option: ${arg}\n`);
+        console.error(HELP);
+        process.exit(1);
+    }
+  }
+  return options;
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  const server = new S3TestServer(options);
+  const shutdown = () => {
+    server.close().finally(() => process.exit(0));
+  };
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  try {
+    await server.run();
+  } catch (err) {
+    console.error(err.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { main };