s3-test-server 0.0.0-stage → 1.0.0

package/lib/controllers/post.js

@@ -0,0 +1,150 @@
+'use strict';
+
+const { S3Error } = require('../models/error');
+const { send, sendDoc, quoteEtag } = require('../models/response');
+const { readBody, awsUriEncode } = require('../utils');
+const { requireBucket } = require('./bucket');
+
+/**
+ * Browser-style form upload: POST / with multipart/form-data.
+ * Policy documents and their signatures are accepted but not enforced.
+ */
+async function postFormUpload(ctx) {
+  await requireBucket(ctx);
+  const contentType = String(ctx.req.headers['content-type'] || '');
+  const boundaryMatch = /boundary="?([^";]+)"?/i.exec(contentType);
+  if (!contentType.startsWith('multipart/form-data') || !boundaryMatch) {
+    throw new S3Error(
+      'MalformedPOSTRequest',
+      'The body of your POST request is not well-formed multipart/form-data.'
+    );
+  }
+  const body = await readBody(ctx.req, { limit: 5 * 1024 * 1024 * 1024 });
+  const { fields, file } = parseMultipartFormData(body, boundaryMatch[1]);
+  if (!file) {
+    throw new S3Error(
+      'InvalidArgument',
+      'POST requires exactly one file upload per request.',
+      { ArgumentName: 'file', ArgumentValue: '0' }
+    );
+  }
+  let key = fields.key;
+  if (!key) {
+    throw new S3Error(
+      'InvalidArgument',
+      "Bucket POST must contain a field named 'key'. If it is specified, please check the order of the fields.",
+      { ArgumentName: 'key', ArgumentValue: '' }
+    );
+  }
+  key = key.replace(/\$\{filename\}/g, file.filename || '');
+
+  const { Readable } = require('stream');
+  const staged = await ctx.store.stageObject(
+    ctx.bucket,
+    Readable.from([file.content]),
+    ['CRC64NVME']
+  );
+  const meta = await staged.commit(key, {
+    contentType:
+      fields['Content-Type'] || file.contentType || 'binary/octet-stream',
+    cacheControl: fields['Cache-Control'],
+    contentDisposition: fields['Content-Disposition'],
+    contentEncoding: fields['Content-Encoding'],
+    userMetadata: Object.fromEntries(
+      Object.entries(fields)
+        .filter(([name]) => name.toLowerCase().startsWith('x-amz-meta-'))
+        .map(([name, value]) => [
+          name.slice('x-amz-meta-'.length).toLowerCase(),
+          value,
+        ])
+    ),
+    tagging: {},
+    checksums: staged.digests.checksums,
+    checksumAlgorithm: 'CRC64NVME',
+    checksumType: 'FULL_OBJECT',
+  });
+  ctx.emitEvent('ObjectCreated:Post', meta);
+
+  const etag = quoteEtag(meta.etag);
+  const redirect = fields.success_action_redirect || fields.redirect;
+  if (redirect) {
+    let location;
+    try {
+      location = new URL(redirect);
+      location.searchParams.set('bucket', ctx.bucket);
+      location.searchParams.set('key', key);
+      location.searchParams.set('etag', etag);
+    } catch {
+      location = null;
+    }
+    if (location) {
+      send(ctx.res, 303, { Location: location.toString(), ETag: etag });
+      return;
+    }
+  }
+  const status = parseInt(fields.success_action_status, 10);
+  if (status === 200 || status === 204) {
+    send(ctx.res, status, { ETag: etag });
+  } else if (status === 201) {
+    const host = ctx.req.headers.host || 'localhost';
+    ctx.res.setHeader('ETag', etag);
+    sendDoc(ctx.res, 201, 'PostResponse', {
+      Location: `http://${host}/${ctx.bucket}/${awsUriEncode(key, false)}`,
+      Bucket: ctx.bucket,
+      Key: key,
+      ETag: etag,
+    });
+  } else {
+    send(ctx.res, 204, { ETag: etag });
+  }
+}
+
+function parseMultipartFormData(body, boundary) {
+  const delimiter = Buffer.from(`--${boundary}`);
+  const fields = {};
+  let file = null;
+
+  let offset = body.indexOf(delimiter);
+  while (offset !== -1) {
+    offset += delimiter.length;
+    // closing delimiter "--boundary--"
+    if (body[offset] === 0x2d && body[offset + 1] === 0x2d) break;
+    // skip the CRLF after the boundary line
+    if (body[offset] === 0x0d && body[offset + 1] === 0x0a) offset += 2;
+
+    const headerEnd = body.indexOf('\r\n\r\n', offset);
+    if (headerEnd === -1) break;
+    const rawHeaders = body.subarray(offset, headerEnd).toString('utf8');
+    const next = body.indexOf(delimiter, headerEnd + 4);
+    if (next === -1) break;
+    // content runs up to the CRLF that precedes the next boundary
+    const content = body.subarray(headerEnd + 4, next - 2);
+
+    const headers = {};
+    for (const line of rawHeaders.split('\r\n')) {
+      const idx = line.indexOf(':');
+      if (idx > 0)
+        headers[line.slice(0, idx).trim().toLowerCase()] = line
+          .slice(idx + 1)
+          .trim();
+    }
+    const disposition = headers['content-disposition'] || '';
+    const nameMatch = /name="([^"]*)"/.exec(disposition);
+    const filenameMatch = /filename="([^"]*)"/.exec(disposition);
+    if (nameMatch) {
+      if (filenameMatch || nameMatch[1] === 'file') {
+        file = {
+          filename: filenameMatch ? filenameMatch[1] : '',
+          content,
+          contentType: headers['content-type'],
+        };
+      } else {
+        fields[nameMatch[1]] = content.toString('utf8');
+      }
+    }
+    offset = next;
+  }
+  return { fields, file };
+}
+
+module.exports = { postFormUpload };

package/lib/controllers/service.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const { sendDoc } = require('../models/response');
+const { isoDate, DUMMY_OWNER } = require('../utils');
+
+async function listBuckets(ctx) {
+  const buckets = await ctx.store.listBuckets();
+  sendDoc(ctx.res, 200, 'ListAllMyBucketsResult', {
+    Owner: DUMMY_OWNER,
+    Buckets: {
+      Bucket: buckets.map((bucket) => ({
+        Name: bucket.name,
+        CreationDate: isoDate(bucket.creationDate),
+      })),
+    },
+  });
+}
+
+module.exports = { listBuckets };

package/lib/index.d.ts

@@ -0,0 +1,193 @@
+/// <reference types="node" />
+
+import { EventEmitter } from 'events';
+import { IncomingMessage, ServerResponse, Server as HttpServer } from 'http';
+
+declare class S3TestServer extends EventEmitter {
+  constructor(options?: S3TestServer.S3TestServerOptions);
+
+  options: S3TestServer.ResolvedOptions;
+
+  /** The underlying HTTP(S) server, available after `run()` resolves. */
+  httpServer: HttpServer | null;
+
+  /** Starts the server. */
+  run(): Promise<S3TestServer.RunAddress>;
+  run(
+    callback: (err: Error | null, address?: S3TestServer.RunAddress) => void
+  ): this;
+
+  /** Stops the server (removing data when `resetOnClose` is set). */
+  close(): Promise<void>;
+  close(callback: (err: Error | null) => void): this;
+
+  /** Returns a request handler for use with `http.createServer()` or Express. */
+  callback(): S3TestServer.RequestHandler;
+  /** Alias of {@link callback}. */
+  getMiddleware(): S3TestServer.RequestHandler;
+
+  /** Provisions the buckets in `options.configureBuckets` without the S3 API. */
+  configureBuckets(): Promise<void>;
+
+  /** Deletes all buckets, objects and configurations in the store. */
+  reset(): Promise<void>;
+
+  on(
+    event: 'event',
+    listener: (notification: S3TestServer.S3EventNotification) => void
+  ): this;
+  on(event: string | symbol, listener: (...args: any[]) => void): this;
+  once(
+    event: 'event',
+    listener: (notification: S3TestServer.S3EventNotification) => void
+  ): this;
+  once(event: string | symbol, listener: (...args: any[]) => void): this;
+}
+
+declare namespace S3TestServer {
+  interface BucketConfiguration {
+    /** Bucket name (must be a valid S3 bucket name). */
+    name: string;
+    /**
+     * Raw XML configuration documents applied to the bucket at startup.
+     * CORS (`<CORSConfiguration>`), website (`<WebsiteConfiguration>`) and
+     * tagging (`<Tagging>`) documents are recognized by their root element.
+     */
+    configs?: Array<string | Buffer>;
+  }
+
+  interface S3TestServerOptions {
+    /** Hostname or IP to bind to. Default: `"localhost"`. */
+    address?: string;
+    /** Port to listen on. Use `0` for an ephemeral port. Default: `4568`. */
+    port?: number;
+    /** Private key for HTTPS. When both `key` and `cert` are set, the server uses TLS. */
+    key?: string | Buffer;
+    /** Certificate for HTTPS. */
+    cert?: string | Buffer;
+    /** Suppress log output. Default: `false`. */
+    silent?: boolean;
+    /**
+     * Directory in which buckets and objects are stored.
+     * Default: a freshly created temporary directory.
+     */
+    directory?: string;
+    /**
+     * Clear the data directory each time the server starts, so a persistent
+     * `directory` does not leak state between runs. Default: `false`.
+     */
+    resetOnStart?: boolean;
+    /** Remove all bucket data when the server closes. Default: `false`. */
+    resetOnClose?: boolean;
+    /**
+     * Accept well-formed signatures even when they do not match
+     * (useful with exotic clients). Default: `false`.
+     */
+    allowMismatchedSignatures?: boolean;
+    /** Enable virtual-host-style bucket addressing. Default: `true`. */
+    vhostBuckets?: boolean;
+    /**
+     * Root domain used to recognize virtual-host-style requests
+     * (e.g. `bucket.s3.<region>.amazonaws.com`). Default: `"amazonaws.com"`.
+     */
+    serviceEndpoint?: string;
+    /** Access key id clients must sign requests with. Default: `"S3TEST"`. */
+    accessKeyId?: string;
+    /** Secret access key clients must sign requests with. Default: `"S3TEST"`. */
+    secretAccessKey?: string;
+    /** Region reported in responses and event records. Default: `"us-east-1"`. */
+    region?: string;
+    /** Buckets to create (and configure) before the server starts. */
+    configureBuckets?: BucketConfiguration[];
+  }
+
+  type ResolvedOptions = S3TestServerOptions &
+    Required<
+      Pick<
+        S3TestServerOptions,
+        | 'address'
+        | 'port'
+        | 'silent'
+        | 'resetOnStart'
+        | 'resetOnClose'
+        | 'allowMismatchedSignatures'
+        | 'vhostBuckets'
+        | 'serviceEndpoint'
+        | 'accessKeyId'
+        | 'secretAccessKey'
+        | 'region'
+      >
+    >;
+
+  interface RunAddress {
+    /** Bound address, as reported by `net.Server#address()`. */
+    address: string;
+    family: string;
+    port: number;
+    /** `"http"` or `"https"`. */
+    protocol: string;
+    /** Convenience endpoint URL, e.g. `"http://localhost:4568"`. */
+    endpoint: string;
+  }
+
+  type S3EventName =
+    | 'ObjectCreated:Put'
+    | 'ObjectCreated:Copy'
+    | 'ObjectCreated:Post'
+    | 'ObjectCreated:CompleteMultipartUpload'
+    | 'ObjectRemoved:Delete';
+
+  interface S3EventRecord {
+    eventVersion: string;
+    eventSource: 'aws:s3';
+    awsRegion: string;
+    eventTime: string;
+    eventName: S3EventName;
+    userIdentity: { principalId: string };
+    requestParameters: { sourceIPAddress: string };
+    responseElements: Record<string, string>;
+    s3: {
+      s3SchemaVersion: string;
+      configurationId: string;
+      bucket: {
+        name: string;
+        ownerIdentity: { principalId: string };
+        arn: string;
+      };
+      object: {
+        /** URL-encoded object key. */
+        key: string;
+        size: number;
+        eTag?: string;
+        sequencer: string;
+      };
+    };
+  }
+
+  interface S3EventNotification {
+    Records: S3EventRecord[];
+  }
+
+  type RequestHandler = (req: IncomingMessage, res: ServerResponse) => void;
+
+  class S3Error extends Error {
+    code: string;
+    statusCode: number;
+    detail: Record<string, unknown>;
+  }
+
+  export {
+    S3TestServer,
+    BucketConfiguration,
+    S3TestServerOptions,
+    ResolvedOptions,
+    RunAddress,
+    S3EventName,
+    S3EventRecord,
+    S3EventNotification,
+    RequestHandler,
+    S3Error,
+  };
+}
+
+export = S3TestServer;

package/lib/index.js

@@ -0,0 +1,241 @@
+'use strict';
+
+const EventEmitter = require('events');
+const fs = require('fs');
+const http = require('http');
+const https = require('https');
+const os = require('os');
+const path = require('path');
+const { FilesystemStore } = require('./stores/filesystem');
+const { createRequestHandler } = require('./routes');
+const { awsUriEncode } = require('./utils');
+const { S3Error } = require('./models/error');
+
+const DEFAULT_OPTIONS = {
+  address: 'localhost',
+  port: 4568,
+  key: undefined,
+  cert: undefined,
+  silent: false,
+  directory: undefined,
+  resetOnStart: false,
+  resetOnClose: false,
+  allowMismatchedSignatures: false,
+  vhostBuckets: true,
+  serviceEndpoint: 'amazonaws.com',
+  accessKeyId: 'S3TEST',
+  secretAccessKey: 'S3TEST',
+  region: 'us-east-1',
+  configureBuckets: [],
+};
+
+class S3TestServer extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.options = { ...DEFAULT_OPTIONS, ...options };
+    this.logger = {
+      info: (...args) => {
+        if (!this.options.silent) console.log(...args);
+      },
+      error: (...args) => {
+        if (!this.options.silent) console.error(...args);
+      },
+    };
+    this.httpServer = null;
+    this.store = null;
+    this._middleware = null;
+    this._sequencer = 0;
+  }
+
+  _ensureStore() {
+    if (this.store) return this.store;
+    let root = this.options.directory;
+    if (!root) {
+      root = fs.mkdtempSync(path.join(os.tmpdir(), 's3-test-server-'));
+      this._createdTempDir = true;
+    }
+    fs.mkdirSync(root, { recursive: true });
+    this.store = new FilesystemStore(root);
+    return this.store;
+  }
+
+  /**
+   * Returns a `(req, res)` handler usable with http.createServer() or as an
+   * Express/Connect middleware.
+   */
+  callback() {
+    this._ensureStore();
+    if (!this._middleware) this._middleware = createRequestHandler(this);
+    return this._middleware;
+  }
+
+  getMiddleware() {
+    return this.callback();
+  }
+
+  /** Starts the server. Returns a promise of `{ address, port, ... }`. */
+  run(callback) {
+    const promise = (async () => {
+      this._ensureStore();
+      await this.store.initialize();
+      // Start from an empty data directory when asked, so a persistent
+      // `directory` does not leak state between runs.
+      if (this.options.resetOnStart) {
+        await this.store.reset();
+      }
+      await this.configureBuckets();
+      const handler = this.callback();
+      this.httpServer =
+        this.options.key && this.options.cert
+          ? https.createServer(
+              { key: this.options.key, cert: this.options.cert },
+              handler
+            )
+          : http.createServer(handler);
+      await new Promise((resolve, reject) => {
+        this.httpServer.once('error', reject);
+        this.httpServer.listen(this.options.port, this.options.address, () => {
+          this.httpServer.removeListener('error', reject);
+          resolve();
+        });
+      });
+      const address = this.httpServer.address();
+      const protocol = this.options.key && this.options.cert ? 'https' : 'http';
+      this.logger.info(
+        `S3 test server listening on ${protocol}://${this.options.address}:${address.port}`
+      );
+      return {
+        ...address,
+        protocol,
+        endpoint: `${protocol}://${this.options.address}:${address.port}`,
+      };
+    })();
+    if (typeof callback === 'function') {
+      promise.then(
+        (address) => callback(null, address),
+        (err) => callback(err)
+      );
+      return this;
+    }
+    return promise;
+  }
+
+  /** Stops the server. With `resetOnClose`, removes all bucket data. */
+  close(callback) {
+    const promise = (async () => {
+      if (this.httpServer) {
+        this.httpServer.closeAllConnections?.();
+        await new Promise((resolve, reject) => {
+          this.httpServer.close((err) => (err ? reject(err) : resolve()));
+        });
+        this.httpServer = null;
+      }
+      if (this.store && this.options.resetOnClose) {
+        if (this._createdTempDir) await this.store.destroy();
+        else await this.store.reset();
+      }
+    })();
+    if (typeof callback === 'function') {
+      promise.then(
+        () => callback(null),
+        (err) => callback(err)
+      );
+      return this;
+    }
+    return promise;
+  }
+
+  /**
+   * Creates the buckets given in `options.configureBuckets` (name plus raw
+   * XML configs: CORS, website or tagging documents, detected by their root
+   * element) without going through the S3 API.
+   */
+  async configureBuckets() {
+    this._ensureStore();
+    await this.store.initialize();
+    const {
+      parseCorsConfig,
+      parseWebsiteConfig,
+      parseTaggingConfig,
+    } = require('./controllers/bucket');
+    const { validateBucketName } = require('./utils');
+    for (const { name, configs = [] } of this.options.configureBuckets || []) {
+      validateBucketName(name);
+      await this.store.createBucket(name);
+      for (const rawConfig of configs) {
+        const xml = Buffer.isBuffer(rawConfig)
+          ? rawConfig.toString('utf8')
+          : String(rawConfig);
+        if (/<\s*(\w+:)?CORSConfiguration[\s>]/.test(xml)) {
+          await this.store.putBucketConfig(name, 'cors', parseCorsConfig(xml));
+        } else if (/<\s*(\w+:)?WebsiteConfiguration[\s>]/.test(xml)) {
+          await this.store.putBucketConfig(
+            name,
+            'website',
+            parseWebsiteConfig(xml)
+          );
+        } else if (/<\s*(\w+:)?Tagging[\s>]/.test(xml)) {
+          await this.store.putBucketConfig(
+            name,
+            'tagging',
+            parseTaggingConfig(xml)
+          );
+        } else {
+          throw new S3Error(
+            'MalformedXML',
+            'Unrecognized bucket configuration document'
+          );
+        }
+      }
+    }
+  }
+
+  /** Deletes all buckets, objects and bucket configurations. */
+  async reset() {
+    this._ensureStore();
+    await this.store.reset();
+  }
+
+  _emitEvent(eventName, bucket, objectMeta) {
+    if (!objectMeta) return;
+    this._sequencer += 1;
+    const record = {
+      eventVersion: '2.1',
+      eventSource: 'aws:s3',
+      awsRegion: this.options.region,
+      eventTime: new Date().toISOString(),
+      eventName,
+      userIdentity: { principalId: 'AIDAJDPLRKLG7UEXAMPLE' },
+      requestParameters: { sourceIPAddress: '127.0.0.1' },
+      responseElements: {
+        'x-amz-request-id': 'C3D13FE58DE4C810',
+        'x-amz-id-2':
+          'FMyUVURIY8/IgAtTv8xRjskZQpcIZ9KG4V5Wp6S7S/JRWeUWerMUE5JgHvANOjpD',
+      },
+      s3: {
+        s3SchemaVersion: '1.0',
+        configurationId: 'testConfigRule',
+        bucket: {
+          name: bucket,
+          ownerIdentity: { principalId: 'A3NL1KOZZKExample' },
+          arn: `arn:aws:s3:::${bucket}`,
+        },
+        object: {
+          key: awsUriEncode(objectMeta.key, false),
+          size: objectMeta.size,
+          eTag: objectMeta.etag,
+          sequencer: this._sequencer
+            .toString(16)
+            .toUpperCase()
+            .padStart(16, '0'),
+        },
+      },
+    };
+    this.emit('event', { Records: [record] });
+  }
+}
+
+module.exports = S3TestServer;
+module.exports.S3TestServer = S3TestServer;
+module.exports.default = S3TestServer;
+module.exports.S3Error = S3Error;