s3-test-server 0.0.0-stage → 1.0.0

package/lib/middleware/cors.js

@@ -0,0 +1,107 @@
+'use strict';
+
+const { S3Error } = require('../models/error');
+const { send } = require('../models/response');
+const { escapeRegExp } = require('../utils');
+
+function matchOrigin(rule, origin) {
+  return rule.allowedOrigins.some((allowed) => {
+    if (allowed === '*') return true;
+    if (allowed.includes('*')) {
+      const pattern = new RegExp(
+        `^${allowed.split('*').map(escapeRegExp).join('.*')}$`
+      );
+      return pattern.test(origin);
+    }
+    return allowed === origin;
+  });
+}
+
+function findCorsRule(corsConfig, origin, method, requestHeaders = []) {
+  if (!corsConfig) return null;
+  for (const rule of corsConfig.rules) {
+    if (!matchOrigin(rule, origin)) continue;
+    if (!rule.allowedMethods.includes(method)) continue;
+    const allowAllHeaders = rule.allowedHeaders.includes('*');
+    const headersOk = requestHeaders.every(
+      (header) =>
+        allowAllHeaders ||
+        rule.allowedHeaders.some(
+          (allowed) =>
+            allowed === header ||
+            (allowed.endsWith('*') && header.startsWith(allowed.slice(0, -1)))
+        )
+    );
+    if (!headersOk) continue;
+    return rule;
+  }
+  return null;
+}
+
+/** Sets simple CORS response headers on non-preflight requests. */
+function applyCorsHeaders(ctx, corsConfig) {
+  const origin = ctx.req.headers.origin;
+  if (!origin || !corsConfig) return;
+  const rule = findCorsRule(corsConfig, origin, ctx.req.method);
+  if (!rule) return;
+  const wildcard = rule.allowedOrigins.includes('*');
+  ctx.res.setHeader('Access-Control-Allow-Origin', wildcard ? '*' : origin);
+  if (!wildcard) {
+    ctx.res.setHeader('Access-Control-Allow-Credentials', 'true');
+    ctx.res.setHeader(
+      'Vary',
+      'Origin, Access-Control-Request-Headers, Access-Control-Request-Method'
+    );
+  }
+  if (rule.exposeHeaders.length > 0) {
+    ctx.res.setHeader(
+      'Access-Control-Expose-Headers',
+      rule.exposeHeaders.join(', ')
+    );
+  }
+}
+
+/** Answers an OPTIONS preflight against the bucket's CORS configuration. */
+function handlePreflight(ctx, corsConfig) {
+  const origin = ctx.req.headers.origin;
+  const method = ctx.req.headers['access-control-request-method'];
+  if (!origin || !method) {
+    throw new S3Error(
+      'BadRequest',
+      'Insufficient information. Origin request header needed.',
+      { statusCode: 400 }
+    );
+  }
+  const requestHeaders = String(
+    ctx.req.headers['access-control-request-headers'] || ''
+  )
+    .split(',')
+    .map((header) => header.trim().toLowerCase())
+    .filter(Boolean);
+  const rule = findCorsRule(corsConfig, origin, method, requestHeaders);
+  if (!rule) {
+    throw new S3Error(
+      'AccessForbidden',
+      corsConfig
+        ? "CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted by the resource's CORS spec."
+        : 'CORSResponse: CORS is not enabled for this bucket.',
+      { statusCode: 403, Method: method, ResourceType: 'OBJECT' }
+    );
+  }
+  const wildcard = rule.allowedOrigins.includes('*');
+  const headers = {
+    'Access-Control-Allow-Origin': wildcard ? '*' : origin,
+    'Access-Control-Allow-Methods': rule.allowedMethods.join(', '),
+    Vary: 'Origin, Access-Control-Request-Headers, Access-Control-Request-Method',
+  };
+  if (!wildcard) headers['Access-Control-Allow-Credentials'] = 'true';
+  if (requestHeaders.length > 0) {
+    headers['Access-Control-Allow-Headers'] = requestHeaders.join(', ');
+  }
+  if (rule.maxAgeSeconds !== undefined) {
+    headers['Access-Control-Max-Age'] = rule.maxAgeSeconds;
+  }
+  send(ctx.res, 200, headers);
+}
+
+module.exports = { applyCorsHeaders, handlePreflight, findCorsRule };

package/lib/middleware/error-handler.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const { S3Error } = require('../models/error');
+const { sendXml } = require('../models/response');
+
+/** Renders any thrown error as an S3-style XML error response. */
+function handleError(server, req, res, err) {
+  const s3Error =
+    err instanceof S3Error
+      ? err
+      : new S3Error(
+          'InternalError',
+          'We encountered an internal error. Please try again.'
+        );
+  if (!(err instanceof S3Error)) {
+    server.logger.error(err.stack || String(err));
+  }
+  if (res.headersSent) {
+    res.destroy();
+    return;
+  }
+  // Drain any unread request body so keep-alive connections stay usable.
+  req.resume();
+  sendXml(
+    res,
+    s3Error.statusCode,
+    s3Error.toXml({ requestId: req.requestId, resource: req.s3Resource })
+  );
+}
+
+module.exports = { handleError };

package/lib/middleware/vhost.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const { escapeRegExp } = require('../utils');
+
+/**
+ * Resolves the bucket name from the Host header for virtual-host-style
+ * requests (bucket.localhost, bucket.s3.<region>.<serviceEndpoint>, ...).
+ * Returns null for path-style requests.
+ */
+function resolveVhostBucket(req, options) {
+  if (!options.vhostBuckets) return null;
+  const host = String(req.headers.host || '')
+    .replace(/:\d+$/, '')
+    .toLowerCase();
+  if (
+    !host ||
+    host === 'localhost' ||
+    /^[\d.]+$/.test(host) ||
+    host === '[::1]'
+  ) {
+    return null;
+  }
+  if (host === options.address || host === options.serviceEndpoint) return null;
+
+  const patterns = [
+    // bucket.s3.region.amazonaws.com / bucket.s3-region.amazonaws.com
+    new RegExp(
+      `^(.+)\\.s3[.-][a-z0-9-]+\\.${escapeRegExp(options.serviceEndpoint)}$`
+    ),
+    // bucket.s3.amazonaws.com
+    new RegExp(`^(.+)\\.s3\\.${escapeRegExp(options.serviceEndpoint)}$`),
+    // bucket.amazonaws.com-style custom endpoints
+    new RegExp(`^(.+)\\.${escapeRegExp(options.serviceEndpoint)}$`),
+    // bucket.localhost
+    /^(.+)\.localhost$/,
+  ];
+  if (
+    options.address &&
+    options.address !== 'localhost' &&
+    !/^[\d.]+$/.test(options.address)
+  ) {
+    patterns.push(new RegExp(`^(.+)\\.${escapeRegExp(options.address)}$`));
+  }
+  for (const pattern of patterns) {
+    const match = pattern.exec(host);
+    if (match) {
+      const prefix = match[1];
+      if (
+        prefix === 's3' ||
+        prefix.startsWith('s3.') ||
+        prefix.startsWith('s3-')
+      ) {
+        return null;
+      }
+      return prefix;
+    }
+  }
+  return null;
+}
+
+module.exports = { resolveVhostBucket };

package/lib/middleware/website.js

@@ -0,0 +1,108 @@
+'use strict';
+
+const { send } = require('../models/response');
+const { escapeXml } = require('../xml');
+
+// Query parameters that indicate an S3 API call rather than a website fetch.
+const API_SUBRESOURCES = [
+  'acl',
+  'cors',
+  'website',
+  'tagging',
+  'location',
+  'versioning',
+  'versions',
+  'uploads',
+  'uploadId',
+  'delete',
+  'policy',
+  'lifecycle',
+  'encryption',
+  'object-lock',
+  'publicAccessBlock',
+  'notification',
+  'requestPayment',
+  'logging',
+  'accelerate',
+  'attributes',
+  'list-type',
+  'replication',
+  'torrent',
+];
+
+/** True when the request should be served in static-website mode. */
+function isWebsiteRequest(ctx, bucketMeta) {
+  return Boolean(
+    ctx.bucket &&
+    bucketMeta &&
+    bucketMeta.configs.website &&
+    (ctx.req.method === 'GET' || ctx.req.method === 'HEAD') &&
+    ctx.auth.mode === 'anonymous' &&
+    !API_SUBRESOURCES.some((subresource) => ctx.query.has(subresource))
+  );
+}
+
+async function serveWebsite(ctx, website) {
+  if (website.redirectAllRequestsTo) {
+    const target = `${website.redirectAllRequestsTo.protocol}://${website.redirectAllRequestsTo.hostName}/${ctx.key}`;
+    send(ctx.res, 301, { Location: target });
+    return;
+  }
+  let key = ctx.key;
+  if (key === '' || key.endsWith('/')) {
+    key += website.indexDocument;
+  }
+  const meta = await ctx.store.getObject(ctx.bucket, key);
+  if (!meta && !ctx.key.endsWith('/') && ctx.key !== '') {
+    // folder-style redirect: /foo → /foo/ when foo/index.html exists
+    const indexKey = `${ctx.key}/${website.indexDocument}`;
+    if (await ctx.store.getObject(ctx.bucket, indexKey)) {
+      send(ctx.res, 302, { Location: `/${ctx.key}/` });
+      return;
+    }
+  }
+  if (meta && meta.websiteRedirectLocation) {
+    send(ctx.res, 301, { Location: meta.websiteRedirectLocation });
+    return;
+  }
+  if (!meta) {
+    if (website.errorDocument) {
+      const errorMeta = await ctx.store.getObject(
+        ctx.bucket,
+        website.errorDocument
+      );
+      if (errorMeta) {
+        ctx.res.writeHead(404, {
+          'Content-Type': errorMeta.contentType || 'text/html',
+          'Content-Length': errorMeta.size,
+        });
+        if (ctx.req.method === 'HEAD') {
+          ctx.res.end();
+          return;
+        }
+        ctx.store
+          .createObjectReadStream(ctx.bucket, website.errorDocument)
+          .pipe(ctx.res);
+        return;
+      }
+    }
+    const html =
+      '<!DOCTYPE html><html><head><title>404 Not Found</title></head>' +
+      `<body><h1>404 Not Found</h1><ul><li>Code: NoSuchKey</li><li>Message: The specified key does not exist.</li><li>Key: ${escapeXml(key)}</li></ul></body></html>`;
+    send(ctx.res, 404, { 'Content-Type': 'text/html' }, html);
+    return;
+  }
+  ctx.res.writeHead(200, {
+    'Content-Type': meta.contentType || 'text/html',
+    'Content-Length': meta.size,
+    ETag: `"${meta.etag}"`,
+    'Last-Modified': new Date(meta.lastModified).toUTCString(),
+  });
+  if (ctx.req.method === 'HEAD') {
+    ctx.res.end();
+    return;
+  }
+  ctx.store.createObjectReadStream(ctx.bucket, key).pipe(ctx.res);
+}
+
+module.exports = { isWebsiteRequest, serveWebsite };

package/lib/models/error.js

@@ -0,0 +1,79 @@
+'use strict';
+
+const STATUS_CODES = {
+  AccessDenied: 403,
+  AuthorizationHeaderMalformed: 400,
+  AuthorizationQueryParametersError: 400,
+  BadDigest: 400,
+  BucketAlreadyExists: 409,
+  BucketAlreadyOwnedByYou: 409,
+  BucketNotEmpty: 409,
+  EntityTooSmall: 400,
+  EntityTooLarge: 400,
+  ExpiredToken: 400,
+  IncompleteBody: 400,
+  InternalError: 500,
+  InvalidAccessKeyId: 403,
+  InvalidArgument: 400,
+  InvalidBucketName: 400,
+  InvalidDigest: 400,
+  InvalidPart: 400,
+  InvalidPartOrder: 400,
+  InvalidRange: 416,
+  InvalidRequest: 400,
+  InvalidToken: 400,
+  InvalidURI: 400,
+  KeyTooLongError: 400,
+  MalformedPOSTRequest: 400,
+  MalformedXML: 400,
+  MethodNotAllowed: 405,
+  MissingContentLength: 411,
+  MissingRequestBodyError: 400,
+  MissingSecurityHeader: 400,
+  NoSuchBucket: 404,
+  NoSuchBucketPolicy: 404,
+  NoSuchCORSConfiguration: 404,
+  NoSuchKey: 404,
+  NoSuchLifecycleConfiguration: 404,
+  NoSuchTagSet: 404,
+  NoSuchUpload: 404,
+  NoSuchWebsiteConfiguration: 404,
+  NotImplemented: 501,
+  PreconditionFailed: 412,
+  RequestTimeTooSkewed: 403,
+  SignatureDoesNotMatch: 403,
+  XAmzContentSHA256Mismatch: 400,
+};
+
+/**
+ * An error that renders as an S3-style XML error response.
+ * `detail` entries become extra XML elements in the <Error> document
+ * (e.g. StringToSign for SignatureDoesNotMatch).
+ */
+class S3Error extends Error {
+  constructor(code, message, detail = {}) {
+    super(message);
+    this.name = 'S3Error';
+    this.code = code;
+    this.statusCode = detail.statusCode || STATUS_CODES[code] || 500;
+    this.detail = { ...detail };
+    delete this.detail.statusCode;
+  }
+
+  toXml({ requestId, resource } = {}) {
+    const { escapeXml } = require('../xml');
+    let body = '<?xml version="1.0" encoding="UTF-8"?>\n<Error>';
+    body += `<Code>${escapeXml(this.code)}</Code>`;
+    body += `<Message>${escapeXml(this.message)}</Message>`;
+    for (const [key, value] of Object.entries(this.detail)) {
+      if (value === undefined || value === null) continue;
+      body += `<${key}>${escapeXml(String(value))}</${key}>`;
+    }
+    if (resource) body += `<Resource>${escapeXml(resource)}</Resource>`;
+    body += `<RequestId>${escapeXml(requestId || '')}</RequestId>`;
+    body += '</Error>';
+    return body;
+  }
+}
+
+module.exports = { S3Error, STATUS_CODES };

package/lib/models/response.js

@@ -0,0 +1,157 @@
+'use strict';
+
+const { buildXml, S3_XMLNS } = require('../xml');
+const { S3Error } = require('./error');
+const { compareKeys, awsUriEncode, isoDate, DUMMY_OWNER } = require('../utils');
+
+function send(res, status, headers, body) {
+  if (res.headersSent) {
+    res.end();
+    return;
+  }
+  for (const [name, value] of Object.entries(headers || {})) {
+    if (value !== undefined && value !== null) res.setHeader(name, value);
+  }
+  const isHead = res.req && res.req.method === 'HEAD';
+  if (body !== undefined && body !== null) {
+    const buf = Buffer.isBuffer(body) ? body : Buffer.from(body);
+    res.setHeader('Content-Length', buf.length);
+    res.writeHead(status);
+    res.end(isHead ? undefined : buf);
+  } else {
+    res.writeHead(status);
+    res.end();
+  }
+}
+
+function sendXml(res, status, xmlString) {
+  send(res, status, { 'Content-Type': 'application/xml' }, xmlString);
+}
+
+function sendDoc(res, status, rootName, content, { xmlns = S3_XMLNS } = {}) {
+  sendXml(res, status, buildXml(rootName, content, { xmlns }));
+}
+
+/**
+ * Shared pagination for ListObjects / ListObjectsV2 / ListObjectVersions.
+ * `metas` must be sorted by key. Returns included object metas, grouped
+ * common prefixes and the last emitted item (for Next* markers).
+ */
+function paginate(
+  metas,
+  { prefix = '', delimiter = '', marker = '', maxKeys = 1000 }
+) {
+  const contents = [];
+  const prefixes = [];
+  const seenPrefixes = new Set();
+  let truncated = false;
+  let lastItem = null;
+
+  if (maxKeys === 0) {
+    return { contents, prefixes, truncated: false, lastItem: null };
+  }
+  for (const meta of metas) {
+    if (prefix && !meta.key.startsWith(prefix)) continue;
+    let item = meta.key;
+    let isPrefix = false;
+    if (delimiter) {
+      const rest = meta.key.slice(prefix.length);
+      const idx = rest.indexOf(delimiter);
+      if (idx !== -1) {
+        item = prefix + rest.slice(0, idx + delimiter.length);
+        isPrefix = true;
+      }
+    }
+    if (marker && compareKeys(item, marker) <= 0) continue;
+    if (isPrefix && seenPrefixes.has(item)) continue;
+    if (contents.length + prefixes.length >= maxKeys) {
+      truncated = true;
+      break;
+    }
+    if (isPrefix) {
+      seenPrefixes.add(item);
+      prefixes.push(item);
+    } else {
+      contents.push(meta);
+    }
+    lastItem = item;
+  }
+  return { contents, prefixes, truncated, lastItem };
+}
+
+function parseListParams(query) {
+  const maxKeysRaw = query.get('max-keys');
+  let maxKeys = 1000;
+  if (maxKeysRaw !== null) {
+    maxKeys = parseInt(maxKeysRaw, 10);
+    if (Number.isNaN(maxKeys) || maxKeys < 0) {
+      throw new S3Error(
+        'InvalidArgument',
+        'Argument max-keys must be an integer between 0 and 2147483647',
+        {
+          ArgumentName: 'max-keys',
+          ArgumentValue: maxKeysRaw,
+        }
+      );
+    }
+    maxKeys = Math.min(maxKeys, 1000);
+  }
+  const encodingType = query.get('encoding-type') || undefined;
+  if (encodingType && encodingType !== 'url') {
+    throw new S3Error(
+      'InvalidArgument',
+      'Invalid Encoding Method specified in Request',
+      {
+        ArgumentName: 'encoding-type',
+        ArgumentValue: encodingType,
+      }
+    );
+  }
+  return {
+    prefix: query.get('prefix') || '',
+    delimiter: query.get('delimiter') || '',
+    maxKeys,
+    encodingType,
+  };
+}
+
+function encodeListValue(value, encodingType) {
+  if (value === undefined || value === null || value === '') return value;
+  return encodingType === 'url' ? awsUriEncode(value, false) : value;
+}
+
+function contentsEntry(meta, encodingType, { owner = true } = {}) {
+  const entry = {
+    Key: encodeListValue(meta.key, encodingType),
+    LastModified: isoDate(meta.lastModified),
+    ETag: `"${meta.etag}"`,
+    Size: meta.size,
+    StorageClass: meta.storageClass || 'STANDARD',
+  };
+  if (meta.checksumAlgorithm) {
+    entry.ChecksumAlgorithm = meta.checksumAlgorithm;
+    if (meta.checksumType) entry.ChecksumType = meta.checksumType;
+  }
+  if (owner) entry.Owner = DUMMY_OWNER;
+  return entry;
+}
+
+function quoteEtag(etag) {
+  return `"${etag}"`;
+}
+
+function stripEtagQuotes(value) {
+  return String(value || '').replace(/^(W\/)?"?|"?$/g, '');
+}
+
+module.exports = {
+  send,
+  sendXml,
+  sendDoc,
+  paginate,
+  parseListParams,
+  encodeListValue,
+  contentsEntry,
+  quoteEtag,
+  stripEtagQuotes,
+};

package/lib/routes.js

@@ -0,0 +1,155 @@
+'use strict';
+
+const { S3Error } = require('./models/error');
+const { authenticate } = require('./signature');
+const { generateRequestId, generateId2 } = require('./utils');
+const { resolveVhostBucket } = require('./middleware/vhost');
+const { applyCorsHeaders, handlePreflight } = require('./middleware/cors');
+const { isWebsiteRequest, serveWebsite } = require('./middleware/website');
+const { handleError } = require('./middleware/error-handler');
+const serviceController = require('./controllers/service');
+const bucketController = require('./controllers/bucket');
+const objectController = require('./controllers/object');
+const { postFormUpload } = require('./controllers/post');
+
+/** Builds the `(req, res)` handler that routes requests to the controllers. */
+function createRequestHandler(server) {
+  return function requestHandler(req, res) {
+    handleRequest(server, req, res).catch((err) => {
+      handleError(server, req, res, err);
+    });
+  };
+}
+
+async function handleRequest(server, req, res) {
+  const { options, store, logger } = server;
+  req.requestId = generateRequestId();
+  res.setHeader('x-amz-request-id', req.requestId);
+  res.setHeader('x-amz-id-2', generateId2());
+
+  res.on('finish', () => {
+    logger.info(`${req.method} ${req.url} ${res.statusCode}`);
+  });
+
+  const queryIndex = req.url.indexOf('?');
+  const rawPath = queryIndex === -1 ? req.url : req.url.slice(0, queryIndex);
+  const rawQuery = queryIndex === -1 ? '' : req.url.slice(queryIndex + 1);
+  const query = new URLSearchParams(rawQuery);
+
+  const vhostBucket = resolveVhostBucket(req, options);
+  let bucket = vhostBucket;
+  let key = '';
+  try {
+    if (vhostBucket) {
+      key = decodeURIComponent(rawPath.slice(1));
+    } else {
+      const trimmed = rawPath.replace(/^\//, '');
+      const slash = trimmed.indexOf('/');
+      if (slash === -1) {
+        bucket = decodeURIComponent(trimmed) || null;
+      } else {
+        bucket = decodeURIComponent(trimmed.slice(0, slash));
+        key = decodeURIComponent(trimmed.slice(slash + 1));
+      }
+    }
+  } catch {
+    throw new S3Error('InvalidURI', "Couldn't parse the specified URI.");
+  }
+
+  const ctx = {
+    req,
+    res,
+    store,
+    options,
+    logger,
+    query,
+    rawPath,
+    bucket,
+    key,
+    auth: null,
+    requestId: req.requestId,
+    emitEvent: (eventName, objectMeta) =>
+      server._emitEvent(eventName, bucket, objectMeta),
+  };
+  req.s3Resource = bucket ? `/${bucket}${key ? `/${key}` : ''}` : '/';
+
+  // CORS
+  const bucketMeta = bucket ? await store.getBucket(bucket) : null;
+  const corsConfig = bucketMeta && bucketMeta.configs.cors;
+  if (req.method === 'OPTIONS') {
+    handlePreflight(ctx, corsConfig);
+    return;
+  }
+  applyCorsHeaders(ctx, corsConfig);
+
+  // Authentication (anonymous requests are allowed, like public buckets)
+  ctx.auth = authenticate({ req, rawPath, query, vhostBucket, options });
+
+  // Static website hosting
+  if (isWebsiteRequest(ctx, bucketMeta)) {
+    await serveWebsite(ctx, bucketMeta.configs.website);
+    return;
+  }
+
+  if (!bucket) {
+    if (req.method === 'GET') {
+      await serviceController.listBuckets(ctx);
+      return;
+    }
+    throw methodNotAllowed(req.method, 'SERVICE');
+  }
+
+  if (!key) {
+    switch (req.method) {
+      case 'GET':
+        await bucketController.getBucket(ctx);
+        return;
+      case 'PUT':
+        await bucketController.putBucket(ctx);
+        return;
+      case 'DELETE':
+        await bucketController.deleteBucket(ctx);
+        return;
+      case 'HEAD':
+        await bucketController.headBucket(ctx);
+        return;
+      case 'POST':
+        if (query.has('delete')) {
+          await objectController.deleteObjects(ctx);
+        } else {
+          await postFormUpload(ctx);
+        }
+        return;
+      default:
+        throw methodNotAllowed(req.method, 'BUCKET');
+    }
+  }
+
+  switch (req.method) {
+    case 'GET':
+    case 'HEAD':
+      await objectController.getOrHeadObject(ctx);
+      return;
+    case 'PUT':
+      await objectController.putObject(ctx);
+      return;
+    case 'POST':
+      await objectController.postObject(ctx);
+      return;
+    case 'DELETE':
+      await objectController.deleteObjectRoute(ctx);
+      return;
+    default:
+      throw methodNotAllowed(req.method, 'OBJECT');
+  }
+}
+
+function methodNotAllowed(method, resourceType) {
+  return new S3Error(
+    'MethodNotAllowed',
+    'The specified method is not allowed against this resource.',
+    { Method: method, ResourceType: resourceType }
+  );
+}
+
+module.exports = { createRequestHandler };