s3mini 0.9.0 → 0.9.2

package/src/S3.ts

@@ -15,6 +15,8 @@ import {
   extractErrCode,
   S3NetworkError,
   S3ServiceError,
+  generateParts,
+  toUint8Array,
 } from './utils.js';
 import type * as IT from './types.js';
 
@@ -55,6 +57,7 @@ class S3mini {
    * @param {number} [config.requestAbortTimeout=undefined] - The timeout in milliseconds after which a request should be aborted (careful on streamed requests).
    * @param {Object} [config.logger=null] - A logger object with methods like info, warn, error.
    * @param {typeof fetch} [config.fetch=globalThis.fetch] - Custom fetch implementation to use for HTTP requests.
+   * @param {number} [config.minPartSize=8388608] - The minimum part size for multipart uploads in bytes (default is 8MB).
    * @throws {TypeError} Will throw an error if required parameters are missing or of incorrect type.
    */
   readonly #accessKeyId: string;
@@ -66,6 +69,7 @@ class S3mini {
   readonly requestAbortTimeout?: number;
   readonly logger?: IT.Logger;
   readonly _fetch: typeof fetch;
+  readonly minPartSize: number;
   private signingKeyDate?: string;
   private signingKey?: ArrayBuffer;
 
@@ -78,6 +82,7 @@ class S3mini {
     requestAbortTimeout = undefined,
     logger = undefined,
     fetch = globalThis.fetch,
+    minPartSize = C.MIN_PART_SIZE,
   }: IT.S3Config) {
     this._validateConstructorParams(accessKeyId, secretAccessKey, endpoint);
     this.#accessKeyId = accessKeyId;
@@ -89,6 +94,7 @@ class S3mini {
     this.requestAbortTimeout = requestAbortTimeout;
     this.logger = logger;
     this._fetch = (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => fetch(input, init);
+    this.minPartSize = minPartSize;
   }
 
   private _sanitize(obj: unknown): unknown {
@@ -241,18 +247,24 @@ class S3mini {
     return { filteredOpts, conditionalHeaders };
   }
 
-  private _validateData(data: unknown): BodyInit {
-    if (!((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string')) {
-      this._log('error', C.ERROR_DATA_BUFFER_REQUIRED);
-      throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED);
-    }
-    return data;
-  }
+  // private _validateData(data: unknown): BodyInit {
+  //   if (data instanceof ArrayBuffer) {
+  //     return data;
+  //   }
+  //   if (data instanceof Uint8Array) {
+  //     return data as unknown as BodyInit;
+  //   }
+  //   if ((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string') {
+  //     return data as BodyInit;
+  //   }
+  //   this._log('error', C.ERROR_DATA_BUFFER_REQUIRED);
+  //   throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED);
+  // }
 
   private _validateUploadPartParams(
     key: string,
     uploadId: string,
-    data: IT.MaybeBuffer | string,
+    data: IT.DataInput,
     partNumber: number,
     opts: object,
   ): BodyInit {
@@ -266,7 +278,7 @@ class S3mini {
       throw new TypeError(`${C.ERROR_PREFIX}partNumber must be a positive integer`);
     }
     this._checkOpts(opts);
-    return this._validateData(data);
+    return data as BodyInit;
   }
 
   private async _sign(
@@ -564,6 +576,7 @@ class S3mini {
     });
 
     if (res.status === 404) {
+      void res.body?.cancel();
       return null;
     }
 
@@ -634,10 +647,16 @@ class S3mini {
 
     // Extract regular objects from Contents
     if (contents) {
-      if (Array.isArray(contents)) {
-        objects.push(...(contents as IT.ListObject[]));
-      } else {
-        objects.push(contents as IT.ListObject);
+      const raw = Array.isArray(contents) ? contents : [contents];
+      for (const item of raw) {
+        const o = item as Record<string, string>;
+        objects.push({
+          Key: o.Key ?? o.key ?? '',
+          Size: Number(o.Size ?? o.size ?? 0),
+          LastModified: new Date(o.LastModified ?? o.lastModified ?? 0),
+          ETag: o.ETag ?? o.etag ?? '',
+          StorageClass: o.StorageClass ?? o.storageClass ?? '',
+        });
       }
     }
 
@@ -742,6 +761,7 @@ class S3mini {
     if (s === 200) {
       return res.text();
     }
+    void res.body?.cancel();
     return null;
   }
 
@@ -766,6 +786,7 @@ class S3mini {
     if (res.status === 200) {
       return res;
     }
+    void res.body?.cancel();
     return null;
   }
 
@@ -790,6 +811,7 @@ class S3mini {
     if (res.status === 200) {
       return res.arrayBuffer();
     }
+    void res.body?.cancel();
     return null;
   }
 
@@ -814,6 +836,7 @@ class S3mini {
     if (res.status === 200) {
       return res.json() as Promise<T>;
     }
+    void res.body?.cancel();
     return null;
   }
 
@@ -838,6 +861,7 @@ class S3mini {
       });
       const s = res.status;
       if (s === 404 || s === 412 || s === 304) {
+        void res.body?.cancel();
         return { etag: null, data: null };
       }
 
@@ -900,7 +924,9 @@ class S3mini {
       return len ? +len : 0;
     } catch (err) {
       this._log('error', `Error getting content length for object ${key}: ${String(err)}`);
-      throw new Error(`${C.ERROR_PREFIX}Error getting content length for object ${key}: ${String(err)}`);
+      throw new Error(`${C.ERROR_PREFIX}Error getting content length for object ${key}: ${String(err)}`, {
+        cause: err,
+      });
     }
   }
 
@@ -969,7 +995,7 @@ class S3mini {
   /**
    * Uploads an object to the S3-compatible service.
    * @param {string} key - The key/path where the object will be stored.
-   * @param {string | Buffer} data - The data to upload (string or Buffer).
+   * @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
    * @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
    * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
    * @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
@@ -985,15 +1011,17 @@ class S3mini {
    */
   public async putObject(
     key: string,
-    data: string | IT.MaybeBuffer,
+    data: string | IT.DataInput | ReadableStream | File | Blob,
     fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
     ssecHeaders?: IT.SSECHeaders,
     additionalHeaders?: IT.AWSHeaders,
+    contentLength?: number,
   ): Promise<Response> {
+    const size = contentLength ?? getByteSize(data);
     return this._signedRequest('PUT', key, {
-      body: this._validateData(data),
+      body: data as BodyInit,
       headers: {
-        [C.HEADER_CONTENT_LENGTH]: getByteSize(data),
+        ...(size && { [C.HEADER_CONTENT_LENGTH]: size }),
         [C.HEADER_CONTENT_TYPE]: fileType,
         ...additionalHeaders,
         ...ssecHeaders,
@@ -1002,6 +1030,235 @@ class S3mini {
     });
   }
 
+  /**
+   * Put object that automatically chooses single PUT vs multipart.
+   * Same signature/shape as putObject so callers don't need to change.
+   * @param {string} key - The key/path where the object will be stored.
+   * @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
+   * @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
+   * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
+   * @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
+   * @param {number} [contentLength] - Optional known content length of data.
+   * @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
+   * @throws {TypeError} If data is not a string or Buffer.
+   * @example
+   * // Upload text file
+   * await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
+   *
+   * // Upload binary data
+   * const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
+   * await s3.putAnyObject('image.png', buffer, 'image/png');
+   */
+  public async putAnyObject(
+    key: string,
+    data: IT.DataInput,
+    fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
+    ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
+    contentLength?: number,
+  ): Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }> {
+    const size = contentLength ?? getByteSize(data);
+
+    // Single PUT for small files
+    if (!Number.isNaN(size) && size <= this.minPartSize) {
+      return this.putObject(key, data, fileType, ssecHeaders, additionalHeaders, contentLength);
+    }
+
+    this._checkKey(key);
+    return this._multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders);
+  }
+
+  private async _multipartUpload(
+    key: string,
+    data: IT.DataInput,
+    fileType: string,
+    ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
+  ): Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }> {
+    const uploadId = await this.getMultipartUploadId(key, fileType, ssecHeaders, additionalHeaders);
+
+    try {
+      const parts = await this._uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders);
+      parts.sort((a, b) => a.partNumber - b.partNumber);
+      const result = await this.completeMultipartUpload(key, uploadId, parts);
+      return this._createSuccessResponse(result.etag || '');
+    } catch (err) {
+      await this._safeAbortUpload(key, uploadId);
+      throw err;
+    }
+  }
+
+  private async _uploadKnownSizePartsParallel(
+    key: string,
+    uploadId: string,
+    data: Uint8Array | Blob,
+    ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
+    concurrency: number = 4,
+    maxRetries: number = 3,
+  ): Promise<IT.UploadPart[]> {
+    const partSize = this.minPartSize;
+    const totalSize = data instanceof Blob ? data.size : data.byteLength;
+    const totalParts = Math.ceil(totalSize / partSize);
+    const results: IT.UploadPart[] = new Array(totalParts) as IT.UploadPart[];
+    let nextIndex = 0;
+
+    const worker = async (): Promise<void> => {
+      while (true) {
+        const index = nextIndex++;
+        if (index >= totalParts) {
+          return;
+        }
+
+        const start = index * partSize;
+        const end = Math.min(start + partSize, totalSize);
+        const part =
+          data instanceof Blob
+            ? await data.slice(start, end).arrayBuffer() // Must await - R2 needs actual bytes
+            : data.subarray(start, end);
+
+        results[index] = await this._uploadPartWithRetry(
+          key,
+          uploadId,
+          part,
+          index + 1,
+          ssecHeaders,
+          additionalHeaders,
+          maxRetries,
+        );
+      }
+    };
+
+    await Promise.all(Array.from({ length: Math.min(concurrency, totalParts) }, () => worker()));
+    return results;
+  }
+
+  private async _uploadPartsOptimized(
+    key: string,
+    uploadId: string,
+    data: IT.DataInput,
+    ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
+    concurrency: number = 4,
+    maxRetries: number = 3,
+  ): Promise<IT.UploadPart[]> {
+    const bytes = toUint8Array(data);
+    if (bytes) {
+      return this._uploadKnownSizePartsParallel(
+        key,
+        uploadId,
+        bytes,
+        ssecHeaders,
+        additionalHeaders,
+        concurrency,
+        maxRetries,
+      );
+    }
+    if (data instanceof Blob) {
+      return this._uploadKnownSizePartsParallel(
+        key,
+        uploadId,
+        data,
+        ssecHeaders,
+        additionalHeaders,
+        concurrency,
+        maxRetries,
+      );
+    }
+    return this._uploadStreamingParts(
+      key,
+      uploadId,
+      data as ReadableStream,
+      ssecHeaders,
+      additionalHeaders,
+      concurrency,
+      maxRetries,
+    );
+  }
+
+  private async _uploadStreamingParts(
+    key: string,
+    uploadId: string,
+    stream: ReadableStream,
+    ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
+    concurrency: number = 4,
+    maxRetries: number = 3,
+  ): Promise<IT.UploadPart[]> {
+    const parts: IT.UploadPart[] = [];
+    const active = new Set<Promise<void>>();
+    let partNumber = 0;
+
+    for await (const partData of generateParts(stream, this.minPartSize)) {
+      const currentPartNumber = ++partNumber;
+
+      while (active.size >= concurrency) {
+        await Promise.race(active);
+      }
+
+      const p = this._uploadPartWithRetry(
+        key,
+        uploadId,
+        partData,
+        currentPartNumber,
+        ssecHeaders,
+        additionalHeaders,
+        maxRetries,
+      ).then(part => {
+        parts.push(part);
+        active.delete(p);
+      });
+
+      active.add(p);
+    }
+
+    await Promise.all(active);
+    return parts;
+  }
+
+  private async _uploadPartWithRetry(
+    key: string,
+    uploadId: string,
+    data: IT.PartData,
+    partNumber: number,
+    ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
+    maxRetries: number = 3,
+  ): Promise<IT.UploadPart> {
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        return await this.uploadPart(key, uploadId, data, partNumber, {}, ssecHeaders, additionalHeaders);
+      } catch (err) {
+        if (attempt === maxRetries) {
+          throw err;
+        }
+        await new Promise(r => setTimeout(r, Math.min(1000 * 2 ** attempt, 10000)));
+      }
+    }
+    throw new Error('Unreachable');
+  }
+
+  private async _safeAbortUpload(key: string, uploadId: string): Promise<void> {
+    try {
+      await this.abortMultipartUpload(key, uploadId);
+    } catch (err) {
+      this._log('warn', `Failed to abort multipart upload: ${String(err)}`);
+    }
+  }
+
+  private _createSuccessResponse(
+    etag: string,
+  ): Response | { ok: boolean; status: number; headers: Map<string, string> } {
+    if (typeof Response !== 'undefined') {
+      const headers = new Headers();
+      if (etag) {
+        headers.set('ETag', etag);
+      }
+      return new Response('', { status: 200, headers });
+    }
+    return { ok: true, status: 200, headers: new Map([['ETag', etag]]) };
+  }
+
   /**
    * Initiates a multipart upload and returns the upload ID.
    * @param {string} key - The key/path where the object will be stored.
@@ -1018,13 +1275,14 @@ class S3mini {
     key: string,
     fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
     ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
   ): Promise<string> {
     this._checkKey(key);
     if (typeof fileType !== 'string') {
       throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`);
     }
     const query = { uploads: '' };
-    const headers = { [C.HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders };
+    const headers = { [C.HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders, ...additionalHeaders };
 
     const res = await this._signedRequest('POST', key, {
       query,
@@ -1056,7 +1314,7 @@ class S3mini {
    * Uploads a part in a multipart upload.
    * @param {string} key - The key of the object being uploaded.
    * @param {string} uploadId - The upload ID from getMultipartUploadId.
-   * @param {Buffer | string} data - The data for this part.
+   * @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
    * @param {number} partNumber - The part number (must be between 1 and 10,000).
    * @param {Record<string, unknown>} [opts={}] - Additional options for the request.
    * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
@@ -1074,20 +1332,23 @@ class S3mini {
   public async uploadPart(
     key: string,
     uploadId: string,
-    data: IT.MaybeBuffer | string,
+    data: IT.DataInput,
     partNumber: number,
     opts: Record<string, unknown> = {},
     ssecHeaders?: IT.SSECHeaders,
+    additionalHeaders?: IT.AWSHeaders,
   ): Promise<IT.UploadPart> {
     const body = this._validateUploadPartParams(key, uploadId, data, partNumber, opts);
 
     const query = { uploadId, partNumber, ...opts };
+    const size = getByteSize(data);
     const res = await this._signedRequest('PUT', key, {
       query,
       body,
       headers: {
-        [C.HEADER_CONTENT_LENGTH]: getByteSize(data),
+        ...(size && !Number.isNaN(size) && { [C.HEADER_CONTENT_LENGTH]: size }),
         ...ssecHeaders,
+        ...additionalHeaders,
       },
     });
 
@@ -1600,6 +1861,84 @@ class S3mini {
       .sort((a, b) => a.localeCompare(b))
       .join('&');
   }
+  /**
+   * Generates a pre-signed URL for direct client access to an S3 object.
+   * The URL embeds authentication in query parameters instead of headers,
+   * allowing unauthenticated HTTP clients to perform the specified operation.
+   *
+   * @param {'GET' | 'PUT'} method - HTTP method ('GET' for download, 'PUT' for upload)
+   * @param {string} key - The object key/path
+   * @param {number} [expiresIn=3600] - URL expiration time in seconds (1–604800)
+   * @param {Record<string, string>} [queryParams={}] - Additional query parameters to sign
+   * @returns {Promise<string>} Pre-signed URL string
+   * @throws {TypeError} If key is empty or expiresIn is out of range
+   * @example
+   * // Download URL valid for 1 hour
+   * const url = await s3.getPresignedUrl('GET', 'photos/vacation.jpg');
+   *
+   * // Upload URL valid for 5 minutes
+   * const url = await s3.getPresignedUrl('PUT', 'uploads/file.bin', 300);
+   *
+   * // Client-side usage (no credentials needed)
+   * await fetch(url, { method: 'PUT', body: data });
+   */
+  public async getPresignedUrl(
+    method: 'GET' | 'PUT',
+    key: string,
+    expiresIn: number = 3600,
+    queryParams: Record<string, string> = {},
+  ): Promise<string> {
+    this._checkKey(key);
+    if (!Number.isFinite(expiresIn) || expiresIn <= 0 || expiresIn > 604800) {
+      throw new TypeError(`${C.ERROR_PREFIX}expiresIn must be between 1 and 604800 seconds`);
+    }
+    return this._presign(method, uriResourceEscape(key), Math.floor(expiresIn), queryParams);
+  }
+
+  private async _presign(
+    method: string,
+    keyPath: string,
+    expiresIn: number,
+    queryParams: Record<string, string>,
+  ): Promise<string> {
+    const url = new URL(this.endpoint);
+    if (keyPath.length > 0) {
+      url.pathname =
+        url.pathname === '/' ? `/${keyPath.replace(/^\/+/, '')}` : `${url.pathname}/${keyPath.replace(/^\/+/, '')}`;
+    }
+
+    const d = new Date();
+    const year = d.getUTCFullYear();
+    const month = String(d.getUTCMonth() + 1).padStart(2, '0');
+    const day = String(d.getUTCDate()).padStart(2, '0');
+    const shortDatetime = `${year}${month}${day}`;
+    const fullDatetime = `${shortDatetime}T${String(d.getUTCHours()).padStart(2, '0')}${String(d.getUTCMinutes()).padStart(2, '0')}${String(d.getUTCSeconds()).padStart(2, '0')}Z`;
+    const credentialScope = `${shortDatetime}/${this.region}/${C.S3_SERVICE}/${C.AWS_REQUEST_TYPE}`;
+
+    const signedHeaders = 'host';
+
+    const allQueryParams: Record<string, string> = {
+      ...queryParams,
+      'X-Amz-Algorithm': C.AWS_ALGORITHM,
+      'X-Amz-Credential': `${this.#accessKeyId}/${credentialScope}`,
+      'X-Amz-Date': fullDatetime,
+      'X-Amz-Expires': String(expiresIn),
+      'X-Amz-SignedHeaders': signedHeaders,
+    };
+
+    const canonicalQueryString = this._buildCanonicalQueryString(allQueryParams);
+    const canonicalRequest = `${method}\n${url.pathname}\n${canonicalQueryString}\nhost:${url.host}\n\n${signedHeaders}\n${C.UNSIGNED_PAYLOAD}`;
+    const stringToSign = `${C.AWS_ALGORITHM}\n${fullDatetime}\n${credentialScope}\n${hexFromBuffer(await sha256(canonicalRequest))}`;
+
+    if (shortDatetime !== this.signingKeyDate || !this.signingKey) {
+      this.signingKeyDate = shortDatetime;
+      this.signingKey = await this._getSignatureKey(shortDatetime);
+    }
+
+    const signature = hexFromBuffer(await hmac(this.signingKey, stringToSign));
+    return `${url.origin}${url.pathname}?${canonicalQueryString}&X-Amz-Signature=${signature}`;
+  }
+
   private async _getSignatureKey(dateStamp: string): Promise<ArrayBuffer> {
     const kDate = await hmac(`AWS4${this.#secretAccessKey}`, dateStamp);
     const kRegion = await hmac(kDate, this.region);

package/src/consts.ts

@@ -11,6 +11,7 @@ export const JSON_CONTENT_TYPE = 'application/json';
 export const SENSITIVE_KEYS_REDACTED = new Set(['accesskeyid', 'secretaccesskey', 'sessiontoken', 'password', 'token']);
 export const IFHEADERS = new Set(['if-match', 'if-none-match', 'if-modified-since', 'if-unmodified-since']);
 export const DEFAULT_REQUEST_SIZE_IN_BYTES = 8 * 1024 * 1024;
+export const MIN_PART_SIZE = 8 * 1024 * 1024;
 
 // Headers
 export const HEADER_AMZ_CONTENT_SHA256 = 'x-amz-content-sha256';

package/src/types.ts

@@ -7,8 +7,11 @@ export interface S3Config {
   requestAbortTimeout?: number;
   logger?: Logger;
   fetch?: typeof fetch;
+  minPartSize?: number;
 }
 
+export type PartData = Uint8Array | Blob | ArrayBuffer;
+
 export interface SSECHeaders {
   'x-amz-server-side-encryption-customer-algorithm': string;
   'x-amz-server-side-encryption-customer-key': string;
@@ -151,12 +154,12 @@ export interface CopyObjectResult {
   lastModified?: Date;
 }
 
-/**
- * Where Buffer is available, e.g. when @types/node is loaded, we want to use it.
- * But it should be excluded in other environments (e.g. Cloudflare).
- */
-export type MaybeBuffer = typeof globalThis extends { Buffer?: infer B }
+type BinaryData = ArrayBuffer | Uint8Array;
+
+type MaybeBuffer = typeof globalThis extends { Buffer?: infer B }
   ? B extends new (...a: unknown[]) => unknown
-    ? InstanceType<B>
-    : ArrayBuffer | Uint8Array
-  : ArrayBuffer | Uint8Array;
+    ? InstanceType<B> | BinaryData
+    : BinaryData
+  : BinaryData;
+
+export type DataInput = string | MaybeBuffer | ReadableStream | File | Blob;