s3mini 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/s3mini.d.ts +42 -12
- package/dist/s3mini.js +250 -20
- package/dist/s3mini.js.map +1 -1
- package/dist/s3mini.min.js +1 -1
- package/dist/s3mini.min.js.map +1 -1
- package/package.json +6 -6
- package/src/S3.ts +264 -17
- package/src/consts.ts +1 -0
- package/src/types.ts +11 -8
- package/src/utils.ts +165 -3
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# s3mini | Tiny & fast S3 client for node and edge platforms.
|
|
2
2
|
|
|
3
|
-
`s3mini` is an ultra-lightweight Typescript client (~
|
|
3
|
+
`s3mini` is an ultra-lightweight Typescript client (~20 KB minified, ≈15 % more ops/s) for S3-compatible object storage. It runs on Node, Bun, Cloudflare Workers, and other edge platforms. It has been tested on Cloudflare R2, Backblaze B2, DigitalOcean Spaces, Ceph, Oracle, Garage and MinIO. (No Browser support!)
|
|
4
4
|
|
|
5
5
|
[[github](https://github.com/good-lly/s3mini)]
|
|
6
6
|
[[issues](https://github.com/good-lly/s3mini/issues)]
|
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
|
|
9
9
|
## Features
|
|
10
10
|
|
|
11
|
-
- 🚀 Light and fast: averages ≈15 % more ops/s and only ~
|
|
11
|
+
- 🚀 Light and fast: averages ≈15 % more ops/s and only ~20 KB (minified, not gzipped).
|
|
12
12
|
- 🔧 Zero dependencies; supports AWS SigV4 (no pre-signed requests) and SSE-C headers (tested only on Cloudflare)
|
|
13
13
|
- 🟠 Works on Cloudflare Workers; ideal for edge computing, Node, and Bun (no browser support).
|
|
14
14
|
- 🔑 Only the essential S3 APIs—improved list, put, get, delete, and a few more.
|
|
15
15
|
- 🛠️ Supports multipart uploads.
|
|
16
|
+
- 🎄 Tree-shakeable ES module.
|
|
16
17
|
- 🎯 TypeScript support with type definitions.
|
|
17
18
|
- 📚 Poorly-documented with examples and tests - But widely tested on various S3-compatible services! (Contributions welcome!)
|
|
18
19
|
- 📦 **BYOS3** — _Bring your own S3-compatible bucket_ (tested on Cloudflare R2, Backblaze B2, DigitalOcean Spaces, MinIO, Garage, Micro/Ceph and Oracle Object Storage, Scaleway).
|
|
@@ -43,9 +44,6 @@ Dev:
|
|
|
43
44
|
|
|
44
45
|
<a href="https://github.com/good-lly/s3mini/issues/"> <img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg" alt="Contributions welcome" /></a>
|
|
45
46
|
|
|
46
|
-
Performance tests was done on local Minio instance. Your results may vary depending on environment and network conditions, so take it with a grain of salt.
|
|
47
|
-

|
|
48
|
-
|
|
49
47
|
## Table of Contents
|
|
50
48
|
|
|
51
49
|
- [Supported Ops](#supported-ops)
|
package/dist/s3mini.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ interface S3Config {
|
|
|
7
7
|
requestAbortTimeout?: number;
|
|
8
8
|
logger?: Logger;
|
|
9
9
|
fetch?: typeof fetch;
|
|
10
|
+
minPartSize?: number;
|
|
10
11
|
}
|
|
11
12
|
interface SSECHeaders {
|
|
12
13
|
'x-amz-server-side-encryption-customer-algorithm': string;
|
|
@@ -124,13 +125,11 @@ interface CopyObjectResult {
|
|
|
124
125
|
etag: string;
|
|
125
126
|
lastModified?: Date;
|
|
126
127
|
}
|
|
127
|
-
|
|
128
|
-
* Where Buffer is available, e.g. when @types/node is loaded, we want to use it.
|
|
129
|
-
* But it should be excluded in other environments (e.g. Cloudflare).
|
|
130
|
-
*/
|
|
128
|
+
type BinaryData = ArrayBuffer | Uint8Array;
|
|
131
129
|
type MaybeBuffer = typeof globalThis extends {
|
|
132
130
|
Buffer?: infer B;
|
|
133
|
-
} ? B extends new (...a: unknown[]) => unknown ? InstanceType<B>
|
|
131
|
+
} ? B extends new (...a: unknown[]) => unknown ? InstanceType<B> | BinaryData : BinaryData : BinaryData;
|
|
132
|
+
type DataInput = string | MaybeBuffer | ReadableStream | File | Blob;
|
|
134
133
|
|
|
135
134
|
/**
|
|
136
135
|
* S3 class for interacting with S3-compatible object storage services.
|
|
@@ -164,9 +163,10 @@ declare class S3mini {
|
|
|
164
163
|
readonly requestAbortTimeout?: number;
|
|
165
164
|
readonly logger?: Logger;
|
|
166
165
|
readonly _fetch: typeof fetch;
|
|
166
|
+
readonly minPartSize: number;
|
|
167
167
|
private signingKeyDate?;
|
|
168
168
|
private signingKey?;
|
|
169
|
-
constructor({ accessKeyId, secretAccessKey, endpoint, region, requestSizeInBytes, requestAbortTimeout, logger, fetch, }: S3Config);
|
|
169
|
+
constructor({ accessKeyId, secretAccessKey, endpoint, region, requestSizeInBytes, requestAbortTimeout, logger, fetch, minPartSize, }: S3Config);
|
|
170
170
|
private _sanitize;
|
|
171
171
|
private _log;
|
|
172
172
|
private _validateConstructorParams;
|
|
@@ -182,7 +182,6 @@ declare class S3mini {
|
|
|
182
182
|
private _checkPrefix;
|
|
183
183
|
private _checkOpts;
|
|
184
184
|
private _filterIfHeaders;
|
|
185
|
-
private _validateData;
|
|
186
185
|
private _validateUploadPartParams;
|
|
187
186
|
private _sign;
|
|
188
187
|
private _signedRequest;
|
|
@@ -353,7 +352,7 @@ declare class S3mini {
|
|
|
353
352
|
/**
|
|
354
353
|
* Uploads an object to the S3-compatible service.
|
|
355
354
|
* @param {string} key - The key/path where the object will be stored.
|
|
356
|
-
* @param {string |
|
|
355
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
357
356
|
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
358
357
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
359
358
|
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
@@ -367,7 +366,38 @@ declare class S3mini {
|
|
|
367
366
|
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
368
367
|
* await s3.putObject('image.png', buffer, 'image/png');
|
|
369
368
|
*/
|
|
370
|
-
putObject(key: string, data: string |
|
|
369
|
+
putObject(key: string, data: string | DataInput | ReadableStream | File | Blob, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders, contentLength?: number): Promise<Response>;
|
|
370
|
+
/**
|
|
371
|
+
* Put object that automatically chooses single PUT vs multipart.
|
|
372
|
+
* Same signature/shape as putObject so callers don't need to change.
|
|
373
|
+
* @param {string} key - The key/path where the object will be stored.
|
|
374
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
375
|
+
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
376
|
+
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
377
|
+
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
378
|
+
* @param {number} [contentLength] - Optional known content length of data.
|
|
379
|
+
* @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
|
|
380
|
+
* @throws {TypeError} If data is not a string or Buffer.
|
|
381
|
+
* @example
|
|
382
|
+
* // Upload text file
|
|
383
|
+
* await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
|
|
384
|
+
*
|
|
385
|
+
* // Upload binary data
|
|
386
|
+
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
387
|
+
* await s3.putAnyObject('image.png', buffer, 'image/png');
|
|
388
|
+
*/
|
|
389
|
+
putAnyObject(key: string, data: DataInput, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders, contentLength?: number): Promise<Response | {
|
|
390
|
+
ok: boolean;
|
|
391
|
+
status: number;
|
|
392
|
+
headers: Map<string, string>;
|
|
393
|
+
}>;
|
|
394
|
+
private _multipartUpload;
|
|
395
|
+
private _uploadKnownSizePartsParallel;
|
|
396
|
+
private _uploadPartsOptimized;
|
|
397
|
+
private _uploadStreamingParts;
|
|
398
|
+
private _uploadPartWithRetry;
|
|
399
|
+
private _safeAbortUpload;
|
|
400
|
+
private _createSuccessResponse;
|
|
371
401
|
/**
|
|
372
402
|
* Initiates a multipart upload and returns the upload ID.
|
|
373
403
|
* @param {string} key - The key/path where the object will be stored.
|
|
@@ -380,12 +410,12 @@ declare class S3mini {
|
|
|
380
410
|
* const uploadId = await s3.getMultipartUploadId('large-file.zip', 'application/zip');
|
|
381
411
|
* console.log(`Started multipart upload: ${uploadId}`);
|
|
382
412
|
*/
|
|
383
|
-
getMultipartUploadId(key: string, fileType?: string, ssecHeaders?: SSECHeaders): Promise<string>;
|
|
413
|
+
getMultipartUploadId(key: string, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders): Promise<string>;
|
|
384
414
|
/**
|
|
385
415
|
* Uploads a part in a multipart upload.
|
|
386
416
|
* @param {string} key - The key of the object being uploaded.
|
|
387
417
|
* @param {string} uploadId - The upload ID from getMultipartUploadId.
|
|
388
|
-
* @param {
|
|
418
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
|
|
389
419
|
* @param {number} partNumber - The part number (must be between 1 and 10,000).
|
|
390
420
|
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
|
|
391
421
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
@@ -400,7 +430,7 @@ declare class S3mini {
|
|
|
400
430
|
* );
|
|
401
431
|
* console.log(`Part ${part.partNumber} uploaded with ETag: ${part.etag}`);
|
|
402
432
|
*/
|
|
403
|
-
uploadPart(key: string, uploadId: string, data:
|
|
433
|
+
uploadPart(key: string, uploadId: string, data: DataInput, partNumber: number, opts?: Record<string, unknown>, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders): Promise<UploadPart>;
|
|
404
434
|
/**
|
|
405
435
|
* Completes a multipart upload by combining all uploaded parts.
|
|
406
436
|
* @param {string} key - The key of the object being uploaded.
|
package/dist/s3mini.js
CHANGED
|
@@ -10,6 +10,7 @@ const XML_CONTENT_TYPE = 'application/xml';
|
|
|
10
10
|
const SENSITIVE_KEYS_REDACTED = new Set(['accesskeyid', 'secretaccesskey', 'sessiontoken', 'password', 'token']);
|
|
11
11
|
const IFHEADERS = new Set(['if-match', 'if-none-match', 'if-modified-since', 'if-unmodified-since']);
|
|
12
12
|
const DEFAULT_REQUEST_SIZE_IN_BYTES = 8 * 1024 * 1024;
|
|
13
|
+
const MIN_PART_SIZE = 8 * 1024 * 1024;
|
|
13
14
|
// Headers
|
|
14
15
|
const HEADER_AMZ_CONTENT_SHA256 = 'x-amz-content-sha256';
|
|
15
16
|
const HEADER_AMZ_CHECKSUM_SHA256 = 'x-amz-checksum-sha256';
|
|
@@ -27,7 +28,6 @@ const ERROR_ENDPOINT_REQUIRED = `${ERROR_PREFIX}endpoint must be a non-empty str
|
|
|
27
28
|
const ERROR_ENDPOINT_FORMAT = `${ERROR_PREFIX}endpoint must be a valid URL. Expected format: https://<host>[:port][/base-path]`;
|
|
28
29
|
const ERROR_KEY_REQUIRED = `${ERROR_PREFIX}key must be a non-empty string`;
|
|
29
30
|
const ERROR_UPLOAD_ID_REQUIRED = `${ERROR_PREFIX}uploadId must be a non-empty string`;
|
|
30
|
-
const ERROR_DATA_BUFFER_REQUIRED = `${ERROR_PREFIX}data must be a Buffer or string`;
|
|
31
31
|
const ERROR_PREFIX_TYPE = `${ERROR_PREFIX}prefix must be a string`;
|
|
32
32
|
const ERROR_DELIMITER_REQUIRED = `${ERROR_PREFIX}delimiter must be a string`;
|
|
33
33
|
|
|
@@ -41,11 +41,30 @@ const getByteSize = (data) => {
|
|
|
41
41
|
if (data instanceof ArrayBuffer || data instanceof Uint8Array) {
|
|
42
42
|
return data.byteLength;
|
|
43
43
|
}
|
|
44
|
-
if (data instanceof Blob) {
|
|
44
|
+
if (data instanceof Blob || data instanceof File) {
|
|
45
45
|
return data.size;
|
|
46
46
|
}
|
|
47
|
+
if (data instanceof ReadableStream) {
|
|
48
|
+
return Number.NaN; // size unknown
|
|
49
|
+
}
|
|
47
50
|
throw new Error('Unsupported data type');
|
|
48
51
|
};
|
|
52
|
+
const toUint8Array = (data) => {
|
|
53
|
+
if (typeof data === 'string') {
|
|
54
|
+
return ENCODR.encode(data);
|
|
55
|
+
}
|
|
56
|
+
if (data instanceof ArrayBuffer) {
|
|
57
|
+
return new Uint8Array(data);
|
|
58
|
+
}
|
|
59
|
+
if (data instanceof Uint8Array) {
|
|
60
|
+
return data;
|
|
61
|
+
}
|
|
62
|
+
// Node Buffer
|
|
63
|
+
if (typeof Buffer !== 'undefined' && Buffer.isBuffer(data)) {
|
|
64
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
};
|
|
49
68
|
/**
|
|
50
69
|
* Turn a raw ArrayBuffer into its hexadecimal representation.
|
|
51
70
|
* @param {ArrayBuffer} buffer The raw bytes.
|
|
@@ -253,6 +272,81 @@ const runInBatches = async (tasks, batchSize = 30, minIntervalMs = 0) => {
|
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
};
|
|
275
|
+
const generateParts = async function* (data, partSize) {
|
|
276
|
+
const bytes = toUint8Array(data);
|
|
277
|
+
if (bytes) {
|
|
278
|
+
yield* generateBufferParts(bytes, partSize);
|
|
279
|
+
}
|
|
280
|
+
else if (data instanceof Blob) {
|
|
281
|
+
yield* generateBlobParts(data, partSize);
|
|
282
|
+
}
|
|
283
|
+
else if (data instanceof ReadableStream) {
|
|
284
|
+
yield* generateStreamParts(data, partSize);
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
throw new TypeError(`${ERROR_PREFIX}Unsupported data type for multipart upload`);
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
function* generateBufferParts(bytes, partSize) {
|
|
291
|
+
for (let offset = 0; offset < bytes.byteLength; offset += partSize) {
|
|
292
|
+
yield bytes.subarray(offset, Math.min(offset + partSize, bytes.byteLength));
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Zero-copy: yields Blob slices. Data is only read when fetch consumes it.
|
|
297
|
+
*/
|
|
298
|
+
const generateBlobParts = function* (blob, partSize) {
|
|
299
|
+
for (let offset = 0; offset < blob.size; offset += partSize) {
|
|
300
|
+
yield blob.slice(offset, Math.min(offset + partSize, blob.size));
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
const generateStreamParts = async function* (stream, partSize) {
|
|
304
|
+
const reader = stream.getReader();
|
|
305
|
+
const chunks = [];
|
|
306
|
+
let buffered = 0;
|
|
307
|
+
try {
|
|
308
|
+
while (true) {
|
|
309
|
+
const { done, value } = await reader.read();
|
|
310
|
+
if (value) {
|
|
311
|
+
chunks.push(value);
|
|
312
|
+
buffered += value.byteLength;
|
|
313
|
+
while (buffered >= partSize) {
|
|
314
|
+
yield extractPart(chunks, partSize);
|
|
315
|
+
buffered -= partSize;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (done) {
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
// Yield remaining
|
|
323
|
+
if (buffered > 0) {
|
|
324
|
+
yield extractPart(chunks, buffered);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
reader.releaseLock();
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
const extractPart = (chunks, size) => {
|
|
332
|
+
const part = new Uint8Array(size);
|
|
333
|
+
let offset = 0;
|
|
334
|
+
while (offset < size && chunks.length > 0) {
|
|
335
|
+
const chunk = chunks[0];
|
|
336
|
+
const needed = size - offset;
|
|
337
|
+
if (chunk.byteLength <= needed) {
|
|
338
|
+
part.set(chunk, offset);
|
|
339
|
+
offset += chunk.byteLength;
|
|
340
|
+
chunks.shift();
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
part.set(chunk.subarray(0, needed), offset);
|
|
344
|
+
chunks[0] = chunk.subarray(needed);
|
|
345
|
+
offset = size;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return part.buffer;
|
|
349
|
+
};
|
|
256
350
|
|
|
257
351
|
/**
|
|
258
352
|
* S3 class for interacting with S3-compatible object storage services.
|
|
@@ -291,6 +385,7 @@ class S3mini {
|
|
|
291
385
|
* @param {number} [config.requestAbortTimeout=undefined] - The timeout in milliseconds after which a request should be aborted (careful on streamed requests).
|
|
292
386
|
* @param {Object} [config.logger=null] - A logger object with methods like info, warn, error.
|
|
293
387
|
* @param {typeof fetch} [config.fetch=globalThis.fetch] - Custom fetch implementation to use for HTTP requests.
|
|
388
|
+
* @param {number} [config.minPartSize=8388608] - The minimum part size for multipart uploads in bytes (default is 8MB).
|
|
294
389
|
* @throws {TypeError} Will throw an error if required parameters are missing or of incorrect type.
|
|
295
390
|
*/
|
|
296
391
|
#accessKeyId;
|
|
@@ -302,9 +397,10 @@ class S3mini {
|
|
|
302
397
|
requestAbortTimeout;
|
|
303
398
|
logger;
|
|
304
399
|
_fetch;
|
|
400
|
+
minPartSize;
|
|
305
401
|
signingKeyDate;
|
|
306
402
|
signingKey;
|
|
307
|
-
constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, }) {
|
|
403
|
+
constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, minPartSize = MIN_PART_SIZE, }) {
|
|
308
404
|
this._validateConstructorParams(accessKeyId, secretAccessKey, endpoint);
|
|
309
405
|
this.#accessKeyId = accessKeyId;
|
|
310
406
|
this.#secretAccessKey = secretAccessKey;
|
|
@@ -315,6 +411,7 @@ class S3mini {
|
|
|
315
411
|
this.requestAbortTimeout = requestAbortTimeout;
|
|
316
412
|
this.logger = logger;
|
|
317
413
|
this._fetch = (input, init) => fetch(input, init);
|
|
414
|
+
this.minPartSize = minPartSize;
|
|
318
415
|
}
|
|
319
416
|
_sanitize(obj) {
|
|
320
417
|
if (typeof obj !== 'object' || obj === null) {
|
|
@@ -441,13 +538,19 @@ class S3mini {
|
|
|
441
538
|
}
|
|
442
539
|
return { filteredOpts, conditionalHeaders };
|
|
443
540
|
}
|
|
444
|
-
_validateData(data) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
}
|
|
541
|
+
// private _validateData(data: unknown): BodyInit {
|
|
542
|
+
// if (data instanceof ArrayBuffer) {
|
|
543
|
+
// return data;
|
|
544
|
+
// }
|
|
545
|
+
// if (data instanceof Uint8Array) {
|
|
546
|
+
// return data as unknown as BodyInit;
|
|
547
|
+
// }
|
|
548
|
+
// if ((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string') {
|
|
549
|
+
// return data as BodyInit;
|
|
550
|
+
// }
|
|
551
|
+
// this._log('error', C.ERROR_DATA_BUFFER_REQUIRED);
|
|
552
|
+
// throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED);
|
|
553
|
+
// }
|
|
451
554
|
_validateUploadPartParams(key, uploadId, data, partNumber, opts) {
|
|
452
555
|
this._checkKey(key);
|
|
453
556
|
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
|
|
@@ -459,7 +562,7 @@ class S3mini {
|
|
|
459
562
|
throw new TypeError(`${ERROR_PREFIX}partNumber must be a positive integer`);
|
|
460
563
|
}
|
|
461
564
|
this._checkOpts(opts);
|
|
462
|
-
return
|
|
565
|
+
return data;
|
|
463
566
|
}
|
|
464
567
|
async _sign(method, keyPath, query = {}, headers = {}) {
|
|
465
568
|
// Create URL without appending keyPath first
|
|
@@ -1006,7 +1109,7 @@ class S3mini {
|
|
|
1006
1109
|
/**
|
|
1007
1110
|
* Uploads an object to the S3-compatible service.
|
|
1008
1111
|
* @param {string} key - The key/path where the object will be stored.
|
|
1009
|
-
* @param {string |
|
|
1112
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
1010
1113
|
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
1011
1114
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
1012
1115
|
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
@@ -1020,11 +1123,12 @@ class S3mini {
|
|
|
1020
1123
|
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
1021
1124
|
* await s3.putObject('image.png', buffer, 'image/png');
|
|
1022
1125
|
*/
|
|
1023
|
-
async putObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders) {
|
|
1126
|
+
async putObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders, contentLength) {
|
|
1127
|
+
const size = contentLength ?? getByteSize(data);
|
|
1024
1128
|
return this._signedRequest('PUT', key, {
|
|
1025
|
-
body:
|
|
1129
|
+
body: data,
|
|
1026
1130
|
headers: {
|
|
1027
|
-
[HEADER_CONTENT_LENGTH]:
|
|
1131
|
+
...(size && { [HEADER_CONTENT_LENGTH]: size }),
|
|
1028
1132
|
[HEADER_CONTENT_TYPE]: fileType,
|
|
1029
1133
|
...additionalHeaders,
|
|
1030
1134
|
...ssecHeaders,
|
|
@@ -1032,6 +1136,130 @@ class S3mini {
|
|
|
1032
1136
|
tolerated: [200],
|
|
1033
1137
|
});
|
|
1034
1138
|
}
|
|
1139
|
+
/**
|
|
1140
|
+
* Put object that automatically chooses single PUT vs multipart.
|
|
1141
|
+
* Same signature/shape as putObject so callers don't need to change.
|
|
1142
|
+
* @param {string} key - The key/path where the object will be stored.
|
|
1143
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
1144
|
+
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
1145
|
+
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
1146
|
+
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
1147
|
+
* @param {number} [contentLength] - Optional known content length of data.
|
|
1148
|
+
* @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
|
|
1149
|
+
* @throws {TypeError} If data is not a string or Buffer.
|
|
1150
|
+
* @example
|
|
1151
|
+
* // Upload text file
|
|
1152
|
+
* await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
|
|
1153
|
+
*
|
|
1154
|
+
* // Upload binary data
|
|
1155
|
+
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
1156
|
+
* await s3.putAnyObject('image.png', buffer, 'image/png');
|
|
1157
|
+
*/
|
|
1158
|
+
async putAnyObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders, contentLength) {
|
|
1159
|
+
const size = contentLength ?? getByteSize(data);
|
|
1160
|
+
// Single PUT for small files
|
|
1161
|
+
if (!Number.isNaN(size) && size <= this.minPartSize) {
|
|
1162
|
+
return this.putObject(key, data, fileType, ssecHeaders, additionalHeaders, contentLength);
|
|
1163
|
+
}
|
|
1164
|
+
this._checkKey(key);
|
|
1165
|
+
return this._multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders);
|
|
1166
|
+
}
|
|
1167
|
+
async _multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders) {
|
|
1168
|
+
const uploadId = await this.getMultipartUploadId(key, fileType, ssecHeaders, additionalHeaders);
|
|
1169
|
+
try {
|
|
1170
|
+
const parts = await this._uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders);
|
|
1171
|
+
parts.sort((a, b) => a.partNumber - b.partNumber);
|
|
1172
|
+
const result = await this.completeMultipartUpload(key, uploadId, parts);
|
|
1173
|
+
return this._createSuccessResponse(result.etag || '');
|
|
1174
|
+
}
|
|
1175
|
+
catch (err) {
|
|
1176
|
+
await this._safeAbortUpload(key, uploadId);
|
|
1177
|
+
throw err;
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
async _uploadKnownSizePartsParallel(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1181
|
+
const partSize = this.minPartSize;
|
|
1182
|
+
const totalSize = data instanceof Blob ? data.size : data.byteLength;
|
|
1183
|
+
const totalParts = Math.ceil(totalSize / partSize);
|
|
1184
|
+
const results = new Array(totalParts);
|
|
1185
|
+
let nextIndex = 0;
|
|
1186
|
+
const worker = async () => {
|
|
1187
|
+
while (true) {
|
|
1188
|
+
const index = nextIndex++;
|
|
1189
|
+
if (index >= totalParts) {
|
|
1190
|
+
return;
|
|
1191
|
+
}
|
|
1192
|
+
const start = index * partSize;
|
|
1193
|
+
const end = Math.min(start + partSize, totalSize);
|
|
1194
|
+
const part = data instanceof Blob
|
|
1195
|
+
? await data.slice(start, end).arrayBuffer() // Must await - R2 needs actual bytes
|
|
1196
|
+
: data.subarray(start, end);
|
|
1197
|
+
results[index] = await this._uploadPartWithRetry(key, uploadId, part, index + 1, ssecHeaders, additionalHeaders, maxRetries);
|
|
1198
|
+
}
|
|
1199
|
+
};
|
|
1200
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, totalParts) }, () => worker()));
|
|
1201
|
+
return results;
|
|
1202
|
+
}
|
|
1203
|
+
async _uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1204
|
+
const bytes = toUint8Array(data);
|
|
1205
|
+
if (bytes) {
|
|
1206
|
+
return this._uploadKnownSizePartsParallel(key, uploadId, bytes, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1207
|
+
}
|
|
1208
|
+
if (data instanceof Blob) {
|
|
1209
|
+
return this._uploadKnownSizePartsParallel(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1210
|
+
}
|
|
1211
|
+
return this._uploadStreamingParts(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1212
|
+
}
|
|
1213
|
+
async _uploadStreamingParts(key, uploadId, stream, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1214
|
+
const parts = [];
|
|
1215
|
+
const active = new Set();
|
|
1216
|
+
let partNumber = 0;
|
|
1217
|
+
for await (const partData of generateParts(stream, this.minPartSize)) {
|
|
1218
|
+
const currentPartNumber = ++partNumber;
|
|
1219
|
+
while (active.size >= concurrency) {
|
|
1220
|
+
await Promise.race(active);
|
|
1221
|
+
}
|
|
1222
|
+
const p = this._uploadPartWithRetry(key, uploadId, partData, currentPartNumber, ssecHeaders, additionalHeaders, maxRetries).then(part => {
|
|
1223
|
+
parts.push(part);
|
|
1224
|
+
active.delete(p);
|
|
1225
|
+
});
|
|
1226
|
+
active.add(p);
|
|
1227
|
+
}
|
|
1228
|
+
await Promise.all(active);
|
|
1229
|
+
return parts;
|
|
1230
|
+
}
|
|
1231
|
+
async _uploadPartWithRetry(key, uploadId, data, partNumber, ssecHeaders, additionalHeaders, maxRetries = 3) {
|
|
1232
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1233
|
+
try {
|
|
1234
|
+
return await this.uploadPart(key, uploadId, data, partNumber, {}, ssecHeaders, additionalHeaders);
|
|
1235
|
+
}
|
|
1236
|
+
catch (err) {
|
|
1237
|
+
if (attempt === maxRetries) {
|
|
1238
|
+
throw err;
|
|
1239
|
+
}
|
|
1240
|
+
await new Promise(r => setTimeout(r, Math.min(1000 * 2 ** attempt, 10000)));
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
throw new Error('Unreachable');
|
|
1244
|
+
}
|
|
1245
|
+
async _safeAbortUpload(key, uploadId) {
|
|
1246
|
+
try {
|
|
1247
|
+
await this.abortMultipartUpload(key, uploadId);
|
|
1248
|
+
}
|
|
1249
|
+
catch (err) {
|
|
1250
|
+
this._log('warn', `Failed to abort multipart upload: ${String(err)}`);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
_createSuccessResponse(etag) {
|
|
1254
|
+
if (typeof Response !== 'undefined') {
|
|
1255
|
+
const headers = new Headers();
|
|
1256
|
+
if (etag) {
|
|
1257
|
+
headers.set('ETag', etag);
|
|
1258
|
+
}
|
|
1259
|
+
return new Response('', { status: 200, headers });
|
|
1260
|
+
}
|
|
1261
|
+
return { ok: true, status: 200, headers: new Map([['ETag', etag]]) };
|
|
1262
|
+
}
|
|
1035
1263
|
/**
|
|
1036
1264
|
* Initiates a multipart upload and returns the upload ID.
|
|
1037
1265
|
* @param {string} key - The key/path where the object will be stored.
|
|
@@ -1044,13 +1272,13 @@ class S3mini {
|
|
|
1044
1272
|
* const uploadId = await s3.getMultipartUploadId('large-file.zip', 'application/zip');
|
|
1045
1273
|
* console.log(`Started multipart upload: ${uploadId}`);
|
|
1046
1274
|
*/
|
|
1047
|
-
async getMultipartUploadId(key, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders) {
|
|
1275
|
+
async getMultipartUploadId(key, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders) {
|
|
1048
1276
|
this._checkKey(key);
|
|
1049
1277
|
if (typeof fileType !== 'string') {
|
|
1050
1278
|
throw new TypeError(`${ERROR_PREFIX}fileType must be a string`);
|
|
1051
1279
|
}
|
|
1052
1280
|
const query = { uploads: '' };
|
|
1053
|
-
const headers = { [HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders };
|
|
1281
|
+
const headers = { [HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders, ...additionalHeaders };
|
|
1054
1282
|
const res = await this._signedRequest('POST', key, {
|
|
1055
1283
|
query,
|
|
1056
1284
|
headers,
|
|
@@ -1075,7 +1303,7 @@ class S3mini {
|
|
|
1075
1303
|
* Uploads a part in a multipart upload.
|
|
1076
1304
|
* @param {string} key - The key of the object being uploaded.
|
|
1077
1305
|
* @param {string} uploadId - The upload ID from getMultipartUploadId.
|
|
1078
|
-
* @param {
|
|
1306
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
|
|
1079
1307
|
* @param {number} partNumber - The part number (must be between 1 and 10,000).
|
|
1080
1308
|
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
|
|
1081
1309
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
@@ -1090,15 +1318,17 @@ class S3mini {
|
|
|
1090
1318
|
* );
|
|
1091
1319
|
* console.log(`Part ${part.partNumber} uploaded with ETag: ${part.etag}`);
|
|
1092
1320
|
*/
|
|
1093
|
-
async uploadPart(key, uploadId, data, partNumber, opts = {}, ssecHeaders) {
|
|
1321
|
+
async uploadPart(key, uploadId, data, partNumber, opts = {}, ssecHeaders, additionalHeaders) {
|
|
1094
1322
|
const body = this._validateUploadPartParams(key, uploadId, data, partNumber, opts);
|
|
1095
1323
|
const query = { uploadId, partNumber, ...opts };
|
|
1324
|
+
const size = getByteSize(data);
|
|
1096
1325
|
const res = await this._signedRequest('PUT', key, {
|
|
1097
1326
|
query,
|
|
1098
1327
|
body,
|
|
1099
1328
|
headers: {
|
|
1100
|
-
[HEADER_CONTENT_LENGTH]:
|
|
1329
|
+
...(size && !Number.isNaN(size) && { [HEADER_CONTENT_LENGTH]: size }),
|
|
1101
1330
|
...ssecHeaders,
|
|
1331
|
+
...additionalHeaders,
|
|
1102
1332
|
},
|
|
1103
1333
|
});
|
|
1104
1334
|
return { partNumber, etag: sanitizeETag(res.headers.get('etag') || '') };
|