s3mini 0.8.1 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/s3mini.d.ts +49 -16
- package/dist/s3mini.js +267 -26
- package/dist/s3mini.js.map +1 -1
- package/dist/s3mini.min.js +1 -1
- package/dist/s3mini.min.js.map +1 -1
- package/package.json +9 -7
- package/src/S3.ts +281 -23
- package/src/consts.ts +2 -1
- package/src/index.ts +2 -0
- package/src/types.ts +17 -11
- package/src/utils.ts +165 -3
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# s3mini | Tiny & fast S3 client for node and edge platforms.
|
|
2
2
|
|
|
3
|
-
`s3mini` is an ultra-lightweight Typescript client (~
|
|
3
|
+
`s3mini` is an ultra-lightweight Typescript client (~20 KB minified, ≈15 % more ops/s) for S3-compatible object storage. It runs on Node, Bun, Cloudflare Workers, and other edge platforms. It has been tested on Cloudflare R2, Backblaze B2, DigitalOcean Spaces, Ceph, Oracle, Garage and MinIO. (No Browser support!)
|
|
4
4
|
|
|
5
5
|
[[github](https://github.com/good-lly/s3mini)]
|
|
6
6
|
[[issues](https://github.com/good-lly/s3mini/issues)]
|
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
|
|
9
9
|
## Features
|
|
10
10
|
|
|
11
|
-
- 🚀 Light and fast: averages ≈15 % more ops/s and only ~
|
|
11
|
+
- 🚀 Light and fast: averages ≈15 % more ops/s and only ~20 KB (minified, not gzipped).
|
|
12
12
|
- 🔧 Zero dependencies; supports AWS SigV4 (no pre-signed requests) and SSE-C headers (tested only on Cloudflare)
|
|
13
13
|
- 🟠 Works on Cloudflare Workers; ideal for edge computing, Node, and Bun (no browser support).
|
|
14
14
|
- 🔑 Only the essential S3 APIs—improved list, put, get, delete, and a few more.
|
|
15
15
|
- 🛠️ Supports multipart uploads.
|
|
16
|
+
- 🎄 Tree-shakeable ES module.
|
|
16
17
|
- 🎯 TypeScript support with type definitions.
|
|
17
18
|
- 📚 Poorly-documented with examples and tests - But widely tested on various S3-compatible services! (Contributions welcome!)
|
|
18
19
|
- 📦 **BYOS3** — _Bring your own S3-compatible bucket_ (tested on Cloudflare R2, Backblaze B2, DigitalOcean Spaces, MinIO, Garage, Micro/Ceph and Oracle Object Storage, Scaleway).
|
|
@@ -43,9 +44,6 @@ Dev:
|
|
|
43
44
|
|
|
44
45
|
<a href="https://github.com/good-lly/s3mini/issues/"> <img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg" alt="Contributions welcome" /></a>
|
|
45
46
|
|
|
46
|
-
Performance tests was done on local Minio instance. Your results may vary depending on environment and network conditions, so take it with a grain of salt.
|
|
47
|
-

|
|
48
|
-
|
|
49
47
|
## Table of Contents
|
|
50
48
|
|
|
51
49
|
- [Supported Ops](#supported-ops)
|
package/dist/s3mini.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ interface S3Config {
|
|
|
7
7
|
requestAbortTimeout?: number;
|
|
8
8
|
logger?: Logger;
|
|
9
9
|
fetch?: typeof fetch;
|
|
10
|
+
minPartSize?: number;
|
|
10
11
|
}
|
|
11
12
|
interface SSECHeaders {
|
|
12
13
|
'x-amz-server-side-encryption-customer-algorithm': string;
|
|
@@ -52,9 +53,7 @@ interface ListBucketError {
|
|
|
52
53
|
}
|
|
53
54
|
type ListBucketResponse = {
|
|
54
55
|
listBucketResult: ListBucketResult;
|
|
55
|
-
} |
|
|
56
|
-
error: ListBucketError;
|
|
57
|
-
};
|
|
56
|
+
} | ListBucketError;
|
|
58
57
|
interface ListMultipartUploadSuccess {
|
|
59
58
|
listMultipartUploadsResult: {
|
|
60
59
|
bucket: string;
|
|
@@ -126,13 +125,11 @@ interface CopyObjectResult {
|
|
|
126
125
|
etag: string;
|
|
127
126
|
lastModified?: Date;
|
|
128
127
|
}
|
|
129
|
-
|
|
130
|
-
* Where Buffer is available, e.g. when @types/node is loaded, we want to use it.
|
|
131
|
-
* But it should be excluded in other environments (e.g. Cloudflare).
|
|
132
|
-
*/
|
|
128
|
+
type BinaryData = ArrayBuffer | Uint8Array;
|
|
133
129
|
type MaybeBuffer = typeof globalThis extends {
|
|
134
130
|
Buffer?: infer B;
|
|
135
|
-
} ? B extends new (...a: unknown[]) => unknown ? InstanceType<B>
|
|
131
|
+
} ? B extends new (...a: unknown[]) => unknown ? InstanceType<B> | BinaryData : BinaryData : BinaryData;
|
|
132
|
+
type DataInput = string | MaybeBuffer | ReadableStream | File | Blob;
|
|
136
133
|
|
|
137
134
|
/**
|
|
138
135
|
* S3 class for interacting with S3-compatible object storage services.
|
|
@@ -166,12 +163,18 @@ declare class S3mini {
|
|
|
166
163
|
readonly requestAbortTimeout?: number;
|
|
167
164
|
readonly logger?: Logger;
|
|
168
165
|
readonly _fetch: typeof fetch;
|
|
166
|
+
readonly minPartSize: number;
|
|
169
167
|
private signingKeyDate?;
|
|
170
168
|
private signingKey?;
|
|
171
|
-
constructor({ accessKeyId, secretAccessKey, endpoint, region, requestSizeInBytes, requestAbortTimeout, logger, fetch, }: S3Config);
|
|
169
|
+
constructor({ accessKeyId, secretAccessKey, endpoint, region, requestSizeInBytes, requestAbortTimeout, logger, fetch, minPartSize, }: S3Config);
|
|
172
170
|
private _sanitize;
|
|
173
171
|
private _log;
|
|
174
172
|
private _validateConstructorParams;
|
|
173
|
+
/**
|
|
174
|
+
* Check if credentials are configured (non-empty).
|
|
175
|
+
* @returns true if both accessKeyId and secretAccessKey are non-empty.
|
|
176
|
+
*/
|
|
177
|
+
private _hasCredentials;
|
|
175
178
|
private _ensureValidUrl;
|
|
176
179
|
private _validateMethodIsGetOrHead;
|
|
177
180
|
private _checkKey;
|
|
@@ -179,7 +182,6 @@ declare class S3mini {
|
|
|
179
182
|
private _checkPrefix;
|
|
180
183
|
private _checkOpts;
|
|
181
184
|
private _filterIfHeaders;
|
|
182
|
-
private _validateData;
|
|
183
185
|
private _validateUploadPartParams;
|
|
184
186
|
private _sign;
|
|
185
187
|
private _signedRequest;
|
|
@@ -350,7 +352,7 @@ declare class S3mini {
|
|
|
350
352
|
/**
|
|
351
353
|
* Uploads an object to the S3-compatible service.
|
|
352
354
|
* @param {string} key - The key/path where the object will be stored.
|
|
353
|
-
* @param {string |
|
|
355
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
354
356
|
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
355
357
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
356
358
|
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
@@ -364,7 +366,38 @@ declare class S3mini {
|
|
|
364
366
|
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
365
367
|
* await s3.putObject('image.png', buffer, 'image/png');
|
|
366
368
|
*/
|
|
367
|
-
putObject(key: string, data: string |
|
|
369
|
+
putObject(key: string, data: string | DataInput | ReadableStream | File | Blob, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders, contentLength?: number): Promise<Response>;
|
|
370
|
+
/**
|
|
371
|
+
* Put object that automatically chooses single PUT vs multipart.
|
|
372
|
+
* Same signature/shape as putObject so callers don't need to change.
|
|
373
|
+
* @param {string} key - The key/path where the object will be stored.
|
|
374
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
375
|
+
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
376
|
+
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
377
|
+
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
378
|
+
* @param {number} [contentLength] - Optional known content length of data.
|
|
379
|
+
* @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
|
|
380
|
+
* @throws {TypeError} If data is not a string or Buffer.
|
|
381
|
+
* @example
|
|
382
|
+
* // Upload text file
|
|
383
|
+
* await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
|
|
384
|
+
*
|
|
385
|
+
* // Upload binary data
|
|
386
|
+
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
387
|
+
* await s3.putAnyObject('image.png', buffer, 'image/png');
|
|
388
|
+
*/
|
|
389
|
+
putAnyObject(key: string, data: DataInput, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders, contentLength?: number): Promise<Response | {
|
|
390
|
+
ok: boolean;
|
|
391
|
+
status: number;
|
|
392
|
+
headers: Map<string, string>;
|
|
393
|
+
}>;
|
|
394
|
+
private _multipartUpload;
|
|
395
|
+
private _uploadKnownSizePartsParallel;
|
|
396
|
+
private _uploadPartsOptimized;
|
|
397
|
+
private _uploadStreamingParts;
|
|
398
|
+
private _uploadPartWithRetry;
|
|
399
|
+
private _safeAbortUpload;
|
|
400
|
+
private _createSuccessResponse;
|
|
368
401
|
/**
|
|
369
402
|
* Initiates a multipart upload and returns the upload ID.
|
|
370
403
|
* @param {string} key - The key/path where the object will be stored.
|
|
@@ -377,12 +410,12 @@ declare class S3mini {
|
|
|
377
410
|
* const uploadId = await s3.getMultipartUploadId('large-file.zip', 'application/zip');
|
|
378
411
|
* console.log(`Started multipart upload: ${uploadId}`);
|
|
379
412
|
*/
|
|
380
|
-
getMultipartUploadId(key: string, fileType?: string, ssecHeaders?: SSECHeaders): Promise<string>;
|
|
413
|
+
getMultipartUploadId(key: string, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders): Promise<string>;
|
|
381
414
|
/**
|
|
382
415
|
* Uploads a part in a multipart upload.
|
|
383
416
|
* @param {string} key - The key of the object being uploaded.
|
|
384
417
|
* @param {string} uploadId - The upload ID from getMultipartUploadId.
|
|
385
|
-
* @param {
|
|
418
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
|
|
386
419
|
* @param {number} partNumber - The part number (must be between 1 and 10,000).
|
|
387
420
|
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
|
|
388
421
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
@@ -397,7 +430,7 @@ declare class S3mini {
|
|
|
397
430
|
* );
|
|
398
431
|
* console.log(`Part ${part.partNumber} uploaded with ETag: ${part.etag}`);
|
|
399
432
|
*/
|
|
400
|
-
uploadPart(key: string, uploadId: string, data:
|
|
433
|
+
uploadPart(key: string, uploadId: string, data: DataInput, partNumber: number, opts?: Record<string, unknown>, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders): Promise<UploadPart>;
|
|
401
434
|
/**
|
|
402
435
|
* Completes a multipart upload by combining all uploaded parts.
|
|
403
436
|
* @param {string} key - The key of the object being uploaded.
|
|
@@ -576,5 +609,5 @@ declare const sanitizeETag: (etag: string) => string;
|
|
|
576
609
|
*/
|
|
577
610
|
declare const runInBatches: <T = unknown>(tasks: Iterable<() => Promise<T>>, batchSize?: number, minIntervalMs?: number) => Promise<Array<PromiseSettledResult<T>>>;
|
|
578
611
|
|
|
579
|
-
export { S3mini, runInBatches, sanitizeETag };
|
|
612
|
+
export { S3mini, S3mini as default, runInBatches, sanitizeETag };
|
|
580
613
|
export type { CompleteMultipartUploadResult, ErrorWithCode, ExistResponseCode, ListBucketResponse, ListMultipartUploadResponse, Logger, S3Config, UploadPart };
|
package/dist/s3mini.js
CHANGED
|
@@ -7,9 +7,10 @@ const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
|
|
|
7
7
|
const DEFAULT_STREAM_CONTENT_TYPE = 'application/octet-stream';
|
|
8
8
|
const XML_CONTENT_TYPE = 'application/xml';
|
|
9
9
|
// List of keys that might contain sensitive information
|
|
10
|
-
const SENSITIVE_KEYS_REDACTED = new Set(['
|
|
10
|
+
const SENSITIVE_KEYS_REDACTED = new Set(['accesskeyid', 'secretaccesskey', 'sessiontoken', 'password', 'token']);
|
|
11
11
|
const IFHEADERS = new Set(['if-match', 'if-none-match', 'if-modified-since', 'if-unmodified-since']);
|
|
12
12
|
const DEFAULT_REQUEST_SIZE_IN_BYTES = 8 * 1024 * 1024;
|
|
13
|
+
const MIN_PART_SIZE = 8 * 1024 * 1024;
|
|
13
14
|
// Headers
|
|
14
15
|
const HEADER_AMZ_CONTENT_SHA256 = 'x-amz-content-sha256';
|
|
15
16
|
const HEADER_AMZ_CHECKSUM_SHA256 = 'x-amz-checksum-sha256';
|
|
@@ -27,7 +28,6 @@ const ERROR_ENDPOINT_REQUIRED = `${ERROR_PREFIX}endpoint must be a non-empty str
|
|
|
27
28
|
const ERROR_ENDPOINT_FORMAT = `${ERROR_PREFIX}endpoint must be a valid URL. Expected format: https://<host>[:port][/base-path]`;
|
|
28
29
|
const ERROR_KEY_REQUIRED = `${ERROR_PREFIX}key must be a non-empty string`;
|
|
29
30
|
const ERROR_UPLOAD_ID_REQUIRED = `${ERROR_PREFIX}uploadId must be a non-empty string`;
|
|
30
|
-
const ERROR_DATA_BUFFER_REQUIRED = `${ERROR_PREFIX}data must be a Buffer or string`;
|
|
31
31
|
const ERROR_PREFIX_TYPE = `${ERROR_PREFIX}prefix must be a string`;
|
|
32
32
|
const ERROR_DELIMITER_REQUIRED = `${ERROR_PREFIX}delimiter must be a string`;
|
|
33
33
|
|
|
@@ -41,11 +41,30 @@ const getByteSize = (data) => {
|
|
|
41
41
|
if (data instanceof ArrayBuffer || data instanceof Uint8Array) {
|
|
42
42
|
return data.byteLength;
|
|
43
43
|
}
|
|
44
|
-
if (data instanceof Blob) {
|
|
44
|
+
if (data instanceof Blob || data instanceof File) {
|
|
45
45
|
return data.size;
|
|
46
46
|
}
|
|
47
|
+
if (data instanceof ReadableStream) {
|
|
48
|
+
return Number.NaN; // size unknown
|
|
49
|
+
}
|
|
47
50
|
throw new Error('Unsupported data type');
|
|
48
51
|
};
|
|
52
|
+
const toUint8Array = (data) => {
|
|
53
|
+
if (typeof data === 'string') {
|
|
54
|
+
return ENCODR.encode(data);
|
|
55
|
+
}
|
|
56
|
+
if (data instanceof ArrayBuffer) {
|
|
57
|
+
return new Uint8Array(data);
|
|
58
|
+
}
|
|
59
|
+
if (data instanceof Uint8Array) {
|
|
60
|
+
return data;
|
|
61
|
+
}
|
|
62
|
+
// Node Buffer
|
|
63
|
+
if (typeof Buffer !== 'undefined' && Buffer.isBuffer(data)) {
|
|
64
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
};
|
|
49
68
|
/**
|
|
50
69
|
* Turn a raw ArrayBuffer into its hexadecimal representation.
|
|
51
70
|
* @param {ArrayBuffer} buffer The raw bytes.
|
|
@@ -253,6 +272,81 @@ const runInBatches = async (tasks, batchSize = 30, minIntervalMs = 0) => {
|
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
};
|
|
275
|
+
const generateParts = async function* (data, partSize) {
|
|
276
|
+
const bytes = toUint8Array(data);
|
|
277
|
+
if (bytes) {
|
|
278
|
+
yield* generateBufferParts(bytes, partSize);
|
|
279
|
+
}
|
|
280
|
+
else if (data instanceof Blob) {
|
|
281
|
+
yield* generateBlobParts(data, partSize);
|
|
282
|
+
}
|
|
283
|
+
else if (data instanceof ReadableStream) {
|
|
284
|
+
yield* generateStreamParts(data, partSize);
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
throw new TypeError(`${ERROR_PREFIX}Unsupported data type for multipart upload`);
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
function* generateBufferParts(bytes, partSize) {
|
|
291
|
+
for (let offset = 0; offset < bytes.byteLength; offset += partSize) {
|
|
292
|
+
yield bytes.subarray(offset, Math.min(offset + partSize, bytes.byteLength));
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Zero-copy: yields Blob slices. Data is only read when fetch consumes it.
|
|
297
|
+
*/
|
|
298
|
+
const generateBlobParts = function* (blob, partSize) {
|
|
299
|
+
for (let offset = 0; offset < blob.size; offset += partSize) {
|
|
300
|
+
yield blob.slice(offset, Math.min(offset + partSize, blob.size));
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
const generateStreamParts = async function* (stream, partSize) {
|
|
304
|
+
const reader = stream.getReader();
|
|
305
|
+
const chunks = [];
|
|
306
|
+
let buffered = 0;
|
|
307
|
+
try {
|
|
308
|
+
while (true) {
|
|
309
|
+
const { done, value } = await reader.read();
|
|
310
|
+
if (value) {
|
|
311
|
+
chunks.push(value);
|
|
312
|
+
buffered += value.byteLength;
|
|
313
|
+
while (buffered >= partSize) {
|
|
314
|
+
yield extractPart(chunks, partSize);
|
|
315
|
+
buffered -= partSize;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (done) {
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
// Yield remaining
|
|
323
|
+
if (buffered > 0) {
|
|
324
|
+
yield extractPart(chunks, buffered);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
reader.releaseLock();
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
const extractPart = (chunks, size) => {
|
|
332
|
+
const part = new Uint8Array(size);
|
|
333
|
+
let offset = 0;
|
|
334
|
+
while (offset < size && chunks.length > 0) {
|
|
335
|
+
const chunk = chunks[0];
|
|
336
|
+
const needed = size - offset;
|
|
337
|
+
if (chunk.byteLength <= needed) {
|
|
338
|
+
part.set(chunk, offset);
|
|
339
|
+
offset += chunk.byteLength;
|
|
340
|
+
chunks.shift();
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
part.set(chunk.subarray(0, needed), offset);
|
|
344
|
+
chunks[0] = chunk.subarray(needed);
|
|
345
|
+
offset = size;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return part.buffer;
|
|
349
|
+
};
|
|
256
350
|
|
|
257
351
|
/**
|
|
258
352
|
* S3 class for interacting with S3-compatible object storage services.
|
|
@@ -291,6 +385,7 @@ class S3mini {
|
|
|
291
385
|
* @param {number} [config.requestAbortTimeout=undefined] - The timeout in milliseconds after which a request should be aborted (careful on streamed requests).
|
|
292
386
|
* @param {Object} [config.logger=null] - A logger object with methods like info, warn, error.
|
|
293
387
|
* @param {typeof fetch} [config.fetch=globalThis.fetch] - Custom fetch implementation to use for HTTP requests.
|
|
388
|
+
* @param {number} [config.minPartSize=8388608] - The minimum part size for multipart uploads in bytes (default is 8MB).
|
|
294
389
|
* @throws {TypeError} Will throw an error if required parameters are missing or of incorrect type.
|
|
295
390
|
*/
|
|
296
391
|
#accessKeyId;
|
|
@@ -302,9 +397,10 @@ class S3mini {
|
|
|
302
397
|
requestAbortTimeout;
|
|
303
398
|
logger;
|
|
304
399
|
_fetch;
|
|
400
|
+
minPartSize;
|
|
305
401
|
signingKeyDate;
|
|
306
402
|
signingKey;
|
|
307
|
-
constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, }) {
|
|
403
|
+
constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, minPartSize = MIN_PART_SIZE, }) {
|
|
308
404
|
this._validateConstructorParams(accessKeyId, secretAccessKey, endpoint);
|
|
309
405
|
this.#accessKeyId = accessKeyId;
|
|
310
406
|
this.#secretAccessKey = secretAccessKey;
|
|
@@ -315,6 +411,7 @@ class S3mini {
|
|
|
315
411
|
this.requestAbortTimeout = requestAbortTimeout;
|
|
316
412
|
this.logger = logger;
|
|
317
413
|
this._fetch = (input, init) => fetch(input, init);
|
|
414
|
+
this.minPartSize = minPartSize;
|
|
318
415
|
}
|
|
319
416
|
_sanitize(obj) {
|
|
320
417
|
if (typeof obj !== 'object' || obj === null) {
|
|
@@ -358,16 +455,23 @@ class S3mini {
|
|
|
358
455
|
}
|
|
359
456
|
}
|
|
360
457
|
_validateConstructorParams(accessKeyId, secretAccessKey, endpoint) {
|
|
361
|
-
if (typeof accessKeyId !== 'string'
|
|
458
|
+
if (typeof accessKeyId !== 'string') {
|
|
362
459
|
throw new TypeError(ERROR_ACCESS_KEY_REQUIRED);
|
|
363
460
|
}
|
|
364
|
-
if (typeof secretAccessKey !== 'string'
|
|
461
|
+
if (typeof secretAccessKey !== 'string') {
|
|
365
462
|
throw new TypeError(ERROR_SECRET_KEY_REQUIRED);
|
|
366
463
|
}
|
|
367
464
|
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
|
|
368
465
|
throw new TypeError(ERROR_ENDPOINT_REQUIRED);
|
|
369
466
|
}
|
|
370
467
|
}
|
|
468
|
+
/**
|
|
469
|
+
* Check if credentials are configured (non-empty).
|
|
470
|
+
* @returns true if both accessKeyId and secretAccessKey are non-empty.
|
|
471
|
+
*/
|
|
472
|
+
_hasCredentials() {
|
|
473
|
+
return this.#accessKeyId.trim().length > 0 && this.#secretAccessKey.trim().length > 0;
|
|
474
|
+
}
|
|
371
475
|
_ensureValidUrl(raw) {
|
|
372
476
|
const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`;
|
|
373
477
|
try {
|
|
@@ -434,13 +538,19 @@ class S3mini {
|
|
|
434
538
|
}
|
|
435
539
|
return { filteredOpts, conditionalHeaders };
|
|
436
540
|
}
|
|
437
|
-
_validateData(data) {
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
}
|
|
541
|
+
// private _validateData(data: unknown): BodyInit {
|
|
542
|
+
// if (data instanceof ArrayBuffer) {
|
|
543
|
+
// return data;
|
|
544
|
+
// }
|
|
545
|
+
// if (data instanceof Uint8Array) {
|
|
546
|
+
// return data as unknown as BodyInit;
|
|
547
|
+
// }
|
|
548
|
+
// if ((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string') {
|
|
549
|
+
// return data as BodyInit;
|
|
550
|
+
// }
|
|
551
|
+
// this._log('error', C.ERROR_DATA_BUFFER_REQUIRED);
|
|
552
|
+
// throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED);
|
|
553
|
+
// }
|
|
444
554
|
_validateUploadPartParams(key, uploadId, data, partNumber, opts) {
|
|
445
555
|
this._checkKey(key);
|
|
446
556
|
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
|
|
@@ -452,7 +562,7 @@ class S3mini {
|
|
|
452
562
|
throw new TypeError(`${ERROR_PREFIX}partNumber must be a positive integer`);
|
|
453
563
|
}
|
|
454
564
|
this._checkOpts(opts);
|
|
455
|
-
return
|
|
565
|
+
return data;
|
|
456
566
|
}
|
|
457
567
|
async _sign(method, keyPath, query = {}, headers = {}) {
|
|
458
568
|
// Create URL without appending keyPath first
|
|
@@ -462,6 +572,11 @@ class S3mini {
|
|
|
462
572
|
url.pathname =
|
|
463
573
|
url.pathname === '/' ? `/${keyPath.replace(/^\/+/, '')}` : `${url.pathname}/${keyPath.replace(/^\/+/, '')}`;
|
|
464
574
|
}
|
|
575
|
+
// If no credentials, return unsigned request (for public bucket access)
|
|
576
|
+
if (!this._hasCredentials()) {
|
|
577
|
+
headers[HEADER_HOST] = url.host;
|
|
578
|
+
return { url: url.toString(), headers };
|
|
579
|
+
}
|
|
465
580
|
const d = new Date();
|
|
466
581
|
const year = d.getUTCFullYear();
|
|
467
582
|
const month = String(d.getUTCMonth() + 1).padStart(2, '0');
|
|
@@ -523,8 +638,7 @@ class S3mini {
|
|
|
523
638
|
if (Object.keys(query).length > 0) {
|
|
524
639
|
withQuery = true; // append query string to signed URL
|
|
525
640
|
}
|
|
526
|
-
const
|
|
527
|
-
const finalUrl = withQuery && Object.keys(filteredOpts).length ? `${url}?${new URLSearchParams(filteredOptsStrings)}` : url;
|
|
641
|
+
const finalUrl = withQuery && Object.keys(filteredOpts).length ? `${url}?${this._buildCanonicalQueryString(filteredOpts)}` : url;
|
|
528
642
|
const signedHeadersString = Object.fromEntries(Object.entries(signedHeaders).map(([k, v]) => [k, String(v)]));
|
|
529
643
|
return this._sendRequest(finalUrl, method, signedHeadersString, body, tolerated);
|
|
530
644
|
}
|
|
@@ -995,7 +1109,7 @@ class S3mini {
|
|
|
995
1109
|
/**
|
|
996
1110
|
* Uploads an object to the S3-compatible service.
|
|
997
1111
|
* @param {string} key - The key/path where the object will be stored.
|
|
998
|
-
* @param {string |
|
|
1112
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
999
1113
|
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
1000
1114
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
1001
1115
|
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
@@ -1009,11 +1123,12 @@ class S3mini {
|
|
|
1009
1123
|
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
1010
1124
|
* await s3.putObject('image.png', buffer, 'image/png');
|
|
1011
1125
|
*/
|
|
1012
|
-
async putObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders) {
|
|
1126
|
+
async putObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders, contentLength) {
|
|
1127
|
+
const size = contentLength ?? getByteSize(data);
|
|
1013
1128
|
return this._signedRequest('PUT', key, {
|
|
1014
|
-
body:
|
|
1129
|
+
body: data,
|
|
1015
1130
|
headers: {
|
|
1016
|
-
[HEADER_CONTENT_LENGTH]:
|
|
1131
|
+
...(size && { [HEADER_CONTENT_LENGTH]: size }),
|
|
1017
1132
|
[HEADER_CONTENT_TYPE]: fileType,
|
|
1018
1133
|
...additionalHeaders,
|
|
1019
1134
|
...ssecHeaders,
|
|
@@ -1021,6 +1136,130 @@ class S3mini {
|
|
|
1021
1136
|
tolerated: [200],
|
|
1022
1137
|
});
|
|
1023
1138
|
}
|
|
1139
|
+
/**
|
|
1140
|
+
* Put object that automatically chooses single PUT vs multipart.
|
|
1141
|
+
* Same signature/shape as putObject so callers don't need to change.
|
|
1142
|
+
* @param {string} key - The key/path where the object will be stored.
|
|
1143
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
1144
|
+
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
1145
|
+
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
1146
|
+
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
1147
|
+
* @param {number} [contentLength] - Optional known content length of data.
|
|
1148
|
+
* @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
|
|
1149
|
+
* @throws {TypeError} If data is not a string or Buffer.
|
|
1150
|
+
* @example
|
|
1151
|
+
* // Upload text file
|
|
1152
|
+
* await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
|
|
1153
|
+
*
|
|
1154
|
+
* // Upload binary data
|
|
1155
|
+
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
1156
|
+
* await s3.putAnyObject('image.png', buffer, 'image/png');
|
|
1157
|
+
*/
|
|
1158
|
+
async putAnyObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders, contentLength) {
|
|
1159
|
+
const size = contentLength ?? getByteSize(data);
|
|
1160
|
+
// Single PUT for small files
|
|
1161
|
+
if (!Number.isNaN(size) && size <= this.minPartSize) {
|
|
1162
|
+
return this.putObject(key, data, fileType, ssecHeaders, additionalHeaders, contentLength);
|
|
1163
|
+
}
|
|
1164
|
+
this._checkKey(key);
|
|
1165
|
+
return this._multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders);
|
|
1166
|
+
}
|
|
1167
|
+
async _multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders) {
|
|
1168
|
+
const uploadId = await this.getMultipartUploadId(key, fileType, ssecHeaders, additionalHeaders);
|
|
1169
|
+
try {
|
|
1170
|
+
const parts = await this._uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders);
|
|
1171
|
+
parts.sort((a, b) => a.partNumber - b.partNumber);
|
|
1172
|
+
const result = await this.completeMultipartUpload(key, uploadId, parts);
|
|
1173
|
+
return this._createSuccessResponse(result.etag || '');
|
|
1174
|
+
}
|
|
1175
|
+
catch (err) {
|
|
1176
|
+
await this._safeAbortUpload(key, uploadId);
|
|
1177
|
+
throw err;
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
async _uploadKnownSizePartsParallel(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1181
|
+
const partSize = this.minPartSize;
|
|
1182
|
+
const totalSize = data instanceof Blob ? data.size : data.byteLength;
|
|
1183
|
+
const totalParts = Math.ceil(totalSize / partSize);
|
|
1184
|
+
const results = new Array(totalParts);
|
|
1185
|
+
let nextIndex = 0;
|
|
1186
|
+
const worker = async () => {
|
|
1187
|
+
while (true) {
|
|
1188
|
+
const index = nextIndex++;
|
|
1189
|
+
if (index >= totalParts) {
|
|
1190
|
+
return;
|
|
1191
|
+
}
|
|
1192
|
+
const start = index * partSize;
|
|
1193
|
+
const end = Math.min(start + partSize, totalSize);
|
|
1194
|
+
const part = data instanceof Blob
|
|
1195
|
+
? await data.slice(start, end).arrayBuffer() // Must await - R2 needs actual bytes
|
|
1196
|
+
: data.subarray(start, end);
|
|
1197
|
+
results[index] = await this._uploadPartWithRetry(key, uploadId, part, index + 1, ssecHeaders, additionalHeaders, maxRetries);
|
|
1198
|
+
}
|
|
1199
|
+
};
|
|
1200
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, totalParts) }, () => worker()));
|
|
1201
|
+
return results;
|
|
1202
|
+
}
|
|
1203
|
+
async _uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1204
|
+
const bytes = toUint8Array(data);
|
|
1205
|
+
if (bytes) {
|
|
1206
|
+
return this._uploadKnownSizePartsParallel(key, uploadId, bytes, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1207
|
+
}
|
|
1208
|
+
if (data instanceof Blob) {
|
|
1209
|
+
return this._uploadKnownSizePartsParallel(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1210
|
+
}
|
|
1211
|
+
return this._uploadStreamingParts(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1212
|
+
}
|
|
1213
|
+
async _uploadStreamingParts(key, uploadId, stream, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1214
|
+
const parts = [];
|
|
1215
|
+
const active = new Set();
|
|
1216
|
+
let partNumber = 0;
|
|
1217
|
+
for await (const partData of generateParts(stream, this.minPartSize)) {
|
|
1218
|
+
const currentPartNumber = ++partNumber;
|
|
1219
|
+
while (active.size >= concurrency) {
|
|
1220
|
+
await Promise.race(active);
|
|
1221
|
+
}
|
|
1222
|
+
const p = this._uploadPartWithRetry(key, uploadId, partData, currentPartNumber, ssecHeaders, additionalHeaders, maxRetries).then(part => {
|
|
1223
|
+
parts.push(part);
|
|
1224
|
+
active.delete(p);
|
|
1225
|
+
});
|
|
1226
|
+
active.add(p);
|
|
1227
|
+
}
|
|
1228
|
+
await Promise.all(active);
|
|
1229
|
+
return parts;
|
|
1230
|
+
}
|
|
1231
|
+
async _uploadPartWithRetry(key, uploadId, data, partNumber, ssecHeaders, additionalHeaders, maxRetries = 3) {
|
|
1232
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1233
|
+
try {
|
|
1234
|
+
return await this.uploadPart(key, uploadId, data, partNumber, {}, ssecHeaders, additionalHeaders);
|
|
1235
|
+
}
|
|
1236
|
+
catch (err) {
|
|
1237
|
+
if (attempt === maxRetries) {
|
|
1238
|
+
throw err;
|
|
1239
|
+
}
|
|
1240
|
+
await new Promise(r => setTimeout(r, Math.min(1000 * 2 ** attempt, 10000)));
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
throw new Error('Unreachable');
|
|
1244
|
+
}
|
|
1245
|
+
async _safeAbortUpload(key, uploadId) {
|
|
1246
|
+
try {
|
|
1247
|
+
await this.abortMultipartUpload(key, uploadId);
|
|
1248
|
+
}
|
|
1249
|
+
catch (err) {
|
|
1250
|
+
this._log('warn', `Failed to abort multipart upload: ${String(err)}`);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
_createSuccessResponse(etag) {
|
|
1254
|
+
if (typeof Response !== 'undefined') {
|
|
1255
|
+
const headers = new Headers();
|
|
1256
|
+
if (etag) {
|
|
1257
|
+
headers.set('ETag', etag);
|
|
1258
|
+
}
|
|
1259
|
+
return new Response('', { status: 200, headers });
|
|
1260
|
+
}
|
|
1261
|
+
return { ok: true, status: 200, headers: new Map([['ETag', etag]]) };
|
|
1262
|
+
}
|
|
1024
1263
|
/**
|
|
1025
1264
|
* Initiates a multipart upload and returns the upload ID.
|
|
1026
1265
|
* @param {string} key - The key/path where the object will be stored.
|
|
@@ -1033,13 +1272,13 @@ class S3mini {
|
|
|
1033
1272
|
* const uploadId = await s3.getMultipartUploadId('large-file.zip', 'application/zip');
|
|
1034
1273
|
* console.log(`Started multipart upload: ${uploadId}`);
|
|
1035
1274
|
*/
|
|
1036
|
-
async getMultipartUploadId(key, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders) {
|
|
1275
|
+
async getMultipartUploadId(key, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders) {
|
|
1037
1276
|
this._checkKey(key);
|
|
1038
1277
|
if (typeof fileType !== 'string') {
|
|
1039
1278
|
throw new TypeError(`${ERROR_PREFIX}fileType must be a string`);
|
|
1040
1279
|
}
|
|
1041
1280
|
const query = { uploads: '' };
|
|
1042
|
-
const headers = { [HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders };
|
|
1281
|
+
const headers = { [HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders, ...additionalHeaders };
|
|
1043
1282
|
const res = await this._signedRequest('POST', key, {
|
|
1044
1283
|
query,
|
|
1045
1284
|
headers,
|
|
@@ -1064,7 +1303,7 @@ class S3mini {
|
|
|
1064
1303
|
* Uploads a part in a multipart upload.
|
|
1065
1304
|
* @param {string} key - The key of the object being uploaded.
|
|
1066
1305
|
* @param {string} uploadId - The upload ID from getMultipartUploadId.
|
|
1067
|
-
* @param {
|
|
1306
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
|
|
1068
1307
|
* @param {number} partNumber - The part number (must be between 1 and 10,000).
|
|
1069
1308
|
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
|
|
1070
1309
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
@@ -1079,15 +1318,17 @@ class S3mini {
|
|
|
1079
1318
|
* );
|
|
1080
1319
|
* console.log(`Part ${part.partNumber} uploaded with ETag: ${part.etag}`);
|
|
1081
1320
|
*/
|
|
1082
|
-
async uploadPart(key, uploadId, data, partNumber, opts = {}, ssecHeaders) {
|
|
1321
|
+
async uploadPart(key, uploadId, data, partNumber, opts = {}, ssecHeaders, additionalHeaders) {
|
|
1083
1322
|
const body = this._validateUploadPartParams(key, uploadId, data, partNumber, opts);
|
|
1084
1323
|
const query = { uploadId, partNumber, ...opts };
|
|
1324
|
+
const size = getByteSize(data);
|
|
1085
1325
|
const res = await this._signedRequest('PUT', key, {
|
|
1086
1326
|
query,
|
|
1087
1327
|
body,
|
|
1088
1328
|
headers: {
|
|
1089
|
-
[HEADER_CONTENT_LENGTH]:
|
|
1329
|
+
...(size && !Number.isNaN(size) && { [HEADER_CONTENT_LENGTH]: size }),
|
|
1090
1330
|
...ssecHeaders,
|
|
1331
|
+
...additionalHeaders,
|
|
1091
1332
|
},
|
|
1092
1333
|
});
|
|
1093
1334
|
return { partNumber, etag: sanitizeETag(res.headers.get('etag') || '') };
|
|
@@ -1535,5 +1776,5 @@ class S3mini {
|
|
|
1535
1776
|
}
|
|
1536
1777
|
}
|
|
1537
1778
|
|
|
1538
|
-
export { S3mini, runInBatches, sanitizeETag };
|
|
1779
|
+
export { S3mini, S3mini as default, runInBatches, sanitizeETag };
|
|
1539
1780
|
//# sourceMappingURL=s3mini.js.map
|