s3mini 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +606 -162
- package/dist/s3mini.d.ts +65 -12
- package/dist/s3mini.js +329 -26
- package/dist/s3mini.js.map +1 -1
- package/dist/s3mini.min.js +1 -1
- package/dist/s3mini.min.js.map +1 -1
- package/package.json +18 -20
- package/src/S3.ts +361 -22
- package/src/consts.ts +1 -0
- package/src/types.ts +11 -8
- package/src/utils.ts +165 -3
package/dist/s3mini.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ interface S3Config {
|
|
|
7
7
|
requestAbortTimeout?: number;
|
|
8
8
|
logger?: Logger;
|
|
9
9
|
fetch?: typeof fetch;
|
|
10
|
+
minPartSize?: number;
|
|
10
11
|
}
|
|
11
12
|
interface SSECHeaders {
|
|
12
13
|
'x-amz-server-side-encryption-customer-algorithm': string;
|
|
@@ -124,13 +125,11 @@ interface CopyObjectResult {
|
|
|
124
125
|
etag: string;
|
|
125
126
|
lastModified?: Date;
|
|
126
127
|
}
|
|
127
|
-
|
|
128
|
-
* Where Buffer is available, e.g. when @types/node is loaded, we want to use it.
|
|
129
|
-
* But it should be excluded in other environments (e.g. Cloudflare).
|
|
130
|
-
*/
|
|
128
|
+
type BinaryData = ArrayBuffer | Uint8Array;
|
|
131
129
|
type MaybeBuffer = typeof globalThis extends {
|
|
132
130
|
Buffer?: infer B;
|
|
133
|
-
} ? B extends new (...a: unknown[]) => unknown ? InstanceType<B>
|
|
131
|
+
} ? B extends new (...a: unknown[]) => unknown ? InstanceType<B> | BinaryData : BinaryData : BinaryData;
|
|
132
|
+
type DataInput = string | MaybeBuffer | ReadableStream | File | Blob;
|
|
134
133
|
|
|
135
134
|
/**
|
|
136
135
|
* S3 class for interacting with S3-compatible object storage services.
|
|
@@ -164,9 +163,10 @@ declare class S3mini {
|
|
|
164
163
|
readonly requestAbortTimeout?: number;
|
|
165
164
|
readonly logger?: Logger;
|
|
166
165
|
readonly _fetch: typeof fetch;
|
|
166
|
+
readonly minPartSize: number;
|
|
167
167
|
private signingKeyDate?;
|
|
168
168
|
private signingKey?;
|
|
169
|
-
constructor({ accessKeyId, secretAccessKey, endpoint, region, requestSizeInBytes, requestAbortTimeout, logger, fetch, }: S3Config);
|
|
169
|
+
constructor({ accessKeyId, secretAccessKey, endpoint, region, requestSizeInBytes, requestAbortTimeout, logger, fetch, minPartSize, }: S3Config);
|
|
170
170
|
private _sanitize;
|
|
171
171
|
private _log;
|
|
172
172
|
private _validateConstructorParams;
|
|
@@ -182,7 +182,6 @@ declare class S3mini {
|
|
|
182
182
|
private _checkPrefix;
|
|
183
183
|
private _checkOpts;
|
|
184
184
|
private _filterIfHeaders;
|
|
185
|
-
private _validateData;
|
|
186
185
|
private _validateUploadPartParams;
|
|
187
186
|
private _sign;
|
|
188
187
|
private _signedRequest;
|
|
@@ -353,7 +352,7 @@ declare class S3mini {
|
|
|
353
352
|
/**
|
|
354
353
|
* Uploads an object to the S3-compatible service.
|
|
355
354
|
* @param {string} key - The key/path where the object will be stored.
|
|
356
|
-
* @param {string |
|
|
355
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
357
356
|
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
358
357
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
359
358
|
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
@@ -367,7 +366,38 @@ declare class S3mini {
|
|
|
367
366
|
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
368
367
|
* await s3.putObject('image.png', buffer, 'image/png');
|
|
369
368
|
*/
|
|
370
|
-
putObject(key: string, data: string |
|
|
369
|
+
putObject(key: string, data: string | DataInput | ReadableStream | File | Blob, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders, contentLength?: number): Promise<Response>;
|
|
370
|
+
/**
|
|
371
|
+
* Put object that automatically chooses single PUT vs multipart.
|
|
372
|
+
* Same signature/shape as putObject so callers don't need to change.
|
|
373
|
+
* @param {string} key - The key/path where the object will be stored.
|
|
374
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
375
|
+
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
376
|
+
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
377
|
+
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
378
|
+
* @param {number} [contentLength] - Optional known content length of data.
|
|
379
|
+
* @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
|
|
380
|
+
* @throws {TypeError} If data is not a string or Buffer.
|
|
381
|
+
* @example
|
|
382
|
+
* // Upload text file
|
|
383
|
+
* await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
|
|
384
|
+
*
|
|
385
|
+
* // Upload binary data
|
|
386
|
+
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
387
|
+
* await s3.putAnyObject('image.png', buffer, 'image/png');
|
|
388
|
+
*/
|
|
389
|
+
putAnyObject(key: string, data: DataInput, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders, contentLength?: number): Promise<Response | {
|
|
390
|
+
ok: boolean;
|
|
391
|
+
status: number;
|
|
392
|
+
headers: Map<string, string>;
|
|
393
|
+
}>;
|
|
394
|
+
private _multipartUpload;
|
|
395
|
+
private _uploadKnownSizePartsParallel;
|
|
396
|
+
private _uploadPartsOptimized;
|
|
397
|
+
private _uploadStreamingParts;
|
|
398
|
+
private _uploadPartWithRetry;
|
|
399
|
+
private _safeAbortUpload;
|
|
400
|
+
private _createSuccessResponse;
|
|
371
401
|
/**
|
|
372
402
|
* Initiates a multipart upload and returns the upload ID.
|
|
373
403
|
* @param {string} key - The key/path where the object will be stored.
|
|
@@ -380,12 +410,12 @@ declare class S3mini {
|
|
|
380
410
|
* const uploadId = await s3.getMultipartUploadId('large-file.zip', 'application/zip');
|
|
381
411
|
* console.log(`Started multipart upload: ${uploadId}`);
|
|
382
412
|
*/
|
|
383
|
-
getMultipartUploadId(key: string, fileType?: string, ssecHeaders?: SSECHeaders): Promise<string>;
|
|
413
|
+
getMultipartUploadId(key: string, fileType?: string, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders): Promise<string>;
|
|
384
414
|
/**
|
|
385
415
|
* Uploads a part in a multipart upload.
|
|
386
416
|
* @param {string} key - The key of the object being uploaded.
|
|
387
417
|
* @param {string} uploadId - The upload ID from getMultipartUploadId.
|
|
388
|
-
* @param {
|
|
418
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
|
|
389
419
|
* @param {number} partNumber - The part number (must be between 1 and 10,000).
|
|
390
420
|
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
|
|
391
421
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
@@ -400,7 +430,7 @@ declare class S3mini {
|
|
|
400
430
|
* );
|
|
401
431
|
* console.log(`Part ${part.partNumber} uploaded with ETag: ${part.etag}`);
|
|
402
432
|
*/
|
|
403
|
-
uploadPart(key: string, uploadId: string, data:
|
|
433
|
+
uploadPart(key: string, uploadId: string, data: DataInput, partNumber: number, opts?: Record<string, unknown>, ssecHeaders?: SSECHeaders, additionalHeaders?: AWSHeaders): Promise<UploadPart>;
|
|
404
434
|
/**
|
|
405
435
|
* Completes a multipart upload by combining all uploaded parts.
|
|
406
436
|
* @param {string} key - The key of the object being uploaded.
|
|
@@ -559,6 +589,29 @@ declare class S3mini {
|
|
|
559
589
|
private _parseErrorXml;
|
|
560
590
|
private _handleErrorResponse;
|
|
561
591
|
private _buildCanonicalQueryString;
|
|
592
|
+
/**
|
|
593
|
+
* Generates a pre-signed URL for direct client access to an S3 object.
|
|
594
|
+
* The URL embeds authentication in query parameters instead of headers,
|
|
595
|
+
* allowing unauthenticated HTTP clients to perform the specified operation.
|
|
596
|
+
*
|
|
597
|
+
* @param {'GET' | 'PUT'} method - HTTP method ('GET' for download, 'PUT' for upload)
|
|
598
|
+
* @param {string} key - The object key/path
|
|
599
|
+
* @param {number} [expiresIn=3600] - URL expiration time in seconds (1–604800)
|
|
600
|
+
* @param {Record<string, string>} [queryParams={}] - Additional query parameters to sign
|
|
601
|
+
* @returns {Promise<string>} Pre-signed URL string
|
|
602
|
+
* @throws {TypeError} If key is empty or expiresIn is out of range
|
|
603
|
+
* @example
|
|
604
|
+
* // Download URL valid for 1 hour
|
|
605
|
+
* const url = await s3.getPresignedUrl('GET', 'photos/vacation.jpg');
|
|
606
|
+
*
|
|
607
|
+
* // Upload URL valid for 5 minutes
|
|
608
|
+
* const url = await s3.getPresignedUrl('PUT', 'uploads/file.bin', 300);
|
|
609
|
+
*
|
|
610
|
+
* // Client-side usage (no credentials needed)
|
|
611
|
+
* await fetch(url, { method: 'PUT', body: data });
|
|
612
|
+
*/
|
|
613
|
+
getPresignedUrl(method: 'GET' | 'PUT', key: string, expiresIn?: number, queryParams?: Record<string, string>): Promise<string>;
|
|
614
|
+
private _presign;
|
|
562
615
|
private _getSignatureKey;
|
|
563
616
|
}
|
|
564
617
|
|
package/dist/s3mini.js
CHANGED
|
@@ -10,6 +10,7 @@ const XML_CONTENT_TYPE = 'application/xml';
|
|
|
10
10
|
const SENSITIVE_KEYS_REDACTED = new Set(['accesskeyid', 'secretaccesskey', 'sessiontoken', 'password', 'token']);
|
|
11
11
|
const IFHEADERS = new Set(['if-match', 'if-none-match', 'if-modified-since', 'if-unmodified-since']);
|
|
12
12
|
const DEFAULT_REQUEST_SIZE_IN_BYTES = 8 * 1024 * 1024;
|
|
13
|
+
const MIN_PART_SIZE = 8 * 1024 * 1024;
|
|
13
14
|
// Headers
|
|
14
15
|
const HEADER_AMZ_CONTENT_SHA256 = 'x-amz-content-sha256';
|
|
15
16
|
const HEADER_AMZ_CHECKSUM_SHA256 = 'x-amz-checksum-sha256';
|
|
@@ -27,7 +28,6 @@ const ERROR_ENDPOINT_REQUIRED = `${ERROR_PREFIX}endpoint must be a non-empty str
|
|
|
27
28
|
const ERROR_ENDPOINT_FORMAT = `${ERROR_PREFIX}endpoint must be a valid URL. Expected format: https://<host>[:port][/base-path]`;
|
|
28
29
|
const ERROR_KEY_REQUIRED = `${ERROR_PREFIX}key must be a non-empty string`;
|
|
29
30
|
const ERROR_UPLOAD_ID_REQUIRED = `${ERROR_PREFIX}uploadId must be a non-empty string`;
|
|
30
|
-
const ERROR_DATA_BUFFER_REQUIRED = `${ERROR_PREFIX}data must be a Buffer or string`;
|
|
31
31
|
const ERROR_PREFIX_TYPE = `${ERROR_PREFIX}prefix must be a string`;
|
|
32
32
|
const ERROR_DELIMITER_REQUIRED = `${ERROR_PREFIX}delimiter must be a string`;
|
|
33
33
|
|
|
@@ -41,11 +41,30 @@ const getByteSize = (data) => {
|
|
|
41
41
|
if (data instanceof ArrayBuffer || data instanceof Uint8Array) {
|
|
42
42
|
return data.byteLength;
|
|
43
43
|
}
|
|
44
|
-
if (data instanceof Blob) {
|
|
44
|
+
if (data instanceof Blob || data instanceof File) {
|
|
45
45
|
return data.size;
|
|
46
46
|
}
|
|
47
|
+
if (data instanceof ReadableStream) {
|
|
48
|
+
return Number.NaN; // size unknown
|
|
49
|
+
}
|
|
47
50
|
throw new Error('Unsupported data type');
|
|
48
51
|
};
|
|
52
|
+
const toUint8Array = (data) => {
|
|
53
|
+
if (typeof data === 'string') {
|
|
54
|
+
return ENCODR.encode(data);
|
|
55
|
+
}
|
|
56
|
+
if (data instanceof ArrayBuffer) {
|
|
57
|
+
return new Uint8Array(data);
|
|
58
|
+
}
|
|
59
|
+
if (data instanceof Uint8Array) {
|
|
60
|
+
return data;
|
|
61
|
+
}
|
|
62
|
+
// Node Buffer
|
|
63
|
+
if (typeof Buffer !== 'undefined' && Buffer.isBuffer(data)) {
|
|
64
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
};
|
|
49
68
|
/**
|
|
50
69
|
* Turn a raw ArrayBuffer into its hexadecimal representation.
|
|
51
70
|
* @param {ArrayBuffer} buffer The raw bytes.
|
|
@@ -253,6 +272,81 @@ const runInBatches = async (tasks, batchSize = 30, minIntervalMs = 0) => {
|
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
};
|
|
275
|
+
const generateParts = async function* (data, partSize) {
|
|
276
|
+
const bytes = toUint8Array(data);
|
|
277
|
+
if (bytes) {
|
|
278
|
+
yield* generateBufferParts(bytes, partSize);
|
|
279
|
+
}
|
|
280
|
+
else if (data instanceof Blob) {
|
|
281
|
+
yield* generateBlobParts(data, partSize);
|
|
282
|
+
}
|
|
283
|
+
else if (data instanceof ReadableStream) {
|
|
284
|
+
yield* generateStreamParts(data, partSize);
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
throw new TypeError(`${ERROR_PREFIX}Unsupported data type for multipart upload`);
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
function* generateBufferParts(bytes, partSize) {
|
|
291
|
+
for (let offset = 0; offset < bytes.byteLength; offset += partSize) {
|
|
292
|
+
yield bytes.subarray(offset, Math.min(offset + partSize, bytes.byteLength));
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Zero-copy: yields Blob slices. Data is only read when fetch consumes it.
|
|
297
|
+
*/
|
|
298
|
+
const generateBlobParts = function* (blob, partSize) {
|
|
299
|
+
for (let offset = 0; offset < blob.size; offset += partSize) {
|
|
300
|
+
yield blob.slice(offset, Math.min(offset + partSize, blob.size));
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
const generateStreamParts = async function* (stream, partSize) {
|
|
304
|
+
const reader = stream.getReader();
|
|
305
|
+
const chunks = [];
|
|
306
|
+
let buffered = 0;
|
|
307
|
+
try {
|
|
308
|
+
while (true) {
|
|
309
|
+
const { done, value } = await reader.read();
|
|
310
|
+
if (value) {
|
|
311
|
+
chunks.push(value);
|
|
312
|
+
buffered += value.byteLength;
|
|
313
|
+
while (buffered >= partSize) {
|
|
314
|
+
yield extractPart(chunks, partSize);
|
|
315
|
+
buffered -= partSize;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (done) {
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
// Yield remaining
|
|
323
|
+
if (buffered > 0) {
|
|
324
|
+
yield extractPart(chunks, buffered);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
reader.releaseLock();
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
const extractPart = (chunks, size) => {
|
|
332
|
+
const part = new Uint8Array(size);
|
|
333
|
+
let offset = 0;
|
|
334
|
+
while (offset < size && chunks.length > 0) {
|
|
335
|
+
const chunk = chunks[0];
|
|
336
|
+
const needed = size - offset;
|
|
337
|
+
if (chunk.byteLength <= needed) {
|
|
338
|
+
part.set(chunk, offset);
|
|
339
|
+
offset += chunk.byteLength;
|
|
340
|
+
chunks.shift();
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
part.set(chunk.subarray(0, needed), offset);
|
|
344
|
+
chunks[0] = chunk.subarray(needed);
|
|
345
|
+
offset = size;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return part.buffer;
|
|
349
|
+
};
|
|
256
350
|
|
|
257
351
|
/**
|
|
258
352
|
* S3 class for interacting with S3-compatible object storage services.
|
|
@@ -291,6 +385,7 @@ class S3mini {
|
|
|
291
385
|
* @param {number} [config.requestAbortTimeout=undefined] - The timeout in milliseconds after which a request should be aborted (careful on streamed requests).
|
|
292
386
|
* @param {Object} [config.logger=null] - A logger object with methods like info, warn, error.
|
|
293
387
|
* @param {typeof fetch} [config.fetch=globalThis.fetch] - Custom fetch implementation to use for HTTP requests.
|
|
388
|
+
* @param {number} [config.minPartSize=8388608] - The minimum part size for multipart uploads in bytes (default is 8MB).
|
|
294
389
|
* @throws {TypeError} Will throw an error if required parameters are missing or of incorrect type.
|
|
295
390
|
*/
|
|
296
391
|
#accessKeyId;
|
|
@@ -302,9 +397,10 @@ class S3mini {
|
|
|
302
397
|
requestAbortTimeout;
|
|
303
398
|
logger;
|
|
304
399
|
_fetch;
|
|
400
|
+
minPartSize;
|
|
305
401
|
signingKeyDate;
|
|
306
402
|
signingKey;
|
|
307
|
-
constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, }) {
|
|
403
|
+
constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, minPartSize = MIN_PART_SIZE, }) {
|
|
308
404
|
this._validateConstructorParams(accessKeyId, secretAccessKey, endpoint);
|
|
309
405
|
this.#accessKeyId = accessKeyId;
|
|
310
406
|
this.#secretAccessKey = secretAccessKey;
|
|
@@ -315,6 +411,7 @@ class S3mini {
|
|
|
315
411
|
this.requestAbortTimeout = requestAbortTimeout;
|
|
316
412
|
this.logger = logger;
|
|
317
413
|
this._fetch = (input, init) => fetch(input, init);
|
|
414
|
+
this.minPartSize = minPartSize;
|
|
318
415
|
}
|
|
319
416
|
_sanitize(obj) {
|
|
320
417
|
if (typeof obj !== 'object' || obj === null) {
|
|
@@ -441,13 +538,19 @@ class S3mini {
|
|
|
441
538
|
}
|
|
442
539
|
return { filteredOpts, conditionalHeaders };
|
|
443
540
|
}
|
|
444
|
-
_validateData(data) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
}
|
|
541
|
+
// private _validateData(data: unknown): BodyInit {
|
|
542
|
+
// if (data instanceof ArrayBuffer) {
|
|
543
|
+
// return data;
|
|
544
|
+
// }
|
|
545
|
+
// if (data instanceof Uint8Array) {
|
|
546
|
+
// return data as unknown as BodyInit;
|
|
547
|
+
// }
|
|
548
|
+
// if ((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string') {
|
|
549
|
+
// return data as BodyInit;
|
|
550
|
+
// }
|
|
551
|
+
// this._log('error', C.ERROR_DATA_BUFFER_REQUIRED);
|
|
552
|
+
// throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED);
|
|
553
|
+
// }
|
|
451
554
|
_validateUploadPartParams(key, uploadId, data, partNumber, opts) {
|
|
452
555
|
this._checkKey(key);
|
|
453
556
|
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
|
|
@@ -459,7 +562,7 @@ class S3mini {
|
|
|
459
562
|
throw new TypeError(`${ERROR_PREFIX}partNumber must be a positive integer`);
|
|
460
563
|
}
|
|
461
564
|
this._checkOpts(opts);
|
|
462
|
-
return
|
|
565
|
+
return data;
|
|
463
566
|
}
|
|
464
567
|
async _sign(method, keyPath, query = {}, headers = {}) {
|
|
465
568
|
// Create URL without appending keyPath first
|
|
@@ -686,6 +789,7 @@ class S3mini {
|
|
|
686
789
|
tolerated: [200, 404],
|
|
687
790
|
});
|
|
688
791
|
if (res.status === 404) {
|
|
792
|
+
void res.body?.cancel();
|
|
689
793
|
return null;
|
|
690
794
|
}
|
|
691
795
|
if (res.status !== 200) {
|
|
@@ -729,11 +833,16 @@ class S3mini {
|
|
|
729
833
|
const objects = [];
|
|
730
834
|
// Extract regular objects from Contents
|
|
731
835
|
if (contents) {
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
836
|
+
const raw = Array.isArray(contents) ? contents : [contents];
|
|
837
|
+
for (const item of raw) {
|
|
838
|
+
const o = item;
|
|
839
|
+
objects.push({
|
|
840
|
+
Key: o.Key ?? o.key ?? '',
|
|
841
|
+
Size: Number(o.Size ?? o.size ?? 0),
|
|
842
|
+
LastModified: new Date(o.LastModified ?? o.lastModified ?? 0),
|
|
843
|
+
ETag: o.ETag ?? o.etag ?? '',
|
|
844
|
+
StorageClass: o.StorageClass ?? o.storageClass ?? '',
|
|
845
|
+
});
|
|
737
846
|
}
|
|
738
847
|
}
|
|
739
848
|
// Extract directory prefixes from CommonPrefixes
|
|
@@ -820,6 +929,7 @@ class S3mini {
|
|
|
820
929
|
if (s === 200) {
|
|
821
930
|
return res.text();
|
|
822
931
|
}
|
|
932
|
+
void res.body?.cancel();
|
|
823
933
|
return null;
|
|
824
934
|
}
|
|
825
935
|
/**
|
|
@@ -839,6 +949,7 @@ class S3mini {
|
|
|
839
949
|
if (res.status === 200) {
|
|
840
950
|
return res;
|
|
841
951
|
}
|
|
952
|
+
void res.body?.cancel();
|
|
842
953
|
return null;
|
|
843
954
|
}
|
|
844
955
|
/**
|
|
@@ -858,6 +969,7 @@ class S3mini {
|
|
|
858
969
|
if (res.status === 200) {
|
|
859
970
|
return res.arrayBuffer();
|
|
860
971
|
}
|
|
972
|
+
void res.body?.cancel();
|
|
861
973
|
return null;
|
|
862
974
|
}
|
|
863
975
|
/**
|
|
@@ -877,6 +989,7 @@ class S3mini {
|
|
|
877
989
|
if (res.status === 200) {
|
|
878
990
|
return res.json();
|
|
879
991
|
}
|
|
992
|
+
void res.body?.cancel();
|
|
880
993
|
return null;
|
|
881
994
|
}
|
|
882
995
|
/**
|
|
@@ -896,6 +1009,7 @@ class S3mini {
|
|
|
896
1009
|
});
|
|
897
1010
|
const s = res.status;
|
|
898
1011
|
if (s === 404 || s === 412 || s === 304) {
|
|
1012
|
+
void res.body?.cancel();
|
|
899
1013
|
return { etag: null, data: null };
|
|
900
1014
|
}
|
|
901
1015
|
const etag = res.headers.get(HEADER_ETAG);
|
|
@@ -949,7 +1063,9 @@ class S3mini {
|
|
|
949
1063
|
}
|
|
950
1064
|
catch (err) {
|
|
951
1065
|
this._log('error', `Error getting content length for object ${key}: ${String(err)}`);
|
|
952
|
-
throw new Error(`${ERROR_PREFIX}Error getting content length for object ${key}: ${String(err)}
|
|
1066
|
+
throw new Error(`${ERROR_PREFIX}Error getting content length for object ${key}: ${String(err)}`, {
|
|
1067
|
+
cause: err,
|
|
1068
|
+
});
|
|
953
1069
|
}
|
|
954
1070
|
}
|
|
955
1071
|
/**
|
|
@@ -1006,7 +1122,7 @@ class S3mini {
|
|
|
1006
1122
|
/**
|
|
1007
1123
|
* Uploads an object to the S3-compatible service.
|
|
1008
1124
|
* @param {string} key - The key/path where the object will be stored.
|
|
1009
|
-
* @param {string |
|
|
1125
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
1010
1126
|
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
1011
1127
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
1012
1128
|
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
@@ -1020,11 +1136,12 @@ class S3mini {
|
|
|
1020
1136
|
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
1021
1137
|
* await s3.putObject('image.png', buffer, 'image/png');
|
|
1022
1138
|
*/
|
|
1023
|
-
async putObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders) {
|
|
1139
|
+
async putObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders, contentLength) {
|
|
1140
|
+
const size = contentLength ?? getByteSize(data);
|
|
1024
1141
|
return this._signedRequest('PUT', key, {
|
|
1025
|
-
body:
|
|
1142
|
+
body: data,
|
|
1026
1143
|
headers: {
|
|
1027
|
-
[HEADER_CONTENT_LENGTH]:
|
|
1144
|
+
...(size && { [HEADER_CONTENT_LENGTH]: size }),
|
|
1028
1145
|
[HEADER_CONTENT_TYPE]: fileType,
|
|
1029
1146
|
...additionalHeaders,
|
|
1030
1147
|
...ssecHeaders,
|
|
@@ -1032,6 +1149,130 @@ class S3mini {
|
|
|
1032
1149
|
tolerated: [200],
|
|
1033
1150
|
});
|
|
1034
1151
|
}
|
|
1152
|
+
/**
|
|
1153
|
+
* Put object that automatically chooses single PUT vs multipart.
|
|
1154
|
+
* Same signature/shape as putObject so callers don't need to change.
|
|
1155
|
+
* @param {string} key - The key/path where the object will be stored.
|
|
1156
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data to upload (string or Buffer).
|
|
1157
|
+
* @param {string} [fileType='application/octet-stream'] - The MIME type of the file.
|
|
1158
|
+
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
1159
|
+
* @param {IT.AWSHeaders} [additionalHeaders] - Additional x-amz-* headers specific to this request, if any.
|
|
1160
|
+
* @param {number} [contentLength] - Optional known content length of data.
|
|
1161
|
+
* @returns {Promise<Response | { ok: boolean; status: number; headers: Map<string, string> }>} A promise that resolves to the Response object from the upload request.
|
|
1162
|
+
* @throws {TypeError} If data is not a string or Buffer.
|
|
1163
|
+
* @example
|
|
1164
|
+
* // Upload text file
|
|
1165
|
+
* await s3.putAnyObject('hello.txt', 'Hello, World!', 'text/plain');
|
|
1166
|
+
*
|
|
1167
|
+
* // Upload binary data
|
|
1168
|
+
* const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
1169
|
+
* await s3.putAnyObject('image.png', buffer, 'image/png');
|
|
1170
|
+
*/
|
|
1171
|
+
async putAnyObject(key, data, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders, contentLength) {
|
|
1172
|
+
const size = contentLength ?? getByteSize(data);
|
|
1173
|
+
// Single PUT for small files
|
|
1174
|
+
if (!Number.isNaN(size) && size <= this.minPartSize) {
|
|
1175
|
+
return this.putObject(key, data, fileType, ssecHeaders, additionalHeaders, contentLength);
|
|
1176
|
+
}
|
|
1177
|
+
this._checkKey(key);
|
|
1178
|
+
return this._multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders);
|
|
1179
|
+
}
|
|
1180
|
+
async _multipartUpload(key, data, fileType, ssecHeaders, additionalHeaders) {
|
|
1181
|
+
const uploadId = await this.getMultipartUploadId(key, fileType, ssecHeaders, additionalHeaders);
|
|
1182
|
+
try {
|
|
1183
|
+
const parts = await this._uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders);
|
|
1184
|
+
parts.sort((a, b) => a.partNumber - b.partNumber);
|
|
1185
|
+
const result = await this.completeMultipartUpload(key, uploadId, parts);
|
|
1186
|
+
return this._createSuccessResponse(result.etag || '');
|
|
1187
|
+
}
|
|
1188
|
+
catch (err) {
|
|
1189
|
+
await this._safeAbortUpload(key, uploadId);
|
|
1190
|
+
throw err;
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
async _uploadKnownSizePartsParallel(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1194
|
+
const partSize = this.minPartSize;
|
|
1195
|
+
const totalSize = data instanceof Blob ? data.size : data.byteLength;
|
|
1196
|
+
const totalParts = Math.ceil(totalSize / partSize);
|
|
1197
|
+
const results = new Array(totalParts);
|
|
1198
|
+
let nextIndex = 0;
|
|
1199
|
+
const worker = async () => {
|
|
1200
|
+
while (true) {
|
|
1201
|
+
const index = nextIndex++;
|
|
1202
|
+
if (index >= totalParts) {
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
1205
|
+
const start = index * partSize;
|
|
1206
|
+
const end = Math.min(start + partSize, totalSize);
|
|
1207
|
+
const part = data instanceof Blob
|
|
1208
|
+
? await data.slice(start, end).arrayBuffer() // Must await - R2 needs actual bytes
|
|
1209
|
+
: data.subarray(start, end);
|
|
1210
|
+
results[index] = await this._uploadPartWithRetry(key, uploadId, part, index + 1, ssecHeaders, additionalHeaders, maxRetries);
|
|
1211
|
+
}
|
|
1212
|
+
};
|
|
1213
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, totalParts) }, () => worker()));
|
|
1214
|
+
return results;
|
|
1215
|
+
}
|
|
1216
|
+
async _uploadPartsOptimized(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1217
|
+
const bytes = toUint8Array(data);
|
|
1218
|
+
if (bytes) {
|
|
1219
|
+
return this._uploadKnownSizePartsParallel(key, uploadId, bytes, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1220
|
+
}
|
|
1221
|
+
if (data instanceof Blob) {
|
|
1222
|
+
return this._uploadKnownSizePartsParallel(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1223
|
+
}
|
|
1224
|
+
return this._uploadStreamingParts(key, uploadId, data, ssecHeaders, additionalHeaders, concurrency, maxRetries);
|
|
1225
|
+
}
|
|
1226
|
+
async _uploadStreamingParts(key, uploadId, stream, ssecHeaders, additionalHeaders, concurrency = 4, maxRetries = 3) {
|
|
1227
|
+
const parts = [];
|
|
1228
|
+
const active = new Set();
|
|
1229
|
+
let partNumber = 0;
|
|
1230
|
+
for await (const partData of generateParts(stream, this.minPartSize)) {
|
|
1231
|
+
const currentPartNumber = ++partNumber;
|
|
1232
|
+
while (active.size >= concurrency) {
|
|
1233
|
+
await Promise.race(active);
|
|
1234
|
+
}
|
|
1235
|
+
const p = this._uploadPartWithRetry(key, uploadId, partData, currentPartNumber, ssecHeaders, additionalHeaders, maxRetries).then(part => {
|
|
1236
|
+
parts.push(part);
|
|
1237
|
+
active.delete(p);
|
|
1238
|
+
});
|
|
1239
|
+
active.add(p);
|
|
1240
|
+
}
|
|
1241
|
+
await Promise.all(active);
|
|
1242
|
+
return parts;
|
|
1243
|
+
}
|
|
1244
|
+
async _uploadPartWithRetry(key, uploadId, data, partNumber, ssecHeaders, additionalHeaders, maxRetries = 3) {
|
|
1245
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1246
|
+
try {
|
|
1247
|
+
return await this.uploadPart(key, uploadId, data, partNumber, {}, ssecHeaders, additionalHeaders);
|
|
1248
|
+
}
|
|
1249
|
+
catch (err) {
|
|
1250
|
+
if (attempt === maxRetries) {
|
|
1251
|
+
throw err;
|
|
1252
|
+
}
|
|
1253
|
+
await new Promise(r => setTimeout(r, Math.min(1000 * 2 ** attempt, 10000)));
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
throw new Error('Unreachable');
|
|
1257
|
+
}
|
|
1258
|
+
async _safeAbortUpload(key, uploadId) {
|
|
1259
|
+
try {
|
|
1260
|
+
await this.abortMultipartUpload(key, uploadId);
|
|
1261
|
+
}
|
|
1262
|
+
catch (err) {
|
|
1263
|
+
this._log('warn', `Failed to abort multipart upload: ${String(err)}`);
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
_createSuccessResponse(etag) {
|
|
1267
|
+
if (typeof Response !== 'undefined') {
|
|
1268
|
+
const headers = new Headers();
|
|
1269
|
+
if (etag) {
|
|
1270
|
+
headers.set('ETag', etag);
|
|
1271
|
+
}
|
|
1272
|
+
return new Response('', { status: 200, headers });
|
|
1273
|
+
}
|
|
1274
|
+
return { ok: true, status: 200, headers: new Map([['ETag', etag]]) };
|
|
1275
|
+
}
|
|
1035
1276
|
/**
|
|
1036
1277
|
* Initiates a multipart upload and returns the upload ID.
|
|
1037
1278
|
* @param {string} key - The key/path where the object will be stored.
|
|
@@ -1044,13 +1285,13 @@ class S3mini {
|
|
|
1044
1285
|
* const uploadId = await s3.getMultipartUploadId('large-file.zip', 'application/zip');
|
|
1045
1286
|
* console.log(`Started multipart upload: ${uploadId}`);
|
|
1046
1287
|
*/
|
|
1047
|
-
async getMultipartUploadId(key, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders) {
|
|
1288
|
+
async getMultipartUploadId(key, fileType = DEFAULT_STREAM_CONTENT_TYPE, ssecHeaders, additionalHeaders) {
|
|
1048
1289
|
this._checkKey(key);
|
|
1049
1290
|
if (typeof fileType !== 'string') {
|
|
1050
1291
|
throw new TypeError(`${ERROR_PREFIX}fileType must be a string`);
|
|
1051
1292
|
}
|
|
1052
1293
|
const query = { uploads: '' };
|
|
1053
|
-
const headers = { [HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders };
|
|
1294
|
+
const headers = { [HEADER_CONTENT_TYPE]: fileType, ...ssecHeaders, ...additionalHeaders };
|
|
1054
1295
|
const res = await this._signedRequest('POST', key, {
|
|
1055
1296
|
query,
|
|
1056
1297
|
headers,
|
|
@@ -1075,7 +1316,7 @@ class S3mini {
|
|
|
1075
1316
|
* Uploads a part in a multipart upload.
|
|
1076
1317
|
* @param {string} key - The key of the object being uploaded.
|
|
1077
1318
|
* @param {string} uploadId - The upload ID from getMultipartUploadId.
|
|
1078
|
-
* @param {
|
|
1319
|
+
* @param {string | IT.MaybeBuffer | ReadableStream | File | Blob} data - The data for this part.
|
|
1079
1320
|
* @param {number} partNumber - The part number (must be between 1 and 10,000).
|
|
1080
1321
|
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
|
|
1081
1322
|
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
|
|
@@ -1090,15 +1331,17 @@ class S3mini {
|
|
|
1090
1331
|
* );
|
|
1091
1332
|
* console.log(`Part ${part.partNumber} uploaded with ETag: ${part.etag}`);
|
|
1092
1333
|
*/
|
|
1093
|
-
async uploadPart(key, uploadId, data, partNumber, opts = {}, ssecHeaders) {
|
|
1334
|
+
async uploadPart(key, uploadId, data, partNumber, opts = {}, ssecHeaders, additionalHeaders) {
|
|
1094
1335
|
const body = this._validateUploadPartParams(key, uploadId, data, partNumber, opts);
|
|
1095
1336
|
const query = { uploadId, partNumber, ...opts };
|
|
1337
|
+
const size = getByteSize(data);
|
|
1096
1338
|
const res = await this._signedRequest('PUT', key, {
|
|
1097
1339
|
query,
|
|
1098
1340
|
body,
|
|
1099
1341
|
headers: {
|
|
1100
|
-
[HEADER_CONTENT_LENGTH]:
|
|
1342
|
+
...(size && !Number.isNaN(size) && { [HEADER_CONTENT_LENGTH]: size }),
|
|
1101
1343
|
...ssecHeaders,
|
|
1344
|
+
...additionalHeaders,
|
|
1102
1345
|
},
|
|
1103
1346
|
});
|
|
1104
1347
|
return { partNumber, etag: sanitizeETag(res.headers.get('etag') || '') };
|
|
@@ -1538,6 +1781,66 @@ class S3mini {
|
|
|
1538
1781
|
.sort((a, b) => a.localeCompare(b))
|
|
1539
1782
|
.join('&');
|
|
1540
1783
|
}
|
|
1784
|
+
/**
|
|
1785
|
+
* Generates a pre-signed URL for direct client access to an S3 object.
|
|
1786
|
+
* The URL embeds authentication in query parameters instead of headers,
|
|
1787
|
+
* allowing unauthenticated HTTP clients to perform the specified operation.
|
|
1788
|
+
*
|
|
1789
|
+
* @param {'GET' | 'PUT'} method - HTTP method ('GET' for download, 'PUT' for upload)
|
|
1790
|
+
* @param {string} key - The object key/path
|
|
1791
|
+
* @param {number} [expiresIn=3600] - URL expiration time in seconds (1–604800)
|
|
1792
|
+
* @param {Record<string, string>} [queryParams={}] - Additional query parameters to sign
|
|
1793
|
+
* @returns {Promise<string>} Pre-signed URL string
|
|
1794
|
+
* @throws {TypeError} If key is empty or expiresIn is out of range
|
|
1795
|
+
* @example
|
|
1796
|
+
* // Download URL valid for 1 hour
|
|
1797
|
+
* const url = await s3.getPresignedUrl('GET', 'photos/vacation.jpg');
|
|
1798
|
+
*
|
|
1799
|
+
* // Upload URL valid for 5 minutes
|
|
1800
|
+
* const url = await s3.getPresignedUrl('PUT', 'uploads/file.bin', 300);
|
|
1801
|
+
*
|
|
1802
|
+
* // Client-side usage (no credentials needed)
|
|
1803
|
+
* await fetch(url, { method: 'PUT', body: data });
|
|
1804
|
+
*/
|
|
1805
|
+
async getPresignedUrl(method, key, expiresIn = 3600, queryParams = {}) {
|
|
1806
|
+
this._checkKey(key);
|
|
1807
|
+
if (!Number.isFinite(expiresIn) || expiresIn <= 0 || expiresIn > 604800) {
|
|
1808
|
+
throw new TypeError(`${ERROR_PREFIX}expiresIn must be between 1 and 604800 seconds`);
|
|
1809
|
+
}
|
|
1810
|
+
return this._presign(method, uriResourceEscape(key), Math.floor(expiresIn), queryParams);
|
|
1811
|
+
}
|
|
1812
|
+
async _presign(method, keyPath, expiresIn, queryParams) {
|
|
1813
|
+
const url = new URL(this.endpoint);
|
|
1814
|
+
if (keyPath.length > 0) {
|
|
1815
|
+
url.pathname =
|
|
1816
|
+
url.pathname === '/' ? `/${keyPath.replace(/^\/+/, '')}` : `${url.pathname}/${keyPath.replace(/^\/+/, '')}`;
|
|
1817
|
+
}
|
|
1818
|
+
const d = new Date();
|
|
1819
|
+
const year = d.getUTCFullYear();
|
|
1820
|
+
const month = String(d.getUTCMonth() + 1).padStart(2, '0');
|
|
1821
|
+
const day = String(d.getUTCDate()).padStart(2, '0');
|
|
1822
|
+
const shortDatetime = `${year}${month}${day}`;
|
|
1823
|
+
const fullDatetime = `${shortDatetime}T${String(d.getUTCHours()).padStart(2, '0')}${String(d.getUTCMinutes()).padStart(2, '0')}${String(d.getUTCSeconds()).padStart(2, '0')}Z`;
|
|
1824
|
+
const credentialScope = `${shortDatetime}/${this.region}/${S3_SERVICE}/${AWS_REQUEST_TYPE}`;
|
|
1825
|
+
const signedHeaders = 'host';
|
|
1826
|
+
const allQueryParams = {
|
|
1827
|
+
...queryParams,
|
|
1828
|
+
'X-Amz-Algorithm': AWS_ALGORITHM,
|
|
1829
|
+
'X-Amz-Credential': `${this.#accessKeyId}/${credentialScope}`,
|
|
1830
|
+
'X-Amz-Date': fullDatetime,
|
|
1831
|
+
'X-Amz-Expires': String(expiresIn),
|
|
1832
|
+
'X-Amz-SignedHeaders': signedHeaders,
|
|
1833
|
+
};
|
|
1834
|
+
const canonicalQueryString = this._buildCanonicalQueryString(allQueryParams);
|
|
1835
|
+
const canonicalRequest = `${method}\n${url.pathname}\n${canonicalQueryString}\nhost:${url.host}\n\n${signedHeaders}\n${UNSIGNED_PAYLOAD}`;
|
|
1836
|
+
const stringToSign = `${AWS_ALGORITHM}\n${fullDatetime}\n${credentialScope}\n${hexFromBuffer(await sha256(canonicalRequest))}`;
|
|
1837
|
+
if (shortDatetime !== this.signingKeyDate || !this.signingKey) {
|
|
1838
|
+
this.signingKeyDate = shortDatetime;
|
|
1839
|
+
this.signingKey = await this._getSignatureKey(shortDatetime);
|
|
1840
|
+
}
|
|
1841
|
+
const signature = hexFromBuffer(await hmac(this.signingKey, stringToSign));
|
|
1842
|
+
return `${url.origin}${url.pathname}?${canonicalQueryString}&X-Amz-Signature=${signature}`;
|
|
1843
|
+
}
|
|
1541
1844
|
async _getSignatureKey(dateStamp) {
|
|
1542
1845
|
const kDate = await hmac(`AWS4${this.#secretAccessKey}`, dateStamp);
|
|
1543
1846
|
const kRegion = await hmac(kDate, this.region);
|