s3-zip 3.2.1 → 3.3.1-a0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/npm-publish.yml +29 -0
- package/.github/workflows/test.yml +29 -0
- package/.travis.yml +2 -2
- package/README.md +28 -7
- package/aws_lambda.md +22 -18
- package/package.json +10 -9
- package/s3-zip.js +9 -1
- package/test/test-coverage-missing-lines.js +97 -0
- package/test/test-password-protected-duplicate.js +37 -0
- package/test/test-s3-error-on-stream.js +15 -15
- package/test/test-s3-password-protected.js +22 -22
- package/test/test-s3-same-file-alt-names.js +33 -24
- package/test/test-s3-v2-client-error.js +55 -0
- package/test/test-s3-zip-alt-names.js +24 -24
- package/test/test-s3-zip-unique-prefix.js +26 -26
- package/test/test-s3-zip.js +22 -22
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: Publish to NPM
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
inputs:
|
|
8
|
+
version:
|
|
9
|
+
description: 'Version to publish (leave empty to use package.json version)'
|
|
10
|
+
required: false
|
|
11
|
+
type: string
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read
|
|
18
|
+
id-token: write
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
# Setup .npmrc file to publish to npm
|
|
22
|
+
- uses: actions/setup-node@v4
|
|
23
|
+
with:
|
|
24
|
+
node-version: '20.x'
|
|
25
|
+
registry-url: 'https://registry.npmjs.org'
|
|
26
|
+
- run: npm i
|
|
27
|
+
- run: npm publish --provenance --access public
|
|
28
|
+
env:
|
|
29
|
+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches: [master]
|
|
6
|
+
types: [opened, synchronize, reopened]
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
node-version: [16, 18, 20]
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Use Node.js ${{ matrix.node-version }}
|
|
21
|
+
uses: actions/setup-node@v4
|
|
22
|
+
with:
|
|
23
|
+
node-version: ${{ matrix.node-version }}
|
|
24
|
+
|
|
25
|
+
- name: Install dependencies
|
|
26
|
+
run: npm i
|
|
27
|
+
|
|
28
|
+
- name: Run tests
|
|
29
|
+
run: npm test
|
package/.travis.yml
CHANGED
package/README.md
CHANGED
|
@@ -51,12 +51,12 @@ s3Zip
|
|
|
51
51
|
You can also pass a custom S3 client. For example if you want to zip files from a S3 compatible storage:
|
|
52
52
|
|
|
53
53
|
```javascript
|
|
54
|
-
const
|
|
54
|
+
const { S3Client } = require('@aws-sdk/client-s3')
|
|
55
55
|
|
|
56
|
-
const s3Client = new
|
|
57
|
-
|
|
58
|
-
s3ForcePathStyle: 'true',
|
|
56
|
+
const s3Client = new S3Client({
|
|
57
|
+
region: 'us-east-1',
|
|
59
58
|
endpoint: 'http://localhost:9000',
|
|
59
|
+
forcePathStyle: true
|
|
60
60
|
})
|
|
61
61
|
|
|
62
62
|
s3Zip
|
|
@@ -64,6 +64,8 @@ s3Zip
|
|
|
64
64
|
.pipe(output)
|
|
65
65
|
```
|
|
66
66
|
|
|
67
|
+
**Note:** When passing a custom S3 client, it must be an AWS SDK v3 client (from `@aws-sdk/client-s3`). AWS SDK v2 clients are not supported and will result in an error.
|
|
68
|
+
|
|
67
69
|
### Zip files with AWS Lambda
|
|
68
70
|
|
|
69
71
|
Example of s3-zip in combination with [AWS Lambda](aws_lambda.md).
|
|
@@ -74,14 +76,16 @@ Example of s3-zip in combination with [AWS Lambda](aws_lambda.md).
|
|
|
74
76
|
```javascript
|
|
75
77
|
const fs = require('fs')
|
|
76
78
|
const join = require('path').join
|
|
77
|
-
const
|
|
79
|
+
const {
|
|
80
|
+
S3Client
|
|
81
|
+
} = require("@aws-sdk/client-s3")
|
|
78
82
|
const s3Zip = require('s3-zip')
|
|
79
83
|
const XmlStream = require('xml-stream')
|
|
80
84
|
|
|
81
85
|
const region = 'bucket-region'
|
|
82
86
|
const bucket = 'name-of-s3-bucket'
|
|
83
87
|
const folder = 'name-of-bucket-folder/'
|
|
84
|
-
const s3 = new
|
|
88
|
+
const s3 = new S3Client({ region: region })
|
|
85
89
|
const params = {
|
|
86
90
|
Bucket: bucket,
|
|
87
91
|
Prefix: folder
|
|
@@ -192,10 +196,27 @@ If you would like a more fancy coverage report:
|
|
|
192
196
|
npm run coverage
|
|
193
197
|
```
|
|
194
198
|
|
|
199
|
+
## Publishing
|
|
200
|
+
|
|
201
|
+
This package is automatically published to NPM when a new release is created on GitHub. The publishing workflow:
|
|
202
|
+
|
|
203
|
+
1. Triggers on GitHub releases
|
|
204
|
+
2. Runs tests to ensure quality
|
|
205
|
+
3. Publishes to NPM using the `NPM_TOKEN` secret
|
|
206
|
+
|
|
207
|
+
### Setup for Maintainers
|
|
208
|
+
|
|
209
|
+
To enable automatic publishing, the repository requires an `NPM_TOKEN` secret to be configured in GitHub:
|
|
210
|
+
|
|
211
|
+
1. Generate an NPM access token with publish permissions
|
|
212
|
+
2. Add it as a repository secret named `NPM_TOKEN` in GitHub Settings > Secrets and variables > Actions
|
|
213
|
+
|
|
214
|
+
The workflow can also be triggered manually from the Actions tab for testing purposes.
|
|
215
|
+
|
|
195
216
|
|
|
196
217
|
|
|
197
218
|
|
|
198
|
-
[aws-sdk-url]:
|
|
219
|
+
[aws-sdk-url]: https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/configuring-the-jssdk.html
|
|
199
220
|
[npm-badge]: https://badge.fury.io/js/s3-zip.svg
|
|
200
221
|
[npm-url]: https://badge.fury.io/js/s3-zip
|
|
201
222
|
[travis-badge]: https://travis-ci.org/orangewise/s3-zip.svg?branch=master
|
package/aws_lambda.md
CHANGED
|
@@ -4,8 +4,10 @@
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
```javascript
|
|
7
|
-
const
|
|
7
|
+
const { Upload } = require("@aws-sdk/lib-storage");
|
|
8
|
+
const { S3 } = require("@aws-sdk/client-s3");
|
|
8
9
|
const s3Zip = require('s3-zip')
|
|
10
|
+
const {Readable} = require('stream')
|
|
9
11
|
|
|
10
12
|
exports.handler = function (event, context) {
|
|
11
13
|
console.log('event', event)
|
|
@@ -20,27 +22,31 @@ exports.handler = function (event, context) {
|
|
|
20
22
|
// Create body stream
|
|
21
23
|
try {
|
|
22
24
|
|
|
23
|
-
const
|
|
25
|
+
const writable = s3Zip.archive({ region: region, bucket: bucket}, folder, files)
|
|
26
|
+
const body = Readable.from(writable)
|
|
24
27
|
const zipParams = { params: { Bucket: bucket, Key: folder + zipFileName } }
|
|
25
|
-
const zipFile = new
|
|
26
|
-
|
|
28
|
+
const zipFile = new S3(zipParams)
|
|
29
|
+
new Upload({
|
|
30
|
+
client: zipFile,
|
|
31
|
+
params: { Body: body }
|
|
32
|
+
})
|
|
27
33
|
.on('httpUploadProgress', function (evt) { console.log(evt) })
|
|
28
|
-
.
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
.done().then(
|
|
35
|
+
(r) => {
|
|
36
|
+
console.log(r)
|
|
37
|
+
context.succeed(r)
|
|
38
|
+
},
|
|
39
|
+
(e) => {
|
|
40
|
+
console.log('zipFile.upload error', e)
|
|
32
41
|
context.fail(err)
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
context.succeed(r)
|
|
36
|
-
})
|
|
42
|
+
}
|
|
43
|
+
)
|
|
37
44
|
|
|
38
45
|
} catch (e) {
|
|
39
46
|
const err = 'catched error: ' + e
|
|
40
47
|
console.log(err)
|
|
41
48
|
context.fail(err)
|
|
42
49
|
}
|
|
43
|
-
|
|
44
50
|
}
|
|
45
51
|
|
|
46
52
|
```
|
|
@@ -48,7 +54,7 @@ exports.handler = function (event, context) {
|
|
|
48
54
|
## Invoke the function
|
|
49
55
|
|
|
50
56
|
```javascript
|
|
51
|
-
const
|
|
57
|
+
const { LambdaClient } = require("@aws-sdk/client-lambda");
|
|
52
58
|
|
|
53
59
|
const region = 'bucket-region'
|
|
54
60
|
const bucket = 'name-of-s3-bucket'
|
|
@@ -59,12 +65,10 @@ const file3 = 'Image C.png'
|
|
|
59
65
|
const file4 = 'Image D.png'
|
|
60
66
|
|
|
61
67
|
|
|
62
|
-
|
|
63
|
-
region
|
|
68
|
+
const lambda = new LambdaClient({
|
|
69
|
+
region
|
|
64
70
|
})
|
|
65
71
|
|
|
66
|
-
const lambda = new AWS.Lambda()
|
|
67
|
-
|
|
68
72
|
const files = [file1, file2, file3, file4]
|
|
69
73
|
const payload = JSON.stringify({
|
|
70
74
|
'region' : region,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "s3-zip",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.1-a0",
|
|
4
4
|
"description": "Download selected files from an Amazon S3 bucket as a zip file.",
|
|
5
5
|
"main": "s3-zip.js",
|
|
6
6
|
"scripts": {
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
},
|
|
10
10
|
"repository": {
|
|
11
11
|
"type": "git",
|
|
12
|
-
"url": "https://github.com/orangewise/s3-zip.git"
|
|
12
|
+
"url": "git+https://github.com/orangewise/s3-zip.git"
|
|
13
13
|
},
|
|
14
14
|
"keywords": [
|
|
15
15
|
"amazon",
|
|
@@ -25,19 +25,20 @@
|
|
|
25
25
|
},
|
|
26
26
|
"homepage": "https://github.com/orangewise/s3-zip#readme",
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"archiver": "^
|
|
29
|
-
"
|
|
28
|
+
"archiver": "^6.0.1",
|
|
29
|
+
"normalize-path": "^3.0.0",
|
|
30
|
+
"s3-files": "^3.0.0"
|
|
30
31
|
},
|
|
31
32
|
"devDependencies": {
|
|
32
33
|
"archiver-zip-encryptable": "^1.0.5",
|
|
33
34
|
"child_process": "^1.0.2",
|
|
34
35
|
"concat-stream": "^2.0.0",
|
|
35
36
|
"proxyquire": "^2.1.3",
|
|
36
|
-
"sinon": "^
|
|
37
|
-
"standard": "^
|
|
37
|
+
"sinon": "^16.0.0",
|
|
38
|
+
"standard": "^17.1.0",
|
|
38
39
|
"stream-array": "^1.1.2",
|
|
39
|
-
"tap": "^
|
|
40
|
-
"tar": "^
|
|
41
|
-
"yauzl": "^2.
|
|
40
|
+
"tap": "^16.3.8",
|
|
41
|
+
"tar": "^6.2.0",
|
|
42
|
+
"yauzl": "^2.10.0"
|
|
42
43
|
}
|
|
43
44
|
}
|
package/s3-zip.js
CHANGED
|
@@ -13,6 +13,11 @@ s3Zip.archive = function (opts, folder, filesS3, filesZip) {
|
|
|
13
13
|
self.debug = opts.debug || false
|
|
14
14
|
|
|
15
15
|
if ('s3' in opts) {
|
|
16
|
+
// Validate that the provided S3 client is compatible with AWS SDK v3
|
|
17
|
+
if (!opts.s3 || typeof opts.s3.send !== 'function') {
|
|
18
|
+
throw new Error('The provided S3 client must be an AWS SDK v3 client with a .send() method. ' +
|
|
19
|
+
'Please use @aws-sdk/client-s3 (v3) instead of aws-sdk (v2).')
|
|
20
|
+
}
|
|
16
21
|
connectionConfig = {
|
|
17
22
|
s3: opts.s3
|
|
18
23
|
}
|
|
@@ -39,7 +44,10 @@ s3Zip.archiveStream = function (stream, filesS3, filesZip) {
|
|
|
39
44
|
const self = this
|
|
40
45
|
const folder = this.folder || ''
|
|
41
46
|
if (this.registerFormat) {
|
|
42
|
-
|
|
47
|
+
// Only register the format if it hasn't been registered before
|
|
48
|
+
if (!archiver.isRegisteredFormat(this.registerFormat)) {
|
|
49
|
+
archiver.registerFormat(this.registerFormat, this.formatModule)
|
|
50
|
+
}
|
|
43
51
|
}
|
|
44
52
|
const archive = archiver(this.format || 'zip', this.archiverOpts || {})
|
|
45
53
|
archive.on('error', function (err) {
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Test to cover missing lines for full coverage
|
|
2
|
+
|
|
3
|
+
const t = require('tap')
|
|
4
|
+
const Stream = require('stream')
|
|
5
|
+
const sinon = require('sinon')
|
|
6
|
+
const proxyquire = require('proxyquire')
|
|
7
|
+
|
|
8
|
+
// Mock s3Files to avoid real S3 calls
|
|
9
|
+
const mockS3Files = {
|
|
10
|
+
connect: sinon.stub().returns({
|
|
11
|
+
createKeyStream: sinon.stub().returns(new Stream())
|
|
12
|
+
}),
|
|
13
|
+
createFileStream: sinon.stub().returns(new Stream())
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const s3Zip = proxyquire('../s3-zip.js', {
|
|
17
|
+
's3-files': mockS3Files
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
t.test('test archive with valid AWS SDK v3 client', function (child) {
|
|
21
|
+
// Mock valid AWS SDK v3 client (has .send method)
|
|
22
|
+
const awsV3Client = {
|
|
23
|
+
send: function (command) {
|
|
24
|
+
return Promise.resolve({ Body: Buffer.from('test data') })
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
// This should work and cover line 21: connectionConfig = { s3: opts.s3 }
|
|
30
|
+
const archive = s3Zip.archive(
|
|
31
|
+
{ s3: awsV3Client, bucket: 'test-bucket' },
|
|
32
|
+
'folder/',
|
|
33
|
+
['test-file.txt']
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
child.type(archive, 'object', 'Should return archive object')
|
|
37
|
+
child.end()
|
|
38
|
+
} catch (error) {
|
|
39
|
+
child.fail(`Should not throw error with valid SDK v3 client: ${error.message}`)
|
|
40
|
+
child.end()
|
|
41
|
+
}
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
t.test('test archiveStream with debug mode and directory paths', function (child) {
|
|
45
|
+
const rs = new Stream()
|
|
46
|
+
rs.readable = true
|
|
47
|
+
|
|
48
|
+
// Enable debug mode to cover lines 59-60
|
|
49
|
+
s3Zip.debug = true
|
|
50
|
+
|
|
51
|
+
const archive = s3Zip.archiveStream(rs, [])
|
|
52
|
+
|
|
53
|
+
// Emit a directory path to trigger the debug log
|
|
54
|
+
rs.emit('data', { data: Buffer.alloc(0), path: 'test-folder/' })
|
|
55
|
+
rs.emit('end')
|
|
56
|
+
|
|
57
|
+
child.type(archive, 'object', 'Should return archive object')
|
|
58
|
+
child.end()
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
t.test('test archiveStream with file not in filesS3 array', function (child) {
|
|
62
|
+
const rs = new Stream()
|
|
63
|
+
rs.readable = true
|
|
64
|
+
|
|
65
|
+
// Test the ternary on line 66 - when file is not found in filesS3 array
|
|
66
|
+
const archive = s3Zip.archiveStream(rs, ['different-file.txt'], ['renamed.txt'])
|
|
67
|
+
|
|
68
|
+
// Emit a file that's not in the filesS3 array
|
|
69
|
+
rs.emit('data', {
|
|
70
|
+
data: Buffer.from('test content'),
|
|
71
|
+
path: 'not-in-list.txt'
|
|
72
|
+
})
|
|
73
|
+
rs.emit('end')
|
|
74
|
+
|
|
75
|
+
child.type(archive, 'object', 'Should return archive object')
|
|
76
|
+
child.end()
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
t.test('test archiveStream with debug and archive error', function (child) {
|
|
80
|
+
const rs = new Stream()
|
|
81
|
+
rs.readable = true
|
|
82
|
+
|
|
83
|
+
// Enable debug mode to cover line 54
|
|
84
|
+
s3Zip.debug = true
|
|
85
|
+
|
|
86
|
+
const archive = s3Zip.archiveStream(rs, [])
|
|
87
|
+
|
|
88
|
+
// Force an archive error to trigger the debug log on line 54
|
|
89
|
+
setImmediate(() => {
|
|
90
|
+
archive.emit('error', new Error('test error'))
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
rs.emit('end')
|
|
94
|
+
|
|
95
|
+
child.type(archive, 'object', 'Should return archive object')
|
|
96
|
+
child.end()
|
|
97
|
+
})
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
const s3Zip = require('../s3-zip.js')
|
|
2
|
+
const t = require('tap')
|
|
3
|
+
const archiverZipEncryptable = require('archiver-zip-encryptable')
|
|
4
|
+
|
|
5
|
+
t.test('test duplicate format registration does not error', function (child) {
|
|
6
|
+
// First registration should work
|
|
7
|
+
s3Zip.setRegisterFormatOptions('zip-encryptable', archiverZipEncryptable)
|
|
8
|
+
|
|
9
|
+
// Try to register the format via archiveStream simulation
|
|
10
|
+
const mockStream = {
|
|
11
|
+
on: function (event, callback) {
|
|
12
|
+
if (event === 'end') {
|
|
13
|
+
setTimeout(callback, 10) // Simulate async behavior
|
|
14
|
+
} else if (event === 'data') {
|
|
15
|
+
// Don't emit any data
|
|
16
|
+
} else if (event === 'error') {
|
|
17
|
+
// Don't emit any errors
|
|
18
|
+
}
|
|
19
|
+
return this
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
try {
|
|
24
|
+
// First call to archiveStream (should register format)
|
|
25
|
+
const archive1 = s3Zip.archiveStream(mockStream, [], [])
|
|
26
|
+
child.ok(archive1, 'First archiveStream call succeeded')
|
|
27
|
+
|
|
28
|
+
// Second call to archiveStream (should NOT fail due to duplicate registration)
|
|
29
|
+
const archive2 = s3Zip.archiveStream(mockStream, [], [])
|
|
30
|
+
child.ok(archive2, 'Second archiveStream call succeeded')
|
|
31
|
+
|
|
32
|
+
child.end()
|
|
33
|
+
} catch (err) {
|
|
34
|
+
child.fail(`archiveStream calls failed: ${err.message}`)
|
|
35
|
+
child.end()
|
|
36
|
+
}
|
|
37
|
+
})
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
let s3Zip = require('../s3-zip.js')
|
|
2
|
+
const t = require('tap')
|
|
3
|
+
const fs = require('fs')
|
|
4
|
+
const Stream = require('stream')
|
|
5
|
+
const concat = require('concat-stream')
|
|
6
|
+
const join = require('path').join
|
|
7
|
+
const proxyquire = require('proxyquire')
|
|
8
|
+
const sinon = require('sinon')
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
const fileStream = function (file) {
|
|
11
|
+
const rs = new Stream()
|
|
12
12
|
rs.readable = true
|
|
13
|
-
|
|
13
|
+
const fileStream = fs.createReadStream(join(__dirname, file))
|
|
14
14
|
fileStream
|
|
15
15
|
.pipe(concat(
|
|
16
16
|
function buffersEmit (buffer) {
|
|
@@ -26,18 +26,18 @@ var fileStream = function (file) {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
t.test('test if error on filestream with archiveStream', function (child) {
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
const stream = fileStream('./fixtures/file.txt')
|
|
30
|
+
const files = ['foo.png']
|
|
31
31
|
s3Zip.archiveStream(stream, files)
|
|
32
32
|
child.end()
|
|
33
33
|
})
|
|
34
34
|
|
|
35
35
|
t.test('test if error on filestream with archive', function (child) {
|
|
36
|
-
|
|
36
|
+
const stream = fileStream('./fixtures/file.txt')
|
|
37
37
|
s3Zip = proxyquire('../s3-zip.js', {
|
|
38
38
|
's3-files': { createFileStream: sinon.stub().returns(stream) }
|
|
39
39
|
})
|
|
40
|
-
|
|
40
|
+
const files = ['foo.png']
|
|
41
41
|
s3Zip.archive({ region: 'region', bucket: 'bucket' }, 'folder', files)
|
|
42
42
|
child.end()
|
|
43
43
|
})
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
let s3Zip = require('../s3-zip.js')
|
|
2
|
+
const t = require('tap')
|
|
3
|
+
const fs = require('fs')
|
|
4
|
+
const Stream = require('stream')
|
|
5
|
+
const concat = require('concat-stream')
|
|
6
|
+
const join = require('path').join
|
|
7
|
+
const streamify = require('stream-array')
|
|
8
|
+
const archiverZipEncryptable = require('archiver-zip-encryptable')
|
|
9
|
+
const { exec } = require('child_process')
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
const fileStreamForFiles = function (files, preserveFolderPath) {
|
|
12
|
+
const rs = new Stream()
|
|
13
13
|
rs.readable = true
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
let fileCounter = 0
|
|
16
16
|
streamify(files).on('data', function (file) {
|
|
17
17
|
fileCounter += 1
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
const fileStream = fs.createReadStream(join(__dirname, file))
|
|
20
20
|
fileStream.pipe(
|
|
21
21
|
concat(function buffersEmit (buffer) {
|
|
22
22
|
// console.log('buffers concatenated, emit data for ', file);
|
|
23
|
-
|
|
24
|
-
rs.emit('data', { data: buffer, path
|
|
23
|
+
const path = preserveFolderPath ? file : file.replace(/^.*[\\/]/, '')
|
|
24
|
+
rs.emit('data', { data: buffer, path })
|
|
25
25
|
})
|
|
26
26
|
)
|
|
27
27
|
fileStream.on('end', function () {
|
|
@@ -35,11 +35,11 @@ var fileStreamForFiles = function (files, preserveFolderPath) {
|
|
|
35
35
|
return rs
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
const file1 = 'a/file.txt'
|
|
39
|
+
const file2 = 'b/file.txt'
|
|
40
|
+
const sinon = require('sinon')
|
|
41
|
+
const proxyquire = require('proxyquire')
|
|
42
|
+
const s3Stub = fileStreamForFiles(
|
|
43
43
|
['/fixtures/folder/a/file.txt', '/fixtures/folder/b/file.txt'],
|
|
44
44
|
true
|
|
45
45
|
)
|
|
@@ -48,8 +48,8 @@ s3Zip = proxyquire('../s3-zip.js', {
|
|
|
48
48
|
})
|
|
49
49
|
|
|
50
50
|
t.test('test archive password protected', async child => {
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
const outputPath = join(__dirname, '/test-password-protected.zip')
|
|
52
|
+
const output = fs.createWriteStream(outputPath)
|
|
53
53
|
|
|
54
54
|
await s3Zip
|
|
55
55
|
.setRegisterFormatOptions('zip-encryptable', archiverZipEncryptable)
|
|
@@ -1,28 +1,30 @@
|
|
|
1
1
|
// Test s3-zip BUT using alternate file names for the same file which is listed multiple times
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
const s3Zip = require('../s3-zip.js')
|
|
4
|
+
const t = require('tap')
|
|
5
|
+
const fs = require('fs')
|
|
6
|
+
const Stream = require('stream')
|
|
7
|
+
const concat = require('concat-stream')
|
|
8
|
+
const join = require('path').join
|
|
9
|
+
const streamify = require('stream-array')
|
|
10
|
+
const tar = require('tar')
|
|
11
|
+
const sinon = require('sinon')
|
|
12
|
+
const proxyquire = require('proxyquire')
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
+
const fileStreamForFiles = function (files, preserveFolderPath) {
|
|
15
|
+
const rs = new Stream()
|
|
14
16
|
rs.readable = true
|
|
15
17
|
|
|
16
|
-
|
|
18
|
+
let fileCounter = 0
|
|
17
19
|
streamify(files).on('data', function (file) {
|
|
18
20
|
fileCounter += 1
|
|
19
21
|
|
|
20
|
-
|
|
22
|
+
const fileStream = fs.createReadStream(join(__dirname, file))
|
|
21
23
|
fileStream.pipe(
|
|
22
24
|
concat(function buffersEmit (buffer) {
|
|
23
25
|
// console.log('buffers concatenated, emit data for ', file);
|
|
24
|
-
|
|
25
|
-
rs.emit('data', { data: buffer, path
|
|
26
|
+
const path = preserveFolderPath ? file : file.replace(/^.*[\\/]/, '')
|
|
27
|
+
rs.emit('data', { data: buffer, path })
|
|
26
28
|
})
|
|
27
29
|
)
|
|
28
30
|
fileStream.on('end', function () {
|
|
@@ -36,20 +38,20 @@ var fileStreamForFiles = function (files, preserveFolderPath) {
|
|
|
36
38
|
return rs
|
|
37
39
|
}
|
|
38
40
|
|
|
39
|
-
|
|
40
|
-
'/fixtures/folder/a/file.txt',
|
|
41
|
-
'/fixtures/folder/a/file.txt'
|
|
42
|
-
]
|
|
43
|
-
var outputFiles = [
|
|
41
|
+
const outputFiles = [
|
|
44
42
|
'FILE_1_ALT_1.TXT',
|
|
45
43
|
'FILE_1_ALT_2.TXT'
|
|
46
44
|
]
|
|
47
|
-
|
|
45
|
+
const filesRead = []
|
|
48
46
|
|
|
49
47
|
t.test('test a tar archive with alternate names for one file listed many times', function (child) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
const inputFiles = [
|
|
49
|
+
'/fixtures/folder/a/file.txt',
|
|
50
|
+
'/fixtures/folder/a/file.txt'
|
|
51
|
+
]
|
|
52
|
+
const outputPath = join(__dirname, '/test-same_file_alt_name.tar')
|
|
53
|
+
const output = fs.createWriteStream(outputPath)
|
|
54
|
+
const archive = s3Zip
|
|
53
55
|
.setFormat('tar')
|
|
54
56
|
.archiveStream(fileStreamForFiles(inputFiles, true), inputFiles, outputFiles)
|
|
55
57
|
.pipe(output)
|
|
@@ -68,7 +70,14 @@ t.test('test a tar archive with alternate names for one file listed many times',
|
|
|
68
70
|
})
|
|
69
71
|
|
|
70
72
|
t.test('test archive with alternate names for one file listed many times', function (child) {
|
|
71
|
-
|
|
73
|
+
const inputFiles = [
|
|
74
|
+
'/fixtures/folder/a/file.txt',
|
|
75
|
+
'/fixtures/folder/a/file.txt'
|
|
76
|
+
]
|
|
77
|
+
const s3Zip = proxyquire('../s3-zip.js', {
|
|
78
|
+
's3-files': { createFileStream: sinon.stub().returns(fileStreamForFiles(inputFiles, true)) }
|
|
79
|
+
})
|
|
80
|
+
const archive = s3Zip
|
|
72
81
|
.archive({ region: 'region', bucket: 'bucket' },
|
|
73
82
|
'',
|
|
74
83
|
inputFiles,
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// Test s3-zip with AWS SDK v2 client which should produce clear error
|
|
2
|
+
|
|
3
|
+
const s3Zip = require('../s3-zip.js')
|
|
4
|
+
const t = require('tap')
|
|
5
|
+
|
|
6
|
+
t.test('test s3-zip with AWS SDK v2 client should fail with clear error', function (child) {
|
|
7
|
+
// Mock AWS SDK v2 client (no .send method)
|
|
8
|
+
const awsV2Client = {
|
|
9
|
+
getObject: function (params) {
|
|
10
|
+
return {
|
|
11
|
+
promise: function () {
|
|
12
|
+
return Promise.resolve({ Body: Buffer.from('test data') })
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
try {
|
|
19
|
+
// This should fail with a clear error message about AWS SDK compatibility
|
|
20
|
+
s3Zip.archive(
|
|
21
|
+
{ s3: awsV2Client, bucket: 'test-bucket' },
|
|
22
|
+
'folder/',
|
|
23
|
+
['test-file.txt']
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
// If we get here without an error, the test failed
|
|
27
|
+
child.fail('Expected an error about AWS SDK compatibility but none was thrown')
|
|
28
|
+
child.end()
|
|
29
|
+
} catch (error) {
|
|
30
|
+
// We should get a clear error about AWS SDK version compatibility
|
|
31
|
+
child.ok(error.message.includes('AWS SDK v3'), 'Should get clear error about AWS SDK v3 requirement')
|
|
32
|
+
child.ok(error.message.includes('@aws-sdk/client-s3'), 'Should mention the correct package')
|
|
33
|
+
child.end()
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
t.test('test s3-zip with null s3 client should fail with clear error', function (child) {
|
|
38
|
+
try {
|
|
39
|
+
// This should fail with a clear error message about AWS SDK compatibility
|
|
40
|
+
s3Zip.archive(
|
|
41
|
+
{ s3: null, bucket: 'test-bucket' },
|
|
42
|
+
'folder/',
|
|
43
|
+
['test-file.txt']
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
// If we get here without an error, the test failed
|
|
47
|
+
child.fail('Expected an error about AWS SDK compatibility but none was thrown')
|
|
48
|
+
child.end()
|
|
49
|
+
} catch (error) {
|
|
50
|
+
// We should get a clear error about AWS SDK version compatibility
|
|
51
|
+
child.ok(error.message.includes('AWS SDK v3'), 'Should get clear error about AWS SDK v3 requirement')
|
|
52
|
+
child.ok(error.message.includes('.send() method'), 'Should mention the .send() method requirement')
|
|
53
|
+
child.end()
|
|
54
|
+
}
|
|
55
|
+
})
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
// Test s3-zip BUT using alternate file names in the resulting zip archive
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
let s3Zip = require('../s3-zip.js')
|
|
4
|
+
const t = require('tap')
|
|
5
|
+
const fs = require('fs')
|
|
6
|
+
const Stream = require('stream')
|
|
7
|
+
const concat = require('concat-stream')
|
|
8
|
+
const yauzl = require('yauzl')
|
|
9
|
+
const join = require('path').join
|
|
10
|
+
const tar = require('tar')
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
const fileStream = function (file, forceError) {
|
|
13
|
+
const rs = new Stream()
|
|
14
14
|
rs.readable = true
|
|
15
|
-
|
|
15
|
+
const fileStream = fs.createReadStream(join(__dirname, file))
|
|
16
16
|
fileStream
|
|
17
17
|
.pipe(concat(
|
|
18
18
|
function buffersEmit (buffer) {
|
|
@@ -32,21 +32,21 @@ var fileStream = function (file, forceError) {
|
|
|
32
32
|
return rs
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
const file1 = '/fixtures/file.txt'
|
|
36
|
+
const file1Alt = 'FILE_ALT.TXT'
|
|
37
|
+
const file1DataEntry = { name: file1Alt, mode: parseInt('0600', 8) }
|
|
38
38
|
// Stub: var fileStream = s3Files.createFileStream(keyStream);
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
39
|
+
const sinon = require('sinon')
|
|
40
|
+
const proxyquire = require('proxyquire')
|
|
41
|
+
const s3Stub = fileStream(file1)
|
|
42
42
|
s3Zip = proxyquire('../s3-zip.js', {
|
|
43
43
|
's3-files': { createFileStream: sinon.stub().returns(s3Stub) }
|
|
44
44
|
})
|
|
45
45
|
|
|
46
46
|
t.test('test archiveStream and zip file with alternate file name in zip archive', function (child) {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
const output = fs.createWriteStream(join(__dirname, '/test-alt.zip'))
|
|
48
|
+
const s = fileStream(file1)
|
|
49
|
+
const archive = s3Zip
|
|
50
50
|
.archiveStream(s, [file1], [file1Alt])
|
|
51
51
|
.pipe(output)
|
|
52
52
|
archive.on('close', function () {
|
|
@@ -69,7 +69,7 @@ t.test('test archiveStream and zip file with alternate file name in zip archive'
|
|
|
69
69
|
})
|
|
70
70
|
|
|
71
71
|
t.test('test archive with alternate zip archive names', function (child) {
|
|
72
|
-
|
|
72
|
+
const archive = s3Zip
|
|
73
73
|
.archive({ region: 'region', bucket: 'bucket' },
|
|
74
74
|
'folder',
|
|
75
75
|
[file1],
|
|
@@ -80,9 +80,9 @@ t.test('test archive with alternate zip archive names', function (child) {
|
|
|
80
80
|
})
|
|
81
81
|
|
|
82
82
|
t.test('test a tar archive with EntryData object', function (child) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
83
|
+
const outputPath = join(__dirname, '/test-entrydata.tar')
|
|
84
|
+
const output = fs.createWriteStream(outputPath)
|
|
85
|
+
const archive = s3Zip
|
|
86
86
|
.setFormat('tar')
|
|
87
87
|
.archiveStream(fileStream(file1), [file1], [file1DataEntry])
|
|
88
88
|
.pipe(output)
|
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
let s3Zip = require('../s3-zip.js')
|
|
2
|
+
const t = require('tap')
|
|
3
|
+
const fs = require('fs')
|
|
4
|
+
const Stream = require('stream')
|
|
5
|
+
const concat = require('concat-stream')
|
|
6
|
+
const yauzl = require('yauzl')
|
|
7
|
+
const join = require('path').join
|
|
8
|
+
const streamify = require('stream-array')
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
const fileStreamForFiles = function (files, preserveFolderPath) {
|
|
11
|
+
const rs = new Stream()
|
|
12
12
|
rs.readable = true
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
let fileCounter = 0
|
|
15
15
|
streamify(files).on('data', function (file) {
|
|
16
16
|
fileCounter += 1
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
const fileStream = fs.createReadStream(join(__dirname, file))
|
|
19
19
|
fileStream.pipe(
|
|
20
20
|
concat(function buffersEmit (buffer) {
|
|
21
21
|
// console.log('buffers concatenated, emit data for ', file);
|
|
22
|
-
|
|
23
|
-
rs.emit('data', { data: buffer, path
|
|
22
|
+
const path = preserveFolderPath ? file : file.replace(/^.*[\\/]/, '')
|
|
23
|
+
rs.emit('data', { data: buffer, path })
|
|
24
24
|
})
|
|
25
25
|
)
|
|
26
26
|
fileStream.on('end', function () {
|
|
@@ -34,13 +34,13 @@ var fileStreamForFiles = function (files, preserveFolderPath) {
|
|
|
34
34
|
return rs
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
37
|
+
const file1 = 'a/file.txt'
|
|
38
|
+
const file1Alt = 'file.txt'
|
|
39
|
+
const file2 = 'b/file.txt'
|
|
40
|
+
const file2Alt = 'file-1.txt'
|
|
41
|
+
const sinon = require('sinon')
|
|
42
|
+
const proxyquire = require('proxyquire')
|
|
43
|
+
const s3Stub = fileStreamForFiles(
|
|
44
44
|
['/fixtures/folder/a/file.txt', '/fixtures/folder/b/file.txt'],
|
|
45
45
|
true
|
|
46
46
|
)
|
|
@@ -51,10 +51,10 @@ s3Zip = proxyquire('../s3-zip.js', {
|
|
|
51
51
|
t.test(
|
|
52
52
|
'test archive with matching alternate zip archive names but unique keys',
|
|
53
53
|
function (child) {
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
const outputPath = join(__dirname, '/test-unique.zip')
|
|
55
|
+
const output = fs.createWriteStream(outputPath)
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
const archive = s3Zip
|
|
58
58
|
.archive(
|
|
59
59
|
{ region: 'region', bucket: 'bucket' },
|
|
60
60
|
'/fixtures/folder/',
|
|
@@ -63,13 +63,13 @@ t.test(
|
|
|
63
63
|
)
|
|
64
64
|
.pipe(output)
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
const altFiles = [file1Alt, file2Alt]
|
|
67
67
|
|
|
68
68
|
archive.on('close', function () {
|
|
69
69
|
yauzl.open(outputPath, function (err, zip) {
|
|
70
70
|
if (err) console.log('err', err)
|
|
71
71
|
zip.on('entry', function (entry) {
|
|
72
|
-
|
|
72
|
+
const i = altFiles.indexOf(entry.fileName)
|
|
73
73
|
if (i > -1) {
|
|
74
74
|
child.same(entry.fileName, altFiles[i])
|
|
75
75
|
altFiles.splice(i, 1)
|
package/test/test-s3-zip.js
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
let s3Zip = require('../s3-zip.js')
|
|
2
|
+
const t = require('tap')
|
|
3
|
+
const fs = require('fs')
|
|
4
|
+
const Stream = require('stream')
|
|
5
|
+
const concat = require('concat-stream')
|
|
6
|
+
const yauzl = require('yauzl')
|
|
7
|
+
const { join } = require('path')
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
const fileStream = function (file, forceError) {
|
|
10
|
+
const rs = new Stream()
|
|
11
11
|
rs.readable = true
|
|
12
|
-
|
|
12
|
+
const fileStream = fs.createReadStream(join(__dirname, file))
|
|
13
13
|
fileStream.pipe(
|
|
14
14
|
concat(function buffersEmit (buffer) {
|
|
15
15
|
if (forceError) {
|
|
@@ -27,20 +27,20 @@ var fileStream = function (file, forceError) {
|
|
|
27
27
|
return rs
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
const file1 = '/fixtures/file.txt'
|
|
31
|
+
const emptyFile = '/fixtures/empty.txt'
|
|
32
32
|
// Stub: var fileStream = s3Files.createFileStream(keyStream);
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
const sinon = require('sinon')
|
|
34
|
+
const proxyquire = require('proxyquire')
|
|
35
|
+
const s3Stub = fileStream(file1)
|
|
36
36
|
s3Zip = proxyquire('../s3-zip.js', {
|
|
37
37
|
's3-files': { createFileStream: sinon.stub().returns(s3Stub) }
|
|
38
38
|
})
|
|
39
39
|
|
|
40
40
|
t.test('test archiveStream and zip file', function (child) {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
const output = fs.createWriteStream(join(__dirname, '/test.zip'))
|
|
42
|
+
const s = fileStream(file1)
|
|
43
|
+
const archive = s3Zip.archiveStream(s).pipe(output)
|
|
44
44
|
archive.on('close', function () {
|
|
45
45
|
console.log('+++++++++++')
|
|
46
46
|
yauzl.open(join(__dirname, '/test.zip'), function (err, zip) {
|
|
@@ -61,7 +61,7 @@ t.test('test archiveStream and zip file', function (child) {
|
|
|
61
61
|
})
|
|
62
62
|
|
|
63
63
|
t.test('test archive', function (child) {
|
|
64
|
-
|
|
64
|
+
const archive = s3Zip.archive(
|
|
65
65
|
{ region: 'region', bucket: 'bucket' },
|
|
66
66
|
'folder',
|
|
67
67
|
[file1]
|
|
@@ -71,9 +71,9 @@ t.test('test archive', function (child) {
|
|
|
71
71
|
})
|
|
72
72
|
|
|
73
73
|
t.test('test archive on empty file', function (child) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
const output = fs.createWriteStream(join(__dirname, '/test.zip'))
|
|
75
|
+
const s = fileStream(emptyFile)
|
|
76
|
+
const archive = s3Zip.archiveStream(s).pipe(output)
|
|
77
77
|
archive.on('close', function () {
|
|
78
78
|
console.log('+++++++++++')
|
|
79
79
|
yauzl.open(join(__dirname, '/test.zip'), function (err, zip) {
|