qdone 1.6.0 → 2.0.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/commonjs/index.js +10 -0
- package/commonjs/package.json +3 -0
- package/commonjs/src/cache.js +142 -0
- package/commonjs/src/cloudWatch.js +148 -0
- package/commonjs/src/consumer.js +483 -0
- package/commonjs/src/defaults.js +107 -0
- package/commonjs/src/enqueue.js +498 -0
- package/commonjs/src/idleQueues.js +466 -0
- package/commonjs/src/qrlCache.js +250 -0
- package/commonjs/src/sqs.js +160 -0
- package/npm-shrinkwrap.json +17598 -264
- package/package.json +41 -29
- package/src/bin.js +3 -0
- package/src/cache.js +21 -25
- package/src/cli.js +269 -181
- package/src/cloudWatch.js +97 -0
- package/src/consumer.js +346 -0
- package/src/defaults.js +114 -0
- package/src/enqueue.js +239 -196
- package/src/idleQueues.js +242 -223
- package/src/monitor.js +53 -0
- package/src/qrlCache.js +110 -83
- package/src/sentry.js +30 -0
- package/src/sqs.js +73 -0
- package/src/worker.js +197 -202
- package/.DS_Store +0 -0
- package/.coveralls.yml +0 -2
- package/.travis.yml +0 -17
- package/CHANGELOG.md +0 -107
- package/dump.rdb +0 -0
- package/index.js +0 -6
- package/package-lock.json.old +0 -3939
- package/qdone +0 -2
- package/test/fixtures/test-child-kill-linux.sh +0 -9
- package/test/fixtures/test-fifo01-x24.batch +0 -24
- package/test/fixtures/test-too-big-1.batch +0 -10
- package/test/fixtures/test-unique01-x24.batch +0 -24
- package/test/fixtures/test-unique02-x24.batch +0 -24
- package/test/fixtures/test-unique24-x24.batch +0 -24
- package/test/fixtures/test-unique24-x240.batch +0 -240
- package/test/test.cache.js +0 -61
- package/test/test.cli.js +0 -1609
package/package.json
CHANGED
|
@@ -1,42 +1,49 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "qdone",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0-alpha",
|
|
4
4
|
"description": "Language agnostic job queue for SQS",
|
|
5
|
-
"
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./index.js",
|
|
7
|
+
"module": "./index.js",
|
|
8
|
+
"bin": "./src/bin.js",
|
|
9
|
+
"files": [
|
|
10
|
+
"src",
|
|
11
|
+
"commonjs"
|
|
12
|
+
],
|
|
6
13
|
"dependencies": {
|
|
7
|
-
"aws-sdk": "^
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
"
|
|
11
|
-
"command-line-
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
"
|
|
14
|
+
"@aws-sdk/client-cloudwatch": "^3.465.0",
|
|
15
|
+
"@aws-sdk/client-sqs": "^3.465.0",
|
|
16
|
+
"@sentry/node": "^7.87.0",
|
|
17
|
+
"chalk": "^4.1.2",
|
|
18
|
+
"command-line-args": "^5.2.1",
|
|
19
|
+
"command-line-commands": "^3.0.2",
|
|
20
|
+
"command-line-usage": "^7.0.1",
|
|
21
|
+
"debug": "^4.3.4",
|
|
22
|
+
"ioredis": "^5.3.2",
|
|
23
|
+
"standard": "^17.1.0",
|
|
24
|
+
"tree-kill": "^1.2.2",
|
|
25
|
+
"uuid": "^9.0.1"
|
|
17
26
|
},
|
|
18
27
|
"devDependencies": {
|
|
19
|
-
"aws-sdk-mock": "^
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"mocha": "^6.1.4",
|
|
24
|
-
"mock-stdin": "^0.3.1",
|
|
25
|
-
"nyc": "^14.1.1",
|
|
26
|
-
"sinon": "^7.3.2",
|
|
27
|
-
"standard": "*",
|
|
28
|
-
"strip-ansi": "^5.0.0"
|
|
28
|
+
"aws-sdk-client-mock": "^3.0.0",
|
|
29
|
+
"aws-sdk-client-mock-jest": "^3.0.0",
|
|
30
|
+
"jest": "^29.7.0",
|
|
31
|
+
"typescript": "^5.3.3"
|
|
29
32
|
},
|
|
30
|
-
"bin": "./qdone",
|
|
31
33
|
"preferGlobal": true,
|
|
32
34
|
"engines": {
|
|
33
|
-
"node": ">=
|
|
35
|
+
"node": ">=16"
|
|
34
36
|
},
|
|
35
37
|
"scripts": {
|
|
36
|
-
"
|
|
38
|
+
"start": "./src/bin.js",
|
|
39
|
+
"test": "NODE_OPTIONS='--experimental-json-modules --experimental-vm-modules --no-warnings' jest",
|
|
40
|
+
"build": "tsc --allowJs index.js --outdir commonjs --esModuleInterop --module commonjs",
|
|
41
|
+
"clean": "rm -rf commonjs/src commonjs/*.js coverage",
|
|
42
|
+
"lint": "standard",
|
|
37
43
|
"coverage": "nyc report --reporter=text-lcov | coveralls",
|
|
38
44
|
"standard": "standard",
|
|
39
|
-
"
|
|
45
|
+
"prepare": "pnpm run clean && pnpm run build && npm shrinkwrap",
|
|
46
|
+
"prep-for-publish": "echo now commit shrinkwrap and use npm run publish-{next,latest}",
|
|
40
47
|
"publish-latest": "npm publish --tag latest",
|
|
41
48
|
"publish-next": "npm publish --tag next"
|
|
42
49
|
},
|
|
@@ -57,9 +64,14 @@
|
|
|
57
64
|
"consumer",
|
|
58
65
|
"amazon"
|
|
59
66
|
],
|
|
60
|
-
"
|
|
61
|
-
"
|
|
62
|
-
|
|
67
|
+
"jest": {
|
|
68
|
+
"testEnvironment": "jest-environment-node",
|
|
69
|
+
"transform": {}
|
|
70
|
+
},
|
|
71
|
+
"standard": {
|
|
72
|
+
"env": [
|
|
73
|
+
"jest",
|
|
74
|
+
"es6"
|
|
63
75
|
]
|
|
64
76
|
},
|
|
65
77
|
"bugs": {
|
package/src/bin.js
ADDED
package/src/cache.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const debug =
|
|
1
|
+
import Redis from 'ioredis'
|
|
2
|
+
import { URL } from 'url'
|
|
3
|
+
import Debug from 'debug'
|
|
4
|
+
const debug = Debug('qdone:cache')
|
|
5
5
|
|
|
6
6
|
class UsageError extends Error {}
|
|
7
7
|
|
|
@@ -11,27 +11,26 @@ let client
|
|
|
11
11
|
* Internal function to setup redis client. Parses URI to figure out
|
|
12
12
|
* how to connect.
|
|
13
13
|
*/
|
|
14
|
-
function
|
|
14
|
+
export function getCacheClient (opt) {
|
|
15
15
|
if (client) {
|
|
16
16
|
return client
|
|
17
|
-
} else if (
|
|
18
|
-
const url = new URL(
|
|
17
|
+
} else if (opt.cacheUri) {
|
|
18
|
+
const url = new URL(opt.cacheUri)
|
|
19
19
|
if (url.protocol === 'redis:') {
|
|
20
20
|
client = new Redis(url.toString())
|
|
21
21
|
} else if (url.protocol === 'redis-cluster:') {
|
|
22
22
|
url.protocol = 'redis:'
|
|
23
|
-
client = new Redis.Cluster([url.toString()], {slotsRefreshInterval: 60 * 1000})
|
|
23
|
+
client = new Redis.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 })
|
|
24
24
|
} else {
|
|
25
25
|
throw new UsageError(`Only redis:// or redis-cluster:// URLs are currently supported. Got: ${url.protocol}`)
|
|
26
26
|
}
|
|
27
|
-
// setTimeout(resetClient, 10000)
|
|
28
27
|
return client
|
|
29
28
|
} else {
|
|
30
29
|
throw new UsageError('Caching requires the --cache-uri option')
|
|
31
30
|
}
|
|
32
31
|
}
|
|
33
32
|
|
|
34
|
-
function
|
|
33
|
+
export function shutdownCache () {
|
|
35
34
|
if (client) client.quit()
|
|
36
35
|
client = undefined
|
|
37
36
|
}
|
|
@@ -40,27 +39,24 @@ function resetClient () {
|
|
|
40
39
|
* Returns a promise for the item. Resolves to false if cache is empty, object
|
|
41
40
|
* if it is found.
|
|
42
41
|
*/
|
|
43
|
-
function getCache (key,
|
|
44
|
-
const client =
|
|
45
|
-
const cacheKey =
|
|
46
|
-
debug({action: 'getCache', cacheKey})
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
})
|
|
42
|
+
export async function getCache (key, opt) {
|
|
43
|
+
const client = getCacheClient(opt)
|
|
44
|
+
const cacheKey = opt.cachePrefix + key
|
|
45
|
+
debug({ action: 'getCache', cacheKey })
|
|
46
|
+
const result = await client.get(cacheKey)
|
|
47
|
+
debug({ action: 'getCache got', cacheKey, result })
|
|
48
|
+
return result ? JSON.parse(result) : undefined
|
|
51
49
|
}
|
|
52
50
|
|
|
53
51
|
/**
|
|
54
52
|
* Returns a promise for the status. Encodes object as JSON
|
|
55
53
|
*/
|
|
56
|
-
function setCache (key, value,
|
|
57
|
-
const client =
|
|
54
|
+
export async function setCache (key, value, opt) {
|
|
55
|
+
const client = getCacheClient(opt)
|
|
58
56
|
const encoded = JSON.stringify(value)
|
|
59
|
-
const cacheKey =
|
|
60
|
-
debug({action: 'setCache', cacheKey, value})
|
|
61
|
-
return client.setex(cacheKey,
|
|
57
|
+
const cacheKey = opt.cachePrefix + key
|
|
58
|
+
debug({ action: 'setCache', cacheKey, value })
|
|
59
|
+
return client.setex(cacheKey, opt.cacheTtlSeconds, encoded)
|
|
62
60
|
}
|
|
63
61
|
|
|
64
|
-
module.exports = { getCache, setCache, getClient, resetClient }
|
|
65
|
-
|
|
66
62
|
debug('loaded')
|