queue-inspector-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/LICENSE +21 -0
- package/README.md +148 -0
- package/dist/backends/asynq-proto.d.ts +16 -0
- package/dist/backends/asynq-proto.js +144 -0
- package/dist/backends/asynq-proto.js.map +1 -0
- package/dist/backends/asynq.d.ts +45 -0
- package/dist/backends/asynq.js +183 -0
- package/dist/backends/asynq.js.map +1 -0
- package/dist/backends/bullmq.d.ts +46 -0
- package/dist/backends/bullmq.js +279 -0
- package/dist/backends/bullmq.js.map +1 -0
- package/dist/backends/index.d.ts +19 -0
- package/dist/backends/index.js +57 -0
- package/dist/backends/index.js.map +1 -0
- package/dist/backends/lua/deleteTask.lua +41 -0
- package/dist/backends/lua/removeJob.lua +346 -0
- package/dist/backends/lua/reprocessJob.lua +113 -0
- package/dist/backends/lua/runTask.lua +39 -0
- package/dist/backends/lua.d.ts +18 -0
- package/dist/backends/lua.js +18 -0
- package/dist/backends/lua.js.map +1 -0
- package/dist/backends/scripting.d.ts +14 -0
- package/dist/backends/scripting.js +34 -0
- package/dist/backends/scripting.js.map +1 -0
- package/dist/config.d.ts +10 -0
- package/dist/config.js +32 -0
- package/dist/config.js.map +1 -0
- package/dist/format.d.ts +16 -0
- package/dist/format.js +49 -0
- package/dist/format.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +32 -0
- package/dist/index.js.map +1 -0
- package/dist/redis.d.ts +13 -0
- package/dist/redis.js +30 -0
- package/dist/redis.js.map +1 -0
- package/dist/server.d.ts +7 -0
- package/dist/server.js +105 -0
- package/dist/server.js.map +1 -0
- package/dist/types.d.ts +64 -0
- package/dist/types.js +14 -0
- package/dist/types.js.map +1 -0
- package/package.json +56 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
-- Vendored verbatim from BullMQ (github.com/taskforcesh/bullmq, MIT license),
|
|
2
|
+
-- assembled script `removeJob`, version 5.79.3. This is the exact atomic script
|
|
3
|
+
-- Job.remove() runs. It handles children, locks and deduplication keys, so it is
|
|
4
|
+
-- used as-is rather than reimplemented, to avoid leaving a queue in a broken state.
|
|
5
|
+
--[[
|
|
6
|
+
Remove a job from all the statuses it may be in as well as all its data.
|
|
7
|
+
In order to be able to remove a job, it cannot be active.
|
|
8
|
+
Input:
|
|
9
|
+
KEYS[1] jobKey
|
|
10
|
+
KEYS[2] repeat key
|
|
11
|
+
ARGV[1] jobId
|
|
12
|
+
ARGV[2] remove children
|
|
13
|
+
ARGV[3] queue prefix
|
|
14
|
+
Events:
|
|
15
|
+
'removed'
|
|
16
|
+
]]
|
|
17
|
+
local rcall = redis.call
|
|
18
|
+
-- Includes
|
|
19
|
+
--[[
|
|
20
|
+
Function to check if the job belongs to a job scheduler and
|
|
21
|
+
current delayed job matches with jobId
|
|
22
|
+
]]
|
|
23
|
+
local function isJobSchedulerJob(jobId, jobKey, jobSchedulersKey)
|
|
24
|
+
local repeatJobKey = rcall("HGET", jobKey, "rjk")
|
|
25
|
+
if repeatJobKey then
|
|
26
|
+
local prevMillis = rcall("ZSCORE", jobSchedulersKey, repeatJobKey)
|
|
27
|
+
if prevMillis then
|
|
28
|
+
local currentDelayedJobId = "repeat:" .. repeatJobKey .. ":" .. prevMillis
|
|
29
|
+
return jobId == currentDelayedJobId
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
return false
|
|
33
|
+
end
|
|
34
|
+
--[[
|
|
35
|
+
Function to recursively check if there are no locks
|
|
36
|
+
on the jobs to be removed.
|
|
37
|
+
returns:
|
|
38
|
+
boolean
|
|
39
|
+
]]
|
|
40
|
+
--[[
|
|
41
|
+
Functions to destructure job key.
|
|
42
|
+
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
|
43
|
+
]]
|
|
44
|
+
local getJobIdFromKey = function (jobKey)
|
|
45
|
+
return string.match(jobKey, ".*:(.*)")
|
|
46
|
+
end
|
|
47
|
+
local getJobKeyPrefix = function (jobKey, jobId)
|
|
48
|
+
return string.sub(jobKey, 0, #jobKey - #jobId)
|
|
49
|
+
end
|
|
50
|
+
local function isLocked( prefix, jobId, removeChildren)
|
|
51
|
+
local jobKey = prefix .. jobId;
|
|
52
|
+
-- Check if this job is locked
|
|
53
|
+
local lockKey = jobKey .. ':lock'
|
|
54
|
+
local lock = rcall("GET", lockKey)
|
|
55
|
+
if not lock then
|
|
56
|
+
if removeChildren == "1" then
|
|
57
|
+
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
|
58
|
+
if (#dependencies > 0) then
|
|
59
|
+
for i, childJobKey in ipairs(dependencies) do
|
|
60
|
+
-- We need to get the jobId for this job.
|
|
61
|
+
local childJobId = getJobIdFromKey(childJobKey)
|
|
62
|
+
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
|
63
|
+
local result = isLocked( childJobPrefix, childJobId, removeChildren )
|
|
64
|
+
if result then
|
|
65
|
+
return true
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
return false
|
|
71
|
+
end
|
|
72
|
+
return true
|
|
73
|
+
end
|
|
74
|
+
--[[
|
|
75
|
+
Remove a job from all the statuses it may be in as well as all its data,
|
|
76
|
+
including its children. Active children can be ignored.
|
|
77
|
+
Events:
|
|
78
|
+
'removed'
|
|
79
|
+
]]
|
|
80
|
+
local rcall = redis.call
|
|
81
|
+
-- Includes
|
|
82
|
+
--[[
|
|
83
|
+
Function to get max events value or set by default 10000.
|
|
84
|
+
]]
|
|
85
|
+
local function getOrSetMaxEvents(metaKey)
|
|
86
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
|
87
|
+
if not maxEvents then
|
|
88
|
+
maxEvents = 10000
|
|
89
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
|
90
|
+
end
|
|
91
|
+
return maxEvents
|
|
92
|
+
end
|
|
93
|
+
--[[
|
|
94
|
+
Function to remove deduplication key if needed
|
|
95
|
+
when a job is being removed.
|
|
96
|
+
]]
|
|
97
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
|
98
|
+
jobId, deduplicationId)
|
|
99
|
+
if deduplicationId then
|
|
100
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
|
101
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
|
102
|
+
if currentJobId and currentJobId == jobId then
|
|
103
|
+
rcall("DEL", deduplicationKey)
|
|
104
|
+
-- Also clean up any pending dedup-next data for this dedup ID
|
|
105
|
+
rcall("DEL", prefixKey .. "dn:" .. deduplicationId)
|
|
106
|
+
return 1
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
--[[
|
|
111
|
+
Function to remove from any state.
|
|
112
|
+
returns:
|
|
113
|
+
prev state
|
|
114
|
+
]]
|
|
115
|
+
local function removeJobFromAnyState( prefix, jobId)
|
|
116
|
+
-- We start with the ZSCORE checks, since they have O(1) complexity
|
|
117
|
+
if rcall("ZSCORE", prefix .. "completed", jobId) then
|
|
118
|
+
rcall("ZREM", prefix .. "completed", jobId)
|
|
119
|
+
return "completed"
|
|
120
|
+
elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
|
|
121
|
+
rcall("ZREM", prefix .. "waiting-children", jobId)
|
|
122
|
+
return "waiting-children"
|
|
123
|
+
elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
|
|
124
|
+
rcall("ZREM", prefix .. "delayed", jobId)
|
|
125
|
+
return "delayed"
|
|
126
|
+
elseif rcall("ZSCORE", prefix .. "failed", jobId) then
|
|
127
|
+
rcall("ZREM", prefix .. "failed", jobId)
|
|
128
|
+
return "failed"
|
|
129
|
+
elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
|
|
130
|
+
rcall("ZREM", prefix .. "prioritized", jobId)
|
|
131
|
+
return "prioritized"
|
|
132
|
+
-- We remove only 1 element from the list, since we assume they are not added multiple times
|
|
133
|
+
elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
|
|
134
|
+
return "wait"
|
|
135
|
+
elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
|
|
136
|
+
return "paused"
|
|
137
|
+
elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
|
|
138
|
+
return "active"
|
|
139
|
+
end
|
|
140
|
+
return "unknown"
|
|
141
|
+
end
|
|
142
|
+
--[[
|
|
143
|
+
Function to remove job keys.
|
|
144
|
+
]]
|
|
145
|
+
local function removeJobKeys(jobKey)
|
|
146
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
|
147
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
|
148
|
+
end
|
|
149
|
+
--[[
|
|
150
|
+
Check if this job has a parent. If so we will just remove it from
|
|
151
|
+
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
|
152
|
+
which requires code from "moveToFinished"
|
|
153
|
+
]]
|
|
154
|
+
-- Includes
|
|
155
|
+
--[[
|
|
156
|
+
Function to add job in target list and add marker if needed.
|
|
157
|
+
]]
|
|
158
|
+
-- Includes
|
|
159
|
+
--[[
|
|
160
|
+
Add marker if needed when a job is available.
|
|
161
|
+
]]
|
|
162
|
+
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
163
|
+
if not isPausedOrMaxed then
|
|
164
|
+
rcall("ZADD", markerKey, 0, "0")
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
|
168
|
+
rcall(pushCmd, targetKey, jobId)
|
|
169
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
170
|
+
end
|
|
171
|
+
--[[
|
|
172
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
|
173
|
+
(since an empty list and !EXISTS are not really the same).
|
|
174
|
+
]]
|
|
175
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
176
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
177
|
+
if queueAttributes[1] then
|
|
178
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
179
|
+
else
|
|
180
|
+
if queueAttributes[2] then
|
|
181
|
+
local activeCount = rcall("LLEN", activeKey)
|
|
182
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
|
183
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
184
|
+
else
|
|
185
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
190
|
+
end
|
|
191
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
192
|
+
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
|
193
|
+
parentPrefix .. "wait", parentPrefix .. "paused")
|
|
194
|
+
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
|
195
|
+
if emitEvent then
|
|
196
|
+
local parentEventStream = parentPrefix .. "events"
|
|
197
|
+
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
|
201
|
+
if parentKey then
|
|
202
|
+
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
203
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
204
|
+
if result > 0 then
|
|
205
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
206
|
+
if pendingDependencies == 0 then
|
|
207
|
+
local parentId = getJobIdFromKey(parentKey)
|
|
208
|
+
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
|
209
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
210
|
+
if numRemovedElements == 1 then
|
|
211
|
+
if hard then -- remove parent in same queue
|
|
212
|
+
if parentPrefix == baseKey then
|
|
213
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
|
214
|
+
removeJobKeys(parentKey)
|
|
215
|
+
if debounceId then
|
|
216
|
+
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
|
217
|
+
end
|
|
218
|
+
else
|
|
219
|
+
_moveParentToWait(parentPrefix, parentId)
|
|
220
|
+
end
|
|
221
|
+
else
|
|
222
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
return true
|
|
227
|
+
end
|
|
228
|
+
else
|
|
229
|
+
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
|
230
|
+
local missedParentKey = parentAttributes[1]
|
|
231
|
+
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
|
232
|
+
and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
233
|
+
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
234
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
235
|
+
if result > 0 then
|
|
236
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
237
|
+
if pendingDependencies == 0 then
|
|
238
|
+
local parentId = getJobIdFromKey(missedParentKey)
|
|
239
|
+
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
240
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
241
|
+
if numRemovedElements == 1 then
|
|
242
|
+
if hard then
|
|
243
|
+
if parentPrefix == baseKey then
|
|
244
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
|
245
|
+
removeJobKeys(missedParentKey)
|
|
246
|
+
if parentAttributes[2] then
|
|
247
|
+
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
|
248
|
+
end
|
|
249
|
+
else
|
|
250
|
+
_moveParentToWait(parentPrefix, parentId)
|
|
251
|
+
end
|
|
252
|
+
else
|
|
253
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
return true
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
return false
|
|
262
|
+
end
|
|
263
|
+
local removeJobChildren
|
|
264
|
+
local removeJobWithChildren
|
|
265
|
+
removeJobChildren = function(prefix, jobKey, options)
|
|
266
|
+
-- Check if this job has children
|
|
267
|
+
-- If so, we are going to try to remove the children recursively in a depth-first way
|
|
268
|
+
-- because if some job is locked, we must exit with an error.
|
|
269
|
+
if not options.ignoreProcessed then
|
|
270
|
+
local processed = rcall("HGETALL", jobKey .. ":processed")
|
|
271
|
+
if #processed > 0 then
|
|
272
|
+
for i = 1, #processed, 2 do
|
|
273
|
+
local childJobId = getJobIdFromKey(processed[i])
|
|
274
|
+
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
|
275
|
+
removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
local failed = rcall("HGETALL", jobKey .. ":failed")
|
|
279
|
+
if #failed > 0 then
|
|
280
|
+
for i = 1, #failed, 2 do
|
|
281
|
+
local childJobId = getJobIdFromKey(failed[i])
|
|
282
|
+
local childJobPrefix = getJobKeyPrefix(failed[i], childJobId)
|
|
283
|
+
removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
local unsuccessful = rcall("ZRANGE", jobKey .. ":unsuccessful", 0, -1)
|
|
287
|
+
if #unsuccessful > 0 then
|
|
288
|
+
for i = 1, #unsuccessful, 1 do
|
|
289
|
+
local childJobId = getJobIdFromKey(unsuccessful[i])
|
|
290
|
+
local childJobPrefix = getJobKeyPrefix(unsuccessful[i], childJobId)
|
|
291
|
+
removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
|
296
|
+
if #dependencies > 0 then
|
|
297
|
+
for i, childJobKey in ipairs(dependencies) do
|
|
298
|
+
local childJobId = getJobIdFromKey(childJobKey)
|
|
299
|
+
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
|
300
|
+
removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
removeJobWithChildren = function(prefix, jobId, parentKey, options)
|
|
305
|
+
local jobKey = prefix .. jobId
|
|
306
|
+
if options.ignoreLocked then
|
|
307
|
+
if isLocked(prefix, jobId) then
|
|
308
|
+
return
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
-- Check if job is in the failed zset
|
|
312
|
+
local failedSet = prefix .. "failed"
|
|
313
|
+
if not (options.ignoreProcessed and rcall("ZSCORE", failedSet, jobId)) then
|
|
314
|
+
removeParentDependencyKey(jobKey, false, parentKey, nil)
|
|
315
|
+
if options.removeChildren then
|
|
316
|
+
removeJobChildren(prefix, jobKey, options)
|
|
317
|
+
end
|
|
318
|
+
local prev = removeJobFromAnyState(prefix, jobId)
|
|
319
|
+
local deduplicationId = rcall("HGET", jobKey, "deid")
|
|
320
|
+
removeDeduplicationKeyIfNeededOnRemoval(prefix, jobId, deduplicationId)
|
|
321
|
+
if removeJobKeys(jobKey) > 0 then
|
|
322
|
+
local metaKey = prefix .. "meta"
|
|
323
|
+
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
324
|
+
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
|
325
|
+
"jobId", jobId, "prev", prev)
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
local jobId = ARGV[1]
|
|
330
|
+
local shouldRemoveChildren = ARGV[2]
|
|
331
|
+
local prefix = ARGV[3]
|
|
332
|
+
local jobKey = KEYS[1]
|
|
333
|
+
local repeatKey = KEYS[2]
|
|
334
|
+
if isJobSchedulerJob(jobId, jobKey, repeatKey) then
|
|
335
|
+
return -8
|
|
336
|
+
end
|
|
337
|
+
if not isLocked(prefix, jobId, shouldRemoveChildren) then
|
|
338
|
+
local options = {
|
|
339
|
+
removeChildren = shouldRemoveChildren == "1",
|
|
340
|
+
ignoreProcessed = false,
|
|
341
|
+
ignoreLocked = false
|
|
342
|
+
}
|
|
343
|
+
removeJobWithChildren(prefix, jobId, nil, options)
|
|
344
|
+
return 1
|
|
345
|
+
end
|
|
346
|
+
return 0
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
-- Vendored verbatim from BullMQ (github.com/taskforcesh/bullmq, MIT license),
|
|
2
|
+
-- assembled script `reprocessJob`, version 5.79.3. This is the exact atomic
|
|
3
|
+
-- script Job.retry() runs to move a job from a terminal state back to wait.
|
|
4
|
+
-- Kept as-is so retry behaviour matches the library rather than a reimplementation.
|
|
5
|
+
--[[
|
|
6
|
+
Attempts to reprocess a job
|
|
7
|
+
Input:
|
|
8
|
+
KEYS[1] job key
|
|
9
|
+
KEYS[2] events stream
|
|
10
|
+
KEYS[3] job state
|
|
11
|
+
KEYS[4] wait key
|
|
12
|
+
KEYS[5] meta
|
|
13
|
+
KEYS[6] paused key
|
|
14
|
+
KEYS[7] active key
|
|
15
|
+
KEYS[8] marker key
|
|
16
|
+
ARGV[1] job.id
|
|
17
|
+
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
|
|
18
|
+
ARGV[3] propVal - failedReason/returnvalue
|
|
19
|
+
ARGV[4] prev state - failed/completed
|
|
20
|
+
ARGV[5] reset attemptsMade - "1" or "0"
|
|
21
|
+
ARGV[6] reset attemptsStarted - "1" or "0"
|
|
22
|
+
Output:
|
|
23
|
+
1 means the operation was a success
|
|
24
|
+
-1 means the job does not exist
|
|
25
|
+
-3 means the job was not found in the expected set.
|
|
26
|
+
]]
|
|
27
|
+
local rcall = redis.call;
|
|
28
|
+
-- Includes
|
|
29
|
+
--[[
|
|
30
|
+
Function to add job in target list and add marker if needed.
|
|
31
|
+
]]
|
|
32
|
+
-- Includes
|
|
33
|
+
--[[
|
|
34
|
+
Add marker if needed when a job is available.
|
|
35
|
+
]]
|
|
36
|
+
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
37
|
+
if not isPausedOrMaxed then
|
|
38
|
+
rcall("ZADD", markerKey, 0, "0")
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
|
42
|
+
rcall(pushCmd, targetKey, jobId)
|
|
43
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
44
|
+
end
|
|
45
|
+
--[[
|
|
46
|
+
Function to get max events value or set by default 10000.
|
|
47
|
+
]]
|
|
48
|
+
local function getOrSetMaxEvents(metaKey)
|
|
49
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
|
50
|
+
if not maxEvents then
|
|
51
|
+
maxEvents = 10000
|
|
52
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
|
53
|
+
end
|
|
54
|
+
return maxEvents
|
|
55
|
+
end
|
|
56
|
+
--[[
|
|
57
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
|
58
|
+
(since an empty list and !EXISTS are not really the same).
|
|
59
|
+
]]
|
|
60
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
61
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
62
|
+
if queueAttributes[1] then
|
|
63
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
64
|
+
else
|
|
65
|
+
if queueAttributes[2] then
|
|
66
|
+
local activeCount = rcall("LLEN", activeKey)
|
|
67
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
|
68
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
69
|
+
else
|
|
70
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
75
|
+
end
|
|
76
|
+
local jobKey = KEYS[1]
|
|
77
|
+
if rcall("EXISTS", jobKey) == 1 then
|
|
78
|
+
local jobId = ARGV[1]
|
|
79
|
+
if (rcall("ZREM", KEYS[3], jobId) == 1) then
|
|
80
|
+
local attributesToRemove = {}
|
|
81
|
+
if ARGV[5] == "1" then
|
|
82
|
+
table.insert(attributesToRemove, "atm")
|
|
83
|
+
end
|
|
84
|
+
if ARGV[6] == "1" then
|
|
85
|
+
table.insert(attributesToRemove, "ats")
|
|
86
|
+
end
|
|
87
|
+
rcall("HDEL", jobKey, "finishedOn", "processedOn", ARGV[3], unpack(attributesToRemove))
|
|
88
|
+
local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[7], KEYS[4], KEYS[6])
|
|
89
|
+
addJobInTargetList(target, KEYS[8], ARGV[2], isPausedOrMaxed, jobId)
|
|
90
|
+
local parentKey = rcall("HGET", jobKey, "parentKey")
|
|
91
|
+
if parentKey and rcall("EXISTS", parentKey) == 1 then
|
|
92
|
+
if ARGV[4] == "failed" then
|
|
93
|
+
if rcall("ZREM", parentKey .. ":unsuccessful", jobKey) == 1 or
|
|
94
|
+
rcall("HDEL", parentKey .. ":failed", jobKey) == 1 then
|
|
95
|
+
rcall("SADD", parentKey .. ":dependencies", jobKey)
|
|
96
|
+
end
|
|
97
|
+
else
|
|
98
|
+
if rcall("HDEL", parentKey .. ":processed", jobKey) == 1 then
|
|
99
|
+
rcall("SADD", parentKey .. ":dependencies", jobKey)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
local maxEvents = getOrSetMaxEvents(KEYS[5])
|
|
104
|
+
-- Emit waiting event
|
|
105
|
+
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
|
106
|
+
"jobId", jobId, "prev", ARGV[4]);
|
|
107
|
+
return 1
|
|
108
|
+
else
|
|
109
|
+
return -3
|
|
110
|
+
end
|
|
111
|
+
else
|
|
112
|
+
return -1
|
|
113
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
-- Transcribed verbatim from Asynq (github.com/hibiken/asynq, MIT license),
|
|
2
|
+
-- internal/rdb/inspect.go `runTaskCmd`, version v0.25.1. This is the exact
|
|
3
|
+
-- atomic script Inspector.RunTask() runs to move a task from scheduled, retry,
|
|
4
|
+
-- archived, completed or aggregating state back to pending. Kept as-is so retry
|
|
5
|
+
-- behaviour matches the library.
|
|
6
|
+
--
|
|
7
|
+
-- KEYS[1] -> task key (asynq:{<qname>}:t:<task_id>)
|
|
8
|
+
-- KEYS[2] -> pending list (asynq:{<qname>}:pending)
|
|
9
|
+
-- KEYS[3] -> all groups set (asynq:{<qname>}:groups)
|
|
10
|
+
-- ARGV[1] -> task id
|
|
11
|
+
-- ARGV[2] -> queue key prefix (asynq:{<qname>}:)
|
|
12
|
+
-- ARGV[3] -> group key prefix (asynq:{<qname>}:g:)
|
|
13
|
+
--
|
|
14
|
+
-- Returns 1 on success, 0 if not found, -1 if active, -2 if already pending.
|
|
15
|
+
if redis.call("EXISTS", KEYS[1]) == 0 then
|
|
16
|
+
return 0
|
|
17
|
+
end
|
|
18
|
+
local state, group = unpack(redis.call("HMGET", KEYS[1], "state", "group"))
|
|
19
|
+
if state == "active" then
|
|
20
|
+
return -1
|
|
21
|
+
elseif state == "pending" then
|
|
22
|
+
return -2
|
|
23
|
+
elseif state == "aggregating" then
|
|
24
|
+
local n = redis.call("ZREM", ARGV[3] .. group, ARGV[1])
|
|
25
|
+
if n == 0 then
|
|
26
|
+
return redis.error_reply("internal error: task id not found in zset " .. tostring(ARGV[3] .. group))
|
|
27
|
+
end
|
|
28
|
+
if redis.call("ZCARD", ARGV[3] .. group) == 0 then
|
|
29
|
+
redis.call("SREM", KEYS[3], group)
|
|
30
|
+
end
|
|
31
|
+
else
|
|
32
|
+
local n = redis.call("ZREM", ARGV[2] .. state, ARGV[1])
|
|
33
|
+
if n == 0 then
|
|
34
|
+
return redis.error_reply("internal error: task id not found in zset " .. tostring(ARGV[2] .. state))
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
redis.call("LPUSH", KEYS[2], ARGV[1])
|
|
38
|
+
redis.call("HSET", KEYS[1], "state", "pending")
|
|
39
|
+
return 1
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export declare const scripts: {
|
|
2
|
+
readonly reprocessJob: {
|
|
3
|
+
readonly source: string;
|
|
4
|
+
readonly numberOfKeys: 8;
|
|
5
|
+
};
|
|
6
|
+
readonly removeJob: {
|
|
7
|
+
readonly source: string;
|
|
8
|
+
readonly numberOfKeys: 2;
|
|
9
|
+
};
|
|
10
|
+
readonly runTask: {
|
|
11
|
+
readonly source: string;
|
|
12
|
+
readonly numberOfKeys: 3;
|
|
13
|
+
};
|
|
14
|
+
readonly deleteTask: {
|
|
15
|
+
readonly source: string;
|
|
16
|
+
readonly numberOfKeys: 2;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
// Vendored Redis scripts live next to the compiled adapters (copied into dist by
|
|
4
|
+
// the build step). They are loaded once at module init and registered on the
|
|
5
|
+
// ioredis client with `defineCommand`, which handles EVALSHA caching.
|
|
6
|
+
const luaDir = fileURLToPath(new URL("./lua/", import.meta.url));
|
|
7
|
+
function load(name) {
|
|
8
|
+
return readFileSync(`${luaDir}${name}.lua`, "utf8");
|
|
9
|
+
}
|
|
10
|
+
export const scripts = {
|
|
11
|
+
// BullMQ: exact assembled scripts from the library.
|
|
12
|
+
reprocessJob: { source: load("reprocessJob"), numberOfKeys: 8 },
|
|
13
|
+
removeJob: { source: load("removeJob"), numberOfKeys: 2 },
|
|
14
|
+
// Asynq: exact scripts from the library's rdb inspector.
|
|
15
|
+
runTask: { source: load("runTask"), numberOfKeys: 3 },
|
|
16
|
+
deleteTask: { source: load("deleteTask"), numberOfKeys: 2 },
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=lua.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lua.js","sourceRoot":"","sources":["../../src/backends/lua.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,iFAAiF;AACjF,6EAA6E;AAC7E,sEAAsE;AACtE,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAEjE,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,YAAY,CAAC,GAAG,MAAM,GAAG,IAAI,MAAM,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,oDAAoD;IACpD,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;IAC/D,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;IACzD,yDAAyD;IACzD,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;IACrD,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;CACnD,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Redis } from "ioredis";
|
|
2
|
+
type Caller = (keys: string[], args: Array<string | number>) => Promise<number>;
|
|
3
|
+
export interface Scripting {
|
|
4
|
+
reprocessJob: Caller;
|
|
5
|
+
removeJob: Caller;
|
|
6
|
+
runTask: Caller;
|
|
7
|
+
deleteTask: Caller;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Registers the vendored scripts on the client with `defineCommand` (so ioredis
|
|
11
|
+
* uses EVALSHA and only ships each script body once) and returns typed callers.
|
|
12
|
+
*/
|
|
13
|
+
export declare function attachScripts(redis: Redis): Scripting;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { scripts } from "./lua.js";
|
|
2
|
+
const METHOD = {
|
|
3
|
+
reprocessJob: "qiReprocessJob",
|
|
4
|
+
removeJob: "qiRemoveJob",
|
|
5
|
+
runTask: "qiRunTask",
|
|
6
|
+
deleteTask: "qiDeleteTask",
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Registers the vendored scripts on the client with `defineCommand` (so ioredis
|
|
10
|
+
* uses EVALSHA and only ships each script body once) and returns typed callers.
|
|
11
|
+
*/
|
|
12
|
+
export function attachScripts(redis) {
|
|
13
|
+
const dyn = redis;
|
|
14
|
+
for (const key of Object.keys(scripts)) {
|
|
15
|
+
const method = METHOD[key];
|
|
16
|
+
if (typeof dyn[method] !== "function") {
|
|
17
|
+
redis.defineCommand(method, {
|
|
18
|
+
numberOfKeys: scripts[key].numberOfKeys,
|
|
19
|
+
lua: scripts[key].source,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const call = (method) => async (keys, args) => {
|
|
24
|
+
const res = await dyn[method](...keys, ...args);
|
|
25
|
+
return Number(res);
|
|
26
|
+
};
|
|
27
|
+
return {
|
|
28
|
+
reprocessJob: call(METHOD.reprocessJob),
|
|
29
|
+
removeJob: call(METHOD.removeJob),
|
|
30
|
+
runTask: call(METHOD.runTask),
|
|
31
|
+
deleteTask: call(METHOD.deleteTask),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=scripting.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scripting.js","sourceRoot":"","sources":["../../src/backends/scripting.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAWnC,MAAM,MAAM,GAAyC;IACnD,YAAY,EAAE,gBAAgB;IAC9B,SAAS,EAAE,aAAa;IACxB,OAAO,EAAE,WAAW;IACpB,UAAU,EAAE,cAAc;CAC3B,CAAC;AAIF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAY;IACxC,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAgC,EAAE,CAAC;QACtE,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,UAAU,EAAE,CAAC;YACtC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE;gBAC1B,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY;gBACvC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GACR,CAAC,MAAc,EAAU,EAAE,CAC3B,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QACnB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC;QACjD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC,CAAC;IAEJ,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QACvC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QACjC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7B,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;KACpC,CAAC;AACJ,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { BackendName } from "./types.js";
|
|
2
|
+
export interface Config {
|
|
3
|
+
redisUrl: string;
|
|
4
|
+
asynqPrefix: string;
|
|
5
|
+
bullPrefix: string;
|
|
6
|
+
backends: BackendName[];
|
|
7
|
+
readOnly: boolean;
|
|
8
|
+
}
|
|
9
|
+
/** Reads configuration from the environment and CLI flags. */
|
|
10
|
+
export declare function loadConfig(argv?: string[]): Config;
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
const ALL_BACKENDS = ["asynq", "bullmq"];
|
|
2
|
+
function parseBackends(raw) {
|
|
3
|
+
if (!raw)
|
|
4
|
+
return [...ALL_BACKENDS];
|
|
5
|
+
const wanted = raw
|
|
6
|
+
.split(",")
|
|
7
|
+
.map((s) => s.trim().toLowerCase())
|
|
8
|
+
.filter(Boolean);
|
|
9
|
+
const picked = ALL_BACKENDS.filter((b) => wanted.includes(b));
|
|
10
|
+
if (picked.length === 0) {
|
|
11
|
+
throw new Error(`QUEUE_INSPECTOR_BACKENDS did not name any known backend (asynq, bullmq): "${raw}"`);
|
|
12
|
+
}
|
|
13
|
+
return picked;
|
|
14
|
+
}
|
|
15
|
+
function envFlag(value) {
|
|
16
|
+
if (!value)
|
|
17
|
+
return false;
|
|
18
|
+
const v = value.trim().toLowerCase();
|
|
19
|
+
return v === "1" || v === "true" || v === "yes";
|
|
20
|
+
}
|
|
21
|
+
/** Reads configuration from the environment and CLI flags. */
|
|
22
|
+
export function loadConfig(argv = process.argv.slice(2)) {
|
|
23
|
+
const readOnly = argv.includes("--read-only") || envFlag(process.env.QUEUE_INSPECTOR_READ_ONLY);
|
|
24
|
+
return {
|
|
25
|
+
redisUrl: process.env.REDIS_URL || "redis://localhost:6379",
|
|
26
|
+
asynqPrefix: process.env.ASYNQ_PREFIX || "asynq",
|
|
27
|
+
bullPrefix: process.env.BULL_PREFIX || "bull",
|
|
28
|
+
backends: parseBackends(process.env.QUEUE_INSPECTOR_BACKENDS),
|
|
29
|
+
readOnly,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAUA,MAAM,YAAY,GAAkB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAExD,SAAS,aAAa,CAAC,GAAuB;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,CAAC,GAAG,YAAY,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,GAAG;SACf,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACb,6EAA6E,GAAG,GAAG,CACpF,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,KAAyB;IACxC,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,UAAU,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAEhG,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,wBAAwB;QAC3D,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO;QAChD,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,MAAM;QAC7C,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;QAC7D,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
package/dist/format.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface DecodedPayload {
|
|
2
|
+
payload: string;
|
|
3
|
+
payloadEncoding: "utf8" | "base64";
|
|
4
|
+
payloadBytes: number;
|
|
5
|
+
payloadTruncated: boolean;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Renders a raw payload for display. Text is returned as UTF-8; binary or
|
|
9
|
+
* otherwise non-text bytes are returned base64-encoded. Either way the result
|
|
10
|
+
* is capped at `maxBytes` of source data so a large payload cannot flood a tool
|
|
11
|
+
* response.
|
|
12
|
+
*/
|
|
13
|
+
export declare function decodePayload(buf: Buffer, maxBytes?: number): DecodedPayload;
|
|
14
|
+
export declare function truncate(text: string, max?: number): string;
|
|
15
|
+
export declare function isoFromUnixSeconds(seconds: number | null | undefined): string | null;
|
|
16
|
+
export declare function isoFromMillis(ms: number | null | undefined): string | null;
|