webspresso 0.0.104 → 0.0.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/AGENTS.md +21 -2
- package/.agents/ARCHITECTURE.md +23 -0
- package/.agents/CACHE.md +147 -0
- package/.agents/CLI.md +220 -8
- package/.agents/EARLY_HINTS.md +120 -0
- package/.agents/FLAGS.md +118 -0
- package/.agents/HEALTH.md +118 -0
- package/.agents/LIMITS.md +109 -0
- package/.agents/LOGGING.md +93 -0
- package/.agents/METRICS.md +127 -0
- package/.agents/MIGRATION_JOBS.md +95 -0
- package/.agents/ORM.md +155 -1
- package/.agents/PLUGINS.md +38 -3
- package/.agents/QUEUE.md +42 -0
- package/.agents/RATE_LIMIT.md +147 -0
- package/.agents/RESILIENCE.md +235 -0
- package/.agents/SCALE_OUT.md +109 -0
- package/.agents/SECRETS.md +117 -0
- package/AGENTS.md +16 -0
- package/README.md +2 -2
- package/bin/commands/add-aws.js +396 -0
- package/bin/commands/add-ci.js +523 -0
- package/bin/commands/add-cloudflare.js +240 -0
- package/bin/commands/add-cloudrun.js +381 -0
- package/bin/commands/add-docker.js +6 -6
- package/bin/commands/add-migration-job.js +396 -0
- package/bin/commands/assets-sync.js +209 -0
- package/bin/commands/client-generate.js +52 -0
- package/bin/commands/console.js +53 -0
- package/bin/commands/db-migrate.js +47 -2
- package/bin/commands/db-studio.js +84 -0
- package/bin/commands/deploy-check.js +242 -13
- package/bin/commands/dev.js +1 -1
- package/bin/commands/export.js +87 -0
- package/bin/commands/new.js +5 -5
- package/bin/commands/openapi.js +1 -1
- package/bin/commands/secrets-pull.js +208 -0
- package/bin/commands/start.js +1 -1
- package/bin/utils/db.js +7 -1
- package/bin/utils/project.js +2 -2
- package/bin/webspresso.js +22 -0
- package/core/app.js +421 -0
- package/core/auth/authenticate.js +93 -0
- package/core/auth/hash.js +5 -4
- package/core/auth/index.js +34 -4
- package/core/auth/jwt.js +5 -54
- package/core/auth/magic-link.js +4 -5
- package/core/auth/manager.js +53 -10
- package/core/auth/middleware.js +16 -68
- package/core/auth/oauth.js +8 -17
- package/core/auth/redis-session.js +1008 -0
- package/core/auth/sessions.js +156 -0
- package/core/auth/totp.js +2 -1
- package/core/cache/byte-size.js +123 -0
- package/core/cache/cdn.js +296 -0
- package/core/cache/index.js +33 -0
- package/core/cache/lru.js +477 -0
- package/core/cache/tagged-lru.js +354 -0
- package/core/commands/registry.js +6 -20
- package/core/console/commands.js +148 -0
- package/core/console/context.js +260 -0
- package/core/console/eval.js +70 -0
- package/core/console/index.js +22 -0
- package/core/console/repl.js +102 -0
- package/core/early-hints/index.js +267 -0
- package/core/errors/domain.js +49 -0
- package/core/errors/http.js +31 -0
- package/core/errors/index.js +10 -0
- package/core/errors/normalize.js +88 -2
- package/core/events/discovery.js +6 -24
- package/core/events/dispatcher.js +9 -7
- package/core/events/signal.js +7 -0
- package/core/export/discovery.js +191 -0
- package/core/export/exporter.js +277 -0
- package/core/export/index.js +24 -0
- package/core/flags/evaluator.js +252 -0
- package/core/flags/hasher.js +37 -0
- package/core/flags/index.js +21 -0
- package/core/flags/manager.js +263 -0
- package/core/flags/store.js +126 -0
- package/core/health/index.js +41 -0
- package/core/health/middleware.js +213 -0
- package/core/health/normalize.js +140 -0
- package/core/health/probes.js +203 -0
- package/core/http/adapter.js +74 -0
- package/core/http/express-adapter.js +46 -0
- package/core/http/hono-adapter.js +531 -0
- package/core/http/index.js +41 -0
- package/core/http/pipeline.js +485 -0
- package/core/index.js +336 -0
- package/core/limits/index.js +34 -0
- package/core/limits/middleware.js +325 -0
- package/core/limits/normalize.js +222 -0
- package/core/logger/formatters.js +264 -0
- package/core/logger/index.js +260 -0
- package/core/logger/trace.js +137 -0
- package/core/metrics/collectors.js +125 -0
- package/core/metrics/index.js +45 -0
- package/core/metrics/middleware.js +230 -0
- package/core/metrics/registry.js +366 -0
- package/core/orm/batch.js +70 -0
- package/core/orm/cache/fingerprint.js +2 -2
- package/core/orm/cache/memory-provider.js +27 -110
- package/core/orm/d1.js +265 -0
- package/core/orm/index.js +176 -13
- package/core/orm/json-fields.js +22 -8
- package/core/orm/migrate-cloud.js +194 -0
- package/core/orm/model.js +10 -0
- package/core/orm/pool.js +97 -0
- package/core/orm/query-builder.js +72 -9
- package/core/orm/repository.js +318 -19
- package/core/orm/schema-helpers.js +8 -10
- package/core/orm/scopes.js +321 -1
- package/core/orm/transaction.js +10 -2
- package/core/orm/utils.js +4 -86
- package/core/queue/adapters/cloudflare.js +131 -0
- package/core/queue/adapters/database.js +4 -8
- package/core/queue/adapters/pubsub.js +400 -0
- package/core/queue/adapters/sqs.js +419 -0
- package/core/queue/adapters/wait-until.js +77 -0
- package/core/queue/index.js +8 -0
- package/core/queue/manager.js +66 -41
- package/core/rate-limit/index.js +22 -0
- package/core/rate-limit/limiter.js +219 -0
- package/core/rate-limit/stores/memory.js +129 -0
- package/core/rate-limit/stores/redis.js +151 -0
- package/core/rate-limit/utils.js +112 -0
- package/core/resilience/circuit-breaker.js +422 -0
- package/core/resilience/errors.js +63 -0
- package/core/resilience/index.js +53 -0
- package/core/resilience/registry.js +97 -0
- package/core/resilience/retry.js +203 -0
- package/core/schema-cache.js +9 -2
- package/core/secrets/index.js +113 -0
- package/core/secrets/manager.js +308 -0
- package/core/secrets/providers/aws.js +288 -0
- package/core/secrets/providers/env.js +44 -0
- package/core/secrets/providers/gcp.js +307 -0
- package/core/secrets/providers/memory.js +76 -0
- package/core/shutdown/http-adapter.js +4 -1
- package/core/shutdown/shutdown-manager.js +129 -19
- package/core/stream/index.js +19 -0
- package/core/stream/text-stream.js +326 -0
- package/core/studio/index.js +43 -0
- package/core/studio/inspector.js +258 -0
- package/core/studio/server.js +325 -0
- package/core/studio/ui.js +681 -0
- package/core/utils/array.js +12 -0
- package/core/utils/base64url.js +8 -0
- package/core/utils/crypto.js +118 -0
- package/core/utils/ensure-leading-slash.js +6 -0
- package/core/utils/escape-regexp.js +7 -0
- package/core/utils/fetch.js +6 -5
- package/core/utils/fs.js +144 -0
- package/core/utils/hmac.js +6 -0
- package/core/utils/json.js +98 -0
- package/core/utils/object.js +106 -0
- package/core/utils/random.js +9 -0
- package/core/utils/regex.js +77 -0
- package/core/utils/sha256.js +6 -0
- package/core/utils/string.js +87 -0
- package/core/utils/strip-trailing-slash.js +6 -0
- package/core/utils/time.js +96 -0
- package/core/utils/timeout.js +348 -0
- package/core/utils/url.js +106 -0
- package/core/utils/walk.js +7 -0
- package/index.d.ts +1549 -2
- package/index.js +272 -2
- package/package.json +3 -2
- package/plugins/admin-panel/core/api-extensions.js +2 -5
- package/plugins/admin-panel/index.js +13 -6
- package/plugins/admin-panel/modules/menu.js +1 -1
- package/plugins/api-key/crypto.js +3 -3
- package/plugins/api-key/index.js +3 -2
- package/plugins/api-key/scopes.js +5 -7
- package/plugins/audit-log/api-handlers.js +6 -15
- package/plugins/audit-log/index.js +3 -1
- package/plugins/audit-log/parse.js +3 -3
- package/plugins/basic-auth/index.js +3 -2
- package/plugins/cors.js +3 -2
- package/plugins/csrf/index.js +9 -12
- package/plugins/csv/index.js +8 -5
- package/plugins/dashboard/app.js +2 -2
- package/plugins/dashboard/index.js +5 -5
- package/plugins/data-exchange/record-selection.js +2 -5
- package/plugins/email/api-handlers.js +5 -18
- package/plugins/email/auth-bridge.js +3 -2
- package/plugins/email/index.js +11 -9
- package/plugins/health-check.js +2 -1
- package/plugins/index.js +5 -0
- package/plugins/lazy.js +5 -6
- package/plugins/mcp/discovery.js +18 -17
- package/plugins/polar/src/api.js +2 -1
- package/plugins/polar/src/webhooks.js +5 -6
- package/plugins/queue/index.js +18 -2
- package/plugins/rate-limit/index.js +3 -1
- package/plugins/redirect/index.js +16 -9
- package/plugins/rls/index.js +170 -0
- package/plugins/seo-checker/index.js +2 -2
- package/plugins/site-analytics/index.js +3 -1
- package/plugins/site-analytics/tracking.js +2 -2
- package/plugins/sitemap.js +3 -2
- package/plugins/swagger.js +12 -5
- package/plugins/upload/gcs-provider.js +148 -0
- package/plugins/upload/index.js +136 -77
- package/plugins/upload/orphan-pruner.js +4 -3
- package/plugins/upload/s3-provider.js +257 -0
- package/plugins/webhooks/crypto.js +6 -11
- package/plugins/webhooks/dispatcher.js +8 -8
- package/plugins/webhooks/index.js +33 -23
- package/plugins/webhooks/matcher.js +7 -11
- package/plugins/webhooks/sender.js +8 -11
- package/plugins/xlsx/index.js +7 -5
- package/src/api/api-loader.js +12 -95
- package/src/api/define-api.js +1 -0
- package/src/app-context.js +41 -5
- package/src/client-runtime/constants.js +10 -0
- package/src/client-runtime/mount.js +2 -3
- package/src/dev/banner.js +6 -2
- package/src/dev/client-gen.js +458 -0
- package/src/discovery/scan-routes.js +5 -2
- package/src/file-router.js +3 -3
- package/src/helpers.js +212 -24
- package/src/modules/module-discovery.js +1 -9
- package/src/pages/page-loader.js +13 -154
- package/src/plugin-manager.js +99 -18
- package/src/server.js +335 -133
- package/src/services/discovery.js +19 -40
- package/src/services/executor.js +95 -25
- package/src/services/memoize.js +3 -25
- package/src/services/registry.js +15 -1
- package/src/services/validator.js +38 -20
- package/src/template/precompile.js +8 -19
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AWS SQS Queue Adapter for Webspresso
|
|
3
|
+
* Zero-dependency: Uses standard Node.js https/crypto with standard AWS SigV4 request signing.
|
|
4
|
+
* Supports official @aws-sdk/client-sqs client, duck-typed client, or direct zero-dependency REST calls.
|
|
5
|
+
* Supports worker polling loop and AWS Lambda event.Records push batch processing.
|
|
6
|
+
*
|
|
7
|
+
* @module core/queue/adapters/sqs
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
'use strict';
|
|
11
|
+
|
|
12
|
+
const https = require('https');
|
|
13
|
+
const { Job } = require('../job');
|
|
14
|
+
const { signAwsV4Request } = require('../../secrets/providers/aws');
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Execute HTTPS request to SQS endpoint
|
|
18
|
+
* @param {string} urlStr
|
|
19
|
+
* @param {Object} options
|
|
20
|
+
* @returns {Promise<{ statusCode: number, data: string, json: any }>}
|
|
21
|
+
*/
|
|
22
|
+
function makeSqsRequest(urlStr, options = {}) {
|
|
23
|
+
return new Promise((resolve, reject) => {
|
|
24
|
+
const parsedUrl = new URL(urlStr);
|
|
25
|
+
|
|
26
|
+
const req = https.request(parsedUrl, {
|
|
27
|
+
method: options.method || 'POST',
|
|
28
|
+
headers: options.headers || {},
|
|
29
|
+
timeout: options.timeout || 15000,
|
|
30
|
+
}, (res) => {
|
|
31
|
+
let body = '';
|
|
32
|
+
res.setEncoding('utf8');
|
|
33
|
+
res.on('data', (chunk) => { body += chunk; });
|
|
34
|
+
res.on('end', () => {
|
|
35
|
+
let json = null;
|
|
36
|
+
try {
|
|
37
|
+
json = JSON.parse(body);
|
|
38
|
+
} catch (_) {}
|
|
39
|
+
resolve({
|
|
40
|
+
statusCode: res.statusCode || 200,
|
|
41
|
+
data: body,
|
|
42
|
+
json,
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
req.on('error', reject);
|
|
48
|
+
req.on('timeout', () => {
|
|
49
|
+
req.destroy();
|
|
50
|
+
reject(new Error(`[SqsQueueAdapter] Request to ${urlStr} timed out`));
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
if (options.body) {
|
|
54
|
+
req.write(typeof options.body === 'string' ? options.body : JSON.stringify(options.body));
|
|
55
|
+
}
|
|
56
|
+
req.end();
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
class SqsQueueAdapter {
|
|
61
|
+
/**
|
|
62
|
+
* @param {Object} options
|
|
63
|
+
* @param {string} options.queueUrl - Full URL to AWS SQS queue
|
|
64
|
+
* @param {string} [options.region] - AWS region (e.g. us-east-1)
|
|
65
|
+
* @param {string} [options.accessKeyId]
|
|
66
|
+
* @param {string} [options.secretAccessKey]
|
|
67
|
+
* @param {string} [options.sessionToken]
|
|
68
|
+
* @param {number} [options.waitTimeSeconds=5] - SQS Long polling wait time (0-20s)
|
|
69
|
+
* @param {number} [options.visibilityTimeout=30] - Visibility timeout in seconds
|
|
70
|
+
* @param {Object} [options.sqsClient] - Optional @aws-sdk/client-sqs client
|
|
71
|
+
*/
|
|
72
|
+
constructor(options = {}) {
|
|
73
|
+
if (!options.queueUrl && !options.sqsClient) {
|
|
74
|
+
throw new Error('[SqsQueueAdapter] Requires queueUrl or an initialized sqsClient');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
this.queueUrl = options.queueUrl || '';
|
|
78
|
+
this.region = options.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || this._extractRegion(this.queueUrl) || 'us-east-1';
|
|
79
|
+
this.accessKeyId = options.accessKeyId || process.env.AWS_ACCESS_KEY_ID || '';
|
|
80
|
+
this.secretAccessKey = options.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY || '';
|
|
81
|
+
this.sessionToken = options.sessionToken || process.env.AWS_SESSION_TOKEN || '';
|
|
82
|
+
this.waitTimeSeconds = Math.max(0, Math.min(20, options.waitTimeSeconds ?? 5));
|
|
83
|
+
this.visibilityTimeout = Math.max(1, options.visibilityTimeout ?? 30);
|
|
84
|
+
this.client = options.sqsClient || null;
|
|
85
|
+
this.httpTransport = options.httpTransport || makeSqsRequest;
|
|
86
|
+
|
|
87
|
+
/** @type {Map<string|number, string>} Map jobId -> receiptHandle */
|
|
88
|
+
this.receiptHandles = new Map();
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
_extractRegion(urlStr) {
|
|
92
|
+
if (!urlStr) return null;
|
|
93
|
+
const match = urlStr.match(/sqs\.([a-z0-9-]+)\.amazonaws\.com/i);
|
|
94
|
+
return match ? match[1] : null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Send SQS request using SDK client or native signed HTTP POST
|
|
99
|
+
* @private
|
|
100
|
+
*/
|
|
101
|
+
async _call(action, payload = {}) {
|
|
102
|
+
if (this.client) {
|
|
103
|
+
if (typeof this.client[action] === 'function') {
|
|
104
|
+
return this.client[action]({ QueueUrl: this.queueUrl, ...payload });
|
|
105
|
+
}
|
|
106
|
+
if (typeof this.client.send === 'function') {
|
|
107
|
+
return this.client.send({ name: action, input: { QueueUrl: this.queueUrl, ...payload } });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Zero-dependency native signed POST using AWS SQS JSON 1.0 protocol
|
|
112
|
+
const parsedUrl = new URL(this.queueUrl);
|
|
113
|
+
const body = JSON.stringify({
|
|
114
|
+
QueueUrl: this.queueUrl,
|
|
115
|
+
...payload,
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const targetHeader = `AmazonSQS.${action}`;
|
|
119
|
+
const baseHeaders = {
|
|
120
|
+
'content-type': 'application/x-amz-json-1.0',
|
|
121
|
+
'x-amz-target': targetHeader,
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
const signedHeaders = signAwsV4Request({
|
|
125
|
+
method: 'POST',
|
|
126
|
+
host: parsedUrl.host,
|
|
127
|
+
path: parsedUrl.pathname,
|
|
128
|
+
headers: baseHeaders,
|
|
129
|
+
body,
|
|
130
|
+
service: 'sqs',
|
|
131
|
+
region: this.region,
|
|
132
|
+
accessKeyId: this.accessKeyId,
|
|
133
|
+
secretAccessKey: this.secretAccessKey,
|
|
134
|
+
sessionToken: this.sessionToken,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
const response = await this.httpTransport(this.queueUrl, {
|
|
138
|
+
method: 'POST',
|
|
139
|
+
headers: signedHeaders,
|
|
140
|
+
body,
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
if (response.statusCode >= 300) {
|
|
144
|
+
const errMsg = response.json?.message || response.json?.Message || response.data || `HTTP ${response.statusCode}`;
|
|
145
|
+
throw new Error(`[SqsQueueAdapter] SQS.${action} failed (${response.statusCode}): ${errMsg}`);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
return response.json || {};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Enqueue job to SQS
|
|
153
|
+
* @param {Job} job
|
|
154
|
+
* @returns {Promise<Job>}
|
|
155
|
+
*/
|
|
156
|
+
async enqueue(job) {
|
|
157
|
+
const jobInstance = job instanceof Job ? job : new Job(job);
|
|
158
|
+
|
|
159
|
+
const messagePayload = {
|
|
160
|
+
id: jobInstance.id,
|
|
161
|
+
name: jobInstance.name,
|
|
162
|
+
data: jobInstance.data,
|
|
163
|
+
attempts: jobInstance.attempts,
|
|
164
|
+
maxAttempts: jobInstance.maxAttempts,
|
|
165
|
+
priority: jobInstance.priority,
|
|
166
|
+
runAt: jobInstance.runAt ? jobInstance.runAt.toISOString() : null,
|
|
167
|
+
createdAt: jobInstance.createdAt ? jobInstance.createdAt.toISOString() : new Date().toISOString(),
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
let delaySeconds = 0;
|
|
171
|
+
if (jobInstance.runAt) {
|
|
172
|
+
const delayMs = jobInstance.runAt.getTime() - Date.now();
|
|
173
|
+
if (delayMs > 0) {
|
|
174
|
+
delaySeconds = Math.min(900, Math.floor(delayMs / 1000));
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const res = await this._call('SendMessage', {
|
|
179
|
+
MessageBody: JSON.stringify(messagePayload),
|
|
180
|
+
DelaySeconds: delaySeconds,
|
|
181
|
+
MessageAttributes: {
|
|
182
|
+
JobName: {
|
|
183
|
+
DataType: 'String',
|
|
184
|
+
StringValue: jobInstance.name,
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
if (res.MessageId) {
|
|
190
|
+
jobInstance._sqsMessageId = res.MessageId;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return jobInstance;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Dequeue single job from SQS
|
|
198
|
+
* @param {string[]} [jobNames]
|
|
199
|
+
* @returns {Promise<Job|null>}
|
|
200
|
+
*/
|
|
201
|
+
async dequeue(jobNames = []) {
|
|
202
|
+
const res = await this._call('ReceiveMessage', {
|
|
203
|
+
MaxNumberOfMessages: 1,
|
|
204
|
+
WaitTimeSeconds: this.waitTimeSeconds,
|
|
205
|
+
VisibilityTimeout: this.visibilityTimeout,
|
|
206
|
+
AttributeNames: ['All'],
|
|
207
|
+
MessageAttributeNames: ['All'],
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
const messages = res.Messages || res.messages || [];
|
|
211
|
+
if (!messages || messages.length === 0) {
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const msg = messages[0];
|
|
216
|
+
const receiptHandle = msg.ReceiptHandle || msg.receiptHandle;
|
|
217
|
+
|
|
218
|
+
let payload = {};
|
|
219
|
+
try {
|
|
220
|
+
payload = JSON.parse(msg.Body || msg.body || '{}');
|
|
221
|
+
} catch (_) {
|
|
222
|
+
payload = { name: 'raw', data: msg.Body };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const job = new Job({
|
|
226
|
+
id: payload.id || msg.MessageId || msg.messageId,
|
|
227
|
+
name: payload.name || 'default',
|
|
228
|
+
data: payload.data || {},
|
|
229
|
+
status: 'running',
|
|
230
|
+
attempts: (payload.attempts || 0) + 1,
|
|
231
|
+
maxAttempts: payload.maxAttempts ?? 3,
|
|
232
|
+
priority: payload.priority ?? 10,
|
|
233
|
+
runAt: payload.runAt ? new Date(payload.runAt) : new Date(),
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
job._receiptHandle = receiptHandle;
|
|
237
|
+
this.receiptHandles.set(job.id, receiptHandle);
|
|
238
|
+
|
|
239
|
+
// If job does not match expected handlers, return visibility immediately
|
|
240
|
+
if (jobNames.length > 0 && !jobNames.includes(job.name)) {
|
|
241
|
+
await this.retry(job.id, 0);
|
|
242
|
+
return null;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return job;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Mark job as completed and delete from SQS
|
|
250
|
+
* @param {string|number} jobId
|
|
251
|
+
* @param {any} [result]
|
|
252
|
+
* @returns {Promise<Job|null>}
|
|
253
|
+
*/
|
|
254
|
+
async complete(jobId, result = null) {
|
|
255
|
+
const handle = this.receiptHandles.get(jobId);
|
|
256
|
+
if (handle) {
|
|
257
|
+
try {
|
|
258
|
+
await this._call('DeleteMessage', { ReceiptHandle: handle });
|
|
259
|
+
} finally {
|
|
260
|
+
this.receiptHandles.delete(jobId);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
return new Job({
|
|
265
|
+
id: jobId,
|
|
266
|
+
status: 'completed',
|
|
267
|
+
result,
|
|
268
|
+
updatedAt: new Date(),
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Mark job as permanently failed
|
|
274
|
+
* @param {string|number} jobId
|
|
275
|
+
* @param {any} [error]
|
|
276
|
+
* @returns {Promise<Job|null>}
|
|
277
|
+
*/
|
|
278
|
+
async fail(jobId, error = null) {
|
|
279
|
+
const handle = this.receiptHandles.get(jobId);
|
|
280
|
+
if (handle) {
|
|
281
|
+
try {
|
|
282
|
+
await this._call('DeleteMessage', { ReceiptHandle: handle });
|
|
283
|
+
} finally {
|
|
284
|
+
this.receiptHandles.delete(jobId);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
return new Job({
|
|
289
|
+
id: jobId,
|
|
290
|
+
status: 'failed',
|
|
291
|
+
error,
|
|
292
|
+
updatedAt: new Date(),
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Change visibility timeout to reschedule job for retry
|
|
298
|
+
* @param {string|number} jobId
|
|
299
|
+
* @param {number} delayMs
|
|
300
|
+
* @param {any} [error]
|
|
301
|
+
* @returns {Promise<Job|null>}
|
|
302
|
+
*/
|
|
303
|
+
async retry(jobId, delayMs, error = null) {
|
|
304
|
+
const handle = this.receiptHandles.get(jobId);
|
|
305
|
+
if (handle) {
|
|
306
|
+
const visibilitySeconds = Math.min(43200, Math.max(0, Math.ceil(delayMs / 1000)));
|
|
307
|
+
try {
|
|
308
|
+
await this._call('ChangeMessageVisibility', {
|
|
309
|
+
ReceiptHandle: handle,
|
|
310
|
+
VisibilityTimeout: visibilitySeconds,
|
|
311
|
+
});
|
|
312
|
+
} finally {
|
|
313
|
+
this.receiptHandles.delete(jobId);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
return new Job({
|
|
318
|
+
id: jobId,
|
|
319
|
+
status: 'pending',
|
|
320
|
+
error,
|
|
321
|
+
updatedAt: new Date(),
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Get queue statistics from SQS
|
|
327
|
+
* @returns {Promise<{ pending: number, running: number, delayed: number, total: number }>}
|
|
328
|
+
*/
|
|
329
|
+
async getStats() {
|
|
330
|
+
try {
|
|
331
|
+
const res = await this._call('GetQueueAttributes', {
|
|
332
|
+
AttributeNames: [
|
|
333
|
+
'ApproximateNumberOfMessages',
|
|
334
|
+
'ApproximateNumberOfMessagesNotVisible',
|
|
335
|
+
'ApproximateNumberOfMessagesDelayed',
|
|
336
|
+
],
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
const attrs = res.Attributes || res.attributes || {};
|
|
340
|
+
const pending = parseInt(attrs.ApproximateNumberOfMessages || '0', 10);
|
|
341
|
+
const running = parseInt(attrs.ApproximateNumberOfMessagesNotVisible || '0', 10);
|
|
342
|
+
const delayed = parseInt(attrs.ApproximateNumberOfMessagesDelayed || '0', 10);
|
|
343
|
+
|
|
344
|
+
return {
|
|
345
|
+
pending,
|
|
346
|
+
running,
|
|
347
|
+
delayed,
|
|
348
|
+
total: pending + running + delayed,
|
|
349
|
+
};
|
|
350
|
+
} catch (_) {
|
|
351
|
+
return { pending: 0, running: 0, delayed: 0, total: 0 };
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Process an AWS Lambda SQS event batch (`event.Records`)
|
|
357
|
+
* @param {Array<Object>} records - SQS event records
|
|
358
|
+
* @param {import('../manager').QueueManager} queueManager
|
|
359
|
+
* @returns {Promise<{ processed: number, failed: number }>}
|
|
360
|
+
*/
|
|
361
|
+
async processBatch(records, queueManager) {
|
|
362
|
+
if (!Array.isArray(records)) {
|
|
363
|
+
return { processed: 0, failed: 0 };
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
let processed = 0;
|
|
367
|
+
let failed = 0;
|
|
368
|
+
|
|
369
|
+
for (const record of records) {
|
|
370
|
+
let payload = {};
|
|
371
|
+
try {
|
|
372
|
+
payload = JSON.parse(record.body || '{}');
|
|
373
|
+
} catch (_) {
|
|
374
|
+
payload = { name: 'raw', data: record.body };
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const jobName = payload.name;
|
|
378
|
+
const handlerDef = queueManager?.handlers?.get(jobName);
|
|
379
|
+
|
|
380
|
+
if (!handlerDef || typeof handlerDef.handler !== 'function') {
|
|
381
|
+
continue;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const job = new Job({
|
|
385
|
+
id: payload.id || record.messageId,
|
|
386
|
+
name: jobName,
|
|
387
|
+
data: payload.data || {},
|
|
388
|
+
priority: payload.priority || 10,
|
|
389
|
+
runAt: payload.runAt ? new Date(payload.runAt) : new Date(),
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
try {
|
|
393
|
+
job.status = 'running';
|
|
394
|
+
job.startedAt = new Date();
|
|
395
|
+
queueManager.emit('job:start', job);
|
|
396
|
+
|
|
397
|
+
const result = await handlerDef.handler(job, { queue: queueManager, record });
|
|
398
|
+
job.status = 'completed';
|
|
399
|
+
job.result = result;
|
|
400
|
+
job.completedAt = new Date();
|
|
401
|
+
queueManager.emit('job:completed', job, result);
|
|
402
|
+
processed++;
|
|
403
|
+
} catch (err) {
|
|
404
|
+
job.status = 'failed';
|
|
405
|
+
job.error = err;
|
|
406
|
+
queueManager.emit('job:failed', job, err);
|
|
407
|
+
failed++;
|
|
408
|
+
throw err; // In Lambda SQS batches, throwing lets SQS retry or trigger DLQ
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
return { processed, failed };
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
module.exports = {
|
|
417
|
+
SqsQueueAdapter,
|
|
418
|
+
makeSqsRequest,
|
|
419
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WaitUntil (Edge/Serverless) Queue Adapter
|
|
3
|
+
* Designed for environments exposing an execution context (e.g. Cloudflare Workers ctx.waitUntil,
|
|
4
|
+
* Vercel Edge waitUntil) where jobs run non-blocking in the background without persistent polling daemons.
|
|
5
|
+
* @module core/queue/adapters/wait-until
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
'use strict';
|
|
9
|
+
|
|
10
|
+
const { MemoryQueueAdapter } = require('./memory');
|
|
11
|
+
const { Job } = require('../job');
|
|
12
|
+
|
|
13
|
+
class WaitUntilQueueAdapter extends MemoryQueueAdapter {
|
|
14
|
+
/**
|
|
15
|
+
* @param {Object} [options]
|
|
16
|
+
* @param {Function} [options.waitUntil] - Optional global waitUntil handler
|
|
17
|
+
* @param {number} [options.maxRetainedJobs=1000] - In-memory retention limit to prevent memory leaks in isolates
|
|
18
|
+
*/
|
|
19
|
+
constructor(options = {}) {
|
|
20
|
+
super();
|
|
21
|
+
this.waitUntilHandler = typeof options.waitUntil === 'function' ? options.waitUntil : null;
|
|
22
|
+
this.maxRetainedJobs = options.maxRetainedJobs || 1000;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Set or update the active waitUntil handler (e.g. from per-request execution context)
|
|
27
|
+
* @param {Function} fn
|
|
28
|
+
*/
|
|
29
|
+
setWaitUntil(fn) {
|
|
30
|
+
if (typeof fn === 'function') {
|
|
31
|
+
this.waitUntilHandler = fn;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Enqueue a job and schedule it for immediate execution via waitUntil if available
|
|
37
|
+
* @param {Job} job
|
|
38
|
+
* @returns {Promise<Job>}
|
|
39
|
+
*/
|
|
40
|
+
async enqueue(job) {
|
|
41
|
+
const jobInstance = await super.enqueue(job);
|
|
42
|
+
|
|
43
|
+
// Evict oldest completed/failed jobs if limit reached to protect isolate memory
|
|
44
|
+
if (this.jobs.size > this.maxRetainedJobs) {
|
|
45
|
+
let evicted = 0;
|
|
46
|
+
for (const [id, j] of this.jobs.entries()) {
|
|
47
|
+
if (j.status === 'completed' || j.status === 'failed') {
|
|
48
|
+
this.jobs.delete(id);
|
|
49
|
+
evicted++;
|
|
50
|
+
if (evicted >= 50) break;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return jobInstance;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Schedule a promise to complete in the background via waitUntil if available,
|
|
60
|
+
* or fall back to unhandled Promise execution.
|
|
61
|
+
* @param {Promise<any>} promise
|
|
62
|
+
*/
|
|
63
|
+
scheduleBackground(promise) {
|
|
64
|
+
if (this.waitUntilHandler) {
|
|
65
|
+
try {
|
|
66
|
+
this.waitUntilHandler(promise);
|
|
67
|
+
return;
|
|
68
|
+
} catch (_) {
|
|
69
|
+
// Fall back to unhandled background promise
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// Standard Node/V8 unhandled background promise fallback
|
|
73
|
+
promise.catch(() => {});
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { WaitUntilQueueAdapter };
|
package/core/queue/index.js
CHANGED
|
@@ -10,6 +10,10 @@ const { QueueManager, createQueueManager } = require('./manager');
|
|
|
10
10
|
const { MemoryQueueAdapter } = require('./adapters/memory');
|
|
11
11
|
const { DatabaseQueueAdapter } = require('./adapters/database');
|
|
12
12
|
const { RedisQueueAdapter } = require('./adapters/redis');
|
|
13
|
+
const { WaitUntilQueueAdapter } = require('./adapters/wait-until');
|
|
14
|
+
const { CloudflareQueueAdapter } = require('./adapters/cloudflare');
|
|
15
|
+
const { SqsQueueAdapter } = require('./adapters/sqs');
|
|
16
|
+
const { PubSubQueueAdapter } = require('./adapters/pubsub');
|
|
13
17
|
const { CronExpression, isValidCron, parseCron, resolveCronAlias } = require('./cron');
|
|
14
18
|
const { TaskScheduler } = require('./scheduler');
|
|
15
19
|
|
|
@@ -20,6 +24,10 @@ module.exports = {
|
|
|
20
24
|
MemoryQueueAdapter,
|
|
21
25
|
DatabaseQueueAdapter,
|
|
22
26
|
RedisQueueAdapter,
|
|
27
|
+
WaitUntilQueueAdapter,
|
|
28
|
+
CloudflareQueueAdapter,
|
|
29
|
+
SqsQueueAdapter,
|
|
30
|
+
PubSubQueueAdapter,
|
|
23
31
|
CronExpression,
|
|
24
32
|
isValidCron,
|
|
25
33
|
parseCron,
|
package/core/queue/manager.js
CHANGED
|
@@ -12,6 +12,8 @@ const { Job } = require('./job');
|
|
|
12
12
|
const { MemoryQueueAdapter } = require('./adapters/memory');
|
|
13
13
|
const { CronExpression, CRON_ALIASES } = require('./cron');
|
|
14
14
|
const { TaskScheduler } = require('./scheduler');
|
|
15
|
+
const { walkDirectoryDetailed } = require('../utils/fs');
|
|
16
|
+
const { parseDurationMs } = require('../utils/time');
|
|
15
17
|
|
|
16
18
|
class QueueManager extends EventEmitter {
|
|
17
19
|
/**
|
|
@@ -31,6 +33,10 @@ class QueueManager extends EventEmitter {
|
|
|
31
33
|
this.maxPollInterval = Math.max(this.pollInterval, options.maxPollInterval || (this.pollInterval * 5));
|
|
32
34
|
this._currentPollInterval = this.pollInterval;
|
|
33
35
|
this.jobsDir = options.jobsDir || null;
|
|
36
|
+
this.timeouts = options.timeouts || null;
|
|
37
|
+
this.defaultTimeout = options.timeout !== undefined
|
|
38
|
+
? parseDurationMs(options.timeout, 30000)
|
|
39
|
+
: (options.timeouts?.queue ? parseDurationMs(options.timeouts.queue, 30000) : 30000);
|
|
34
40
|
|
|
35
41
|
/** @type {Map<string, { handler: Function, options: Object }>} */
|
|
36
42
|
this.handlers = new Map();
|
|
@@ -117,7 +123,7 @@ class QueueManager extends EventEmitter {
|
|
|
117
123
|
handler,
|
|
118
124
|
options: {
|
|
119
125
|
attempts: 3,
|
|
120
|
-
timeout:
|
|
126
|
+
timeout: options.timeout !== undefined ? parseDurationMs(options.timeout, this.defaultTimeout) : this.defaultTimeout,
|
|
121
127
|
priority: 10,
|
|
122
128
|
...options,
|
|
123
129
|
},
|
|
@@ -126,6 +132,17 @@ class QueueManager extends EventEmitter {
|
|
|
126
132
|
return this;
|
|
127
133
|
}
|
|
128
134
|
|
|
135
|
+
/**
|
|
136
|
+
* Alias for define
|
|
137
|
+
* @param {string} name
|
|
138
|
+
* @param {Function} handler
|
|
139
|
+
* @param {Object} [options]
|
|
140
|
+
* @returns {JobQueueManager}
|
|
141
|
+
*/
|
|
142
|
+
register(name, handler, options = {}) {
|
|
143
|
+
return this.define(name, handler, options);
|
|
144
|
+
}
|
|
145
|
+
|
|
129
146
|
/**
|
|
130
147
|
* Check if a job handler is defined
|
|
131
148
|
* @param {string} name
|
|
@@ -152,6 +169,8 @@ class QueueManager extends EventEmitter {
|
|
|
152
169
|
runAt = new Date(Date.now() + options.delay);
|
|
153
170
|
} else if (options.delay instanceof Date) {
|
|
154
171
|
runAt = options.delay;
|
|
172
|
+
} else if (typeof options.delay === 'string') {
|
|
173
|
+
runAt = new Date(Date.now() + parseDurationMs(options.delay, 0));
|
|
155
174
|
}
|
|
156
175
|
}
|
|
157
176
|
|
|
@@ -161,7 +180,7 @@ class QueueManager extends EventEmitter {
|
|
|
161
180
|
runAt,
|
|
162
181
|
priority: options.priority !== undefined ? options.priority : defaults.priority,
|
|
163
182
|
maxAttempts: options.attempts !== undefined ? options.attempts : defaults.attempts,
|
|
164
|
-
timeout: options.timeout !== undefined ? options.timeout : defaults.timeout,
|
|
183
|
+
timeout: options.timeout !== undefined ? parseDurationMs(options.timeout, defaults.timeout) : defaults.timeout,
|
|
165
184
|
backoff: options.backoff !== undefined ? options.backoff : defaults.backoff,
|
|
166
185
|
onProgress: (j, percent, msg) => {
|
|
167
186
|
this.emit('job:progress', j, percent, msg);
|
|
@@ -450,45 +469,44 @@ class QueueManager extends EventEmitter {
|
|
|
450
469
|
loadJobsFromDirectory(dirPath) {
|
|
451
470
|
if (!fs.existsSync(dirPath)) return;
|
|
452
471
|
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
console.error(`[QueueManager] Failed to load job from ${fullPath}:`, err.message);
|
|
472
|
+
const files = walkDirectoryDetailed(dirPath, {
|
|
473
|
+
extensions: ['.js', '.cjs'],
|
|
474
|
+
ignorePrefixes: ['.', '_'],
|
|
475
|
+
ignoreTests: true,
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
for (const item of files) {
|
|
479
|
+
const relNormalized = item.relativePath.split(/[\/\\]/).join('.');
|
|
480
|
+
const isCronFile = item.name.endsWith('.cron.js') || item.name.endsWith('.cron.cjs');
|
|
481
|
+
const baseNameWithoutExt = isCronFile
|
|
482
|
+
? item.name.replace(/\.cron\.(c)?js$/, '')
|
|
483
|
+
: item.name.replace(/\.(c)?js$/, '');
|
|
484
|
+
const jobName = isCronFile
|
|
485
|
+
? relNormalized.replace(/\.cron\.(c)?js$/, '')
|
|
486
|
+
: relNormalized.replace(/\.(c)?js$/, '');
|
|
487
|
+
|
|
488
|
+
try {
|
|
489
|
+
const mod = require(item.absolutePath);
|
|
490
|
+
const handler = typeof mod === 'function' ? mod : (mod.handler || mod.default);
|
|
491
|
+
const options = mod.options || {};
|
|
492
|
+
|
|
493
|
+
if (typeof handler === 'function') {
|
|
494
|
+
this.define(jobName, handler, options);
|
|
495
|
+
|
|
496
|
+
const cronExpr = mod.cron || mod.schedule || (isCronFile ? (mod.expression || (CRON_ALIASES[`@${baseNameWithoutExt}`] ? `@${baseNameWithoutExt}` : null)) : null);
|
|
497
|
+
if (cronExpr) {
|
|
498
|
+
this.schedule(
|
|
499
|
+
cronExpr,
|
|
500
|
+
jobName,
|
|
501
|
+
mod.data || {},
|
|
502
|
+
{ ...options, ...(mod.scheduleOptions || {}) }
|
|
503
|
+
);
|
|
486
504
|
}
|
|
487
505
|
}
|
|
506
|
+
} catch (err) {
|
|
507
|
+
console.error(`[QueueManager] Failed to load job from ${item.absolutePath}:`, err.message);
|
|
488
508
|
}
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
readRecursive(dirPath);
|
|
509
|
+
}
|
|
492
510
|
}
|
|
493
511
|
|
|
494
512
|
/**
|
|
@@ -592,13 +610,16 @@ class QueueManager extends EventEmitter {
|
|
|
592
610
|
this.emit('job:started', job);
|
|
593
611
|
|
|
594
612
|
const { handler } = registered;
|
|
595
|
-
const timeoutMs = job.timeout || 30000;
|
|
613
|
+
const timeoutMs = job.timeout || this.defaultTimeout || 30000;
|
|
596
614
|
|
|
597
615
|
let timeoutTimer = null;
|
|
598
616
|
const timeoutPromise = new Promise((_, reject) => {
|
|
599
617
|
timeoutTimer = setTimeout(() => {
|
|
600
618
|
reject(new Error(`Job '${job.name}' (${job.id}) exceeded timeout of ${timeoutMs}ms`));
|
|
601
619
|
}, timeoutMs);
|
|
620
|
+
if (timeoutTimer && typeof timeoutTimer.unref === 'function') {
|
|
621
|
+
timeoutTimer.unref();
|
|
622
|
+
}
|
|
602
623
|
});
|
|
603
624
|
|
|
604
625
|
try {
|
|
@@ -606,12 +627,16 @@ class QueueManager extends EventEmitter {
|
|
|
606
627
|
handler(job, { queue: this }),
|
|
607
628
|
timeoutPromise,
|
|
608
629
|
]);
|
|
609
|
-
|
|
630
|
+
if (timeoutTimer) {
|
|
631
|
+
clearTimeout(timeoutTimer);
|
|
632
|
+
}
|
|
610
633
|
|
|
611
634
|
const completedJob = await this.adapter.complete(job.id, result);
|
|
612
635
|
this.emit('job:completed', completedJob || job, result);
|
|
613
636
|
} catch (err) {
|
|
614
|
-
|
|
637
|
+
if (timeoutTimer) {
|
|
638
|
+
clearTimeout(timeoutTimer);
|
|
639
|
+
}
|
|
615
640
|
|
|
616
641
|
if (job.attempts < job.maxAttempts) {
|
|
617
642
|
const delayMs = job.getNextRetryDelay();
|