ravenflow 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,10 +1,10 @@
1
1
  'use strict';
2
2
 
3
3
  var zod = require('zod');
4
+ var ioredis = require('ioredis');
4
5
  var pino = require('pino');
5
6
  var mongodb = require('mongodb');
6
7
  var bullmq = require('bullmq');
7
- var ioredis = require('ioredis');
8
8
  var util = require('util');
9
9
 
10
10
  function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
@@ -266,145 +266,6 @@ function throwIfAny(errors) {
266
266
  throw new Error(`[ravenflow] Config validation failed:
267
267
  - ${errors.join("\n - ")}`);
268
268
  }
269
- function createLogger(options = {}) {
270
- const isDev = process.env.NODE_ENV !== "production";
271
- const usePretty = options.pretty ?? isDev;
272
- return pino__default.default({
273
- level: options.level ?? (isDev ? "debug" : "info"),
274
- transport: usePretty ? {
275
- target: "pino-pretty",
276
- options: {
277
- colorize: true,
278
- translateTime: "HH:MM:ss",
279
- ignore: "pid,hostname,job",
280
- messageFormat: "{if job}({job}) {end}{msg}"
281
- }
282
- } : void 0
283
- });
284
- }
285
- var MongoConnection = class {
286
- constructor(uri) {
287
- this.uri = uri;
288
- this.client = null;
289
- this.database = null;
290
- this.connecting = null;
291
- }
292
- async connect() {
293
- if (this.database) return this.database;
294
- this.connecting ?? (this.connecting = this.openTheDatabase().catch(
295
- (error) => this.forgetTheAttempt(error)
296
- ));
297
- return this.connecting;
298
- }
299
- async close() {
300
- const closing = this.client;
301
- this.client = null;
302
- this.database = null;
303
- this.connecting = null;
304
- await closing?.close();
305
- }
306
- async openTheDatabase() {
307
- const opened = new mongodb.MongoClient(this.uri);
308
- await opened.connect();
309
- this.client = opened;
310
- this.database = opened.db();
311
- return this.database;
312
- }
313
- forgetTheAttempt(error) {
314
- this.connecting = null;
315
- throw error;
316
- }
317
- };
318
-
319
- // src/queues/declared-queues.ts
320
- function declaredQueueNames(config) {
321
- const queueOfEachJob = Object.values(config.jobs).map(
322
- (job) => queueOf(job, config)
323
- );
324
- return [.../* @__PURE__ */ new Set([config.defaultQueue, ...queueOfEachJob])];
325
- }
326
- function jobsDeclaredOn(queue, config) {
327
- return Object.fromEntries(
328
- Object.entries(config.jobs).filter(
329
- ([, job]) => queueOf(job, config) === queue
330
- )
331
- );
332
- }
333
- function queueOf(job, config) {
334
- return job.queue ?? config.defaultQueue;
335
- }
336
-
337
- // src/queues/open-queues.ts
338
- function openQueues(config, connection) {
339
- const settings = { connection, retention: config.queuesRetention };
340
- return Object.fromEntries(
341
- declaredQueueNames(config).map((name) => [
342
- name,
343
- openQueue(name, settings)
344
- ])
345
- );
346
- }
347
- function openQueue(name, settings) {
348
- return new bullmq.Queue(name, {
349
- connection: settings.connection,
350
- defaultJobOptions: defaultJobOptionsFor(settings.retention)
351
- });
352
- }
353
- function defaultJobOptionsFor(retention = {}) {
354
- return {
355
- removeOnComplete: {
356
- count: retention.completedCount ?? DEFAULT_COMPLETED_COUNT_RETENTION
357
- },
358
- removeOnFail: {
359
- count: retention.failedCount ?? DEFAULT_FAILED_COUNT_RETENTION
360
- }
361
- };
362
- }
363
-
364
- // src/queues/close-queues.ts
365
- async function closeQueues(queues) {
366
- await Promise.all(Object.values(queues).map((queue) => queue.close()));
367
- }
368
-
369
- // src/queues/build-job-options.ts
370
- function buildJobOptions(job) {
371
- const options = {};
372
- if (job.attempts !== void 0) options.attempts = job.attempts;
373
- if (job.backoff !== void 0) options.backoff = backoffOptions(job.backoff);
374
- return options;
375
- }
376
- function backoffOptions(backoff) {
377
- return {
378
- type: backoff.type,
379
- delay: backoff.delayMs,
380
- jitter: backoff.jitter
381
- };
382
- }
383
-
384
- // src/queues/link-id.ts
385
- var LINK_SEPARATOR = ".";
386
- var A_NUMBERED_LINK = /^(.+)\.([1-9]\d*)$/;
387
- function nextLinkIdOf(linkId) {
388
- const { runId, link } = linkOfItsRun(linkId);
389
- return linkIdOf({ runId, link: link + 1 });
390
- }
391
- function linkIdOf({ runId, link }) {
392
- if (link === 0) return runId;
393
- return `${asACustomJobId(runId)}${LINK_SEPARATOR}${link}`;
394
- }
395
- function linkOfItsRun(linkId) {
396
- const numbered = A_NUMBERED_LINK.exec(linkId);
397
- if (numbered === null) return { runId: linkId, link: 0 };
398
- return { runId: numbered[1], link: Number(numbered[2]) };
399
- }
400
- function asACustomJobId(runId) {
401
- if (holdsColonsBullmqRefuses(runId)) return encodeURIComponent(runId);
402
- return runId;
403
- }
404
- function holdsColonsBullmqRefuses(id) {
405
- const parts = id.split(":").length;
406
- return parts !== 1 && parts !== 3;
407
- }
408
269
 
409
270
  // src/control-plane/control-key.ts
410
271
  var JOB_PREFIX = "job:";
@@ -890,83 +751,222 @@ function initControlPlane(runtime) {
890
751
  });
891
752
  return new ControlPlane({ store, channel, logger: runtime.logger });
892
753
  }
893
- var BACKLOG_COLLECTION_PREFIX = "rav:spool-backlog:";
894
- var NEVER_CLAIMED = { claimedAt: { $exists: false } };
895
- var CLAIMED = { claimedAt: { $type: "date" } };
896
- var CLAIM_INDEX = { claimedAt: 1, _id: 1 };
897
- var OLDEST_FIRST = { _id: 1 };
898
- var MS_PER_SECOND = 1e3;
899
- var Backlog = class {
900
- constructor(spoolName, connection) {
901
- this.spoolName = spoolName;
902
- this.connection = connection;
903
- }
904
- async push(item) {
905
- const backlog = await this.collection();
906
- const { insertedId } = await backlog.insertOne({ item });
907
- return insertedId.toHexString();
908
- }
909
- async pushMany(items) {
910
- if (items.length === 0) return [];
911
- const backlog = await this.collection();
912
- const { insertedIds } = await backlog.insertMany(
913
- items.map((item) => ({ item }))
914
- );
915
- return items.map((_, position) => insertedIds[position].toHexString());
916
- }
917
- async purge() {
918
- const backlog = await this.collection();
919
- const { deletedCount } = await backlog.deleteMany(NEVER_CLAIMED);
920
- return {
921
- dropped: deletedCount,
922
- inFlight: await backlog.countDocuments(CLAIMED)
923
- };
924
- }
925
- async indexForTheClaim() {
926
- const backlog = await this.collection();
927
- await backlog.createIndex(CLAIM_INDEX);
928
- }
929
- async countInFlight() {
930
- const backlog = await this.collection();
931
- return backlog.countDocuments(CLAIMED);
754
+ function createLogger(options = {}) {
755
+ const isDev = process.env.NODE_ENV !== "production";
756
+ const usePretty = options.pretty ?? isDev;
757
+ return pino__default.default({
758
+ level: options.level ?? (isDev ? "debug" : "info"),
759
+ transport: usePretty ? {
760
+ target: "pino-pretty",
761
+ options: {
762
+ colorize: true,
763
+ translateTime: "HH:MM:ss",
764
+ ignore: "pid,hostname,job",
765
+ messageFormat: "{if job}({job}) {end}{msg}"
766
+ }
767
+ } : void 0
768
+ });
769
+ }
770
+ var MongoConnection = class {
771
+ constructor(uri) {
772
+ this.uri = uri;
773
+ this.client = null;
774
+ this.database = null;
775
+ this.connecting = null;
932
776
  }
933
- async expireUnclaimedPushedBefore(pushedBefore) {
934
- const backlog = await this.collection();
935
- const { deletedCount } = await backlog.deleteMany({
936
- ...NEVER_CLAIMED,
937
- _id: { $lt: mongodb.ObjectId.createFromTime(secondsOf(pushedBefore)) }
938
- });
939
- return deletedCount;
777
+ async connect() {
778
+ if (this.database) return this.database;
779
+ this.connecting ?? (this.connecting = this.openTheDatabase().catch(
780
+ (error) => this.forgetTheAttempt(error)
781
+ ));
782
+ return this.connecting;
940
783
  }
941
- async reclaimStaleClaims(claimedBefore, now) {
942
- const staleClaim = {
943
- claimedAt: { $lte: claimedBefore }
944
- };
945
- const backlog = await this.collection();
946
- const stale = await backlog.find(staleClaim).toArray();
947
- return this.claimAll(stale, staleClaim, now);
784
+ async close() {
785
+ const closing = this.client;
786
+ this.client = null;
787
+ this.database = null;
788
+ this.connecting = null;
789
+ await closing?.close();
948
790
  }
949
- async claimOldestNeverClaimed(count, now) {
950
- if (count <= 0) return [];
951
- const backlog = await this.collection();
952
- const oldest = await backlog.find(NEVER_CLAIMED, { sort: OLDEST_FIRST, limit: count }).toArray();
953
- return this.claimAll(oldest, NEVER_CLAIMED, now);
791
+ async openTheDatabase() {
792
+ const opened = new mongodb.MongoClient(this.uri);
793
+ await opened.connect();
794
+ this.client = opened;
795
+ this.database = opened.db();
796
+ return this.database;
954
797
  }
955
- async removeAll(ids) {
956
- if (ids.length === 0) return;
957
- const backlog = await this.collection();
958
- await backlog.deleteMany({ _id: { $in: ids } });
798
+ forgetTheAttempt(error) {
799
+ this.connecting = null;
800
+ throw error;
959
801
  }
960
- async claimAll(entries, stillClaimable, claimedAt) {
961
- if (entries.length === 0) return [];
962
- const ids = entries.map(({ _id }) => _id);
963
- const backlog = await this.collection();
964
- const { modifiedCount } = await backlog.updateMany(
965
- { _id: { $in: ids }, ...stillClaimable },
966
- { $set: { claimedAt } }
967
- );
968
- if (modifiedCount < entries.length) {
969
- return this.whatThisClaimTook(ids, claimedAt);
802
+ };
803
+
804
+ // src/queues/declared-queues.ts
805
+ function declaredQueueNames(config) {
806
+ const queueOfEachJob = Object.values(config.jobs).map(
807
+ (job) => queueOf(job, config)
808
+ );
809
+ return [.../* @__PURE__ */ new Set([config.defaultQueue, ...queueOfEachJob])];
810
+ }
811
+ function jobsDeclaredOn(queue, config) {
812
+ return Object.fromEntries(
813
+ Object.entries(config.jobs).filter(
814
+ ([, job]) => queueOf(job, config) === queue
815
+ )
816
+ );
817
+ }
818
+ function queueOf(job, config) {
819
+ return job.queue ?? config.defaultQueue;
820
+ }
821
+
822
+ // src/queues/open-queues.ts
823
+ function openQueues(config, connection) {
824
+ const settings = { connection, retention: config.queuesRetention };
825
+ return Object.fromEntries(
826
+ declaredQueueNames(config).map((name) => [
827
+ name,
828
+ openQueue(name, settings)
829
+ ])
830
+ );
831
+ }
832
+ function openQueue(name, settings) {
833
+ return new bullmq.Queue(name, {
834
+ connection: settings.connection,
835
+ defaultJobOptions: defaultJobOptionsFor(settings.retention)
836
+ });
837
+ }
838
+ function defaultJobOptionsFor(retention = {}) {
839
+ return {
840
+ removeOnComplete: {
841
+ count: retention.completedCount ?? DEFAULT_COMPLETED_COUNT_RETENTION
842
+ },
843
+ removeOnFail: {
844
+ count: retention.failedCount ?? DEFAULT_FAILED_COUNT_RETENTION
845
+ }
846
+ };
847
+ }
848
+
849
+ // src/queues/close-queues.ts
850
+ async function closeQueues(queues) {
851
+ await Promise.all(Object.values(queues).map((queue) => queue.close()));
852
+ }
853
+
854
+ // src/queues/build-job-options.ts
855
+ function buildJobOptions(job) {
856
+ const options = {};
857
+ if (job.attempts !== void 0) options.attempts = job.attempts;
858
+ if (job.backoff !== void 0) options.backoff = backoffOptions(job.backoff);
859
+ return options;
860
+ }
861
+ function backoffOptions(backoff) {
862
+ return {
863
+ type: backoff.type,
864
+ delay: backoff.delayMs,
865
+ jitter: backoff.jitter
866
+ };
867
+ }
868
+
869
+ // src/queues/link-id.ts
870
+ var LINK_SEPARATOR = ".";
871
+ var A_NUMBERED_LINK = /^(.+)\.([1-9]\d*)$/;
872
+ function nextLinkIdOf(linkId) {
873
+ const { runId, link } = linkOfItsRun(linkId);
874
+ return linkIdOf({ runId, link: link + 1 });
875
+ }
876
+ function linkIdOf({ runId, link }) {
877
+ if (link === 0) return runId;
878
+ return `${asACustomJobId(runId)}${LINK_SEPARATOR}${link}`;
879
+ }
880
+ function linkOfItsRun(linkId) {
881
+ const numbered = A_NUMBERED_LINK.exec(linkId);
882
+ if (numbered === null) return { runId: linkId, link: 0 };
883
+ return { runId: numbered[1], link: Number(numbered[2]) };
884
+ }
885
+ function asACustomJobId(runId) {
886
+ if (holdsColonsBullmqRefuses(runId)) return encodeURIComponent(runId);
887
+ return runId;
888
+ }
889
+ function holdsColonsBullmqRefuses(id) {
890
+ const parts = id.split(":").length;
891
+ return parts !== 1 && parts !== 3;
892
+ }
893
+ var BACKLOG_COLLECTION_PREFIX = "rav:spool-backlog:";
894
+ var NEVER_CLAIMED = { claimedAt: { $exists: false } };
895
+ var CLAIMED = { claimedAt: { $type: "date" } };
896
+ var CLAIM_INDEX = { claimedAt: 1, _id: 1 };
897
+ var OLDEST_FIRST = { _id: 1 };
898
+ var MS_PER_SECOND = 1e3;
899
+ var Backlog = class {
900
+ constructor(spoolName, connection) {
901
+ this.spoolName = spoolName;
902
+ this.connection = connection;
903
+ }
904
+ async push(item) {
905
+ const backlog = await this.collection();
906
+ const { insertedId } = await backlog.insertOne({ item });
907
+ return insertedId.toHexString();
908
+ }
909
+ async pushMany(items) {
910
+ if (items.length === 0) return [];
911
+ const backlog = await this.collection();
912
+ const { insertedIds } = await backlog.insertMany(
913
+ items.map((item) => ({ item }))
914
+ );
915
+ return items.map((_, position) => insertedIds[position].toHexString());
916
+ }
917
+ async purge() {
918
+ const backlog = await this.collection();
919
+ const { deletedCount } = await backlog.deleteMany(NEVER_CLAIMED);
920
+ return {
921
+ dropped: deletedCount,
922
+ inFlight: await backlog.countDocuments(CLAIMED)
923
+ };
924
+ }
925
+ async indexForTheClaim() {
926
+ const backlog = await this.collection();
927
+ await backlog.createIndex(CLAIM_INDEX);
928
+ }
929
+ async countInFlight() {
930
+ const backlog = await this.collection();
931
+ return backlog.countDocuments(CLAIMED);
932
+ }
933
+ async expireUnclaimedPushedBefore(pushedBefore) {
934
+ const backlog = await this.collection();
935
+ const { deletedCount } = await backlog.deleteMany({
936
+ ...NEVER_CLAIMED,
937
+ _id: { $lt: mongodb.ObjectId.createFromTime(secondsOf(pushedBefore)) }
938
+ });
939
+ return deletedCount;
940
+ }
941
+ async reclaimStaleClaims(claimedBefore, now) {
942
+ const staleClaim = {
943
+ claimedAt: { $lte: claimedBefore }
944
+ };
945
+ const backlog = await this.collection();
946
+ const stale = await backlog.find(staleClaim).toArray();
947
+ return this.claimAll(stale, staleClaim, now);
948
+ }
949
+ async claimOldestNeverClaimed(count, now) {
950
+ if (count <= 0) return [];
951
+ const backlog = await this.collection();
952
+ const oldest = await backlog.find(NEVER_CLAIMED, { sort: OLDEST_FIRST, limit: count }).toArray();
953
+ return this.claimAll(oldest, NEVER_CLAIMED, now);
954
+ }
955
+ async removeAll(ids) {
956
+ if (ids.length === 0) return;
957
+ const backlog = await this.collection();
958
+ await backlog.deleteMany({ _id: { $in: ids } });
959
+ }
960
+ async claimAll(entries, stillClaimable, claimedAt) {
961
+ if (entries.length === 0) return [];
962
+ const ids = entries.map(({ _id }) => _id);
963
+ const backlog = await this.collection();
964
+ const { modifiedCount } = await backlog.updateMany(
965
+ { _id: { $in: ids }, ...stillClaimable },
966
+ { $set: { claimedAt } }
967
+ );
968
+ if (modifiedCount < entries.length) {
969
+ return this.whatThisClaimTook(ids, claimedAt);
970
970
  }
971
971
  return entries;
972
972
  }
@@ -1178,172 +1178,68 @@ function buildSendOrIngest(job, controls) {
1178
1178
  return { send: buildSend(job) };
1179
1179
  }
1180
1180
 
1181
- // src/client/ravenflow-client.ts
1182
- var RavenflowClient = class {
1183
- constructor(config, options) {
1184
- validateConfig(config);
1185
- this.config = config;
1186
- this.mongoUri = options.mongo;
1187
- this.redisConnectionOptions = options.redis;
1188
- this.logger = createLogger(config.logger);
1189
- this.mongo = new MongoConnection(options.mongo);
1190
- this.queues = openQueues(config, this.redisConnectionOptions);
1181
+ // src/client/create-ravenflow-client.ts
1182
+ function createRavenflowClient(config, options) {
1183
+ validateConfig(config);
1184
+ const logger = createLogger(config.logger);
1185
+ const mongo = new MongoConnection(options.mongo);
1186
+ const queues = openQueues(config, options.redis);
1187
+ const controlPlane = initControlPlane({ redis: options.redis, mongo, logger });
1188
+ return {
1189
+ jobs: initJobProxies(config, { queues, mongo, controls: controlPlane }),
1190
+ close: async () => {
1191
+ await controlPlane.close();
1192
+ await closeQueues(queues);
1193
+ await mongo.close();
1194
+ }
1195
+ };
1196
+ }
1197
+
1198
+ // src/worker/control-plane-reader.ts
1199
+ var ControlPlaneReader = class {
1200
+ constructor(settings) {
1201
+ this.mirrorConnection = new MongoConnection(settings.mongo);
1191
1202
  this.controlPlane = initControlPlane({
1192
- redis: this.redisConnectionOptions,
1193
- mongo: this.mongo,
1194
- logger: this.logger
1195
- });
1196
- this.jobs = initJobProxies(config, {
1197
- queues: this.queues,
1198
- mongo: this.mongo,
1199
- controls: this.controlPlane
1203
+ redis: settings.redis,
1204
+ mongo: this.mirrorConnection,
1205
+ logger: settings.logger
1200
1206
  });
1201
1207
  }
1208
+ async open() {
1209
+ await this.controlPlane.rehydrate();
1210
+ await this.mirrorConnection.close();
1211
+ }
1212
+ read(jobName) {
1213
+ return this.controlPlane.read(jobName);
1214
+ }
1202
1215
  async close() {
1203
1216
  await this.controlPlane.close();
1204
- await closeQueues(this.queues);
1205
- await this.mongo.close();
1217
+ await this.mirrorConnection.close();
1206
1218
  }
1207
1219
  };
1208
1220
 
1209
- // src/client/create-ravenflow-client.ts
1210
- function createRavenflowClient(config, options) {
1211
- return new RavenflowClient(config, options);
1221
+ // src/worker/log-bullmq-errors.ts
1222
+ function logBullmqErrors(source, logger) {
1223
+ source.on("error", (error) => {
1224
+ logger.error(
1225
+ { err: error, queue: source.name },
1226
+ `[ravenflow] BullMQ reported an error on "${source.name}".`
1227
+ );
1228
+ });
1212
1229
  }
1213
1230
 
1214
- // src/describe-error.ts
1215
- function describeError(error) {
1216
- return error instanceof Error ? error.message : String(error);
1217
- }
1218
- var RAVENFLOW = `
1219
- \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557
1220
- \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551
1221
- \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2557 \u2588\u2588\u2551
1222
- \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255A\u2588\u2588\u2557 \u2588\u2588\u2554\u255D\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551\u255A\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551\u2588\u2588\u2588\u2557\u2588\u2588\u2551
1223
- \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u255A\u2588\u2588\u2588\u2554\u2588\u2588\u2588\u2554\u255D
1224
- \u255A\u2550\u255D \u255A\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u255D\u255A\u2550\u2550\u255D
1225
- `;
1226
- var RULE = "\u2550".repeat(85);
1227
- var FEEDS_IN_ORDER = ["nothing", "clock", "spool"];
1228
- function printBanner(facts) {
1229
- console.log(
1230
- [
1231
- RAVENFLOW,
1232
- RULE,
1233
- "[Config]",
1234
- ...configLines(facts),
1235
- "",
1236
- "[Registered Jobs]",
1237
- ...registeredJobLines(facts.jobs),
1238
- "",
1239
- "[Scheduled Jobs]",
1240
- ...schedulerLines(facts.schedulers),
1241
- RULE,
1242
- "Ready to process jobs..."
1243
- ].join("\n")
1244
- );
1231
+ // src/worker/mongodb-deadline-on-close.ts
1232
+ var MONGODB_DEADLINE_ON_CLOSE_MS = 5e3;
1233
+ async function waitOrCutShort(waitingOnMongoDB, cutItShort) {
1234
+ if (await settlesBeforeTheDeadline(waitingOnMongoDB)) return;
1235
+ await cutItShort();
1236
+ await waitingOnMongoDB;
1245
1237
  }
1246
- function configLines(facts) {
1247
- return [
1248
- ` -> queue: ${facts.queue}`,
1249
- ` -> concurrency: ${facts.concurrency}`,
1250
- ` -> redis: ${addressOf(facts.redis)}`,
1251
- ` -> mongo: ${mongoLineOf(facts)}`
1252
- ];
1253
- }
1254
- function addressOf(redis) {
1255
- if (bullmq.isRedisInstance(redis)) return "an ioredis client";
1256
- const {
1257
- url,
1258
- host = "localhost",
1259
- port = 6379,
1260
- db = 0
1261
- } = redis;
1262
- return url ? withoutPassword(url) : `${host}:${port}/${db}`;
1263
- }
1264
- function mongoLineOf({ mongo, recorderOpening }) {
1265
- const address = withoutPassword(mongo);
1266
- if (!recorderOpening.opened) {
1267
- return `${address} (job results are not recorded: ${recorderOpening.because})`;
1268
- }
1269
- return address;
1270
- }
1271
- function withoutPassword(uri) {
1272
- return uri.replace(/\/\/([^:/@]*):[^@]*@/, "//$1:***@");
1273
- }
1274
- function registeredJobLines(jobs) {
1275
- const lines = Object.entries(jobs).map(([name, job]) => ({ name, feed: feedOf(job) })).sort((a, b) => feedRank(a.feed) - feedRank(b.feed)).map(({ name, feed }) => ` . ${name}${feed.label}`);
1276
- return lines.length > 0 ? lines : [" (none)"];
1277
- }
1278
- function feedOf(job) {
1279
- const cron = job.trigger?.cron;
1280
- if (cron) return { kind: "clock", label: ` (\u21BB ${cron})` };
1281
- if (isSpoolFed(job)) return { kind: "spool", label: spoolLabelOf(job) };
1282
- return { kind: "nothing", label: "" };
1283
- }
1284
- function spoolLabelOf({ trigger, pace }) {
1285
- const ceiling = inFlightCeilingOf(trigger.spool, pace);
1286
- return ` (\u2190 ${trigger.spool.name}, ${ceiling} in flight at most)`;
1287
- }
1288
- function feedRank(feed) {
1289
- return FEEDS_IN_ORDER.indexOf(feed.kind);
1290
- }
1291
- function schedulerLines(changes) {
1292
- const lines = [
1293
- ...changes.added.map(({ name, pattern }) => ` + ${name} \u2192 ${pattern}`),
1294
- ...changes.rescheduled.map(
1295
- ({ name, pattern, previousPattern }) => ` \u21BB ${name} \u2192 ${pattern} (was ${previousPattern})`
1296
- ),
1297
- ...changes.removed.map((name) => ` - ${name}`)
1298
- ];
1299
- return lines.length > 0 ? lines : [" (no change)"];
1300
- }
1301
-
1302
- // src/worker/control-plane-reader.ts
1303
- var ControlPlaneReader = class {
1304
- constructor(settings) {
1305
- this.mirrorConnection = new MongoConnection(settings.mongo);
1306
- this.controlPlane = initControlPlane({
1307
- redis: settings.redis,
1308
- mongo: this.mirrorConnection,
1309
- logger: settings.logger
1310
- });
1311
- }
1312
- async open() {
1313
- await this.controlPlane.rehydrate();
1314
- await this.mirrorConnection.close();
1315
- }
1316
- read(jobName) {
1317
- return this.controlPlane.read(jobName);
1318
- }
1319
- async close() {
1320
- await this.controlPlane.close();
1321
- await this.mirrorConnection.close();
1322
- }
1323
- };
1324
-
1325
- // src/worker/log-bullmq-errors.ts
1326
- function logBullmqErrors(source, logger) {
1327
- source.on("error", (error) => {
1328
- logger.error(
1329
- { err: error, queue: source.name },
1330
- `[ravenflow] BullMQ reported an error on "${source.name}".`
1331
- );
1332
- });
1333
- }
1334
-
1335
- // src/worker/mongodb-deadline-on-close.ts
1336
- var MONGODB_DEADLINE_ON_CLOSE_MS = 5e3;
1337
- async function waitOrCutShort(waitingOnMongoDB, cutItShort) {
1338
- if (await settlesBeforeTheDeadline(waitingOnMongoDB)) return;
1339
- await cutItShort();
1340
- await waitingOnMongoDB;
1341
- }
1342
- async function settlesBeforeTheDeadline(work) {
1343
- const deadline = new Promise((resolve) => {
1344
- setTimeout(() => resolve(false), MONGODB_DEADLINE_ON_CLOSE_MS).unref();
1345
- });
1346
- return Promise.race([work.then(() => true), deadline]);
1238
+ async function settlesBeforeTheDeadline(work) {
1239
+ const deadline = new Promise((resolve) => {
1240
+ setTimeout(() => resolve(false), MONGODB_DEADLINE_ON_CLOSE_MS).unref();
1241
+ });
1242
+ return Promise.race([work.then(() => true), deadline]);
1347
1243
  }
1348
1244
  var FLUSH_INTERVAL_MS = 3e4;
1349
1245
  var JOB_RESULTS_PER_BATCH = 1e3;
@@ -2706,100 +2602,6 @@ function abortWhenClosing(worker, shutdown) {
2706
2602
  shutdown.abort(new Error("[ravenflow] the worker is shutting down"));
2707
2603
  });
2708
2604
  }
2709
-
2710
- // src/worker/schedulers.ts
2711
- async function reconcileSchedulers(queue, jobs) {
2712
- const declared = cronSchedulesOf(jobs);
2713
- const existing = await patternsByScheduler(queue);
2714
- const changes = changesBetween(declared, existing);
2715
- await Promise.all(
2716
- changes.removed.map((key) => queue.removeJobScheduler(key))
2717
- );
2718
- await Promise.all(declared.map((schedule) => upsert(queue, schedule)));
2719
- return changes;
2720
- }
2721
- function cronSchedulesOf(jobs) {
2722
- return Object.entries(jobs).flatMap(([name, job]) => {
2723
- const pattern = job.trigger?.cron;
2724
- if (pattern === void 0) return [];
2725
- return [{ name, pattern, options: buildJobOptions(job) }];
2726
- });
2727
- }
2728
- async function patternsByScheduler(queue) {
2729
- const schedulers = await queue.getJobSchedulers();
2730
- return new Map(
2731
- schedulers.map(({ key, pattern }) => [key, pattern ?? "no pattern"])
2732
- );
2733
- }
2734
- function changesBetween(declared, existing) {
2735
- const declaredNames = new Set(declared.map(({ name }) => name));
2736
- return {
2737
- added: declared.filter(({ name }) => !existing.has(name)),
2738
- rescheduled: declared.flatMap(
2739
- (schedule) => reschedulingOf(schedule, existing)
2740
- ),
2741
- removed: [...existing.keys()].filter((key) => !declaredNames.has(key))
2742
- };
2743
- }
2744
- function reschedulingOf(schedule, existing) {
2745
- const previousPattern = existing.get(schedule.name);
2746
- if (previousPattern === void 0) return [];
2747
- if (previousPattern === schedule.pattern) return [];
2748
- return [{ ...schedule, previousPattern }];
2749
- }
2750
- async function upsert(queue, { name, pattern, options }) {
2751
- try {
2752
- await queue.upsertJobScheduler(name, { pattern }, { name, opts: options });
2753
- } catch (error) {
2754
- throw new Error(
2755
- `[ravenflow] "${name}" could not be scheduled on "${pattern}": ${describeError(error)}`
2756
- );
2757
- }
2758
- }
2759
-
2760
- // src/worker/served-queue.ts
2761
- function refuseAQueueItsConfigDoesNotDeclare(queue, config) {
2762
- const declared = declaredQueueNames(config);
2763
- if (declared.includes(queue)) return;
2764
- const quoted = declared.map((name) => `"${name}"`).join(", ");
2765
- throw new Error(
2766
- `[ravenflow] no job of this config runs on "${queue}": its queues are ${quoted}.`
2767
- );
2768
- }
2769
-
2770
- // src/worker/serving.ts
2771
- async function closeInOrder(serving) {
2772
- await stopWatchingMongoDB(serving);
2773
- await stopTheTicks(serving);
2774
- await stopTheWorker(serving);
2775
- await writeWhatItsJobsLeft(serving);
2776
- await closeTheConnections(serving);
2777
- }
2778
- async function stopWatchingMongoDB(serving) {
2779
- await serving.mongoWatch.close();
2780
- }
2781
- async function stopTheTicks(serving) {
2782
- await Promise.all([
2783
- serving.jobResultsAggregator.stopTicking(),
2784
- serving.spoolFeeder.stopTicking()
2785
- ]);
2786
- }
2787
- async function stopTheWorker(serving) {
2788
- serving.jobResultsRecorder.stopHoldingJobs();
2789
- await serving.worker.close();
2790
- }
2791
- async function writeWhatItsJobsLeft(serving) {
2792
- await Promise.all([
2793
- serving.jobResultsRecorder.close(),
2794
- serving.spoolFeeder.finishTheRemovals()
2795
- ]);
2796
- }
2797
- async function closeTheConnections(serving) {
2798
- await serving.spoolFeeder.close();
2799
- await serving.jobResultsAggregator.close();
2800
- await serving.controlPlaneReader.close();
2801
- await serving.queue.close();
2802
- }
2803
2605
  var THE_DRIVERS_RECOVERY_LAG_MS = 1e4;
2804
2606
  var MongoWatch = class {
2805
2607
  constructor(settings) {
@@ -2921,189 +2723,374 @@ async function callOnWithdraw({ logger, onWithdraw }, reason) {
2921
2723
  }
2922
2724
  }
2923
2725
 
2924
- // src/worker/ravenflow-worker.ts
2925
- var RavenflowWorker = class {
2926
- constructor(config, options) {
2927
- this.serving = null;
2928
- this.starting = null;
2929
- this.closing = null;
2930
- validateConfig(config);
2931
- refuseAQueueItsConfigDoesNotDeclare(options.queue, config);
2932
- this.config = config;
2933
- this.options = options;
2934
- this.jobs = jobsDeclaredOn(options.queue, config);
2935
- this.logger = createLogger(config.logger);
2936
- }
2937
- async start() {
2938
- this.refuseASecondStart();
2939
- const serving = this.openServing();
2940
- this.serving = serving;
2941
- this.starting = this.serveOrStop(serving);
2726
+ // src/worker/open-serving.ts
2727
+ function openServing(settings, onDurableLoss) {
2728
+ const { config, options, jobs, logger } = settings;
2729
+ const queue = openQueue(options.queue, {
2730
+ connection: options.redis,
2731
+ retention: config.queuesRetention
2732
+ });
2733
+ logBullmqErrors(queue, logger);
2734
+ const controlPlaneReader = buildControlPlaneReader(settings);
2735
+ const jobResultsRecorder = buildJobResultsRecorder(settings, queue);
2736
+ const worker = openWorker(options, {
2737
+ jobs,
2738
+ logger,
2739
+ queue,
2740
+ controls: controlPlaneReader,
2741
+ jobResultsRecorder
2742
+ });
2743
+ return {
2744
+ queue,
2745
+ controlPlaneReader,
2746
+ jobResultsRecorder,
2747
+ jobResultsAggregator: buildJobResultsAggregator(settings),
2748
+ spoolFeeder: buildSpoolFeeder(settings, { queue, controlPlaneReader }),
2749
+ mongoWatch: buildMongoWatch(settings, onDurableLoss),
2750
+ worker
2751
+ };
2752
+ }
2753
+ function buildControlPlaneReader({
2754
+ options,
2755
+ logger
2756
+ }) {
2757
+ return new ControlPlaneReader({
2758
+ redis: options.redis,
2759
+ mongo: options.mongo,
2760
+ logger
2761
+ });
2762
+ }
2763
+ function buildJobResultsRecorder({ config, options, jobs, logger }, queue) {
2764
+ return new JobResultsRecorder({
2765
+ mongo: options.mongo,
2766
+ retentionHours: config.jobResultsRetentionHours ?? DEFAULT_JOB_RESULTS_RETENTION_HOURS,
2767
+ queue,
2768
+ jobs,
2769
+ logger
2770
+ });
2771
+ }
2772
+ function buildJobResultsAggregator({
2773
+ config,
2774
+ options,
2775
+ logger
2776
+ }) {
2777
+ return new JobResultsAggregator({
2778
+ redis: options.redis,
2779
+ mongo: options.mongo,
2780
+ retentionHours: config.aggregatedResultsRetentionHours ?? DEFAULT_AGGREGATED_RESULTS_RETENTION_HOURS,
2781
+ logger
2782
+ });
2783
+ }
2784
+ function buildSpoolFeeder({ options, jobs, logger }, { queue, controlPlaneReader }) {
2785
+ return new SpoolFeeder({
2786
+ redis: options.redis,
2787
+ mongo: options.mongo,
2788
+ queue,
2789
+ controls: controlPlaneReader,
2790
+ jobs,
2791
+ logger
2792
+ });
2793
+ }
2794
+ function buildMongoWatch({ options, logger }, onDurableLoss) {
2795
+ return new MongoWatch({
2796
+ mongo: options.mongo,
2797
+ gracePeriodMs: options.withdrawal?.afterMs ?? DEFAULT_GRACE_PERIOD_MS,
2798
+ logger,
2799
+ onDurableLoss
2800
+ });
2801
+ }
2802
+
2803
+ // src/worker/served-queue.ts
2804
+ function refuseAQueueItsConfigDoesNotDeclare(queue, config) {
2805
+ const declared = declaredQueueNames(config);
2806
+ if (declared.includes(queue)) return;
2807
+ const quoted = declared.map((name) => `"${name}"`).join(", ");
2808
+ throw new Error(
2809
+ `[ravenflow] no job of this config runs on "${queue}": its queues are ${quoted}.`
2810
+ );
2811
+ }
2812
+
2813
+ // src/describe-error.ts
2814
+ function describeError(error) {
2815
+ return error instanceof Error ? error.message : String(error);
2816
+ }
2817
+ var RAVENFLOW = `
2818
+ \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557
2819
+ \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551
2820
+ \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2557 \u2588\u2588\u2551
2821
+ \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255A\u2588\u2588\u2557 \u2588\u2588\u2554\u255D\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551\u255A\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551\u2588\u2588\u2588\u2557\u2588\u2588\u2551
2822
+ \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u255A\u2588\u2588\u2588\u2554\u2588\u2588\u2588\u2554\u255D
2823
+ \u255A\u2550\u255D \u255A\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u255D\u255A\u2550\u2550\u255D
2824
+ `;
2825
+ var RULE = "\u2550".repeat(85);
2826
+ var FEEDS_IN_ORDER = ["nothing", "clock", "spool"];
2827
+ function printBanner(facts) {
2828
+ console.log(
2829
+ [
2830
+ RAVENFLOW,
2831
+ RULE,
2832
+ "[Config]",
2833
+ ...configLines(facts),
2834
+ "",
2835
+ "[Registered Jobs]",
2836
+ ...registeredJobLines(facts.jobs),
2837
+ "",
2838
+ "[Scheduled Jobs]",
2839
+ ...schedulerLines(facts.schedulers),
2840
+ RULE,
2841
+ "Ready to process jobs..."
2842
+ ].join("\n")
2843
+ );
2844
+ }
2845
+ function configLines(facts) {
2846
+ return [
2847
+ ` -> queue: ${facts.queue}`,
2848
+ ` -> concurrency: ${facts.concurrency}`,
2849
+ ` -> redis: ${addressOf(facts.redis)}`,
2850
+ ` -> mongo: ${mongoLineOf(facts)}`
2851
+ ];
2852
+ }
2853
+ function addressOf(redis) {
2854
+ if (bullmq.isRedisInstance(redis)) return "an ioredis client";
2855
+ const {
2856
+ url,
2857
+ host = "localhost",
2858
+ port = 6379,
2859
+ db = 0
2860
+ } = redis;
2861
+ return url ? withoutPassword(url) : `${host}:${port}/${db}`;
2862
+ }
2863
+ function mongoLineOf({ mongo, recorderOpening }) {
2864
+ const address = withoutPassword(mongo);
2865
+ if (!recorderOpening.opened) {
2866
+ return `${address} (job results are not recorded: ${recorderOpening.because})`;
2867
+ }
2868
+ return address;
2869
+ }
2870
+ function withoutPassword(uri) {
2871
+ return uri.replace(/\/\/([^:/@]*):[^@]*@/, "//$1:***@");
2872
+ }
2873
+ function registeredJobLines(jobs) {
2874
+ const lines = Object.entries(jobs).map(([name, job]) => ({ name, feed: feedOf(job) })).sort((a, b) => feedRank(a.feed) - feedRank(b.feed)).map(({ name, feed }) => ` . ${name}${feed.label}`);
2875
+ return lines.length > 0 ? lines : [" (none)"];
2876
+ }
2877
+ function feedOf(job) {
2878
+ const cron = job.trigger?.cron;
2879
+ if (cron) return { kind: "clock", label: ` (\u21BB ${cron})` };
2880
+ if (isSpoolFed(job)) return { kind: "spool", label: spoolLabelOf(job) };
2881
+ return { kind: "nothing", label: "" };
2882
+ }
2883
+ function spoolLabelOf({ trigger, pace }) {
2884
+ const ceiling = inFlightCeilingOf(trigger.spool, pace);
2885
+ return ` (\u2190 ${trigger.spool.name}, ${ceiling} in flight at most)`;
2886
+ }
2887
+ function feedRank(feed) {
2888
+ return FEEDS_IN_ORDER.indexOf(feed.kind);
2889
+ }
2890
+ function schedulerLines(changes) {
2891
+ const lines = [
2892
+ ...changes.added.map(({ name, pattern }) => ` + ${name} \u2192 ${pattern}`),
2893
+ ...changes.rescheduled.map(
2894
+ ({ name, pattern, previousPattern }) => ` \u21BB ${name} \u2192 ${pattern} (was ${previousPattern})`
2895
+ ),
2896
+ ...changes.removed.map((name) => ` - ${name}`)
2897
+ ];
2898
+ return lines.length > 0 ? lines : [" (no change)"];
2899
+ }
2900
+
2901
+ // src/worker/schedulers.ts
2902
+ async function reconcileSchedulers(queue, jobs) {
2903
+ const declared = cronSchedulesOf(jobs);
2904
+ const existing = await patternsByScheduler(queue);
2905
+ const changes = changesBetween(declared, existing);
2906
+ await Promise.all(
2907
+ changes.removed.map((key) => queue.removeJobScheduler(key))
2908
+ );
2909
+ await Promise.all(declared.map((schedule) => upsert(queue, schedule)));
2910
+ return changes;
2911
+ }
2912
+ function cronSchedulesOf(jobs) {
2913
+ return Object.entries(jobs).flatMap(([name, job]) => {
2914
+ const pattern = job.trigger?.cron;
2915
+ if (pattern === void 0) return [];
2916
+ return [{ name, pattern, options: buildJobOptions(job) }];
2917
+ });
2918
+ }
2919
+ async function patternsByScheduler(queue) {
2920
+ const schedulers = await queue.getJobSchedulers();
2921
+ return new Map(
2922
+ schedulers.map(({ key, pattern }) => [key, pattern ?? "no pattern"])
2923
+ );
2924
+ }
2925
+ function changesBetween(declared, existing) {
2926
+ const declaredNames = new Set(declared.map(({ name }) => name));
2927
+ return {
2928
+ added: declared.filter(({ name }) => !existing.has(name)),
2929
+ rescheduled: declared.flatMap(
2930
+ (schedule) => reschedulingOf(schedule, existing)
2931
+ ),
2932
+ removed: [...existing.keys()].filter((key) => !declaredNames.has(key))
2933
+ };
2934
+ }
2935
+ function reschedulingOf(schedule, existing) {
2936
+ const previousPattern = existing.get(schedule.name);
2937
+ if (previousPattern === void 0) return [];
2938
+ if (previousPattern === schedule.pattern) return [];
2939
+ return [{ ...schedule, previousPattern }];
2940
+ }
2941
+ async function upsert(queue, { name, pattern, options }) {
2942
+ try {
2943
+ await queue.upsertJobScheduler(name, { pattern }, { name, opts: options });
2944
+ } catch (error) {
2945
+ throw new Error(
2946
+ `[ravenflow] "${name}" could not be scheduled on "${pattern}": ${describeError(error)}`
2947
+ );
2948
+ }
2949
+ }
2950
+
2951
+ // src/worker/serving.ts
2952
+ async function serve(serving, settings) {
2953
+ const { options, jobs } = settings;
2954
+ const { queue, worker } = serving;
2955
+ serving.mongoWatch.open();
2956
+ await serving.controlPlaneReader.open();
2957
+ const recorderOpening = await openRecorder(serving, settings);
2958
+ await serving.jobResultsAggregator.open();
2959
+ await serving.spoolFeeder.open(worker);
2960
+ const schedulers = await reconcileSchedulers(queue, jobs);
2961
+ takeJobs(worker, settings);
2962
+ await worker.waitUntilReady();
2963
+ printBanner({
2964
+ queue: options.queue,
2965
+ concurrency: worker.concurrency,
2966
+ redis: options.redis,
2967
+ mongo: options.mongo,
2968
+ recorderOpening,
2969
+ jobs,
2970
+ schedulers
2971
+ });
2972
+ }
2973
+ async function openRecorder({ jobResultsRecorder, worker }, settings) {
2974
+ try {
2975
+ await jobResultsRecorder.open(worker);
2976
+ } catch (error) {
2977
+ sayJobResultsAreNotRecorded(settings, error);
2978
+ return { opened: false, because: describeError(error) };
2979
+ }
2980
+ return { opened: true };
2981
+ }
2982
+ function sayJobResultsAreNotRecorded({ options, logger }, error) {
2983
+ logger.error(
2984
+ { err: error },
2985
+ `[ravenflow] the job results recorder could not start: the worker serves "${options.queue}", and its job results are not recorded.`
2986
+ );
2987
+ }
2988
+ function takeJobs(worker, { options, logger }) {
2989
+ worker.run().catch((error) => {
2990
+ logger.error(
2991
+ { err: error },
2992
+ `[ravenflow] the worker serving "${options.queue}" stopped taking jobs.`
2993
+ );
2994
+ });
2995
+ }
2996
+ async function closeInOrder(serving) {
2997
+ await stopWatchingMongoDB(serving);
2998
+ await stopTheTicks(serving);
2999
+ await stopTheWorker(serving);
3000
+ await writeWhatItsJobsLeft(serving);
3001
+ await closeTheConnections(serving);
3002
+ }
3003
+ async function stopWatchingMongoDB(serving) {
3004
+ await serving.mongoWatch.close();
3005
+ }
3006
+ async function stopTheTicks(serving) {
3007
+ await Promise.all([
3008
+ serving.jobResultsAggregator.stopTicking(),
3009
+ serving.spoolFeeder.stopTicking()
3010
+ ]);
3011
+ }
3012
+ async function stopTheWorker(serving) {
3013
+ serving.jobResultsRecorder.stopHoldingJobs();
3014
+ await serving.worker.close();
3015
+ }
3016
+ async function writeWhatItsJobsLeft(serving) {
3017
+ await Promise.all([
3018
+ serving.jobResultsRecorder.close(),
3019
+ serving.spoolFeeder.finishTheRemovals()
3020
+ ]);
3021
+ }
3022
+ async function closeTheConnections(serving) {
3023
+ await serving.spoolFeeder.close();
3024
+ await serving.jobResultsAggregator.close();
3025
+ await serving.controlPlaneReader.close();
3026
+ await serving.queue.close();
3027
+ }
3028
+
3029
+ // src/worker/create-ravenflow-worker.ts
3030
+ function createRavenflowWorker(config, options) {
3031
+ validateConfig(config);
3032
+ refuseAQueueItsConfigDoesNotDeclare(options.queue, config);
3033
+ return workerOf({
3034
+ config,
3035
+ options,
3036
+ jobs: jobsDeclaredOn(options.queue, config),
3037
+ logger: createLogger(config.logger)
3038
+ });
3039
+ }
3040
+ function workerOf(settings) {
3041
+ let serving = null;
3042
+ let starting = null;
3043
+ let closing = null;
3044
+ return { start, close };
3045
+ async function start() {
3046
+ if (serving) throw aSecondStart(settings);
3047
+ const opened = openServing(settings, beginAWithdrawal);
3048
+ serving = opened;
3049
+ starting = serveOrStop(opened);
2942
3050
  try {
2943
- await this.starting;
3051
+ await starting;
2944
3052
  } finally {
2945
- this.starting = null;
3053
+ starting = null;
2946
3054
  }
2947
3055
  }
2948
- async close() {
2949
- await Promise.allSettled([this.starting]);
2950
- if (this.serving === null) return;
2951
- this.closing ?? (this.closing = this.stopServing(this.serving));
2952
- await this.closing;
3056
+ async function close() {
3057
+ await Promise.allSettled([starting]);
3058
+ if (serving === null) return;
3059
+ closing ?? (closing = stopServing(serving));
3060
+ await closing;
2953
3061
  }
2954
- async serveOrStop(serving) {
3062
+ async function serveOrStop(opened) {
2955
3063
  try {
2956
- await this.serve(serving);
3064
+ await serve(opened, settings);
2957
3065
  } catch (error) {
2958
- await this.stopServing(serving);
3066
+ await stopServing(opened);
2959
3067
  throw error;
2960
3068
  }
2961
3069
  }
2962
- async stopServing(serving) {
3070
+ async function stopServing(opened) {
2963
3071
  try {
2964
- await closeInOrder(serving);
3072
+ await closeInOrder(opened);
2965
3073
  } finally {
2966
- this.serving = null;
2967
- this.closing = null;
2968
- }
2969
- }
2970
- refuseASecondStart() {
2971
- if (!this.serving) return;
2972
- throw new Error(
2973
- `[ravenflow] this worker has already started on "${this.options.queue}".`
2974
- );
2975
- }
2976
- openServing() {
2977
- const queue = openQueue(this.options.queue, {
2978
- connection: this.options.redis,
2979
- retention: this.config.queuesRetention
2980
- });
2981
- logBullmqErrors(queue, this.logger);
2982
- const controlPlaneReader = this.buildControlPlaneReader();
2983
- const jobResultsRecorder = this.buildJobResultsRecorder(queue);
2984
- const worker = openWorker(this.options, {
2985
- jobs: this.jobs,
2986
- logger: this.logger,
2987
- queue,
2988
- controls: controlPlaneReader,
2989
- jobResultsRecorder
2990
- });
2991
- return {
2992
- queue,
2993
- controlPlaneReader,
2994
- jobResultsRecorder,
2995
- jobResultsAggregator: this.buildJobResultsAggregator(),
2996
- spoolFeeder: this.buildSpoolFeeder(queue, controlPlaneReader),
2997
- mongoWatch: this.buildMongoWatch(),
2998
- worker
2999
- };
3000
- }
3001
- buildControlPlaneReader() {
3002
- return new ControlPlaneReader({
3003
- redis: this.options.redis,
3004
- mongo: this.options.mongo,
3005
- logger: this.logger
3006
- });
3007
- }
3008
- buildJobResultsRecorder(queue) {
3009
- return new JobResultsRecorder({
3010
- mongo: this.options.mongo,
3011
- retentionHours: this.config.jobResultsRetentionHours ?? DEFAULT_JOB_RESULTS_RETENTION_HOURS,
3012
- queue,
3013
- jobs: this.jobs,
3014
- logger: this.logger
3015
- });
3016
- }
3017
- buildJobResultsAggregator() {
3018
- return new JobResultsAggregator({
3019
- redis: this.options.redis,
3020
- mongo: this.options.mongo,
3021
- retentionHours: this.config.aggregatedResultsRetentionHours ?? DEFAULT_AGGREGATED_RESULTS_RETENTION_HOURS,
3022
- logger: this.logger
3023
- });
3024
- }
3025
- buildSpoolFeeder(queue, controls) {
3026
- return new SpoolFeeder({
3027
- redis: this.options.redis,
3028
- mongo: this.options.mongo,
3029
- queue,
3030
- controls,
3031
- jobs: this.jobs,
3032
- logger: this.logger
3033
- });
3034
- }
3035
- buildMongoWatch() {
3036
- return new MongoWatch({
3037
- mongo: this.options.mongo,
3038
- gracePeriodMs: this.options.withdrawal?.afterMs ?? DEFAULT_GRACE_PERIOD_MS,
3039
- logger: this.logger,
3040
- onDurableLoss: (unreachableForMs) => this.beginAWithdrawal(unreachableForMs)
3041
- });
3042
- }
3043
- beginAWithdrawal(unreachableForMs) {
3044
- const withdrawing = {
3045
- queue: this.options.queue,
3046
- logger: this.logger,
3047
- onWithdraw: this.options.withdrawal?.onWithdraw,
3048
- close: () => this.close()
3049
- };
3050
- void withdraw(withdrawing, unreachableForMs);
3051
- }
3052
- async serve({
3053
- queue,
3054
- controlPlaneReader,
3055
- jobResultsRecorder,
3056
- jobResultsAggregator,
3057
- spoolFeeder,
3058
- mongoWatch,
3059
- worker
3060
- }) {
3061
- mongoWatch.open();
3062
- await controlPlaneReader.open();
3063
- const recorderOpening = await this.openRecorder(jobResultsRecorder, worker);
3064
- await jobResultsAggregator.open();
3065
- await spoolFeeder.open(worker);
3066
- const schedulers = await reconcileSchedulers(queue, this.jobs);
3067
- this.takeJobs(worker);
3068
- await worker.waitUntilReady();
3069
- printBanner({
3070
- queue: this.options.queue,
3071
- concurrency: worker.concurrency,
3072
- redis: this.options.redis,
3073
- mongo: this.options.mongo,
3074
- recorderOpening,
3075
- jobs: this.jobs,
3076
- schedulers
3077
- });
3078
- }
3079
- async openRecorder(jobResultsRecorder, worker) {
3080
- try {
3081
- await jobResultsRecorder.open(worker);
3082
- } catch (error) {
3083
- this.sayJobResultsAreNotRecorded(error);
3084
- return { opened: false, because: describeError(error) };
3074
+ serving = null;
3075
+ closing = null;
3085
3076
  }
3086
- return { opened: true };
3087
3077
  }
3088
- sayJobResultsAreNotRecorded(error) {
3089
- this.logger.error(
3090
- { err: error },
3091
- `[ravenflow] the job results recorder could not start: the worker serves "${this.options.queue}", and its job results are not recorded.`
3092
- );
3078
+ function beginAWithdrawal(unreachableForMs) {
3079
+ void withdraw(withdrawingOf(settings, close), unreachableForMs);
3093
3080
  }
3094
- takeJobs(worker) {
3095
- worker.run().catch((error) => {
3096
- this.logger.error(
3097
- { err: error },
3098
- `[ravenflow] the worker serving "${this.options.queue}" stopped taking jobs.`
3099
- );
3100
- });
3101
- }
3102
- };
3103
-
3104
- // src/worker/create-ravenflow-worker.ts
3105
- function createRavenflowWorker(config, options) {
3106
- return new RavenflowWorker(config, options);
3081
+ }
3082
+ function aSecondStart({ options }) {
3083
+ return new Error(
3084
+ `[ravenflow] this worker has already started on "${options.queue}".`
3085
+ );
3086
+ }
3087
+ function withdrawingOf({ options, logger }, close) {
3088
+ return {
3089
+ queue: options.queue,
3090
+ logger,
3091
+ onWithdraw: options.withdrawal?.onWithdraw,
3092
+ close
3093
+ };
3107
3094
  }
3108
3095
 
3109
3096
  // src/config/ravenflow-config.ts