vona-module-a-queue 5.0.9 → 5.0.11

package/dist/.metadata/index.d.ts

@@ -0,0 +1,70 @@
+import type { BeanScopeUtil, TypeModuleConfig } from 'vona';
+/** startup: begin */
+import type { IDecoratorStartupOptions } from 'vona-module-a-startup';
+import type { config } from '../config/config.ts';
+/** service: end */
+/** service: begin */
+import type { ServiceQueue } from '../service/queue.ts';
+/** service: end */
+/** service: begin */
+/** monkey: end */
+/** scope: begin */
+import { BeanScopeBase } from 'vona';
+import 'vona';
+import 'vona';
+import 'vona';
+export * from '../bean/startup.loadQueueWorkers.ts';
+declare module 'vona-module-a-startup' {
+    interface IStartupRecord {
+        'a-queue:loadQueueWorkers': IDecoratorStartupOptions;
+    }
+}
+declare module 'vona-module-a-queue' {
+    interface StartupLoadQueueWorkers {
+    }
+}
+/** service: end */
+/** config: begin */
+export * from '../config/config.ts';
+declare module 'vona-module-a-web' {
+    interface IServiceRecord {
+        'a-queue:queue': never;
+    }
+}
+declare module 'vona-module-a-queue' {
+    interface ServiceQueue {
+    }
+}
+export interface IModuleService {
+    queue: ServiceQueue;
+}
+declare module 'vona' {
+    interface IBeanRecordGeneral {
+        'a-queue.service.queue': ServiceQueue;
+    }
+}
+/** config: end */
+/** monkey: begin */
+export * from '../monkey.ts';
+/** startup: end */
+/** service: begin */
+export * from '../service/queue.ts';
+export declare class ScopeModuleAQueue extends BeanScopeBase {
+}
+export interface ScopeModuleAQueue {
+    util: BeanScopeUtil;
+    config: TypeModuleConfig<typeof config>;
+    service: IModuleService;
+}
+declare module 'vona' {
+    interface IBeanScopeRecord {
+        'a-queue': ScopeModuleAQueue;
+    }
+    interface IBeanScopeContainer {
+        queue: ScopeModuleAQueue;
+    }
+    interface IBeanScopeConfig {
+        'a-queue': ReturnType<typeof config>;
+    }
+}
+/** scope: end */

package/dist/.metadata/this.d.ts

@@ -0,0 +1,2 @@
+export declare const __ThisModule__ = "a-queue";
+export { ScopeModuleAQueue as ScopeModule } from './index.ts';

package/dist/bean/startup.loadQueueWorkers.d.ts

@@ -0,0 +1,5 @@
+import type { IStartupExecute } from 'vona-module-a-startup';
+import { BeanBase } from 'vona';
+export declare class StartupLoadQueueWorkers extends BeanBase implements IStartupExecute {
+    execute(): Promise<void>;
+}

package/dist/config/config.d.ts

@@ -0,0 +1,5 @@
+import type * as Bull from 'bullmq';
+import type { VonaApplication } from 'vona';
+export declare function config(_app: VonaApplication): {
+    worker: Bull.WorkerOptions;
+};

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from './.metadata/index.ts';
+export * from './lib/index.ts';
+export * from './types/index.ts';

package/dist/index.js

@@ -0,0 +1,334 @@
+import { BeanInfo, BeanBase, BeanSimple, uuidv4, deepExtend, instanceDesp, beanFullNameFromOnionName, BeanScopeBase, createBeanDecorator } from 'vona';
+import { Scope } from 'vona-module-a-bean';
+import { Startup } from 'vona-module-a-startup';
+import * as Bull from 'bullmq';
+import { Service } from 'vona-module-a-web';
+
+var _dec$2, _dec2$2, _class$2;
+let StartupLoadQueueWorkers = (_dec$2 = Startup({
+  instance: true,
+  after: true
+}), _dec2$2 = BeanInfo({
+  module: "a-queue"
+}), _dec$2(_class$2 = _dec2$2(_class$2 = class StartupLoadQueueWorkers extends BeanBase {
+  async execute() {
+    // load queue workers
+    if (!this.app.meta.isTest) {
+      this.scope.service.queue.loadQueueWorkers(this.ctx.instanceName);
+    }
+  }
+}) || _class$2) || _class$2);
+
+function config(_app) {
+  return {
+    worker: {
+      lockDuration: 30 * 1000,
+      maxStalledCount: 1000,
+      stalledInterval: 10 * 1000
+    }
+  };
+}
+
+const __ThisModule__ = 'a-queue';
+
+class Monkey extends BeanSimple {
+  async appClose() {
+    const scopeSelf = this.bean.scope(__ThisModule__);
+    await scopeSelf.service.queue.clearWorkers();
+  }
+  async appClosed() {
+    const scopeSelf = this.bean.scope(__ThisModule__);
+    await scopeSelf.service.queue.clearQueues();
+  }
+}
+
+var _dec$1, _dec2$1, _class$1;
+let ServiceQueue = (_dec$1 = Service(), _dec2$1 = BeanInfo({
+  module: "a-queue"
+}), _dec$1(_class$1 = _dec2$1(_class$1 = class ServiceQueue extends BeanBase {
+  constructor(...args) {
+    super(...args);
+    this._workers = {};
+    this._queues = {};
+    this._queueCallbacks = {};
+  }
+  push(info) {
+    if (!info.options?.dbInfo) throw new Error('should specify the options.dbInfo');
+    this._queuePush(info, false);
+  }
+
+  // { locale, instanceName, module, queueName,queueNameSub,data }
+  pushAsync(info) {
+    if (!info.options?.dbInfo) throw new Error('should specify the options.dbInfo');
+    return this._queuePush(info, true);
+  }
+  loadQueueWorkers(instanceName) {
+    for (const queueItem of this.bean.onion.queue.getOnionsEnabled()) {
+      const info = {
+        queueName: queueItem.name,
+        data: undefined,
+        options: {
+          instanceName
+        }
+      };
+      this._ensureWorker(info);
+    }
+  }
+  async clearWorkers() {
+    for (const queueKey in this._workers) {
+      const _worker = this._workers[queueKey];
+      await _worker.worker.close();
+    }
+    this._workers = {};
+  }
+  async clearQueues() {
+    for (const queueKey in this._queues) {
+      const _queue = this._queues[queueKey];
+      await _queue.queue.close();
+      await _queue.queueEvents.close();
+    }
+    this._queues = {};
+  }
+  _createWorker(info, queueKey) {
+    const app = this.app;
+    // worker
+    const _worker = {};
+    // prefix
+    const prefix = `bull_${app.name}:queue`;
+    // queue config
+    const queueConfig = app.bean.onion.queue.getOnionOptions(info.queueName);
+    // queueConfig.options: queue/worker/job/redlock
+    const workerOptions = queueConfig?.options?.worker;
+    const redlockOptions = queueConfig?.options?.redlock;
+    const _redlockOptions = Object.assign({}, this.$scope.redlock.config.redlock.options, redlockOptions);
+    const _lockTTL = redlockOptions?.lockTTL ?? this.$scope.redlock.config.redlock.lockTTL;
+
+    // redlock
+    if (!queueConfig?.concurrency) {
+      _worker.redlock = this.$scope.redlock.service.redlock.create(_redlockOptions);
+    }
+
+    // create work
+    const connectionWorker = app.bean.redis.get('queue');
+    const _workerOptions = Object.assign({}, this.scope.config.worker, workerOptions, {
+      prefix,
+      connection: connectionWorker
+    });
+    _worker.workerOptions = _workerOptions;
+    _worker.worker = new Bull.Worker(queueKey, async job => {
+      // concurrency
+      if (queueConfig?.concurrency) {
+        return await this._performTask(job);
+      }
+      // redlock
+      const info = job.data;
+      const queueNameSub = info.options?.queueNameSub;
+      const _lockResource = `queue:${queueKey}${queueNameSub ? `#${queueNameSub}` : ''}`;
+      return await this.$scope.redlock.service.redlock.lock(_lockResource, async () => {
+        return await this._performTask(job);
+      }, {
+        // instanceName: job.data.instanceName, // need not
+        redlock: _worker.redlock,
+        lockTTL: _lockTTL
+      });
+    }, _workerOptions);
+    _worker.worker.on('failed', (_job, err) => {
+      this.$logger.error(err);
+    });
+    _worker.worker.on('error', err => {
+      if (err.message && err.message.includes('Missing lock for job')) {
+        const workerInner = _worker.worker;
+        if (!workerInner.running) {
+          _worker.worker.run().catch(err => {
+            this.$logger.error(err);
+          });
+        }
+      }
+    });
+
+    // ok
+    return _worker;
+  }
+  _createQueue(info, queueKey) {
+    const app = this.app;
+    // queue
+    const _queue = {};
+    // prefix
+    const prefix = `bull_${app.name}:queue`;
+    // queue config
+    const queueConfig = app.bean.onion.queue.getOnionOptions(info.queueName);
+    // queueConfig.options: queue/worker/job/limiter
+    const queueOptions = queueConfig?.options?.queue;
+
+    // create queue
+    const connectionQueue = app.bean.redis.get('queue');
+    const _queueOptions = Object.assign({}, queueOptions, {
+      prefix,
+      connection: connectionQueue
+    });
+    _queue.config = queueConfig;
+    _queue.options = _queueOptions;
+    _queue.queue = new Bull.Queue(queueKey, _queueOptions);
+
+    // create events
+    const connectionEvents = app.bean.redis.get('queue');
+    const _queueEventsOptions = {
+      prefix,
+      connection: connectionEvents
+    };
+    _queue.queueEventsOptions = _queueEventsOptions;
+    _queue.queueEvents = new Bull.QueueEvents(queueKey, _queueEventsOptions);
+    _queue.queueEvents.on('completed', ({
+      jobId,
+      returnvalue
+    }) => {
+      this._callCallback(jobId, undefined, returnvalue);
+    });
+    _queue.queueEvents.on('failed', ({
+      jobId,
+      failedReason
+    }) => {
+      this._callCallback(jobId, failedReason, undefined);
+    });
+
+    // ok
+    return _queue;
+  }
+  _ensureWorker(info) {
+    // queueKey
+    const queueKey = this._combineQueueKey(info);
+    // worker
+    if (!this._workers[queueKey]) {
+      this._workers[queueKey] = this._createWorker(info, queueKey);
+    }
+  }
+  _ensureQueue(info) {
+    // worker
+    this._ensureWorker(info);
+    // queueKey
+    const queueKey = this._combineQueueKey(info);
+    // queue
+    if (!this._queues[queueKey]) {
+      this._queues[queueKey] = this._createQueue(info, queueKey);
+    }
+    // ok
+    return this._queues[queueKey];
+  }
+  getQueue(queueName, instanceName) {
+    return this._getQueue({
+      queueName,
+      data: undefined,
+      options: {
+        instanceName: instanceName ?? this.ctx.instanceName
+      }
+    });
+  }
+  _getQueue(info) {
+    return this._ensureQueue(info).queue;
+  }
+  _callCallback(jobId, failedReason, data) {
+    const _callback = this._queueCallbacks[jobId];
+    if (_callback) {
+      delete this._queueCallbacks[jobId];
+      _callback.callback(failedReason ? new Error(failedReason) : undefined, data);
+    }
+  }
+  async _queuePush(info, isAsync) {
+    // queue config
+    const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
+    // queueConfig.options: queue/worker/job/limiter
+    const jobOptionsBase = queueConfig?.options?.job;
+    // queue
+    const queue = this._getQueue(info);
+    // job
+    const jobId = info.options?.jobOptions?.jobId || uuidv4();
+    const jobName = info.options?.jobName || jobId;
+    const jobOptions = deepExtend({
+      jobId
+    }, jobOptionsBase, info.options?.jobOptions);
+    // should not change info, hold original info.options?.jobName, info.options?.jobOptions
+    // info = deepExtend({}, info, { options: { jobName, jobOptions } });
+    // not async
+    if (!isAsync) {
+      // add job
+      queue.add(jobName, info, jobOptions);
+      return undefined;
+    }
+    // async
+    return new Promise((resolve, reject) => {
+      // callback
+      this._queueCallbacks[jobId] = {
+        info,
+        callback: (err, data) => {
+          if (err) return reject(err);
+          resolve(data);
+        }
+      };
+      // add job
+      return queue.add(jobName, info, jobOptions);
+    });
+  }
+  _combineQueueKey(info) {
+    const instanceName = instanceDesp(info.options?.instanceName);
+    return `${instanceName}||${beanFullNameFromOnionName(info.queueName, 'queue')}`;
+  }
+  async _performTask(job) {
+    const info = job.data;
+    // queue config
+    const queueItem = this.bean.onion.queue.getOnionSlice(info.queueName);
+    const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
+    // execute
+    return await this.bean.executor.newCtx(async () => {
+      const beanFullName = queueItem.beanOptions.beanFullName;
+      const beanInstance = this.app.bean._getBean(beanFullName);
+      return await beanInstance.execute(info.data, info.options, job);
+    }, {
+      dbInfo: info.options?.dbInfo,
+      locale: info.options?.locale,
+      instanceName: info.options?.instanceName,
+      extraData: info.options?.extraData,
+      transaction: queueConfig?.transaction
+    });
+  }
+  getRepeatKey(jobName, repeat) {
+    const endDate = repeat.endDate ? new Date(repeat.endDate).getTime() : '';
+    const tz = repeat.tz || '';
+    const pattern = repeat.pattern;
+    const suffix = pattern || String(repeat.every) || '';
+    return `${jobName}:${endDate}:${tz}:${suffix}`;
+  }
+  prepareJobInfo(queueName, data, options) {
+    options = this.$scope.executor.service.executor.prepareGeneralInfo(options);
+    // info
+    return {
+      queueName,
+      data,
+      options
+    };
+  }
+}) || _class$1) || _class$1);
+
+var _dec, _dec2, _class;
+let ScopeModuleAQueue = (_dec = Scope(), _dec2 = BeanInfo({
+  module: "a-queue"
+}), _dec(_class = _dec2(_class = class ScopeModuleAQueue extends BeanScopeBase {}) || _class) || _class);
+
+/** scope: end */
+
+class BeanQueueBase extends BeanBase {
+  async pushAsync(data, options) {
+    return await this.$scope.queue.service.queue.pushAsync(this.$scope.queue.service.queue.prepareJobInfo(this.$onionName, data, options));
+  }
+  push(data, options) {
+    return this.$scope.queue.service.queue.push(this.$scope.queue.service.queue.prepareJobInfo(this.$onionName, data, options));
+  }
+  getQueue(instanceName) {
+    return this.$scope.queue.service.queue.getQueue(this.$onionName, instanceName);
+  }
+}
+
+function Queue(options) {
+  return createBeanDecorator('queue', options);
+}
+
+export { BeanQueueBase, Monkey, Queue, ScopeModuleAQueue, ServiceQueue, StartupLoadQueueWorkers, config };

package/dist/lib/beanQueueBase.d.ts

@@ -0,0 +1,7 @@
+import type { IQueuePushOptions } from '../types/queue.ts';
+import { BeanBase } from 'vona';
+export declare class BeanQueueBase<DATA = unknown, RESULT = unknown> extends BeanBase {
+    pushAsync(data: DATA, options?: IQueuePushOptions): Promise<RESULT>;
+    push(data: DATA, options?: IQueuePushOptions): void;
+    getQueue(instanceName?: string): import("bullmq").Queue<any, any, string, any, any, string>;
+}

package/dist/lib/index.d.ts

@@ -0,0 +1,2 @@
+export * from './beanQueueBase.ts';
+export * from './queue.ts';

package/dist/lib/queue.d.ts

@@ -0,0 +1,2 @@
+import type { IDecoratorQueueOptions } from '../types/queue.ts';
+export declare function Queue(options?: IDecoratorQueueOptions): ClassDecorator;

package/dist/monkey.d.ts

@@ -0,0 +1,6 @@
+import type { IMonkeyAppClose, IMonkeyAppClosed } from 'vona';
+import { BeanSimple } from 'vona';
+export declare class Monkey extends BeanSimple implements IMonkeyAppClose, IMonkeyAppClosed {
+    appClose(): Promise<void>;
+    appClosed(): Promise<void>;
+}

package/dist/service/queue.d.ts

@@ -0,0 +1,25 @@
+import type { IQueueCallbacks, IQueueJobContext, IQueuePushOptions, IQueueQueue, IQueueQueues, IQueueRecord, IQueueWork, IQueueWorks, TypeQueueJob } from '../types/queue.ts';
+import * as Bull from 'bullmq';
+import { BeanBase } from 'vona';
+export declare class ServiceQueue extends BeanBase {
+    _workers: IQueueWorks;
+    _queues: IQueueQueues;
+    _queueCallbacks: IQueueCallbacks;
+    push<DATA>(info: IQueueJobContext<DATA>): void;
+    pushAsync<DATA, RESULT>(info: IQueueJobContext<DATA>): Promise<RESULT>;
+    loadQueueWorkers(instanceName?: string | undefined | null): void;
+    clearWorkers(): Promise<void>;
+    clearQueues(): Promise<void>;
+    _createWorker<DATA>(info: IQueueJobContext<DATA>, queueKey: string): IQueueWork;
+    _createQueue<DATA>(info: IQueueJobContext<DATA>, queueKey: string): IQueueQueue;
+    _ensureWorker<DATA>(info: IQueueJobContext<DATA>): void;
+    _ensureQueue<DATA>(info: IQueueJobContext<DATA>): IQueueQueue;
+    getQueue(queueName: keyof IQueueRecord, instanceName?: string | undefined | null): Bull.Queue<any, any, string, any, any, string>;
+    _getQueue<DATA>(info: IQueueJobContext<DATA>): Bull.Queue<any, any, string, any, any, string>;
+    _callCallback<DATA>(jobId: string | number, failedReason: string | undefined, data: DATA | undefined): void;
+    _queuePush<DATA, RESULT>(info: IQueueJobContext<DATA>, isAsync: boolean): Promise<RESULT>;
+    _combineQueueKey<DATA>(info: IQueueJobContext<DATA>): string;
+    _performTask<DATA, RESULT>(job: TypeQueueJob<DATA, RESULT>): Promise<unknown>;
+    getRepeatKey(jobName: string, repeat: Bull.RepeatOptions): string;
+    prepareJobInfo<DATA>(queueName: keyof IQueueRecord, data: DATA, options?: IQueuePushOptions): IQueueJobContext<DATA>;
+}

package/dist/types/index.d.ts

@@ -0,0 +1 @@
+export * from './queue.ts';

package/dist/types/queue.d.ts

@@ -0,0 +1,71 @@
+import type Redlock from '@sesamecare-oss/redlock';
+import type * as Bull from 'bullmq';
+import type { OmitNever } from 'vona';
+import type { IGeneralInfoOptions } from 'vona-module-a-executor';
+import type { IOnionOptionsEnable, ServiceOnion } from 'vona-module-a-onion';
+export interface IQueuePushOptions extends IGeneralInfoOptions {
+    queueNameSub?: string;
+    jobName?: string;
+    jobOptions?: Bull.JobsOptions;
+}
+export type TypeQueueJob<DATA = unknown, RESULT = unknown> = Bull.Job<IQueueJobContext<DATA>, RESULT>;
+export interface IQueueExecute<DATA = unknown, RESULT = unknown> {
+    execute: (data: DATA, options?: IQueuePushOptions, job?: TypeQueueJob<DATA, RESULT>) => Promise<RESULT>;
+}
+export interface IQueueJobContext<DATA> {
+    queueName: keyof IQueueRecord;
+    data: DATA;
+    options?: IQueuePushOptions;
+}
+export interface IQueueWork {
+    redlock: Redlock.Redlock;
+    worker: Bull.Worker;
+    workerOptions: Bull.WorkerOptions;
+}
+export interface IQueueWorks {
+    [queueKey: string]: IQueueWork;
+}
+export interface IQueueQueue {
+    config?: IDecoratorQueueOptions;
+    options: Bull.QueueOptions;
+    queue: Bull.Queue;
+    queueEvents: Bull.QueueEvents;
+    queueEventsOptions: Bull.QueueEventsOptions;
+}
+export interface IQueueQueues {
+    [queueKey: string]: IQueueQueue;
+}
+export interface IQueueCallback<DATA, RESULT> {
+    info: IQueueJobContext<DATA>;
+    callback: (err: Error | undefined, data: RESULT | undefined) => void;
+}
+export interface IQueueCallbacks {
+    [jobId: string | number]: IQueueCallback<unknown, unknown>;
+}
+export interface IQueueRecord {
+}
+export interface IDecoratorQueueOptions extends IOnionOptionsEnable {
+    concurrency?: boolean;
+    transaction?: boolean;
+    options?: {
+        queue?: Bull.QueueOptions;
+        worker?: Partial<Bull.WorkerOptions>;
+        redlock?: Redlock.Redlock & {
+            lockTTL?: number;
+        };
+        job?: Bull.JobsOptions;
+    };
+}
+declare module 'vona-module-a-onion' {
+    interface BeanOnion {
+        queue: ServiceOnion<IDecoratorQueueOptions, keyof IQueueRecord>;
+    }
+}
+declare module 'vona' {
+    interface ConfigOnions {
+        queue: OmitNever<IQueueRecord>;
+    }
+    interface IBeanSceneRecord {
+        queue: never;
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vona-module-a-queue",
   "type": "module",
-  "version": "5.0.9",
+  "version": "5.0.11",
   "title": "a-queue",
   "vonaModule": {
     "dependencies": {},
@@ -43,8 +43,12 @@
   "dependencies": {
     "bullmq": "^5.49.1"
   },
+  "devDependencies": {
+    "clean-package": "^2.2.0",
+    "rimraf": "^6.0.1"
+  },
   "scripts": {
-    "clean": "rimraf dist tsconfig.tsbuildinfo",
-    "tsc:publish": "npm run clean && tsc"
+    "clean": "rimraf dist tsconfig.build.tsbuildinfo",
+    "tsc:publish": "npm run clean && vona :bin:buildModule && tsc -p tsconfig.build.json"
   }
 }