plugin-cluster-manager 1.1.10 → 1.1.13

package/dist/server/actions/orchestrator.js

@@ -122,6 +122,10 @@ const orchestratorActions = {
    * POST /workerOrchestrator:scale
    * Body: { stackId: 1, replicas: 3 }
    * Leader-only
+   *
+   * Before scaling, resolves queue-to-stack mappings and injects
+   * WORKER_MODE into the stack's envVars so new containers only
+   * process assigned queues.
    */
   async scale(ctx, next) {
     assertLeader(ctx);
@@ -130,6 +134,39 @@ const orchestratorActions = {
     if (replicas === void 0 || replicas === null) ctx.throw(400, "replicas is required");
     if (replicas < 0 || replicas > 20) ctx.throw(400, "replicas must be between 0 and 20");
     const stack = await getStack(ctx, stackId);
+    try {
+      const mappingsRepo = ctx.db.getRepository("workerQueueMappings");
+      if (mappingsRepo) {
+        const assigned = await mappingsRepo.find({
+          filter: {
+            stackId: stack.id,
+            enabled: true
+          }
+        });
+        const queueNames = assigned.map((m) => m.get("queueName")).filter(Boolean);
+        if (queueNames.length > 0) {
+          const workerMode = queueNames.join(",");
+          ctx.app.logger.info(
+            `[Orchestrator] Injecting WORKER_MODE=${workerMode} for stack "${stack.name}" (${queueNames.length} queue(s) assigned)`
+          );
+          stack.envVars = {
+            ...stack.envVars || {},
+            WORKER_MODE: workerMode
+          };
+        } else {
+          stack.envVars = {
+            ...stack.envVars || {},
+            WORKER_MODE: "*"
+          };
+        }
+      }
+    } catch (err) {
+      ctx.app.logger.debug(`[Orchestrator] Queue mappings not available: ${err.message}`);
+      stack.envVars = {
+        ...stack.envVars || {},
+        WORKER_MODE: "*"
+      };
+    }
     const result = await adapter.scale(stack, Number(replicas));
     const repo = ctx.db.getRepository("orchestratorStacks");
     await repo.update({

package/dist/server/actions/queue-mappings.js

@@ -0,0 +1,107 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var queue_mappings_exports = {};
+__export(queue_mappings_exports, {
+  queueMappingsActions: () => queueMappingsActions
+});
+module.exports = __toCommonJS(queue_mappings_exports);
+var import_queue_scanner = require("../queue-scanner");
+const queueMappingsActions = {
+  /**
+   * GET /workerQueueMappings:scanQueues
+   * Scans all registered queues (EventQueue + Redis) and merges with existing mappings.
+   */
+  async scanQueues(ctx, next) {
+    const result = await (0, import_queue_scanner.scanQueues)(ctx.app);
+    const repo = ctx.db.getRepository("workerQueueMappings");
+    let existingMappings = [];
+    try {
+      existingMappings = await repo.find();
+    } catch {
+    }
+    const mappedNames = new Set(existingMappings.map((m) => m.get("queueName")));
+    ctx.body = {
+      discovered: result.queues,
+      total: result.total,
+      registered: existingMappings.map((m) => ({
+        id: m.get("id"),
+        queueName: m.get("queueName"),
+        label: m.get("label"),
+        stackId: m.get("stackId"),
+        enabled: m.get("enabled"),
+        type: m.get("type")
+      })),
+      unmapped: result.queues.filter((q) => !mappedNames.has(q.name)).map((q) => ({
+        name: q.name,
+        type: q.type,
+        label: q.label,
+        description: q.description
+      }))
+    };
+    await next();
+  },
+  /**
+   * POST /workerQueueMappings:autoMap
+   * Auto-create mappings for any discovered queues that don't have one yet.
+   * Body: { stackId?: number } — optional default stack for new mappings
+   */
+  async autoMap(ctx, next) {
+    const { stackId } = ctx.action.params.values || {};
+    const result = await (0, import_queue_scanner.scanQueues)(ctx.app);
+    const repo = ctx.db.getRepository("workerQueueMappings");
+    let existingMappings = [];
+    try {
+      existingMappings = await repo.find();
+    } catch {
+    }
+    const mappedNames = new Set(existingMappings.map((m) => m.get("queueName")));
+    const created = [];
+    for (const q of result.queues) {
+      if (mappedNames.has(q.name)) continue;
+      await repo.create({
+        values: {
+          queueName: q.name,
+          label: q.label,
+          description: q.description,
+          type: q.type,
+          stackId: stackId || null,
+          enabled: true
+        }
+      });
+      created.push(q.name);
+    }
+    ctx.body = {
+      created,
+      count: created.length
+    };
+    await next();
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  queueMappingsActions
+});

package/dist/server/collections/cluster-manager-doctor-runs.js

@@ -0,0 +1,52 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var cluster_manager_doctor_runs_exports = {};
+__export(cluster_manager_doctor_runs_exports, {
+  default: () => cluster_manager_doctor_runs_default
+});
+module.exports = __toCommonJS(cluster_manager_doctor_runs_exports);
+var cluster_manager_doctor_runs_default = {
+  name: "clusterManagerDoctorRuns",
+  title: "Cluster Manager Doctor Runs",
+  fields: [
+    { name: "id", type: "bigInt", autoIncrement: true, primaryKey: true },
+    { name: "runId", type: "string", length: 64, unique: true, allowNull: false },
+    { name: "status", type: "string", length: 20, defaultValue: "running", allowNull: false },
+    { name: "durationMs", type: "integer", defaultValue: 12e4, allowNull: false },
+    { name: "progress", type: "integer", defaultValue: 0, allowNull: false },
+    { name: "startedAt", type: "date", allowNull: false },
+    { name: "deadlineAt", type: "date", allowNull: false },
+    { name: "finishedAt", type: "date", allowNull: true },
+    { name: "finishReason", type: "string", length: 40, allowNull: true },
+    { name: "startedBy", type: "string", length: 200, allowNull: true },
+    { name: "summary", type: "json", allowNull: true },
+    { name: "report", type: "json", allowNull: true },
+    { name: "error", type: "text", allowNull: true },
+    { name: "createdAt", type: "date" },
+    { name: "updatedAt", type: "date" }
+  ]
+};

package/dist/server/collections/cluster-manager-doctor.js

@@ -0,0 +1,44 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var cluster_manager_doctor_exports = {};
+__export(cluster_manager_doctor_exports, {
+  default: () => cluster_manager_doctor_default
+});
+module.exports = __toCommonJS(cluster_manager_doctor_exports);
+var cluster_manager_doctor_default = {
+  name: "clusterManagerDoctor",
+  dumpRules: "skip",
+  autoGenId: true,
+  createdAt: false,
+  updatedAt: false,
+  fields: [
+    {
+      name: "name",
+      type: "string"
+    }
+  ]
+};

package/dist/server/collections/worker-queue-mappings.js

@@ -0,0 +1,106 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var worker_queue_mappings_exports = {};
+__export(worker_queue_mappings_exports, {
+  default: () => worker_queue_mappings_default
+});
+module.exports = __toCommonJS(worker_queue_mappings_exports);
+var worker_queue_mappings_default = {
+  name: "workerQueueMappings",
+  autoGenId: true,
+  createdAt: true,
+  updatedAt: true,
+  fields: [
+    {
+      name: "queueName",
+      type: "string",
+      unique: true,
+      interface: "input",
+      uiSchema: {
+        title: "Queue Name",
+        "x-component": "Input",
+        required: true
+      }
+    },
+    {
+      name: "label",
+      type: "string",
+      interface: "input",
+      uiSchema: {
+        title: "Label",
+        "x-component": "Input",
+        description: "Human-readable display name"
+      }
+    },
+    {
+      name: "description",
+      type: "text",
+      interface: "textarea",
+      uiSchema: {
+        title: "Description",
+        "x-component": "Input.TextArea"
+      }
+    },
+    {
+      name: "type",
+      type: "string",
+      interface: "select",
+      defaultValue: "event-queue",
+      uiSchema: {
+        title: "Source Type",
+        "x-component": "Select",
+        enum: [
+          { value: "event-queue", label: "EventQueue" },
+          { value: "redis-list", label: "Redis List" }
+        ]
+      }
+    },
+    {
+      name: "stackId",
+      type: "integer",
+      interface: "select",
+      uiSchema: {
+        title: "Assigned Stack",
+        "x-component": "Select",
+        "x-component-props": {
+          allowClear: true,
+          placeholder: "Unassigned (worker runs all queues)"
+        }
+      }
+    },
+    {
+      name: "enabled",
+      type: "boolean",
+      defaultValue: true,
+      interface: "checkbox",
+      uiSchema: {
+        title: "Enabled",
+        "x-component": "Checkbox"
+      }
+    }
+  ]
+};

package/dist/server/hooks/cacheInvalidationHooks.js

@@ -0,0 +1,81 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var cacheInvalidationHooks_exports = {};
+__export(cacheInvalidationHooks_exports, {
+  registerCacheHooks: () => registerCacheHooks
+});
+module.exports = __toCommonJS(cacheInvalidationHooks_exports);
+var import_versionManager = require("../utils/versionManager");
+function registerCacheHooks(app) {
+  const db = app.db;
+  db.on("collections.afterSave", async () => {
+    await import_versionManager.cacheVersionManager.incrementCollectionVersion(app);
+  });
+  db.on("collections.afterDestroy", async () => {
+    await import_versionManager.cacheVersionManager.incrementCollectionVersion(app);
+  });
+  db.on("fields.afterSave", async () => {
+    await import_versionManager.cacheVersionManager.incrementCollectionVersion(app);
+  });
+  db.on("fields.afterDestroy", async () => {
+    await import_versionManager.cacheVersionManager.incrementCollectionVersion(app);
+  });
+  db.on("uiSchemas.afterSave", async () => {
+    await import_versionManager.cacheVersionManager.incrementSchemaVersion(app);
+  });
+  db.on("uiSchemas.afterDestroy", async () => {
+    await import_versionManager.cacheVersionManager.incrementSchemaVersion(app);
+  });
+  const invalidateRole = async (model) => {
+    var _a, _b;
+    const roleName = ((_a = model.get) == null ? void 0 : _a.call(model, "roleName")) || ((_b = model.get) == null ? void 0 : _b.call(model, "name"));
+    if (roleName) {
+      await import_versionManager.cacheVersionManager.incrementAclVersion(app, roleName);
+    } else {
+      await import_versionManager.cacheVersionManager.incrementAllAclVersions(app);
+    }
+  };
+  db.on("roles.afterSave", invalidateRole);
+  db.on("roles.afterDestroy", invalidateRole);
+  db.on("rolesResources.afterSave", invalidateRole);
+  db.on("rolesResources.afterDestroy", invalidateRole);
+  db.on("rolesResourcesActions.afterSave", invalidateRole);
+  db.on("rolesResourcesActions.afterDestroy", invalidateRole);
+  db.on("rolesUsers.afterSave", invalidateRole);
+  db.on("rolesUsers.afterDestroy", invalidateRole);
+  db.on("scopes.afterSave", async () => {
+    await import_versionManager.cacheVersionManager.incrementAllAclVersions(app);
+  });
+  db.on("scopes.afterDestroy", async () => {
+    await import_versionManager.cacheVersionManager.incrementAllAclVersions(app);
+  });
+  app.logger.info("[ClusterManager] Cache invalidation hooks registered successfully");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  registerCacheHooks
+});

package/dist/server/middlewares/listMetaCacheMiddleware.js

@@ -0,0 +1,79 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var listMetaCacheMiddleware_exports = {};
+__export(listMetaCacheMiddleware_exports, {
+  createListMetaCacheMiddleware: () => createListMetaCacheMiddleware
+});
+module.exports = __toCommonJS(listMetaCacheMiddleware_exports);
+var import_versionManager = require("../utils/versionManager");
+const LIST_META_CACHE_TTL = 1e3 * 60 * 10;
+function getErrorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function createListMetaCacheMiddleware(app) {
+  return async function listMetaCacheMiddleware(ctx, next) {
+    var _a, _b, _c, _d, _e, _f, _g;
+    const cache = app.cache;
+    if (!cache || ((_a = ctx.action) == null ? void 0 : _a.resourceName) !== "collections" || ((_b = ctx.action) == null ? void 0 : _b.actionName) !== "listMeta") {
+      return next();
+    }
+    const currentRole = ((_c = ctx.state) == null ? void 0 : _c.currentRole) || "anonymous";
+    const appName = ctx.headers["x-app"] || "main";
+    const dataSource = ctx.headers["x-data-source"] || "main";
+    const locale = ctx.headers["x-locale"] || ctx.headers["accept-language"] || "en-US";
+    let cacheKey = "";
+    let version = 0;
+    try {
+      version = await import_versionManager.cacheVersionManager.getCollectionVersion(app);
+      cacheKey = `nb:cache:${appName}:meta:v${version}:ds:${dataSource}:role:${currentRole}:lang:${locale}`;
+      const cached = await cache.get(cacheKey);
+      if (cached !== void 0 && cached !== null) {
+        ctx.body = typeof cached === "string" ? JSON.parse(cached) : cached;
+        (_d = ctx.set) == null ? void 0 : _d.call(ctx, "X-Cache", "HIT");
+        (_e = ctx.set) == null ? void 0 : _e.call(ctx, "X-Collection-Version", String(version));
+        return;
+      }
+    } catch (err) {
+      app.logger.warn(`[ClusterManager] listMeta cache read skipped: ${getErrorMessage(err)}`);
+    }
+    await next();
+    if (ctx.status === 200 && ctx.body && cacheKey) {
+      try {
+        const valueToCache = typeof ctx.body === "string" ? ctx.body : JSON.stringify(ctx.body);
+        await cache.set(cacheKey, valueToCache, LIST_META_CACHE_TTL);
+        (_f = ctx.set) == null ? void 0 : _f.call(ctx, "X-Cache", "MISS");
+        (_g = ctx.set) == null ? void 0 : _g.call(ctx, "X-Collection-Version", String(version));
+      } catch (err) {
+        app.logger.warn(`[ClusterManager] listMeta cache write skipped: ${getErrorMessage(err)}`);
+      }
+    }
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createListMetaCacheMiddleware
+});

package/dist/server/orchestrator/PackageManager.js

@@ -41,6 +41,7 @@ __export(PackageManager_exports, {
 module.exports = __toCommonJS(PackageManager_exports);
 var import_child_process = require("child_process");
 var import_redis = require("../utils/redis");
+var import_node = require("../utils/node");
 var import_fs = require("fs");
 var import_path = __toESM(require("path"));
 const SAFE_PKG_RE = /^(?:[a-zA-Z0-9_.@/-]|\[|\])+$/;
@@ -84,14 +85,7 @@ function redactUrl(value) {
   }
 }
 function getCurrentRole() {
-  if (process.env.APP_ROLE === "app" || process.env.APP_ROLE === "worker" || process.env.APP_ROLE === "sandbox") {
-    return process.env.APP_ROLE;
-  }
-  if (process.env.SKILL_HUB_SANDBOX === "true") {
-    return "sandbox";
-  }
-  const workerMode = process.env.WORKER_MODE || "main";
-  return workerMode === "worker" || workerMode === "task" || workerMode === "*" ? "worker" : "app";
+  return (0, import_node.getLocalRole)();
 }
 function formatCommand(command, args) {
   return [command, ...args].map((part) => {
@@ -408,22 +402,25 @@ ${logs.join("\n")}`);
       const redisClient = (0, import_redis.getRedisClient)(this.app);
       const podName = process.env.POD_NAME || require("os").hostname();
       if (redisClient) {
-        const key = `orchestrator:pkg-status:${podName}`;
-        await redisClient.sendCommand([
-          "SET",
-          key,
-          JSON.stringify({
-            initStatus,
-            initProgressPercent,
-            initProgressLog,
-            lastInitAt: /* @__PURE__ */ new Date(),
-            lastInitLog: logs.join("\n"),
-            ...extraValues
-          }),
-          "EX",
-          "86400"
-          // expire after 1 day
-        ]);
+        const statusPayload = JSON.stringify({
+          initStatus,
+          initProgressPercent,
+          initProgressLog,
+          lastInitAt: /* @__PURE__ */ new Date(),
+          lastInitLog: logs.join("\n"),
+          ...extraValues
+        });
+        const keys = [`orchestrator:pkg-status:${podName}`, `cluster-manager:pkg-status:${(0, import_node.getLocalNodeId)(this.app)}`];
+        for (const key of keys) {
+          await redisClient.sendCommand([
+            "SET",
+            key,
+            statusPayload,
+            "EX",
+            "86400"
+            // expire after 1 day
+          ]);
+        }
       }
       const repo = this.app.db.getRepository("workerPackagesConfigs");
       let config = await repo.findOne();