plugin-cluster-manager 1.1.7 → 1.1.11

package/dist/server/orchestrator/leader-election.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Leader Election using native Redis commands
+ *
+ * Ensures only ONE app instance runs orchestrator write operations (scale, start, stop).
+ * Other instances remain read-only followers.
+ *
+ * Flow:
+ *   1. After app starts, tryBecomeLeader() attempts to acquire the Redis lock using SET NX PX.
+ *   2. If acquired → this node is leader; a renewal timer extends the lock periodically using a Lua script.
+ *   3. If not acquired → this node is follower; it retries periodically.
+ *   4. On app stop → release the lock gracefully using a Lua script.
+ *   5. On crash → lock auto-expires after TTL.
+ */
+export declare class LeaderElection {
+    private app;
+    private renewTimer;
+    private retryTimer;
+    private _isLeader;
+    private _leaderId;
+    private standaloneMode;
+    private enabled;
+    private _disabledReason;
+    private redis;
+    constructor(app: any, options?: {
+        standaloneMode?: boolean;
+        enabled?: boolean;
+        disabledReason?: string;
+    });
+    get isLeader(): boolean;
+    get leaderId(): string;
+    get disabledReason(): string;
+    /**
+     * Initialize Redis client. Must be called after Redis is connected.
+     */
+    init(): Promise<boolean>;
+    /**
+     * Attempt to become the orchestrator leader.
+     * If successful, starts a renewal loop.
+     * If not, starts a retry loop.
+     */
+    tryBecomeLeader(): Promise<boolean>;
+    /**
+     * Gracefully release leadership.
+     */
+    release(): Promise<void>;
+    private startRenewal;
+    private startRetry;
+}

package/dist/server/orchestrator/types.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Orchestrator Adapter Interface
+ *
+ * Abstraction layer for container runtimes (Docker, K8s, Swarm).
+ * Each adapter implements this interface so the plugin logic is runtime-agnostic.
+ */
+export interface ContainerInfo {
+    id: string;
+    name: string;
+    status: 'running' | 'stopped' | 'pending' | 'error' | 'creating' | 'exited';
+    image: string;
+    createdAt: Date;
+    cpu?: number;
+    memory?: number;
+    labels?: Record<string, string>;
+}
+export interface ScaleResult {
+    previousReplicas: number;
+    currentReplicas: number;
+    containersCreated?: string[];
+    containersRemoved?: string[];
+}
+export interface ContainerStats {
+    cpu: number;
+    memory: number;
+    memoryLimit: number;
+    networkRx: number;
+    networkTx: number;
+}
+export interface StackConfig {
+    id?: number;
+    name: string;
+    adapter: 'docker' | 'kubernetes';
+    image: string;
+    command?: string;
+    envVars?: Record<string, string>;
+    volumes?: string[];
+    networks?: string[];
+    resourceLimits?: {
+        memory?: string;
+        cpu?: string;
+    };
+    replicas: number;
+    desiredReplicas: number;
+    enabled: boolean;
+    namespace?: string;
+    deploymentName?: string;
+    serviceAccountName?: string;
+    imagePullPolicy?: string;
+    k8sContainerName?: string;
+    k8sEnv?: Record<string, any>[];
+    k8sEnvFrom?: Record<string, any>[];
+    k8sVolumeMounts?: Record<string, any>[];
+    k8sVolumes?: Record<string, any>[];
+    networkMode?: string;
+    restartPolicy?: string;
+}
+export interface IOrchestratorAdapter {
+    /** Human-readable adapter name */
+    readonly name: string;
+    /** Test connectivity to the runtime */
+    ping(): Promise<boolean>;
+    /** List containers managed by a stack */
+    listContainers(stack: StackConfig): Promise<ContainerInfo[]>;
+    /** Verify that a runtime container/pod belongs to the requested stack */
+    assertManagedByStack(stack: StackConfig, containerId: string): Promise<void>;
+    /** Scale stack to N replicas */
+    scale(stack: StackConfig, replicas: number): Promise<ScaleResult>;
+    /** Start a stopped container */
+    startContainer(containerId: string): Promise<void>;
+    /** Gracefully stop a running container */
+    stopContainer(containerId: string, timeoutSecs?: number): Promise<void>;
+    /** Remove/delete a container (force) */
+    removeContainer(containerId: string): Promise<void>;
+    /** Get real-time resource stats for a container */
+    getStats(containerId: string): Promise<ContainerStats>;
+    /** Get tail logs from a container */
+    getLogs(containerId: string, tail?: number): Promise<string>;
+    /** List available networks (if supported by adapter) */
+    listNetworks?(): Promise<{
+        id: string;
+        name: string;
+    }[]>;
+}

package/dist/server/plugin.d.ts

@@ -0,0 +1,26 @@
+import { Plugin } from '@nocobase/server';
+import { RedisNodeRegistry } from './adapters/redis-node-registry';
+import type { IOrchestratorAdapter } from './orchestrator/types';
+import { LeaderElection } from './orchestrator/leader-election';
+export declare class PluginClusterManagerServer extends Plugin {
+    nodeRegistry: RedisNodeRegistry;
+    orchestrator: IOrchestratorAdapter | null;
+    leaderElection: LeaderElection | null;
+    beforeLoad(): Promise<void>;
+    load(): Promise<void>;
+    private registerPubSubAdapter;
+    /**
+     * Initialize the Container Orchestrator subsystem.
+     * Config is loaded from DB (orchestratorSettings collection) first,
+     * then falls back to ORCHESTRATOR_ADAPTER env var.
+     * This allows manual configuration via the NocoBase admin UI.
+     */
+    private initOrchestrator;
+    /**
+     * Connect (or reconnect) the orchestrator adapter based on settings.
+     * Can be called at startup or when user saves new settings via UI.
+     */
+    connectAdapter(settings: any): Promise<boolean>;
+    private isWorkerOnlyNode;
+}
+export default PluginClusterManagerServer;

package/dist/server/plugin.js

@@ -63,6 +63,9 @@ var import_k8s_adapter = require("./orchestrator/k8s-adapter");
 var import_leader_election = require("./orchestrator/leader-election");
 var import_package_manager = require("./actions/package-manager");
 var import_PackageManager = require("./orchestrator/PackageManager");
+var import_listMetaCacheMiddleware = require("./middlewares/listMetaCacheMiddleware");
+var import_cacheInvalidationHooks = require("./hooks/cacheInvalidationHooks");
+var import_doctor = require("./actions/doctor");
 class PluginClusterManagerServer extends import_server.Plugin {
   nodeRegistry;
   orchestrator = null;
@@ -71,6 +74,15 @@ class PluginClusterManagerServer extends import_server.Plugin {
     await this.db.import({ directory: import_path.default.resolve(__dirname, "collections") });
   }
   async load() {
+    this.db.extendCollection({
+      name: "attachments",
+      fields: [
+        {
+          type: "bigInt",
+          name: "createdById"
+        }
+      ]
+    });
     this.nodeRegistry = new import_redis_node_registry.RedisNodeRegistry(this.app);
     this.app.on("afterStart", () => {
       var _a;
@@ -98,7 +110,7 @@ class PluginClusterManagerServer extends import_server.Plugin {
               try {
                 const customRaw = config.get("customPackages");
                 if (customRaw) custom = typeof customRaw === "string" ? JSON.parse(customRaw) : customRaw;
-              } catch {
+              } catch (err) {
               }
               const unique = (arr) => Array.from(new Set(arr.filter(Boolean)));
               const packages = {
@@ -161,16 +173,51 @@ class PluginClusterManagerServer extends import_server.Plugin {
           if (!redis || !requestId) return;
           const logData = await (0, import_cluster_nodes.readLocalLogs)(this.app, lines || 200);
           const responseKey = `cluster-manager:log-response:${requestId}`;
+          await redis.sendCommand(["SET", responseKey, JSON.stringify(logData), "EX", "30"]);
+          this.app.logger.debug(`[ClusterManager] Served log request ${requestId} for ${targetNodeId}`);
+        } catch (err) {
+          this.app.logger.error(`[ClusterManager] Error handling log request: ${err.message}`);
+        }
+      });
+      pubSub.subscribe(`cluster-manager:doctor-collect:${myNodeId}`, async (msg) => {
+        const redis = (0, import_redis.getRedisClient)(this.app);
+        let requestId = "";
+        try {
+          const parsed = typeof msg === "string" ? JSON.parse(msg) : msg;
+          requestId = parsed.requestId;
+          if (!redis || !requestId) return;
+          const snapshot = await (0, import_doctor.collectLocalDoctorSnapshot)(this.app, {
+            runId: parsed.runId,
+            sinceMs: parsed.sinceMs,
+            untilMs: parsed.untilMs,
+            maxLines: parsed.maxLines
+          });
           await redis.sendCommand([
             "SET",
-            responseKey,
-            JSON.stringify(logData),
+            `cluster-manager:doctor-response:${requestId}`,
+            JSON.stringify(snapshot),
             "EX",
-            "30"
+            "90"
           ]);
-          this.app.logger.debug(`[ClusterManager] Served log request ${requestId} for ${targetNodeId}`);
+          this.app.logger.debug(`[ClusterManager] Served doctor snapshot request ${requestId}`);
         } catch (err) {
-          this.app.logger.error(`[ClusterManager] Error handling log request: ${err.message}`);
+          const message = err instanceof Error ? err.message : String(err);
+          this.app.logger.error(`[ClusterManager] Error handling doctor snapshot request: ${message}`);
+          if (redis && requestId) {
+            const fallback = {
+              nodeId: (0, import_node.getLocalNodeId)(this.app),
+              collectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+              error: message
+            };
+            await redis.sendCommand([
+              "SET",
+              `cluster-manager:doctor-response:${requestId}`,
+              JSON.stringify(fallback),
+              "EX",
+              "90"
+            ]).catch(() => {
+            });
+          }
         }
       });
       pubSub.subscribe("cluster-manager.install-packages", async (payload) => {
@@ -186,12 +233,16 @@ class PluginClusterManagerServer extends import_server.Plugin {
         try {
           let target = msg;
           let mode = "hard";
+          let targetNodeId = "";
           if (msg.startsWith("{")) {
             const parsed = JSON.parse(msg);
-            target = parsed.hostname;
+            target = parsed.hostname || parsed.target || "";
+            targetNodeId = parsed.targetNodeId || "";
             mode = parsed.mode || "hard";
           }
-          if (target === os.hostname() || target === "*") {
+          const myNodeId2 = (0, import_node.getLocalNodeId)(this.app);
+          const shouldRestart = targetNodeId ? targetNodeId === myNodeId2 : target === os.hostname() || target === "*";
+          if (shouldRestart) {
             this.app.logger.warn(`[ClusterManager] Received ${mode} restart command for node ${os.hostname()}...`);
             setTimeout(async () => {
               try {
@@ -232,6 +283,10 @@ class PluginClusterManagerServer extends import_server.Plugin {
       name: "clusterManagerCluster",
       actions: import_cluster_nodes.clusterActions
     });
+    this.app.resourcer.define({
+      name: "clusterManagerDoctor",
+      actions: import_doctor.doctorActions
+    });
     this.app.resourcer.define({
       name: "clusterManagerQueue",
       actions: import_event_queue_monitor.eventQueueActions
@@ -258,6 +313,12 @@ class PluginClusterManagerServer extends import_server.Plugin {
       before: "core",
       after: "allow-manager"
     });
+    const listMetaCacheMiddleware = (0, import_listMetaCacheMiddleware.createListMetaCacheMiddleware)(this.app);
+    this.app.resourcer.use(listMetaCacheMiddleware, {
+      tag: "listMetaCache",
+      after: "setCurrentRole"
+    });
+    (0, import_cacheInvalidationHooks.registerCacheHooks)(this.app);
     this.app.use(async (ctx, next) => {
       if (ctx.path === "/api/clusterManager:health" && (ctx.method === "GET" || ctx.method === "HEAD")) {
         ctx.body = {
@@ -277,6 +338,7 @@ class PluginClusterManagerServer extends import_server.Plugin {
         "clusterManagerRedis:*",
         "clusterManagerAclCache:*",
         "clusterManagerCluster:*",
+        "clusterManagerDoctor:*",
         "clusterManagerQueue:*",
         "clusterManagerLock:*",
         "clusterManagerCacheMgr:*",

package/dist/server/utils/node.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Generate a universally unique identifier for this specific Node.js process.
+ * Combines app name, worker mode, hostname, port, and PID to ensure uniqueness
+ * even when multiple workers run on the exact same host.
+ */
+export declare function getLocalNodeId(app: any): string;

package/dist/server/utils/redis.d.ts

@@ -0,0 +1,29 @@
+import { Context } from '@nocobase/actions';
+export declare function getRedisClient(app?: any): any;
+/**
+ * Get the shared Redis connection from the app's connection manager.
+ * Returns undefined if Redis is not configured.
+ */
+export declare function getRedis(ctx: Context): any;
+/**
+ * Get Redis connection or throw 503 if not available.
+ */
+export declare function getRedisOrThrow(ctx: Context): any;
+/**
+ * Scan Redis keys using SCAN (cursor-based) instead of the blocking KEYS command.
+ * Safe for production use — never blocks the Redis event loop.
+ *
+ * @param redis - Redis client instance
+ * @param pattern - Glob pattern to match keys (e.g. "acl:can:*")
+ * @param batchSize - Number of keys to scan per iteration (default 200)
+ */
+export declare function scanKeys(redis: any, pattern: string, batchSize?: number): Promise<string[]>;
+/**
+ * Delete keys in chunked batches to avoid exceeding Redis command argument limits.
+ *
+ * @param redis - Redis client instance
+ * @param keys - Array of keys to delete
+ * @param chunkSize - Max keys per DEL command (default 500)
+ * @returns Total number of keys deleted
+ */
+export declare function deleteKeysChunked(redis: any, keys: string[], chunkSize?: number): Promise<number>;

package/dist/server/utils/versionManager.d.ts

@@ -0,0 +1,10 @@
+export declare const cacheVersionManager: {
+    getCollectionVersion(app: any): Promise<number>;
+    incrementCollectionVersion(app: any): Promise<number>;
+    getSchemaVersion(app: any): Promise<number>;
+    incrementSchemaVersion(app: any): Promise<number>;
+    getAclVersion(app: any, roleName: string): Promise<number>;
+    incrementAclVersion(app: any, roleName: string): Promise<number>;
+    incrementAllAclVersions(app: any): Promise<number>;
+    getGlobalAclVersion(app: any): Promise<number>;
+};

package/dist/server/utils/versionManager.js

@@ -0,0 +1,91 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var versionManager_exports = {};
+__export(versionManager_exports, {
+  cacheVersionManager: () => cacheVersionManager
+});
+module.exports = __toCommonJS(versionManager_exports);
+var import_redis = require("./redis");
+const localCounters = /* @__PURE__ */ new Map();
+async function getVersion(app, key) {
+  const redis = (0, import_redis.getRedisClient)(app);
+  if (redis) {
+    try {
+      const val = await redis.sendCommand(["GET", key]);
+      return val ? parseInt(val, 10) : 1;
+    } catch {
+    }
+  }
+  return localCounters.get(key) || 1;
+}
+async function incrementVersion(app, key) {
+  const redis = (0, import_redis.getRedisClient)(app);
+  if (redis) {
+    try {
+      const val = await redis.sendCommand(["INCR", key]);
+      return parseInt(val, 10);
+    } catch {
+    }
+  }
+  const next = (localCounters.get(key) || 1) + 1;
+  localCounters.set(key, next);
+  return next;
+}
+const cacheVersionManager = {
+  async getCollectionVersion(app) {
+    return getVersion(app, "nocobase:version:collections");
+  },
+  async incrementCollectionVersion(app) {
+    app.logger.info("[ClusterManager] Incrementing collections cache version due to schema change");
+    return incrementVersion(app, "nocobase:version:collections");
+  },
+  async getSchemaVersion(app) {
+    return getVersion(app, "nocobase:version:schemas");
+  },
+  async incrementSchemaVersion(app) {
+    app.logger.info("[ClusterManager] Incrementing uiSchemas cache version due to UI change");
+    return incrementVersion(app, "nocobase:version:schemas");
+  },
+  async getAclVersion(app, roleName) {
+    return getVersion(app, `nocobase:version:acl:role:${roleName}`);
+  },
+  async incrementAclVersion(app, roleName) {
+    app.logger.info(`[ClusterManager] Incrementing ACL cache version for role: ${roleName}`);
+    return incrementVersion(app, `nocobase:version:acl:role:${roleName}`);
+  },
+  async incrementAllAclVersions(app) {
+    app.logger.info("[ClusterManager] Incrementing global ACL cache version");
+    return incrementVersion(app, "nocobase:version:acl:global");
+  },
+  async getGlobalAclVersion(app) {
+    return getVersion(app, "nocobase:version:acl:global");
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  cacheVersionManager
+});

package/dist/shared/packages.d.ts

@@ -0,0 +1,23 @@
+export declare const DEFAULT_WORKER_PACKAGES: {
+    apt: string[];
+    python: string[];
+    npm: string[];
+};
+export interface WorkerPackageMap {
+    apt?: string[];
+    npm?: string[];
+    python?: string[];
+}
+export interface CustomPackageMap {
+    python?: string[];
+    node?: string[];
+    npm?: string[];
+}
+export declare function normalizePackages(packages?: Array<string | undefined>): string[];
+export declare function parsePackageText(value: unknown, fallback?: string[]): string[];
+export declare function formatPackageText(packages?: string[]): string;
+export declare function packagesFromConfig(config: {
+    aptPackages?: unknown;
+    pythonPackages?: unknown;
+    npmPackages?: unknown;
+}): WorkerPackageMap;

package/package.json

@@ -1,42 +1,42 @@
-{
-  "name": "plugin-cluster-manager",
-  "displayName": "Cluster Manager",
-  "displayName.zh-CN": "Cluster Manager",
-  "description": "Cluster node tracking, task orchestration, worker management, and package distribution",
-  "version": "1.1.7",
-  "license": "Apache-2.0",
-  "main": "./dist/server/index.js",
-  "keywords": [
-    "Monitoring",
-    "DevOps"
-  ],
-  "files": [
-    "dist",
-    "src",
-    "client.js",
-    "server.js",
-    "client.d.ts",
-    "server.d.ts"
-  ],
-  "peerDependencies": {
-    "@nocobase/client": "2.x",
-    "@nocobase/server": "2.x",
-    "@nocobase/database": "2.x"
-  },
-  "dependencies": {
-    "@nocobase/lock-manager": "2.x"
-  },
-  "devDependencies": {
-    "@kubernetes/client-node": "^0.22.3",
-    "dockerode": "^3.3.5",
-    "redis": "^5.10.0",
-    "redlock": "5.0.0-beta.2",
-    "uuid": "^9.0.1"
-  },
-  "nocobase": {
-    "supportedVersions": [
-      "2.x"
-    ],
-    "editionLevel": 0
-  }
+{
+  "name": "plugin-cluster-manager",
+  "displayName": "Cluster Manager",
+  "displayName.zh-CN": "Cluster Manager",
+  "description": "Cluster node tracking, task orchestration, worker management, and package distribution",
+  "version": "1.1.11",
+  "license": "Apache-2.0",
+  "main": "./dist/server/index.js",
+  "keywords": [
+    "Monitoring",
+    "DevOps"
+  ],
+  "files": [
+    "dist",
+    "src",
+    "client.js",
+    "server.js",
+    "client.d.ts",
+    "server.d.ts"
+  ],
+  "peerDependencies": {
+    "@nocobase/client": "2.x",
+    "@nocobase/server": "2.x",
+    "@nocobase/database": "2.x"
+  },
+  "dependencies": {
+    "@nocobase/lock-manager": "2.x"
+  },
+  "devDependencies": {
+    "@kubernetes/client-node": "^0.22.3",
+    "dockerode": "^3.3.5",
+    "redis": "^5.10.0",
+    "redlock": "5.0.0-beta.2",
+    "uuid": "^9.0.1"
+  },
+  "nocobase": {
+    "supportedVersions": [
+      "2.x"
+    ],
+    "editionLevel": 0
+  }
 }

package/server.js

@@ -0,0 +1 @@
+module.exports = require('./dist/server/index.js');