plugin-cluster-manager 1.1.10 → 1.1.11

package/dist/server/actions/cluster-nodes.js

@@ -40,7 +40,6 @@ __export(cluster_nodes_exports, {
   readLocalLogs: () => readLocalLogs
 });
 module.exports = __toCommonJS(cluster_nodes_exports);
-var import_server = require("@nocobase/server");
 var import_os = __toESM(require("os"));
 var import_fs = require("fs");
 var import_path = __toESM(require("path"));
@@ -48,11 +47,162 @@ var import_crypto = __toESM(require("crypto"));
 var import_redis_node_registry = require("../adapters/redis-node-registry");
 var import_redis = require("../utils/redis");
 var import_node = require("../utils/node");
+var import_packages = require("../../shared/packages");
 const LOG_RESPONSE_KEY_PREFIX = "cluster-manager:log-response:";
-const LOG_RESPONSE_TTL = 30;
+const LEGACY_MULTI_APP_PLUGINS = ["multi-app-manager", "multi-app-share-collection"];
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+function normalizeList(value) {
+  if (!Array.isArray(value)) return [];
+  return Array.from(
+    new Set(
+      value.filter((item) => typeof item === "string").map((item) => item.trim()).filter(Boolean)
+    )
+  );
+}
+function normalizePackageMap(packages) {
+  return {
+    apt: normalizeList(packages == null ? void 0 : packages.apt),
+    npm: normalizeList(packages == null ? void 0 : packages.npm),
+    python: normalizeList(packages == null ? void 0 : packages.python)
+  };
+}
+function parseCustomPackages(value) {
+  if (!value) {
+    return { python: [], node: [], npm: [] };
+  }
+  let customValue = value;
+  if (typeof customValue === "string") {
+    try {
+      customValue = JSON.parse(customValue);
+    } catch {
+      return { python: [], node: [], npm: [] };
+    }
+  }
+  if (!customValue || typeof customValue !== "object" || Array.isArray(customValue)) {
+    return { python: [], node: [], npm: [] };
+  }
+  const custom = customValue;
+  return {
+    python: normalizeList(custom.python),
+    node: normalizeList(custom.node),
+    npm: normalizeList(custom.npm)
+  };
+}
+function parsePackageWhitelist(status) {
+  if (!(status == null ? void 0 : status.packageWhitelist)) {
+    return { apt: [], npm: [], python: [] };
+  }
+  let whitelistValue = status.packageWhitelist;
+  if (typeof whitelistValue === "string") {
+    try {
+      whitelistValue = JSON.parse(whitelistValue);
+    } catch {
+      return { apt: [], npm: [], python: [] };
+    }
+  }
+  if (!whitelistValue || typeof whitelistValue !== "object" || Array.isArray(whitelistValue)) {
+    return { apt: [], npm: [], python: [] };
+  }
+  const whitelist = whitelistValue;
+  const npmPackages = [
+    ...Array.isArray(whitelist.npm) ? whitelist.npm : [],
+    ...Array.isArray(whitelist.node) ? whitelist.node : []
+  ];
+  return {
+    apt: normalizeList(whitelist.apt),
+    npm: normalizeList(npmPackages),
+    python: normalizeList(whitelist.python)
+  };
+}
+function diffPackages(expected, installed) {
+  return {
+    apt: expected.apt.filter((pkg) => !installed.apt.includes(pkg)),
+    npm: expected.npm.filter((pkg) => !installed.npm.includes(pkg)),
+    python: expected.python.filter((pkg) => !installed.python.includes(pkg))
+  };
+}
+function hasMissingPackages(packages) {
+  return packages.apt.length > 0 || packages.npm.length > 0 || packages.python.length > 0;
+}
+function getErrorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function getNodeRole(node) {
+  if (node.isSandbox) {
+    return "sandbox";
+  }
+  const workerMode = node.workerMode || "main";
+  return workerMode === "worker" || workerMode === "task" || workerMode === "*" ? "worker" : "app";
+}
+function getReferenceVersion(nodes) {
+  var _a;
+  const appNode = nodes.find((node) => getNodeRole(node) === "app" && node.appVersion);
+  if (appNode == null ? void 0 : appNode.appVersion) {
+    return appNode.appVersion;
+  }
+  const counts = /* @__PURE__ */ new Map();
+  for (const node of nodes) {
+    if (!node.appVersion) continue;
+    counts.set(node.appVersion, (counts.get(node.appVersion) || 0) + 1);
+  }
+  return ((_a = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]) == null ? void 0 : _a[0]) || null;
+}
+async function getClusterNodes(ctx) {
+  var _a, _b;
+  const plugin = (_b = (_a = ctx.app.pm) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, "plugin-cluster-manager");
+  const registry = (plugin == null ? void 0 : plugin.nodeRegistry) ?? new import_redis_node_registry.RedisNodeRegistry(ctx.app);
+  return registry.getNodes();
+}
+async function getExpectedPackages(ctx) {
+  var _a;
+  const repo = ctx.db.getRepository("workerPackagesConfigs");
+  const config = await ((_a = repo == null ? void 0 : repo.findOne) == null ? void 0 : _a.call(repo));
+  if (!config) {
+    return normalizePackageMap((0, import_packages.packagesFromConfig)({}));
+  }
+  const configured = (0, import_packages.packagesFromConfig)({
+    aptPackages: config.get("aptPackages"),
+    pythonPackages: config.get("pythonPackages"),
+    npmPackages: config.get("npmPackages")
+  });
+  const custom = parseCustomPackages(config.get("customPackages"));
+  return normalizePackageMap({
+    apt: configured.apt,
+    npm: [...configured.npm || [], ...custom.node || [], ...custom.npm || []],
+    python: [...configured.python || [], ...custom.python || []]
+  });
+}
+async function readPackageStatus(ctx, node) {
+  const redis = (0, import_redis.getRedis)(ctx);
+  if (!redis) return null;
+  const keys = [
+    node.id ? `cluster-manager:pkg-status:${node.id}` : null,
+    node.hostname ? `orchestrator:pkg-status:${node.hostname}` : null,
+    node.name ? `orchestrator:pkg-status:${node.name}` : null
+  ].filter(Boolean);
+  for (const key of keys) {
+    try {
+      const raw = await redis.sendCommand(["GET", key]);
+      if (raw && typeof raw === "string") {
+        return JSON.parse(raw);
+      }
+    } catch {
+    }
+  }
+  return null;
+}
+async function getApplicationPluginRows(ctx) {
+  const repo = ctx.db.getRepository("applicationPlugins");
+  if (!repo) return [];
+  const rows = await repo.find({ sort: ["name"] });
+  return rows.map((row) => row.toJSON());
+}
+function getPayload(ctx) {
+  var _a, _b, _c;
+  return ctx.action.params.values || ((_b = (_a = ctx.request) == null ? void 0 : _a.body) == null ? void 0 : _b.values) || ((_c = ctx.request) == null ? void 0 : _c.body) || {};
+}
 async function readLocalLogs(app, maxLines) {
   const logBasePath = process.env.LOGGER_BASE_PATH || import_path.default.resolve(process.cwd(), "storage", "logs");
   const appName = process.env.APP_NAME || app.name || "main";
@@ -129,9 +279,7 @@ const clusterActions = {
       const plugin = (_b = (_a = ctx.app.pm) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, "plugin-cluster-manager");
       const registry = (plugin == null ? void 0 : plugin.nodeRegistry) ?? new import_redis_node_registry.RedisNodeRegistry(ctx.app);
       const nodes = await registry.getNodes();
-      const appNode = nodes.find(
-        (n) => n.workerMode === "main" || n.workerMode === "" || n.workerMode === "app"
-      );
+      const appNode = nodes.find((n) => n.workerMode === "main" || n.workerMode === "" || n.workerMode === "app");
       if (appNode == null ? void 0 : appNode.nodeDetails) {
         ctx.body = appNode.nodeDetails;
       } else {
@@ -173,12 +321,8 @@ const clusterActions = {
    * Returns all known cluster environments/nodes (if discovery adapter supports it)
    */
   async list(ctx, next) {
-    var _a, _b;
-    const supervisor = import_server.AppSupervisor.getInstance();
     const environments = [];
-    const plugin = (_b = (_a = ctx.app.pm) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, "plugin-cluster-manager");
-    const registry = (plugin == null ? void 0 : plugin.nodeRegistry) ?? new import_redis_node_registry.RedisNodeRegistry(ctx.app);
-    const nodes = await registry.getNodes();
+    const nodes = await getClusterNodes(ctx);
     if (nodes && nodes.length > 0) {
       for (const env of nodes) {
         environments.push({
@@ -210,6 +354,168 @@ const clusterActions = {
     ctx.body = { data: environments, meta: { count: environments.length } };
     await next();
   },
+  /**
+   * GET /clusterManagerCluster:drift
+   * Reports version/runtime/package drift across active cluster nodes.
+   */
+  async drift(ctx, next) {
+    var _a, _b;
+    const nodes = await getClusterNodes(ctx);
+    const referenceVersion = getReferenceVersion(nodes);
+    const expectedPackages = await getExpectedPackages(ctx);
+    const versionDrifts = nodes.filter((node) => node.status !== "offline").filter((node) => referenceVersion && node.appVersion && node.appVersion !== referenceVersion).map((node) => ({
+      id: node.id,
+      name: node.name,
+      hostname: node.hostname,
+      role: getNodeRole(node),
+      expectedVersion: referenceVersion,
+      actualVersion: node.appVersion
+    }));
+    const runtimeReference = (_b = (_a = nodes.find((node) => getNodeRole(node) === "app")) == null ? void 0 : _a.nodeDetails) == null ? void 0 : _b.node;
+    const runtimeDrifts = runtimeReference ? nodes.filter((node) => node.status !== "offline").filter((node) => {
+      var _a2;
+      const runtime = (_a2 = node.nodeDetails) == null ? void 0 : _a2.node;
+      if (!runtime) return false;
+      return runtime.nodeVersion !== runtimeReference.nodeVersion || runtime.platform !== runtimeReference.platform || runtime.arch !== runtimeReference.arch;
+    }).map((node) => {
+      var _a2, _b2, _c, _d, _e, _f;
+      return {
+        id: node.id,
+        name: node.name,
+        hostname: node.hostname,
+        role: getNodeRole(node),
+        expected: {
+          nodeVersion: runtimeReference.nodeVersion,
+          platform: runtimeReference.platform,
+          arch: runtimeReference.arch
+        },
+        actual: {
+          nodeVersion: (_b2 = (_a2 = node.nodeDetails) == null ? void 0 : _a2.node) == null ? void 0 : _b2.nodeVersion,
+          platform: (_d = (_c = node.nodeDetails) == null ? void 0 : _c.node) == null ? void 0 : _d.platform,
+          arch: (_f = (_e = node.nodeDetails) == null ? void 0 : _e.node) == null ? void 0 : _f.arch
+        }
+      };
+    }) : [];
+    const packageDrifts = [];
+    for (const node of nodes.filter((item) => item.status !== "offline" && getNodeRole(item) !== "app")) {
+      const status = await readPackageStatus(ctx, node);
+      const installedPackages = parsePackageWhitelist(status);
+      const missingPackages = diffPackages(expectedPackages, installedPackages);
+      const hasPackageStatus = Boolean(status);
+      const statusOk = (status == null ? void 0 : status.initStatus) === "succeeded";
+      if (!hasPackageStatus || !statusOk || hasMissingPackages(missingPackages)) {
+        packageDrifts.push({
+          id: node.id,
+          name: node.name,
+          hostname: node.hostname,
+          role: getNodeRole(node),
+          status: (status == null ? void 0 : status.initStatus) || "unknown",
+          lastInitAt: (status == null ? void 0 : status.lastInitAt) || null,
+          missingPackages,
+          installedPackages,
+          initProgressLog: (status == null ? void 0 : status.initProgressLog) || ""
+        });
+      }
+    }
+    ctx.body = {
+      healthy: versionDrifts.length === 0 && runtimeDrifts.length === 0 && packageDrifts.length === 0,
+      referenceVersion,
+      expectedPackages,
+      versionDrifts,
+      runtimeDrifts,
+      packageDrifts,
+      checkedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      summary: {
+        nodes: nodes.length,
+        versionDrifts: versionDrifts.length,
+        runtimeDrifts: runtimeDrifts.length,
+        packageDrifts: packageDrifts.length
+      }
+    };
+    await next();
+  },
+  /**
+   * GET /clusterManagerCluster:legacyDiagnostics
+   * Detects deprecated legacy multi-app plugins and leftover application records.
+   */
+  async legacyDiagnostics(ctx, next) {
+    var _a, _b;
+    const rows = await getApplicationPluginRows(ctx);
+    const plugins = LEGACY_MULTI_APP_PLUGINS.map((name) => {
+      var _a2, _b2, _c, _d;
+      const row = rows.find((item) => item.name === name || item.packageName === `@nocobase/plugin-${name}`);
+      const loaded = Boolean(
+        ((_b2 = (_a2 = ctx.app.pm) == null ? void 0 : _a2.get) == null ? void 0 : _b2.call(_a2, name)) || ((_d = (_c = ctx.app.pm) == null ? void 0 : _c.get) == null ? void 0 : _d.call(_c, `@nocobase/plugin-${name}`))
+      );
+      return {
+        name,
+        packageName: `@nocobase/plugin-${name}`,
+        installed: Boolean(row),
+        enabled: Boolean(row == null ? void 0 : row.enabled),
+        loaded,
+        version: row == null ? void 0 : row.version
+      };
+    });
+    let legacyApplicationCount = 0;
+    if ((_b = (_a = ctx.db).hasCollection) == null ? void 0 : _b.call(_a, "applications")) {
+      try {
+        legacyApplicationCount = await ctx.db.getRepository("applications").count();
+      } catch {
+        legacyApplicationCount = 0;
+      }
+    }
+    const findings = [];
+    const manager = plugins.find((plugin) => plugin.name === "multi-app-manager");
+    const shareCollection = plugins.find((plugin) => plugin.name === "multi-app-share-collection");
+    const appSupervisor = rows.find(
+      (item) => item.name === "app-supervisor" || item.packageName === "@nocobase/plugin-app-supervisor"
+    );
+    if ((manager == null ? void 0 : manager.enabled) || (manager == null ? void 0 : manager.loaded)) {
+      findings.push({
+        level: "warning",
+        code: "legacy_multi_app_manager_active",
+        messageKey: "Deprecated multi-app manager is active. It runs apps in shared process memory and should not be used for production cluster isolation.",
+        message: "Deprecated multi-app manager is active. It runs apps in shared process memory and should not be used for production cluster isolation."
+      });
+    }
+    if ((shareCollection == null ? void 0 : shareCollection.enabled) || (shareCollection == null ? void 0 : shareCollection.loaded)) {
+      findings.push({
+        level: "warning",
+        code: "legacy_share_collection_active",
+        messageKey: "Deprecated multi-app share collection is active. Avoid schema/table sharing for new cluster deployments.",
+        message: "Deprecated multi-app share collection is active. Avoid schema/table sharing for new cluster deployments."
+      });
+    }
+    if (legacyApplicationCount > 0) {
+      findings.push({
+        level: "warning",
+        code: "legacy_app_records_found",
+        messageKey: "{count} legacy application record(s) were found in the applications collection.",
+        messageArgs: { count: legacyApplicationCount },
+        message: `${legacyApplicationCount} legacy application record(s) were found in the applications collection.`
+      });
+    }
+    if (!(appSupervisor == null ? void 0 : appSupervisor.enabled)) {
+      findings.push({
+        level: "info",
+        code: "app_supervisor_not_enabled",
+        messageKey: "App Supervisor is not enabled. Use it for new multi-application management instead of deprecated multi-app plugins.",
+        message: "App Supervisor is not enabled. Use it for new multi-application management instead of deprecated multi-app plugins."
+      });
+    }
+    ctx.body = {
+      healthy: findings.every((finding) => finding.level !== "warning"),
+      plugins,
+      appSupervisor: appSupervisor ? {
+        installed: true,
+        enabled: Boolean(appSupervisor.enabled),
+        version: appSupervisor.version
+      } : { installed: false, enabled: false },
+      legacyApplicationCount,
+      findings
+    };
+    await next();
+  },
   /**
    * GET /clusterManagerCluster:health
    * Health check for all subsystems
@@ -290,6 +596,84 @@ const clusterActions = {
     }
     await next();
   },
+  /**
+   * POST /clusterManagerCluster:rollingRestart
+   * Restarts online nodes one-by-one, optionally filtered by role.
+   */
+  async rollingRestart(ctx, next) {
+    const payload = getPayload(ctx);
+    const mode = payload.mode === "soft" ? "soft" : "hard";
+    const role = payload.role || "worker";
+    const delayMs = Math.min(Math.max(Number(payload.delayMs) || 5e3, 1e3), 6e4);
+    const requestedNodeIds = Array.isArray(payload.nodeIds) ? payload.nodeIds.map(String) : [];
+    const pubSub = ctx.app.pubSubManager;
+    if (!pubSub) {
+      ctx.throw(500, "PubSub manager is not initialized. HA requires PUBSUB_ADAPTER_REDIS_URL to be set.");
+    }
+    const nodes = (await getClusterNodes(ctx)).filter((node) => {
+      if (node.status === "offline") return false;
+      if (requestedNodeIds.length > 0) return node.id && requestedNodeIds.includes(node.id);
+      if (role === "all") return true;
+      return getNodeRole(node) === role;
+    });
+    if (nodes.length === 0) {
+      ctx.throw(404, "No online nodes match the rolling restart target.");
+    }
+    const myNodeId = (0, import_node.getLocalNodeId)(ctx.app);
+    const sortedNodes = nodes.sort((a, b) => {
+      if (a.id === myNodeId) return 1;
+      if (b.id === myNodeId) return -1;
+      return String(a.name || a.id).localeCompare(String(b.name || b.id));
+    });
+    const restartId = import_crypto.default.randomBytes(8).toString("hex");
+    const startedAt = Date.now();
+    const logger = ctx.app.logger;
+    const published = sortedNodes.map((node, index) => ({
+      id: node.id,
+      name: node.name,
+      hostname: node.hostname,
+      role: getNodeRole(node),
+      mode,
+      order: index + 1,
+      scheduledDelayMs: index * delayMs,
+      scheduledAt: new Date(startedAt + index * delayMs).toISOString()
+    }));
+    sortedNodes.forEach((node, index) => {
+      setTimeout(() => {
+        try {
+          const publishResult = pubSub.publish(
+            "cluster-manager:restart",
+            JSON.stringify({
+              restartId,
+              targetNodeId: node.id,
+              hostname: node.hostname,
+              mode
+            })
+          );
+          Promise.resolve(publishResult).catch((error) => {
+            logger.error(
+              `[ClusterManager] Failed to publish rolling restart ${restartId} for ${node.id || node.hostname}: ${getErrorMessage(error)}`
+            );
+          });
+        } catch (error) {
+          logger.error(
+            `[ClusterManager] Failed to schedule rolling restart ${restartId} for ${node.id || node.hostname}: ${getErrorMessage(error)}`
+          );
+        }
+      }, index * delayMs);
+    });
+    ctx.body = {
+      success: true,
+      restartId,
+      mode,
+      role,
+      delayMs,
+      scheduled: true,
+      estimatedDurationMs: Math.max(0, (sortedNodes.length - 1) * delayMs),
+      published
+    };
+    await next();
+  },
   /**
    * GET /clusterManagerCluster:logs?targetNodeId=xxx&lines=200
    *

package/dist/server/actions/doctor.d.ts

@@ -0,0 +1,82 @@
+/// <reference types="node" />
+import { Context } from '@nocobase/actions';
+import Application from '@nocobase/server';
+interface DiagnosticLogLine {
+    source: string;
+    line: string;
+    timestamp?: string;
+    level?: string;
+    message?: string;
+    stack?: string;
+}
+interface LogSignature {
+    signature: string;
+    level: string;
+    count: number;
+    firstSeen?: string;
+    lastSeen?: string;
+    sources: string[];
+    samples: string[];
+}
+interface LogAnalysis {
+    totalLines: number;
+    levels: Record<string, number>;
+    signatures: LogSignature[];
+}
+interface PluginSnapshot {
+    name?: string;
+    packageName?: string;
+    enabled?: boolean;
+    dbVersion?: string;
+    loaded: boolean;
+    runtimeVersion?: string;
+}
+interface DoctorNodeSnapshot {
+    nodeId: string;
+    node: {
+        hostname: string;
+        pid: number;
+        workerMode: string;
+        role: string;
+        appVersion: string;
+        nodeVersion: string;
+        platform: string;
+        arch: string;
+        uptime: number;
+        isSandbox: boolean;
+    };
+    memory: NodeJS.MemoryUsage;
+    os: {
+        totalMemory: number;
+        freeMemory: number;
+        cpuCount: number;
+        loadAvg: number[];
+    };
+    env: Record<string, string | undefined>;
+    plugins: PluginSnapshot[];
+    logs: {
+        files: Array<{
+            file: string;
+            lineCount: number;
+        }>;
+        lines: DiagnosticLogLine[];
+        analysis: LogAnalysis;
+    };
+    collectedAt: string;
+    error?: string;
+}
+interface DoctorSnapshotOptions {
+    runId?: string;
+    sinceMs?: number;
+    untilMs?: number;
+    maxLines?: number;
+}
+export declare function collectLocalDoctorSnapshot(app: Application, options?: DoctorSnapshotOptions): Promise<DoctorNodeSnapshot>;
+export declare const doctorActions: {
+    start(ctx: Context, next: () => Promise<void>): Promise<void>;
+    stop(ctx: Context, next: () => Promise<void>): Promise<void>;
+    status(ctx: Context, next: () => Promise<void>): Promise<void>;
+    report(ctx: Context, next: () => Promise<void>): Promise<void>;
+    download(ctx: Context, next: () => Promise<void>): Promise<void>;
+};
+export {};