plugin-cluster-manager 1.0.12

package/src/server/actions/cluster-nodes.ts

@@ -0,0 +1,378 @@
+import { Context } from '@nocobase/actions';
+import { AppSupervisor } from '@nocobase/server';
+import os from 'os';
+import { promises as fsp } from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { RedisNodeRegistry } from '../adapters/redis-node-registry';
+import { getRedis } from '../utils/redis';
+import { getLocalNodeId } from '../utils/node';
+
+const LOG_RESPONSE_KEY_PREFIX = 'cluster-manager:log-response:';
+const LOG_RESPONSE_TTL = 30; // seconds
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Read the last N lines from the local system log file.
+ * Extracted so it can be called from both the HTTP action and the PubSub subscriber.
+ */
+export async function readLocalLogs(app: any, maxLines: number) {
+  const logBasePath = process.env.LOGGER_BASE_PATH || path.resolve(process.cwd(), 'storage', 'logs');
+  const appName = process.env.APP_NAME || app.name || 'main';
+  const logDir = path.resolve(logBasePath, appName);
+
+  let logFiles: string[] = [];
+  try {
+    const files = await fsp.readdir(logDir);
+    logFiles = files
+      .filter((f) => f.startsWith('system') && f.endsWith('.log') && !f.includes('error'))
+      .sort()
+      .reverse();
+  } catch {
+    // logDir doesn't exist or not readable
+  }
+
+  const nodeInfo = {
+    hostname: os.hostname(),
+    pid: process.pid,
+    workerMode: process.env.WORKER_MODE || 'main',
+  };
+
+  if (logFiles.length === 0) {
+    return { node: nodeInfo, lines: [] as string[], file: null };
+  }
+
+  const logFilePath = path.resolve(logDir, logFiles[0]);
+  const result: string[] = [];
+  try {
+    const stat = await fsp.stat(logFilePath);
+    const bufferSize = Math.min(stat.size, maxLines * 2048);
+    const buffer = Buffer.alloc(bufferSize);
+    const fh = await fsp.open(logFilePath, 'r');
+    await fh.read(buffer, 0, bufferSize, Math.max(0, stat.size - bufferSize));
+    await fh.close();
+
+    const content = buffer.toString('utf8');
+    const allLines = content.split('\n').filter((l) => l.trim());
+    result.push(...allLines.slice(-maxLines));
+  } catch {
+    // File read error
+  }
+
+  return { node: nodeInfo, lines: result, file: logFiles[0] };
+}
+
+
+export const clusterActions = {
+  /**
+   * GET /clusterManagerCluster:current
+   * Always returns info about the APP node (not workers).
+   * If this request is handled by a worker, we look up the APP node from Redis.
+   */
+  async current(ctx: Context, next: () => Promise<void>) {
+    const currentMode = process.env.WORKER_MODE || 'main';
+    const isApp = currentMode === 'main' || currentMode === '' || currentMode === 'app';
+
+    if (isApp) {
+      // This process IS the APP node — return local data directly
+      const mem = process.memoryUsage();
+      ctx.body = {
+        node: {
+          hostname: os.hostname(),
+          pid: process.pid,
+          nodeVersion: process.version,
+          platform: process.platform,
+          arch: process.arch,
+          uptime: process.uptime(),
+          workerMode: currentMode,
+          appPort: process.env.APP_PORT || '',
+          clusterMode: process.env.CLUSTER_MODE || '',
+        },
+        memory: {
+          rss: mem.rss,
+          heapUsed: mem.heapUsed,
+          heapTotal: mem.heapTotal,
+          external: mem.external,
+          arrayBuffers: mem.arrayBuffers || 0,
+        },
+        os: {
+          totalMemory: os.totalmem(),
+          freeMemory: os.freemem(),
+          cpuCount: os.cpus().length,
+          loadAvg: os.loadavg(),
+        },
+      };
+    } else {
+      // This process is a WORKER — find the APP node from Redis heartbeat data
+      const plugin = (ctx.app as any).pm?.get?.('plugin-cluster-manager') as any;
+      const registry = plugin?.nodeRegistry ?? new RedisNodeRegistry(ctx.app);
+      const nodes = await registry.getNodes();
+      const appNode = nodes.find(
+        (n: any) => n.workerMode === 'main' || n.workerMode === '' || n.workerMode === 'app',
+      );
+
+      if (appNode?.nodeDetails) {
+        ctx.body = appNode.nodeDetails;
+      } else {
+        // Fallback: return local data with a flag so the UI knows
+        const mem = process.memoryUsage();
+        ctx.body = {
+          node: {
+            hostname: os.hostname(),
+            pid: process.pid,
+            nodeVersion: process.version,
+            platform: process.platform,
+            arch: process.arch,
+            uptime: process.uptime(),
+            workerMode: currentMode,
+            appPort: process.env.APP_PORT || '',
+            clusterMode: process.env.CLUSTER_MODE || '',
+          },
+          memory: {
+            rss: mem.rss,
+            heapUsed: mem.heapUsed,
+            heapTotal: mem.heapTotal,
+            external: mem.external,
+            arrayBuffers: mem.arrayBuffers || 0,
+          },
+          os: {
+            totalMemory: os.totalmem(),
+            freeMemory: os.freemem(),
+            cpuCount: os.cpus().length,
+            loadAvg: os.loadavg(),
+          },
+          _fallback: true,
+          _note: 'APP node not found in Redis; showing responding worker data',
+        };
+      }
+    }
+
+    await next();
+  },
+
+  /**
+   * GET /clusterManagerCluster:list
+   * Returns all known cluster environments/nodes (if discovery adapter supports it)
+   */
+  async list(ctx: Context, next: () => Promise<void>) {
+    const supervisor = AppSupervisor.getInstance();
+    const environments: any[] = [];
+
+    const plugin = (ctx.app as any).pm?.get?.('plugin-cluster-manager') as any;
+    const registry = plugin?.nodeRegistry ?? new RedisNodeRegistry(ctx.app);
+    const nodes = await registry.getNodes();
+    
+    if (nodes && nodes.length > 0) {
+      for (const env of nodes) {
+        environments.push({
+          id: env.id || env.name,
+          name: env.name,
+          hostname: env.hostname || os.hostname(),
+          url: env.url,
+          available: env.available,
+          appVersion: env.appVersion,
+          lastHeartbeatAt: env.lastHeartbeatAt ? new Date(env.lastHeartbeatAt).toISOString() : null,
+          status: env.status || 'online',
+          workerMode: env.workerMode,
+          isSandbox: env.isSandbox,
+          pid: env.pid
+        });
+      }
+    }
+
+    // If no discovery adapter or empty, at least return current node
+    if (environments.length === 0) {
+      environments.push({
+        name: os.hostname(),
+        hostname: os.hostname(),
+        url: null,
+        available: true,
+        appVersion: null,
+        lastHeartbeatAt: new Date().toISOString(),
+        status: 'online',
+      });
+    }
+
+    ctx.body = { data: environments, meta: { count: environments.length } };
+    await next();
+  },
+
+  /**
+   * GET /clusterManagerCluster:health
+   * Health check for all subsystems
+   */
+  async health(ctx: Context, next: () => Promise<void>) {
+    const checks: Record<string, { status: string; latency?: number; detail?: string }> = {};
+
+    // Redis check
+    try {
+      const redis = getRedis(ctx);
+      if (redis) {
+        const start = Date.now();
+        await redis.ping();
+        checks.redis = { status: 'ok', latency: Date.now() - start };
+      } else {
+        checks.redis = { status: 'not_configured' };
+      }
+    } catch (e: any) {
+      checks.redis = { status: 'error', detail: e.message };
+    }
+
+    // Database check
+    try {
+      const start = Date.now();
+      await ctx.db.sequelize.query('SELECT 1');
+      checks.database = { status: 'ok', latency: Date.now() - start };
+    } catch (e: any) {
+      checks.database = { status: 'error', detail: e.message };
+    }
+
+    // PubSub check
+    try {
+      const connected = await ctx.app.pubSubManager?.isConnected();
+      const pubSubAdapter = (ctx.app.pubSubManager as any)?.adapter;
+      checks.pubsub = { 
+        status: connected ? 'connected' : 'disconnected',
+        detail: pubSubAdapter?.constructor?.name || 'no adapter',
+      };
+    } catch (e: any) {
+      checks.pubsub = { status: 'error', detail: e.message };
+    }
+
+    // Event Queue check
+    try {
+      const connected = ctx.app.eventQueue?.isConnected();
+      const adapterType = (ctx.app.eventQueue as any)?.adapter?.constructor?.name || 'unknown';
+      checks.eventQueue = {
+        status: connected ? 'connected' : 'disconnected',
+        detail: adapterType,
+      };
+    } catch (e: any) {
+      checks.eventQueue = { status: 'error', detail: e.message };
+    }
+
+    // Lock Manager check
+    try {
+      const lockOptions = (ctx.app.lockManager as any)?.options;
+      const adapterType = lockOptions?.defaultAdapter || 'local';
+      checks.lockManager = { status: 'ok', detail: `adapter: ${adapterType}` };
+    } catch (e: any) {
+      checks.lockManager = { status: 'error', detail: e.message };
+    }
+
+    // Cache check
+    try {
+      const defaultStore = ctx.app.cacheManager?.defaultStore || 'memory';
+      checks.cache = { status: 'ok', detail: `store: ${defaultStore}` };
+    } catch (e: any) {
+      checks.cache = { status: 'error', detail: e.message };
+    }
+
+    const allOk = Object.values(checks).every(
+      (c) => c.status === 'ok' || c.status === 'connected' || c.status === 'not_configured',
+    );
+
+    ctx.body = { healthy: allOk, checks };
+    await next();
+  },
+
+  /**
+   * POST /clusterManagerCluster:restart
+   * Publishes a restart signal to target nodes orchestrating a soft NocoBase restart or a hard docker daemon rebirth
+   */
+  async restart(ctx: Context, next: () => Promise<void>) {
+    const { hostname, mode = 'hard' } = ctx.action.params.values || ctx.action.params;
+    if (!hostname) ctx.throw(400, 'Hostname required');
+
+    // NocoBase initializes pubSubManager ONLY IF PUBSUB_ADAPTER_REDIS_URL is provided natively.
+    if ((ctx.app as any).pubSubManager) {
+      await (ctx.app as any).pubSubManager.publish('cluster-manager:restart', JSON.stringify({ hostname, mode }));
+      ctx.body = { success: true, target: hostname, mode };
+    } else {
+      ctx.throw(500, 'PubSub manager is not initialized. HA requires PUBSUB_ADAPTER_REDIS_URL to be set.');
+    }
+    await next();
+  },
+
+  /**
+   * GET /clusterManagerCluster:logs?targetNodeId=xxx&lines=200
+   *
+   * HA-aware log viewer. Reads logs from a specific node in the cluster.
+   *
+   * Flow:
+   *  1. If targetNodeId matches current node (or is empty) → read local FS directly
+   *  2. Otherwise → publish a log request via PubSub → target node reads its local FS
+   *     and writes the result to a Redis key → this handler polls Redis until the
+   *     response arrives (max 10s) → returns it to the client
+   */
+  async logs(ctx: Context, next: () => Promise<void>) {
+    const { lines = 200, targetNodeId } = ctx.action.params;
+    const maxLines = Math.min(Number(lines) || 200, 1000);
+    const myNodeId = getLocalNodeId(ctx.app);
+
+    // ── Case 1: Local read (no target specified, or target is this node) ──
+    if (!targetNodeId || targetNodeId === myNodeId) {
+      ctx.body = await readLocalLogs(ctx.app, maxLines);
+      await next();
+      return;
+    }
+
+    // ── Case 2: Remote read via PubSub → Redis response pattern ──
+    const redis = getRedis(ctx);
+    const pubSub = (ctx.app as any).pubSubManager;
+
+    if (!redis || !pubSub) {
+      // No HA infrastructure — fall back to local logs with a warning
+      const localResult = await readLocalLogs(ctx.app, maxLines);
+      (localResult as any)._fallback = true;
+      (localResult as any)._note = `PubSub/Redis not available; showing logs from local node instead of ${targetNodeId}`;
+      ctx.body = localResult;
+      await next();
+      return;
+    }
+
+    // Generate a unique request ID for the response channel
+    const requestId = crypto.randomBytes(8).toString('hex');
+    const responseKey = `${LOG_RESPONSE_KEY_PREFIX}${requestId}`;
+
+    // Publish the log request — ONLY the target node is subscribed to this specific channel
+    await pubSub.publish(
+      `cluster-manager:log-request:${targetNodeId}`,
+      JSON.stringify({ requestId, targetNodeId, lines: maxLines }),
+    );
+
+    // Poll Redis for the response (200ms interval, max 10s = 50 iterations)
+    let responseData: any = null;
+    for (let i = 0; i < 50; i++) {
+      await sleep(200);
+      try {
+        const raw = await redis.sendCommand(['GET', responseKey]);
+        if (raw) {
+          responseData = JSON.parse(raw);
+          // Clean up the response key immediately
+          redis.sendCommand(['DEL', responseKey]).catch(() => {});
+          break;
+        }
+      } catch {
+        // Parse error or Redis error — continue polling
+      }
+    }
+
+    if (responseData) {
+      ctx.body = responseData;
+    } else {
+      // Timeout — target node may be unreachable
+      ctx.body = {
+        node: { hostname: 'unknown', pid: null, workerMode: 'unknown', id: targetNodeId },
+        lines: [],
+        file: null,
+        _error: `Timeout waiting for logs from ${targetNodeId}. Node may be offline or PubSub is not connected.`,
+      };
+    }
+
+    await next();
+  },
+};

package/src/server/actions/event-queue-monitor.ts

@@ -0,0 +1,95 @@
+import { Context } from '@nocobase/actions';
+
+export const eventQueueActions = {
+  /**
+   * GET /clusterManagerQueue:stats
+   * Returns event queue statistics
+   */
+  async stats(ctx: Context, next: () => Promise<void>) {
+    const eq = ctx.app.eventQueue;
+    if (!eq) {
+      ctx.throw(503, 'Event queue is not available');
+    }
+
+    const adapter = (eq as any).adapter;
+    const events = (eq as any).events as Map<string, any>;
+    const adapterName = adapter?.constructor?.name || 'unknown';
+    const connected = eq.isConnected();
+
+    const channels: any[] = [];
+    if (events) {
+      for (const [channel, opts] of events.entries()) {
+        const queueData: any = { channel };
+        queueData.concurrency = opts.concurrency || 1;
+        queueData.interval = opts.interval || 250;
+
+        // For MemoryEventQueueAdapter, we can peek at queue depth
+        if (adapter?.queues) {
+          const fullChannel = eq.getFullChannel(channel, opts.shared);
+          const queue = adapter.queues.get(fullChannel);
+          queueData.pending = queue?.length || 0;
+        } else {
+          queueData.pending = null; // Unknown for external adapters
+        }
+
+        channels.push(queueData);
+      }
+    }
+
+    const totalPending = channels.reduce((sum, c) => sum + (c.pending || 0), 0);
+
+    ctx.body = {
+      adapter: adapterName,
+      connected,
+      totalChannels: channels.length,
+      totalPending,
+      channels,
+    };
+    await next();
+  },
+
+  /**
+   * GET /clusterManagerQueue:messages
+   * List pending messages in a specific channel (memory adapter only)
+   */
+  async messages(ctx: Context, next: () => Promise<void>) {
+    const eq = ctx.app.eventQueue;
+    if (!eq) {
+      ctx.throw(503, 'Event queue is not available');
+    }
+
+    const { channel, page = 1, pageSize = 20 } = ctx.action.params;
+    const adapter = (eq as any).adapter;
+
+    if (!adapter?.queues) {
+      ctx.body = {
+        data: [],
+        meta: { count: 0, page: 1, pageSize: 20, note: 'Message inspection not available for this adapter' },
+      };
+      await next();
+      return;
+    }
+
+    const events = (eq as any).events as Map<string, any>;
+    let fullChannel = channel;
+    if (events?.has(channel)) {
+      fullChannel = eq.getFullChannel(channel, events.get(channel)?.shared);
+    }
+
+    const queue = adapter.queues.get(fullChannel) || [];
+    const start = (Number(page) - 1) * Number(pageSize);
+    const slice = queue.slice(start, start + Number(pageSize));
+
+    ctx.body = {
+      data: slice.map((msg: any) => ({
+        id: msg.id,
+        content: msg.content,
+        timestamp: msg.options?.timestamp,
+        retried: msg.options?.retried || 0,
+        maxRetries: msg.options?.maxRetries || 0,
+      })),
+      meta: { count: queue.length, page: Number(page), pageSize: Number(pageSize) },
+    };
+    await next();
+  },
+};

package/src/server/actions/lock-monitor.ts

@@ -0,0 +1,169 @@
+import { Context } from '@nocobase/actions';
+import { scanKeys } from '../utils/redis';
+
+// Must match the prefix used by RedisLockAdapter
+const REDIS_LOCK_PREFIX = 'nocobase:lock:';
+
+function getRedis(ctx: Context) {
+  return ctx.app.redisConnectionManager?.getConnection();
+}
+
+export const lockActions = {
+  /**
+   * GET /clusterManagerLock:info
+   * Returns lock manager info
+   */
+  async info(ctx: Context, next: () => Promise<void>) {
+    const lm = ctx.app.lockManager;
+    if (!lm) {
+      ctx.throw(503, 'Lock manager is not available');
+    }
+
+    const options = (lm as any).options || {};
+    const adapterType = options.defaultAdapter || 'local';
+
+    let activeLocks = 0;
+
+    if (adapterType === 'local') {
+      // LocalLockAdapter uses static Map
+      try {
+        // LocalLockAdapter uses a static Map; access it via the adapter instance
+        const adapter = (lm as any).adapters?.get?.('local');
+        const lockMap = adapter?.constructor?.locks;
+        if (lockMap instanceof Map) {
+          for (const mutex of lockMap.values()) {
+            if (mutex.isLocked?.()) activeLocks++;
+          }
+        }
+      } catch {
+        // Ignore
+      }
+    } else {
+      // For Redis adapter, scan for lock keys (using correct prefix)
+      try {
+        const redis = getRedis(ctx);
+        if (redis) {
+          const keys = await scanKeys(redis, `${REDIS_LOCK_PREFIX}*`);
+          activeLocks = keys.length;
+        }
+      } catch {
+        // Ignore
+      }
+    }
+
+    ctx.body = {
+      adapter: adapterType,
+      activeLocks,
+    };
+    await next();
+  },
+
+  /**
+   * GET /clusterManagerLock:list
+   * List active locks
+   */
+  async list(ctx: Context, next: () => Promise<void>) {
+    const lm = ctx.app.lockManager;
+    if (!lm) {
+      ctx.throw(503, 'Lock manager is not available');
+    }
+
+    const options = (lm as any).options || {};
+    const adapterType = options.defaultAdapter || 'local';
+    const locks: any[] = [];
+
+    if (adapterType === 'local') {
+      try {
+        const adapter = (lm as any).adapters?.get?.('local');
+        const lockMap = adapter?.constructor?.locks;
+        if (lockMap instanceof Map) {
+          for (const [key, mutex] of lockMap.entries()) {
+            if (mutex.isLocked?.()) {
+              locks.push({ key, locked: true, ttl: null, adapter: 'local' });
+            }
+          }
+        }
+      } catch {
+        // Ignore
+      }
+    } else {
+      // Redis adapter: scan lock keys using correct prefix and get TTL
+      try {
+        const redis = getRedis(ctx);
+        if (redis) {
+          const keys = await scanKeys(redis, `${REDIS_LOCK_PREFIX}*`);
+          for (const key of keys) {
+            const ttl = (await redis.sendCommand(['PTTL', key])) as number;
+            locks.push({
+              key,
+              displayKey: key.replace(REDIS_LOCK_PREFIX, ''),
+              locked: true,
+              ttl: ttl > 0 ? ttl : null,
+              adapter: 'redis',
+            });
+          }
+        }
+      } catch {
+        // Ignore
+      }
+    }
+
+    ctx.body = { data: locks, meta: { count: locks.length } };
+    await next();
+  },
+
+  /**
+   * POST /clusterManagerLock:release
+   * Force release a stuck lock (admin emergency action)
+   * Uses compare-and-swap via DEL for Redis locks with the correct key prefix.
+   */
+  async release(ctx: Context, next: () => Promise<void>) {
+    const { key } = ctx.action.params.values || ctx.action.params;
+    if (!key) {
+      ctx.throw(400, 'Lock key is required');
+    }
+
+    const lm = ctx.app.lockManager;
+    const options = (lm as any).options || {};
+    const adapterType = options.defaultAdapter || 'local';
+    let released = false;
+
+    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
+    ctx.app.logger.warn(`[cluster-manager] Force releasing lock "${key}" by user ${user}`);
+
+    if (adapterType === 'local') {
+      try {
+        const adapter = (lm as any).adapters?.get?.('local');
+        const lockMap = adapter?.constructor?.locks;
+        if (lockMap instanceof Map && lockMap.has(key)) {
+          const mutex = lockMap.get(key);
+          if (mutex.isLocked?.()) {
+            mutex.release?.();
+          }
+          lockMap.delete(key);
+          released = true;
+        }
+      } catch {
+        // Ignore
+      }
+    } else {
+      try {
+        const redis = getRedis(ctx);
+        if (redis) {
+          // The key from the UI may be the full Redis key or just the lock name.
+          // Try the exact key first, then try with prefix.
+          let result = await redis.sendCommand(['DEL', key]);
+          if (Number(result) === 0 && !key.startsWith(REDIS_LOCK_PREFIX)) {
+            result = await redis.sendCommand(['DEL', `${REDIS_LOCK_PREFIX}${key}`]);
+          }
+          released = Number(result) > 0;
+        }
+      } catch {
+        // Ignore
+      }
+    }
+
+    ctx.body = { success: released };
+    await next();
+  },
+};