pm2-perfmonitor 1.0.3 → 1.1.2

package/README.md

@@ -1,2 +1,37 @@
 # pm2-perfmonitor
-A pm2 module for performance monitor
+
+A pm2 module for performance monitor.
+
+# Features
+
+- Automatically detect **zombie** processes and restart it.
+- Monitor the number of zombie process restarts (pm2 monit).
+
+# Installation
+
+```bash
+$ pm2 install pm2-perfmonitor
+```
+
+> NOTE: the command is `pm2 install` NOT `npm install`
+
+# Configure
+
+|            Property             | Default Value |                               Description                                |
+| :-----------------------------: | :-----------: | :----------------------------------------------------------------------: |
+|            `enabled`            |    `true`     |                  Specify whether to enable this module                   |
+|          `excludeApps`          |       -       | Specify the application name that needs to be excluded from guardianship |
+|          `includeApps`          |       -       |          Specify the application name that needs to be guarded           |
+|        `workerInterval`         |    `60000`    |                    Timed task execution interval (ms)                    |
+|        `zombieDetection`        |    `true`     |           Specify whether to enable zombie process protection            |
+|         `zombieMaxHits`         |     `10`      |        Specify the maximum occurrence frequency of zombie status         |
+| `autoRestartWhenZombieDetected` |    `true`     |        Specify whether to automatically restart zombie processes         |
+
+# How to set these values ?
+
+After having installed the module you have to type : `pm2 set pm2-perfmonitor:<param> <value>`
+
+**e.g:**
+
+- `pm2 set pm2-perfmonitor:includeApps myNuxtApp1, myNextApp2` （Only detect applications named `myNuxtApp1` and `myNextApp2`）
+- `pm2 set pm2-perfmonitor:workerInterval 120000` （Check every `2` minutes）

package/lib/app.js

@@ -0,0 +1,241 @@
+const pmx = require('pmx')
+const pm2 = require('pm2')
+const { parseParamToArray, parseParamToNumber, parseBool } = require('./utils')
+
+const defaultOptions = {
+  enabled: true,
+  /**
+   * 排除的 app 名
+   */
+  excludeApps: [],
+  /**
+   * 包含的 app 名
+   */
+  includeApps: [],
+  /**
+   * 定时检测间隔（ms）
+   */
+  workerInterval: 60000,
+  /**
+   * 是否开启僵尸进程守护
+   */
+  zombieDetection: true,
+  /**
+   * 僵尸状态最大出现次数
+   */
+  zombieMaxHits: 10,
+  /**
+   * 僵尸状态达到最大容忍度时，是否自动重启僵尸进程
+   */
+  autoRestartWhenZombieDetected: true,
+}
+
+const conf = pmx.initModule({}, (err, incomingConf) => {
+  if (err) {
+    console.error(`[${incomingConf.module_name}] init module error:`, err)
+    process.exit(2)
+  }
+
+  return {
+    ...defaultOptions,
+    ...incomingConf,
+  }
+})
+
+const Probe = pmx.probe()
+const MODULE_NAME = conf.module_name
+const MODULE_ENABLED = parseBool(conf.enabled)
+const WORKER_INTERVAL = parseParamToNumber(conf.workerInterval)
+const INCLUDE_APPS = parseParamToArray(conf.includeApps)
+const EXCLUDE_APPS = parseParamToArray(conf.excludeApps)
+const ZOMBIE_DETECTION = parseBool(conf.zombieDetection)
+const AUTO_RESTART_WHEN_ZOMBIE_DETECTED = parseBool(
+  conf.autoRestartWhenZombieDetected,
+)
+const ZOMBIE_MAX_HITS = parseParamToNumber(conf.zombieMaxHits)
+
+// 存储每个进程的 CPU 采样历史（pm_id -> [cpu1, cpu2, ...]）
+const cpuHistory = new Map()
+const zombieRestartHistory = new Map()
+const restartFailedHistory = new Map()
+
+/**
+ * @param {'log' | 'info' | 'error' | 'warn'} type
+ *
+ */
+const logger = (type, ...args) => {
+  return console[type](`[${MODULE_NAME}]`, ...args)
+}
+
+/**
+ * 判断是否为僵尸进程：最近 ZOMBIE_MAX_HITS 次全是 0%
+ */
+const isZombie = (history) => {
+  return history.length >= ZOMBIE_MAX_HITS && history.every((v) => v === 0)
+}
+
+/**
+ * check zombie process
+ */
+const zombieProcessChecker = () => {
+  if (!ZOMBIE_DETECTION) return
+
+  pm2.list((err, apps) => {
+    apps.forEach((app) => {
+      const { name, pm_id, monit, pm2_env } = app
+
+      const appStatus = pm2_env?.status
+      const appCpuUsage = monit?.cpu || 0
+
+      if (
+        MODULE_NAME === name ||
+        (INCLUDE_APPS.length > 0 && !INCLUDE_APPS.includes(name)) ||
+        (EXCLUDE_APPS.length > 0 && EXCLUDE_APPS.includes(name))
+      ) {
+        return
+      }
+
+      // 2. 只处理 online 状态的进程
+      if (appStatus !== 'online') {
+        // 进程不在 online 状态时，清空其历史记录，避免干扰
+        cpuHistory.delete(pm_id)
+        return
+      }
+
+      if (!cpuHistory.has(pm_id)) {
+        cpuHistory.set(pm_id, [])
+      }
+
+      const history = cpuHistory.get(pm_id)
+
+      history.push(appCpuUsage)
+
+      // 只保留最近 ZOMBIE_MAX_HITS 次记录
+      if (history.length > ZOMBIE_MAX_HITS) {
+        history.shift()
+      }
+
+      // 4. 判断是否为僵尸：最近 ZOMBIE_MAX_HITS 次全是 0%
+
+      if (isZombie(history)) {
+        logger('info', `Zombie detected: ${name} (pm_id: ${pm_id})`)
+
+        if (AUTO_RESTART_WHEN_ZOMBIE_DETECTED) {
+          logger('info', 'restarting...')
+
+          pm2.restart(pm_id, (restartErr) => {
+            if (restartErr) {
+              logger(
+                'error',
+                `Restart failed for ${name} (pm_id: ${pm_id}):`,
+                restartErr,
+              )
+
+              if (!restartFailedHistory.has(pm_id)) {
+                restartFailedHistory.set(pm_id, 1)
+              } else {
+                restartFailedHistory.set(
+                  pm_id,
+                  restartFailedHistory.get(pm_id) + 1,
+                )
+              }
+
+              return
+            }
+
+            if (!zombieRestartHistory.has(pm_id)) {
+              zombieRestartHistory.set(pm_id, 1)
+            } else {
+              const history = zombieRestartHistory.get(pm_id)
+
+              zombieRestartHistory.set(pm_id, history + 1)
+            }
+
+            logger(
+              'info',
+              `Restarted ${name} (pm_id: ${pm_id}) successfully!!! Restarted ${zombieRestartHistory.get(pm_id)} times`,
+            )
+
+            // 重启后清除该进程的历史记录，避免刚重启又被判定为僵尸
+            cpuHistory.delete(pm_id)
+          })
+        }
+      }
+    })
+  })
+}
+
+const runModule = () => {
+  if (!MODULE_ENABLED) return
+
+  // connect to local pm2
+  pm2.connect((err) => {
+    if (err) {
+      logger('error', `PM2 connection error:`, err)
+
+      process.exit(1)
+    }
+
+    logger('info', 'Connected to PM2, starting monitor...')
+
+    zombieProcessChecker()
+
+    setInterval(() => {
+      zombieProcessChecker()
+    }, WORKER_INTERVAL)
+  })
+
+  /** PROB PMX **/
+  Probe.metric({
+    name: 'Zombie Restarts',
+    value: () => {
+      const res = []
+
+      for (const [k, v] of zombieRestartHistory) {
+        if (v > 0) {
+          res.push([k, v])
+        }
+      }
+
+      if (!res.length) return 'N/A'
+
+      return res.map((v) => `[${v[0]}]:${v[1]}`).join(' ; ')
+    },
+  })
+
+  Probe.metric({
+    name: 'Zombie Restarts (failed)',
+    value: () => {
+      const res = []
+
+      for (const [k, v] of restartFailedHistory) {
+        if (v > 0) {
+          res.push([k, v])
+        }
+      }
+
+      if (!res.length) return 'N/A'
+
+      return res.map((v) => `[${v[0]}]:${v[1]}`).join(' ; ')
+    },
+  })
+
+  Probe.metric({
+    name: 'Zombie Processes',
+    value: () => {
+      const res = []
+
+      for (const [pmId, arr] of cpuHistory) {
+        if (isZombie(arr)) {
+          res.push(pmId)
+        }
+      }
+
+      if (!res.length) return 'N/A'
+
+      return res.join(',')
+    },
+  })
+}
+
+runModule()

package/lib/utils.js

@@ -0,0 +1,35 @@
+const parseParamToArray = (value, defaultVal = []) => {
+  if (Array.isArray(value)) return value
+
+  if (typeof value === 'string') {
+    return value.split(',').map((v) => v.trim())
+  }
+
+  return defaultVal
+}
+
+const parseParamToNumber = (value) => {
+  if (typeof value === 'number') return value
+  if (!value) return 0
+
+  if (typeof value === 'string') {
+    return Number(value)
+  }
+
+  return 0
+}
+
+const parseBool = (value, defaultVal = false) => {
+  if (typeof value === 'boolean') return value
+
+  if (value === 'true') return true
+  if (value === 'false') return false
+
+  return defaultVal
+}
+
+module.exports = {
+  parseParamToArray,
+  parseParamToNumber,
+  parseBool,
+}

package/package.json

@@ -1,10 +1,16 @@
 {
   "name": "pm2-perfmonitor",
-  "version": "1.0.3",
-  "description": "A pm2 module for performance monitoring",
-  "author": "elenh",
+  "version": "1.1.2",
+  "description": "A pm2 module for performance monitoring. Automatically detect zombie processes and restart it",
+  "author": {
+    "name": "elenh",
+    "email": "yisiwings@163.com"
+  },
   "license": "MIT",
   "main": "lib/app.js",
+  "files": [
+    "lib"
+  ],
   "repository": {
     "type": "git",
     "url": "https://github.com/yisibell/pm2-perfmonitor.git"
@@ -14,12 +20,14 @@
   },
   "homepage": "https://github.com/yisibell/pm2-perfmonitor",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "pm2 delete app1 || true && pm2 start ecosystem.app.config.cjs",
+    "dev": "pm2 start ecosystem.dev.config.cjs",
     "release": "changelogen --release && npm publish --access=public && git push --follow-tags"
   },
   "keywords": [
     "pm2",
-    "monitor"
+    "monitor",
+    "zombie process"
   ],
   "apps": [
     {
@@ -28,6 +36,7 @@
       "max_memory_restart": "500M"
     }
   ],
+  "config": {},
   "dependencies": {
     "pm2": "latest",
     "pmx": "latest"

package/.czrc

@@ -1,3 +0,0 @@
-{
-  "path": "cz-conventional-changelog"
-}

package/CHANGELOG.md

@@ -1,31 +0,0 @@
-# Changelog
-
-
-## v1.0.3
-
-[compare changes](https://github.com/yisibell/pm2-perfmonitor/compare/v1.0.2...v1.0.3)
-
-## v1.0.2
-
-[compare changes](https://github.com/yisibell/pm2-perfmonitor/compare/v1.0.1...v1.0.2)
-
-### 🏡 Chore
-
-- Update author name ([8fbec40](https://github.com/yisibell/pm2-perfmonitor/commit/8fbec40))
-
-### ❤️ Contributors
-
-- Wenqing <wenqing@kerrylan.com>
-
-## v1.0.1
-
-
-### 🏡 Chore
-
-- Init ([297fff0](https://github.com/yisibell/pm2-perfmonitor/commit/297fff0))
-- Init release ([72dc521](https://github.com/yisibell/pm2-perfmonitor/commit/72dc521))
-
-### ❤️ Contributors
-
-- Wenqing <wenqing@kerrylan.com>
-