snow-flow 2.9.5 โ 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mcp/servicenow-machine-learning-mcp.js +80 -13
- package/dist/utils/mcp-process-manager-safe.d.ts +70 -0
- package/dist/utils/mcp-process-manager-safe.js +319 -0
- package/dist/utils/mcp-process-manager.backup.d.ts +58 -0
- package/dist/utils/mcp-process-manager.backup.js +220 -0
- package/dist/utils/mcp-process-manager.d.ts +22 -10
- package/dist/utils/mcp-process-manager.js +194 -95
- package/package.json +3 -1
|
@@ -1375,13 +1375,19 @@ class ServiceNowMachineLearningMCP {
|
|
|
1375
1375
|
// Create tokenizer
|
|
1376
1376
|
const tokenizer = new Map();
|
|
1377
1377
|
let tokenIndex = 1;
|
|
1378
|
-
// Get unique categories
|
|
1379
|
-
|
|
1378
|
+
// Get unique categories with fallback
|
|
1379
|
+
let categories = [...new Set(incidents.map(i => i.category).filter(c => c))];
|
|
1380
|
+
// CRITICAL FIX: Ensure we have at least 2 categories to prevent shape (1,1) error
|
|
1381
|
+
if (categories.length < 2) {
|
|
1382
|
+
categories = [...categories, 'other', 'uncategorized'];
|
|
1383
|
+
this.logger.warn(`Only ${categories.length - 2} unique categories found, added fallback categories`);
|
|
1384
|
+
}
|
|
1385
|
+
this.logger.info(`Training with ${categories.length} categories: ${categories.join(', ')}`);
|
|
1380
1386
|
// Tokenize all text
|
|
1381
1387
|
const sequences = [];
|
|
1382
1388
|
for (const incident of incidents) {
|
|
1383
|
-
const text = `${incident.short_description} ${incident.description}`.toLowerCase();
|
|
1384
|
-
const words = text.split(/\s+/);
|
|
1389
|
+
const text = `${incident.short_description || ''} ${incident.description || ''}`.toLowerCase();
|
|
1390
|
+
const words = text.split(/\s+/).filter(w => w.length > 0);
|
|
1385
1391
|
const sequence = [];
|
|
1386
1392
|
for (const word of words) {
|
|
1387
1393
|
if (!tokenizer.has(word)) {
|
|
@@ -1401,9 +1407,27 @@ class ServiceNowMachineLearningMCP {
|
|
|
1401
1407
|
return [...seq, ...new Array(maxLength - seq.length).fill(0)];
|
|
1402
1408
|
}
|
|
1403
1409
|
});
|
|
1410
|
+
// Create category indices with improved error handling
|
|
1411
|
+
const categoryIndices = incidents.map(incident => {
|
|
1412
|
+
const category = incident.category || 'uncategorized';
|
|
1413
|
+
const index = categories.indexOf(category);
|
|
1414
|
+
if (index >= 0) {
|
|
1415
|
+
return index;
|
|
1416
|
+
}
|
|
1417
|
+
else {
|
|
1418
|
+
// Fallback to 'other' or first category
|
|
1419
|
+
const otherIndex = categories.indexOf('other');
|
|
1420
|
+
return otherIndex >= 0 ? otherIndex : 0;
|
|
1421
|
+
}
|
|
1422
|
+
});
|
|
1423
|
+
// Validate before creating tensors
|
|
1424
|
+
if (categories.length < 2) {
|
|
1425
|
+
throw new Error(`Insufficient categories for classification: ${categories.length}. Need at least 2 categories.`);
|
|
1426
|
+
}
|
|
1404
1427
|
// Create features and labels
|
|
1405
1428
|
const features = tf.tensor2d(paddedSequences);
|
|
1406
|
-
const labels = tf.oneHot(tf.tensor1d(
|
|
1429
|
+
const labels = tf.oneHot(tf.tensor1d(categoryIndices, 'int32'), categories.length);
|
|
1430
|
+
this.logger.info(`Prepared training data: features [${paddedSequences.length}, ${maxLength}], labels [${incidents.length}, ${categories.length}]`);
|
|
1407
1431
|
return { features, labels, tokenizer, categories };
|
|
1408
1432
|
}
|
|
1409
1433
|
tokenizeText(text, tokenizer, maxLength) {
|
|
@@ -1866,18 +1890,43 @@ class ServiceNowMachineLearningMCP {
|
|
|
1866
1890
|
processBatchWithHashing(incidents, categories, hasher) {
|
|
1867
1891
|
const sequences = [];
|
|
1868
1892
|
const labels = [];
|
|
1893
|
+
// CRITICAL FIX: Ensure we have at least 2 categories to prevent shape (1,1) error
|
|
1894
|
+
const validCategories = [...categories];
|
|
1895
|
+
if (validCategories.length < 2) {
|
|
1896
|
+
validCategories.push('other', 'uncategorized'); // Add fallback categories
|
|
1897
|
+
}
|
|
1898
|
+
this.logger.info(`Processing batch with ${incidents.length} incidents and ${validCategories.length} categories`);
|
|
1869
1899
|
for (const incident of incidents) {
|
|
1870
|
-
const text = `${incident.short_description} ${incident.description}`;
|
|
1900
|
+
const text = `${incident.short_description || ''} ${incident.description || ''}`;
|
|
1871
1901
|
const sequence = hasher(text);
|
|
1872
1902
|
sequences.push(sequence);
|
|
1873
|
-
// One-hot encode category
|
|
1874
|
-
const
|
|
1875
|
-
const
|
|
1903
|
+
// One-hot encode category with improved error handling
|
|
1904
|
+
const category = incident.category || 'uncategorized';
|
|
1905
|
+
const categoryIndex = validCategories.indexOf(category);
|
|
1906
|
+
const label = new Array(validCategories.length).fill(0);
|
|
1876
1907
|
if (categoryIndex >= 0) {
|
|
1877
1908
|
label[categoryIndex] = 1;
|
|
1878
1909
|
}
|
|
1910
|
+
else {
|
|
1911
|
+
// Fallback to 'other' category if not found
|
|
1912
|
+
const otherIndex = validCategories.indexOf('other');
|
|
1913
|
+
if (otherIndex >= 0) {
|
|
1914
|
+
label[otherIndex] = 1;
|
|
1915
|
+
}
|
|
1916
|
+
else {
|
|
1917
|
+
label[0] = 1; // Use first category as fallback
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1879
1920
|
labels.push(label);
|
|
1880
1921
|
}
|
|
1922
|
+
// Validate shapes before creating tensors
|
|
1923
|
+
if (sequences.length === 0 || labels.length === 0) {
|
|
1924
|
+
throw new Error('No valid data for training - empty sequences or labels');
|
|
1925
|
+
}
|
|
1926
|
+
if (labels[0].length < 2) {
|
|
1927
|
+
throw new Error(`Insufficient categories for classification: ${labels[0].length}. Need at least 2 categories.`);
|
|
1928
|
+
}
|
|
1929
|
+
this.logger.info(`Creating tensors: features [${sequences.length}, ${sequences[0]?.length}], labels [${labels.length}, ${labels[0]?.length}]`);
|
|
1881
1930
|
return {
|
|
1882
1931
|
features: tf.tensor2d(sequences),
|
|
1883
1932
|
labels: tf.tensor2d(labels)
|
|
@@ -1967,10 +2016,18 @@ class ServiceNowMachineLearningMCP {
|
|
|
1967
2016
|
const validVocabularySize = Math.max(1000, maxVocabularySize || 5000);
|
|
1968
2017
|
this.logger.info(`Using vocabulary size: ${validVocabularySize} for data preparation`);
|
|
1969
2018
|
const hasher = this.createFeatureHasher(validVocabularySize);
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
2019
|
+
let categories = [...new Set(incidents.map(i => i.category))].filter(c => c); // Filter out empty categories
|
|
2020
|
+
// CRITICAL FIX: Ensure we have at least 2 categories to prevent shape (1,1) error
|
|
2021
|
+
if (categories.length < 2) {
|
|
2022
|
+
if (categories.length === 0) {
|
|
2023
|
+
categories = ['uncategorized', 'other'];
|
|
2024
|
+
}
|
|
2025
|
+
else {
|
|
2026
|
+
categories.push('other');
|
|
2027
|
+
}
|
|
2028
|
+
this.logger.warn(`Insufficient categories found, using fallback categories: ${categories.join(', ')}`);
|
|
1973
2029
|
}
|
|
2030
|
+
this.logger.info(`Processing with ${categories.length} categories: ${categories.join(', ')}`);
|
|
1974
2031
|
const sequences = [];
|
|
1975
2032
|
const labels = [];
|
|
1976
2033
|
for (const incident of incidents) {
|
|
@@ -1985,13 +2042,23 @@ class ServiceNowMachineLearningMCP {
|
|
|
1985
2042
|
return idx;
|
|
1986
2043
|
});
|
|
1987
2044
|
sequences.push(validatedSequence);
|
|
1988
|
-
// One-hot encode category
|
|
2045
|
+
// One-hot encode category with improved error handling
|
|
1989
2046
|
const category = incident.category || 'uncategorized';
|
|
1990
2047
|
const categoryIndex = categories.indexOf(category);
|
|
1991
2048
|
const label = new Array(categories.length).fill(0);
|
|
1992
2049
|
if (categoryIndex >= 0) {
|
|
1993
2050
|
label[categoryIndex] = 1;
|
|
1994
2051
|
}
|
|
2052
|
+
else {
|
|
2053
|
+
// Fallback to 'other' category if not found
|
|
2054
|
+
const otherIndex = categories.indexOf('other');
|
|
2055
|
+
if (otherIndex >= 0) {
|
|
2056
|
+
label[otherIndex] = 1;
|
|
2057
|
+
}
|
|
2058
|
+
else {
|
|
2059
|
+
label[0] = 1; // Use first category as final fallback
|
|
2060
|
+
}
|
|
2061
|
+
}
|
|
1995
2062
|
labels.push(label);
|
|
1996
2063
|
}
|
|
1997
2064
|
// Validate sequences before creating tensors
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCP Process Manager - SAFE VERSION
|
|
3
|
+
* Emergency fix for memory crash issues
|
|
4
|
+
* Implements graceful shutdown and memory-safe cleanup
|
|
5
|
+
*/
|
|
6
|
+
export declare class MCPProcessManager {
|
|
7
|
+
private static instance;
|
|
8
|
+
private readonly MAX_MCP_SERVERS;
|
|
9
|
+
private readonly MAX_MEMORY_MB;
|
|
10
|
+
private readonly CLEANUP_ENABLED;
|
|
11
|
+
private readonly CLEANUP_INTERVAL;
|
|
12
|
+
private cleanupTimer?;
|
|
13
|
+
private isCleaningUp;
|
|
14
|
+
private constructor();
|
|
15
|
+
static getInstance(): MCPProcessManager;
|
|
16
|
+
/**
|
|
17
|
+
* Check if we can spawn a new MCP server
|
|
18
|
+
*/
|
|
19
|
+
canSpawnServer(): boolean;
|
|
20
|
+
/**
|
|
21
|
+
* Get current MCP system status - SAFER VERSION
|
|
22
|
+
*/
|
|
23
|
+
getSystemStatus(): {
|
|
24
|
+
processCount: number;
|
|
25
|
+
memoryUsageMB: number;
|
|
26
|
+
processes: Array<{
|
|
27
|
+
pid: number;
|
|
28
|
+
memory: number;
|
|
29
|
+
name: string;
|
|
30
|
+
}>;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* Gracefully shutdown a process with timeout
|
|
34
|
+
*/
|
|
35
|
+
private gracefulKill;
|
|
36
|
+
/**
|
|
37
|
+
* Kill duplicate MCP servers - SAFER VERSION
|
|
38
|
+
*/
|
|
39
|
+
killDuplicates(): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Emergency cleanup - only for critical situations
|
|
42
|
+
*/
|
|
43
|
+
emergencyCleanup(): Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* Safe cleanup - only when absolutely necessary
|
|
46
|
+
*/
|
|
47
|
+
cleanup(): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Start periodic cleanup - MUCH SAFER
|
|
50
|
+
*/
|
|
51
|
+
private startPeriodicCleanup;
|
|
52
|
+
/**
|
|
53
|
+
* Stop periodic cleanup
|
|
54
|
+
*/
|
|
55
|
+
stopPeriodicCleanup(): void;
|
|
56
|
+
/**
|
|
57
|
+
* Kill all MCP servers - USE WITH CAUTION
|
|
58
|
+
*/
|
|
59
|
+
killAll(): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Get resource usage summary
|
|
62
|
+
*/
|
|
63
|
+
getResourceSummary(): string;
|
|
64
|
+
/**
|
|
65
|
+
* Get health status
|
|
66
|
+
*/
|
|
67
|
+
getHealthStatus(): 'healthy' | 'warning' | 'critical';
|
|
68
|
+
}
|
|
69
|
+
export declare const mcpProcessManager: MCPProcessManager;
|
|
70
|
+
//# sourceMappingURL=mcp-process-manager-safe.d.ts.map
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* MCP Process Manager - SAFE VERSION
|
|
4
|
+
* Emergency fix for memory crash issues
|
|
5
|
+
* Implements graceful shutdown and memory-safe cleanup
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.mcpProcessManager = exports.MCPProcessManager = void 0;
|
|
9
|
+
const child_process_1 = require("child_process");
|
|
10
|
+
const logger_js_1 = require("./logger.js");
|
|
11
|
+
const util_1 = require("util");
|
|
12
|
+
const execAsync = (0, util_1.promisify)(child_process_1.exec);
|
|
13
|
+
const logger = new logger_js_1.Logger('MCPProcessManager');
|
|
14
|
+
class MCPProcessManager {
|
|
15
|
+
constructor() {
|
|
16
|
+
// INCREASED LIMITS TO PREVENT AGGRESSIVE CLEANUP
|
|
17
|
+
this.MAX_MCP_SERVERS = parseInt(process.env.SNOW_MAX_MCP_SERVERS || '30'); // Increased from 10
|
|
18
|
+
this.MAX_MEMORY_MB = parseInt(process.env.SNOW_MCP_MEMORY_LIMIT || '3000'); // Increased from 1500
|
|
19
|
+
// DISABLED AUTOMATIC CLEANUP BY DEFAULT
|
|
20
|
+
this.CLEANUP_ENABLED = process.env.SNOW_MCP_CLEANUP_ENABLED === 'true'; // Off by default
|
|
21
|
+
this.CLEANUP_INTERVAL = parseInt(process.env.SNOW_MCP_CLEANUP_INTERVAL || '300000'); // 5 minutes instead of 1
|
|
22
|
+
this.isCleaningUp = false;
|
|
23
|
+
// Only start cleanup if explicitly enabled
|
|
24
|
+
if (this.CLEANUP_ENABLED) {
|
|
25
|
+
logger.warn('โ ๏ธ MCP cleanup is ENABLED - monitor for memory issues');
|
|
26
|
+
this.startPeriodicCleanup();
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
logger.info('โ
MCP cleanup is DISABLED for stability');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
static getInstance() {
|
|
33
|
+
if (!MCPProcessManager.instance) {
|
|
34
|
+
MCPProcessManager.instance = new MCPProcessManager();
|
|
35
|
+
}
|
|
36
|
+
return MCPProcessManager.instance;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Check if we can spawn a new MCP server
|
|
40
|
+
*/
|
|
41
|
+
canSpawnServer() {
|
|
42
|
+
const status = this.getSystemStatus();
|
|
43
|
+
// More lenient limits
|
|
44
|
+
if (status.processCount >= this.MAX_MCP_SERVERS) {
|
|
45
|
+
logger.warn(`โ ๏ธ At server limit (${status.processCount}/${this.MAX_MCP_SERVERS}) - consider manual cleanup`);
|
|
46
|
+
// Still allow spawning unless critically high
|
|
47
|
+
if (status.processCount >= this.MAX_MCP_SERVERS * 1.5) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
|
|
52
|
+
logger.warn(`โ ๏ธ High memory usage (${status.memoryUsageMB}MB > ${this.MAX_MEMORY_MB}MB)`);
|
|
53
|
+
// Still allow spawning unless critically high
|
|
54
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB * 1.5) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Get current MCP system status - SAFER VERSION
|
|
62
|
+
*/
|
|
63
|
+
getSystemStatus() {
|
|
64
|
+
try {
|
|
65
|
+
// More careful process detection
|
|
66
|
+
const psOutput = (0, child_process_1.execSync)('ps aux | grep -E "mcp|servicenow.*mcp" | grep -v grep || true', {
|
|
67
|
+
encoding: 'utf8',
|
|
68
|
+
maxBuffer: 1024 * 1024 // 1MB buffer limit
|
|
69
|
+
}).trim();
|
|
70
|
+
if (!psOutput) {
|
|
71
|
+
return {
|
|
72
|
+
processCount: 0,
|
|
73
|
+
memoryUsageMB: 0,
|
|
74
|
+
processes: []
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
const lines = psOutput.split('\n').slice(0, 100); // Limit to 100 processes
|
|
78
|
+
const processes = [];
|
|
79
|
+
let totalMemory = 0;
|
|
80
|
+
for (const line of lines) {
|
|
81
|
+
try {
|
|
82
|
+
const parts = line.split(/\s+/);
|
|
83
|
+
if (parts.length > 10) {
|
|
84
|
+
const pid = parseInt(parts[1]);
|
|
85
|
+
const memory = Math.round(parseInt(parts[5]) / 1024); // Convert KB to MB
|
|
86
|
+
const name = parts.slice(10).join(' ').substring(0, 100); // Limit name length
|
|
87
|
+
if (!isNaN(pid) && !isNaN(memory)) {
|
|
88
|
+
processes.push({ pid, memory, name });
|
|
89
|
+
totalMemory += memory;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
// Skip malformed lines
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
processCount: processes.length,
|
|
99
|
+
memoryUsageMB: totalMemory,
|
|
100
|
+
processes
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
logger.error('Failed to get system status:', error);
|
|
105
|
+
return {
|
|
106
|
+
processCount: 0,
|
|
107
|
+
memoryUsageMB: 0,
|
|
108
|
+
processes: []
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Gracefully shutdown a process with timeout
|
|
114
|
+
*/
|
|
115
|
+
async gracefulKill(pid, name) {
|
|
116
|
+
try {
|
|
117
|
+
// First try SIGTERM for graceful shutdown
|
|
118
|
+
process.kill(pid, 'SIGTERM');
|
|
119
|
+
// Wait up to 5 seconds for graceful shutdown
|
|
120
|
+
let waited = 0;
|
|
121
|
+
while (waited < 5000) {
|
|
122
|
+
try {
|
|
123
|
+
process.kill(pid, 0); // Check if still alive
|
|
124
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
125
|
+
waited += 500;
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
// Process terminated
|
|
129
|
+
logger.info(`โ
Gracefully stopped ${name} (PID: ${pid})`);
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Force kill if still alive
|
|
134
|
+
process.kill(pid, 'SIGKILL');
|
|
135
|
+
logger.warn(`โ ๏ธ Force killed ${name} (PID: ${pid})`);
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
if (error.code === 'ESRCH') {
|
|
140
|
+
// Process already dead
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
logger.error(`Failed to kill ${name} (PID: ${pid}):`, error);
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Kill duplicate MCP servers - SAFER VERSION
|
|
149
|
+
*/
|
|
150
|
+
async killDuplicates() {
|
|
151
|
+
if (this.isCleaningUp) {
|
|
152
|
+
logger.warn('Cleanup already in progress, skipping...');
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
this.isCleaningUp = true;
|
|
156
|
+
try {
|
|
157
|
+
const status = this.getSystemStatus();
|
|
158
|
+
// Group processes by server type
|
|
159
|
+
const serverGroups = new Map();
|
|
160
|
+
for (const proc of status.processes) {
|
|
161
|
+
const match = proc.name.match(/servicenow-([^-]+)-mcp\.js/);
|
|
162
|
+
if (match) {
|
|
163
|
+
const serverType = match[1];
|
|
164
|
+
if (!serverGroups.has(serverType)) {
|
|
165
|
+
serverGroups.set(serverType, []);
|
|
166
|
+
}
|
|
167
|
+
serverGroups.get(serverType).push({
|
|
168
|
+
pid: proc.pid,
|
|
169
|
+
memory: proc.memory
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
// Kill duplicates gracefully
|
|
174
|
+
for (const [serverType, procs] of serverGroups) {
|
|
175
|
+
if (procs.length > 2) { // Only clean if more than 2 duplicates
|
|
176
|
+
logger.info(`Found ${procs.length} instances of ${serverType}-mcp`);
|
|
177
|
+
// Sort by memory usage (kill highest consumers first)
|
|
178
|
+
procs.sort((a, b) => b.memory - a.memory);
|
|
179
|
+
// Keep 2 instances, kill the rest
|
|
180
|
+
for (let i = 2; i < procs.length; i++) {
|
|
181
|
+
await this.gracefulKill(procs[i].pid, `${serverType}-mcp`);
|
|
182
|
+
// Wait between kills to avoid memory spike
|
|
183
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
finally {
|
|
189
|
+
this.isCleaningUp = false;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Emergency cleanup - only for critical situations
|
|
194
|
+
*/
|
|
195
|
+
async emergencyCleanup() {
|
|
196
|
+
logger.warn('๐จ EMERGENCY CLEANUP INITIATED');
|
|
197
|
+
const status = this.getSystemStatus();
|
|
198
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB * 2) {
|
|
199
|
+
logger.error(`๐ด CRITICAL: Memory usage ${status.memoryUsageMB}MB - killing highest consumers`);
|
|
200
|
+
// Sort by memory usage
|
|
201
|
+
const sorted = status.processes.sort((a, b) => b.memory - a.memory);
|
|
202
|
+
// Kill top 3 memory consumers
|
|
203
|
+
for (let i = 0; i < Math.min(3, sorted.length); i++) {
|
|
204
|
+
await this.gracefulKill(sorted[i].pid, sorted[i].name);
|
|
205
|
+
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait 2s between kills
|
|
206
|
+
}
|
|
207
|
+
// Force garbage collection if available
|
|
208
|
+
if (global.gc) {
|
|
209
|
+
global.gc();
|
|
210
|
+
logger.info('Forced garbage collection');
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Safe cleanup - only when absolutely necessary
|
|
216
|
+
*/
|
|
217
|
+
async cleanup() {
|
|
218
|
+
if (!this.CLEANUP_ENABLED) {
|
|
219
|
+
logger.info('Cleanup disabled for stability');
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
if (this.isCleaningUp) {
|
|
223
|
+
logger.warn('Cleanup already in progress');
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
const status = this.getSystemStatus();
|
|
227
|
+
// Only cleanup if REALLY necessary
|
|
228
|
+
if (status.processCount > this.MAX_MCP_SERVERS * 1.5) {
|
|
229
|
+
logger.warn(`๐งน Too many MCP servers (${status.processCount}), cleaning duplicates...`);
|
|
230
|
+
await this.killDuplicates();
|
|
231
|
+
}
|
|
232
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB * 1.5) {
|
|
233
|
+
await this.emergencyCleanup();
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Start periodic cleanup - MUCH SAFER
|
|
238
|
+
*/
|
|
239
|
+
startPeriodicCleanup() {
|
|
240
|
+
if (this.cleanupTimer) {
|
|
241
|
+
clearInterval(this.cleanupTimer);
|
|
242
|
+
}
|
|
243
|
+
// Only run cleanup when critically necessary
|
|
244
|
+
this.cleanupTimer = setInterval(async () => {
|
|
245
|
+
try {
|
|
246
|
+
const status = this.getSystemStatus();
|
|
247
|
+
// Only cleanup if CRITICALLY high
|
|
248
|
+
if (status.processCount > this.MAX_MCP_SERVERS * 2 ||
|
|
249
|
+
status.memoryUsageMB > this.MAX_MEMORY_MB * 2) {
|
|
250
|
+
logger.warn('๐ Critical resource usage detected, running cleanup...');
|
|
251
|
+
await this.cleanup();
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
logger.error('Cleanup failed:', error);
|
|
256
|
+
}
|
|
257
|
+
}, this.CLEANUP_INTERVAL);
|
|
258
|
+
// Don't block process exit
|
|
259
|
+
this.cleanupTimer.unref();
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Stop periodic cleanup
|
|
263
|
+
*/
|
|
264
|
+
stopPeriodicCleanup() {
|
|
265
|
+
if (this.cleanupTimer) {
|
|
266
|
+
clearInterval(this.cleanupTimer);
|
|
267
|
+
this.cleanupTimer = undefined;
|
|
268
|
+
logger.info('โ
Periodic cleanup stopped');
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Kill all MCP servers - USE WITH CAUTION
|
|
273
|
+
*/
|
|
274
|
+
async killAll() {
|
|
275
|
+
logger.warn('๐ด KILLING ALL MCP PROCESSES');
|
|
276
|
+
try {
|
|
277
|
+
// First try graceful shutdown
|
|
278
|
+
await execAsync('pkill -TERM -f mcp');
|
|
279
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
280
|
+
// Then force kill any remaining
|
|
281
|
+
await execAsync('pkill -KILL -f mcp');
|
|
282
|
+
logger.info('โ
All MCP processes terminated');
|
|
283
|
+
}
|
|
284
|
+
catch (error) {
|
|
285
|
+
// pkill returns non-zero if no processes found
|
|
286
|
+
logger.info('No MCP processes to kill');
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Get resource usage summary
|
|
291
|
+
*/
|
|
292
|
+
getResourceSummary() {
|
|
293
|
+
const status = this.getSystemStatus();
|
|
294
|
+
return `MCP Resources:
|
|
295
|
+
Processes: ${status.processCount}/${this.MAX_MCP_SERVERS} (${Math.round(status.processCount / this.MAX_MCP_SERVERS * 100)}%)
|
|
296
|
+
Memory: ${status.memoryUsageMB}MB/${this.MAX_MEMORY_MB}MB (${Math.round(status.memoryUsageMB / this.MAX_MEMORY_MB * 100)}%)
|
|
297
|
+
Cleanup: ${this.CLEANUP_ENABLED ? 'ENABLED' : 'DISABLED'}
|
|
298
|
+
Status: ${this.getHealthStatus()}`;
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Get health status
|
|
302
|
+
*/
|
|
303
|
+
getHealthStatus() {
|
|
304
|
+
const status = this.getSystemStatus();
|
|
305
|
+
const processPercent = status.processCount / this.MAX_MCP_SERVERS;
|
|
306
|
+
const memoryPercent = status.memoryUsageMB / this.MAX_MEMORY_MB;
|
|
307
|
+
if (processPercent > 1.5 || memoryPercent > 1.5) {
|
|
308
|
+
return 'critical';
|
|
309
|
+
}
|
|
310
|
+
if (processPercent > 1.0 || memoryPercent > 1.0) {
|
|
311
|
+
return 'warning';
|
|
312
|
+
}
|
|
313
|
+
return 'healthy';
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
exports.MCPProcessManager = MCPProcessManager;
|
|
317
|
+
// Export singleton instance
|
|
318
|
+
exports.mcpProcessManager = MCPProcessManager.getInstance();
|
|
319
|
+
//# sourceMappingURL=mcp-process-manager-safe.js.map
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCP Process Manager
|
|
3
|
+
* Prevents excessive MCP server spawning and manages resource limits
|
|
4
|
+
*/
|
|
5
|
+
export declare class MCPProcessManager {
|
|
6
|
+
private static instance;
|
|
7
|
+
private readonly MAX_MCP_SERVERS;
|
|
8
|
+
private readonly MAX_MEMORY_MB;
|
|
9
|
+
private readonly CLEANUP_INTERVAL;
|
|
10
|
+
private cleanupTimer?;
|
|
11
|
+
private constructor();
|
|
12
|
+
static getInstance(): MCPProcessManager;
|
|
13
|
+
/**
|
|
14
|
+
* Check if we can spawn a new MCP server
|
|
15
|
+
*/
|
|
16
|
+
canSpawnServer(): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Get current MCP system status
|
|
19
|
+
*/
|
|
20
|
+
getSystemStatus(): {
|
|
21
|
+
processCount: number;
|
|
22
|
+
memoryUsageMB: number;
|
|
23
|
+
processes: Array<{
|
|
24
|
+
pid: number;
|
|
25
|
+
memory: number;
|
|
26
|
+
name: string;
|
|
27
|
+
}>;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Kill duplicate MCP servers (keep only the newest)
|
|
31
|
+
*/
|
|
32
|
+
killDuplicates(): void;
|
|
33
|
+
/**
|
|
34
|
+
* Kill all MCP servers
|
|
35
|
+
*/
|
|
36
|
+
killAll(): void;
|
|
37
|
+
/**
|
|
38
|
+
* Clean up excessive resources
|
|
39
|
+
*/
|
|
40
|
+
cleanup(): void;
|
|
41
|
+
/**
|
|
42
|
+
* Start periodic cleanup
|
|
43
|
+
*/
|
|
44
|
+
private startPeriodicCleanup;
|
|
45
|
+
/**
|
|
46
|
+
* Stop periodic cleanup
|
|
47
|
+
*/
|
|
48
|
+
stopPeriodicCleanup(): void;
|
|
49
|
+
/**
|
|
50
|
+
* Get resource usage summary
|
|
51
|
+
*/
|
|
52
|
+
getResourceSummary(): string;
|
|
53
|
+
/**
|
|
54
|
+
* Get health status
|
|
55
|
+
*/
|
|
56
|
+
getHealthStatus(): 'healthy' | 'warning' | 'critical';
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=mcp-process-manager.backup.d.ts.map
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* MCP Process Manager
|
|
4
|
+
* Prevents excessive MCP server spawning and manages resource limits
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.MCPProcessManager = void 0;
|
|
8
|
+
const child_process_1 = require("child_process");
|
|
9
|
+
const logger_js_1 = require("./logger.js");
|
|
10
|
+
const logger = new logger_js_1.Logger('MCPProcessManager');
|
|
11
|
+
class MCPProcessManager {
|
|
12
|
+
constructor() {
|
|
13
|
+
this.MAX_MCP_SERVERS = parseInt(process.env.SNOW_MAX_MCP_SERVERS || '10');
|
|
14
|
+
this.MAX_MEMORY_MB = parseInt(process.env.SNOW_MCP_MEMORY_LIMIT || '1500');
|
|
15
|
+
this.CLEANUP_INTERVAL = 60000; // 1 minute
|
|
16
|
+
// Start periodic cleanup
|
|
17
|
+
this.startPeriodicCleanup();
|
|
18
|
+
}
|
|
19
|
+
static getInstance() {
|
|
20
|
+
if (!MCPProcessManager.instance) {
|
|
21
|
+
MCPProcessManager.instance = new MCPProcessManager();
|
|
22
|
+
}
|
|
23
|
+
return MCPProcessManager.instance;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Check if we can spawn a new MCP server
|
|
27
|
+
*/
|
|
28
|
+
canSpawnServer() {
|
|
29
|
+
const status = this.getSystemStatus();
|
|
30
|
+
if (status.processCount >= this.MAX_MCP_SERVERS) {
|
|
31
|
+
logger.warn(`โ Cannot spawn: Already at max servers (${status.processCount}/${this.MAX_MCP_SERVERS})`);
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
|
|
35
|
+
logger.warn(`โ Cannot spawn: Memory limit exceeded (${status.memoryUsageMB}MB > ${this.MAX_MEMORY_MB}MB)`);
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Get current MCP system status
|
|
42
|
+
*/
|
|
43
|
+
getSystemStatus() {
|
|
44
|
+
try {
|
|
45
|
+
// Count MCP processes
|
|
46
|
+
const psOutput = (0, child_process_1.execSync)('ps aux | grep -E "mcp|servicenow.*mcp" | grep -v grep', {
|
|
47
|
+
encoding: 'utf8'
|
|
48
|
+
}).trim();
|
|
49
|
+
if (!psOutput) {
|
|
50
|
+
return {
|
|
51
|
+
processCount: 0,
|
|
52
|
+
memoryUsageMB: 0,
|
|
53
|
+
processes: []
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
const lines = psOutput.split('\n');
|
|
57
|
+
const processes = [];
|
|
58
|
+
let totalMemory = 0;
|
|
59
|
+
for (const line of lines) {
|
|
60
|
+
const parts = line.split(/\s+/);
|
|
61
|
+
if (parts.length > 10) {
|
|
62
|
+
const pid = parseInt(parts[1]);
|
|
63
|
+
const memory = Math.round(parseInt(parts[5]) / 1024); // Convert KB to MB
|
|
64
|
+
const name = parts.slice(10).join(' ');
|
|
65
|
+
processes.push({ pid, memory, name });
|
|
66
|
+
totalMemory += memory;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
processCount: processes.length,
|
|
71
|
+
memoryUsageMB: totalMemory,
|
|
72
|
+
processes
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
// No MCP processes found
|
|
77
|
+
return {
|
|
78
|
+
processCount: 0,
|
|
79
|
+
memoryUsageMB: 0,
|
|
80
|
+
processes: []
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Kill duplicate MCP servers (keep only the newest)
|
|
86
|
+
*/
|
|
87
|
+
killDuplicates() {
|
|
88
|
+
const status = this.getSystemStatus();
|
|
89
|
+
// Group processes by server type
|
|
90
|
+
const serverGroups = new Map();
|
|
91
|
+
for (const proc of status.processes) {
|
|
92
|
+
// Extract server type from process name
|
|
93
|
+
const match = proc.name.match(/servicenow-([^-]+)-mcp\.js/);
|
|
94
|
+
if (match) {
|
|
95
|
+
const serverType = match[1];
|
|
96
|
+
if (!serverGroups.has(serverType)) {
|
|
97
|
+
serverGroups.set(serverType, []);
|
|
98
|
+
}
|
|
99
|
+
serverGroups.get(serverType).push({
|
|
100
|
+
pid: proc.pid,
|
|
101
|
+
memory: proc.memory
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
// Kill duplicates (keep the first one)
|
|
106
|
+
for (const [serverType, procs] of serverGroups) {
|
|
107
|
+
if (procs.length > 1) {
|
|
108
|
+
logger.warn(`Found ${procs.length} instances of ${serverType}-mcp, killing duplicates...`);
|
|
109
|
+
// Sort by PID (older PIDs first) and keep the first one
|
|
110
|
+
procs.sort((a, b) => a.pid - b.pid);
|
|
111
|
+
for (let i = 1; i < procs.length; i++) {
|
|
112
|
+
try {
|
|
113
|
+
process.kill(procs[i].pid, 'SIGTERM');
|
|
114
|
+
logger.info(`Killed duplicate ${serverType}-mcp (PID: ${procs[i].pid})`);
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
// Process might already be dead
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Kill all MCP servers
|
|
125
|
+
*/
|
|
126
|
+
killAll() {
|
|
127
|
+
try {
|
|
128
|
+
(0, child_process_1.execSync)('pkill -f mcp', { encoding: 'utf8' });
|
|
129
|
+
logger.info('โ
Killed all MCP processes');
|
|
130
|
+
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
// pkill returns non-zero if no processes found
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Clean up excessive resources
|
|
137
|
+
*/
|
|
138
|
+
cleanup() {
|
|
139
|
+
const status = this.getSystemStatus();
|
|
140
|
+
if (status.processCount > this.MAX_MCP_SERVERS) {
|
|
141
|
+
logger.warn(`๐งน Cleaning up excessive MCP servers (${status.processCount} > ${this.MAX_MCP_SERVERS})`);
|
|
142
|
+
this.killDuplicates();
|
|
143
|
+
}
|
|
144
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
|
|
145
|
+
logger.warn(`๐งน Memory usage too high (${status.memoryUsageMB}MB), killing oldest processes...`);
|
|
146
|
+
// Sort by memory usage and kill the highest consumers
|
|
147
|
+
const sorted = status.processes.sort((a, b) => b.memory - a.memory);
|
|
148
|
+
let memoryFreed = 0;
|
|
149
|
+
for (const proc of sorted) {
|
|
150
|
+
if (status.memoryUsageMB - memoryFreed <= this.MAX_MEMORY_MB * 0.8) {
|
|
151
|
+
break; // Stop when we're at 80% of limit
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
process.kill(proc.pid, 'SIGTERM');
|
|
155
|
+
memoryFreed += proc.memory;
|
|
156
|
+
logger.info(`Killed high-memory process (PID: ${proc.pid}, ${proc.memory}MB)`);
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
// Process might already be dead
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Start periodic cleanup
|
|
166
|
+
*/
|
|
167
|
+
startPeriodicCleanup() {
|
|
168
|
+
// Clear existing timer
|
|
169
|
+
if (this.cleanupTimer) {
|
|
170
|
+
clearInterval(this.cleanupTimer);
|
|
171
|
+
}
|
|
172
|
+
// Run cleanup periodically
|
|
173
|
+
this.cleanupTimer = setInterval(() => {
|
|
174
|
+
const status = this.getSystemStatus();
|
|
175
|
+
if (status.processCount > this.MAX_MCP_SERVERS * 0.8 ||
|
|
176
|
+
status.memoryUsageMB > this.MAX_MEMORY_MB * 0.8) {
|
|
177
|
+
logger.info('๐ Running periodic MCP cleanup...');
|
|
178
|
+
this.cleanup();
|
|
179
|
+
}
|
|
180
|
+
}, this.CLEANUP_INTERVAL);
|
|
181
|
+
// Don't block process exit
|
|
182
|
+
this.cleanupTimer.unref();
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Stop periodic cleanup
|
|
186
|
+
*/
|
|
187
|
+
stopPeriodicCleanup() {
|
|
188
|
+
if (this.cleanupTimer) {
|
|
189
|
+
clearInterval(this.cleanupTimer);
|
|
190
|
+
this.cleanupTimer = undefined;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Get resource usage summary
|
|
195
|
+
*/
|
|
196
|
+
getResourceSummary() {
|
|
197
|
+
const status = this.getSystemStatus();
|
|
198
|
+
return `MCP Resources:
|
|
199
|
+
Processes: ${status.processCount}/${this.MAX_MCP_SERVERS} (${Math.round(status.processCount / this.MAX_MCP_SERVERS * 100)}%)
|
|
200
|
+
Memory: ${status.memoryUsageMB}MB/${this.MAX_MEMORY_MB}MB (${Math.round(status.memoryUsageMB / this.MAX_MEMORY_MB * 100)}%)
|
|
201
|
+
Status: ${this.getHealthStatus()}`;
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Get health status
|
|
205
|
+
*/
|
|
206
|
+
getHealthStatus() {
|
|
207
|
+
const status = this.getSystemStatus();
|
|
208
|
+
const processPercent = status.processCount / this.MAX_MCP_SERVERS;
|
|
209
|
+
const memoryPercent = status.memoryUsageMB / this.MAX_MEMORY_MB;
|
|
210
|
+
if (processPercent > 0.9 || memoryPercent > 0.9) {
|
|
211
|
+
return 'critical';
|
|
212
|
+
}
|
|
213
|
+
if (processPercent > 0.7 || memoryPercent > 0.7) {
|
|
214
|
+
return 'warning';
|
|
215
|
+
}
|
|
216
|
+
return 'healthy';
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
exports.MCPProcessManager = MCPProcessManager;
|
|
220
|
+
//# sourceMappingURL=mcp-process-manager.backup.js.map
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* MCP Process Manager
|
|
3
|
-
*
|
|
2
|
+
* MCP Process Manager - SAFE VERSION
|
|
3
|
+
* Emergency fix for memory crash issues
|
|
4
|
+
* Implements graceful shutdown and memory-safe cleanup
|
|
4
5
|
*/
|
|
5
6
|
export declare class MCPProcessManager {
|
|
6
7
|
private static instance;
|
|
7
8
|
private readonly MAX_MCP_SERVERS;
|
|
8
9
|
private readonly MAX_MEMORY_MB;
|
|
10
|
+
private readonly CLEANUP_ENABLED;
|
|
9
11
|
private readonly CLEANUP_INTERVAL;
|
|
10
12
|
private cleanupTimer?;
|
|
13
|
+
private isCleaningUp;
|
|
11
14
|
private constructor();
|
|
12
15
|
static getInstance(): MCPProcessManager;
|
|
13
16
|
/**
|
|
@@ -15,7 +18,7 @@ export declare class MCPProcessManager {
|
|
|
15
18
|
*/
|
|
16
19
|
canSpawnServer(): boolean;
|
|
17
20
|
/**
|
|
18
|
-
* Get current MCP system status
|
|
21
|
+
* Get current MCP system status - SAFER VERSION
|
|
19
22
|
*/
|
|
20
23
|
getSystemStatus(): {
|
|
21
24
|
processCount: number;
|
|
@@ -27,25 +30,33 @@ export declare class MCPProcessManager {
|
|
|
27
30
|
}>;
|
|
28
31
|
};
|
|
29
32
|
/**
|
|
30
|
-
*
|
|
33
|
+
* Gracefully shutdown a process with timeout
|
|
31
34
|
*/
|
|
32
|
-
|
|
35
|
+
private gracefulKill;
|
|
33
36
|
/**
|
|
34
|
-
* Kill
|
|
37
|
+
* Kill duplicate MCP servers - SAFER VERSION
|
|
35
38
|
*/
|
|
36
|
-
|
|
39
|
+
killDuplicates(): Promise<void>;
|
|
37
40
|
/**
|
|
38
|
-
*
|
|
41
|
+
* Emergency cleanup - only for critical situations
|
|
39
42
|
*/
|
|
40
|
-
|
|
43
|
+
emergencyCleanup(): Promise<void>;
|
|
41
44
|
/**
|
|
42
|
-
*
|
|
45
|
+
* Safe cleanup - only when absolutely necessary
|
|
46
|
+
*/
|
|
47
|
+
cleanup(): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Start periodic cleanup - MUCH SAFER
|
|
43
50
|
*/
|
|
44
51
|
private startPeriodicCleanup;
|
|
45
52
|
/**
|
|
46
53
|
* Stop periodic cleanup
|
|
47
54
|
*/
|
|
48
55
|
stopPeriodicCleanup(): void;
|
|
56
|
+
/**
|
|
57
|
+
* Kill all MCP servers - USE WITH CAUTION
|
|
58
|
+
*/
|
|
59
|
+
killAll(): Promise<void>;
|
|
49
60
|
/**
|
|
50
61
|
* Get resource usage summary
|
|
51
62
|
*/
|
|
@@ -55,4 +66,5 @@ export declare class MCPProcessManager {
|
|
|
55
66
|
*/
|
|
56
67
|
getHealthStatus(): 'healthy' | 'warning' | 'critical';
|
|
57
68
|
}
|
|
69
|
+
export declare const mcpProcessManager: MCPProcessManager;
|
|
58
70
|
//# sourceMappingURL=mcp-process-manager.d.ts.map
|
|
@@ -1,20 +1,33 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* MCP Process Manager
|
|
4
|
-
*
|
|
3
|
+
* MCP Process Manager - SAFE VERSION
|
|
4
|
+
* Emergency fix for memory crash issues
|
|
5
|
+
* Implements graceful shutdown and memory-safe cleanup
|
|
5
6
|
*/
|
|
6
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.MCPProcessManager = void 0;
|
|
8
|
+
exports.mcpProcessManager = exports.MCPProcessManager = void 0;
|
|
8
9
|
const child_process_1 = require("child_process");
|
|
9
10
|
const logger_js_1 = require("./logger.js");
|
|
11
|
+
const util_1 = require("util");
|
|
12
|
+
const execAsync = (0, util_1.promisify)(child_process_1.exec);
|
|
10
13
|
const logger = new logger_js_1.Logger('MCPProcessManager');
|
|
11
14
|
class MCPProcessManager {
|
|
12
15
|
constructor() {
|
|
13
|
-
|
|
14
|
-
this.
|
|
15
|
-
this.
|
|
16
|
-
//
|
|
17
|
-
this.
|
|
16
|
+
// INCREASED LIMITS TO PREVENT AGGRESSIVE CLEANUP
|
|
17
|
+
this.MAX_MCP_SERVERS = parseInt(process.env.SNOW_MAX_MCP_SERVERS || '30'); // Increased from 10
|
|
18
|
+
this.MAX_MEMORY_MB = parseInt(process.env.SNOW_MCP_MEMORY_LIMIT || '3000'); // Increased from 1500
|
|
19
|
+
// DISABLED AUTOMATIC CLEANUP BY DEFAULT
|
|
20
|
+
this.CLEANUP_ENABLED = process.env.SNOW_MCP_CLEANUP_ENABLED === 'true'; // Off by default
|
|
21
|
+
this.CLEANUP_INTERVAL = parseInt(process.env.SNOW_MCP_CLEANUP_INTERVAL || '300000'); // 5 minutes instead of 1
|
|
22
|
+
this.isCleaningUp = false;
|
|
23
|
+
// Only start cleanup if explicitly enabled
|
|
24
|
+
if (this.CLEANUP_ENABLED) {
|
|
25
|
+
logger.warn('โ ๏ธ MCP cleanup is ENABLED - monitor for memory issues');
|
|
26
|
+
this.startPeriodicCleanup();
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
logger.info('โ
MCP cleanup is DISABLED for stability');
|
|
30
|
+
}
|
|
18
31
|
}
|
|
19
32
|
static getInstance() {
|
|
20
33
|
if (!MCPProcessManager.instance) {
|
|
@@ -27,24 +40,32 @@ class MCPProcessManager {
|
|
|
27
40
|
*/
|
|
28
41
|
canSpawnServer() {
|
|
29
42
|
const status = this.getSystemStatus();
|
|
43
|
+
// More lenient limits
|
|
30
44
|
if (status.processCount >= this.MAX_MCP_SERVERS) {
|
|
31
|
-
logger.warn(
|
|
32
|
-
|
|
45
|
+
logger.warn(`โ ๏ธ At server limit (${status.processCount}/${this.MAX_MCP_SERVERS}) - consider manual cleanup`);
|
|
46
|
+
// Still allow spawning unless critically high
|
|
47
|
+
if (status.processCount >= this.MAX_MCP_SERVERS * 1.5) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
33
50
|
}
|
|
34
51
|
if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
|
|
35
|
-
logger.warn(
|
|
36
|
-
|
|
52
|
+
logger.warn(`โ ๏ธ High memory usage (${status.memoryUsageMB}MB > ${this.MAX_MEMORY_MB}MB)`);
|
|
53
|
+
// Still allow spawning unless critically high
|
|
54
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB * 1.5) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
37
57
|
}
|
|
38
58
|
return true;
|
|
39
59
|
}
|
|
40
60
|
/**
|
|
41
|
-
* Get current MCP system status
|
|
61
|
+
* Get current MCP system status - SAFER VERSION
|
|
42
62
|
*/
|
|
43
63
|
getSystemStatus() {
|
|
44
64
|
try {
|
|
45
|
-
//
|
|
46
|
-
const psOutput = (0, child_process_1.execSync)('ps aux | grep -E "mcp|servicenow.*mcp" | grep -v grep', {
|
|
47
|
-
encoding: 'utf8'
|
|
65
|
+
// More careful process detection
|
|
66
|
+
const psOutput = (0, child_process_1.execSync)('ps aux | grep -E "mcp|servicenow.*mcp" | grep -v grep || true', {
|
|
67
|
+
encoding: 'utf8',
|
|
68
|
+
maxBuffer: 1024 * 1024 // 1MB buffer limit
|
|
48
69
|
}).trim();
|
|
49
70
|
if (!psOutput) {
|
|
50
71
|
return {
|
|
@@ -53,17 +74,24 @@ class MCPProcessManager {
|
|
|
53
74
|
processes: []
|
|
54
75
|
};
|
|
55
76
|
}
|
|
56
|
-
const lines = psOutput.split('\n');
|
|
77
|
+
const lines = psOutput.split('\n').slice(0, 100); // Limit to 100 processes
|
|
57
78
|
const processes = [];
|
|
58
79
|
let totalMemory = 0;
|
|
59
80
|
for (const line of lines) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
81
|
+
try {
|
|
82
|
+
const parts = line.split(/\s+/);
|
|
83
|
+
if (parts.length > 10) {
|
|
84
|
+
const pid = parseInt(parts[1]);
|
|
85
|
+
const memory = Math.round(parseInt(parts[5]) / 1024); // Convert KB to MB
|
|
86
|
+
const name = parts.slice(10).join(' ').substring(0, 100); // Limit name length
|
|
87
|
+
if (!isNaN(pid) && !isNaN(memory)) {
|
|
88
|
+
processes.push({ pid, memory, name });
|
|
89
|
+
totalMemory += memory;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
// Skip malformed lines
|
|
67
95
|
}
|
|
68
96
|
}
|
|
69
97
|
return {
|
|
@@ -73,7 +101,7 @@ class MCPProcessManager {
|
|
|
73
101
|
};
|
|
74
102
|
}
|
|
75
103
|
catch (error) {
|
|
76
|
-
|
|
104
|
+
logger.error('Failed to get system status:', error);
|
|
77
105
|
return {
|
|
78
106
|
processCount: 0,
|
|
79
107
|
memoryUsageMB: 0,
|
|
@@ -82,100 +110,149 @@ class MCPProcessManager {
|
|
|
82
110
|
}
|
|
83
111
|
}
|
|
84
112
|
/**
|
|
85
|
-
*
|
|
113
|
+
* Gracefully shutdown a process with timeout
|
|
86
114
|
*/
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
115
|
+
async gracefulKill(pid, name) {
|
|
116
|
+
try {
|
|
117
|
+
// First try SIGTERM for graceful shutdown
|
|
118
|
+
process.kill(pid, 'SIGTERM');
|
|
119
|
+
// Wait up to 5 seconds for graceful shutdown
|
|
120
|
+
let waited = 0;
|
|
121
|
+
while (waited < 5000) {
|
|
122
|
+
try {
|
|
123
|
+
process.kill(pid, 0); // Check if still alive
|
|
124
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
125
|
+
waited += 500;
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
// Process terminated
|
|
129
|
+
logger.info(`โ
Gracefully stopped ${name} (PID: ${pid})`);
|
|
130
|
+
return true;
|
|
98
131
|
}
|
|
99
|
-
serverGroups.get(serverType).push({
|
|
100
|
-
pid: proc.pid,
|
|
101
|
-
memory: proc.memory
|
|
102
|
-
});
|
|
103
132
|
}
|
|
133
|
+
// Force kill if still alive
|
|
134
|
+
process.kill(pid, 'SIGKILL');
|
|
135
|
+
logger.warn(`โ ๏ธ Force killed ${name} (PID: ${pid})`);
|
|
136
|
+
return true;
|
|
104
137
|
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
// Sort by PID (older PIDs first) and keep the first one
|
|
110
|
-
procs.sort((a, b) => a.pid - b.pid);
|
|
111
|
-
for (let i = 1; i < procs.length; i++) {
|
|
112
|
-
try {
|
|
113
|
-
process.kill(procs[i].pid, 'SIGTERM');
|
|
114
|
-
logger.info(`Killed duplicate ${serverType}-mcp (PID: ${procs[i].pid})`);
|
|
115
|
-
}
|
|
116
|
-
catch (error) {
|
|
117
|
-
// Process might already be dead
|
|
118
|
-
}
|
|
119
|
-
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
if (error.code === 'ESRCH') {
|
|
140
|
+
// Process already dead
|
|
141
|
+
return true;
|
|
120
142
|
}
|
|
143
|
+
logger.error(`Failed to kill ${name} (PID: ${pid}):`, error);
|
|
144
|
+
return false;
|
|
121
145
|
}
|
|
122
146
|
}
|
|
123
147
|
/**
|
|
124
|
-
* Kill
|
|
148
|
+
* Kill duplicate MCP servers - SAFER VERSION
|
|
125
149
|
*/
|
|
126
|
-
|
|
150
|
+
async killDuplicates() {
|
|
151
|
+
if (this.isCleaningUp) {
|
|
152
|
+
logger.warn('Cleanup already in progress, skipping...');
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
this.isCleaningUp = true;
|
|
127
156
|
try {
|
|
128
|
-
|
|
129
|
-
|
|
157
|
+
const status = this.getSystemStatus();
|
|
158
|
+
// Group processes by server type
|
|
159
|
+
const serverGroups = new Map();
|
|
160
|
+
for (const proc of status.processes) {
|
|
161
|
+
const match = proc.name.match(/servicenow-([^-]+)-mcp\.js/);
|
|
162
|
+
if (match) {
|
|
163
|
+
const serverType = match[1];
|
|
164
|
+
if (!serverGroups.has(serverType)) {
|
|
165
|
+
serverGroups.set(serverType, []);
|
|
166
|
+
}
|
|
167
|
+
serverGroups.get(serverType).push({
|
|
168
|
+
pid: proc.pid,
|
|
169
|
+
memory: proc.memory
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
// Kill duplicates gracefully
|
|
174
|
+
for (const [serverType, procs] of serverGroups) {
|
|
175
|
+
if (procs.length > 2) { // Only clean if more than 2 duplicates
|
|
176
|
+
logger.info(`Found ${procs.length} instances of ${serverType}-mcp`);
|
|
177
|
+
// Sort by memory usage (kill highest consumers first)
|
|
178
|
+
procs.sort((a, b) => b.memory - a.memory);
|
|
179
|
+
// Keep 2 instances, kill the rest
|
|
180
|
+
for (let i = 2; i < procs.length; i++) {
|
|
181
|
+
await this.gracefulKill(procs[i].pid, `${serverType}-mcp`);
|
|
182
|
+
// Wait between kills to avoid memory spike
|
|
183
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
130
187
|
}
|
|
131
|
-
|
|
132
|
-
|
|
188
|
+
finally {
|
|
189
|
+
this.isCleaningUp = false;
|
|
133
190
|
}
|
|
134
191
|
}
|
|
135
192
|
/**
|
|
136
|
-
*
|
|
193
|
+
* Emergency cleanup - only for critical situations
|
|
137
194
|
*/
|
|
138
|
-
|
|
195
|
+
async emergencyCleanup() {
|
|
196
|
+
logger.warn('๐จ EMERGENCY CLEANUP INITIATED');
|
|
139
197
|
const status = this.getSystemStatus();
|
|
140
|
-
if (status.
|
|
141
|
-
logger.
|
|
142
|
-
|
|
143
|
-
}
|
|
144
|
-
if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
|
|
145
|
-
logger.warn(`๐งน Memory usage too high (${status.memoryUsageMB}MB), killing oldest processes...`);
|
|
146
|
-
// Sort by memory usage and kill the highest consumers
|
|
198
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB * 2) {
|
|
199
|
+
logger.error(`๐ด CRITICAL: Memory usage ${status.memoryUsageMB}MB - killing highest consumers`);
|
|
200
|
+
// Sort by memory usage
|
|
147
201
|
const sorted = status.processes.sort((a, b) => b.memory - a.memory);
|
|
148
|
-
|
|
149
|
-
for (
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
}
|
|
158
|
-
catch (error) {
|
|
159
|
-
// Process might already be dead
|
|
160
|
-
}
|
|
202
|
+
// Kill top 3 memory consumers
|
|
203
|
+
for (let i = 0; i < Math.min(3, sorted.length); i++) {
|
|
204
|
+
await this.gracefulKill(sorted[i].pid, sorted[i].name);
|
|
205
|
+
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait 2s between kills
|
|
206
|
+
}
|
|
207
|
+
// Force garbage collection if available
|
|
208
|
+
if (global.gc) {
|
|
209
|
+
global.gc();
|
|
210
|
+
logger.info('Forced garbage collection');
|
|
161
211
|
}
|
|
162
212
|
}
|
|
163
213
|
}
|
|
164
214
|
/**
|
|
165
|
-
*
|
|
215
|
+
* Safe cleanup - only when absolutely necessary
|
|
216
|
+
*/
|
|
217
|
+
async cleanup() {
|
|
218
|
+
if (!this.CLEANUP_ENABLED) {
|
|
219
|
+
logger.info('Cleanup disabled for stability');
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
if (this.isCleaningUp) {
|
|
223
|
+
logger.warn('Cleanup already in progress');
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
const status = this.getSystemStatus();
|
|
227
|
+
// Only cleanup if REALLY necessary
|
|
228
|
+
if (status.processCount > this.MAX_MCP_SERVERS * 1.5) {
|
|
229
|
+
logger.warn(`๐งน Too many MCP servers (${status.processCount}), cleaning duplicates...`);
|
|
230
|
+
await this.killDuplicates();
|
|
231
|
+
}
|
|
232
|
+
if (status.memoryUsageMB > this.MAX_MEMORY_MB * 1.5) {
|
|
233
|
+
await this.emergencyCleanup();
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Start periodic cleanup - MUCH SAFER
|
|
166
238
|
*/
|
|
167
239
|
startPeriodicCleanup() {
|
|
168
|
-
// Clear existing timer
|
|
169
240
|
if (this.cleanupTimer) {
|
|
170
241
|
clearInterval(this.cleanupTimer);
|
|
171
242
|
}
|
|
172
|
-
//
|
|
173
|
-
this.cleanupTimer = setInterval(() => {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
243
|
+
// Only run cleanup when critically necessary
|
|
244
|
+
this.cleanupTimer = setInterval(async () => {
|
|
245
|
+
try {
|
|
246
|
+
const status = this.getSystemStatus();
|
|
247
|
+
// Only cleanup if CRITICALLY high
|
|
248
|
+
if (status.processCount > this.MAX_MCP_SERVERS * 2 ||
|
|
249
|
+
status.memoryUsageMB > this.MAX_MEMORY_MB * 2) {
|
|
250
|
+
logger.warn('๐ Critical resource usage detected, running cleanup...');
|
|
251
|
+
await this.cleanup();
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
logger.error('Cleanup failed:', error);
|
|
179
256
|
}
|
|
180
257
|
}, this.CLEANUP_INTERVAL);
|
|
181
258
|
// Don't block process exit
|
|
@@ -188,6 +265,25 @@ class MCPProcessManager {
|
|
|
188
265
|
if (this.cleanupTimer) {
|
|
189
266
|
clearInterval(this.cleanupTimer);
|
|
190
267
|
this.cleanupTimer = undefined;
|
|
268
|
+
logger.info('โ
Periodic cleanup stopped');
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Kill all MCP servers - USE WITH CAUTION
|
|
273
|
+
*/
|
|
274
|
+
async killAll() {
|
|
275
|
+
logger.warn('๐ด KILLING ALL MCP PROCESSES');
|
|
276
|
+
try {
|
|
277
|
+
// First try graceful shutdown
|
|
278
|
+
await execAsync('pkill -TERM -f mcp');
|
|
279
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
280
|
+
// Then force kill any remaining
|
|
281
|
+
await execAsync('pkill -KILL -f mcp');
|
|
282
|
+
logger.info('โ
All MCP processes terminated');
|
|
283
|
+
}
|
|
284
|
+
catch (error) {
|
|
285
|
+
// pkill returns non-zero if no processes found
|
|
286
|
+
logger.info('No MCP processes to kill');
|
|
191
287
|
}
|
|
192
288
|
}
|
|
193
289
|
/**
|
|
@@ -198,6 +294,7 @@ class MCPProcessManager {
|
|
|
198
294
|
return `MCP Resources:
|
|
199
295
|
Processes: ${status.processCount}/${this.MAX_MCP_SERVERS} (${Math.round(status.processCount / this.MAX_MCP_SERVERS * 100)}%)
|
|
200
296
|
Memory: ${status.memoryUsageMB}MB/${this.MAX_MEMORY_MB}MB (${Math.round(status.memoryUsageMB / this.MAX_MEMORY_MB * 100)}%)
|
|
297
|
+
Cleanup: ${this.CLEANUP_ENABLED ? 'ENABLED' : 'DISABLED'}
|
|
201
298
|
Status: ${this.getHealthStatus()}`;
|
|
202
299
|
}
|
|
203
300
|
/**
|
|
@@ -207,14 +304,16 @@ class MCPProcessManager {
|
|
|
207
304
|
const status = this.getSystemStatus();
|
|
208
305
|
const processPercent = status.processCount / this.MAX_MCP_SERVERS;
|
|
209
306
|
const memoryPercent = status.memoryUsageMB / this.MAX_MEMORY_MB;
|
|
210
|
-
if (processPercent >
|
|
307
|
+
if (processPercent > 1.5 || memoryPercent > 1.5) {
|
|
211
308
|
return 'critical';
|
|
212
309
|
}
|
|
213
|
-
if (processPercent > 0
|
|
310
|
+
if (processPercent > 1.0 || memoryPercent > 1.0) {
|
|
214
311
|
return 'warning';
|
|
215
312
|
}
|
|
216
313
|
return 'healthy';
|
|
217
314
|
}
|
|
218
315
|
}
|
|
219
316
|
exports.MCPProcessManager = MCPProcessManager;
|
|
317
|
+
// Export singleton instance
|
|
318
|
+
exports.mcpProcessManager = MCPProcessManager.getInstance();
|
|
220
319
|
//# sourceMappingURL=mcp-process-manager.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "snow-flow",
|
|
3
|
-
"version": "2.9.
|
|
3
|
+
"version": "2.9.7",
|
|
4
4
|
"description": "Snow-Flow: ServiceNow Advanced Intelligence Platform - 100+ real MCP tools with AI-powered swarm orchestration and neural networks. Dynamic task categorization using AI. Machine learning for incident classification, change risk prediction, and anomaly detection. Zero Mock Data, 100% Real API Integration.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -22,6 +22,8 @@
|
|
|
22
22
|
"reset-mcp": "node scripts/reset-mcp-servers.js",
|
|
23
23
|
"reset-mcp:restart": "node scripts/reset-mcp-servers.js --restart",
|
|
24
24
|
"cleanup-mcp": "node scripts/cleanup-mcp-servers.js",
|
|
25
|
+
"mcp:safe-cleanup": "node scripts/safe-mcp-cleanup.js",
|
|
26
|
+
"mcp:emergency-stop": "pkill -f mcp || true",
|
|
25
27
|
"mcp:clean": "node scripts/cleanup-mcp-servers.js && npm run build",
|
|
26
28
|
"mcp:start": "node scripts/start-mcp-proper.js",
|
|
27
29
|
"mcp:start-proper": "node scripts/start-mcp-proper.js",
|