unreal-engine-mcp-server 0.4.7 → 0.5.0

package/tests/test-runner.mjs

@@ -0,0 +1,933 @@
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import net from 'node:net';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const repoRoot = path.resolve(__dirname, '..');
+const reportsDir = path.join(__dirname, 'reports');
+
+// Common failure keywords to check against
+const failureKeywords = ['failed', 'error', 'exception', 'invalid', 'not found', 'missing', 'timed out', 'timeout', 'unsupported', 'unknown'];
+const successKeywords = ['success', 'created', 'updated', 'deleted', 'completed', 'done', 'ok'];
+
+// Defaults for spawning the MCP server.
+let serverCommand = process.env.UNREAL_MCP_SERVER_CMD ?? 'node';
+let serverArgs = process.env.UNREAL_MCP_SERVER_ARGS ? process.env.UNREAL_MCP_SERVER_ARGS.split(',') : [path.join(repoRoot, 'dist', 'cli.js')];
+const serverCwd = process.env.UNREAL_MCP_SERVER_CWD ?? repoRoot;
+const serverEnv = Object.assign({}, process.env);
+
+function formatResultLine(testCase, status, detail, durationMs) {
+  const durationText = typeof durationMs === 'number' ? ` (${durationMs.toFixed(1)} ms)` : '';
+  return `[${status.toUpperCase()}] ${testCase.scenario}${durationText}${detail ? ` => ${detail}` : ''}`;
+}
+
+async function persistResults(toolName, results) {
+  await fs.mkdir(reportsDir, { recursive: true });
+  const timestamp = new Date().toISOString().replace(/[:]/g, '-');
+  const resultsPath = path.join(reportsDir, `${toolName}-test-results-${timestamp}.json`);
+  const serializable = results.map((result) => ({
+    scenario: result.scenario,
+    toolName: result.toolName,
+    arguments: result.arguments,
+    status: result.status,
+    durationMs: result.durationMs,
+    detail: result.detail
+  }));
+  await fs.writeFile(resultsPath, JSON.stringify({ generatedAt: new Date().toISOString(), toolName, results: serializable }, null, 2));
+  return resultsPath;
+}
+
+function summarize(toolName, results, resultsPath) {
+  const totals = results.reduce((acc, result) => { acc.total += 1; acc[result.status] = (acc[result.status] ?? 0) + 1; return acc; }, { total: 0, passed: 0, failed: 0, skipped: 0 });
+  console.log('\n' + '='.repeat(60));
+  console.log(`${toolName} Test Summary`);
+  console.log('='.repeat(60));
+  console.log(`Total cases: ${totals.total}`);
+  console.log(`✅ Passed: ${totals.passed ?? 0}`);
+  console.log(`❌ Failed: ${totals.failed ?? 0}`);
+  console.log(`⏭️  Skipped: ${totals.skipped ?? 0}`);
+  if (totals.passed && totals.total > 0) console.log(`Pass rate: ${((totals.passed / totals.total) * 100).toFixed(1)}%`);
+  console.log(`Results saved to: ${resultsPath}`);
+  console.log('='.repeat(60));
+}
+
+/**
+ * Evaluates whether a test case passed based on expected outcome
+ */
+function evaluateExpectation(testCase, response) {
+  const expectation = testCase.expected;
+
+  // Normalize expected into a comparable form. If expected is an object
+  // (e.g. {condition: 'success|error', errorPattern: 'SC_DISABLED'}), then
+  // we extract the condition string as the primary expectation string.
+  const expectedCondition = (typeof expectation === 'object' && expectation !== null && expectation.condition)
+    ? expectation.condition
+    : (typeof expectation === 'string' ? expectation : String(expectation));
+
+  const lowerExpected = expectedCondition.toLowerCase();
+
+  // Determine failure/success intent from condition keywords
+  const containsFailure = failureKeywords.some((word) => lowerExpected.includes(word));
+  const containsSuccess = successKeywords.some((word) => lowerExpected.includes(word));
+
+  const structuredSuccess = typeof response.structuredContent?.success === 'boolean'
+    ? response.structuredContent.success
+    : undefined;
+  const actualSuccess = structuredSuccess ?? !response.isError;
+
+  // Extract actual error/message from response
+  let actualError = null;
+  let actualMessage = null;
+  if (response.structuredContent) {
+    actualError = response.structuredContent.error;
+    actualMessage = response.structuredContent.message;
+  }
+
+  // Also extract flattened plain-text content for matching when structured
+  // fields are missing or when MCP errors (e.g. timeouts) are only reported
+  // via the textual content array.
+  let contentText = '';
+  if (Array.isArray(response.content) && response.content.length > 0) {
+    contentText = response.content
+      .map((entry) => (entry && typeof entry.text === 'string' ? entry.text : ''))
+      .filter((t) => t.length > 0)
+      .join('\n');
+  }
+
+  // Helper to get effective actual strings for matching
+  const messageStr = (actualMessage || '').toString().toLowerCase();
+  const errorStr = (actualError || '').toString().toLowerCase();
+  const contentStr = contentText.toString().toLowerCase();
+  const combined = `${messageStr} ${errorStr} ${contentStr}`;
+
+  // If expectation is an object with specific pattern constraints, apply them
+  if (typeof expectation === 'object' && expectation !== null) {
+    // If actual outcome was success, check successPattern
+    if (actualSuccess && expectation.successPattern) {
+      const pattern = expectation.successPattern.toLowerCase();
+      if (combined.includes(pattern)) {
+        return { passed: true, reason: `Success pattern matched: ${expectation.successPattern}` };
+      }
+    }
+    // If actual outcome was error/failure, check errorPattern
+    if (!actualSuccess && expectation.errorPattern) {
+      const pattern = expectation.errorPattern.toLowerCase();
+      if (combined.includes(pattern)) {
+        return { passed: true, reason: `Error pattern matched: ${expectation.errorPattern}` };
+      }
+    }
+  }
+
+  // Handle multi-condition expectations using "or" or pipe separators
+  // e.g., "success or LOAD_FAILED" or "success|no_instances|load_failed"
+  if (lowerExpected.includes(' or ') || lowerExpected.includes('|')) {
+    const separator = lowerExpected.includes(' or ') ? ' or ' : '|';
+    const conditions = lowerExpected.split(separator).map((c) => c.trim()).filter(Boolean);
+    for (const condition of conditions) {
+      if (successKeywords.some((kw) => condition.includes(kw)) && actualSuccess === true) {
+        return { passed: true, reason: JSON.stringify(response.structuredContent) };
+      }
+      if (condition === 'handled' && response.structuredContent && response.structuredContent.handled === true) {
+        return { passed: true, reason: 'Handled gracefully' };
+      }
+
+      // Special-case timeout expectations so that MCP transport timeouts
+      // (e.g. "Request timed out") satisfy conditions containing "timeout".
+      if (condition === 'timeout' || condition.includes('timeout')) {
+        if (combined.includes('timeout') || combined.includes('timed out')) {
+          return { passed: true, reason: `Expected timeout condition met: ${condition}` };
+        }
+      }
+
+      if (combined.includes(condition)) {
+        return { passed: true, reason: `Expected condition met: ${condition}` };
+      }
+    }
+    // If none of the OR/pipe conditions matched, it's a failure
+    return { passed: false, reason: `None of the expected conditions matched: ${expectedCondition}` };
+  }
+
+  // Also flag common automation/plugin failure phrases
+  const pluginFailureIndicators = ['does not match prefix', 'unknown', 'not implemented', 'unavailable', 'unsupported'];
+  const hasPluginFailure = pluginFailureIndicators.some(term => combined.includes(term));
+
+  if (!containsFailure && hasPluginFailure) {
+    return {
+      passed: false,
+      reason: `Expected success but plugin reported failure: ${actualMessage || actualError}`
+    };
+  }
+
+  // CRITICAL: Check if message says "failed" but success is true (FALSE POSITIVE)
+  if (actualSuccess && (
+    messageStr.includes('failed') ||
+    messageStr.includes('python execution failed') ||
+    errorStr.includes('failed')
+  )) {
+    return {
+      passed: false,
+      reason: `False positive: success=true but message indicates failure: ${actualMessage}`
+    };
+  }
+
+  // CRITICAL FIX: UE_NOT_CONNECTED errors should ALWAYS fail tests unless explicitly expected
+  if (actualError === 'UE_NOT_CONNECTED') {
+    const explicitlyExpectsDisconnection = lowerExpected.includes('not connected') ||
+      lowerExpected.includes('ue_not_connected') ||
+      lowerExpected.includes('disconnected');
+    if (!explicitlyExpectsDisconnection) {
+      return {
+        passed: false,
+        reason: `Test requires Unreal Engine connection, but got: ${actualError} - ${actualMessage}`
+      };
+    }
+  }
+
+  // For tests that expect specific error types, validate the actual error matches
+  const expectedFailure = containsFailure && !containsSuccess;
+  if (expectedFailure && !actualSuccess) {
+    // Test expects failure and got failure - but verify it's the RIGHT kind of failure
+    const lowerReason = actualMessage?.toLowerCase() || actualError?.toLowerCase() || contentStr || '';
+
+    // Check for specific error types (not just generic "error" keyword)
+    const specificErrorTypes = ['not found', 'invalid', 'missing', 'already exists', 'does not exist', 'sc_disabled'];
+    const expectedErrorType = specificErrorTypes.find(type => lowerExpected.includes(type));
+    let errorTypeMatch = expectedErrorType ? lowerReason.includes(expectedErrorType) :
+      failureKeywords.some(keyword => lowerExpected.includes(keyword) && lowerReason.includes(keyword));
+
+    // Also check detail field if main error check failed (handles wrapped exceptions)
+    if (!errorTypeMatch && response.detail && typeof response.detail === 'string') {
+      const lowerDetail = response.detail.toLowerCase();
+      if (expectedErrorType) {
+        if (lowerDetail.includes(expectedErrorType)) errorTypeMatch = true;
+      } else {
+        // If no specific error type, just check if detail contains expected string
+        if (lowerDetail.includes(lowerExpected)) errorTypeMatch = true;
+      }
+    }
+
+    // If expected outcome specifies an error type, actual error should match it
+    if (lowerExpected.includes('not found') || lowerExpected.includes('invalid') ||
+      lowerExpected.includes('missing') || lowerExpected.includes('already exists') || lowerExpected.includes('sc_disabled')) {
+      const passed = errorTypeMatch;
+      let reason;
+      if (response.isError) {
+        reason = response.content?.map((entry) => ('text' in entry ? entry.text : JSON.stringify(entry))).join('\n');
+      } else if (response.structuredContent) {
+        reason = JSON.stringify(response.structuredContent);
+      } else {
+        reason = 'No structured response returned';
+      }
+      return { passed, reason };
+    }
+  }
+
+  // Default evaluation logic
+  const passed = expectedFailure ? !actualSuccess : !!actualSuccess;
+  let reason;
+  if (response.isError) {
+    reason = response.content?.map((entry) => ('text' in entry ? entry.text : JSON.stringify(entry))).join('\n');
+  } else if (response.structuredContent) {
+    reason = JSON.stringify(response.structuredContent);
+  } else if (response.content?.length) {
+    reason = response.content.map((entry) => ('text' in entry ? entry.text : JSON.stringify(entry))).join('\n');
+  } else {
+    reason = 'No structured response returned';
+  }
+  return { passed, reason };
+}
+
+/**
+ * Main test runner function
+ */
+export async function runToolTests(toolName, testCases) {
+  console.log(`Total test cases: ${testCases.length}`);
+  console.log('='.repeat(60));
+  console.log('');
+
+  let transport;
+  let client;
+  const results = [];
+  // callToolOnce is assigned after the MCP client is initialized. Declare here so
+  // the test loop can call it regardless of block scoping rules.
+  let callToolOnce;
+
+  try {
+    // Wait for the automation bridge ports to be available so the spawned MCP server
+    // process can successfully connect to the editor plugin.
+    const bridgeHost = process.env.MCP_AUTOMATION_WS_HOST ?? '127.0.0.1';
+    const envPorts = process.env.MCP_AUTOMATION_WS_PORTS
+      ? process.env.MCP_AUTOMATION_WS_PORTS.split(',').map((p) => parseInt(p.trim(), 10)).filter(Boolean)
+      : [8090, 8091];
+    const waitMs = 10000; // Hardcoded increased timeout
+
+    console.log(`Waiting up to ${waitMs}ms for automation bridge on ${bridgeHost}:${envPorts.join(',')}`);
+
+    async function waitForAnyPort(host, ports, timeoutMs = 10000) {
+      const start = Date.now();
+      while (Date.now() - start < timeoutMs) {
+        for (const port of ports) {
+          try {
+            await new Promise((resolve, reject) => {
+              const sock = new net.Socket();
+              let settled = false;
+              sock.setTimeout(1000);
+              sock.once('connect', () => { settled = true; sock.destroy(); resolve(true); });
+              sock.once('timeout', () => { if (!settled) { settled = true; sock.destroy(); reject(new Error('timeout')); } });
+              sock.once('error', () => { if (!settled) { settled = true; sock.destroy(); reject(new Error('error')); } });
+              sock.connect(port, host);
+            });
+            console.log(`✅ Automation bridge appears to be listening on ${host}:${port}`);
+            return port;
+          } catch {
+            // ignore and try next port
+          }
+        }
+        // Yield to the event loop once instead of sleeping.
+        await new Promise((r) => setImmediate(r));
+      }
+      throw new Error(`Timed out waiting for automation bridge on ports: ${ports.join(',')}`);
+    }
+
+    try {
+      await waitForAnyPort(bridgeHost, envPorts, waitMs);
+    } catch (err) {
+      console.warn('Automation bridge did not become available before tests started:', err.message);
+    }
+
+    // Decide whether to run the built server (dist/cli.js) or to run the
+    // TypeScript source directly. Prefer the built dist when it is up-to-date
+    // with the src tree. Fall back to running src with ts-node when dist is
+    // missing or older than the src modification time to avoid running stale code.
+    const distPath = path.join(repoRoot, 'dist', 'cli.js');
+    const srcDir = path.join(repoRoot, 'src');
+
+    async function getLatestMtime(dir) {
+      let latest = 0;
+      try {
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (const e of entries) {
+          const full = path.join(dir, e.name);
+          if (e.isDirectory()) {
+            const child = await getLatestMtime(full);
+            if (child > latest) latest = child;
+          } else {
+            try {
+              const st = await fs.stat(full);
+              const m = st.mtimeMs || 0;
+              if (m > latest) latest = m;
+            } catch (_) { }
+          }
+        }
+      } catch (_) {
+        // ignore
+      }
+      return latest;
+    }
+
+    // Choose how to launch the server. Prefer using the built `dist/` executable so
+    // Node resolves ESM imports cleanly. If `dist/` is missing, attempt an automatic
+    // `npm run build` so users that run live tests don't hit ts-node resolution errors.
+    let useDist = false;
+    let distExists = false;
+    try {
+      await fs.access(distPath);
+      distExists = true;
+    } catch (e) {
+      distExists = false;
+    }
+
+    if (process.env.UNREAL_MCP_FORCE_DIST === '1') {
+      useDist = true;
+      console.log('Forcing use of dist build via UNREAL_MCP_FORCE_DIST=1');
+    } else if (distExists) {
+      try {
+        const distStat = await fs.stat(distPath);
+        const srcLatest = await getLatestMtime(srcDir);
+        const srcIsNewer = srcLatest > (distStat.mtimeMs || 0);
+        const autoBuildEnabled = process.env.UNREAL_MCP_AUTO_BUILD === '1';
+        const autoBuildDisabled = process.env.UNREAL_MCP_NO_AUTO_BUILD === '1';
+        if (srcIsNewer) {
+          if (!autoBuildEnabled && !autoBuildDisabled) {
+            console.log('Detected newer source files than dist; attempting automatic build to refresh dist/ (set UNREAL_MCP_NO_AUTO_BUILD=1 to disable)');
+          }
+          if (autoBuildEnabled || !autoBuildDisabled) {
+            const { spawn } = await import('node:child_process');
+            try {
+              await new Promise((resolve, reject) => {
+                const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+                const ps = process.platform === 'win32'
+                  ? spawn(`${npmCmd} run build`, { cwd: repoRoot, stdio: 'inherit', shell: true })
+                  : spawn(npmCmd, ['run', 'build'], { cwd: repoRoot, stdio: 'inherit' });
+                ps.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`Build failed with code ${code}`))));
+                ps.on('error', (err) => reject(err));
+              });
+              console.log('Build succeeded — using dist/ for live tests');
+              useDist = true;
+            } catch (buildErr) {
+              console.warn('Automatic build failed or could not stat files — falling back to TypeScript source for live tests:', String(buildErr));
+              useDist = false;
+            }
+          } else {
+            console.log('Detected newer source files than dist but automatic build is disabled.');
+            console.log('Set UNREAL_MCP_AUTO_BUILD=1 to enable automatic builds, or run `npm run build` manually.');
+            useDist = false;
+          }
+        } else {
+          useDist = true;
+          console.log('Using built dist for live tests');
+        }
+      } catch (buildErr) {
+        console.warn('Automatic build failed or could not stat files — falling back to TypeScript source for live tests:', String(buildErr));
+        useDist = false;
+        console.log('Preferring TypeScript source for tests to pick up local changes (set UNREAL_MCP_FORCE_DIST=1 to force dist)');
+      }
+    } else {
+      console.log('dist not found — attempting to run `npm run build` to produce dist/ for live tests');
+      try {
+        const { spawn } = await import('node:child_process');
+        await new Promise((resolve, reject) => {
+          const ps = spawn(process.platform === 'win32' ? 'npm.cmd' : 'npm', ['run', 'build'], { cwd: repoRoot, stdio: 'inherit' });
+          ps.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`Build failed with code ${code}`))));
+          ps.on('error', (err) => reject(err));
+        });
+        useDist = true;
+        console.log('Build succeeded — using dist/ for live tests');
+      } catch (buildErr) {
+        console.warn('Automatic build failed — falling back to running TypeScript source with ts-node-esm:', String(buildErr));
+        useDist = false;
+      }
+    }
+
+    if (!useDist) {
+      serverCommand = process.env.UNREAL_MCP_SERVER_CMD ?? 'npx';
+      serverArgs = ['ts-node-esm', path.join(repoRoot, 'src', 'cli.ts')];
+    } else {
+      serverCommand = process.env.UNREAL_MCP_SERVER_CMD ?? serverCommand;
+      serverArgs = process.env.UNREAL_MCP_SERVER_ARGS?.split(',') ?? serverArgs;
+    }
+
+    transport = new StdioClientTransport({
+      command: serverCommand,
+      args: serverArgs,
+      cwd: serverCwd,
+      stderr: 'inherit',
+      env: serverEnv
+    });
+
+    client = new Client({
+      name: 'unreal-mcp-test-runner',
+      version: '1.0.0'
+    });
+
+    await client.connect(transport);
+    await client.listTools({});
+    console.log('✅ Connected to Unreal MCP Server\n');
+
+    // Single-attempt call helper (no retries). This forwards a timeoutMs
+    // argument to the server so server-side automation calls use the same
+    // timeout the test harness expects.
+    callToolOnce = async function (callOptions, baseTimeoutMs) {
+      const envDefault = Number(process.env.UNREAL_MCP_TEST_CALL_TIMEOUT_MS ?? '60000') || 60000;
+      const perCall = Number(callOptions?.arguments?.timeoutMs) || undefined;
+      const base = typeof baseTimeoutMs === 'number' && baseTimeoutMs > 0 ? baseTimeoutMs : (perCall || envDefault);
+      const timeoutMs = base;
+      try {
+        console.log(`[CALL] ${callOptions.name} (timeout ${timeoutMs}ms)`);
+        const outgoing = Object.assign({}, callOptions, { arguments: { ...(callOptions.arguments || {}), timeoutMs } });
+        // Prefer instructing the MCP client to use a matching timeout if
+        // the client library supports per-call options; fall back to the
+        // plain call if not supported.
+        let callPromise;
+        try {
+          // Correct parameter order: (params, resultSchema?, options)
+          callPromise = client.callTool(outgoing, undefined, { timeout: timeoutMs });
+        } catch (err) {
+          // Fall back to calling the older signature where options might be second param
+          try {
+            callPromise = client.callTool(outgoing, { timeout: timeoutMs });
+          } catch (inner) {
+            try {
+              callPromise = client.callTool(outgoing);
+            } catch (inner2) {
+              throw inner2 || inner || err;
+            }
+          }
+        }
+
+        let timeoutId;
+        const timeoutPromise = new Promise((_, rej) => {
+          timeoutId = setTimeout(() => rej(new Error(`Local test runner timeout after ${timeoutMs}ms`)), timeoutMs);
+          if (timeoutId && typeof timeoutId.unref === 'function') {
+            timeoutId.unref();
+          }
+        });
+        try {
+          const timed = Promise.race([
+            callPromise,
+            timeoutPromise
+          ]);
+          return await timed;
+        } finally {
+          if (timeoutId) {
+            clearTimeout(timeoutId);
+          }
+        }
+      } catch (e) {
+        const msg = String(e?.message || e || '');
+        if (msg.includes('Unknown blueprint action')) {
+          return { structuredContent: { success: false, error: msg } };
+        }
+        throw e;
+      }
+    };
+
+    // Run each test case
+    for (let i = 0; i < testCases.length; i++) {
+      const testCase = testCases[i];
+      const testCaseTimeoutMs = Number(process.env.UNREAL_MCP_TEST_CASE_TIMEOUT_MS ?? testCase.arguments?.timeoutMs ?? '180000');
+      const startTime = performance.now();
+
+      try {
+        // Log test start to Unreal Engine console
+        const cleanScenario = (testCase.scenario || 'Unknown Test').replace(/"/g, "'");
+        await callToolOnce({
+          name: 'system_control',
+          arguments: { action: 'console_command', command: `Log "---- STARTING TEST: ${cleanScenario} ----"` }
+        }, 5000).catch(() => { });
+      } catch (e) { /* ignore */ }
+
+      try {
+        const response = await callToolOnce({ name: testCase.toolName, arguments: testCase.arguments }, testCaseTimeoutMs);
+
+        const endTime = performance.now();
+        const durationMs = endTime - startTime;
+
+        let structuredContent = response.structuredContent ?? null;
+        if (structuredContent === null && response.content?.length) {
+          for (const entry of response.content) {
+            if (entry?.type !== 'text' || typeof entry.text !== 'string') continue;
+            try { structuredContent = JSON.parse(entry.text); break; } catch { }
+          }
+        }
+        const normalizedResponse = { ...response, structuredContent };
+        const { passed, reason } = evaluateExpectation(testCase, normalizedResponse);
+
+        if (!passed) {
+          console.log(`[FAILED] ${testCase.scenario} (${durationMs.toFixed(1)} ms) => ${reason}`);
+          if (normalizedResponse) {
+            console.log(`[DEBUG] Full response for ${testCase.scenario}:`, JSON.stringify(normalizedResponse, null, 2));
+          }
+          results.push({
+            scenario: testCase.scenario,
+            toolName: testCase.toolName,
+            arguments: testCase.arguments,
+            status: 'failed',
+            durationMs,
+            detail: reason,
+            response: normalizedResponse
+          });
+        } else {
+          console.log(`[PASSED] ${testCase.scenario} (${durationMs.toFixed(1)} ms)`);
+          results.push({
+            scenario: testCase.scenario,
+            toolName: testCase.toolName,
+            arguments: testCase.arguments,
+            status: 'passed',
+            durationMs,
+            detail: reason
+          });
+        }
+
+      } catch (error) {
+        const endTime = performance.now();
+        const durationMs = endTime - startTime;
+        const errorMessage = String(error?.message || error || '');
+        const lowerExpected = (testCase.expected || '').toString().toLowerCase();
+        const lowerError = errorMessage.toLowerCase();
+
+        // If the test explicitly expects a timeout (e.g. "timeout|error"), then
+        // an MCP/client timeout should be treated as the expected outcome rather
+        // than as a hard harness failure. Accept both "timeout" and "timed out"
+        // phrasing from different MCP client implementations.
+        if (lowerExpected.includes('timeout') && (lowerError.includes('timeout') || lowerError.includes('timed out'))) {
+          console.log(`[PASSED] ${testCase.scenario} (${durationMs.toFixed(1)} ms)`);
+          results.push({
+            scenario: testCase.scenario,
+            toolName: testCase.toolName,
+            arguments: testCase.arguments,
+            status: 'passed',
+            durationMs,
+            detail: errorMessage
+          });
+          continue;
+        }
+
+        console.log(`[FAILED] ${testCase.scenario} (${durationMs.toFixed(1)} ms) => Error: ${errorMessage}`);
+        results.push({
+          scenario: testCase.scenario,
+          toolName: testCase.toolName,
+          arguments: testCase.arguments,
+          status: 'failed',
+          durationMs,
+          detail: errorMessage
+        });
+      }
+    }
+
+    const resultsPath = await persistResults(toolName, results);
+    summarize(toolName, results, resultsPath);
+
+    const hasFailures = results.some((result) => result.status === 'failed');
+    process.exitCode = hasFailures ? 1 : 0;
+
+  } catch (error) {
+    console.error('Test runner failed:', error);
+    process.exit(1);
+  } finally {
+    if (client) {
+      try {
+        await client.close();
+      } catch {
+        // ignore
+      }
+    }
+    if (transport) {
+      try {
+        await transport.close();
+      } catch {
+        // ignore
+      }
+    }
+  }
+}
+
+export class TestRunner {
+  constructor(suiteName) {
+    this.suiteName = suiteName || 'Test Suite';
+    this.steps = [];
+  }
+
+  addStep(name, fn) {
+    this.steps.push({ name, fn });
+  }
+
+  async run() {
+    if (this.steps.length === 0) {
+      console.warn(`No steps registered for ${this.suiteName}`);
+      return;
+    }
+
+    console.log('\n' + '='.repeat(60));
+    console.log(`${this.suiteName}`);
+    console.log('='.repeat(60));
+    console.log(`Total steps: ${this.steps.length}`);
+    console.log('');
+
+    let transport;
+    let client;
+    const results = [];
+
+    try {
+      const bridgeHost = process.env.MCP_AUTOMATION_WS_HOST ?? '127.0.0.1';
+      const envPorts = process.env.MCP_AUTOMATION_WS_PORTS
+        ? process.env.MCP_AUTOMATION_WS_PORTS.split(',').map((p) => parseInt(p.trim(), 10)).filter(Boolean)
+        : [8090, 8091];
+      const waitMs = parseInt(process.env.UNREAL_MCP_WAIT_PORT_MS ?? '5000', 10);
+
+      async function waitForAnyPort(host, ports, timeoutMs = 10000) {
+        const start = Date.now();
+        while (Date.now() - start < timeoutMs) {
+          for (const port of ports) {
+            try {
+              await new Promise((resolve, reject) => {
+                const sock = new net.Socket();
+                let settled = false;
+                sock.setTimeout(1000);
+                sock.once('connect', () => { settled = true; sock.destroy(); resolve(true); });
+                sock.once('timeout', () => { if (!settled) { settled = true; sock.destroy(); reject(new Error('timeout')); } });
+                sock.once('error', () => { if (!settled) { settled = true; sock.destroy(); reject(new Error('error')); } });
+                sock.connect(port, host);
+              });
+              console.log(`✅ Automation bridge appears to be listening on ${host}:${port}`);
+              return port;
+            } catch {
+            }
+          }
+          await new Promise((r) => setImmediate(r));
+        }
+        throw new Error(`Timed out waiting for automation bridge on ports: ${ports.join(',')}`);
+      }
+
+      try {
+        await waitForAnyPort(bridgeHost, envPorts, waitMs);
+      } catch (err) {
+        console.warn('Automation bridge did not become available before tests started:', err.message);
+      }
+
+      const distPath = path.join(repoRoot, 'dist', 'cli.js');
+      const srcDir = path.join(repoRoot, 'src');
+
+      async function getLatestMtime(dir) {
+        let latest = 0;
+        try {
+          const entries = await fs.readdir(dir, { withFileTypes: true });
+          for (const e of entries) {
+            const full = path.join(dir, e.name);
+            if (e.isDirectory()) {
+              const child = await getLatestMtime(full);
+              if (child > latest) latest = child;
+            } else {
+              try {
+                const st = await fs.stat(full);
+                const m = st.mtimeMs || 0;
+                if (m > latest) latest = m;
+              } catch (_) { }
+            }
+          }
+        } catch (_) {
+        }
+        return latest;
+      }
+
+      let useDist = false;
+      let distExists = false;
+      try {
+        await fs.access(distPath);
+        distExists = true;
+      } catch (e) {
+        distExists = false;
+      }
+
+      if (process.env.UNREAL_MCP_FORCE_DIST === '1') {
+        useDist = true;
+        console.log('Forcing use of dist build via UNREAL_MCP_FORCE_DIST=1');
+      } else if (distExists) {
+        try {
+          const distStat = await fs.stat(distPath);
+          const srcLatest = await getLatestMtime(srcDir);
+          const srcIsNewer = srcLatest > (distStat.mtimeMs || 0);
+          const autoBuildEnabled = process.env.UNREAL_MCP_AUTO_BUILD === '1';
+          const autoBuildDisabled = process.env.UNREAL_MCP_NO_AUTO_BUILD === '1';
+          if (srcIsNewer) {
+            if (!autoBuildEnabled && !autoBuildDisabled) {
+              console.log('Detected newer source files than dist; attempting automatic build to refresh dist/ (set UNREAL_MCP_NO_AUTO_BUILD=1 to disable)');
+            }
+            if (autoBuildEnabled || !autoBuildDisabled) {
+              const { spawn } = await import('node:child_process');
+              try {
+                await new Promise((resolve, reject) => {
+                  const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+                  const ps = spawn(npmCmd, ['run', 'build'], { cwd: repoRoot, stdio: 'inherit', shell: process.platform === 'win32' });
+                  ps.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`Build failed with code ${code}`))));
+                  ps.on('error', (err) => reject(err));
+                });
+                console.log('Build succeeded — using dist/ for live tests');
+                useDist = true;
+              } catch (buildErr) {
+                console.warn('Automatic build failed or could not stat files — falling back to TypeScript source for live tests:', String(buildErr));
+                useDist = false;
+              }
+            } else {
+              console.log('Detected newer source files than dist but automatic build is disabled.');
+              console.log('Set UNREAL_MCP_AUTO_BUILD=1 to enable automatic builds, or run `npm run build` manually.');
+              useDist = false;
+            }
+          } else {
+            useDist = true;
+            console.log('Using built dist for live tests');
+          }
+        } catch (buildErr) {
+          console.warn('Automatic build failed or could not stat files — falling back to TypeScript source for live tests:', String(buildErr));
+          useDist = false;
+          console.log('Preferring TypeScript source for tests to pick up local changes (set UNREAL_MCP_FORCE_DIST=1 to force dist)');
+        }
+      } else {
+        console.log('dist not found — attempting to run `npm run build` to produce dist/ for live tests');
+        try {
+          const { spawn } = await import('node:child_process');
+          await new Promise((resolve, reject) => {
+            const ps = spawn(process.platform === 'win32' ? 'npm.cmd' : 'npm', ['run', 'build'], { cwd: repoRoot, stdio: 'inherit' });
+            ps.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`Build failed with code ${code}`))));
+            ps.on('error', (err) => reject(err));
+          });
+          useDist = true;
+          console.log('Build succeeded — using dist/ for live tests');
+        } catch (buildErr) {
+          console.warn('Automatic build failed — falling back to running TypeScript source with ts-node-esm:', String(buildErr));
+          useDist = false;
+        }
+      }
+
+      if (!useDist) {
+        serverCommand = process.env.UNREAL_MCP_SERVER_CMD ?? 'npx';
+        serverArgs = ['ts-node-esm', path.join(repoRoot, 'src', 'cli.ts')];
+      } else {
+        serverCommand = process.env.UNREAL_MCP_SERVER_CMD ?? serverCommand;
+        serverArgs = process.env.UNREAL_MCP_SERVER_ARGS?.split(',') ?? serverArgs;
+      }
+
+      transport = new StdioClientTransport({
+        command: serverCommand,
+        args: serverArgs,
+        cwd: serverCwd,
+        stderr: 'inherit',
+        env: serverEnv
+      });
+
+      client = new Client({
+        name: 'unreal-mcp-step-runner',
+        version: '1.0.0'
+      });
+
+      await client.connect(transport);
+      await client.listTools({});
+      console.log('✅ Connected to Unreal MCP Server\n');
+
+      const callToolOnce = async function (callOptions, baseTimeoutMs) {
+        const envDefault = Number(process.env.UNREAL_MCP_TEST_CALL_TIMEOUT_MS ?? '60000') || 60000;
+        const perCall = Number(callOptions?.arguments?.timeoutMs) || undefined;
+        const base = typeof baseTimeoutMs === 'number' && baseTimeoutMs > 0 ? baseTimeoutMs : (perCall || envDefault);
+        const timeoutMs = base;
+        try {
+          console.log(`[CALL] ${callOptions.name} (timeout ${timeoutMs}ms)`);
+          const outgoing = Object.assign({}, callOptions, { arguments: { ...(callOptions.arguments || {}), timeoutMs } });
+          let callPromise;
+          try {
+            callPromise = client.callTool(outgoing, undefined, { timeout: timeoutMs });
+          } catch (err) {
+            try {
+              callPromise = client.callTool(outgoing, { timeout: timeoutMs });
+            } catch (inner) {
+              try {
+                callPromise = client.callTool(outgoing);
+              } catch (inner2) {
+                throw inner2 || inner || err;
+              }
+            }
+          }
+
+          let timeoutId;
+          const timeoutPromise = new Promise((_, rej) => {
+            timeoutId = setTimeout(() => rej(new Error(`Local test runner timeout after ${timeoutMs}ms`)), timeoutMs);
+            if (timeoutId && typeof timeoutId.unref === 'function') {
+              timeoutId.unref();
+            }
+          });
+          try {
+            const timed = Promise.race([
+              callPromise,
+              timeoutPromise
+            ]);
+            return await timed;
+          } finally {
+            if (timeoutId) {
+              clearTimeout(timeoutId);
+            }
+          }
+        } catch (e) {
+          const msg = String(e?.message || e || '');
+          if (msg.includes('Unknown blueprint action')) {
+            return { structuredContent: { success: false, error: msg } };
+          }
+          throw e;
+        }
+      };
+
+      const tools = {
+        async executeTool(toolName, args, options = {}) {
+          const timeoutMs = typeof options.timeoutMs === 'number' ? options.timeoutMs : undefined;
+          const response = await callToolOnce({ name: toolName, arguments: args }, timeoutMs);
+          let structuredContent = response.structuredContent ?? null;
+          if (structuredContent === null && response.content?.length) {
+            for (const entry of response.content) {
+              if (entry?.type !== 'text' || typeof entry.text !== 'string') continue;
+              try {
+                structuredContent = JSON.parse(entry.text);
+                break;
+              } catch {
+              }
+            }
+          }
+
+          if (structuredContent && typeof structuredContent === 'object') {
+            return structuredContent;
+          }
+
+          return {
+            success: !response.isError,
+            message: undefined,
+            error: undefined
+          };
+        }
+      };
+
+      for (const step of this.steps) {
+        const startTime = performance.now();
+
+        try {
+          // Log step start to Unreal Engine console
+          const cleanName = (step.name || 'Unknown Step').replace(/"/g, "'");
+          await callToolOnce({
+            name: 'system_control',
+            arguments: { action: 'console_command', command: `Log "---- STARTING STEP: ${cleanName} ----"` }
+          }, 5000).catch(() => { });
+        } catch (e) { /* ignore */ }
+
+        try {
+          const ok = await step.fn(tools);
+          const durationMs = performance.now() - startTime;
+          const status = ok ? 'passed' : 'failed';
+          console.log(formatResultLine({ scenario: step.name }, status, ok ? '' : 'Step returned false', durationMs));
+          results.push({
+            scenario: step.name,
+            toolName: null,
+            arguments: null,
+            status,
+            durationMs,
+            detail: ok ? undefined : 'Step returned false'
+          });
+        } catch (err) {
+          const durationMs = performance.now() - startTime;
+          const detail = err?.message || String(err);
+          console.log(formatResultLine({ scenario: step.name }, 'failed', detail, durationMs));
+          results.push({
+            scenario: step.name,
+            toolName: null,
+            arguments: null,
+            status: 'failed',
+            durationMs,
+            detail
+          });
+        }
+      }
+
+      const resultsPath = await persistResults(this.suiteName, results);
+      summarize(this.suiteName, results, resultsPath);
+
+      const hasFailures = results.some((result) => result.status === 'failed');
+      process.exitCode = hasFailures ? 1 : 0;
+    } catch (error) {
+      console.error('Step-based test runner failed:', error);
+      process.exit(1);
+    } finally {
+      if (client) {
+        try {
+          await client.close();
+        } catch {
+        }
+      }
+      if (transport) {
+        try {
+          await transport.close();
+        } catch {
+        }
+      }
+    }
+  }
+}
+