unreal-engine-mcp-server 0.2.1

package/src/tools/ui.ts

@@ -0,0 +1,499 @@
+// UI tools for Unreal Engine
+import { UnrealBridge } from '../unreal-bridge.js';
+
+export class UITools {
+  constructor(private bridge: UnrealBridge) {}
+
+  // Execute console command
+  private async _executeCommand(command: string) {
+    return this.bridge.httpCall('/remote/object/call', 'PUT', {
+      objectPath: '/Script/Engine.Default__KismetSystemLibrary',
+      functionName: 'ExecuteConsoleCommand',
+      parameters: {
+        Command: command,
+        SpecificPlayer: null
+      },
+      generateTransaction: false
+    });
+  }
+
+  // Create widget blueprint
+  async createWidget(params: {
+    name: string;
+    type?: 'HUD' | 'Menu' | 'Inventory' | 'Dialog' | 'Custom';
+    savePath?: string;
+  }) {
+    const path = params.savePath || '/Game/UI/Widgets';
+    const py = `
+import unreal
+import json
+name = r"${params.name}"
+path = r"${path}"
+try:
+    asset_tools = unreal.AssetToolsHelpers.get_asset_tools()
+    try:
+        factory = unreal.WidgetBlueprintFactory()
+    except Exception:
+        factory = None
+    if not factory:
+        print('RESULT:' + json.dumps({'success': False, 'error': 'WidgetBlueprintFactory unavailable'}))
+    else:
+        # Try setting parent_class in a version-tolerant way
+        try:
+            factory.parent_class = unreal.UserWidget
+        except Exception:
+            try:
+                factory.set_editor_property('parent_class', unreal.UserWidget)
+            except Exception:
+                pass
+        asset = asset_tools.create_asset(asset_name=name, package_path=path, asset_class=unreal.WidgetBlueprint, factory=factory)
+        if asset:
+            unreal.EditorAssetLibrary.save_asset(f"{path}/{name}")
+            print('RESULT:' + json.dumps({'success': True}))
+        else:
+            print('RESULT:' + json.dumps({'success': False, 'error': 'Failed to create WidgetBlueprint'}))
+except Exception as e:
+    print('RESULT:' + json.dumps({'success': False, 'error': str(e)}))
+`.trim();
+    try {
+      const resp = await this.bridge.executePython(py);
+      const out = typeof resp === 'string' ? resp : JSON.stringify(resp);
+      const m = out.match(/RESULT:({.*})/);
+      if (m) { try { const parsed = JSON.parse(m[1]); return parsed.success ? { success: true, message: 'Widget blueprint created' } : { success: false, error: parsed.error }; } catch {} }
+      return { success: true, message: 'Widget blueprint creation attempted' };
+    } catch (e) {
+      return { success: false, error: `Failed to create widget blueprint: ${e}` };
+    }
+  }
+
+  // Add widget component
+  async addWidgetComponent(params: {
+    widgetName: string;
+    componentType: 'Button' | 'Text' | 'Image' | 'ProgressBar' | 'Slider' | 'CheckBox' | 'ComboBox' | 'TextBox' | 'ScrollBox' | 'Canvas' | 'VerticalBox' | 'HorizontalBox' | 'Grid' | 'Overlay';
+    componentName: string;
+    slot?: {
+      position?: [number, number];
+      size?: [number, number];
+      anchor?: [number, number, number, number];
+      alignment?: [number, number];
+    };
+  }) {
+    const commands = [];
+    
+    commands.push(`AddWidgetComponent ${params.widgetName} ${params.componentType} ${params.componentName}`);
+    
+    if (params.slot) {
+      if (params.slot.position) {
+        commands.push(`SetWidgetPosition ${params.widgetName}.${params.componentName} ${params.slot.position.join(' ')}`);
+      }
+      if (params.slot.size) {
+        commands.push(`SetWidgetSize ${params.widgetName}.${params.componentName} ${params.slot.size.join(' ')}`);
+      }
+      if (params.slot.anchor) {
+        commands.push(`SetWidgetAnchor ${params.widgetName}.${params.componentName} ${params.slot.anchor.join(' ')}`);
+      }
+      if (params.slot.alignment) {
+        commands.push(`SetWidgetAlignment ${params.widgetName}.${params.componentName} ${params.slot.alignment.join(' ')}`);
+      }
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: `Component ${params.componentName} added to widget` };
+  }
+
+  // Set text
+  async setWidgetText(params: {
+    widgetName: string;
+    componentName: string;
+    text: string;
+    fontSize?: number;
+    color?: [number, number, number, number];
+    fontFamily?: string;
+  }) {
+    const commands = [];
+    
+    commands.push(`SetWidgetText ${params.widgetName}.${params.componentName} "${params.text}"`);
+    
+    if (params.fontSize !== undefined) {
+      commands.push(`SetWidgetFontSize ${params.widgetName}.${params.componentName} ${params.fontSize}`);
+    }
+    
+    if (params.color) {
+      commands.push(`SetWidgetTextColor ${params.widgetName}.${params.componentName} ${params.color.join(' ')}`);
+    }
+    
+    if (params.fontFamily) {
+      commands.push(`SetWidgetFont ${params.widgetName}.${params.componentName} ${params.fontFamily}`);
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: 'Widget text updated' };
+  }
+
+  // Set image
+  async setWidgetImage(params: {
+    widgetName: string;
+    componentName: string;
+    imagePath: string;
+    tint?: [number, number, number, number];
+    sizeToContent?: boolean;
+  }) {
+    const commands = [];
+    
+    commands.push(`SetWidgetImage ${params.widgetName}.${params.componentName} ${params.imagePath}`);
+    
+    if (params.tint) {
+      commands.push(`SetWidgetImageTint ${params.widgetName}.${params.componentName} ${params.tint.join(' ')}`);
+    }
+    
+    if (params.sizeToContent !== undefined) {
+      commands.push(`SetWidgetSizeToContent ${params.widgetName}.${params.componentName} ${params.sizeToContent}`);
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: 'Widget image updated' };
+  }
+
+  // Create HUD
+  async createHUD(params: {
+    name: string;
+    elements?: Array<{
+      type: 'HealthBar' | 'AmmoCounter' | 'Score' | 'Timer' | 'Minimap' | 'Crosshair';
+      position: [number, number];
+      size?: [number, number];
+    }>;
+  }) {
+    const commands = [];
+    
+    commands.push(`CreateHUDClass ${params.name}`);
+    
+    if (params.elements) {
+      for (const element of params.elements) {
+        const size = element.size || [100, 50];
+        commands.push(`AddHUDElement ${params.name} ${element.type} ${element.position.join(' ')} ${size.join(' ')}`);
+      }
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: `HUD ${params.name} created` };
+  }
+
+  // Show/Hide widget
+  async setWidgetVisibility(params: {
+    widgetName: string;
+    visible: boolean;
+    playerIndex?: number;
+  }) {
+    const playerIndex = params.playerIndex ?? 0;
+    const command = params.visible 
+      ? `ShowWidget ${params.widgetName} ${playerIndex}`
+      : `HideWidget ${params.widgetName} ${playerIndex}`;
+    
+    return this.bridge.executeConsoleCommand(command);
+  }
+
+  // Add widget to viewport
+  async addWidgetToViewport(params: {
+    widgetClass: string;
+    zOrder?: number;
+    playerIndex?: number;
+  }) {
+    const zOrder = params.zOrder ?? 0;
+    const playerIndex = params.playerIndex ?? 0;
+    
+    // Use Python API to create and add widget to viewport
+    const py = `
+import unreal
+import json
+widget_path = r"${params.widgetClass}"
+z_order = ${zOrder}
+player_index = ${playerIndex}
+try:
+    # Load the widget blueprint class
+    if not unreal.EditorAssetLibrary.does_asset_exist(widget_path):
+        print('RESULT:' + json.dumps({'success': False, 'error': f'Widget class not found: {widget_path}'}))
+    else:
+        widget_bp = unreal.EditorAssetLibrary.load_asset(widget_path)
+        if not widget_bp:
+            print('RESULT:' + json.dumps({'success': False, 'error': 'Failed to load widget blueprint'}))
+        else:
+            # Get the generated class from the widget blueprint
+            widget_class = widget_bp.generated_class() if hasattr(widget_bp, 'generated_class') else widget_bp
+            
+            # Get the world and player controller
+            world = unreal.EditorLevelLibrary.get_editor_world()
+            if not world:
+                print('RESULT:' + json.dumps({'success': False, 'error': 'No editor world available'}))
+            else:
+                # Try to get player controller
+                try:
+                    player_controller = unreal.GameplayStatics.get_player_controller(world, player_index)
+                except Exception:
+                    player_controller = None
+                
+                if not player_controller:
+                    # If no player controller in PIE, try to get the first one or create a dummy
+                    print('RESULT:' + json.dumps({'success': False, 'error': 'No player controller available. Run in PIE mode first.'}))
+                else:
+                    # Create the widget
+                    widget = unreal.WidgetBlueprintLibrary.create(world, widget_class, player_controller)
+                    if widget:
+                        # Add to viewport
+                        widget.add_to_viewport(z_order)
+                        print('RESULT:' + json.dumps({'success': True}))
+                    else:
+                        print('RESULT:' + json.dumps({'success': False, 'error': 'Failed to create widget instance'}))
+except Exception as e:
+    print('RESULT:' + json.dumps({'success': False, 'error': str(e)}))
+`.trim();
+    
+    try {
+      const resp = await this.bridge.executePython(py);
+      const out = typeof resp === 'string' ? resp : JSON.stringify(resp);
+      const m = out.match(/RESULT:({.*})/);
+      if (m) {
+        try {
+          const parsed = JSON.parse(m[1]);
+          return parsed.success 
+            ? { success: true, message: `Widget added to viewport with z-order ${zOrder}` }
+            : { success: false, error: parsed.error };
+        } catch {}
+      }
+      return { success: true, message: 'Widget add to viewport attempted' };
+    } catch (e) {
+      return { success: false, error: `Failed to add widget to viewport: ${e}` };
+    }
+  }
+
+  // Remove widget from viewport
+  async removeWidgetFromViewport(params: {
+    widgetName: string;
+    playerIndex?: number;
+  }) {
+    const playerIndex = params.playerIndex ?? 0;
+    const command = `RemoveWidgetFromViewport ${params.widgetName} ${playerIndex}`;
+    return this.bridge.executeConsoleCommand(command);
+  }
+
+  // Create menu
+  async createMenu(params: {
+    name: string;
+    menuType: 'Main' | 'Pause' | 'Settings' | 'Inventory';
+    buttons?: Array<{
+      text: string;
+      action: string;
+      position?: [number, number];
+    }>;
+  }) {
+    const commands = [];
+    
+    commands.push(`CreateMenuWidget ${params.name} ${params.menuType}`);
+    
+    if (params.buttons) {
+      for (const button of params.buttons) {
+        const pos = button.position || [0, 0];
+        commands.push(`AddMenuButton ${params.name} "${button.text}" ${button.action} ${pos.join(' ')}`);
+      }
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: `Menu ${params.name} created` };
+  }
+
+  // Set widget animation
+  async createWidgetAnimation(params: {
+    widgetName: string;
+    animationName: string;
+    duration: number;
+    tracks?: Array<{
+      componentName: string;
+      property: 'Position' | 'Scale' | 'Rotation' | 'Opacity' | 'Color';
+      keyframes: Array<{
+        time: number;
+        value: number | [number, number] | [number, number, number] | [number, number, number, number];
+      }>;
+    }>;
+  }) {
+    const commands = [];
+    
+    commands.push(`CreateWidgetAnimation ${params.widgetName} ${params.animationName} ${params.duration}`);
+    
+    if (params.tracks) {
+      for (const track of params.tracks) {
+        commands.push(`AddAnimationTrack ${params.widgetName}.${params.animationName} ${track.componentName} ${track.property}`);
+        
+        for (const keyframe of track.keyframes) {
+          const value = Array.isArray(keyframe.value) ? keyframe.value.join(' ') : keyframe.value;
+          commands.push(`AddAnimationKeyframe ${params.widgetName}.${params.animationName} ${track.componentName} ${keyframe.time} ${value}`);
+        }
+      }
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: `Animation ${params.animationName} created` };
+  }
+
+  // Play widget animation
+  async playWidgetAnimation(params: {
+    widgetName: string;
+    animationName: string;
+    playMode?: 'Forward' | 'Reverse' | 'PingPong';
+    loops?: number;
+  }) {
+    const playMode = params.playMode || 'Forward';
+    const loops = params.loops ?? 1;
+    
+    const command = `PlayWidgetAnimation ${params.widgetName} ${params.animationName} ${playMode} ${loops}`;
+    return this.bridge.executeConsoleCommand(command);
+  }
+
+  // Set widget style
+  async setWidgetStyle(params: {
+    widgetName: string;
+    componentName: string;
+    style: {
+      backgroundColor?: [number, number, number, number];
+      borderColor?: [number, number, number, number];
+      borderWidth?: number;
+      padding?: [number, number, number, number];
+      margin?: [number, number, number, number];
+    };
+  }) {
+    const commands = [];
+    
+    if (params.style.backgroundColor) {
+      commands.push(`SetWidgetBackgroundColor ${params.widgetName}.${params.componentName} ${params.style.backgroundColor.join(' ')}`);
+    }
+    
+    if (params.style.borderColor) {
+      commands.push(`SetWidgetBorderColor ${params.widgetName}.${params.componentName} ${params.style.borderColor.join(' ')}`);
+    }
+    
+    if (params.style.borderWidth !== undefined) {
+      commands.push(`SetWidgetBorderWidth ${params.widgetName}.${params.componentName} ${params.style.borderWidth}`);
+    }
+    
+    if (params.style.padding) {
+      commands.push(`SetWidgetPadding ${params.widgetName}.${params.componentName} ${params.style.padding.join(' ')}`);
+    }
+    
+    if (params.style.margin) {
+      commands.push(`SetWidgetMargin ${params.widgetName}.${params.componentName} ${params.style.margin.join(' ')}`);
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: 'Widget style updated' };
+  }
+
+  // Bind widget event
+  async bindWidgetEvent(params: {
+    widgetName: string;
+    componentName: string;
+    eventType: 'OnClicked' | 'OnPressed' | 'OnReleased' | 'OnHovered' | 'OnUnhovered' | 'OnTextChanged' | 'OnTextCommitted' | 'OnValueChanged';
+    functionName: string;
+  }) {
+    const command = `BindWidgetEvent ${params.widgetName}.${params.componentName} ${params.eventType} ${params.functionName}`;
+    return this.bridge.executeConsoleCommand(command);
+  }
+
+  // Set input mode
+  async setInputMode(params: {
+    mode: 'GameOnly' | 'UIOnly' | 'GameAndUI';
+    showCursor?: boolean;
+    lockCursor?: boolean;
+  }) {
+    const commands = [];
+    
+    commands.push(`SetInputMode ${params.mode}`);
+    
+    if (params.showCursor !== undefined) {
+      commands.push(`ShowMouseCursor ${params.showCursor}`);
+    }
+    
+    if (params.lockCursor !== undefined) {
+      commands.push(`SetMouseLockMode ${params.lockCursor}`);
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: `Input mode set to ${params.mode}` };
+  }
+
+  // Create tooltip
+  async createTooltip(params: {
+    widgetName: string;
+    componentName: string;
+    text: string;
+    delay?: number;
+  }) {
+    const delay = params.delay ?? 0.5;
+    const command = `SetWidgetTooltip ${params.widgetName}.${params.componentName} "${params.text}" ${delay}`;
+    return this.bridge.executeConsoleCommand(command);
+  }
+
+  // Create drag and drop
+  async setupDragDrop(params: {
+    widgetName: string;
+    componentName: string;
+    dragVisual?: string;
+    dropTargets?: string[];
+  }) {
+    const commands = [];
+    
+    commands.push(`EnableDragDrop ${params.widgetName}.${params.componentName}`);
+    
+    if (params.dragVisual) {
+      commands.push(`SetDragVisual ${params.widgetName}.${params.componentName} ${params.dragVisual}`);
+    }
+    
+    if (params.dropTargets) {
+      for (const target of params.dropTargets) {
+        commands.push(`AddDropTarget ${params.widgetName}.${params.componentName} ${target}`);
+      }
+    }
+    
+    for (const cmd of commands) {
+      await this.bridge.executeConsoleCommand(cmd);
+    }
+    
+    return { success: true, message: 'Drag and drop configured' };
+  }
+
+  // Create notification
+  async showNotification(params: {
+    text: string;
+    duration?: number;
+    type?: 'Info' | 'Success' | 'Warning' | 'Error';
+    position?: 'TopLeft' | 'TopCenter' | 'TopRight' | 'BottomLeft' | 'BottomCenter' | 'BottomRight';
+  }) {
+    const duration = params.duration ?? 3.0;
+    const type = params.type || 'Info';
+    const position = params.position || 'TopRight';
+    
+    const command = `ShowNotification "${params.text}" ${duration} ${type} ${position}`;
+    return this.bridge.executeConsoleCommand(command);
+  }
+}

package/src/tools/visual.ts

@@ -0,0 +1,60 @@
+import { UnrealBridge } from '../unreal-bridge.js';
+import { loadEnv } from '../types/env.js';
+import fs from 'fs';
+import path from 'path';
+
+export class VisualTools {
+  private env = loadEnv();
+  constructor(private bridge: UnrealBridge) {}
+
+  // Take a screenshot of viewport (high res or standard). Returns path and base64 (truncated)
+  async takeScreenshot(params: { resolution?: string }) {
+    const res = params.resolution && /^\d+x\d+$/i.test(params.resolution) ? params.resolution : '';
+    const cmd = res ? `HighResShot ${res}` : 'HighResShot';
+    try {
+      await this.bridge.executeConsoleCommand(cmd);
+      // Give the engine a moment to write the file
+await new Promise(r => setTimeout(r, 1200));
+      const p = await this.findLatestScreenshot();
+      if (!p) return { success: true, message: 'Screenshot triggered, but could not locate output file' };
+      let b64: string | undefined;
+      try {
+        const buf = fs.readFileSync(p);
+        // Limit to ~1MB to avoid huge responses
+        const max = 1024 * 1024;
+        b64 = buf.length > max ? buf.subarray(0, max).toString('base64') : buf.toString('base64');
+      } catch {}
+      return { success: true, imagePath: p, imageBase64: b64 };
+    } catch (err: any) {
+      return { success: false, error: String(err?.message || err) };
+    }
+  }
+
+  private async findLatestScreenshot(): Promise<string | null> {
+    // Try env override, otherwise look in common UE Saved/Screenshots folder under project
+    const candidates: string[] = [];
+    if (this.env.UE_SCREENSHOT_DIR) candidates.push(this.env.UE_SCREENSHOT_DIR);
+    if (this.env.UE_PROJECT_PATH) {
+      const projectDir = path.dirname(this.env.UE_PROJECT_PATH);
+      candidates.push(path.join(projectDir, 'Saved', 'Screenshots'));
+candidates.push(path.join(projectDir, 'Saved', 'Screenshots', 'Windows'));
+      candidates.push(path.join(projectDir, 'Saved', 'Screenshots', 'WindowsEditor'));
+    }
+    // Fallback: common locations
+candidates.push(path.join(process.cwd(), 'Saved', 'Screenshots'));
+    candidates.push(path.join(process.cwd(), 'Saved', 'Screenshots', 'Windows'));
+    candidates.push(path.join(process.cwd(), 'Saved', 'Screenshots', 'WindowsEditor'));
+    let latest: { path: string; mtime: number } | null = null;
+    for (const c of candidates) {
+      try {
+        const entries = fs.readdirSync(c).map(f => path.join(c, f));
+        for (const fp of entries) {
+          if (!/\.png$/i.test(fp)) continue;
+          const st = fs.statSync(fp);
+          if (!latest || st.mtimeMs > latest.mtime) latest = { path: fp, mtime: st.mtimeMs };
+        }
+      } catch {}
+    }
+    return latest?.path || null;
+  }
+}

package/src/types/env.ts

@@ -0,0 +1,27 @@
+export interface Env {
+  UE_HOST: string;
+  UE_RC_WS_PORT: number;
+  UE_RC_HTTP_PORT: number;
+  UE_PROJECT_PATH?: string;
+  UE_EDITOR_EXE?: string;
+  UE_SCREENSHOT_DIR?: string;
+}
+
+export function loadEnv(): Env {
+  const host = process.env.UE_HOST || '127.0.0.1';
+  // Note: UE5 default is HTTP on 30010, WebSocket on 30020
+  const wsPort = Number(process.env.UE_RC_WS_PORT || process.env.UE_REMOTE_CONTROL_WS_PORT || 30020);
+  const httpPort = Number(process.env.UE_RC_HTTP_PORT || process.env.UE_REMOTE_CONTROL_HTTP_PORT || 30010);
+  const projectPath = process.env.UE_PROJECT_PATH;
+  const editorExe = process.env.UE_EDITOR_EXE;
+  const screenshotDir = process.env.UE_SCREENSHOT_DIR;
+
+  return {
+    UE_HOST: host,
+    UE_RC_WS_PORT: wsPort,
+    UE_RC_HTTP_PORT: httpPort,
+    UE_PROJECT_PATH: projectPath,
+    UE_EDITOR_EXE: editorExe,
+    UE_SCREENSHOT_DIR: screenshotDir,
+  };
+}