screenhand 0.1.0 → 0.1.1

package/native/windows-bridge/Program.cs

@@ -0,0 +1,265 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace WindowsBridge;
+
+/// <summary>
+/// JSON-RPC over stdio bridge for Windows native APIs.
+/// Reads JSON requests from stdin (one per line), dispatches to the appropriate bridge,
+/// and writes JSON responses to stdout (one per line).
+/// Mirrors the protocol of the macOS Swift bridge exactly.
+/// </summary>
+class Program
+{
+    private static readonly AppManagement _appManagement = new();
+    private static readonly UIAutomationBridge _uiAutomation = new();
+    private static readonly InputBridge _input = new();
+    private static readonly ScreenCapture _screenCapture = new();
+
+    private static readonly object _outputLock = new();
+    private static readonly JsonSerializerOptions _jsonOptions = new()
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        WriteIndented = false,
+    };
+
+    static void Main(string[] args)
+    {
+        Console.InputEncoding = System.Text.Encoding.UTF8;
+        Console.OutputEncoding = System.Text.Encoding.UTF8;
+
+        string? line;
+        while ((line = Console.ReadLine()) != null)
+        {
+            if (string.IsNullOrWhiteSpace(line)) continue;
+
+            try
+            {
+                var request = JsonSerializer.Deserialize<JsonRpcRequest>(line, _jsonOptions);
+                if (request == null)
+                {
+                    WriteError(0, -32700, "Parse error: null request");
+                    continue;
+                }
+
+                try
+                {
+                    var result = Dispatch(request.Method, request.Params);
+                    WriteResult(request.Id, result);
+                }
+                catch (BridgeException ex)
+                {
+                    WriteError(request.Id, -1, ex.Message);
+                }
+                catch (Exception ex)
+                {
+                    WriteError(request.Id, -1, ex.Message);
+                }
+            }
+            catch (Exception ex)
+            {
+                WriteError(0, -32700, $"Parse error: {ex.Message}");
+            }
+        }
+    }
+
+    private static object Dispatch(string method, JsonObject? p)
+    {
+        return method switch
+        {
+            // Lifecycle
+            "ping" => new Dictionary<string, object>
+            {
+                ["pong"] = true,
+                ["pid"] = Environment.ProcessId,
+                ["accessible"] = true, // UI Automation doesn't need special permissions on Windows
+            },
+            "check_permissions" => new Dictionary<string, object>
+            {
+                ["trusted"] = true, // No special permissions needed on Windows for UIA
+            },
+
+            // App Management
+            "app.launch" => _appManagement.LaunchApp(RequiredParam<string>(p, "bundleId")),
+            "app.focus" => _appManagement.FocusApp(RequiredParam<string>(p, "bundleId")),
+            "app.list" => _appManagement.ListRunningApps(),
+            "app.windows" => _appManagement.ListWindows(),
+            "app.frontmost" => _appManagement.FrontmostApp(),
+
+            // UI Automation (Accessibility equivalent)
+            "ax.findElement" => _uiAutomation.FindElement(
+                RequiredParam<int>(p, "pid"),
+                Param<string>(p, "role"),
+                Param<string>(p, "title"),
+                Param<string>(p, "value"),
+                Param<string>(p, "identifier"),
+                Param<bool>(p, "exact") ?? true),
+            "ax.getElementTree" => _uiAutomation.GetElementTree(
+                RequiredParam<int>(p, "pid"),
+                Param<int>(p, "maxDepth") ?? 5),
+            "ax.performAction" => _uiAutomation.PerformAction(
+                RequiredParam<int>(p, "pid"),
+                RequiredParam<int[]>(p, "elementPath"),
+                Param<string>(p, "action") ?? "AXPress"),
+            "ax.setElementValue" => _uiAutomation.SetElementValue(
+                RequiredParam<int>(p, "pid"),
+                RequiredParam<int[]>(p, "elementPath"),
+                RequiredParam<string>(p, "value")),
+            "ax.getElementValue" => _uiAutomation.GetElementValue(
+                RequiredParam<int>(p, "pid"),
+                RequiredParam<int[]>(p, "elementPath")),
+            "ax.menuClick" => _uiAutomation.MenuClick(
+                RequiredParam<int>(p, "pid"),
+                RequiredParam<string[]>(p, "menuPath")),
+
+            // Observer (stub — Windows UIA events could be added later)
+            "observer.start" => new Dictionary<string, object>
+            {
+                ["ok"] = true,
+                ["stub"] = true,
+                ["message"] = "UI Automation event observation not yet implemented on Windows",
+            },
+            "observer.stop" => new Dictionary<string, object>
+            {
+                ["ok"] = true,
+                ["stub"] = true,
+                ["message"] = "UI Automation event observation not yet implemented on Windows",
+            },
+
+            // Input (CoreGraphics equivalent)
+            "cg.mouseClick" => _input.MouseClick(
+                RequiredParam<double>(p, "x"),
+                RequiredParam<double>(p, "y"),
+                Param<string>(p, "button") ?? "left",
+                Param<int>(p, "clickCount") ?? 1),
+            "cg.mouseMove" => _input.MouseMove(
+                RequiredParam<double>(p, "x"),
+                RequiredParam<double>(p, "y")),
+            "cg.mouseDrag" => _input.MouseDrag(
+                RequiredParam<double>(p, "fromX"),
+                RequiredParam<double>(p, "fromY"),
+                RequiredParam<double>(p, "toX"),
+                RequiredParam<double>(p, "toY")),
+            "cg.mouseFlick" => _input.MouseDrag( // Map flick to fast drag on Windows
+                RequiredParam<double>(p, "fromX"),
+                RequiredParam<double>(p, "fromY"),
+                RequiredParam<double>(p, "toX"),
+                RequiredParam<double>(p, "toY")),
+            "cg.keyCombo" => _input.KeyCombo(RequiredParam<string[]>(p, "keys")),
+            "cg.typeText" => _input.TypeText(RequiredParam<string>(p, "text")),
+            "cg.scroll" => _input.Scroll(
+                RequiredParam<double>(p, "x"),
+                RequiredParam<double>(p, "y"),
+                Param<int>(p, "deltaX") ?? 0,
+                Param<int>(p, "deltaY") ?? 0),
+            "cg.captureScreen" => _screenCapture.CaptureScreen(
+                Param<Dictionary<string, double>>(p, "region")),
+            "cg.captureWindow" => _screenCapture.CaptureWindow(
+                RequiredParam<int>(p, "windowId")),
+
+            // Vision (OCR)
+            "vision.findText" => _screenCapture.FindText(
+                RequiredParam<string>(p, "imagePath"),
+                Param<string>(p, "searchText")),
+            "vision.ocr" => _screenCapture.Ocr(
+                RequiredParam<string>(p, "imagePath")),
+
+            _ => throw new BridgeException($"Unknown method: {method}"),
+        };
+    }
+
+    // Parameter helpers (mirror Swift's param/requiredParam)
+    private static T? Param<T>(JsonObject? p, string key)
+    {
+        if (p == null || !p.ContainsKey(key) || p[key] == null) return default;
+
+        var node = p[key]!;
+
+        // Handle numeric coercion
+        if (typeof(T) == typeof(double) && node is JsonValue jv)
+        {
+            if (jv.TryGetValue<double>(out var d)) return (T)(object)d;
+            if (jv.TryGetValue<int>(out var i)) return (T)(object)(double)i;
+            if (jv.TryGetValue<long>(out var l)) return (T)(object)(double)l;
+        }
+        if (typeof(T) == typeof(int) && node is JsonValue jv2)
+        {
+            if (jv2.TryGetValue<int>(out var i)) return (T)(object)i;
+            if (jv2.TryGetValue<double>(out var d)) return (T)(object)(int)d;
+            if (jv2.TryGetValue<long>(out var l)) return (T)(object)(int)l;
+        }
+
+        try
+        {
+            return node.Deserialize<T>(_jsonOptions);
+        }
+        catch
+        {
+            return default;
+        }
+    }
+
+    private static T RequiredParam<T>(JsonObject? p, string key)
+    {
+        var value = Param<T>(p, key);
+        if (value == null)
+            throw new BridgeException($"Missing required parameter: {key}");
+        return value;
+    }
+
+    // Output helpers
+    private static void WriteResult(int id, object result)
+    {
+        var response = new Dictionary<string, object?>
+        {
+            ["id"] = id,
+            ["result"] = result,
+            ["error"] = null,
+        };
+        WriteLine(response);
+    }
+
+    private static void WriteError(int id, int code, string message)
+    {
+        var response = new Dictionary<string, object?>
+        {
+            ["id"] = id,
+            ["result"] = null,
+            ["error"] = new Dictionary<string, object> { ["code"] = code, ["message"] = message },
+        };
+        WriteLine(response);
+    }
+
+    public static void WriteEvent(Dictionary<string, object> eventData)
+    {
+        var wrapped = new Dictionary<string, object>
+        {
+            ["id"] = 0,
+            ["event"] = eventData,
+        };
+        WriteLine(wrapped);
+    }
+
+    private static void WriteLine(object obj)
+    {
+        var json = JsonSerializer.Serialize(obj, _jsonOptions);
+        lock (_outputLock)
+        {
+            Console.WriteLine(json);
+            Console.Out.Flush();
+        }
+    }
+}
+
+// JSON-RPC types
+class JsonRpcRequest
+{
+    public int Id { get; set; }
+    public string Method { get; set; } = "";
+    public JsonObject? Params { get; set; }
+}
+
+class BridgeException : Exception
+{
+    public BridgeException(string message) : base(message) { }
+}

package/native/windows-bridge/ScreenCapture.cs

@@ -0,0 +1,329 @@
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Runtime.InteropServices;
+
+namespace WindowsBridge;
+
+/// <summary>
+/// Screenshot capture and OCR.
+/// Equivalent to macOS CoreGraphicsBridge (capture) + VisionBridge (OCR).
+/// Uses GDI+ for screenshots and Windows.Media.Ocr for text recognition.
+/// </summary>
+class ScreenCapture
+{
+    [DllImport("user32.dll")]
+    private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
+
+    [DllImport("user32.dll")]
+    private static extern bool PrintWindow(IntPtr hWnd, IntPtr hdcBlt, uint nFlags);
+
+    [DllImport("user32.dll")]
+    private static extern IntPtr GetDesktopWindow();
+
+    [DllImport("user32.dll")]
+    private static extern IntPtr GetWindowDC(IntPtr hWnd);
+
+    [DllImport("user32.dll")]
+    private static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC);
+
+    [DllImport("gdi32.dll")]
+    private static extern IntPtr CreateCompatibleDC(IntPtr hdc);
+
+    [DllImport("gdi32.dll")]
+    private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
+
+    [DllImport("gdi32.dll")]
+    private static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
+
+    [DllImport("gdi32.dll")]
+    private static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int wDest, int hDest,
+        IntPtr hdcSrc, int xSrc, int ySrc, uint rop);
+
+    [DllImport("gdi32.dll")]
+    private static extern bool DeleteDC(IntPtr hdc);
+
+    [DllImport("gdi32.dll")]
+    private static extern bool DeleteObject(IntPtr hObject);
+
+    [DllImport("user32.dll")]
+    private static extern int GetSystemMetrics(int nIndex);
+
+    [StructLayout(LayoutKind.Sequential)]
+    private struct RECT
+    {
+        public int Left, Top, Right, Bottom;
+    }
+
+    private const int SM_CXSCREEN = 0;
+    private const int SM_CYSCREEN = 1;
+    private const int SM_XVIRTUALSCREEN = 76;
+    private const int SM_YVIRTUALSCREEN = 77;
+    private const int SM_CXVIRTUALSCREEN = 78;
+    private const int SM_CYVIRTUALSCREEN = 79;
+    private const uint SRCCOPY = 0x00CC0020;
+    private const uint PW_RENDERFULLCONTENT = 0x00000002;
+
+    private static readonly string _tempDir = Path.Combine(Path.GetTempPath(), "screenhand");
+
+    static ScreenCapture()
+    {
+        Directory.CreateDirectory(_tempDir);
+    }
+
+    /// <summary>
+    /// Capture the full screen or a region.
+    /// </summary>
+    public Dictionary<string, object> CaptureScreen(Dictionary<string, double>? region)
+    {
+        int x, y, width, height;
+
+        if (region != null)
+        {
+            x = (int)region.GetValueOrDefault("x", 0);
+            y = (int)region.GetValueOrDefault("y", 0);
+            width = (int)region.GetValueOrDefault("width", GetSystemMetrics(SM_CXSCREEN));
+            height = (int)region.GetValueOrDefault("height", GetSystemMetrics(SM_CYSCREEN));
+        }
+        else
+        {
+            // Capture virtual screen (all monitors)
+            x = GetSystemMetrics(SM_XVIRTUALSCREEN);
+            y = GetSystemMetrics(SM_YVIRTUALSCREEN);
+            width = GetSystemMetrics(SM_CXVIRTUALSCREEN);
+            height = GetSystemMetrics(SM_CYVIRTUALSCREEN);
+
+            // Fallback to primary monitor
+            if (width == 0 || height == 0)
+            {
+                x = 0;
+                y = 0;
+                width = GetSystemMetrics(SM_CXSCREEN);
+                height = GetSystemMetrics(SM_CYSCREEN);
+            }
+        }
+
+        using var bitmap = new Bitmap(width, height, PixelFormat.Format32bppArgb);
+        using var graphics = Graphics.FromImage(bitmap);
+        graphics.CopyFromScreen(x, y, 0, 0, new Size(width, height), CopyPixelOperation.SourceCopy);
+
+        var filePath = Path.Combine(_tempDir, $"screen_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}.png");
+        bitmap.Save(filePath, ImageFormat.Png);
+
+        return new Dictionary<string, object>
+        {
+            ["path"] = filePath,
+            ["width"] = width,
+            ["height"] = height,
+        };
+    }
+
+    /// <summary>
+    /// Capture a specific window by its window handle (passed as windowId).
+    /// </summary>
+    public Dictionary<string, object> CaptureWindow(int windowId)
+    {
+        var hWnd = new IntPtr(windowId);
+        GetWindowRect(hWnd, out RECT rect);
+
+        int width = rect.Right - rect.Left;
+        int height = rect.Bottom - rect.Top;
+
+        if (width <= 0 || height <= 0)
+            throw new BridgeException($"Window {windowId} has invalid dimensions");
+
+        using var bitmap = new Bitmap(width, height, PixelFormat.Format32bppArgb);
+        using var graphics = Graphics.FromImage(bitmap);
+
+        // Try PrintWindow first (works for off-screen windows)
+        var hdc = graphics.GetHdc();
+        bool success = PrintWindow(hWnd, hdc, PW_RENDERFULLCONTENT);
+        graphics.ReleaseHdc(hdc);
+
+        if (!success)
+        {
+            // Fallback to screen capture of the window area
+            graphics.CopyFromScreen(rect.Left, rect.Top, 0, 0,
+                new Size(width, height), CopyPixelOperation.SourceCopy);
+        }
+
+        var filePath = Path.Combine(_tempDir, $"window_{windowId}_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}.png");
+        bitmap.Save(filePath, ImageFormat.Png);
+
+        return new Dictionary<string, object>
+        {
+            ["path"] = filePath,
+            ["width"] = width,
+            ["height"] = height,
+        };
+    }
+
+    /// <summary>
+    /// OCR an image file. Uses Windows.Media.Ocr when available, falls back to basic implementation.
+    /// </summary>
+    public Dictionary<string, object> Ocr(string imagePath)
+    {
+        if (!File.Exists(imagePath))
+            throw new BridgeException($"Image file not found: {imagePath}");
+
+        try
+        {
+            return OcrWithWindowsMediaOcr(imagePath);
+        }
+        catch
+        {
+            // Fallback: return empty result with a message
+            return new Dictionary<string, object>
+            {
+                ["text"] = "",
+                ["regions"] = new List<object>(),
+                ["error"] = "Windows.Media.Ocr not available. Install Windows 10 1809+ for built-in OCR.",
+            };
+        }
+    }
+
+    /// <summary>
+    /// Find text in an image using OCR.
+    /// </summary>
+    public Dictionary<string, object> FindText(string imagePath, string? searchText)
+    {
+        var ocrResult = Ocr(imagePath);
+
+        if (string.IsNullOrEmpty(searchText))
+            return ocrResult;
+
+        var regions = ocrResult["regions"] as List<object> ?? new List<object>();
+        var matches = regions
+            .Cast<Dictionary<string, object>>()
+            .Where(r => r.ContainsKey("text") &&
+                       r["text"].ToString()!.Contains(searchText, StringComparison.OrdinalIgnoreCase))
+            .ToList();
+
+        return new Dictionary<string, object>
+        {
+            ["text"] = ocrResult["text"],
+            ["matches"] = matches,
+            ["matchCount"] = matches.Count,
+        };
+    }
+
+    /// <summary>
+    /// OCR using Windows.Media.Ocr (available on Windows 10 1809+).
+    /// Uses dynamic loading to avoid compile-time dependency on WinRT.
+    /// </summary>
+    private Dictionary<string, object> OcrWithWindowsMediaOcr(string imagePath)
+    {
+        // Use PowerShell to invoke Windows.Media.Ocr
+        // This avoids WinRT interop complexity while still using the built-in OCR engine
+        var script = $@"
+Add-Type -AssemblyName System.Runtime.WindowsRuntime
+$null = [Windows.Media.Ocr.OcrEngine, Windows.Foundation.UniversalApiContract, ContentType = WindowsRuntime]
+$null = [Windows.Graphics.Imaging.BitmapDecoder, Windows.Foundation.UniversalApiContract, ContentType = WindowsRuntime]
+$null = [Windows.Storage.StorageFile, Windows.Foundation.UniversalApiContract, ContentType = WindowsRuntime]
+
+function Await($WinRtTask, $ResultType) {{
+    $asTask = $WinRtTask.GetType().GetMethod('AsTask', [Type[]]@())
+    if ($asTask -eq $null) {{
+        $asTaskGeneric = [System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object {{ $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.IsGenericMethod }} | Select-Object -First 1
+        $asTask = $asTaskGeneric.MakeGenericMethod($ResultType)
+        $task = $asTask.Invoke($null, @($WinRtTask))
+    }} else {{
+        $task = $asTask.Invoke($WinRtTask, @())
+    }}
+    $task.Wait()
+    return $task.Result
+}}
+
+$file = Await ([Windows.Storage.StorageFile]::GetFileFromPathAsync('{imagePath.Replace("'", "''")}')) ([Windows.Storage.StorageFile])
+$stream = Await ($file.OpenAsync([Windows.Storage.FileAccessMode]::Read)) ([Windows.Storage.Streams.IRandomAccessStream])
+$decoder = Await ([Windows.Graphics.Imaging.BitmapDecoder]::CreateAsync($stream)) ([Windows.Graphics.Imaging.BitmapDecoder])
+$bitmap = Await ($decoder.GetSoftwareBitmapAsync()) ([Windows.Graphics.Imaging.SoftwareBitmap])
+
+$engine = [Windows.Media.Ocr.OcrEngine]::TryCreateFromUserProfileLanguages()
+$result = Await ($engine.RecognizeAsync($bitmap)) ([Windows.Media.Ocr.OcrResult])
+
+$output = @{{
+    text = $result.Text
+    regions = @()
+}}
+
+foreach ($line in $result.Lines) {{
+    foreach ($word in $line.Words) {{
+        $output.regions += @{{
+            text = $word.Text
+            bounds = @{{
+                x = $word.BoundingRect.X
+                y = $word.BoundingRect.Y
+                width = $word.BoundingRect.Width
+                height = $word.BoundingRect.Height
+            }}
+        }}
+    }}
+}}
+
+$output | ConvertTo-Json -Depth 5
+";
+
+        try
+        {
+            var psi = new System.Diagnostics.ProcessStartInfo
+            {
+                FileName = "powershell.exe",
+                Arguments = $"-NoProfile -NonInteractive -Command -",
+                UseShellExecute = false,
+                RedirectStandardInput = true,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true,
+            };
+
+            using var process = System.Diagnostics.Process.Start(psi)!;
+            process.StandardInput.Write(script);
+            process.StandardInput.Close();
+
+            var output = process.StandardOutput.ReadToEnd();
+            process.WaitForExit(15000);
+
+            if (process.ExitCode != 0)
+            {
+                var stderr = process.StandardError.ReadToEnd();
+                throw new Exception($"PowerShell OCR failed: {stderr}");
+            }
+
+            // Parse the JSON output
+            var jsonDoc = System.Text.Json.JsonDocument.Parse(output);
+            var root = jsonDoc.RootElement;
+
+            var text = root.GetProperty("text").GetString() ?? "";
+            var regions = new List<object>();
+
+            if (root.TryGetProperty("regions", out var regionsElement))
+            {
+                foreach (var region in regionsElement.EnumerateArray())
+                {
+                    var bounds = region.GetProperty("bounds");
+                    regions.Add(new Dictionary<string, object>
+                    {
+                        ["text"] = region.GetProperty("text").GetString() ?? "",
+                        ["bounds"] = new Dictionary<string, object>
+                        {
+                            ["x"] = bounds.GetProperty("x").GetDouble(),
+                            ["y"] = bounds.GetProperty("y").GetDouble(),
+                            ["width"] = bounds.GetProperty("width").GetDouble(),
+                            ["height"] = bounds.GetProperty("height").GetDouble(),
+                        },
+                    });
+                }
+            }
+
+            return new Dictionary<string, object>
+            {
+                ["text"] = text,
+                ["regions"] = regions,
+            };
+        }
+        catch (Exception ex)
+        {
+            throw new BridgeException($"OCR failed: {ex.Message}");
+        }
+    }
+}