screenhand 0.2.0 → 0.3.1

package/native/macos-bridge/Sources/StreamCapture.swift

@@ -0,0 +1,136 @@
+import Foundation
+import ScreenCaptureKit
+import CoreMedia
+import AppKit
+
+/// Continuous screen capture using SCStream.
+/// Keeps the latest frame as a temp PNG file on disk.
+/// Replaces one-shot CGWindowListCreateImage (~200ms) with pre-captured frames (~0ms read).
+class StreamCapture: NSObject, SCStreamOutput {
+    private var stream: SCStream?
+    private var _running = false
+    private let queue = DispatchQueue(label: "streamcapture.state")
+
+    /// Path to the latest captured frame (PNG file)
+    private var _latestFramePath: String?
+    private var _latestWidth: Int = 0
+    private var _latestHeight: Int = 0
+    private var _latestFrameTime: Date?
+    private var _frameCount: UInt64 = 0
+    private var saveEveryN: Int = 1
+
+    /// Start continuous capture for a specific window.
+    func start(windowId: Int, fps: Int = 30) async throws {
+        var alreadyRunning = false
+        queue.sync { alreadyRunning = self._running }
+        if alreadyRunning { return }
+
+        let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+        guard let window = content.windows.first(where: { $0.windowID == CGWindowID(windowId) }) else {
+            throw BridgeError.general("Window \(windowId) not found for stream capture")
+        }
+
+        self.saveEveryN = max(1, fps / 30)
+
+        let filter = SCContentFilter(desktopIndependentWindow: window)
+        let config = SCStreamConfiguration()
+        config.width = window.frame.width > 0 ? Int(window.frame.width) * 2 : 2880
+        config.height = window.frame.height > 0 ? Int(window.frame.height) * 2 : 1800
+        config.showsCursor = false
+        config.capturesAudio = false
+        config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
+        config.queueDepth = 3
+
+        let newStream = SCStream(filter: filter, configuration: config, delegate: nil)
+        try newStream.addStreamOutput(self, type: .screen, sampleHandlerQueue: .global(qos: .userInitiated))
+        try await newStream.startCapture()
+
+        queue.sync {
+            self.stream = newStream
+            self._running = true
+            self._frameCount = 0
+        }
+    }
+
+    /// Stop the stream and clean up.
+    func stop() async {
+        var s: SCStream?
+        var pathToClean: String?
+
+        queue.sync {
+            s = self.stream
+            self._running = false
+            self.stream = nil
+            pathToClean = self._latestFramePath
+            self._latestFramePath = nil
+        }
+
+        if let s = s {
+            try? await s.stopCapture()
+        }
+
+        if let path = pathToClean {
+            try? FileManager.default.removeItem(atPath: path)
+        }
+    }
+
+    var isRunning: Bool {
+        queue.sync { _running }
+    }
+
+    /// Get info about the latest frame.
+    func getLatestInfo() -> [String: Any]? {
+        queue.sync {
+            guard let path = _latestFramePath, let time = _latestFrameTime else { return nil }
+            return [
+                "path": path,
+                "width": _latestWidth,
+                "height": _latestHeight,
+                "ageMs": Int(Date().timeIntervalSince(time) * 1000),
+                "frameCount": _frameCount,
+            ]
+        }
+    }
+
+    // MARK: - SCStreamOutput
+
+    func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+        guard type == .screen else { return }
+
+        var shouldSave = false
+        queue.sync {
+            guard self._running else { shouldSave = false; return }
+            _frameCount += 1
+            shouldSave = _frameCount % UInt64(saveEveryN) == 0
+        }
+        guard shouldSave else { return }
+
+        guard let imageBuffer = sampleBuffer.imageBuffer else { return }
+        let ciImage = CIImage(cvImageBuffer: imageBuffer)
+        let context = CIContext()
+        let width = CVPixelBufferGetWidth(imageBuffer)
+        let height = CVPixelBufferGetHeight(imageBuffer)
+
+        guard let cgImage = context.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: width, height: height)) else { return }
+
+        let tempDir = FileManager.default.temporaryDirectory
+        let fileURL = tempDir.appendingPathComponent("stream_frame_latest.png")
+        let bitmapRep = NSBitmapImageRep(cgImage: cgImage)
+        guard let pngData = bitmapRep.representation(using: .png, properties: [:]) else { return }
+
+        do {
+            let tmpURL = tempDir.appendingPathComponent("stream_frame_tmp_\(ProcessInfo.processInfo.processIdentifier).png")
+            try pngData.write(to: tmpURL)
+            _ = try FileManager.default.replaceItemAt(fileURL, withItemAt: tmpURL)
+
+            queue.sync {
+                self._latestFramePath = fileURL.path
+                self._latestWidth = width
+                self._latestHeight = height
+                self._latestFrameTime = Date()
+            }
+        } catch {
+            // Skip frame on write failure
+        }
+    }
+}

package/native/macos-bridge/Sources/VisionBridge.swift

@@ -0,0 +1,238 @@
+import Foundation
+import Vision
+import AppKit
+import CoreML
+
+class VisionBridge {
+
+    // MARK: - YOLO Element Detection
+
+    private var yoloModel: VNCoreMLModel?
+    private let yoloClassNames = ["button", "field", "heading", "iframe", "image", "label", "link", "text"]
+
+    /// Load the YOLO CoreML model for UI element detection.
+    /// Called lazily on first detectElements call.
+    private func ensureYoloModel() throws {
+        if yoloModel != nil { return }
+
+        // Look for model relative to the binary or via known paths
+        let execPath = URL(fileURLWithPath: CommandLine.arguments[0]).resolvingSymlinksInPath()
+        let execDir = execPath.deletingLastPathComponent()
+        let possiblePaths = [
+            execDir.appendingPathComponent("../Resources/ui-elements.mlpackage"),
+            execDir.appendingPathComponent("../../Resources/ui-elements.mlpackage"),
+            execDir.appendingPathComponent("Resources/ui-elements.mlpackage"),
+            URL(fileURLWithPath: "/tmp/vins-yolo/web-ui-8cls.mlpackage"), // Development fallback
+        ]
+
+        for modelURL in possiblePaths {
+            if FileManager.default.fileExists(atPath: modelURL.path) {
+                let config = MLModelConfiguration()
+                config.computeUnits = .all // Use ANE when available
+                let coreMLModel = try MLModel(contentsOf: modelURL, configuration: config)
+                yoloModel = try VNCoreMLModel(for: coreMLModel)
+                return
+            }
+        }
+
+        throw BridgeError.general("YOLO model not found")
+    }
+
+    /// Detect UI elements in an image using the YOLO CoreML model.
+    /// Returns bounding boxes with class labels and confidence scores.
+    func detectElements(imagePath: String, confidence: Double = 0.25) throws -> [[String: Any]] {
+        try ensureYoloModel()
+        guard let model = yoloModel else {
+            throw BridgeError.general("YOLO model not loaded")
+        }
+
+        let url = URL(fileURLWithPath: imagePath)
+        guard let image = NSImage(contentsOf: url),
+              let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else {
+            throw BridgeError.general("Failed to load image at \(imagePath)")
+        }
+
+        let imageWidth = CGFloat(cgImage.width)
+        let imageHeight = CGFloat(cgImage.height)
+
+        let request = VNCoreMLRequest(model: model)
+        request.imageCropAndScaleOption = .scaleFit
+
+        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+        try handler.perform([request])
+
+        guard let observations = request.results as? [VNRecognizedObjectObservation] else {
+            return []
+        }
+
+        var results: [[String: Any]] = []
+        for obs in observations {
+            guard obs.confidence >= Float(confidence) else { continue }
+
+            let bbox = obs.boundingBox
+            // Convert from Vision normalized coords (origin bottom-left) to screen coords
+            let x = bbox.origin.x * imageWidth
+            let y = (1 - bbox.origin.y - bbox.height) * imageHeight
+            let width = bbox.width * imageWidth
+            let height = bbox.height * imageHeight
+
+            let className = obs.labels.first?.identifier ?? "unknown"
+
+            results.append([
+                "class": className,
+                "confidence": Double(obs.confidence),
+                "bounds": [
+                    "x": Double(x),
+                    "y": Double(y),
+                    "width": Double(width),
+                    "height": Double(height),
+                ] as [String: Any],
+            ] as [String: Any])
+        }
+
+        return results
+    }
+
+    /// Perform OCR on an image, optionally searching for specific text.
+    /// Returns all recognized text with bounding boxes.
+    func findText(imagePath: String, searchText: String?, mode: String = "accurate") throws -> [[String: Any]] {
+        let results = try performOCR(imagePath: imagePath, mode: mode)
+
+        guard let search = searchText?.lowercased() else {
+            return results
+        }
+
+        return results.filter { result in
+            guard let text = result["text"] as? String else { return false }
+            return text.lowercased().contains(search)
+        }
+    }
+
+    /// Full OCR of an image — returns all recognized text.
+    /// mode: "fast" (perception loop, 10x faster) or "accurate" (tool actions, default)
+    func ocr(imagePath: String, mode: String = "accurate") throws -> [String: Any] {
+        let results = try performOCR(imagePath: imagePath, mode: mode)
+        let fullText = results.compactMap { $0["text"] as? String }.joined(separator: "\n")
+        return [
+            "text": fullText,
+            "regions": results,
+        ]
+    }
+
+    /// OCR a specific region of a window — captures the window, crops to the ROI,
+    /// and runs text recognition on just that region. Returns bounds in window coordinates.
+    func ocrRegion(windowId: Int, region: [String: Double], mode: String = "accurate") throws -> [String: Any] {
+        let roiX = region["x"] ?? 0
+        let roiY = region["y"] ?? 0
+        let roiW = region["width"] ?? 0
+        let roiH = region["height"] ?? 0
+
+        // Capture the full window
+        guard let fullImage = CGWindowListCreateImage(
+            .null, .optionIncludingWindow, CGWindowID(windowId), [.bestResolution, .boundsIgnoreFraming]
+        ) else {
+            throw BridgeError.general("CGWindowListCreateImage returned nil for window \(windowId)")
+        }
+
+        // Crop to the ROI (CGImage coordinates have origin top-left, same as our ROI)
+        let cropRect = CGRect(x: roiX, y: roiY, width: roiW, height: roiH)
+            .intersection(CGRect(x: 0, y: 0, width: fullImage.width, height: fullImage.height))
+
+        guard !cropRect.isEmpty,
+              let cropped = fullImage.cropping(to: cropRect) else {
+            return ["text": "", "regions": [] as [Any]]
+        }
+
+        // Run OCR on cropped image
+        let results = try performOCROnImage(cropped, mode: mode)
+
+        // Translate bounds from cropped-image coordinates back to window coordinates
+        let adjustedResults: [[String: Any]] = results.map { entry in
+            var adjusted = entry
+            if var bounds = entry["bounds"] as? [String: Double] {
+                bounds["x"] = (bounds["x"] ?? 0) + roiX
+                bounds["y"] = (bounds["y"] ?? 0) + roiY
+                adjusted["bounds"] = bounds
+            }
+            return adjusted
+        }
+
+        let fullText = adjustedResults.compactMap { $0["text"] as? String }.joined(separator: "\n")
+        return [
+            "text": fullText,
+            "regions": adjustedResults,
+        ]
+    }
+
+    private func performOCR(imagePath: String, mode: String = "accurate") throws -> [[String: Any]] {
+        let url = URL(fileURLWithPath: imagePath)
+
+        guard let image = NSImage(contentsOf: url),
+              let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else {
+            throw BridgeError.general("Failed to load image at \(imagePath)")
+        }
+
+        return try performOCROnImage(cgImage, mode: mode)
+    }
+
+    private func performOCROnImage(_ cgImage: CGImage, mode: String = "accurate") throws -> [[String: Any]] {
+        let imageWidth = CGFloat(cgImage.width)
+        let imageHeight = CGFloat(cgImage.height)
+
+        let request = VNRecognizeTextRequest()
+
+        if mode == "fast" {
+            // FAST mode: 10x faster (~60ms vs ~631ms), used for perception loop
+            // Trades ~20% text coverage for speed — acceptable for frame diffing
+            request.recognitionLevel = .fast
+            request.usesLanguageCorrection = false
+            request.recognitionLanguages = ["en-US"]
+        } else {
+            // ACCURATE mode: full precision, used for tool actions (ocr, click_text)
+            request.recognitionLevel = .accurate
+            request.usesLanguageCorrection = true
+            if #available(macOS 13.0, *) {
+                request.automaticallyDetectsLanguage = true
+            }
+            let supportedLangs = try? request.supportedRecognitionLanguages()
+            if let supported = supportedLangs {
+                request.recognitionLanguages = supported
+            } else {
+                request.recognitionLanguages = ["en-US", "zh-Hans", "zh-Hant", "ja", "ko", "hi", "ar", "de", "fr", "es", "pt", "it", "ru"]
+            }
+        }
+
+        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+        try handler.perform([request])
+
+        guard let observations = request.results else {
+            return []
+        }
+
+        var results: [[String: Any]] = []
+
+        for observation in observations {
+            guard let candidate = observation.topCandidates(1).first else { continue }
+
+            let boundingBox = observation.boundingBox
+            // Convert from Vision's normalized coordinates (origin bottom-left) to screen coordinates
+            let x = boundingBox.origin.x * imageWidth
+            let y = (1 - boundingBox.origin.y - boundingBox.height) * imageHeight
+            let width = boundingBox.width * imageWidth
+            let height = boundingBox.height * imageHeight
+
+            results.append([
+                "text": candidate.string,
+                "confidence": Double(candidate.confidence),
+                "bounds": [
+                    "x": Double(x),
+                    "y": Double(y),
+                    "width": Double(width),
+                    "height": Double(height),
+                ] as [String: Any],
+            ] as [String: Any])
+        }
+
+        return results
+    }
+}