simframe 0.4.2 → 0.6.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +334 -85
  2. package/native/simframed/Package.swift +16 -0
  3. package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift +379 -0
  4. package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +523 -0
  5. package/native/simframed/Sources/PrivateAPI/HIDKeyboard.swift +70 -0
  6. package/native/simframed/Sources/PrivateAPI/IndigoHID.swift +121 -0
  7. package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +149 -0
  8. package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +112 -0
  9. package/native/simframed/Sources/SimframeCore/Bitmap.swift +61 -0
  10. package/native/simframed/Sources/SimframeCore/ControlSocket.swift +122 -0
  11. package/native/simframed/Sources/SimframeCore/CoreGraphicsScaler.swift +70 -0
  12. package/native/simframed/Sources/SimframeCore/Element.swift +148 -0
  13. package/native/simframed/Sources/SimframeCore/FrameStore.swift +303 -0
  14. package/native/simframed/Sources/SimframeCore/Hashing.swift +119 -0
  15. package/native/simframed/Sources/SimframeCore/Motion.swift +431 -0
  16. package/native/simframed/Sources/SimframeCore/PNGWriter.swift +40 -0
  17. package/native/simframed/Sources/SimframeCore/VisionOCR.swift +75 -0
  18. package/native/simframed/Sources/simframed/main.swift +485 -0
  19. package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +270 -0
  20. package/package.json +12 -4
  21. package/scripts/bench-flow.mjs +54 -0
  22. package/scripts/bench.sh +98 -0
  23. package/scripts/check-package.mjs +99 -0
  24. package/scripts/ci-memory.mjs +416 -0
  25. package/scripts/eval-fingerprint.mjs +192 -0
  26. package/scripts/smoke.mjs +76 -0
  27. package/scripts/sync-server-version.mjs +39 -0
  28. package/scripts/verify-baseline.mjs +65 -0
  29. package/skills/simframe/SKILL.md +173 -0
  30. package/src/actions.js +264 -18
  31. package/src/cli.js +561 -89
  32. package/src/control.js +77 -0
  33. package/src/daemon.js +8 -1
  34. package/src/engine.js +99 -0
  35. package/src/fingerprint.js +183 -0
  36. package/src/graph.js +411 -0
  37. package/src/index.js +351 -24
  38. package/src/input.js +179 -2
  39. package/src/matching.js +265 -0
  40. package/src/mcp.js +425 -112
  41. package/src/navigate.js +120 -0
  42. package/src/refs.js +141 -0
  43. package/src/regions.js +267 -0
  44. package/src/screenmap.js +119 -22
  45. package/src/simctl.js +74 -5
  46. package/src/store.js +8 -0
  47. package/src/view.js +342 -0
@@ -0,0 +1,121 @@
1
+ import CoreGraphics
2
+ import Foundation
3
+ import ObjectiveC.runtime
4
+
5
+ /// Raw Indigo HID message plumbing.
6
+ ///
7
+ /// Signatures here were read out of the SimulatorKit binary, which carries its
8
+ /// own C prototypes as strings, and confirmed end to end by sending a tap and
9
+ /// watching the screen change. See docs/PRIVATE_API.md. If an Xcode upgrade
10
+ /// breaks input, re-read the prototypes first:
11
+ ///
12
+ /// strings SimulatorKit | grep '^IndigoHIDMessage'
13
+ public final class IndigoHID {
14
+ /// The digitizer. Touch events routed anywhere else are ignored.
15
+ public static let digitizerTarget: UInt32 = 0x32
16
+ /// Hardware buttons do NOT use the digitizer target. Sending a home press
17
+ /// to 0x32 is silently swallowed; target 0 is what actually reaches the
18
+ /// device. Verified by sweeping the target with a known-good button code.
19
+ public static let buttonTarget: UInt32 = 0
20
+
21
+ /// AppKit NSEventType values, spelled out so this file needs no AppKit.
22
+ public enum MouseEvent: UInt {
23
+ case down = 1
24
+ case up = 2
25
+ case dragged = 6
26
+ }
27
+
28
+ public enum ButtonOp: UInt32 {
29
+ case down = 1
30
+ case up = 2
31
+ }
32
+
33
+ private let client: NSObject
34
+ private let sendSel = NSSelectorFromString("sendWithMessage:freeWhenDone:completionQueue:completion:")
35
+ private let send: @convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, Bool, AnyObject?, AnyObject?) -> Void
36
+
37
+ private typealias MouseFn = @convention(c) (
38
+ UnsafeMutablePointer<CGPoint>, UnsafeMutablePointer<CGPoint>?, UInt32, UInt, CGSize, UInt32
39
+ ) -> UnsafeMutableRawPointer?
40
+ private typealias ButtonFn = @convention(c) (UInt32, UInt32, UInt32) -> UnsafeMutableRawPointer?
41
+ private typealias KeyboardFn = @convention(c) (UInt32, UInt32) -> UnsafeMutableRawPointer?
42
+ /// Takes an NSEvent, which carries the *characters* rather than a key
43
+ /// position — this is how Simulator.app types, and why typing there is
44
+ /// layout-correct while raw usage codes are not.
45
+ private typealias KeyboardEventFn = @convention(c) (AnyObject) -> UnsafeMutableRawPointer?
46
+
47
+ private let makeMouse: MouseFn
48
+ private let makeButton: ButtonFn?
49
+ private let makeKeyboard: KeyboardFn?
50
+ private let makeKeyboardEvent: KeyboardEventFn?
51
+
52
+ public init(device: NSObject, simulatorKit: UnsafeMutableRawPointer?) throws {
53
+ // A Swift class, so it is registered under its qualified name; a bare
54
+ // "SimDeviceLegacyHIDClient" does not resolve.
55
+ guard let cls = NSClassFromString("SimulatorKit.SimDeviceLegacyHIDClient") else {
56
+ throw PrivateAPIError.frameworksUnavailable("SimDeviceLegacyHIDClient missing")
57
+ }
58
+ // `alloc` is unavailable in Swift; go through the runtime.
59
+ typealias AllocFn = @convention(c) (AnyClass, Selector) -> AnyObject?
60
+ let allocSel = NSSelectorFromString("alloc")
61
+ guard let allocMethod = class_getClassMethod(cls, allocSel),
62
+ let allocated = unsafeBitCast(method_getImplementation(allocMethod), to: AllocFn.self)(cls, allocSel) else {
63
+ throw PrivateAPIError.frameworksUnavailable("could not allocate the HID client")
64
+ }
65
+ let initSel = NSSelectorFromString("initWithDevice:error:")
66
+ typealias InitFn = @convention(c) (AnyObject, Selector, AnyObject, UnsafeMutableRawPointer?) -> AnyObject?
67
+ guard let initIMP = (allocated as AnyObject).method(for: initSel),
68
+ let created = unsafeBitCast(initIMP, to: InitFn.self)(allocated, initSel, device, nil) as? NSObject else {
69
+ throw PrivateAPIError.hidUnavailable("initWithDevice: returned nil")
70
+ }
71
+ client = created
72
+ guard client.responds(to: sendSel), let sendIMP = client.method(for: sendSel) else {
73
+ throw PrivateAPIError.hidUnavailable("client cannot send messages")
74
+ }
75
+ send = unsafeBitCast(sendIMP, to: (@convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, Bool, AnyObject?, AnyObject?) -> Void).self)
76
+
77
+ guard let mouseSym = dlsym(simulatorKit, "IndigoHIDMessageForMouseNSEvent") else {
78
+ throw PrivateAPIError.hidUnavailable("IndigoHIDMessageForMouseNSEvent missing")
79
+ }
80
+ makeMouse = unsafeBitCast(mouseSym, to: MouseFn.self)
81
+ makeButton = dlsym(simulatorKit, "IndigoHIDMessageForButton").map { unsafeBitCast($0, to: ButtonFn.self) }
82
+ makeKeyboard = dlsym(simulatorKit, "IndigoHIDMessageForKeyboardArbitrary").map { unsafeBitCast($0, to: KeyboardFn.self) }
83
+ makeKeyboardEvent = dlsym(simulatorKit, "IndigoHIDMessageForKeyboardNSEvent").map { unsafeBitCast($0, to: KeyboardEventFn.self) }
84
+ }
85
+
86
+ private func dispatch(_ message: UnsafeMutableRawPointer?) {
87
+ guard let message else { return }
88
+ // freeWhenDone: the framework allocated it and owns it after this.
89
+ send(client, sendSel, message, true, nil, nil)
90
+ }
91
+
92
+ /// One mouse event at a point in device points.
93
+ public func mouse(at point: CGPoint, event: MouseEvent, screen: CGSize) {
94
+ var p = point
95
+ dispatch(makeMouse(&p, nil, Self.digitizerTarget, event.rawValue, screen, 0))
96
+ }
97
+
98
+ public func button(keyCode: UInt32, op: ButtonOp) {
99
+ guard let makeButton else { return }
100
+ dispatch(makeButton(keyCode, op.rawValue, Self.buttonTarget))
101
+ }
102
+
103
+ public func key(usage: UInt32, op: ButtonOp) {
104
+ guard let makeKeyboard else { return }
105
+ dispatch(makeKeyboard(usage, op.rawValue))
106
+ }
107
+
108
+ /// True when the character-carrying keyboard path is available.
109
+ public var canSendKeyboardEvents: Bool { makeKeyboardEvent != nil }
110
+
111
+ /// Send a key event built from characters rather than a key position.
112
+ public func keyboardEvent(_ event: AnyObject) {
113
+ guard let makeKeyboardEvent else { return }
114
+ dispatch(makeKeyboardEvent(event))
115
+ }
116
+
117
+ public func resetSession() {
118
+ let sel = NSSelectorFromString("resetHIDSession")
119
+ if client.responds(to: sel) { _ = client.perform(sel) }
120
+ }
121
+ }
@@ -0,0 +1,149 @@
1
+ import Foundation
2
+
3
+ /// A frame as the simulator hands it to us: tightly-packed BGRA is not
4
+ /// guaranteed, so `bytesPerRow` must always be honoured rather than assumed
5
+ /// to equal `width * 4`.
6
+ public struct RawFrame {
7
+ public let width: Int
8
+ public let height: Int
9
+ public let bytesPerRow: Int
10
+ public let pixels: UnsafePointer<UInt8>
11
+
12
+ public init(width: Int, height: Int, bytesPerRow: Int, pixels: UnsafePointer<UInt8>) {
13
+ self.width = width
14
+ self.height = height
15
+ self.bytesPerRow = bytesPerRow
16
+ self.pixels = pixels
17
+ }
18
+ }
19
+
20
+ public struct DeviceInfo: Sendable {
21
+ public let udid: String
22
+ public let name: String
23
+ public let runtime: String
24
+ /// Native pixels, as the framebuffer reports them.
25
+ public let pixelWidth: Int
26
+ public let pixelHeight: Int
27
+ /// Points per pixel. Input coordinates are in points, frames are in pixels.
28
+ public let scale: Double
29
+
30
+ public var pointWidth: Int { scale > 0 ? Int((Double(pixelWidth) / scale).rounded()) : pixelWidth }
31
+ public var pointHeight: Int { scale > 0 ? Int((Double(pixelHeight) / scale).rounded()) : pixelHeight }
32
+
33
+ public init(udid: String, name: String, runtime: String,
34
+ pixelWidth: Int = 0, pixelHeight: Int = 0, scale: Double = 1) {
35
+ self.udid = udid
36
+ self.name = name
37
+ self.runtime = runtime
38
+ self.pixelWidth = pixelWidth
39
+ self.pixelHeight = pixelHeight
40
+ self.scale = scale
41
+ }
42
+ }
43
+
44
+ /// Hardware buttons, named rather than numbered at call sites.
45
+ public enum HardwareButton: String, Sendable, CaseIterable {
46
+ case home, lock, siri, volumeUp, volumeDown
47
+ }
48
+
49
+ public enum PrivateAPIError: Error, CustomStringConvertible {
50
+ case frameworksUnavailable(String)
51
+ case noBootedDevice
52
+ case deviceNotFound(String)
53
+ case noDisplayPort
54
+ case surfaceUnavailable
55
+ case hidUnavailable(String)
56
+ case simctlFailed(String)
57
+
58
+ public var description: String {
59
+ switch self {
60
+ case .frameworksUnavailable(let d): return "simulator frameworks unavailable: \(d)"
61
+ case .noBootedDevice: return "no booted simulator"
62
+ case .deviceNotFound(let u): return "no simulator matching \(u)"
63
+ case .noDisplayPort: return "the device exposes no active display port"
64
+ case .surfaceUnavailable: return "the display surface could not be read"
65
+ case .hidUnavailable(let d): return "input is unavailable: \(d)"
66
+ case .simctlFailed(let d): return "simctl \(d)"
67
+ }
68
+ }
69
+ }
70
+
71
+ /// The whole private surface simframe depends on. Anything that talks to
72
+ /// CoreSimulator or SimulatorKit goes through this.
73
+ public protocol SimulatorPlatform: AnyObject {
74
+ func bootedDevices() throws -> [DeviceInfo]
75
+ /// Bind to a device's display. Must be called before `withFrame`.
76
+ func attach(udid: String?) throws -> DeviceInfo
77
+ /// Re-resolve the display port on the device already attached.
78
+ ///
79
+ /// The descriptor handed out by `attach` can die under a live daemon: the
80
+ /// simulator tears its display port down and builds a new one, and every
81
+ /// `framebufferSurface` read on the old object returns nil from then on.
82
+ /// Measured on one long session — capture reported "the display surface
83
+ /// could not be read" for six minutes on a device that was awake and
84
+ /// perfectly visible, and a daemon restart fixed it instantly. Without a
85
+ /// way to re-resolve, a restart is the only cure.
86
+ func reattachDisplay() throws -> DeviceInfo
87
+ /// Borrow the current framebuffer. The pointer is only valid inside `body`.
88
+ func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T
89
+ /// Called whenever the display reports damage — the per-redraw signal, so a
90
+ /// capture loop can be driven by the screen rather than by a timer.
91
+ func observeChanges(_ handler: @escaping () -> Void) throws
92
+ func detach()
93
+
94
+ // MARK: Input. Optional: a platform may observe without being able to touch.
95
+
96
+ /// Whether input is available, and why not when it is not.
97
+ func inputStatus() -> (available: Bool, detail: String)
98
+ /// A press and release at one point.
99
+ func tap(at point: CGPoint, durationMs: Double) throws
100
+ /// A deliberate hold. Named separately from `tap` because callers mean
101
+ /// something different by it, even though the mechanism is a longer hold.
102
+ func longPress(at point: CGPoint, durationMs: Double) throws
103
+ /// Press, hold in place, then move and release. The hold is what separates
104
+ /// a drag from a swipe: reorderable lists and drag-and-drop need the UI to
105
+ /// register a pick-up before movement starts.
106
+ func drag(from: CGPoint, to: CGPoint, holdMs: Double, durationMs: Double) throws
107
+ /// A real down, interpolated moves, then up — never a teleporting jump.
108
+ func swipe(from: CGPoint, to: CGPoint, durationMs: Double) throws
109
+ func type(_ text: String) throws
110
+ /// Put text on the device pasteboard and paste it.
111
+ ///
112
+ /// Key events are mapped by whatever keyboard layout iOS has active, so
113
+ /// typing "Fryer 3" against a Persian layout yields Persian text with no
114
+ /// error. The pasteboard carries characters rather than key positions, so
115
+ /// it is the only reliable route for content that must be exact.
116
+ func paste(_ text: String) throws
117
+ func press(_ button: HardwareButton) throws
118
+ /// Rebuild the HID session.
119
+ ///
120
+ /// Input has no feedback channel: a dispatched Indigo message reports
121
+ /// success when the send succeeds, and there is no way to ask the device
122
+ /// whether anything happened. Measured on a long-running daemon, a HOME
123
+ /// press returned in 66ms and the screen never moved, while the same press
124
+ /// on a freshly started daemon worked — so the session can stop delivering
125
+ /// while still accepting. This is the recovery for that, and the caller
126
+ /// that notices is the one with the frames.
127
+ func resetInput() throws
128
+
129
+ // MARK: Accessibility. Optional in the same way input is: a platform may
130
+ // see the screen without being able to read the app's own tree.
131
+
132
+ /// Whether the accessibility tree can be read, and why not when it cannot.
133
+ func accessibilityStatus() -> (available: Bool, detail: String)
134
+ /// The frontmost application's accessibility tree, flattened depth-first,
135
+ /// carrying whether it is all of one.
136
+ ///
137
+ /// An app still launching has no tree yet, and this reports that as it is —
138
+ /// one node, no children — rather than retrying until it looks populated.
139
+ func accessibilityTree() throws -> AXTree
140
+
141
+ // MARK: App lifecycle. These are simctl, not private API — no HID needed.
142
+
143
+ @discardableResult
144
+ func launch(bundleId: String, arguments: [String], environment: [String: String]) throws -> Int32
145
+ func terminate(bundleId: String) throws
146
+ func openURL(_ url: String) throws
147
+ /// `action` is grant/revoke/reset, `service` one of simctl's privacy services.
148
+ func permission(action: String, service: String, bundleId: String?) throws
149
+ }
@@ -0,0 +1,112 @@
1
+ import CoreGraphics
2
+ import Foundation
3
+
4
+ /// A deterministic platform for tests and for machines without Xcode.
5
+ /// Renders a synthetic screen so hashing and frame-store logic can be tested
6
+ /// with no simulator present.
7
+ public final class StubPlatform: SimulatorPlatform {
8
+ public var width: Int
9
+ public var height: Int
10
+ public var tint: UInt8
11
+ private var buffer: [UInt8] = []
12
+ /// What was asked of it, so tests can assert on gestures without a device.
13
+ public var recorded: [String] = []
14
+ /// What `accessibilityTree()` should answer. Empty by default, which is
15
+ /// what a device with no app in the foreground genuinely looks like.
16
+ public var stubTree: AXTree = AXTree(nodes: [])
17
+
18
+ public init(width: Int = 1206, height: Int = 2622, tint: UInt8 = 0) {
19
+ self.width = width
20
+ self.height = height
21
+ self.tint = tint
22
+ }
23
+
24
+ public func bootedDevices() throws -> [DeviceInfo] {
25
+ [DeviceInfo(udid: "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")]
26
+ }
27
+
28
+ public func attach(udid: String?) throws -> DeviceInfo {
29
+ attachedUdid = udid
30
+ return DeviceInfo(udid: udid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
31
+ }
32
+
33
+ /// Counted so a test can assert the capture loop actually tries to recover
34
+ /// rather than logging the same failure forever.
35
+ public private(set) var reattachCount = 0
36
+ private var attachedUdid: String?
37
+
38
+ /// Counted, so a test can assert that a dead input path is actually retried.
39
+ public private(set) var resetInputCount = 0
40
+ public func resetInput() throws { resetInputCount += 1 }
41
+
42
+ public func reattachDisplay() throws -> DeviceInfo {
43
+ reattachCount += 1
44
+ return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
45
+ }
46
+
47
+ public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
48
+ let bytesPerRow = width * 4 + 40 // deliberate padding: mirrors real surfaces
49
+ if buffer.count != bytesPerRow * height {
50
+ buffer = [UInt8](repeating: 0, count: bytesPerRow * height)
51
+ }
52
+ for y in 0..<height {
53
+ for x in 0..<width {
54
+ let i = y * bytesPerRow + x * 4
55
+ let band: UInt8 = x < width / 2 ? 40 : 200
56
+ buffer[i] = band &+ tint // B
57
+ buffer[i + 1] = band // G
58
+ buffer[i + 2] = band // R
59
+ buffer[i + 3] = 255 // A
60
+ }
61
+ }
62
+ return try buffer.withUnsafeBufferPointer { ptr in
63
+ try body(RawFrame(width: width, height: height, bytesPerRow: bytesPerRow, pixels: ptr.baseAddress!))
64
+ }
65
+ }
66
+
67
+ public func observeChanges(_ handler: @escaping () -> Void) throws {
68
+ // The stub never changes on its own; tests drive `tint` and call this.
69
+ stubHandler = handler
70
+ }
71
+
72
+ /// Test hook: mutate the synthetic screen and announce it.
73
+ public func simulateChange() {
74
+ tint = tint &+ 40
75
+ stubHandler?()
76
+ }
77
+
78
+ private var stubHandler: (() -> Void)?
79
+
80
+ public func detach() { stubHandler = nil }
81
+ }
82
+
83
+ extension StubPlatform {
84
+ public func inputStatus() -> (available: Bool, detail: String) { (true, "stub") }
85
+ public func tap(at point: CGPoint, durationMs: Double) throws { recorded.append("tap(\(Int(point.x)),\(Int(point.y)))") }
86
+ public func swipe(from: CGPoint, to: CGPoint, durationMs: Double) throws {
87
+ recorded.append("swipe(\(Int(from.x)),\(Int(from.y))->\(Int(to.x)),\(Int(to.y)))")
88
+ }
89
+ public func type(_ text: String) throws { recorded.append("type(\(text))") }
90
+ public func paste(_ text: String) throws { recorded.append("paste(\(text))") }
91
+ public func press(_ button: HardwareButton) throws { recorded.append("press(\(button.rawValue))") }
92
+ public func longPress(at point: CGPoint, durationMs: Double) throws {
93
+ recorded.append("longPress(\(Int(point.x)),\(Int(point.y)),\(Int(durationMs)))")
94
+ }
95
+ public func drag(from: CGPoint, to: CGPoint, holdMs: Double, durationMs: Double) throws {
96
+ recorded.append("drag(\(Int(from.x)),\(Int(from.y))->\(Int(to.x)),\(Int(to.y)),hold=\(Int(holdMs)))")
97
+ }
98
+ @discardableResult
99
+ public func launch(bundleId: String, arguments: [String], environment: [String: String]) throws -> Int32 {
100
+ recorded.append("launch(\(bundleId))"); return 1234
101
+ }
102
+ public func terminate(bundleId: String) throws { recorded.append("terminate(\(bundleId))") }
103
+ public func openURL(_ url: String) throws { recorded.append("openURL(\(url))") }
104
+ public func permission(action: String, service: String, bundleId: String?) throws {
105
+ recorded.append("permission(\(action),\(service),\(bundleId ?? "-"))")
106
+ }
107
+ public func accessibilityStatus() -> (available: Bool, detail: String) { (true, "stub") }
108
+ public func accessibilityTree() throws -> AXTree {
109
+ recorded.append("accessibilityTree()")
110
+ return stubTree
111
+ }
112
+ }
@@ -0,0 +1,61 @@
1
+ import Foundation
2
+ import PrivateAPI
3
+
4
+ /// A tightly-packed RGBA bitmap, which is what every algorithm below expects.
5
+ /// The simulator hands us padded BGRA, so conversion happens exactly once.
6
+ public struct Bitmap {
7
+ public let width: Int
8
+ public let height: Int
9
+ public var data: [UInt8] // RGBA, 4 bytes per pixel, no row padding
10
+
11
+ public init(width: Int, height: Int, data: [UInt8]) {
12
+ self.width = width
13
+ self.height = height
14
+ self.data = data
15
+ }
16
+
17
+ /// Convert a padded BGRA surface into packed RGBA, downscaling by box
18
+ /// average in the same pass. Box averaging is what `grayGrid` does anyway,
19
+ /// so doing it here keeps the scaled image consistent with the hashes.
20
+ public static func from(_ frame: RawFrame, targetLongEdge: Int) -> Bitmap {
21
+ let (tw, th) = Self.fit(width: frame.width, height: frame.height, longEdge: targetLongEdge)
22
+ var out = [UInt8](repeating: 0, count: tw * th * 4)
23
+ let src = frame.pixels
24
+
25
+ for ty in 0..<th {
26
+ let y0 = (ty * frame.height) / th
27
+ let y1 = max(y0 + 1, ((ty + 1) * frame.height) / th)
28
+ for tx in 0..<tw {
29
+ let x0 = (tx * frame.width) / tw
30
+ let x1 = max(x0 + 1, ((tx + 1) * frame.width) / tw)
31
+ var r = 0, g = 0, b = 0, n = 0
32
+ for y in y0..<y1 {
33
+ let row = y * frame.bytesPerRow
34
+ for x in x0..<x1 {
35
+ let i = row + x * 4
36
+ b += Int(src[i]) // surfaces are BGRA
37
+ g += Int(src[i + 1])
38
+ r += Int(src[i + 2])
39
+ n += 1
40
+ }
41
+ }
42
+ let o = (ty * tw + tx) * 4
43
+ out[o] = UInt8(r / n)
44
+ out[o + 1] = UInt8(g / n)
45
+ out[o + 2] = UInt8(b / n)
46
+ out[o + 3] = 255
47
+ }
48
+ }
49
+ return Bitmap(width: tw, height: th, data: out)
50
+ }
51
+
52
+ /// Match `sips -Z`: the longer edge becomes `longEdge`, aspect preserved.
53
+ public static func fit(width: Int, height: Int, longEdge: Int) -> (Int, Int) {
54
+ guard width > 0, height > 0 else { return (1, 1) }
55
+ if max(width, height) <= longEdge { return (width, height) }
56
+ if height >= width {
57
+ return (max(1, Int((Double(width) * Double(longEdge) / Double(height)).rounded())), longEdge)
58
+ }
59
+ return (longEdge, max(1, Int((Double(height) * Double(longEdge) / Double(width)).rounded())))
60
+ }
61
+ }
@@ -0,0 +1,122 @@
1
+ import Foundation
2
+ import PrivateAPI
3
+
4
+ /// A per-device Unix domain socket carrying one JSON object per line.
5
+ ///
6
+ /// Unix sockets rather than TCP because the permission model is the file
7
+ /// system's: mode 0600 means only this user can drive the simulator, with no
8
+ /// port to collide and nothing reachable from the network.
9
+ public final class ControlSocket {
10
+ public typealias Handler = ([String: Any]) -> [String: Any]
11
+
12
+ private let path: String
13
+ private let handler: Handler
14
+ private var listenFD: Int32 = -1
15
+ private var acceptSource: DispatchSourceRead?
16
+ // Requests run off the capture loop: a swipe sleeps for its whole duration
17
+ // and must not stall frame capture.
18
+ private let workQueue = DispatchQueue(label: "simframe.control", qos: .userInitiated)
19
+ /// Identity of the socket file this instance created, so shutdown can tell
20
+ /// its own socket from one a successor has since bound at the same path.
21
+ private var boundIno: ino_t = 0
22
+ private var boundDev: dev_t = 0
23
+
24
+ public init(path: String, handler: @escaping Handler) {
25
+ self.path = path
26
+ self.handler = handler
27
+ }
28
+
29
+ public func start() throws {
30
+ unlink(path)
31
+ listenFD = socket(AF_UNIX, SOCK_STREAM, 0)
32
+ guard listenFD >= 0 else { throw SocketError.failed("socket(): \(errno)") }
33
+
34
+ var addr = sockaddr_un()
35
+ addr.sun_family = sa_family_t(AF_UNIX)
36
+ let maxLen = MemoryLayout.size(ofValue: addr.sun_path)
37
+ guard path.utf8.count < maxLen else { throw SocketError.failed("socket path too long") }
38
+ withUnsafeMutablePointer(to: &addr.sun_path) { p in
39
+ p.withMemoryRebound(to: CChar.self, capacity: maxLen) { dst in
40
+ _ = strcpy(dst, path)
41
+ }
42
+ }
43
+ let size = socklen_t(MemoryLayout<sockaddr_un>.size)
44
+ let bound = withUnsafePointer(to: &addr) {
45
+ $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(listenFD, $0, size) }
46
+ }
47
+ guard bound == 0 else { close(listenFD); throw SocketError.failed("bind(): \(errno)") }
48
+ // Only this user may drive the device.
49
+ chmod(path, 0o600)
50
+ var bornStat = stat()
51
+ if stat(path, &bornStat) == 0 {
52
+ boundIno = bornStat.st_ino
53
+ boundDev = bornStat.st_dev
54
+ }
55
+ guard listen(listenFD, 8) == 0 else { close(listenFD); throw SocketError.failed("listen(): \(errno)") }
56
+
57
+ let source = DispatchSource.makeReadSource(fileDescriptor: listenFD, queue: workQueue)
58
+ source.setEventHandler { [weak self] in self?.acceptOne() }
59
+ source.resume()
60
+ acceptSource = source
61
+ }
62
+
63
+ private func acceptOne() {
64
+ let fd = accept(listenFD, nil, nil)
65
+ guard fd >= 0 else { return }
66
+ defer { close(fd) }
67
+ guard let request = readLine(fd) else { return }
68
+
69
+ var response: [String: Any]
70
+ if let data = request.data(using: .utf8),
71
+ let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
72
+ response = handler(object)
73
+ if let id = object["id"] { response["id"] = id }
74
+ } else {
75
+ response = ["ok": false, "error": "malformed request"]
76
+ }
77
+ if let out = try? JSONSerialization.data(withJSONObject: response) {
78
+ var line = out
79
+ line.append(0x0A)
80
+ _ = line.withUnsafeBytes { write(fd, $0.baseAddress, $0.count) }
81
+ }
82
+ }
83
+
84
+ private func readLine(_ fd: Int32) -> String? {
85
+ var buffer = [UInt8]()
86
+ var byte: UInt8 = 0
87
+ while read(fd, &byte, 1) == 1 {
88
+ if byte == 0x0A { break }
89
+ buffer.append(byte)
90
+ if buffer.count > 1 << 20 { return nil }
91
+ }
92
+ return buffer.isEmpty ? nil : String(decoding: buffer, as: UTF8.self)
93
+ }
94
+
95
+ public func stop() {
96
+ acceptSource?.cancel()
97
+ acceptSource = nil
98
+ if listenFD >= 0 { close(listenFD) }
99
+ listenFD = -1
100
+ // Only remove the socket if it is still the one we created.
101
+ //
102
+ // A restart binds a fresh socket at the same path, and a dying daemon
103
+ // that unlinks blindly deletes its *successor's* socket. Capture keeps
104
+ // working — that is file-based — so the only symptom is input quietly
105
+ // dropping to idb, which is slower and has different semantics. That is
106
+ // exactly the kind of invisible downgrade this tool is supposed to
107
+ // refuse to have.
108
+ var current = stat()
109
+ if stat(path, &current) == 0, current.st_ino == boundIno, current.st_dev == boundDev {
110
+ unlink(path)
111
+ }
112
+ boundIno = 0
113
+ boundDev = 0
114
+ }
115
+
116
+ public enum SocketError: Error, CustomStringConvertible {
117
+ case failed(String)
118
+ public var description: String {
119
+ switch self { case .failed(let d): return "control socket: \(d)" }
120
+ }
121
+ }
122
+ }
@@ -0,0 +1,70 @@
1
+ import CoreGraphics
2
+ import ImageIO
3
+ import Foundation
4
+ import PrivateAPI
5
+
6
+ /// Downscale through CoreGraphics rather than by box average.
7
+ ///
8
+ /// `sips` — which the JavaScript capture loop uses — is a thin wrapper over
9
+ /// ImageIO/CoreGraphics, so resampling the same way is the closest we can get
10
+ /// to producing byte-identical hashes without reimplementing its filter.
11
+ public enum CoreGraphicsScaler {
12
+ public static func bitmap(from frame: RawFrame, targetLongEdge: Int) -> Bitmap? {
13
+ let (tw, th) = Bitmap.fit(width: frame.width, height: frame.height, longEdge: targetLongEdge)
14
+ let srcBytes = frame.bytesPerRow * frame.height
15
+ guard let provider = CGDataProvider(dataInfo: nil, data: frame.pixels, size: srcBytes, releaseData: { _, _, _ in }) else {
16
+ return nil
17
+ }
18
+ // Surfaces are BGRA; this bitmapInfo is how CoreGraphics is told so.
19
+ let srcInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue).union(.byteOrder32Little)
20
+ guard let cgImage = CGImage(
21
+ width: frame.width, height: frame.height,
22
+ bitsPerComponent: 8, bitsPerPixel: 32,
23
+ bytesPerRow: frame.bytesPerRow,
24
+ space: CGColorSpaceCreateDeviceRGB(),
25
+ bitmapInfo: srcInfo,
26
+ provider: provider, decode: nil,
27
+ shouldInterpolate: false, intent: .defaultIntent
28
+ ) else { return nil }
29
+
30
+ var out = [UInt8](repeating: 0, count: tw * th * 4)
31
+ let ok = out.withUnsafeMutableBytes { buf -> Bool in
32
+ guard let ctx = CGContext(
33
+ data: buf.baseAddress, width: tw, height: th,
34
+ bitsPerComponent: 8, bytesPerRow: tw * 4,
35
+ space: CGColorSpaceCreateDeviceRGB(),
36
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
37
+ ) else { return false }
38
+ ctx.interpolationQuality = .high
39
+ ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: tw, height: th))
40
+ return true
41
+ }
42
+ return ok ? Bitmap(width: tw, height: th, data: out) : nil
43
+ }
44
+ }
45
+
46
+ extension CoreGraphicsScaler {
47
+ /// Re-encode an existing PNG at a fraction of its size. Frames that have
48
+ /// aged out of the fine window stay legible enough to tell which screen was
49
+ /// showing, at roughly a quarter of the bytes.
50
+ public static func downscalePNG(at url: URL, factor: Double) -> Bitmap? {
51
+ guard factor > 0, factor < 1,
52
+ let src = CGImageSourceCreateWithURL(url as CFURL, nil),
53
+ let image = CGImageSourceCreateImageAtIndex(src, 0, nil) else { return nil }
54
+ let tw = max(1, Int(Double(image.width) * factor))
55
+ let th = max(1, Int(Double(image.height) * factor))
56
+ var out = [UInt8](repeating: 0, count: tw * th * 4)
57
+ let ok = out.withUnsafeMutableBytes { buf -> Bool in
58
+ guard let ctx = CGContext(
59
+ data: buf.baseAddress, width: tw, height: th,
60
+ bitsPerComponent: 8, bytesPerRow: tw * 4,
61
+ space: CGColorSpaceCreateDeviceRGB(),
62
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
63
+ ) else { return false }
64
+ ctx.interpolationQuality = .high
65
+ ctx.draw(image, in: CGRect(x: 0, y: 0, width: tw, height: th))
66
+ return true
67
+ }
68
+ return ok ? Bitmap(width: tw, height: th, data: out) : nil
69
+ }
70
+ }