simframe 0.4.2 → 0.6.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +334 -85
- package/native/simframed/Package.swift +16 -0
- package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift +379 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +523 -0
- package/native/simframed/Sources/PrivateAPI/HIDKeyboard.swift +70 -0
- package/native/simframed/Sources/PrivateAPI/IndigoHID.swift +121 -0
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +149 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +112 -0
- package/native/simframed/Sources/SimframeCore/Bitmap.swift +61 -0
- package/native/simframed/Sources/SimframeCore/ControlSocket.swift +122 -0
- package/native/simframed/Sources/SimframeCore/CoreGraphicsScaler.swift +70 -0
- package/native/simframed/Sources/SimframeCore/Element.swift +148 -0
- package/native/simframed/Sources/SimframeCore/FrameStore.swift +303 -0
- package/native/simframed/Sources/SimframeCore/Hashing.swift +119 -0
- package/native/simframed/Sources/SimframeCore/Motion.swift +431 -0
- package/native/simframed/Sources/SimframeCore/PNGWriter.swift +40 -0
- package/native/simframed/Sources/SimframeCore/VisionOCR.swift +75 -0
- package/native/simframed/Sources/simframed/main.swift +485 -0
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +270 -0
- package/package.json +12 -4
- package/scripts/bench-flow.mjs +54 -0
- package/scripts/bench.sh +98 -0
- package/scripts/check-package.mjs +99 -0
- package/scripts/ci-memory.mjs +416 -0
- package/scripts/eval-fingerprint.mjs +192 -0
- package/scripts/smoke.mjs +76 -0
- package/scripts/sync-server-version.mjs +39 -0
- package/scripts/verify-baseline.mjs +65 -0
- package/skills/simframe/SKILL.md +173 -0
- package/src/actions.js +264 -18
- package/src/cli.js +561 -89
- package/src/control.js +77 -0
- package/src/daemon.js +8 -1
- package/src/engine.js +99 -0
- package/src/fingerprint.js +183 -0
- package/src/graph.js +411 -0
- package/src/index.js +351 -24
- package/src/input.js +179 -2
- package/src/matching.js +265 -0
- package/src/mcp.js +425 -112
- package/src/navigate.js +120 -0
- package/src/refs.js +141 -0
- package/src/regions.js +267 -0
- package/src/screenmap.js +119 -22
- package/src/simctl.js +74 -5
- package/src/store.js +8 -0
- package/src/view.js +342 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import CoreGraphics
|
|
2
|
+
import Foundation
|
|
3
|
+
import ObjectiveC.runtime
|
|
4
|
+
|
|
5
|
+
/// Raw Indigo HID message plumbing.
|
|
6
|
+
///
|
|
7
|
+
/// Signatures here were read out of the SimulatorKit binary, which carries its
|
|
8
|
+
/// own C prototypes as strings, and confirmed end to end by sending a tap and
|
|
9
|
+
/// watching the screen change. See docs/PRIVATE_API.md. If an Xcode upgrade
|
|
10
|
+
/// breaks input, re-read the prototypes first:
|
|
11
|
+
///
|
|
12
|
+
/// strings SimulatorKit | grep '^IndigoHIDMessage'
|
|
13
|
+
public final class IndigoHID {
|
|
14
|
+
/// The digitizer. Touch events routed anywhere else are ignored.
|
|
15
|
+
public static let digitizerTarget: UInt32 = 0x32
|
|
16
|
+
/// Hardware buttons do NOT use the digitizer target. Sending a home press
|
|
17
|
+
/// to 0x32 is silently swallowed; target 0 is what actually reaches the
|
|
18
|
+
/// device. Verified by sweeping the target with a known-good button code.
|
|
19
|
+
public static let buttonTarget: UInt32 = 0
|
|
20
|
+
|
|
21
|
+
/// AppKit NSEventType values, spelled out so this file needs no AppKit.
|
|
22
|
+
public enum MouseEvent: UInt {
|
|
23
|
+
case down = 1
|
|
24
|
+
case up = 2
|
|
25
|
+
case dragged = 6
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
public enum ButtonOp: UInt32 {
|
|
29
|
+
case down = 1
|
|
30
|
+
case up = 2
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
private let client: NSObject
|
|
34
|
+
private let sendSel = NSSelectorFromString("sendWithMessage:freeWhenDone:completionQueue:completion:")
|
|
35
|
+
private let send: @convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, Bool, AnyObject?, AnyObject?) -> Void
|
|
36
|
+
|
|
37
|
+
private typealias MouseFn = @convention(c) (
|
|
38
|
+
UnsafeMutablePointer<CGPoint>, UnsafeMutablePointer<CGPoint>?, UInt32, UInt, CGSize, UInt32
|
|
39
|
+
) -> UnsafeMutableRawPointer?
|
|
40
|
+
private typealias ButtonFn = @convention(c) (UInt32, UInt32, UInt32) -> UnsafeMutableRawPointer?
|
|
41
|
+
private typealias KeyboardFn = @convention(c) (UInt32, UInt32) -> UnsafeMutableRawPointer?
|
|
42
|
+
/// Takes an NSEvent, which carries the *characters* rather than a key
|
|
43
|
+
/// position — this is how Simulator.app types, and why typing there is
|
|
44
|
+
/// layout-correct while raw usage codes are not.
|
|
45
|
+
private typealias KeyboardEventFn = @convention(c) (AnyObject) -> UnsafeMutableRawPointer?
|
|
46
|
+
|
|
47
|
+
private let makeMouse: MouseFn
|
|
48
|
+
private let makeButton: ButtonFn?
|
|
49
|
+
private let makeKeyboard: KeyboardFn?
|
|
50
|
+
private let makeKeyboardEvent: KeyboardEventFn?
|
|
51
|
+
|
|
52
|
+
public init(device: NSObject, simulatorKit: UnsafeMutableRawPointer?) throws {
|
|
53
|
+
// A Swift class, so it is registered under its qualified name; a bare
|
|
54
|
+
// "SimDeviceLegacyHIDClient" does not resolve.
|
|
55
|
+
guard let cls = NSClassFromString("SimulatorKit.SimDeviceLegacyHIDClient") else {
|
|
56
|
+
throw PrivateAPIError.frameworksUnavailable("SimDeviceLegacyHIDClient missing")
|
|
57
|
+
}
|
|
58
|
+
// `alloc` is unavailable in Swift; go through the runtime.
|
|
59
|
+
typealias AllocFn = @convention(c) (AnyClass, Selector) -> AnyObject?
|
|
60
|
+
let allocSel = NSSelectorFromString("alloc")
|
|
61
|
+
guard let allocMethod = class_getClassMethod(cls, allocSel),
|
|
62
|
+
let allocated = unsafeBitCast(method_getImplementation(allocMethod), to: AllocFn.self)(cls, allocSel) else {
|
|
63
|
+
throw PrivateAPIError.frameworksUnavailable("could not allocate the HID client")
|
|
64
|
+
}
|
|
65
|
+
let initSel = NSSelectorFromString("initWithDevice:error:")
|
|
66
|
+
typealias InitFn = @convention(c) (AnyObject, Selector, AnyObject, UnsafeMutableRawPointer?) -> AnyObject?
|
|
67
|
+
guard let initIMP = (allocated as AnyObject).method(for: initSel),
|
|
68
|
+
let created = unsafeBitCast(initIMP, to: InitFn.self)(allocated, initSel, device, nil) as? NSObject else {
|
|
69
|
+
throw PrivateAPIError.hidUnavailable("initWithDevice: returned nil")
|
|
70
|
+
}
|
|
71
|
+
client = created
|
|
72
|
+
guard client.responds(to: sendSel), let sendIMP = client.method(for: sendSel) else {
|
|
73
|
+
throw PrivateAPIError.hidUnavailable("client cannot send messages")
|
|
74
|
+
}
|
|
75
|
+
send = unsafeBitCast(sendIMP, to: (@convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, Bool, AnyObject?, AnyObject?) -> Void).self)
|
|
76
|
+
|
|
77
|
+
guard let mouseSym = dlsym(simulatorKit, "IndigoHIDMessageForMouseNSEvent") else {
|
|
78
|
+
throw PrivateAPIError.hidUnavailable("IndigoHIDMessageForMouseNSEvent missing")
|
|
79
|
+
}
|
|
80
|
+
makeMouse = unsafeBitCast(mouseSym, to: MouseFn.self)
|
|
81
|
+
makeButton = dlsym(simulatorKit, "IndigoHIDMessageForButton").map { unsafeBitCast($0, to: ButtonFn.self) }
|
|
82
|
+
makeKeyboard = dlsym(simulatorKit, "IndigoHIDMessageForKeyboardArbitrary").map { unsafeBitCast($0, to: KeyboardFn.self) }
|
|
83
|
+
makeKeyboardEvent = dlsym(simulatorKit, "IndigoHIDMessageForKeyboardNSEvent").map { unsafeBitCast($0, to: KeyboardEventFn.self) }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
private func dispatch(_ message: UnsafeMutableRawPointer?) {
|
|
87
|
+
guard let message else { return }
|
|
88
|
+
// freeWhenDone: the framework allocated it and owns it after this.
|
|
89
|
+
send(client, sendSel, message, true, nil, nil)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/// One mouse event at a point in device points.
|
|
93
|
+
public func mouse(at point: CGPoint, event: MouseEvent, screen: CGSize) {
|
|
94
|
+
var p = point
|
|
95
|
+
dispatch(makeMouse(&p, nil, Self.digitizerTarget, event.rawValue, screen, 0))
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
public func button(keyCode: UInt32, op: ButtonOp) {
|
|
99
|
+
guard let makeButton else { return }
|
|
100
|
+
dispatch(makeButton(keyCode, op.rawValue, Self.buttonTarget))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
public func key(usage: UInt32, op: ButtonOp) {
|
|
104
|
+
guard let makeKeyboard else { return }
|
|
105
|
+
dispatch(makeKeyboard(usage, op.rawValue))
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/// True when the character-carrying keyboard path is available.
|
|
109
|
+
public var canSendKeyboardEvents: Bool { makeKeyboardEvent != nil }
|
|
110
|
+
|
|
111
|
+
/// Send a key event built from characters rather than a key position.
|
|
112
|
+
public func keyboardEvent(_ event: AnyObject) {
|
|
113
|
+
guard let makeKeyboardEvent else { return }
|
|
114
|
+
dispatch(makeKeyboardEvent(event))
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
public func resetSession() {
|
|
118
|
+
let sel = NSSelectorFromString("resetHIDSession")
|
|
119
|
+
if client.responds(to: sel) { _ = client.perform(sel) }
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// A frame as the simulator hands it to us: tightly-packed BGRA is not
|
|
4
|
+
/// guaranteed, so `bytesPerRow` must always be honoured rather than assumed
|
|
5
|
+
/// to equal `width * 4`.
|
|
6
|
+
public struct RawFrame {
|
|
7
|
+
public let width: Int
|
|
8
|
+
public let height: Int
|
|
9
|
+
public let bytesPerRow: Int
|
|
10
|
+
public let pixels: UnsafePointer<UInt8>
|
|
11
|
+
|
|
12
|
+
public init(width: Int, height: Int, bytesPerRow: Int, pixels: UnsafePointer<UInt8>) {
|
|
13
|
+
self.width = width
|
|
14
|
+
self.height = height
|
|
15
|
+
self.bytesPerRow = bytesPerRow
|
|
16
|
+
self.pixels = pixels
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
public struct DeviceInfo: Sendable {
|
|
21
|
+
public let udid: String
|
|
22
|
+
public let name: String
|
|
23
|
+
public let runtime: String
|
|
24
|
+
/// Native pixels, as the framebuffer reports them.
|
|
25
|
+
public let pixelWidth: Int
|
|
26
|
+
public let pixelHeight: Int
|
|
27
|
+
/// Points per pixel. Input coordinates are in points, frames are in pixels.
|
|
28
|
+
public let scale: Double
|
|
29
|
+
|
|
30
|
+
public var pointWidth: Int { scale > 0 ? Int((Double(pixelWidth) / scale).rounded()) : pixelWidth }
|
|
31
|
+
public var pointHeight: Int { scale > 0 ? Int((Double(pixelHeight) / scale).rounded()) : pixelHeight }
|
|
32
|
+
|
|
33
|
+
public init(udid: String, name: String, runtime: String,
|
|
34
|
+
pixelWidth: Int = 0, pixelHeight: Int = 0, scale: Double = 1) {
|
|
35
|
+
self.udid = udid
|
|
36
|
+
self.name = name
|
|
37
|
+
self.runtime = runtime
|
|
38
|
+
self.pixelWidth = pixelWidth
|
|
39
|
+
self.pixelHeight = pixelHeight
|
|
40
|
+
self.scale = scale
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Hardware buttons, named rather than numbered at call sites.
|
|
45
|
+
public enum HardwareButton: String, Sendable, CaseIterable {
|
|
46
|
+
case home, lock, siri, volumeUp, volumeDown
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
public enum PrivateAPIError: Error, CustomStringConvertible {
|
|
50
|
+
case frameworksUnavailable(String)
|
|
51
|
+
case noBootedDevice
|
|
52
|
+
case deviceNotFound(String)
|
|
53
|
+
case noDisplayPort
|
|
54
|
+
case surfaceUnavailable
|
|
55
|
+
case hidUnavailable(String)
|
|
56
|
+
case simctlFailed(String)
|
|
57
|
+
|
|
58
|
+
public var description: String {
|
|
59
|
+
switch self {
|
|
60
|
+
case .frameworksUnavailable(let d): return "simulator frameworks unavailable: \(d)"
|
|
61
|
+
case .noBootedDevice: return "no booted simulator"
|
|
62
|
+
case .deviceNotFound(let u): return "no simulator matching \(u)"
|
|
63
|
+
case .noDisplayPort: return "the device exposes no active display port"
|
|
64
|
+
case .surfaceUnavailable: return "the display surface could not be read"
|
|
65
|
+
case .hidUnavailable(let d): return "input is unavailable: \(d)"
|
|
66
|
+
case .simctlFailed(let d): return "simctl \(d)"
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/// The whole private surface simframe depends on. Anything that talks to
|
|
72
|
+
/// CoreSimulator or SimulatorKit goes through this.
|
|
73
|
+
public protocol SimulatorPlatform: AnyObject {
|
|
74
|
+
func bootedDevices() throws -> [DeviceInfo]
|
|
75
|
+
/// Bind to a device's display. Must be called before `withFrame`.
|
|
76
|
+
func attach(udid: String?) throws -> DeviceInfo
|
|
77
|
+
/// Re-resolve the display port on the device already attached.
|
|
78
|
+
///
|
|
79
|
+
/// The descriptor handed out by `attach` can die under a live daemon: the
|
|
80
|
+
/// simulator tears its display port down and builds a new one, and every
|
|
81
|
+
/// `framebufferSurface` read on the old object returns nil from then on.
|
|
82
|
+
/// Measured on one long session — capture reported "the display surface
|
|
83
|
+
/// could not be read" for six minutes on a device that was awake and
|
|
84
|
+
/// perfectly visible, and a daemon restart fixed it instantly. Without a
|
|
85
|
+
/// way to re-resolve, a restart is the only cure.
|
|
86
|
+
func reattachDisplay() throws -> DeviceInfo
|
|
87
|
+
/// Borrow the current framebuffer. The pointer is only valid inside `body`.
|
|
88
|
+
func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T
|
|
89
|
+
/// Called whenever the display reports damage — the per-redraw signal, so a
|
|
90
|
+
/// capture loop can be driven by the screen rather than by a timer.
|
|
91
|
+
func observeChanges(_ handler: @escaping () -> Void) throws
|
|
92
|
+
func detach()
|
|
93
|
+
|
|
94
|
+
// MARK: Input. Optional: a platform may observe without being able to touch.
|
|
95
|
+
|
|
96
|
+
/// Whether input is available, and why not when it is not.
|
|
97
|
+
func inputStatus() -> (available: Bool, detail: String)
|
|
98
|
+
/// A press and release at one point.
|
|
99
|
+
func tap(at point: CGPoint, durationMs: Double) throws
|
|
100
|
+
/// A deliberate hold. Named separately from `tap` because callers mean
|
|
101
|
+
/// something different by it, even though the mechanism is a longer hold.
|
|
102
|
+
func longPress(at point: CGPoint, durationMs: Double) throws
|
|
103
|
+
/// Press, hold in place, then move and release. The hold is what separates
|
|
104
|
+
/// a drag from a swipe: reorderable lists and drag-and-drop need the UI to
|
|
105
|
+
/// register a pick-up before movement starts.
|
|
106
|
+
func drag(from: CGPoint, to: CGPoint, holdMs: Double, durationMs: Double) throws
|
|
107
|
+
/// A real down, interpolated moves, then up — never a teleporting jump.
|
|
108
|
+
func swipe(from: CGPoint, to: CGPoint, durationMs: Double) throws
|
|
109
|
+
func type(_ text: String) throws
|
|
110
|
+
/// Put text on the device pasteboard and paste it.
|
|
111
|
+
///
|
|
112
|
+
/// Key events are mapped by whatever keyboard layout iOS has active, so
|
|
113
|
+
/// typing "Fryer 3" against a Persian layout yields Persian text with no
|
|
114
|
+
/// error. The pasteboard carries characters rather than key positions, so
|
|
115
|
+
/// it is the only reliable route for content that must be exact.
|
|
116
|
+
func paste(_ text: String) throws
|
|
117
|
+
func press(_ button: HardwareButton) throws
|
|
118
|
+
/// Rebuild the HID session.
|
|
119
|
+
///
|
|
120
|
+
/// Input has no feedback channel: a dispatched Indigo message reports
|
|
121
|
+
/// success when the send succeeds, and there is no way to ask the device
|
|
122
|
+
/// whether anything happened. Measured on a long-running daemon, a HOME
|
|
123
|
+
/// press returned in 66ms and the screen never moved, while the same press
|
|
124
|
+
/// on a freshly started daemon worked — so the session can stop delivering
|
|
125
|
+
/// while still accepting. This is the recovery for that, and the caller
|
|
126
|
+
/// that notices is the one with the frames.
|
|
127
|
+
func resetInput() throws
|
|
128
|
+
|
|
129
|
+
// MARK: Accessibility. Optional in the same way input is: a platform may
|
|
130
|
+
// see the screen without being able to read the app's own tree.
|
|
131
|
+
|
|
132
|
+
/// Whether the accessibility tree can be read, and why not when it cannot.
|
|
133
|
+
func accessibilityStatus() -> (available: Bool, detail: String)
|
|
134
|
+
/// The frontmost application's accessibility tree, flattened depth-first,
|
|
135
|
+
/// carrying whether it is all of one.
|
|
136
|
+
///
|
|
137
|
+
/// An app still launching has no tree yet, and this reports that as it is —
|
|
138
|
+
/// one node, no children — rather than retrying until it looks populated.
|
|
139
|
+
func accessibilityTree() throws -> AXTree
|
|
140
|
+
|
|
141
|
+
// MARK: App lifecycle. These are simctl, not private API — no HID needed.
|
|
142
|
+
|
|
143
|
+
@discardableResult
|
|
144
|
+
func launch(bundleId: String, arguments: [String], environment: [String: String]) throws -> Int32
|
|
145
|
+
func terminate(bundleId: String) throws
|
|
146
|
+
func openURL(_ url: String) throws
|
|
147
|
+
/// `action` is grant/revoke/reset, `service` one of simctl's privacy services.
|
|
148
|
+
func permission(action: String, service: String, bundleId: String?) throws
|
|
149
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import CoreGraphics
|
|
2
|
+
import Foundation
|
|
3
|
+
|
|
4
|
+
/// A deterministic platform for tests and for machines without Xcode.
|
|
5
|
+
/// Renders a synthetic screen so hashing and frame-store logic can be tested
|
|
6
|
+
/// with no simulator present.
|
|
7
|
+
public final class StubPlatform: SimulatorPlatform {
|
|
8
|
+
public var width: Int
|
|
9
|
+
public var height: Int
|
|
10
|
+
public var tint: UInt8
|
|
11
|
+
private var buffer: [UInt8] = []
|
|
12
|
+
/// What was asked of it, so tests can assert on gestures without a device.
|
|
13
|
+
public var recorded: [String] = []
|
|
14
|
+
/// What `accessibilityTree()` should answer. Empty by default, which is
|
|
15
|
+
/// what a device with no app in the foreground genuinely looks like.
|
|
16
|
+
public var stubTree: AXTree = AXTree(nodes: [])
|
|
17
|
+
|
|
18
|
+
public init(width: Int = 1206, height: Int = 2622, tint: UInt8 = 0) {
|
|
19
|
+
self.width = width
|
|
20
|
+
self.height = height
|
|
21
|
+
self.tint = tint
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
public func bootedDevices() throws -> [DeviceInfo] {
|
|
25
|
+
[DeviceInfo(udid: "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")]
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
public func attach(udid: String?) throws -> DeviceInfo {
|
|
29
|
+
attachedUdid = udid
|
|
30
|
+
return DeviceInfo(udid: udid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// Counted so a test can assert the capture loop actually tries to recover
|
|
34
|
+
/// rather than logging the same failure forever.
|
|
35
|
+
public private(set) var reattachCount = 0
|
|
36
|
+
private var attachedUdid: String?
|
|
37
|
+
|
|
38
|
+
/// Counted, so a test can assert that a dead input path is actually retried.
|
|
39
|
+
public private(set) var resetInputCount = 0
|
|
40
|
+
public func resetInput() throws { resetInputCount += 1 }
|
|
41
|
+
|
|
42
|
+
public func reattachDisplay() throws -> DeviceInfo {
|
|
43
|
+
reattachCount += 1
|
|
44
|
+
return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
|
|
48
|
+
let bytesPerRow = width * 4 + 40 // deliberate padding: mirrors real surfaces
|
|
49
|
+
if buffer.count != bytesPerRow * height {
|
|
50
|
+
buffer = [UInt8](repeating: 0, count: bytesPerRow * height)
|
|
51
|
+
}
|
|
52
|
+
for y in 0..<height {
|
|
53
|
+
for x in 0..<width {
|
|
54
|
+
let i = y * bytesPerRow + x * 4
|
|
55
|
+
let band: UInt8 = x < width / 2 ? 40 : 200
|
|
56
|
+
buffer[i] = band &+ tint // B
|
|
57
|
+
buffer[i + 1] = band // G
|
|
58
|
+
buffer[i + 2] = band // R
|
|
59
|
+
buffer[i + 3] = 255 // A
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return try buffer.withUnsafeBufferPointer { ptr in
|
|
63
|
+
try body(RawFrame(width: width, height: height, bytesPerRow: bytesPerRow, pixels: ptr.baseAddress!))
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
public func observeChanges(_ handler: @escaping () -> Void) throws {
|
|
68
|
+
// The stub never changes on its own; tests drive `tint` and call this.
|
|
69
|
+
stubHandler = handler
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/// Test hook: mutate the synthetic screen and announce it.
|
|
73
|
+
public func simulateChange() {
|
|
74
|
+
tint = tint &+ 40
|
|
75
|
+
stubHandler?()
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private var stubHandler: (() -> Void)?
|
|
79
|
+
|
|
80
|
+
public func detach() { stubHandler = nil }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
extension StubPlatform {
|
|
84
|
+
public func inputStatus() -> (available: Bool, detail: String) { (true, "stub") }
|
|
85
|
+
public func tap(at point: CGPoint, durationMs: Double) throws { recorded.append("tap(\(Int(point.x)),\(Int(point.y)))") }
|
|
86
|
+
public func swipe(from: CGPoint, to: CGPoint, durationMs: Double) throws {
|
|
87
|
+
recorded.append("swipe(\(Int(from.x)),\(Int(from.y))->\(Int(to.x)),\(Int(to.y)))")
|
|
88
|
+
}
|
|
89
|
+
public func type(_ text: String) throws { recorded.append("type(\(text))") }
|
|
90
|
+
public func paste(_ text: String) throws { recorded.append("paste(\(text))") }
|
|
91
|
+
public func press(_ button: HardwareButton) throws { recorded.append("press(\(button.rawValue))") }
|
|
92
|
+
public func longPress(at point: CGPoint, durationMs: Double) throws {
|
|
93
|
+
recorded.append("longPress(\(Int(point.x)),\(Int(point.y)),\(Int(durationMs)))")
|
|
94
|
+
}
|
|
95
|
+
public func drag(from: CGPoint, to: CGPoint, holdMs: Double, durationMs: Double) throws {
|
|
96
|
+
recorded.append("drag(\(Int(from.x)),\(Int(from.y))->\(Int(to.x)),\(Int(to.y)),hold=\(Int(holdMs)))")
|
|
97
|
+
}
|
|
98
|
+
@discardableResult
|
|
99
|
+
public func launch(bundleId: String, arguments: [String], environment: [String: String]) throws -> Int32 {
|
|
100
|
+
recorded.append("launch(\(bundleId))"); return 1234
|
|
101
|
+
}
|
|
102
|
+
public func terminate(bundleId: String) throws { recorded.append("terminate(\(bundleId))") }
|
|
103
|
+
public func openURL(_ url: String) throws { recorded.append("openURL(\(url))") }
|
|
104
|
+
public func permission(action: String, service: String, bundleId: String?) throws {
|
|
105
|
+
recorded.append("permission(\(action),\(service),\(bundleId ?? "-"))")
|
|
106
|
+
}
|
|
107
|
+
public func accessibilityStatus() -> (available: Bool, detail: String) { (true, "stub") }
|
|
108
|
+
public func accessibilityTree() throws -> AXTree {
|
|
109
|
+
recorded.append("accessibilityTree()")
|
|
110
|
+
return stubTree
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import PrivateAPI
|
|
3
|
+
|
|
4
|
+
/// A tightly-packed RGBA bitmap, which is what every algorithm below expects.
|
|
5
|
+
/// The simulator hands us padded BGRA, so conversion happens exactly once.
|
|
6
|
+
public struct Bitmap {
|
|
7
|
+
public let width: Int
|
|
8
|
+
public let height: Int
|
|
9
|
+
public var data: [UInt8] // RGBA, 4 bytes per pixel, no row padding
|
|
10
|
+
|
|
11
|
+
public init(width: Int, height: Int, data: [UInt8]) {
|
|
12
|
+
self.width = width
|
|
13
|
+
self.height = height
|
|
14
|
+
self.data = data
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/// Convert a padded BGRA surface into packed RGBA, downscaling by box
|
|
18
|
+
/// average in the same pass. Box averaging is what `grayGrid` does anyway,
|
|
19
|
+
/// so doing it here keeps the scaled image consistent with the hashes.
|
|
20
|
+
public static func from(_ frame: RawFrame, targetLongEdge: Int) -> Bitmap {
|
|
21
|
+
let (tw, th) = Self.fit(width: frame.width, height: frame.height, longEdge: targetLongEdge)
|
|
22
|
+
var out = [UInt8](repeating: 0, count: tw * th * 4)
|
|
23
|
+
let src = frame.pixels
|
|
24
|
+
|
|
25
|
+
for ty in 0..<th {
|
|
26
|
+
let y0 = (ty * frame.height) / th
|
|
27
|
+
let y1 = max(y0 + 1, ((ty + 1) * frame.height) / th)
|
|
28
|
+
for tx in 0..<tw {
|
|
29
|
+
let x0 = (tx * frame.width) / tw
|
|
30
|
+
let x1 = max(x0 + 1, ((tx + 1) * frame.width) / tw)
|
|
31
|
+
var r = 0, g = 0, b = 0, n = 0
|
|
32
|
+
for y in y0..<y1 {
|
|
33
|
+
let row = y * frame.bytesPerRow
|
|
34
|
+
for x in x0..<x1 {
|
|
35
|
+
let i = row + x * 4
|
|
36
|
+
b += Int(src[i]) // surfaces are BGRA
|
|
37
|
+
g += Int(src[i + 1])
|
|
38
|
+
r += Int(src[i + 2])
|
|
39
|
+
n += 1
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
let o = (ty * tw + tx) * 4
|
|
43
|
+
out[o] = UInt8(r / n)
|
|
44
|
+
out[o + 1] = UInt8(g / n)
|
|
45
|
+
out[o + 2] = UInt8(b / n)
|
|
46
|
+
out[o + 3] = 255
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return Bitmap(width: tw, height: th, data: out)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/// Match `sips -Z`: the longer edge becomes `longEdge`, aspect preserved.
|
|
53
|
+
public static func fit(width: Int, height: Int, longEdge: Int) -> (Int, Int) {
|
|
54
|
+
guard width > 0, height > 0 else { return (1, 1) }
|
|
55
|
+
if max(width, height) <= longEdge { return (width, height) }
|
|
56
|
+
if height >= width {
|
|
57
|
+
return (max(1, Int((Double(width) * Double(longEdge) / Double(height)).rounded())), longEdge)
|
|
58
|
+
}
|
|
59
|
+
return (longEdge, max(1, Int((Double(height) * Double(longEdge) / Double(width)).rounded())))
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import PrivateAPI
|
|
3
|
+
|
|
4
|
+
/// A per-device Unix domain socket carrying one JSON object per line.
|
|
5
|
+
///
|
|
6
|
+
/// Unix sockets rather than TCP because the permission model is the file
|
|
7
|
+
/// system's: mode 0600 means only this user can drive the simulator, with no
|
|
8
|
+
/// port to collide and nothing reachable from the network.
|
|
9
|
+
public final class ControlSocket {
|
|
10
|
+
public typealias Handler = ([String: Any]) -> [String: Any]
|
|
11
|
+
|
|
12
|
+
private let path: String
|
|
13
|
+
private let handler: Handler
|
|
14
|
+
private var listenFD: Int32 = -1
|
|
15
|
+
private var acceptSource: DispatchSourceRead?
|
|
16
|
+
// Requests run off the capture loop: a swipe sleeps for its whole duration
|
|
17
|
+
// and must not stall frame capture.
|
|
18
|
+
private let workQueue = DispatchQueue(label: "simframe.control", qos: .userInitiated)
|
|
19
|
+
/// Identity of the socket file this instance created, so shutdown can tell
|
|
20
|
+
/// its own socket from one a successor has since bound at the same path.
|
|
21
|
+
private var boundIno: ino_t = 0
|
|
22
|
+
private var boundDev: dev_t = 0
|
|
23
|
+
|
|
24
|
+
public init(path: String, handler: @escaping Handler) {
|
|
25
|
+
self.path = path
|
|
26
|
+
self.handler = handler
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
public func start() throws {
|
|
30
|
+
unlink(path)
|
|
31
|
+
listenFD = socket(AF_UNIX, SOCK_STREAM, 0)
|
|
32
|
+
guard listenFD >= 0 else { throw SocketError.failed("socket(): \(errno)") }
|
|
33
|
+
|
|
34
|
+
var addr = sockaddr_un()
|
|
35
|
+
addr.sun_family = sa_family_t(AF_UNIX)
|
|
36
|
+
let maxLen = MemoryLayout.size(ofValue: addr.sun_path)
|
|
37
|
+
guard path.utf8.count < maxLen else { throw SocketError.failed("socket path too long") }
|
|
38
|
+
withUnsafeMutablePointer(to: &addr.sun_path) { p in
|
|
39
|
+
p.withMemoryRebound(to: CChar.self, capacity: maxLen) { dst in
|
|
40
|
+
_ = strcpy(dst, path)
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
let size = socklen_t(MemoryLayout<sockaddr_un>.size)
|
|
44
|
+
let bound = withUnsafePointer(to: &addr) {
|
|
45
|
+
$0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(listenFD, $0, size) }
|
|
46
|
+
}
|
|
47
|
+
guard bound == 0 else { close(listenFD); throw SocketError.failed("bind(): \(errno)") }
|
|
48
|
+
// Only this user may drive the device.
|
|
49
|
+
chmod(path, 0o600)
|
|
50
|
+
var bornStat = stat()
|
|
51
|
+
if stat(path, &bornStat) == 0 {
|
|
52
|
+
boundIno = bornStat.st_ino
|
|
53
|
+
boundDev = bornStat.st_dev
|
|
54
|
+
}
|
|
55
|
+
guard listen(listenFD, 8) == 0 else { close(listenFD); throw SocketError.failed("listen(): \(errno)") }
|
|
56
|
+
|
|
57
|
+
let source = DispatchSource.makeReadSource(fileDescriptor: listenFD, queue: workQueue)
|
|
58
|
+
source.setEventHandler { [weak self] in self?.acceptOne() }
|
|
59
|
+
source.resume()
|
|
60
|
+
acceptSource = source
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private func acceptOne() {
|
|
64
|
+
let fd = accept(listenFD, nil, nil)
|
|
65
|
+
guard fd >= 0 else { return }
|
|
66
|
+
defer { close(fd) }
|
|
67
|
+
guard let request = readLine(fd) else { return }
|
|
68
|
+
|
|
69
|
+
var response: [String: Any]
|
|
70
|
+
if let data = request.data(using: .utf8),
|
|
71
|
+
let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
|
|
72
|
+
response = handler(object)
|
|
73
|
+
if let id = object["id"] { response["id"] = id }
|
|
74
|
+
} else {
|
|
75
|
+
response = ["ok": false, "error": "malformed request"]
|
|
76
|
+
}
|
|
77
|
+
if let out = try? JSONSerialization.data(withJSONObject: response) {
|
|
78
|
+
var line = out
|
|
79
|
+
line.append(0x0A)
|
|
80
|
+
_ = line.withUnsafeBytes { write(fd, $0.baseAddress, $0.count) }
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
private func readLine(_ fd: Int32) -> String? {
|
|
85
|
+
var buffer = [UInt8]()
|
|
86
|
+
var byte: UInt8 = 0
|
|
87
|
+
while read(fd, &byte, 1) == 1 {
|
|
88
|
+
if byte == 0x0A { break }
|
|
89
|
+
buffer.append(byte)
|
|
90
|
+
if buffer.count > 1 << 20 { return nil }
|
|
91
|
+
}
|
|
92
|
+
return buffer.isEmpty ? nil : String(decoding: buffer, as: UTF8.self)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
public func stop() {
|
|
96
|
+
acceptSource?.cancel()
|
|
97
|
+
acceptSource = nil
|
|
98
|
+
if listenFD >= 0 { close(listenFD) }
|
|
99
|
+
listenFD = -1
|
|
100
|
+
// Only remove the socket if it is still the one we created.
|
|
101
|
+
//
|
|
102
|
+
// A restart binds a fresh socket at the same path, and a dying daemon
|
|
103
|
+
// that unlinks blindly deletes its *successor's* socket. Capture keeps
|
|
104
|
+
// working — that is file-based — so the only symptom is input quietly
|
|
105
|
+
// dropping to idb, which is slower and has different semantics. That is
|
|
106
|
+
// exactly the kind of invisible downgrade this tool is supposed to
|
|
107
|
+
// refuse to have.
|
|
108
|
+
var current = stat()
|
|
109
|
+
if stat(path, ¤t) == 0, current.st_ino == boundIno, current.st_dev == boundDev {
|
|
110
|
+
unlink(path)
|
|
111
|
+
}
|
|
112
|
+
boundIno = 0
|
|
113
|
+
boundDev = 0
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
public enum SocketError: Error, CustomStringConvertible {
|
|
117
|
+
case failed(String)
|
|
118
|
+
public var description: String {
|
|
119
|
+
switch self { case .failed(let d): return "control socket: \(d)" }
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import CoreGraphics
|
|
2
|
+
import ImageIO
|
|
3
|
+
import Foundation
|
|
4
|
+
import PrivateAPI
|
|
5
|
+
|
|
6
|
+
/// Downscale through CoreGraphics rather than by box average.
|
|
7
|
+
///
|
|
8
|
+
/// `sips` — which the JavaScript capture loop uses — is a thin wrapper over
|
|
9
|
+
/// ImageIO/CoreGraphics, so resampling the same way is the closest we can get
|
|
10
|
+
/// to producing byte-identical hashes without reimplementing its filter.
|
|
11
|
+
public enum CoreGraphicsScaler {
|
|
12
|
+
public static func bitmap(from frame: RawFrame, targetLongEdge: Int) -> Bitmap? {
|
|
13
|
+
let (tw, th) = Bitmap.fit(width: frame.width, height: frame.height, longEdge: targetLongEdge)
|
|
14
|
+
let srcBytes = frame.bytesPerRow * frame.height
|
|
15
|
+
guard let provider = CGDataProvider(dataInfo: nil, data: frame.pixels, size: srcBytes, releaseData: { _, _, _ in }) else {
|
|
16
|
+
return nil
|
|
17
|
+
}
|
|
18
|
+
// Surfaces are BGRA; this bitmapInfo is how CoreGraphics is told so.
|
|
19
|
+
let srcInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue).union(.byteOrder32Little)
|
|
20
|
+
guard let cgImage = CGImage(
|
|
21
|
+
width: frame.width, height: frame.height,
|
|
22
|
+
bitsPerComponent: 8, bitsPerPixel: 32,
|
|
23
|
+
bytesPerRow: frame.bytesPerRow,
|
|
24
|
+
space: CGColorSpaceCreateDeviceRGB(),
|
|
25
|
+
bitmapInfo: srcInfo,
|
|
26
|
+
provider: provider, decode: nil,
|
|
27
|
+
shouldInterpolate: false, intent: .defaultIntent
|
|
28
|
+
) else { return nil }
|
|
29
|
+
|
|
30
|
+
var out = [UInt8](repeating: 0, count: tw * th * 4)
|
|
31
|
+
let ok = out.withUnsafeMutableBytes { buf -> Bool in
|
|
32
|
+
guard let ctx = CGContext(
|
|
33
|
+
data: buf.baseAddress, width: tw, height: th,
|
|
34
|
+
bitsPerComponent: 8, bytesPerRow: tw * 4,
|
|
35
|
+
space: CGColorSpaceCreateDeviceRGB(),
|
|
36
|
+
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
|
37
|
+
) else { return false }
|
|
38
|
+
ctx.interpolationQuality = .high
|
|
39
|
+
ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: tw, height: th))
|
|
40
|
+
return true
|
|
41
|
+
}
|
|
42
|
+
return ok ? Bitmap(width: tw, height: th, data: out) : nil
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
extension CoreGraphicsScaler {
|
|
47
|
+
/// Re-encode an existing PNG at a fraction of its size. Frames that have
|
|
48
|
+
/// aged out of the fine window stay legible enough to tell which screen was
|
|
49
|
+
/// showing, at roughly a quarter of the bytes.
|
|
50
|
+
public static func downscalePNG(at url: URL, factor: Double) -> Bitmap? {
|
|
51
|
+
guard factor > 0, factor < 1,
|
|
52
|
+
let src = CGImageSourceCreateWithURL(url as CFURL, nil),
|
|
53
|
+
let image = CGImageSourceCreateImageAtIndex(src, 0, nil) else { return nil }
|
|
54
|
+
let tw = max(1, Int(Double(image.width) * factor))
|
|
55
|
+
let th = max(1, Int(Double(image.height) * factor))
|
|
56
|
+
var out = [UInt8](repeating: 0, count: tw * th * 4)
|
|
57
|
+
let ok = out.withUnsafeMutableBytes { buf -> Bool in
|
|
58
|
+
guard let ctx = CGContext(
|
|
59
|
+
data: buf.baseAddress, width: tw, height: th,
|
|
60
|
+
bitsPerComponent: 8, bytesPerRow: tw * 4,
|
|
61
|
+
space: CGColorSpaceCreateDeviceRGB(),
|
|
62
|
+
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
|
63
|
+
) else { return false }
|
|
64
|
+
ctx.interpolationQuality = .high
|
|
65
|
+
ctx.draw(image, in: CGRect(x: 0, y: 0, width: tw, height: th))
|
|
66
|
+
return true
|
|
67
|
+
}
|
|
68
|
+
return ok ? Bitmap(width: tw, height: th, data: out) : nil
|
|
69
|
+
}
|
|
70
|
+
}
|