simframe 0.1.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -155
- package/native/ocr.swift +29 -0
- package/package.json +2 -1
- package/src/actions.js +244 -0
- package/src/analyze.js +59 -0
- package/src/cli.js +198 -15
- package/src/daemon.js +139 -9
- package/src/index.js +382 -15
- package/src/input.js +221 -0
- package/src/intent.js +100 -0
- package/src/mcp.js +284 -22
- package/src/ocr.js +71 -0
- package/src/screenmap.js +229 -0
- package/src/simctl.js +26 -1
package/src/daemon.js
CHANGED
|
@@ -2,22 +2,40 @@
|
|
|
2
2
|
// newest frame permanently warm on disk so a reader never waits on simctl.
|
|
3
3
|
import fs from 'node:fs';
|
|
4
4
|
import path from 'node:path';
|
|
5
|
-
import { decodePng } from './png.js';
|
|
6
|
-
import {
|
|
5
|
+
import { decodePng, encodePng, scaleBitmap } from './png.js';
|
|
6
|
+
import {
|
|
7
|
+
frameHash,
|
|
8
|
+
layoutHash,
|
|
9
|
+
regionSignature,
|
|
10
|
+
signatureDiff,
|
|
11
|
+
regionDeltas,
|
|
12
|
+
signatureToHex,
|
|
13
|
+
} from './analyze.js';
|
|
7
14
|
import * as store from './store.js';
|
|
8
15
|
import { isBootedSync, resize, screenshot } from './simctl.js';
|
|
9
16
|
|
|
10
17
|
// Bump whenever the shape of state.json changes, so an upgraded client retires
|
|
11
18
|
// a capture loop left running by an older install instead of misreading it.
|
|
12
|
-
export const STATE_VERSION =
|
|
19
|
+
export const STATE_VERSION = 5;
|
|
13
20
|
|
|
14
21
|
export const DEFAULTS = {
|
|
15
22
|
fps: 4,
|
|
16
23
|
idleFps: 1.5,
|
|
17
24
|
idleAfterMs: 2500,
|
|
18
25
|
maxDim: 700,
|
|
19
|
-
|
|
20
|
-
|
|
26
|
+
// Frame memory: every frame for the last few seconds, thinned to roughly
|
|
27
|
+
// 2fps further back. Fine detail where transitions live, cheap recall beyond.
|
|
28
|
+
retainMs: 60_000,
|
|
29
|
+
fineMs: 6_000,
|
|
30
|
+
keyframeMs: 450,
|
|
31
|
+
// Frames older than fineMs are re-encoded at half size: still legible enough
|
|
32
|
+
// to tell which screen was showing, at roughly a quarter of the bytes.
|
|
33
|
+
recallScale: 0.5,
|
|
34
|
+
maxRingBytes: 12 << 20,
|
|
35
|
+
ringSize: 400,
|
|
36
|
+
historySize: 400,
|
|
37
|
+
historyMs: 90_000,
|
|
38
|
+
fullKeep: 8,
|
|
21
39
|
changeThreshold: 0.004,
|
|
22
40
|
idleExitMs: 15 * 60_000,
|
|
23
41
|
};
|
|
@@ -53,6 +71,11 @@ export async function runDaemon(device, options = {}) {
|
|
|
53
71
|
|
|
54
72
|
let seq = 0;
|
|
55
73
|
let prevSignature = null;
|
|
74
|
+
/** Recent frames, so a caller can diff against whatever it last saw rather
|
|
75
|
+
* than only against the frame that happened to precede this one. */
|
|
76
|
+
let history = [];
|
|
77
|
+
/** seq + timestamp for every frame still on disk, so retention can be thinned by age. */
|
|
78
|
+
let ringIndex = [];
|
|
56
79
|
let lastChangeAt = Date.now();
|
|
57
80
|
let consecutiveErrors = 0;
|
|
58
81
|
let lastBootCheck = Date.now();
|
|
@@ -102,6 +125,7 @@ export async function runDaemon(device, options = {}) {
|
|
|
102
125
|
|
|
103
126
|
const bmp = decodePng(fs.readFileSync(ringFile));
|
|
104
127
|
const signature = regionSignature(bmp);
|
|
128
|
+
const prevWasNull = prevSignature === null;
|
|
105
129
|
const diff = signatureDiff(signature, prevSignature);
|
|
106
130
|
const deltas = regionDeltas(signature, prevSignature);
|
|
107
131
|
const changed = diff > opts.changeThreshold;
|
|
@@ -112,6 +136,18 @@ export async function runDaemon(device, options = {}) {
|
|
|
112
136
|
prevSignature = signature;
|
|
113
137
|
consecutiveErrors = 0;
|
|
114
138
|
|
|
139
|
+
const hash = frameHash(bmp);
|
|
140
|
+
const layout = layoutHash(bmp);
|
|
141
|
+
history.push({
|
|
142
|
+
seq,
|
|
143
|
+
at: now,
|
|
144
|
+
hash,
|
|
145
|
+
sig: signatureToHex(signature),
|
|
146
|
+
diff: prevWasNull ? 0 : Number(diff.toFixed(5)),
|
|
147
|
+
});
|
|
148
|
+
const historyCutoff = now - opts.historyMs;
|
|
149
|
+
history = history.filter((h) => h.at >= historyCutoff).slice(-opts.historySize);
|
|
150
|
+
|
|
115
151
|
fs.copyFileSync(ringFile, path.join(p.dir, 'latest.png.tmp'));
|
|
116
152
|
fs.renameSync(path.join(p.dir, 'latest.png.tmp'), path.join(p.dir, 'latest.png'));
|
|
117
153
|
|
|
@@ -123,17 +159,34 @@ export async function runDaemon(device, options = {}) {
|
|
|
123
159
|
captureMs: now - tickStart,
|
|
124
160
|
width: bmp.width,
|
|
125
161
|
height: bmp.height,
|
|
126
|
-
hash
|
|
127
|
-
|
|
128
|
-
|
|
162
|
+
hash,
|
|
163
|
+
layoutHash: layout,
|
|
164
|
+
diff: prevWasNull ? null : Number(diff.toFixed(5)),
|
|
165
|
+
changed: prevWasNull ? false : changed,
|
|
166
|
+
firstFrame: prevWasNull,
|
|
129
167
|
stableForMs: now - lastChangeAt,
|
|
130
|
-
regions: deltas.map((d) => Number(d.toFixed(4))),
|
|
168
|
+
regions: prevWasNull ? deltas.map(() => 0) : deltas.map((d) => Number(d.toFixed(4))),
|
|
169
|
+
// Absolute instant of the last detected change: lets a caller reason
|
|
170
|
+
// about baselines older than the history window.
|
|
171
|
+
lastChangeAt,
|
|
172
|
+
history,
|
|
173
|
+
ring: ringIndex,
|
|
131
174
|
fullFile,
|
|
132
175
|
ringFile,
|
|
133
176
|
device,
|
|
134
177
|
}),
|
|
135
178
|
);
|
|
136
179
|
|
|
180
|
+
ringIndex.push({ seq, at: now });
|
|
181
|
+
ringIndex = thinRing(ringIndex, now, opts, (dropped) => {
|
|
182
|
+
try {
|
|
183
|
+
fs.unlinkSync(path.join(p.ring, `${dropped}.png`));
|
|
184
|
+
} catch {
|
|
185
|
+
/* already gone */
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
shrinkAgedFrames(p.ring, ringIndex, now, opts, log);
|
|
189
|
+
enforceByteBudget(p.ring, ringIndex, opts);
|
|
137
190
|
store.pruneDir(p.ring, opts.ringSize);
|
|
138
191
|
store.pruneDir(p.full, opts.fullKeep);
|
|
139
192
|
} catch (err) {
|
|
@@ -168,3 +221,80 @@ export async function runDaemon(device, options = {}) {
|
|
|
168
221
|
}
|
|
169
222
|
return outcome;
|
|
170
223
|
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Decide which buffered frames to keep. Everything inside `fineMs` survives, so
|
|
227
|
+
* a transition can be replayed frame by frame; beyond that only one frame per
|
|
228
|
+
* `keyframeMs` is kept, out to `retainMs`. Calls `drop` for each discarded seq
|
|
229
|
+
* and returns the retained index.
|
|
230
|
+
*/
|
|
231
|
+
export function thinRing(index, now, opts, drop = () => {}) {
|
|
232
|
+
const kept = [];
|
|
233
|
+
let lastKeptAt = null;
|
|
234
|
+
for (let i = index.length - 1; i >= 0; i--) {
|
|
235
|
+
const frame = index[i];
|
|
236
|
+
const age = now - frame.at;
|
|
237
|
+
if (age > opts.retainMs) {
|
|
238
|
+
drop(frame.seq);
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
if (age <= opts.fineMs || lastKeptAt === null || lastKeptAt - frame.at >= opts.keyframeMs) {
|
|
242
|
+
kept.push(frame);
|
|
243
|
+
lastKeptAt = frame.at;
|
|
244
|
+
} else {
|
|
245
|
+
drop(frame.seq);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return kept.reverse();
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Re-encode frames that have aged out of the fine window at a smaller size.
|
|
253
|
+
* Done in-process with the bundled PNG codec, so recall stays cheap on disk
|
|
254
|
+
* without adding a dependency or another process spawn per frame.
|
|
255
|
+
*/
|
|
256
|
+
function shrinkAgedFrames(dir, index, now, opts, log) {
|
|
257
|
+
for (const frame of index) {
|
|
258
|
+
if (frame.small || now - frame.at <= opts.fineMs) continue;
|
|
259
|
+
const file = path.join(dir, `${frame.seq}.png`);
|
|
260
|
+
try {
|
|
261
|
+
const bmp = decodePng(fs.readFileSync(file));
|
|
262
|
+
const small = scaleBitmap(
|
|
263
|
+
bmp,
|
|
264
|
+
Math.max(1, Math.round(bmp.width * opts.recallScale)),
|
|
265
|
+
Math.max(1, Math.round(bmp.height * opts.recallScale)),
|
|
266
|
+
);
|
|
267
|
+
store.writeAtomic(file, encodePng(small));
|
|
268
|
+
frame.small = true;
|
|
269
|
+
} catch (err) {
|
|
270
|
+
// A frame we cannot shrink is still a frame we can serve.
|
|
271
|
+
frame.small = true;
|
|
272
|
+
log?.(`shrink failed for #${frame.seq}: ${err.message}`);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Last-resort cap so a long session cannot grow the buffer without bound. */
|
|
278
|
+
function enforceByteBudget(dir, index, opts) {
|
|
279
|
+
let total = 0;
|
|
280
|
+
const sizes = index.map((frame) => {
|
|
281
|
+
let size = 0;
|
|
282
|
+
try {
|
|
283
|
+
size = fs.statSync(path.join(dir, `${frame.seq}.png`)).size;
|
|
284
|
+
} catch {
|
|
285
|
+
/* counted as zero */
|
|
286
|
+
}
|
|
287
|
+
total += size;
|
|
288
|
+
return size;
|
|
289
|
+
});
|
|
290
|
+
for (let i = 0; i < index.length && total > opts.maxRingBytes; i++) {
|
|
291
|
+
try {
|
|
292
|
+
fs.unlinkSync(path.join(dir, `${index[i].seq}.png`));
|
|
293
|
+
total -= sizes[i];
|
|
294
|
+
index[i].dropped = true;
|
|
295
|
+
} catch {
|
|
296
|
+
/* already gone */
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
for (let i = index.length - 1; i >= 0; i--) if (index[i].dropped) index.splice(i, 1);
|
|
300
|
+
}
|
package/src/index.js
CHANGED
|
@@ -5,8 +5,16 @@ import path from 'node:path';
|
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { DEFAULTS, STATE_VERSION } from './daemon.js';
|
|
7
7
|
import { decodePng, encodePng, scaleBitmap } from './png.js';
|
|
8
|
-
import {
|
|
9
|
-
|
|
8
|
+
import {
|
|
9
|
+
REGION_COLS,
|
|
10
|
+
hexToSignature,
|
|
11
|
+
regionDeltas,
|
|
12
|
+
regionMap,
|
|
13
|
+
signatureDiff,
|
|
14
|
+
} from './analyze.js';
|
|
15
|
+
import * as input from './input.js';
|
|
16
|
+
import * as screenmap from './screenmap.js';
|
|
17
|
+
import { resolveDevice, resize, screenshot } from './simctl.js';
|
|
10
18
|
import * as store from './store.js';
|
|
11
19
|
|
|
12
20
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
@@ -21,6 +29,39 @@ export function resolveMaxDim(detail) {
|
|
|
21
29
|
return DETAIL_LEVELS.normal;
|
|
22
30
|
}
|
|
23
31
|
|
|
32
|
+
/**
|
|
33
|
+
* A full-resolution frame to read text from. The capture loop prunes these
|
|
34
|
+
* aggressively, so by the time a caller wants one it is often already gone —
|
|
35
|
+
* in which case take a fresh shot rather than silently skipping OCR.
|
|
36
|
+
*/
|
|
37
|
+
async function fullFrameFor(udid, state) {
|
|
38
|
+
if (state.fullFile && fs.existsSync(state.fullFile)) return state.fullFile;
|
|
39
|
+
const p = store.paths(udid);
|
|
40
|
+
const file = path.join(p.dir, 'ocr-source.png');
|
|
41
|
+
await screenshot(udid, file, { mask: 'ignored' });
|
|
42
|
+
return file;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Points-per-pixel and screen size for this device. idb reports both exactly;
|
|
47
|
+
* without it, fall back to the captured frame's aspect and a 3x guess, which is
|
|
48
|
+
* only used for OCR coordinates that nothing can tap anyway.
|
|
49
|
+
*/
|
|
50
|
+
async function deviceGeometry(udid, state) {
|
|
51
|
+
try {
|
|
52
|
+
const geo = await input.screenInfo(udid);
|
|
53
|
+
if (geo.pointWidth && geo.pointHeight) return geo;
|
|
54
|
+
} catch {
|
|
55
|
+
/* idb absent: fall through */
|
|
56
|
+
}
|
|
57
|
+
const density = 3;
|
|
58
|
+
return {
|
|
59
|
+
density,
|
|
60
|
+
pointWidth: Math.round((state.width * (state.nativeScale ?? 1)) / 1) || 402,
|
|
61
|
+
pointHeight: Math.round((state.height * (state.nativeScale ?? 1)) / 1) || 874,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
24
65
|
export function daemonStatus(udid) {
|
|
25
66
|
const meta = store.readJson(store.paths(udid).meta);
|
|
26
67
|
const pid = meta?.pid ?? null;
|
|
@@ -120,9 +161,100 @@ function readLogTail(file, lines = 6) {
|
|
|
120
161
|
return safeRead(file).trim().split('\n').slice(-lines).join('\n');
|
|
121
162
|
}
|
|
122
163
|
|
|
123
|
-
|
|
164
|
+
/** A frame this old means the capture loop is wedged, not that the screen is calm. */
|
|
165
|
+
export const STALE_FRAME_MS = 2500;
|
|
166
|
+
|
|
167
|
+
/** Below this a "change" is a clock digit or a caret, not a new screen. */
|
|
168
|
+
export const MINOR_CHANGE = 0.004;
|
|
169
|
+
export const MAJOR_CHANGE = 0.03;
|
|
170
|
+
|
|
171
|
+
export function changeLevel(diff) {
|
|
172
|
+
if (diff > MAJOR_CHANGE) return 'major';
|
|
173
|
+
if (diff > MINOR_CHANGE) return 'minor';
|
|
174
|
+
return 'none';
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export function liveness(udid, state) {
|
|
178
|
+
const ageMs = Date.now() - state.capturedAt;
|
|
179
|
+
const { running } = daemonStatus(udid);
|
|
180
|
+
if (!running) {
|
|
181
|
+
return { ok: false, ageMs, note: 'the capture loop has died; the frame you are looking at is the last one it wrote' };
|
|
182
|
+
}
|
|
183
|
+
if (ageMs > STALE_FRAME_MS) {
|
|
184
|
+
return { ok: false, ageMs, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
|
|
185
|
+
}
|
|
186
|
+
return { ok: true, ageMs, note: null };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Find the frame a caller is diffing against. `since` may be a frame hash, a
|
|
191
|
+
* sequence number, or a millisecond timestamp. Baselines older than the history
|
|
192
|
+
* window fall back to a coarse answer derived from lastChangeAt, which is still
|
|
193
|
+
* correct about *whether* anything changed.
|
|
194
|
+
*/
|
|
195
|
+
export function resolveBaseline(state, since) {
|
|
196
|
+
if (since == null) return null;
|
|
197
|
+
const history = state.history || [];
|
|
198
|
+
const key = String(since);
|
|
199
|
+
let entry = null;
|
|
200
|
+
for (let i = history.length - 1; i >= 0; i--) {
|
|
201
|
+
const h = history[i];
|
|
202
|
+
if (h.hash === key || String(h.seq) === key) {
|
|
203
|
+
entry = h;
|
|
204
|
+
break;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (entry) return { kind: 'history', entry };
|
|
208
|
+
|
|
209
|
+
const at = Number(since);
|
|
210
|
+
if (Number.isFinite(at) && at > 1e12) {
|
|
211
|
+
return { kind: 'coarse', at, changed: (state.lastChangeAt ?? 0) > at };
|
|
212
|
+
}
|
|
213
|
+
return { kind: 'unmatched', requested: key };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function compareToBaseline(state, baseline) {
|
|
217
|
+
if (!baseline) return null;
|
|
218
|
+
if (baseline.kind === 'history') {
|
|
219
|
+
const from = hexToSignature(baseline.entry.sig);
|
|
220
|
+
const to = hexToSignature(
|
|
221
|
+
(state.history || []).find((h) => h.seq === state.seq)?.sig || '',
|
|
222
|
+
);
|
|
223
|
+
if (!to.length) return { kind: 'unmatched', requested: String(baseline.entry.seq) };
|
|
224
|
+
const diff = signatureDiff(to, from);
|
|
225
|
+
const deltas = regionDeltas(to, from);
|
|
226
|
+
return {
|
|
227
|
+
kind: 'history',
|
|
228
|
+
matched: true,
|
|
229
|
+
seq: baseline.entry.seq,
|
|
230
|
+
hash: baseline.entry.hash,
|
|
231
|
+
at: baseline.entry.at,
|
|
232
|
+
ageMs: Date.now() - baseline.entry.at,
|
|
233
|
+
changed: diff > MINOR_CHANGE,
|
|
234
|
+
level: changeLevel(diff),
|
|
235
|
+
diff: Number(diff.toFixed(5)),
|
|
236
|
+
regions: deltas.map((d) => Number(d.toFixed(4))),
|
|
237
|
+
map: regionMap(deltas, REGION_COLS),
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
if (baseline.kind === 'coarse') {
|
|
241
|
+
return {
|
|
242
|
+
kind: 'coarse',
|
|
243
|
+
matched: false,
|
|
244
|
+
at: baseline.at,
|
|
245
|
+
ageMs: Date.now() - baseline.at,
|
|
246
|
+
changed: baseline.changed,
|
|
247
|
+
note: 'baseline is older than the buffered history; only whether-it-changed is known',
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
return { kind: 'unmatched', matched: false, requested: baseline.requested };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export function stopDaemon(udid, { force = false } = {}) {
|
|
124
254
|
const { pid, running } = daemonStatus(udid);
|
|
125
255
|
if (!running) return false;
|
|
256
|
+
// Another client may be mid-session on this device; do not yank it away.
|
|
257
|
+
if (!force && store.heartbeatAge(udid) < 60_000) return 'in-use';
|
|
126
258
|
try {
|
|
127
259
|
process.kill(pid, 'SIGTERM');
|
|
128
260
|
return true;
|
|
@@ -171,13 +303,15 @@ function pngSize(png) {
|
|
|
171
303
|
return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
|
|
172
304
|
}
|
|
173
305
|
|
|
174
|
-
export async function getState(deviceQuery, { options } = {}) {
|
|
306
|
+
export async function getState(deviceQuery, { since, options } = {}) {
|
|
175
307
|
const { device, state } = await ensureDaemon(deviceQuery, options);
|
|
176
308
|
return {
|
|
177
309
|
device,
|
|
178
310
|
state,
|
|
179
311
|
ageMs: Date.now() - state.capturedAt,
|
|
180
312
|
map: regionMap(state.regions || [], REGION_COLS),
|
|
313
|
+
since: compareToBaseline(state, resolveBaseline(state, since)),
|
|
314
|
+
live: liveness(device.udid, state),
|
|
181
315
|
};
|
|
182
316
|
}
|
|
183
317
|
|
|
@@ -185,29 +319,86 @@ export async function getState(deviceQuery, { options } = {}) {
|
|
|
185
319
|
* Wait for the screen to settle (`mode: 'stable'`) or to move away from what it
|
|
186
320
|
* shows right now (`mode: 'change'`). Removes the screenshot-retry loop.
|
|
187
321
|
*/
|
|
322
|
+
/**
|
|
323
|
+
* Wait for the screen to do something.
|
|
324
|
+
*
|
|
325
|
+
* change the screen differs from `since` (or from now, if no baseline)
|
|
326
|
+
* stable the screen holds still for `stableMs`, observed within this call
|
|
327
|
+
* settle change first, then stable — what you want after a tap or a launch
|
|
328
|
+
*
|
|
329
|
+
* Pass `since` (a hash captured BEFORE the action) whenever you can: a baseline
|
|
330
|
+
* sampled after the fact is the single most common way to wait for a change
|
|
331
|
+
* that has already happened.
|
|
332
|
+
*/
|
|
188
333
|
export async function waitFor(
|
|
189
334
|
deviceQuery,
|
|
190
|
-
{ mode = '
|
|
335
|
+
{ mode = 'settle', since, stableMs = 600, timeoutMs = 8000, reactionMs = 2500, baselineHash, options } = {},
|
|
191
336
|
) {
|
|
192
337
|
const { device, state: first } = await ensureDaemon(deviceQuery, options);
|
|
193
338
|
const p = store.paths(device.udid);
|
|
194
|
-
const
|
|
195
|
-
const
|
|
339
|
+
const requested = since ?? baselineHash;
|
|
340
|
+
const resolved = resolveBaseline(first, requested);
|
|
341
|
+
const baselineHashValue =
|
|
342
|
+
resolved?.kind === 'history' ? resolved.entry.hash : (requested ?? first.hash);
|
|
343
|
+
const baselineResolved = resolved?.kind === 'history' || requested == null;
|
|
344
|
+
|
|
345
|
+
const startedAt = Date.now();
|
|
346
|
+
const deadline = startedAt + timeoutMs;
|
|
347
|
+
const startSeq = first.seq;
|
|
196
348
|
let last = first;
|
|
349
|
+
let sawChange = mode === 'stable' || first.hash !== baselineHashValue;
|
|
350
|
+
const changedAtStart = sawChange && mode !== 'stable';
|
|
351
|
+
|
|
352
|
+
const done = (satisfied, extra = {}) => ({
|
|
353
|
+
device,
|
|
354
|
+
state: last,
|
|
355
|
+
satisfied,
|
|
356
|
+
mode,
|
|
357
|
+
sawChange,
|
|
358
|
+
changedBeforeWait: changedAtStart,
|
|
359
|
+
baselineHash: baselineHashValue,
|
|
360
|
+
baselineResolved,
|
|
361
|
+
waitedMs: Date.now() - startedAt,
|
|
362
|
+
live: liveness(device.udid, last),
|
|
363
|
+
...extra,
|
|
364
|
+
});
|
|
197
365
|
|
|
198
366
|
while (Date.now() < deadline) {
|
|
199
367
|
const state = store.readJson(p.state);
|
|
200
368
|
if (state) {
|
|
201
369
|
last = state;
|
|
370
|
+
// A wedged capture loop must not look like a calm screen.
|
|
371
|
+
const live = liveness(device.udid, state);
|
|
372
|
+
if (!live.ok) return done(false, { stalled: true });
|
|
373
|
+
|
|
374
|
+
if (!sawChange && state.hash !== baselineHashValue) sawChange = true;
|
|
375
|
+
|
|
376
|
+
// Some controls barely move the screen at all — a radio dot, a checkbox,
|
|
377
|
+
// a button changing state. Waiting the full timeout for a change that
|
|
378
|
+
// will never be visible turns a 100ms action into a 12s one, so give up
|
|
379
|
+
// early and say so, rather than silently burning the clock.
|
|
380
|
+
if (!sawChange && Date.now() - startedAt > reactionMs && state.stableForMs >= stableMs) {
|
|
381
|
+
return done(false, { noVisibleChange: true });
|
|
382
|
+
}
|
|
383
|
+
|
|
202
384
|
if (mode === 'change') {
|
|
203
|
-
if (
|
|
204
|
-
} else
|
|
205
|
-
|
|
385
|
+
if (sawChange) return done(true);
|
|
386
|
+
} else {
|
|
387
|
+
// "settle" requires a change first, so accumulated stillness from before
|
|
388
|
+
// the caller acted can never satisfy it; once the change is seen,
|
|
389
|
+
// stableForMs is measured from that change. Plain "stable" has no such
|
|
390
|
+
// requirement — an already-still screen genuinely is stable.
|
|
391
|
+
// At least one frame must arrive during the call, so the answer is
|
|
392
|
+
// never derived purely from what was already on disk.
|
|
393
|
+
const freshFrames = state.seq - startSeq;
|
|
394
|
+
if (sawChange && freshFrames >= 1 && state.stableForMs >= stableMs) {
|
|
395
|
+
return done(true);
|
|
396
|
+
}
|
|
206
397
|
}
|
|
207
398
|
}
|
|
208
399
|
await sleep(60);
|
|
209
400
|
}
|
|
210
|
-
return
|
|
401
|
+
return done(false, { timedOut: true });
|
|
211
402
|
}
|
|
212
403
|
|
|
213
404
|
/**
|
|
@@ -224,13 +415,24 @@ export async function getStrip(deviceQuery, { count = 5, spanMs, thumbMaxDim = 2
|
|
|
224
415
|
.filter((e) => Number.isFinite(e.seq))
|
|
225
416
|
.sort((a, b) => a.seq - b.seq);
|
|
226
417
|
|
|
227
|
-
|
|
418
|
+
const stamps = new Map(((await ensureDaemon(deviceQuery, options)).state.ring || []).map((f) => [f.seq, f.at]));
|
|
419
|
+
entries = entries
|
|
420
|
+
.map((e) => ({ ...e, mtimeMs: stamps.get(e.seq) ?? safeMtime(e.file) }))
|
|
421
|
+
.filter((e) => e.mtimeMs);
|
|
422
|
+
const want = Math.max(1, count);
|
|
228
423
|
if (spanMs) {
|
|
424
|
+
// "Show me the last 40 seconds" means frames spread ACROSS that window, not
|
|
425
|
+
// the newest few frames that happen to fall inside it.
|
|
229
426
|
const cutoff = Date.now() - spanMs;
|
|
230
427
|
const within = entries.filter((e) => e.mtimeMs >= cutoff);
|
|
231
|
-
if (within.length)
|
|
428
|
+
if (within.length) {
|
|
429
|
+
entries = within.length <= want ? within : spreadEvenly(within, want);
|
|
430
|
+
} else {
|
|
431
|
+
entries = entries.slice(-want);
|
|
432
|
+
}
|
|
433
|
+
} else {
|
|
434
|
+
entries = entries.slice(-want);
|
|
232
435
|
}
|
|
233
|
-
entries = entries.slice(-Math.max(1, count));
|
|
234
436
|
if (!entries.length) throw new Error('no frames buffered yet');
|
|
235
437
|
|
|
236
438
|
const frames = entries.map((e) => decodePng(fs.readFileSync(e.file)));
|
|
@@ -260,6 +462,17 @@ export async function getStrip(deviceQuery, { count = 5, spanMs, thumbMaxDim = 2
|
|
|
260
462
|
};
|
|
261
463
|
}
|
|
262
464
|
|
|
465
|
+
/** Pick `count` items spaced as evenly as possible across a list, keeping the ends. */
|
|
466
|
+
export function spreadEvenly(items, count) {
|
|
467
|
+
if (count >= items.length) return items;
|
|
468
|
+
if (count === 1) return [items[items.length - 1]];
|
|
469
|
+
const out = [];
|
|
470
|
+
for (let i = 0; i < count; i++) {
|
|
471
|
+
out.push(items[Math.round((i * (items.length - 1)) / (count - 1))]);
|
|
472
|
+
}
|
|
473
|
+
return out;
|
|
474
|
+
}
|
|
475
|
+
|
|
263
476
|
function safeMtime(file) {
|
|
264
477
|
try {
|
|
265
478
|
return fs.statSync(file).mtimeMs;
|
|
@@ -268,4 +481,158 @@ function safeMtime(file) {
|
|
|
268
481
|
}
|
|
269
482
|
}
|
|
270
483
|
|
|
271
|
-
|
|
484
|
+
/**
|
|
485
|
+
* What happened on screen over the last `spanMs`, derived from the buffered
|
|
486
|
+
* frame signatures. This is the "memory" view: not every frame, but the events
|
|
487
|
+
* worth knowing about, each with when it started, how long it took and how much
|
|
488
|
+
* of the screen it moved.
|
|
489
|
+
*/
|
|
490
|
+
export async function getTimeline(deviceQuery, { spanMs = 60_000, options } = {}) {
|
|
491
|
+
const { device, state } = await ensureDaemon(deviceQuery, options);
|
|
492
|
+
const now = Date.now();
|
|
493
|
+
const hist = (state.history || []).filter((h) => h.at >= now - spanMs);
|
|
494
|
+
const events = [];
|
|
495
|
+
let current = null;
|
|
496
|
+
|
|
497
|
+
hist.forEach((h, i) => {
|
|
498
|
+
if ((h.diff ?? 0) > MINOR_CHANGE) {
|
|
499
|
+
if (!current) {
|
|
500
|
+
const before = hist[i - 1] || h;
|
|
501
|
+
current = { startAt: before.at, fromSig: before.sig, endAt: h.at, frames: 1, peak: h.diff };
|
|
502
|
+
} else {
|
|
503
|
+
current.endAt = h.at;
|
|
504
|
+
current.frames += 1;
|
|
505
|
+
current.peak = Math.max(current.peak, h.diff);
|
|
506
|
+
}
|
|
507
|
+
current.toSig = h.sig;
|
|
508
|
+
} else if (current) {
|
|
509
|
+
current.endAt = h.at;
|
|
510
|
+
current.toSig = h.sig;
|
|
511
|
+
events.push(current);
|
|
512
|
+
current = null;
|
|
513
|
+
}
|
|
514
|
+
});
|
|
515
|
+
if (current) events.push(current);
|
|
516
|
+
|
|
517
|
+
const shaped = events.map((e) => {
|
|
518
|
+
const from = hexToSignature(e.fromSig || '');
|
|
519
|
+
const to = hexToSignature(e.toSig || '');
|
|
520
|
+
const magnitude = from.length && to.length ? signatureDiff(to, from) : e.peak;
|
|
521
|
+
const deltas = from.length && to.length ? regionDeltas(to, from) : [];
|
|
522
|
+
return {
|
|
523
|
+
startedMsAgo: now - e.startAt,
|
|
524
|
+
endedMsAgo: now - e.endAt,
|
|
525
|
+
durationMs: Math.max(0, e.endAt - e.startAt),
|
|
526
|
+
magnitude: Number(magnitude.toFixed(4)),
|
|
527
|
+
level: changeLevel(magnitude),
|
|
528
|
+
frames: e.frames,
|
|
529
|
+
map: deltas.length ? regionMap(deltas, REGION_COLS) : null,
|
|
530
|
+
};
|
|
531
|
+
});
|
|
532
|
+
|
|
533
|
+
return {
|
|
534
|
+
device,
|
|
535
|
+
state,
|
|
536
|
+
spanMs,
|
|
537
|
+
coveredMs: hist.length ? now - hist[0].at : 0,
|
|
538
|
+
frames: hist.length,
|
|
539
|
+
buffered: (state.ring || []).length,
|
|
540
|
+
events: shaped,
|
|
541
|
+
idleForMs: state.stableForMs,
|
|
542
|
+
live: liveness(device.udid, state),
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/** The buffered frame closest to a moment in the past. */
|
|
547
|
+
export async function getFrameAt(deviceQuery, { msAgo = 0, options } = {}) {
|
|
548
|
+
const { device, state } = await ensureDaemon(deviceQuery, options);
|
|
549
|
+
const ring = state.ring || [];
|
|
550
|
+
if (!ring.length) throw new Error('no frames buffered yet');
|
|
551
|
+
const target = Date.now() - msAgo;
|
|
552
|
+
let best = ring[0];
|
|
553
|
+
for (const frame of ring) {
|
|
554
|
+
if (Math.abs(frame.at - target) < Math.abs(best.at - target)) best = frame;
|
|
555
|
+
}
|
|
556
|
+
const file = path.join(store.paths(device.udid).ring, `${best.seq}.png`);
|
|
557
|
+
if (!fs.existsSync(file)) throw new Error(`frame #${best.seq} is no longer buffered`);
|
|
558
|
+
const png = fs.readFileSync(file);
|
|
559
|
+
return {
|
|
560
|
+
device,
|
|
561
|
+
state,
|
|
562
|
+
png,
|
|
563
|
+
seq: best.seq,
|
|
564
|
+
at: best.at,
|
|
565
|
+
actualMsAgo: Date.now() - best.at,
|
|
566
|
+
requestedMsAgo: msAgo,
|
|
567
|
+
width: png.readUInt32BE(16),
|
|
568
|
+
height: png.readUInt32BE(20),
|
|
569
|
+
oldestMsAgo: Date.now() - ring[0].at,
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Find where to tap for a label on the screen showing right now.
|
|
575
|
+
*
|
|
576
|
+
* Familiar screens answer from memory: no accessibility read, no OCR, no image.
|
|
577
|
+
* A screen seen for the first time pays once to build its map, and every later
|
|
578
|
+
* visit is a file read.
|
|
579
|
+
*/
|
|
580
|
+
export async function locate(deviceQuery, query, { index, refresh = false, useAx = true, useOcr = true, options } = {}) {
|
|
581
|
+
const { device, state } = await ensureDaemon(deviceQuery, options);
|
|
582
|
+
const udid = device.udid;
|
|
583
|
+
let entry = null;
|
|
584
|
+
let from = 'memory';
|
|
585
|
+
let distance = 0;
|
|
586
|
+
|
|
587
|
+
if (!refresh) {
|
|
588
|
+
const near = screenmap.recallNearest(udid, state.layoutHash);
|
|
589
|
+
if (near) {
|
|
590
|
+
entry = near.entry;
|
|
591
|
+
distance = near.distance;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
if (!entry) {
|
|
596
|
+
const geo = await deviceGeometry(udid, state);
|
|
597
|
+
entry = await screenmap.build(udid, {
|
|
598
|
+
hash: state.hash,
|
|
599
|
+
layoutHash: state.layoutHash,
|
|
600
|
+
fullFrame: await fullFrameFor(udid, state),
|
|
601
|
+
density: geo.density,
|
|
602
|
+
screen: { width: geo.pointWidth, height: geo.pointHeight },
|
|
603
|
+
useAx,
|
|
604
|
+
useOcr,
|
|
605
|
+
});
|
|
606
|
+
from = 'built';
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
const candidates = screenmap.rank(entry, query);
|
|
610
|
+
if (candidates.length > 1 && index == null) {
|
|
611
|
+
const top = candidates[0];
|
|
612
|
+
const second = candidates[1];
|
|
613
|
+
const decisive = screenmap.isInteractive(top) && !screenmap.isInteractive(second);
|
|
614
|
+
if (!decisive) {
|
|
615
|
+
const list = candidates
|
|
616
|
+
.slice(0, 6)
|
|
617
|
+
.map((t, i) => `[${i}] "${t.label}" (${t.x},${t.y}) ${t.type}/${t.source}`)
|
|
618
|
+
.join(', ');
|
|
619
|
+
throw new Error(
|
|
620
|
+
`"${query}" matches ${candidates.length} things on this screen — pass index to choose: ${list}`,
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
const target = index != null ? candidates[index] : candidates[0];
|
|
625
|
+
if (!target) {
|
|
626
|
+
const sample = entry.targets
|
|
627
|
+
.filter((t) => t.label)
|
|
628
|
+
.slice(0, 12)
|
|
629
|
+
.map((t) => t.label)
|
|
630
|
+
.join(', ');
|
|
631
|
+
throw new Error(
|
|
632
|
+
`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`,
|
|
633
|
+
);
|
|
634
|
+
}
|
|
635
|
+
return { device, state, entry, target, from, distance, screens: screenmap.stats(udid).screens };
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
export { DEFAULTS, screenmap, store };
|