simframe 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +181 -2
- package/data/vocabulary/en.json +148 -0
- package/native/ocr.swift +13 -1
- package/native/rank.swift +87 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +72 -4
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +27 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +4 -0
- package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +19 -0
- package/native/simframed/Sources/simframed/main.swift +33 -3
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +41 -0
- package/native/supervise.swift +216 -0
- package/package.json +4 -1
- package/scripts/check-package.mjs +22 -2
- package/scripts/check-private.mjs +9 -0
- package/scripts/ci-integration-local.sh +79 -0
- package/scripts/ci-memory.mjs +104 -20
- package/scripts/collect-rulings.mjs +312 -0
- package/scripts/eval-fingerprint.mjs +100 -23
- package/scripts/eval-perception.mjs +33 -0
- package/scripts/phase17-corpus.mjs +176 -0
- package/scripts/probe-network.mjs +118 -0
- package/scripts/soak-capture.mjs +72 -0
- package/skills/simframe/SKILL.md +257 -7
- package/src/actions.js +1791 -44
- package/src/cli.js +216 -14
- package/src/control.js +1 -0
- package/src/fingerprint.js +43 -1
- package/src/graph.js +136 -7
- package/src/index.js +252 -14
- package/src/input.js +66 -3
- package/src/localhelper.js +161 -0
- package/src/matching.js +64 -1
- package/src/mcp.js +333 -32
- package/src/metrics.js +148 -3
- package/src/ocr.js +18 -1
- package/src/planner.js +195 -0
- package/src/platform/android.js +25 -1
- package/src/platform/index.js +11 -1
- package/src/platform/ios.js +25 -1
- package/src/png.js +26 -0
- package/src/refs.js +51 -8
- package/src/regions.js +215 -1
- package/src/screenmap.js +89 -9
- package/src/supervisor.js +161 -0
- package/src/view.js +375 -11
- package/src/vocabulary.js +134 -0
- package/src/wrote.js +136 -0
package/src/wrote.js
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What simframe itself typed into this device, and whether it is still there.
|
|
3
|
+
*
|
|
4
|
+
* A peer filled three fields on a web form, and six calls later they were
|
|
5
|
+
* empty. Every one of those six calls returned `ok`. simframe had *recorded*
|
|
6
|
+
* all three values two calls earlier and said nothing when they vanished — so
|
|
7
|
+
* the agent only found out by paying for a screenshot, and an agent that had
|
|
8
|
+
* trusted the verdicts would have submitted an empty order.
|
|
9
|
+
*
|
|
10
|
+
* Two design decisions, both from how that failure actually presented.
|
|
11
|
+
*
|
|
12
|
+
* **Not keyed on screen identity.** The obvious home for this is the screen
|
|
13
|
+
* map, and it would have missed the whole thing: the wipe was observed under a
|
|
14
|
+
* different screen hash than the one the values were written under, because a
|
|
15
|
+
* pinch-zoom and a horizontal scroll had both moved the identity. What stayed
|
|
16
|
+
* constant was the field's own label. So this is a small per-device journal,
|
|
17
|
+
* and a screen is only relevant in that its labels are what we match against.
|
|
18
|
+
*
|
|
19
|
+
* **Only confirmed writes are journalled.** An unconfirmed write is not
|
|
20
|
+
* evidence a value was ever there, and journalling one would manufacture a
|
|
21
|
+
* "your text disappeared" warning for text that never arrived — trading a
|
|
22
|
+
* silent failure for a confident wrong answer, which is the worse of the two.
|
|
23
|
+
*/
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
import * as store from './store.js';
|
|
26
|
+
|
|
27
|
+
const FILE = 'wrote.json';
|
|
28
|
+
|
|
29
|
+
/** Keep the journal small: this is a recency signal, not a history. */
|
|
30
|
+
export const KEEP = 24;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* How long a journalled value stays interesting.
|
|
34
|
+
*
|
|
35
|
+
* Long enough to span the kind of sequence that lost the peer's three fields
|
|
36
|
+
* (six calls, a little over two minutes), short enough that yesterday's form
|
|
37
|
+
* never comments on today's.
|
|
38
|
+
*/
|
|
39
|
+
export const MAX_AGE_MS = 15 * 60 * 1000;
|
|
40
|
+
|
|
41
|
+
const file = (udid) => path.join(store.deviceDir(udid), FILE);
|
|
42
|
+
|
|
43
|
+
/** Comparison that ignores what OCR adds — the same rule the readback uses. */
|
|
44
|
+
const alnum = (v) => String(v ?? '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
|
|
45
|
+
|
|
46
|
+
/** Does this read as a label at all, rather than as a coordinate or a ref? */
|
|
47
|
+
const hasLetters = (v) => /\p{L}/u.test(String(v ?? ''));
|
|
48
|
+
|
|
49
|
+
export function read(udid) {
|
|
50
|
+
const got = store.readJson(file(udid));
|
|
51
|
+
return Array.isArray(got) ? got : [];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Record a value that was *confirmed* to have landed.
|
|
56
|
+
*
|
|
57
|
+
* Re-writing the same field replaces its entry rather than adding one, so a
|
|
58
|
+
* field filled twice cannot warn about its own earlier contents.
|
|
59
|
+
*/
|
|
60
|
+
export function record(udid, { selector, value, screen }) {
|
|
61
|
+
if (!udid || !selector || !alnum(value)) return null;
|
|
62
|
+
// A selector with no letters in it is not a label: "(125,325)", "@120,400",
|
|
63
|
+
// "#3". There is nothing to look for on a later screen, and its digits would
|
|
64
|
+
// happily match some unrelated number — so it is never journalled at all,
|
|
65
|
+
// rather than journalled and then skipped.
|
|
66
|
+
if (!hasLetters(selector)) return null;
|
|
67
|
+
const now = Date.now();
|
|
68
|
+
const key = alnum(selector);
|
|
69
|
+
const kept = read(udid).filter((e) => alnum(e.selector) !== key);
|
|
70
|
+
const next = [{ selector: String(selector), value: String(value), screen: screen ?? null, at: now }, ...kept]
|
|
71
|
+
.slice(0, KEEP);
|
|
72
|
+
try {
|
|
73
|
+
store.ensureDirs(udid);
|
|
74
|
+
store.writeAtomic(file(udid), JSON.stringify(next));
|
|
75
|
+
} catch {
|
|
76
|
+
// A journal that cannot be written must never fail the step that wrote the
|
|
77
|
+
// field. Losing the warning is a smaller cost than losing the write.
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
return next[0];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Which journalled values look as though they have been cleared.
|
|
85
|
+
*
|
|
86
|
+
* Deliberately narrow, because a false alarm here would teach an agent to
|
|
87
|
+
* ignore the line. An entry only counts as missing when its own **label is on
|
|
88
|
+
* this screen** — so we are plausibly looking at the same form — and its value
|
|
89
|
+
* appears nowhere in the map. Anything else (the label absent, the value
|
|
90
|
+
* present, the screen empty) says nothing and is reported as nothing.
|
|
91
|
+
*/
|
|
92
|
+
export function missing(udid, targets, { now = Date.now() } = {}) {
|
|
93
|
+
const rows = targets ?? [];
|
|
94
|
+
if (!rows.length) return [];
|
|
95
|
+
const haystack = rows.map((t) => alnum(`${t.label ?? ''} ${t.value ?? ''}`)).filter(Boolean);
|
|
96
|
+
if (!haystack.length) return [];
|
|
97
|
+
const gone = [];
|
|
98
|
+
for (const e of read(udid)) {
|
|
99
|
+
if (!Number.isFinite(e.at) || now - e.at > MAX_AGE_MS) continue;
|
|
100
|
+
const label = alnum(e.selector);
|
|
101
|
+
const value = alnum(e.value);
|
|
102
|
+
if (!label || !value) continue;
|
|
103
|
+
// `startsWith`, not `includes`. A field's row begins with its own label —
|
|
104
|
+
// and OCR fuses the value onto the end of it ("Telephone: 5551234567"),
|
|
105
|
+
// which is why this cannot be an equality test. A substring test looked
|
|
106
|
+
// equivalent and was not: on the very first field run after this shipped, a
|
|
107
|
+
// journalled "Email" matched the page footer's newsletter box, "Enter your
|
|
108
|
+
// email address", and announced a value gone that was merely on a different
|
|
109
|
+
// part of the page. A false alarm here teaches an agent to ignore the line,
|
|
110
|
+
// which costs more than the line is worth.
|
|
111
|
+
if (!haystack.some((h) => h.startsWith(label))) continue;
|
|
112
|
+
if (haystack.some((h) => h.includes(value))) continue;
|
|
113
|
+
gone.push(e);
|
|
114
|
+
}
|
|
115
|
+
return gone;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The line the map prints, or null when there is nothing to say. */
|
|
119
|
+
export function missingLine(gone) {
|
|
120
|
+
const list = (gone ?? []).slice(0, 3);
|
|
121
|
+
if (!list.length) return null;
|
|
122
|
+
const parts = list.map((e) => {
|
|
123
|
+
const v = String(e.value).length > 24 ? `${String(e.value).slice(0, 24)}…` : String(e.value);
|
|
124
|
+
return `${JSON.stringify(v)} in ${JSON.stringify(String(e.selector))}`;
|
|
125
|
+
});
|
|
126
|
+
const more = (gone?.length ?? 0) - list.length;
|
|
127
|
+
return `a value simframe wrote here is gone: ${parts.join(', ')}${more > 0 ? `, and ${more} more` : ''}`
|
|
128
|
+
+ ' — the field is on this screen and its contents are not, so something cleared it. Re-fill before continuing.';
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Forget the journal for a device — used when its memory is reset. */
|
|
132
|
+
export function forget(udid) {
|
|
133
|
+
try {
|
|
134
|
+
store.writeAtomic(file(udid), JSON.stringify([]));
|
|
135
|
+
} catch { /* nothing to forget */ }
|
|
136
|
+
}
|