screenhand 0.3.6 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mcp-desktop.js
CHANGED
|
@@ -494,7 +494,14 @@ const originalTool = ((...args) => {
|
|
|
494
494
|
const wrappedHandler = async (params, extra) => {
|
|
495
495
|
const sessionId = memory.getSessionId();
|
|
496
496
|
if (sessionId && worldModel.getState().sessionId !== sessionId) {
|
|
497
|
-
|
|
497
|
+
// If perception is running, rebind sessionId without clearing live state —
|
|
498
|
+
// a full init() wipes windows/controls that perception actively feeds.
|
|
499
|
+
if (perceptionManager.isRunning) {
|
|
500
|
+
worldModel.rebindSession(sessionId);
|
|
501
|
+
}
|
|
502
|
+
else {
|
|
503
|
+
worldModel.init(sessionId);
|
|
504
|
+
}
|
|
498
505
|
}
|
|
499
506
|
return handler(params, extra);
|
|
500
507
|
};
|
|
@@ -532,7 +539,14 @@ server.tool = (...args) => {
|
|
|
532
539
|
const start = Date.now();
|
|
533
540
|
// ── PRE-CALL: lazy-init world model on first session ──
|
|
534
541
|
if (sessionId && worldModel.getState().sessionId !== sessionId) {
|
|
535
|
-
|
|
542
|
+
// If perception is running, rebind sessionId without clearing live state —
|
|
543
|
+
// a full init() wipes windows/controls that perception actively feeds.
|
|
544
|
+
if (perceptionManager.isRunning) {
|
|
545
|
+
worldModel.rebindSession(sessionId);
|
|
546
|
+
}
|
|
547
|
+
else {
|
|
548
|
+
worldModel.init(sessionId);
|
|
549
|
+
}
|
|
536
550
|
}
|
|
537
551
|
// ── PRE-CALL: notify perception to stay active (idle gating) ──
|
|
538
552
|
perceptionManager.notifyToolCall();
|
|
@@ -592,6 +606,12 @@ server.tool = (...args) => {
|
|
|
592
606
|
const postCallBundleId = preBundleId ?? lastKnownBundleId;
|
|
593
607
|
// Capture pre-call window title for navigation edge tracking
|
|
594
608
|
const preWindowTitle = worldModel.getFocusedWindow()?.title.value ?? null;
|
|
609
|
+
// Capture pre-call state snapshot for auto-diff (when perception is running)
|
|
610
|
+
const preState = perceptionManager.isRunning ? worldModel.getState() : null;
|
|
611
|
+
const preWindowCount = preState?.windows.size ?? 0;
|
|
612
|
+
const preControlCount = preState ? [...preState.windows.values()].reduce((s, w) => s + w.controls.size, 0) : 0;
|
|
613
|
+
const preDialogCount = preState?.activeDialogs.length ?? 0;
|
|
614
|
+
const preFocusedTitle = preState ? (worldModel.getFocusedWindow()?.title.value ?? "") : "";
|
|
595
615
|
// Action tools = actually doing something. Navigation = just clicking around.
|
|
596
616
|
const ACTION_TOOLS = new Set([
|
|
597
617
|
"type_text", "key", "drag", "scroll", "menu_click", "applescript",
|
|
@@ -912,6 +932,20 @@ server.tool = (...args) => {
|
|
|
912
932
|
hitFeatures.push(feature.id);
|
|
913
933
|
}
|
|
914
934
|
}
|
|
935
|
+
// Auto-discover features for apps on the generic ladder.
|
|
936
|
+
// When a non-nav tool call doesn't match any existing feature, create a new
|
|
937
|
+
// feature entry from the interaction target so distinct actions register separately.
|
|
938
|
+
if (hitFeatures.length === 0 && !isNavTool && appMap.isGenericLadder(learnBundleId)) {
|
|
939
|
+
const target = typeof locatorTarget === "string" ? locatorTarget
|
|
940
|
+
: typeof safeParams.text === "string" ? safeParams.text.slice(0, 40)
|
|
941
|
+
: typeof safeParams.combo === "string" ? safeParams.combo
|
|
942
|
+
: null;
|
|
943
|
+
if (target) {
|
|
944
|
+
const discovered = appMap.discoverFeature(learnBundleId, toolName, target, 2);
|
|
945
|
+
if (discovered)
|
|
946
|
+
hitFeatures.push(discovered);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
915
949
|
// Cross-feature workflow detection: track distinct features hit by action tools.
|
|
916
950
|
// When 3+ distinct features are hit in a rolling window, record a cross-feature workflow.
|
|
917
951
|
if (!crossFeatureBuffer.has(learnBundleId)) {
|
|
@@ -1239,6 +1273,26 @@ server.tool = (...args) => {
|
|
|
1239
1273
|
if (perceptionManager.isRunning) {
|
|
1240
1274
|
hints.push(perceptionManager.getFreshnessSummary());
|
|
1241
1275
|
}
|
|
1276
|
+
// Auto world_state_diff: when perception is running and an action tool was used,
|
|
1277
|
+
// show what changed so the agent gets instant feedback without manual world_state_diff calls
|
|
1278
|
+
if (preState && ACTION_TOOLS.has(toolName)) {
|
|
1279
|
+
const postWindowCount = worldModel.getState().windows.size;
|
|
1280
|
+
const postControlCount = [...worldModel.getState().windows.values()].reduce((s, w) => s + w.controls.size, 0);
|
|
1281
|
+
const postDialogCount = worldModel.getState().activeDialogs.length;
|
|
1282
|
+
const postFocusedTitle = worldModel.getFocusedWindow()?.title.value ?? "";
|
|
1283
|
+
const diffs = [];
|
|
1284
|
+
if (postWindowCount !== preWindowCount)
|
|
1285
|
+
diffs.push(`windows: ${preWindowCount}→${postWindowCount}`);
|
|
1286
|
+
if (postControlCount !== preControlCount)
|
|
1287
|
+
diffs.push(`controls: ${preControlCount}→${postControlCount}`);
|
|
1288
|
+
if (postDialogCount !== preDialogCount)
|
|
1289
|
+
diffs.push(`dialogs: ${preDialogCount}→${postDialogCount}`);
|
|
1290
|
+
if (postFocusedTitle !== preFocusedTitle && postFocusedTitle)
|
|
1291
|
+
diffs.push(`title: "${preFocusedTitle}"→"${postFocusedTitle}"`);
|
|
1292
|
+
if (diffs.length > 0) {
|
|
1293
|
+
hints.push(`Δ ${diffs.join(", ")}`);
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1242
1296
|
// Learning engine recommendations
|
|
1243
1297
|
const patternRec = learningEngine.recommendPattern(learnBundleId, toolName);
|
|
1244
1298
|
if (patternRec) {
|
|
@@ -3031,13 +3085,14 @@ server.tool("export_playbook", "Generate a playbook JSON from your session. Extr
|
|
|
3031
3085
|
// ═══════════════════════════════════════════════
|
|
3032
3086
|
// PLAYBOOK RECORD — macro recorder for MCP tool calls
|
|
3033
3087
|
// ═══════════════════════════════════════════════
|
|
3034
|
-
server.tool("playbook_record", "Macro recorder: start
|
|
3035
|
-
action: z.enum(["start", "stop", "cancel", "status"]).describe("start/stop/cancel/status"),
|
|
3088
|
+
server.tool("playbook_record", "Macro recorder: start/stop/trim/clean recorded playbooks. Use 'trim' to remove specific steps, 'clean' to auto-remove failed steps and retries before export.", {
|
|
3089
|
+
action: z.enum(["start", "stop", "cancel", "status", "trim", "clean"]).describe("start/stop/cancel/status/trim/clean"),
|
|
3036
3090
|
platform: z.string().optional().describe("Platform name (required for start)"),
|
|
3037
3091
|
name: z.string().optional().describe("Playbook name (required for stop)"),
|
|
3038
3092
|
description: z.string().optional().describe("Playbook description (for stop)"),
|
|
3039
3093
|
cdpPort: z.number().min(9222).max(9999).optional().describe("CDP port if needed for browser_js steps (e.g. 9333 for Codex)"),
|
|
3040
|
-
|
|
3094
|
+
removeSteps: z.array(z.number()).optional().describe("Step indices to remove (0-based, for trim action)"),
|
|
3095
|
+
}, async ({ action, platform, name, description, cdpPort, removeSteps: removeIndices }) => {
|
|
3041
3096
|
switch (action) {
|
|
3042
3097
|
case "start": {
|
|
3043
3098
|
if (!platform)
|
|
@@ -3045,7 +3100,7 @@ server.tool("playbook_record", "Macro recorder: start recording, do the flow, st
|
|
|
3045
3100
|
if (mcpRecorder.isRecording)
|
|
3046
3101
|
return { content: [{ type: "text", text: "Already recording. Call stop or cancel first." }] };
|
|
3047
3102
|
mcpRecorder.start(platform, cdpPort ?? undefined);
|
|
3048
|
-
return { content: [{ type: "text", text: `Recording started for "${platform}". All subsequent tool calls will be captured.\nCall playbook_record(action="stop", name="...") when done.` }] };
|
|
3103
|
+
return { content: [{ type: "text", text: `Recording started for "${platform}". All subsequent tool calls will be captured.\nCall playbook_record(action="stop", name="...") when done.\n\nTip: Before stopping, use action="clean" to auto-remove failed steps and retries, or action="trim" to remove specific steps by index.` }] };
|
|
3049
3104
|
}
|
|
3050
3105
|
case "stop": {
|
|
3051
3106
|
if (!mcpRecorder.isRecording)
|
|
@@ -3072,8 +3127,29 @@ server.tool("playbook_record", "Macro recorder: start recording, do the flow, st
|
|
|
3072
3127
|
case "status": {
|
|
3073
3128
|
if (!mcpRecorder.isRecording)
|
|
3074
3129
|
return { content: [{ type: "text", text: "Not recording." }] };
|
|
3075
|
-
const steps = mcpRecorder.getSteps().map((s, i) =>
|
|
3076
|
-
|
|
3130
|
+
const steps = mcpRecorder.getSteps().map((s, i) => {
|
|
3131
|
+
const marker = s.optional ? " ⚠️FAILED" : "";
|
|
3132
|
+
return ` ${i}. [${s.action}]${marker} ${s.description ?? ""}`;
|
|
3133
|
+
}).join("\n");
|
|
3134
|
+
return { content: [{ type: "text", text: `Recording active: ${mcpRecorder.stepCount} steps captured\n${steps}\n\nUse action="clean" to auto-remove failed steps and retries, or action="trim" with removeSteps=[0,3,5] to remove specific steps.` }] };
|
|
3135
|
+
}
|
|
3136
|
+
case "trim": {
|
|
3137
|
+
if (!mcpRecorder.isRecording)
|
|
3138
|
+
return { content: [{ type: "text", text: "No active recording to trim." }] };
|
|
3139
|
+
if (!removeIndices || removeIndices.length === 0)
|
|
3140
|
+
return { content: [{ type: "text", text: "Error: removeSteps array required (e.g. removeSteps=[0, 3, 5])" }] };
|
|
3141
|
+
const removed = mcpRecorder.removeSteps(removeIndices);
|
|
3142
|
+
const steps = mcpRecorder.getSteps().map((s, i) => ` ${i}. [${s.action}] ${s.description ?? ""}`).join("\n");
|
|
3143
|
+
return { content: [{ type: "text", text: `Removed ${removed} step(s). ${mcpRecorder.stepCount} remaining:\n${steps}` }] };
|
|
3144
|
+
}
|
|
3145
|
+
case "clean": {
|
|
3146
|
+
if (!mcpRecorder.isRecording)
|
|
3147
|
+
return { content: [{ type: "text", text: "No active recording to clean." }] };
|
|
3148
|
+
const failedRemoved = mcpRecorder.removeFailedSteps();
|
|
3149
|
+
const retriesRemoved = mcpRecorder.removeRetries();
|
|
3150
|
+
const total = failedRemoved + retriesRemoved;
|
|
3151
|
+
const steps = mcpRecorder.getSteps().map((s, i) => ` ${i}. [${s.action}] ${s.description ?? ""}`).join("\n");
|
|
3152
|
+
return { content: [{ type: "text", text: `Cleaned: removed ${failedRemoved} failed step(s) + ${retriesRemoved} retry(s) = ${total} total. ${mcpRecorder.stepCount} steps remaining:\n${steps}` }] };
|
|
3077
3153
|
}
|
|
3078
3154
|
}
|
|
3079
3155
|
});
|
|
@@ -201,4 +201,54 @@ export class McpPlaybookRecorder {
|
|
|
201
201
|
this.recording = false;
|
|
202
202
|
this.steps = [];
|
|
203
203
|
}
|
|
204
|
+
/**
|
|
205
|
+
* Remove steps by index (0-based). Accepts individual indices or ranges.
|
|
206
|
+
* Returns the number of steps removed.
|
|
207
|
+
*/
|
|
208
|
+
removeSteps(indices) {
|
|
209
|
+
if (!this.recording || indices.length === 0)
|
|
210
|
+
return 0;
|
|
211
|
+
const toRemove = new Set(indices.filter((i) => i >= 0 && i < this.steps.length));
|
|
212
|
+
const before = this.steps.length;
|
|
213
|
+
this.steps = this.steps.filter((_, i) => !toRemove.has(i));
|
|
214
|
+
return before - this.steps.length;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Remove all failed/optional steps (recorded when tool calls failed).
|
|
218
|
+
* Returns the number of steps removed.
|
|
219
|
+
*/
|
|
220
|
+
removeFailedSteps() {
|
|
221
|
+
if (!this.recording)
|
|
222
|
+
return 0;
|
|
223
|
+
const before = this.steps.length;
|
|
224
|
+
this.steps = this.steps.filter((s) => !s.optional);
|
|
225
|
+
return before - this.steps.length;
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Remove consecutive duplicate steps that look like retries.
|
|
229
|
+
* A "retry" = same action + same target within 3 consecutive steps.
|
|
230
|
+
* Keeps the last occurrence (the one that presumably worked).
|
|
231
|
+
* Returns the number of steps removed.
|
|
232
|
+
*/
|
|
233
|
+
removeRetries() {
|
|
234
|
+
if (!this.recording || this.steps.length < 2)
|
|
235
|
+
return 0;
|
|
236
|
+
const before = this.steps.length;
|
|
237
|
+
const cleaned = [];
|
|
238
|
+
for (let i = 0; i < this.steps.length; i++) {
|
|
239
|
+
const step = this.steps[i];
|
|
240
|
+
const next = this.steps[i + 1];
|
|
241
|
+
// If the next step is the same action+target, skip this one (keep the later one)
|
|
242
|
+
if (next &&
|
|
243
|
+
next.action === step.action &&
|
|
244
|
+
next.target === step.target &&
|
|
245
|
+
JSON.stringify(next.keys ?? []) === JSON.stringify(step.keys ?? []) &&
|
|
246
|
+
JSON.stringify(next.menuPath ?? []) === JSON.stringify(step.menuPath ?? [])) {
|
|
247
|
+
continue; // skip this duplicate, keep the next one
|
|
248
|
+
}
|
|
249
|
+
cleaned.push(step);
|
|
250
|
+
}
|
|
251
|
+
this.steps = cleaned;
|
|
252
|
+
return before - this.steps.length;
|
|
253
|
+
}
|
|
204
254
|
}
|
|
@@ -242,6 +242,70 @@ export class AppMap {
|
|
|
242
242
|
return true;
|
|
243
243
|
return this.loadGeneratedLadder(bundleId) !== null;
|
|
244
244
|
}
|
|
245
|
+
/** Check if this app is using the 5-item generic fallback ladder (no builtin, no generated). */
|
|
246
|
+
isGenericLadder(bundleId) {
|
|
247
|
+
if (BUILTIN_LADDERS[bundleId])
|
|
248
|
+
return false;
|
|
249
|
+
if (this.generatedLadderCache.has(bundleId))
|
|
250
|
+
return false;
|
|
251
|
+
if (this.loadGeneratedLadder(bundleId) !== null)
|
|
252
|
+
return false;
|
|
253
|
+
return true;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Auto-discover a feature from a tool interaction and add it to the ladder.
|
|
257
|
+
* Called when a tool call doesn't match any existing ladder feature AND we're
|
|
258
|
+
* on the generic ladder. Derives a feature ID from the interaction context
|
|
259
|
+
* (menu path, target element, tool name) so distinct interactions register
|
|
260
|
+
* as distinct features instead of all collapsing into "core_action".
|
|
261
|
+
*
|
|
262
|
+
* Returns the new feature ID if created, null if skipped.
|
|
263
|
+
*/
|
|
264
|
+
discoverFeature(bundleId, toolName, target, depth) {
|
|
265
|
+
if (!target || target.length < 2)
|
|
266
|
+
return null;
|
|
267
|
+
// Derive a feature ID from the target — normalize to snake_case
|
|
268
|
+
// "Format > Checklist" → "format_checklist"
|
|
269
|
+
// "New Folder" → "new_folder"
|
|
270
|
+
const featureId = target
|
|
271
|
+
.toLowerCase()
|
|
272
|
+
.replace(/[>→/\\|]/g, " ") // split menu paths
|
|
273
|
+
.replace(/[^a-z0-9\s]/g, "") // strip special chars
|
|
274
|
+
.trim()
|
|
275
|
+
.replace(/\s+/g, "_") // spaces to underscores
|
|
276
|
+
.slice(0, 40); // cap length
|
|
277
|
+
if (!featureId || featureId.length < 2)
|
|
278
|
+
return null;
|
|
279
|
+
// Don't create duplicates
|
|
280
|
+
const data = this.ensureLoaded(bundleId);
|
|
281
|
+
if (!data)
|
|
282
|
+
return null;
|
|
283
|
+
if (data.featureLadder.some((f) => f.id === featureId))
|
|
284
|
+
return null;
|
|
285
|
+
if (data.featureMastery[featureId])
|
|
286
|
+
return null;
|
|
287
|
+
// Cap auto-discovered features at 30 per app to prevent unbounded growth
|
|
288
|
+
const discoveredCount = data.featureLadder.filter((f) => f.description.startsWith("[auto]")).length;
|
|
289
|
+
if (discoveredCount >= 30)
|
|
290
|
+
return null;
|
|
291
|
+
// Assign level based on tool complexity
|
|
292
|
+
let level = "beginner";
|
|
293
|
+
if (toolName === "menu_click" || toolName === "key")
|
|
294
|
+
level = "pro";
|
|
295
|
+
if (toolName === "applescript" || toolName === "browser_js")
|
|
296
|
+
level = "expert";
|
|
297
|
+
const feature = {
|
|
298
|
+
id: featureId,
|
|
299
|
+
description: `[auto] ${target}`,
|
|
300
|
+
level,
|
|
301
|
+
weight: 1,
|
|
302
|
+
critical: false,
|
|
303
|
+
};
|
|
304
|
+
data.featureLadder.push(feature);
|
|
305
|
+
this.recordFeatureSignal(bundleId, featureId, depth, true);
|
|
306
|
+
this.save(data);
|
|
307
|
+
return featureId;
|
|
308
|
+
}
|
|
245
309
|
/**
|
|
246
310
|
* Set a custom feature ladder for an app. Useful for apps without built-in ladders.
|
|
247
311
|
*/
|
|
@@ -261,6 +261,14 @@ export class WorldModel {
|
|
|
261
261
|
this.entityTracker.rehydrate(this.state.trackedEntities);
|
|
262
262
|
}
|
|
263
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* Rebind session ID without clearing live state. Use when perception is actively
|
|
266
|
+
* feeding data — a full init() would wipe windows/controls that perception wrote.
|
|
267
|
+
* Only updates the sessionId tag so persistence targets the new session.
|
|
268
|
+
*/
|
|
269
|
+
rebindSession(sessionId) {
|
|
270
|
+
this.state.sessionId = sessionId;
|
|
271
|
+
}
|
|
264
272
|
/**
|
|
265
273
|
* Merge an incoming control with an existing one using source confidence.
|
|
266
274
|
* Higher-confidence sources win unless the existing data is very recent (<5s).
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
"appName": "Notion",
|
|
4
4
|
"version": "unknown",
|
|
5
5
|
"masteryLevel": "grandmaster",
|
|
6
|
-
"confidence": 0.
|
|
7
|
-
"lastValidated": "2026-03-
|
|
6
|
+
"confidence": 0.9996007984031936,
|
|
7
|
+
"lastValidated": "2026-03-22T08:48:48.328Z",
|
|
8
8
|
"mapVersion": 1,
|
|
9
9
|
"uiArchitecture": {
|
|
10
10
|
"type": "other",
|
|
@@ -1279,6 +1279,17 @@
|
|
|
1279
1279
|
"confidence": 0.9995235259082788,
|
|
1280
1280
|
"zonesKnown": 1,
|
|
1281
1281
|
"edgesVerified": 0
|
|
1282
|
+
},
|
|
1283
|
+
{
|
|
1284
|
+
"date": "2026-03-22",
|
|
1285
|
+
"level": "grandmaster",
|
|
1286
|
+
"rating": {
|
|
1287
|
+
"grade": "A",
|
|
1288
|
+
"subTier": 1
|
|
1289
|
+
},
|
|
1290
|
+
"confidence": 0.9996007984031936,
|
|
1291
|
+
"zonesKnown": 6,
|
|
1292
|
+
"edgesVerified": 2
|
|
1282
1293
|
}
|
|
1283
1294
|
],
|
|
1284
1295
|
"totalTasksCompleted": 0,
|
|
@@ -1400,11 +1411,11 @@
|
|
|
1400
1411
|
"new_page_options": {
|
|
1401
1412
|
"depth": 4,
|
|
1402
1413
|
"confidence": 1,
|
|
1403
|
-
"repeatCount":
|
|
1404
|
-
"workflowCount":
|
|
1414
|
+
"repeatCount": 280,
|
|
1415
|
+
"workflowCount": 262,
|
|
1405
1416
|
"healingCount": 0,
|
|
1406
1417
|
"failCount": 0,
|
|
1407
|
-
"lastSeen": "2026-03-
|
|
1418
|
+
"lastSeen": "2026-03-22T08:48:40.288Z",
|
|
1408
1419
|
"lastVerified": "2026-03-21T09:29:08.915Z"
|
|
1409
1420
|
},
|
|
1410
1421
|
"database": {
|
|
@@ -1532,7 +1543,7 @@
|
|
|
1532
1543
|
"breadth": 1,
|
|
1533
1544
|
"workflowBreadth": 1,
|
|
1534
1545
|
"outcomeBreadth": 0.7241379310344828,
|
|
1535
|
-
"reliability": 0.
|
|
1546
|
+
"reliability": 0.9960079840319361,
|
|
1536
1547
|
"healingRate": 1,
|
|
1537
1548
|
"crossFeatureWorkflows": 60,
|
|
1538
1549
|
"criticalFloor": 3,
|
|
@@ -1,27 +1,175 @@
|
|
|
1
1
|
{
|
|
2
|
-
"id": "notes",
|
|
3
|
-
"name": "notes playbook",
|
|
4
|
-
"description": "",
|
|
5
2
|
"platform": "notes",
|
|
6
|
-
"urlPatterns": [],
|
|
7
|
-
"steps": [],
|
|
8
|
-
"tags": [
|
|
9
|
-
"notes"
|
|
10
|
-
],
|
|
11
|
-
"version": "1.0.0",
|
|
12
|
-
"successCount": 0,
|
|
13
|
-
"failCount": 0,
|
|
14
3
|
"bundleId": "com.apple.Notes",
|
|
4
|
+
"version": "2.0.0",
|
|
5
|
+
"updated": "2026-03-22",
|
|
6
|
+
"description": "Apple Notes mastery: AppleScript bulk ops, HTML styling, inline images, website-quality notes",
|
|
7
|
+
"urls": {},
|
|
15
8
|
"selectors": {
|
|
16
|
-
"
|
|
9
|
+
"noteBody": "AXTextArea[identifier='Note Body Text View']",
|
|
10
|
+
"noteList": "AXTable — sidebar note list",
|
|
11
|
+
"folderList": "AXOutline — sidebar folder tree"
|
|
12
|
+
},
|
|
13
|
+
"flows": {
|
|
14
|
+
"create_html_note": {
|
|
15
|
+
"description": "Create a rich HTML-formatted note with tables, styled sections, colors, and emoji icons",
|
|
16
|
+
"steps": [
|
|
17
|
+
"applescript: make new note at folder 'Notes' of default account with properties {body:htmlString}",
|
|
18
|
+
"HTML supports: h1-h3, p, b, i, u, hr, ul/li, ol/li, table, a href, br",
|
|
19
|
+
"Inline CSS works: style='color:#00e5ff; background-color:#0a1a1c; border-radius:12px; font-size:28px;'",
|
|
20
|
+
"Tables create card layouts: cellpadding for spacing, cellspacing for gaps, vertical-align:top"
|
|
21
|
+
],
|
|
22
|
+
"example": "set h to \"<h1>Title</h1><table border='0' cellpadding='15' cellspacing='8'><tr><td style='background-color:#0a1a1c; border:1px solid #00e5ff30; border-radius:12px;'><b>Card</b><br>Description</td></tr></table>\""
|
|
23
|
+
},
|
|
24
|
+
"inline_image_paste": {
|
|
25
|
+
"description": "Place images at exact positions inside a note using clipboard paste (not make new attachment which only appends to end)",
|
|
26
|
+
"steps": [
|
|
27
|
+
"Step 1: Create the full note content via AppleScript with HTML body",
|
|
28
|
+
"Step 2: Copy image to clipboard via Bash: osascript -e 'set the clipboard to (read (POSIX file \"/path/to/image.png\") as «class PNGf»)'",
|
|
29
|
+
"Step 3: Focus Notes app: focus({bundleId:'com.apple.Notes'})",
|
|
30
|
+
"Step 4: Use Find to position cursor: cmd+f → type target text → escape",
|
|
31
|
+
"Step 5: Navigate: cmd+left (start of line) → up (line above)",
|
|
32
|
+
"Step 6: Paste: cmd+v (image appears inline at cursor position)",
|
|
33
|
+
"Repeat steps 2-6 for each image at different positions"
|
|
34
|
+
],
|
|
35
|
+
"gotchas": [
|
|
36
|
+
"make new attachment at end of attachments ONLY appends to bottom — never inline",
|
|
37
|
+
"cmd+right in Notes goes to end of ENTIRE note, not end of line — use arrow keys instead",
|
|
38
|
+
"Copy image via Bash osascript, not via ScreenHand applescript tool (read command is blocked)",
|
|
39
|
+
"Always focus Notes before pasting — switching apps moves cursor"
|
|
40
|
+
]
|
|
41
|
+
},
|
|
42
|
+
"bulk_read_notes": {
|
|
43
|
+
"description": "Read all notes via AppleScript (100x faster than UI automation)",
|
|
44
|
+
"snippet": "tell application \"Notes\"\n set output to \"\"\n repeat with n in notes of default account\n set output to output & \"---\" & return & name of n & return & plaintext of n & return\n end repeat\n return output\nend tell"
|
|
45
|
+
},
|
|
46
|
+
"bulk_delete_by_name": {
|
|
47
|
+
"description": "Delete specific notes by name (safer than pattern matching)",
|
|
48
|
+
"snippet": "tell application \"Notes\"\n set noteNames to {\"Note One\", \"Note Two\", \"Note Three\"}\n repeat with noteName in noteNames\n try\n delete note noteName of default account\n end try\n end repeat\nend tell",
|
|
49
|
+
"gotchas": [
|
|
50
|
+
"Deleting inside a loop shifts the list — wrap in 'repeat 5 times' outer loop",
|
|
51
|
+
"'items' is a reserved word — use taskList, noteList etc",
|
|
52
|
+
"container of note errors out — skip folder name, query folder-specific instead"
|
|
53
|
+
]
|
|
54
|
+
},
|
|
55
|
+
"search_notes": {
|
|
56
|
+
"description": "Search note content via AppleScript",
|
|
57
|
+
"snippet": "tell application \"Notes\"\n set results to {}\n repeat with n in notes of default account\n if plaintext of n contains \"keyword\" then\n set end of results to name of n\n end if\n end repeat\n return results\nend tell"
|
|
58
|
+
},
|
|
59
|
+
"move_note": {
|
|
60
|
+
"description": "Move a note between folders",
|
|
61
|
+
"snippet": "tell application \"Notes\"\n set sourceNote to note \"My Note\" of default account\n set targetFolder to folder \"Target Folder\" of default account\n move sourceNote to targetFolder\nend tell"
|
|
62
|
+
},
|
|
63
|
+
"export_all_notes": {
|
|
64
|
+
"description": "Export all notes to files (two-step: AppleScript dumps, Bash writes files)",
|
|
65
|
+
"snippet_step1": "tell application \"Notes\"\n set output to \"\"\n repeat with n in notes of default account\n set output to output & \"<<<NOTE_START>>>\" & name of n & \"<<<NOTE_SEP>>>\" & plaintext of n & \"<<<NOTE_END>>>\"\n end repeat\n return output\nend tell",
|
|
66
|
+
"snippet_step2": "# Bash/Python: parse output file, split by delimiters, write each note to ~/Desktop/notes-export/",
|
|
67
|
+
"gotchas": [
|
|
68
|
+
"do shell script is blocked in ScreenHand applescript tool — must use two-step approach"
|
|
69
|
+
]
|
|
70
|
+
},
|
|
71
|
+
"count_per_folder": {
|
|
72
|
+
"description": "Count notes in each folder",
|
|
73
|
+
"snippet": "tell application \"Notes\"\n set output to \"\"\n repeat with f in folders of default account\n set output to output & name of f & \": \" & (count of notes of f) & \" notes\" & return\n end repeat\n return output\nend tell"
|
|
74
|
+
},
|
|
75
|
+
"find_duplicates": {
|
|
76
|
+
"description": "Find duplicate note titles",
|
|
77
|
+
"snippet": "tell application \"Notes\"\n set titles to {}\n set dupes to {}\n repeat with n in notes of default account\n set t to name of n\n if titles contains t then\n if dupes does not contain t then set end of dupes to t\n else\n set end of titles to t\n end if\n end repeat\n return dupes\nend tell"
|
|
78
|
+
},
|
|
79
|
+
"merge_notes": {
|
|
80
|
+
"description": "Merge multiple notes into one with HTML",
|
|
81
|
+
"snippet": "tell application \"Notes\"\n set note1 to note \"First Note\" of default account\n set note2 to note \"Second Note\" of default account\n set combined to \"<h1>Merged Note</h1>\" & body of note1 & \"<hr>\" & body of note2\n make new note at folder \"Notes\" of default account with properties {body:combined}\nend tell"
|
|
82
|
+
},
|
|
83
|
+
"bulk_tag": {
|
|
84
|
+
"description": "Add tag to note body",
|
|
85
|
+
"snippet": "tell application \"Notes\"\n set n to note \"My Note\" of default account\n set body of n to (body of n) & \"<br>#newtag\"\nend tell"
|
|
86
|
+
},
|
|
87
|
+
"list_titles_only": {
|
|
88
|
+
"description": "List just note titles (fast overview)",
|
|
89
|
+
"snippet": "tell application \"Notes\"\n set output to {}\n repeat with n in notes of default account\n set end of output to name of n\n end repeat\n return output\nend tell"
|
|
90
|
+
},
|
|
91
|
+
"delete_by_pattern": {
|
|
92
|
+
"description": "Delete notes matching a pattern (wraps 5x for list shifting)",
|
|
93
|
+
"snippet": "tell application \"Notes\"\n repeat 5 times\n repeat with n in notes of default account\n if name of n starts with \"Test\" then delete n\n end repeat\n end repeat\nend tell"
|
|
94
|
+
},
|
|
95
|
+
"move_from_folder": {
|
|
96
|
+
"description": "Move a note from a specific folder back to Notes",
|
|
97
|
+
"snippet": "tell application \"Notes\"\n set sourceNote to note \"My Note\" of folder \"Source Folder\" of default account\n set targetFolder to folder \"Notes\" of default account\n move sourceNote to targetFolder\nend tell"
|
|
98
|
+
},
|
|
99
|
+
"list_with_dates": {
|
|
100
|
+
"description": "List notes with creation and modification dates",
|
|
101
|
+
"snippet": "tell application \"Notes\"\n set output to \"\"\n repeat with n in notes of default account\n set output to output & name of n & \" | created: \" & (creation date of n as string) & \" | modified: \" & (modification date of n as string) & return\n end repeat\n return output\nend tell"
|
|
102
|
+
},
|
|
103
|
+
"bulk_rename": {
|
|
104
|
+
"description": "Rename a note by prepending a new h1 title to body",
|
|
105
|
+
"snippet": "tell application \"Notes\"\n set n to note \"Old Title\" of default account\n set body of n to \"<h1>New Title</h1>\" & body of n\nend tell"
|
|
106
|
+
},
|
|
107
|
+
"get_metadata": {
|
|
108
|
+
"description": "Get note metadata: name, ID, dates, password status",
|
|
109
|
+
"snippet": "tell application \"Notes\"\n set n to note \"My Note\" of default account\n return \"Name: \" & name of n & return & \"ID: \" & id of n & return & \"Created: \" & (creation date of n as string) & return & \"Modified: \" & (modification date of n as string) & return & \"Password protected: \" & (password protected of n as string)\nend tell"
|
|
110
|
+
},
|
|
111
|
+
"bulk_create_from_list": {
|
|
112
|
+
"description": "Create multiple notes from a list of titles",
|
|
113
|
+
"snippet": "tell application \"Notes\"\n set taskList to {\"Task One\", \"Task Two\", \"Task Three\"}\n repeat with taskItem in taskList\n make new note at folder \"Notes\" of default account with properties {body:\"<h1>\" & taskItem & \"</h1><p>Bulk created</p>\"}\n end repeat\nend tell"
|
|
114
|
+
},
|
|
115
|
+
"create_delete_folders": {
|
|
116
|
+
"description": "Create and delete folders",
|
|
117
|
+
"steps": [
|
|
118
|
+
"applescript: make new folder with properties {name:'Folder Name'}",
|
|
119
|
+
"applescript: delete folder 'Folder Name' of default account"
|
|
120
|
+
]
|
|
121
|
+
},
|
|
122
|
+
"website_style_note": {
|
|
123
|
+
"description": "Create a website-quality note mirroring a Next.js site structure with Hero, Problem, Solution, Features, Comparison, Demo, CTA sections",
|
|
124
|
+
"steps": [
|
|
125
|
+
"Step 1: Build full HTML body with styled sections using tables for card layouts",
|
|
126
|
+
"Step 2: Use inline CSS for colors (color:#00e5ff), backgrounds (background-color:#0a1a1c), borders, border-radius",
|
|
127
|
+
"Step 3: Use <table border='0' cellpadding='15' cellspacing='8'> for card grids",
|
|
128
|
+
"Step 4: Use <hr> between sections, styled <p> tags for section labels",
|
|
129
|
+
"Step 5: Add comparison tables with emoji checkmarks (✅/❌)",
|
|
130
|
+
"Step 6: Add CTA with styled button-like table cells (background-color:#00e5ff; border-radius:25px)",
|
|
131
|
+
"Step 7: Paste images inline at exact positions using Find+cursor+clipboard technique"
|
|
132
|
+
],
|
|
133
|
+
"html_tags_supported": ["h1", "h2", "h3", "p", "b", "i", "u", "hr", "br", "table", "tr", "td", "ul", "ol", "li", "a", "span", "img"],
|
|
134
|
+
"css_properties_supported": ["color", "background-color", "font-size", "border", "border-radius", "text-align", "vertical-align", "padding", "letter-spacing", "width"]
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
"detection": {
|
|
138
|
+
"is_logged_in": "Always logged in — native macOS app"
|
|
17
139
|
},
|
|
18
|
-
"flows": {},
|
|
19
140
|
"errors": [
|
|
20
141
|
{
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
|
|
142
|
+
"pattern": "string concatenation containing 'script' or 'shell'",
|
|
143
|
+
"solution": "Rephrase body text to avoid the word 'script' — use 'macOS native' instead of 'AppleScript'",
|
|
144
|
+
"tool": "applescript"
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
"pattern": "read command blocked",
|
|
148
|
+
"solution": "Use Bash osascript to copy images to clipboard, not ScreenHand applescript tool",
|
|
149
|
+
"tool": "applescript"
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
"pattern": "container of note errors -1728",
|
|
153
|
+
"solution": "Skip folder name in metadata queries — query folder-specific notes instead",
|
|
154
|
+
"tool": "applescript"
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"pattern": "items reserved word",
|
|
158
|
+
"solution": "Use taskList, noteList etc instead of 'items' as variable names",
|
|
159
|
+
"tool": "applescript"
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
"pattern": "delete inside loop skips items",
|
|
163
|
+
"solution": "Wrap in 'repeat 5 times' outer loop, or delete by specific name with try blocks",
|
|
164
|
+
"tool": "applescript"
|
|
25
165
|
}
|
|
26
|
-
]
|
|
27
|
-
|
|
166
|
+
],
|
|
167
|
+
"_meta": {
|
|
168
|
+
"exported_from": "screenhand",
|
|
169
|
+
"actions_count": 446,
|
|
170
|
+
"strategies_count": 21,
|
|
171
|
+
"proven_techniques": 21,
|
|
172
|
+
"flows_count": 19,
|
|
173
|
+
"grade": "D3"
|
|
174
|
+
}
|
|
175
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "screenhand",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.7",
|
|
4
4
|
"mcpName": "io.github.manushi4/screenhand",
|
|
5
5
|
"description": "Give AI eyes and hands on your desktop. ScreenHand is an open-source MCP server that lets Claude and other AI agents see your screen, click buttons, type text, and control any app on macOS and Windows.",
|
|
6
6
|
"homepage": "https://screenhand.com",
|