screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -36,11 +36,19 @@ export function writeFileAtomicSync(filePath, data) {
|
|
|
36
36
|
try {
|
|
37
37
|
fs.writeFileSync(tmp, data, { mode: 0o644 });
|
|
38
38
|
// Back up current file before overwriting (ignore if it doesn't exist yet)
|
|
39
|
+
// V3: Check for symlinks before backup to prevent data exfiltration
|
|
39
40
|
try {
|
|
40
|
-
fs.
|
|
41
|
+
const stat = fs.lstatSync(filePath);
|
|
42
|
+
if (stat.isSymbolicLink()) {
|
|
43
|
+
// Target is a symlink — skip backup and remove the symlink before writing
|
|
44
|
+
fs.unlinkSync(filePath);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
fs.copyFileSync(filePath, filePath + ".bak");
|
|
48
|
+
}
|
|
41
49
|
}
|
|
42
50
|
catch {
|
|
43
|
-
// No existing file to back up — fine
|
|
51
|
+
// No existing file to back up (ENOENT) — fine
|
|
44
52
|
}
|
|
45
53
|
fs.renameSync(tmp, filePath);
|
|
46
54
|
}
|
|
@@ -70,10 +78,17 @@ export function writeFileAtomic(filePath, data, callback) {
|
|
|
70
78
|
return;
|
|
71
79
|
}
|
|
72
80
|
// Back up current file (best-effort, sync is fine for a copy)
|
|
81
|
+
// V3: Check for symlinks before backup to prevent data exfiltration
|
|
73
82
|
try {
|
|
74
|
-
fs.
|
|
83
|
+
const stat = fs.lstatSync(filePath);
|
|
84
|
+
if (stat.isSymbolicLink()) {
|
|
85
|
+
fs.unlinkSync(filePath);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
fs.copyFileSync(filePath, filePath + ".bak");
|
|
89
|
+
}
|
|
75
90
|
}
|
|
76
|
-
catch { /* ignore */ }
|
|
91
|
+
catch { /* ignore — ENOENT means no file to back up */ }
|
|
77
92
|
fs.rename(tmp, filePath, (renameErr) => {
|
|
78
93
|
if (renameErr) {
|
|
79
94
|
try {
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
/**
|
|
4
|
+
* Shared sanitization utilities for redacting sensitive data from tool outputs.
|
|
5
|
+
* Used by both the world model (state persistence) and MCP tool responses.
|
|
6
|
+
*/
|
|
7
|
+
import { execSync } from "node:child_process";
|
|
8
|
+
import os from "node:os";
|
|
9
|
+
const MAX_STRING_LENGTH = 1000;
|
|
10
|
+
const ALLOWED_URL_PROTOCOLS = new Set(["http:", "https:", "about:", "chrome:", "chrome-extension:"]);
|
|
11
|
+
const SENSITIVE_URL_PARAMS = new Set([
|
|
12
|
+
"code", "token", "access_token", "refresh_token", "id_token",
|
|
13
|
+
"secret", "password", "key", "api_key", "apikey", "auth",
|
|
14
|
+
"session", "session_id", "sessionid", "state", "nonce",
|
|
15
|
+
]);
|
|
16
|
+
const SENSITIVE_LABEL_PATTERNS = [
|
|
17
|
+
// email:password in window titles (e.g. "user@example.com:P@ssw0rd! - Chrome")
|
|
18
|
+
[/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+:[^\s]+/g, "[CREDENTIALS_REDACTED]"],
|
|
19
|
+
// Bearer tokens
|
|
20
|
+
[/Bearer\s+[A-Za-z0-9\-._~+/]+=*/g, "[BEARER_REDACTED]"],
|
|
21
|
+
];
|
|
22
|
+
/**
|
|
23
|
+
* Sanitize untrusted strings: truncate + strip control characters.
|
|
24
|
+
*/
|
|
25
|
+
export function sanitizeString(s) {
|
|
26
|
+
// eslint-disable-next-line no-control-regex
|
|
27
|
+
const stripped = s.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
|
|
28
|
+
return stripped.length > MAX_STRING_LENGTH
|
|
29
|
+
? stripped.slice(0, MAX_STRING_LENGTH)
|
|
30
|
+
: stripped;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Sanitize a URL: validate protocol + redact sensitive query params.
|
|
34
|
+
*/
|
|
35
|
+
export function sanitizeUrl(url) {
|
|
36
|
+
try {
|
|
37
|
+
const parsed = new URL(url);
|
|
38
|
+
if (!ALLOWED_URL_PROTOCOLS.has(parsed.protocol))
|
|
39
|
+
return "about:blocked";
|
|
40
|
+
let redacted = false;
|
|
41
|
+
for (const paramName of parsed.searchParams.keys()) {
|
|
42
|
+
if (SENSITIVE_URL_PARAMS.has(paramName.toLowerCase())) {
|
|
43
|
+
parsed.searchParams.set(paramName, "[REDACTED]");
|
|
44
|
+
redacted = true;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return redacted ? parsed.toString() : url;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return "about:blocked";
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Check if a string looks like a token/key (32+ chars, 3+ character classes).
|
|
55
|
+
*/
|
|
56
|
+
function looksLikeToken(s) {
|
|
57
|
+
if (s.length < 32)
|
|
58
|
+
return false;
|
|
59
|
+
const hasUpper = /[A-Z]/.test(s);
|
|
60
|
+
const hasLower = /[a-z]/.test(s);
|
|
61
|
+
const hasDigit = /[0-9]/.test(s);
|
|
62
|
+
const hasSpecial = /[\-._~+/]/.test(s);
|
|
63
|
+
const classes = [hasUpper, hasLower, hasDigit, hasSpecial].filter(Boolean).length;
|
|
64
|
+
return classes >= 3;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Redact sensitive patterns from labels/titles: credentials, bearer tokens, long token strings.
|
|
68
|
+
*/
|
|
69
|
+
export function redactSensitiveLabel(label) {
|
|
70
|
+
let result = label;
|
|
71
|
+
for (const [pattern, replacement] of SENSITIVE_LABEL_PATTERNS) {
|
|
72
|
+
result = result.replace(pattern, replacement);
|
|
73
|
+
}
|
|
74
|
+
result = result.replace(/[A-Za-z0-9\-._~+/]{32,}={0,2}/g, (match) => looksLikeToken(match) ? "[TOKEN_REDACTED]" : match);
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Redact PII patterns from text for persistence paths (Option C: redact on write, not on read).
|
|
79
|
+
* Catches: email addresses, phone numbers, and the local user's name parts.
|
|
80
|
+
* Applied to: memory actions/strategies, playbook exports, timeline logs.
|
|
81
|
+
* NOT applied to: live tool responses (ocr, ui_tree, screenshot, browser_dom, world_state).
|
|
82
|
+
*/
|
|
83
|
+
export function redactPII(text) {
|
|
84
|
+
let result = text;
|
|
85
|
+
// Email addresses
|
|
86
|
+
result = result.replace(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, "[EMAIL_REDACTED]");
|
|
87
|
+
// Phone numbers — international and domestic formats
|
|
88
|
+
result = result.replace(/(?:\+\d{1,3}[\s\-]?)?\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4}/g, "[PHONE_REDACTED]");
|
|
89
|
+
// Local user name parts (from OS)
|
|
90
|
+
result = redactUsername(result);
|
|
91
|
+
// Credential patterns already handled by redactSensitiveLabel
|
|
92
|
+
result = redactSensitiveLabel(result);
|
|
93
|
+
return result;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Redact the macOS username from strings (e.g. menu labels like "Log Out username").
|
|
97
|
+
*/
|
|
98
|
+
let _cachedNameParts = null;
|
|
99
|
+
/** Collect all name parts to redact: short username, full name from id -F, home dir name. */
|
|
100
|
+
function getNameParts() {
|
|
101
|
+
if (_cachedNameParts !== null)
|
|
102
|
+
return _cachedNameParts;
|
|
103
|
+
const parts = new Set();
|
|
104
|
+
// Short username from env
|
|
105
|
+
const username = process.env.USER || process.env.USERNAME || "";
|
|
106
|
+
if (username.length >= 2)
|
|
107
|
+
parts.add(username);
|
|
108
|
+
// Home directory basename (e.g. /Users/khushi → khushi)
|
|
109
|
+
try {
|
|
110
|
+
const home = os.homedir();
|
|
111
|
+
const homeName = home.split("/").pop() || "";
|
|
112
|
+
if (homeName.length >= 2)
|
|
113
|
+
parts.add(homeName);
|
|
114
|
+
}
|
|
115
|
+
catch { /* ignore */ }
|
|
116
|
+
// Full display name from id -F (macOS only)
|
|
117
|
+
try {
|
|
118
|
+
const fullName = execSync("id -F", { encoding: "utf-8", timeout: 2000, stdio: ["pipe", "pipe", "pipe"] }).trim();
|
|
119
|
+
if (fullName.length >= 2) {
|
|
120
|
+
parts.add(fullName);
|
|
121
|
+
// Also add individual name words (e.g. "khushi goyal" → "khushi", "goyal")
|
|
122
|
+
for (const word of fullName.split(/\s+/)) {
|
|
123
|
+
if (word.length >= 2)
|
|
124
|
+
parts.add(word);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch { /* not macOS or id -F unavailable */ }
|
|
129
|
+
// Sort longest first so longer matches take priority
|
|
130
|
+
_cachedNameParts = [...parts].sort((a, b) => b.length - a.length);
|
|
131
|
+
return _cachedNameParts;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Redact the macOS username, full display name, and individual name parts from strings.
|
|
135
|
+
* Also catches common patterns like "Log Out <name>" where the name may not match env vars.
|
|
136
|
+
*/
|
|
137
|
+
export function redactUsername(text) {
|
|
138
|
+
let result = text;
|
|
139
|
+
for (const part of getNameParts()) {
|
|
140
|
+
result = result.replaceAll(part, "[USER]");
|
|
141
|
+
}
|
|
142
|
+
// Catch "Log Out <name>" pattern — macOS always formats this as "Log Out <full display name>"
|
|
143
|
+
// The name part may already be partially redacted (e.g. "[USER] goyal"), so match everything after "Log Out "
|
|
144
|
+
result = result.replace(/Log Out [^\n:]+/g, "Log Out [USER]");
|
|
145
|
+
return result;
|
|
146
|
+
}
|