unbrowse 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +455 -96
- package/dist/index.js +2 -6
- package/dist/mcp.js +695 -46
- package/dist/server.js +25811 -0
- package/package.json +1 -2
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +7 -10
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -317
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1823
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1794
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1572
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3397
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1524
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -103
- package/runtime-src/mcp.ts +0 -1747
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -912
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
- package/vendor/kuri/win-x64/kuri.exe +0 -0
|
@@ -1,196 +0,0 @@
|
|
|
1
|
-
import { createCipheriv, createDecipheriv, randomBytes } from "crypto";
|
|
2
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
3
|
-
import { join } from "path";
|
|
4
|
-
import { homedir } from "os";
|
|
5
|
-
import { log } from "../logger.js";
|
|
6
|
-
|
|
7
|
-
type KeytarClient = {
|
|
8
|
-
setPassword: (service: string, account: string, password: string) => Promise<unknown>;
|
|
9
|
-
getPassword: (service: string, account: string) => Promise<string | null>;
|
|
10
|
-
deletePassword: (service: string, account: string) => Promise<boolean>;
|
|
11
|
-
};
|
|
12
|
-
|
|
13
|
-
const KEYTAR_UNAVAILABLE = Symbol("KEYTAR_UNAVAILABLE");
|
|
14
|
-
const KEYTAR_BINDING_ERROR_RE = /(keytar(?:\.node)?|native bindings?|bindings file|no native build was found|could not locate the bindings file|module did not self-register|err_dlopen_failed|dlopen\(|compiled against a different node\.js version|cannot find module .*keytar|wasm is not supported on this platform|(set|get|delete)password is not a function)/i;
|
|
15
|
-
|
|
16
|
-
export function normalizeKeytarModule(mod: unknown): KeytarClient | null {
|
|
17
|
-
let candidate: unknown = mod;
|
|
18
|
-
for (let depth = 0; depth < 3; depth++) {
|
|
19
|
-
if (!candidate || typeof candidate !== "object" || !("default" in candidate)) break;
|
|
20
|
-
candidate = (candidate as { default?: unknown }).default;
|
|
21
|
-
}
|
|
22
|
-
if (!candidate) return null;
|
|
23
|
-
if (
|
|
24
|
-
typeof (candidate as Partial<KeytarClient>).setPassword === "function" &&
|
|
25
|
-
typeof (candidate as Partial<KeytarClient>).getPassword === "function" &&
|
|
26
|
-
typeof (candidate as Partial<KeytarClient>).deletePassword === "function"
|
|
27
|
-
) {
|
|
28
|
-
return candidate as KeytarClient;
|
|
29
|
-
}
|
|
30
|
-
return null;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
let keytar: KeytarClient | null = null;
|
|
34
|
-
try {
|
|
35
|
-
keytar = normalizeKeytarModule(await import("keytar"));
|
|
36
|
-
} catch {
|
|
37
|
-
// keytar unavailable -- use encrypted file fallback
|
|
38
|
-
}
|
|
39
|
-
const importedKeytar = keytar;
|
|
40
|
-
let keytarFallbackLogged = false;
|
|
41
|
-
|
|
42
|
-
function isKeytarBindingError(error: unknown): boolean {
|
|
43
|
-
const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
44
|
-
return KEYTAR_BINDING_ERROR_RE.test(message);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function disableKeytar(error: unknown): void {
|
|
48
|
-
keytar = null;
|
|
49
|
-
if (keytarFallbackLogged) return;
|
|
50
|
-
const summary = error instanceof Error ? error.message : String(error);
|
|
51
|
-
log("vault", `keytar runtime unavailable (${summary}); using encrypted file fallback`);
|
|
52
|
-
keytarFallbackLogged = true;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
async function callKeytar<T>(op: (client: KeytarClient) => Promise<T>): Promise<T | typeof KEYTAR_UNAVAILABLE> {
|
|
56
|
-
if (!keytar) return KEYTAR_UNAVAILABLE;
|
|
57
|
-
try {
|
|
58
|
-
return await op(keytar);
|
|
59
|
-
} catch (error) {
|
|
60
|
-
if (!isKeytarBindingError(error)) throw error;
|
|
61
|
-
disableKeytar(error);
|
|
62
|
-
return KEYTAR_UNAVAILABLE;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export function setKeytarClientForTests(client: KeytarClient | null): void {
|
|
67
|
-
keytar = client;
|
|
68
|
-
keytarFallbackLogged = false;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export function resetKeytarClientForTests(): void {
|
|
72
|
-
keytar = importedKeytar;
|
|
73
|
-
keytarFallbackLogged = false;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export interface StoredCredential {
|
|
77
|
-
value: string;
|
|
78
|
-
stored_at: string;
|
|
79
|
-
expires_at?: string;
|
|
80
|
-
max_age_ms?: number;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const SERVICE = "unbrowse";
|
|
84
|
-
// Use a fixed location so the vault works regardless of server CWD
|
|
85
|
-
const VAULT_DIR = join(homedir(), ".unbrowse", "vault");
|
|
86
|
-
const VAULT_FILE = join(VAULT_DIR, "credentials.enc");
|
|
87
|
-
const KEY_FILE = join(VAULT_DIR, ".key");
|
|
88
|
-
|
|
89
|
-
function getOrCreateKey(): Buffer {
|
|
90
|
-
if (!existsSync(VAULT_DIR)) mkdirSync(VAULT_DIR, { recursive: true, mode: 0o700 });
|
|
91
|
-
if (existsSync(KEY_FILE)) return readFileSync(KEY_FILE);
|
|
92
|
-
const key = randomBytes(32);
|
|
93
|
-
writeFileSync(KEY_FILE, key, { mode: 0o600 });
|
|
94
|
-
return key;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// Async mutex to prevent concurrent read-modify-write races
|
|
98
|
-
let vaultLock: Promise<void> = Promise.resolve();
|
|
99
|
-
function withVaultLock<T>(fn: () => T | Promise<T>): Promise<T> {
|
|
100
|
-
const prev = vaultLock;
|
|
101
|
-
let release: () => void;
|
|
102
|
-
vaultLock = new Promise<void>((r) => { release = r; });
|
|
103
|
-
return prev.then(fn).finally(() => release!());
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function readVaultFile(): Record<string, string> {
|
|
107
|
-
if (!existsSync(VAULT_FILE)) return {};
|
|
108
|
-
try {
|
|
109
|
-
const key = getOrCreateKey();
|
|
110
|
-
const raw = readFileSync(VAULT_FILE);
|
|
111
|
-
const iv = raw.subarray(0, 16);
|
|
112
|
-
const enc = raw.subarray(16);
|
|
113
|
-
const decipher = createDecipheriv("aes-256-cbc", key, iv);
|
|
114
|
-
const dec = Buffer.concat([decipher.update(enc), decipher.final()]);
|
|
115
|
-
return JSON.parse(dec.toString("utf8")) as Record<string, string>;
|
|
116
|
-
} catch {
|
|
117
|
-
return {};
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
function writeVaultFile(data: Record<string, string>): void {
|
|
122
|
-
const key = getOrCreateKey();
|
|
123
|
-
const iv = randomBytes(16);
|
|
124
|
-
const cipher = createCipheriv("aes-256-cbc", key, iv);
|
|
125
|
-
const enc = Buffer.concat([cipher.update(JSON.stringify(data), "utf8"), cipher.final()]);
|
|
126
|
-
writeFileSync(VAULT_FILE, Buffer.concat([iv, enc]), { mode: 0o600 });
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
export async function storeCredential(
|
|
130
|
-
account: string,
|
|
131
|
-
value: string,
|
|
132
|
-
opts?: { expires_at?: string; max_age_ms?: number }
|
|
133
|
-
): Promise<void> {
|
|
134
|
-
const wrapped: StoredCredential = {
|
|
135
|
-
value,
|
|
136
|
-
stored_at: new Date().toISOString(),
|
|
137
|
-
expires_at: opts?.expires_at,
|
|
138
|
-
max_age_ms: opts?.max_age_ms,
|
|
139
|
-
};
|
|
140
|
-
const serialized = JSON.stringify(wrapped);
|
|
141
|
-
const keytarResult = await callKeytar((client) => client.setPassword(SERVICE, account, serialized));
|
|
142
|
-
if (keytarResult !== KEYTAR_UNAVAILABLE) return;
|
|
143
|
-
await withVaultLock(() => {
|
|
144
|
-
const data = readVaultFile();
|
|
145
|
-
data[account] = serialized;
|
|
146
|
-
writeVaultFile(data);
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
function isExpired(cred: StoredCredential): boolean {
|
|
151
|
-
if (cred.expires_at) {
|
|
152
|
-
return new Date(cred.expires_at).getTime() <= Date.now();
|
|
153
|
-
}
|
|
154
|
-
if (cred.max_age_ms) {
|
|
155
|
-
return new Date(cred.stored_at).getTime() + cred.max_age_ms <= Date.now();
|
|
156
|
-
}
|
|
157
|
-
return false;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
export async function getCredential(account: string): Promise<string | null> {
|
|
161
|
-
let raw: string | null;
|
|
162
|
-
const keytarResult = await callKeytar((client) => client.getPassword(SERVICE, account));
|
|
163
|
-
if (keytarResult !== KEYTAR_UNAVAILABLE) {
|
|
164
|
-
raw = keytarResult;
|
|
165
|
-
} else {
|
|
166
|
-
const data = readVaultFile();
|
|
167
|
-
raw = data[account] ?? null;
|
|
168
|
-
}
|
|
169
|
-
if (!raw) return null;
|
|
170
|
-
|
|
171
|
-
// Try to parse as StoredCredential; backward-compat: raw strings are legacy (no expiry)
|
|
172
|
-
try {
|
|
173
|
-
const parsed = JSON.parse(raw) as StoredCredential;
|
|
174
|
-
if (parsed.value && parsed.stored_at) {
|
|
175
|
-
// It's a wrapped credential — check expiry
|
|
176
|
-
if (isExpired(parsed)) {
|
|
177
|
-
await deleteCredential(account);
|
|
178
|
-
return null;
|
|
179
|
-
}
|
|
180
|
-
return parsed.value;
|
|
181
|
-
}
|
|
182
|
-
} catch {
|
|
183
|
-
// Not JSON — legacy raw string, return as-is
|
|
184
|
-
}
|
|
185
|
-
return raw;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
export async function deleteCredential(account: string): Promise<void> {
|
|
189
|
-
const keytarResult = await callKeytar((client) => client.deletePassword(SERVICE, account));
|
|
190
|
-
if (keytarResult !== KEYTAR_UNAVAILABLE) return;
|
|
191
|
-
await withVaultLock(() => {
|
|
192
|
-
const data = readVaultFile();
|
|
193
|
-
delete data[account];
|
|
194
|
-
writeVaultFile(data);
|
|
195
|
-
});
|
|
196
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { EndpointDescriptor, SkillManifest } from "../types/index.js";
|
|
2
|
-
|
|
3
|
-
export function isAuthGatedEndpoint(
|
|
4
|
-
skill: SkillManifest,
|
|
5
|
-
endpoint: EndpointDescriptor,
|
|
6
|
-
): boolean {
|
|
7
|
-
return Boolean(skill.auth_profile_ref || endpoint.semantic?.auth_required === true);
|
|
8
|
-
}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import type { EndpointDescriptor, SkillManifest } from "../types/index.js";
|
|
2
|
-
|
|
3
|
-
const STALE_THRESHOLD_MS = 24 * 60 * 60 * 1000; // 24 hours
|
|
4
|
-
|
|
5
|
-
export function selectVerificationCandidates(
|
|
6
|
-
skill: SkillManifest,
|
|
7
|
-
options?: {
|
|
8
|
-
staleOnly?: boolean;
|
|
9
|
-
limit?: number;
|
|
10
|
-
now?: number;
|
|
11
|
-
},
|
|
12
|
-
): EndpointDescriptor[] {
|
|
13
|
-
const now = options?.now ?? Date.now();
|
|
14
|
-
const candidates = skill.endpoints.filter((endpoint) => {
|
|
15
|
-
if (endpoint.method !== "GET") return false;
|
|
16
|
-
if (!options?.staleOnly) return true;
|
|
17
|
-
const isDisabled = endpoint.verification_status === "disabled";
|
|
18
|
-
const isFailed = endpoint.verification_status === "failed";
|
|
19
|
-
const lowReliability = typeof endpoint.reliability_score === "number" && endpoint.reliability_score < 0.2;
|
|
20
|
-
const lastVerified = endpoint.last_verified_at
|
|
21
|
-
? new Date(endpoint.last_verified_at).getTime()
|
|
22
|
-
: 0;
|
|
23
|
-
return isDisabled || isFailed || lowReliability || now - lastVerified > STALE_THRESHOLD_MS;
|
|
24
|
-
});
|
|
25
|
-
const limit = options?.limit && options.limit > 0 ? options.limit : candidates.length;
|
|
26
|
-
return candidates.slice(0, limit);
|
|
27
|
-
}
|
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
import { executeInBrowser } from "../capture/index.js";
|
|
2
|
-
import { updateEndpointScore } from "../marketplace/index.js";
|
|
3
|
-
import { listSkills, getSkill } from "../marketplace/index.js";
|
|
4
|
-
import { detectSchemaDrift } from "../transform/drift.js";
|
|
5
|
-
import { computeVerificationCoverage, INITIAL_MATRIX } from "./matrix.js";
|
|
6
|
-
import { isAuthGatedEndpoint } from "./auth-gate.js";
|
|
7
|
-
import type { VerificationMatrix } from "./matrix.js";
|
|
8
|
-
import type { EndpointDescriptor, SkillManifest, VerificationStatus } from "../types/index.js";
|
|
9
|
-
import { selectVerificationCandidates } from "./candidates.js";
|
|
10
|
-
|
|
11
|
-
const VERIFICATION_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
12
|
-
const VERIFY_ENDPOINT_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_VERIFY_ENDPOINT_BATCH_SIZE ?? 3));
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Verify a single endpoint by test-executing safe (GET) endpoints.
|
|
16
|
-
* Returns the new verification status.
|
|
17
|
-
*/
|
|
18
|
-
export async function verifyEndpoint(
|
|
19
|
-
skill: SkillManifest,
|
|
20
|
-
endpoint: EndpointDescriptor
|
|
21
|
-
): Promise<VerificationStatus> {
|
|
22
|
-
// Only verify safe (GET) endpoints automatically
|
|
23
|
-
if (endpoint.method !== "GET") return endpoint.verification_status;
|
|
24
|
-
|
|
25
|
-
try {
|
|
26
|
-
const { status, data } = await executeInBrowser(
|
|
27
|
-
endpoint.url_template,
|
|
28
|
-
endpoint.method,
|
|
29
|
-
endpoint.headers_template ?? {},
|
|
30
|
-
undefined,
|
|
31
|
-
undefined,
|
|
32
|
-
undefined
|
|
33
|
-
);
|
|
34
|
-
|
|
35
|
-
if (status < 200 || status >= 300) {
|
|
36
|
-
await updateEndpointScore(skill.skill_id, endpoint.endpoint_id, endpoint.reliability_score, "failed");
|
|
37
|
-
return "failed";
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Check for schema drift if we have a response schema
|
|
41
|
-
let hasCriticalDrift = false;
|
|
42
|
-
if (endpoint.response_schema && data != null) {
|
|
43
|
-
const drift = detectSchemaDrift(endpoint.response_schema, data);
|
|
44
|
-
if (drift.drifted && (drift.removed_fields.length > 0 || drift.type_changes.length > 0)) {
|
|
45
|
-
hasCriticalDrift = true;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const newStatus: VerificationStatus = hasCriticalDrift ? "pending" : "verified";
|
|
50
|
-
// Reset score for recovered disabled endpoints so they become usable again
|
|
51
|
-
const newScore = endpoint.verification_status === "disabled" && newStatus === "verified"
|
|
52
|
-
? 0.5
|
|
53
|
-
: endpoint.reliability_score;
|
|
54
|
-
await updateEndpointScore(skill.skill_id, endpoint.endpoint_id, newScore, newStatus);
|
|
55
|
-
// Update last_verified_at
|
|
56
|
-
const fullSkill = await getSkill(skill.skill_id);
|
|
57
|
-
if (fullSkill) {
|
|
58
|
-
const ep = fullSkill.endpoints.find((e) => e.endpoint_id === endpoint.endpoint_id);
|
|
59
|
-
if (ep) ep.last_verified_at = new Date().toISOString();
|
|
60
|
-
}
|
|
61
|
-
return newStatus;
|
|
62
|
-
} catch {
|
|
63
|
-
await updateEndpointScore(skill.skill_id, endpoint.endpoint_id, endpoint.reliability_score, "failed");
|
|
64
|
-
return "failed";
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* Verify all safe endpoints in a skill.
|
|
70
|
-
* Returns a map of endpoint_id -> verification status.
|
|
71
|
-
*/
|
|
72
|
-
export async function verifySkill(
|
|
73
|
-
skill: SkillManifest,
|
|
74
|
-
options?: {
|
|
75
|
-
endpoints?: EndpointDescriptor[];
|
|
76
|
-
staleOnly?: boolean;
|
|
77
|
-
limit?: number;
|
|
78
|
-
now?: number;
|
|
79
|
-
},
|
|
80
|
-
): Promise<Record<string, VerificationStatus>> {
|
|
81
|
-
const results: Record<string, VerificationStatus> = {};
|
|
82
|
-
const endpoints = options?.endpoints ?? selectVerificationCandidates(skill, options);
|
|
83
|
-
for (const endpoint of endpoints) {
|
|
84
|
-
results[endpoint.endpoint_id] = await verifyEndpoint(skill, endpoint);
|
|
85
|
-
}
|
|
86
|
-
return results;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Verify all safe endpoints in a skill and compute verification coverage
|
|
91
|
-
* from the integration matrix. Returns endpoint results plus a coverage ratio.
|
|
92
|
-
*/
|
|
93
|
-
export async function verifySkillWithCoverage(
|
|
94
|
-
skill: SkillManifest,
|
|
95
|
-
matrix: VerificationMatrix = INITIAL_MATRIX,
|
|
96
|
-
): Promise<{ results: Record<string, VerificationStatus>; coverage: number }> {
|
|
97
|
-
const results = await verifySkill(skill);
|
|
98
|
-
const coverage = computeVerificationCoverage(matrix);
|
|
99
|
-
return { results, coverage };
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* Schedule periodic re-verification of stale endpoints.
|
|
104
|
-
*/
|
|
105
|
-
export function schedulePeriodicVerification(): ReturnType<typeof setInterval> {
|
|
106
|
-
return setInterval(async () => {
|
|
107
|
-
const skills = await listSkills();
|
|
108
|
-
for (const skill of skills) {
|
|
109
|
-
if (skill.lifecycle !== "active") continue;
|
|
110
|
-
const endpoints = selectVerificationCandidates(skill, {
|
|
111
|
-
staleOnly: true,
|
|
112
|
-
limit: VERIFY_ENDPOINT_BATCH_SIZE,
|
|
113
|
-
});
|
|
114
|
-
for (const endpoint of endpoints) {
|
|
115
|
-
if (isAuthGatedEndpoint(skill, endpoint)) continue;
|
|
116
|
-
await verifyEndpoint(skill, endpoint).catch(() => {});
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
}, VERIFICATION_INTERVAL_MS);
|
|
120
|
-
}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
export interface IntegrationCheck {
|
|
2
|
-
host: string;
|
|
3
|
-
capability: string;
|
|
4
|
-
status: "pass" | "fail" | "skip" | "untested";
|
|
5
|
-
last_verified?: string;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
export type VerificationMatrix = IntegrationCheck[];
|
|
9
|
-
|
|
10
|
-
export function computeVerificationCoverage(matrix: VerificationMatrix): number {
|
|
11
|
-
if (matrix.length === 0) return 0;
|
|
12
|
-
const tested = matrix.filter((c) => c.status !== "untested").length;
|
|
13
|
-
return tested / matrix.length;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export const INITIAL_MATRIX: VerificationMatrix = [
|
|
17
|
-
{ host: "openclaw", capability: "capture", status: "pass", last_verified: "2026-03-31" },
|
|
18
|
-
{ host: "openclaw", capability: "execute", status: "pass", last_verified: "2026-03-31" },
|
|
19
|
-
{ host: "openclaw", capability: "search", status: "pass", last_verified: "2026-03-31" },
|
|
20
|
-
{ host: "mcp", capability: "execute", status: "pass", last_verified: "2026-03-31" },
|
|
21
|
-
{ host: "mcp", capability: "search", status: "pass", last_verified: "2026-03-31" },
|
|
22
|
-
{ host: "cli", capability: "capture", status: "pass", last_verified: "2026-03-31" },
|
|
23
|
-
{ host: "cli", capability: "execute", status: "pass", last_verified: "2026-03-31" },
|
|
24
|
-
{ host: "cli", capability: "search", status: "pass", last_verified: "2026-03-31" },
|
|
25
|
-
{ host: "cli", capability: "publish", status: "pass", last_verified: "2026-03-31" },
|
|
26
|
-
{ host: "hermes", capability: "execute", status: "untested" },
|
|
27
|
-
{ host: "elizaos", capability: "execute", status: "untested" },
|
|
28
|
-
{ host: "langchain", capability: "execute", status: "untested" },
|
|
29
|
-
{ host: "langchain", capability: "search", status: "untested" },
|
|
30
|
-
];
|
package/runtime-src/version.ts
DELETED
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
import { createHash } from "crypto";
|
|
2
|
-
import { existsSync, readFileSync, readdirSync } from "fs";
|
|
3
|
-
import { dirname, join, parse } from "path";
|
|
4
|
-
import { fileURLToPath } from "url";
|
|
5
|
-
import {
|
|
6
|
-
BUILD_CODE_HASH,
|
|
7
|
-
BUILD_DEFAULT_BACKEND_URL,
|
|
8
|
-
BUILD_GIT_SHA,
|
|
9
|
-
BUILD_RELEASE_MANIFEST_BASE64,
|
|
10
|
-
BUILD_RELEASE_MANIFEST_SIGNATURE,
|
|
11
|
-
BUILD_RELEASE_VERSION,
|
|
12
|
-
} from "./build-info.generated.js";
|
|
13
|
-
|
|
14
|
-
// Deterministic version hash of all src/*.ts files.
|
|
15
|
-
// Computed once at startup. Same code = same hash.
|
|
16
|
-
// Used to stamp every trace so real user sessions become versioned evals.
|
|
17
|
-
|
|
18
|
-
const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
|
|
19
|
-
|
|
20
|
-
function collectTsFiles(dir: string): string[] {
|
|
21
|
-
const results: string[] = [];
|
|
22
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
23
|
-
const full = join(dir, entry.name);
|
|
24
|
-
if (entry.isDirectory() && entry.name !== "node_modules") {
|
|
25
|
-
results.push(...collectTsFiles(full));
|
|
26
|
-
} else if (entry.name.endsWith(".ts")) {
|
|
27
|
-
results.push(full);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
return results;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function hashFiles(srcDir: string, files: string[]): string {
|
|
34
|
-
const hash = createHash("sha256");
|
|
35
|
-
for (const file of files) {
|
|
36
|
-
hash.update(file.slice(srcDir.length));
|
|
37
|
-
hash.update(readFileSync(file, "utf-8"));
|
|
38
|
-
}
|
|
39
|
-
return hash.digest("hex").slice(0, 12);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export function resolveCodeHashSourceDir(moduleDir: string): string | null {
|
|
43
|
-
const candidates = [
|
|
44
|
-
moduleDir,
|
|
45
|
-
join(moduleDir, "runtime-src"),
|
|
46
|
-
join(moduleDir, "..", "runtime-src"),
|
|
47
|
-
join(moduleDir, "src"),
|
|
48
|
-
join(moduleDir, "..", "src"),
|
|
49
|
-
];
|
|
50
|
-
|
|
51
|
-
for (const candidate of candidates) {
|
|
52
|
-
try {
|
|
53
|
-
if (!existsSync(candidate)) continue;
|
|
54
|
-
const files = collectTsFiles(candidate);
|
|
55
|
-
if (files.length > 0) return candidate;
|
|
56
|
-
} catch {
|
|
57
|
-
// ignore candidate
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
return null;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export function computeCodeHashForDir(srcDir: string): string {
|
|
65
|
-
const files = collectTsFiles(srcDir).sort();
|
|
66
|
-
if (files.length === 0) throw new Error(`No TypeScript sources found in ${srcDir}`);
|
|
67
|
-
return hashFiles(srcDir, files);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function computeCodeHash(): string {
|
|
71
|
-
try {
|
|
72
|
-
const srcDir = resolveCodeHashSourceDir(MODULE_DIR);
|
|
73
|
-
if (srcDir) return computeCodeHashForDir(srcDir);
|
|
74
|
-
} catch {
|
|
75
|
-
// fall through
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const pkgVersion = getPackageVersion();
|
|
79
|
-
if (pkgVersion !== "unknown") {
|
|
80
|
-
return createHash("sha256").update(`package:${pkgVersion}`).digest("hex").slice(0, 12);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// Compiled binary: filesystem not available, use a static hash
|
|
84
|
-
return "compiled";
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function getGitSha(): string {
|
|
88
|
-
return BUILD_GIT_SHA?.trim() || "unknown";
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function decodeBase64UrlJson<T>(value: string): T | null {
|
|
92
|
-
try {
|
|
93
|
-
if (!value) return null;
|
|
94
|
-
return JSON.parse(Buffer.from(value, "base64url").toString("utf-8")) as T;
|
|
95
|
-
} catch {
|
|
96
|
-
return null;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export function getEmbeddedReleaseVersion(): string | null {
|
|
101
|
-
if (BUILD_RELEASE_VERSION?.trim()) return BUILD_RELEASE_VERSION.trim();
|
|
102
|
-
const manifest = decodeBase64UrlJson<{ release_version?: string }>(BUILD_RELEASE_MANIFEST_BASE64?.trim() || "");
|
|
103
|
-
return typeof manifest?.release_version === "string" && manifest.release_version.trim()
|
|
104
|
-
? manifest.release_version.trim()
|
|
105
|
-
: null;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
export function getPackageVersionForModuleDir(moduleDir: string): string {
|
|
109
|
-
let dir = moduleDir;
|
|
110
|
-
const root = parse(dir).root;
|
|
111
|
-
|
|
112
|
-
while (true) {
|
|
113
|
-
try {
|
|
114
|
-
const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf-8")) as { version?: string };
|
|
115
|
-
return typeof pkg.version === "string" ? pkg.version : "unknown";
|
|
116
|
-
} catch {
|
|
117
|
-
// keep walking
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
if (dir === root) return "unknown";
|
|
121
|
-
dir = dirname(dir);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function getPackageVersion(): string {
|
|
126
|
-
const packageVersion = getPackageVersionForModuleDir(MODULE_DIR);
|
|
127
|
-
if (packageVersion !== "unknown") return packageVersion;
|
|
128
|
-
return getEmbeddedReleaseVersion() ?? "unknown";
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/** 12-char hex hash of all source file contents */
|
|
132
|
-
export const CODE_HASH: string = BUILD_CODE_HASH?.trim() || computeCodeHash();
|
|
133
|
-
|
|
134
|
-
/** Short git commit SHA */
|
|
135
|
-
export const GIT_SHA: string = getGitSha();
|
|
136
|
-
|
|
137
|
-
/** package.json version for CLI/runtime mismatch checks */
|
|
138
|
-
export const PACKAGE_VERSION: string = getPackageVersion();
|
|
139
|
-
|
|
140
|
-
/** Embedded default backend target; preview binaries override this at build time. */
|
|
141
|
-
export const DEFAULT_BACKEND_URL: string = BUILD_DEFAULT_BACKEND_URL?.trim() || "https://beta-api.unbrowse.ai";
|
|
142
|
-
|
|
143
|
-
/** Combined version: "{code_hash}@{git_sha}" — stamped on every trace */
|
|
144
|
-
export const TRACE_VERSION: string = `${CODE_HASH}@${GIT_SHA}`;
|
|
145
|
-
|
|
146
|
-
/** Signed release attestation; embedded in compiled binaries when available. */
|
|
147
|
-
export const RELEASE_MANIFEST_BASE64: string = BUILD_RELEASE_MANIFEST_BASE64?.trim() || "";
|
|
148
|
-
export const RELEASE_MANIFEST_SIGNATURE: string = BUILD_RELEASE_MANIFEST_SIGNATURE?.trim() || "";
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "fs";
|
|
2
|
-
import { homedir } from "os";
|
|
3
|
-
import { join } from "path";
|
|
4
|
-
import { log } from "../logger.js";
|
|
5
|
-
import type { WorkflowArtifact, WorkflowRecipe, WorkflowStepStrategy } from "../types/index.js";
|
|
6
|
-
|
|
7
|
-
function sanitizeProfileName(value: string): string {
|
|
8
|
-
return value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
function getConfigDir(): string {
|
|
12
|
-
if (process.env.UNBROWSE_CONFIG_DIR) return process.env.UNBROWSE_CONFIG_DIR;
|
|
13
|
-
const profile = sanitizeProfileName(process.env.UNBROWSE_PROFILE ?? "");
|
|
14
|
-
return profile
|
|
15
|
-
? join(homedir(), ".unbrowse", "profiles", profile)
|
|
16
|
-
: join(homedir(), ".unbrowse");
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
function getWorkflowArtifactDir(): string {
|
|
20
|
-
return process.env.UNBROWSE_WORKFLOW_ARTIFACT_DIR
|
|
21
|
-
?? join(getConfigDir(), "workflow-artifacts");
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export function workflowArtifactPathForSkill(skillId: string): string {
|
|
25
|
-
return join(getWorkflowArtifactDir(), `${skillId}.json`);
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export function readWorkflowArtifact(skillId: string): WorkflowArtifact | null {
|
|
29
|
-
const path = workflowArtifactPathForSkill(skillId);
|
|
30
|
-
if (!existsSync(path)) return null;
|
|
31
|
-
try {
|
|
32
|
-
return JSON.parse(readFileSync(path, "utf-8")) as WorkflowArtifact;
|
|
33
|
-
} catch {
|
|
34
|
-
return null;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function writeWorkflowArtifact(artifact: WorkflowArtifact): string | undefined {
|
|
39
|
-
try {
|
|
40
|
-
const dir = getWorkflowArtifactDir();
|
|
41
|
-
mkdirSync(dir, { recursive: true });
|
|
42
|
-
const target = workflowArtifactPathForSkill(artifact.skill_id);
|
|
43
|
-
writeFileSync(target, JSON.stringify(artifact, null, 2), "utf-8");
|
|
44
|
-
if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
|
|
45
|
-
log("workflow", `wrote artifact ${artifact.skill_id} -> ${target} (${artifact.recipes.length} recipes)`);
|
|
46
|
-
}
|
|
47
|
-
return target;
|
|
48
|
-
} catch (error) {
|
|
49
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
50
|
-
log("workflow", `artifact write failed for ${artifact.skill_id}: ${message}`);
|
|
51
|
-
return undefined;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function mergeRecipeState(
|
|
56
|
-
nextRecipe: WorkflowRecipe,
|
|
57
|
-
prevRecipe?: WorkflowRecipe,
|
|
58
|
-
): WorkflowRecipe {
|
|
59
|
-
if (!prevRecipe) return nextRecipe;
|
|
60
|
-
const previousSteps = new Map(prevRecipe.steps.map((step) => [step.strategy, step]));
|
|
61
|
-
const mergedSteps = nextRecipe.steps.map((step) => {
|
|
62
|
-
const previous = previousSteps.get(step.strategy);
|
|
63
|
-
if (!previous) return step;
|
|
64
|
-
return {
|
|
65
|
-
...step,
|
|
66
|
-
success_count: previous.success_count ?? step.success_count,
|
|
67
|
-
failure_count: previous.failure_count ?? step.failure_count,
|
|
68
|
-
last_status: previous.last_status ?? step.last_status,
|
|
69
|
-
last_error: previous.last_error ?? step.last_error,
|
|
70
|
-
last_used_at: previous.last_used_at ?? step.last_used_at,
|
|
71
|
-
};
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
if (prevRecipe.last_successful_strategy) {
|
|
75
|
-
mergedSteps.sort((lhs, rhs) => {
|
|
76
|
-
if (lhs.strategy === prevRecipe.last_successful_strategy) return -1;
|
|
77
|
-
if (rhs.strategy === prevRecipe.last_successful_strategy) return 1;
|
|
78
|
-
return 0;
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
return {
|
|
83
|
-
...nextRecipe,
|
|
84
|
-
steps: mergedSteps,
|
|
85
|
-
preferred: prevRecipe.preferred || nextRecipe.preferred,
|
|
86
|
-
last_successful_strategy: prevRecipe.last_successful_strategy ?? nextRecipe.last_successful_strategy,
|
|
87
|
-
last_used_at: prevRecipe.last_used_at ?? nextRecipe.last_used_at,
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
export function mergeWorkflowArtifacts(
|
|
92
|
-
nextArtifact: WorkflowArtifact,
|
|
93
|
-
prevArtifact?: WorkflowArtifact | null,
|
|
94
|
-
): WorkflowArtifact {
|
|
95
|
-
if (!prevArtifact || prevArtifact.skill_id !== nextArtifact.skill_id) return nextArtifact;
|
|
96
|
-
const previousRecipes = new Map(prevArtifact.recipes.map((recipe) => [recipe.endpoint_id, recipe]));
|
|
97
|
-
return {
|
|
98
|
-
...nextArtifact,
|
|
99
|
-
recipes: nextArtifact.recipes.map((recipe) => mergeRecipeState(recipe, previousRecipes.get(recipe.endpoint_id))),
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
export function recordWorkflowRecipeOutcome(
|
|
104
|
-
artifact: WorkflowArtifact,
|
|
105
|
-
endpointId: string,
|
|
106
|
-
strategy: WorkflowStepStrategy,
|
|
107
|
-
outcome: {
|
|
108
|
-
success: boolean;
|
|
109
|
-
status?: number;
|
|
110
|
-
error?: string;
|
|
111
|
-
selectedBindings?: Array<{ target_name: string; source_kind: string; source_name: string }>;
|
|
112
|
-
},
|
|
113
|
-
): WorkflowArtifact {
|
|
114
|
-
const now = new Date().toISOString();
|
|
115
|
-
return {
|
|
116
|
-
...artifact,
|
|
117
|
-
recipes: artifact.recipes.map((recipe) => {
|
|
118
|
-
if (recipe.endpoint_id !== endpointId) return recipe;
|
|
119
|
-
return {
|
|
120
|
-
...recipe,
|
|
121
|
-
preferred: outcome.success ? true : recipe.preferred,
|
|
122
|
-
last_successful_strategy: outcome.success ? strategy : recipe.last_successful_strategy,
|
|
123
|
-
last_used_at: now,
|
|
124
|
-
token_bindings: recipe.token_bindings.map((binding) => {
|
|
125
|
-
const selected = outcome.selectedBindings?.find((entry) => entry.target_name === binding.target_name);
|
|
126
|
-
if (!selected) return binding;
|
|
127
|
-
return {
|
|
128
|
-
...binding,
|
|
129
|
-
selected_source_kind: selected.source_kind as typeof binding.selected_source_kind,
|
|
130
|
-
selected_source_name: selected.source_name,
|
|
131
|
-
};
|
|
132
|
-
}),
|
|
133
|
-
steps: recipe.steps
|
|
134
|
-
.map((step) => {
|
|
135
|
-
if (step.strategy !== strategy) return step;
|
|
136
|
-
return {
|
|
137
|
-
...step,
|
|
138
|
-
success_count: (step.success_count ?? 0) + (outcome.success ? 1 : 0),
|
|
139
|
-
failure_count: (step.failure_count ?? 0) + (outcome.success ? 0 : 1),
|
|
140
|
-
last_status: outcome.status ?? step.last_status,
|
|
141
|
-
last_error: outcome.error,
|
|
142
|
-
last_used_at: now,
|
|
143
|
-
};
|
|
144
|
-
})
|
|
145
|
-
.sort((lhs, rhs) => {
|
|
146
|
-
if (outcome.success && lhs.strategy === strategy) return -1;
|
|
147
|
-
if (outcome.success && rhs.strategy === strategy) return 1;
|
|
148
|
-
return 0;
|
|
149
|
-
}),
|
|
150
|
-
};
|
|
151
|
-
}),
|
|
152
|
-
};
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
export function listWorkflowArtifacts(): string[] {
|
|
156
|
-
const dir = getWorkflowArtifactDir();
|
|
157
|
-
if (!existsSync(dir)) return [];
|
|
158
|
-
return readdirSync(dir)
|
|
159
|
-
.filter((entry) => entry.endsWith(".json"))
|
|
160
|
-
.map((entry) => join(dir, entry));
|
|
161
|
-
}
|