unbrowse 2.12.4 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +60 -23
- package/dist/cli.js +110 -172
- package/dist/mcp.js +251 -64
- package/package.json +1 -1
- package/runtime-src/api/browse-index.ts +74 -11
- package/runtime-src/api/browse-session.ts +19 -95
- package/runtime-src/api/routes.ts +148 -132
- package/runtime-src/browser/index.ts +33 -17
- package/runtime-src/build-info.generated.ts +4 -2
- package/runtime-src/capture/index.ts +251 -34
- package/runtime-src/capture/prefetch.ts +3 -30
- package/runtime-src/cli.ts +41 -157
- package/runtime-src/client/graph-client.ts +2 -1
- package/runtime-src/client/index.ts +20 -7
- package/runtime-src/execution/index.ts +12 -4
- package/runtime-src/foundry/publish-bundle.ts +392 -0
- package/runtime-src/graph/index.ts +581 -11
- package/runtime-src/indexer/index.ts +37 -13
- package/runtime-src/kuri/client.ts +20 -5
- package/runtime-src/mcp.ts +220 -44
- package/runtime-src/orchestrator/dag-feedback.ts +2 -1
- package/runtime-src/orchestrator/first-pass-action.ts +2 -2
- package/runtime-src/orchestrator/index.ts +318 -183
- package/runtime-src/orchestrator/passive-publish.ts +9 -4
- package/runtime-src/payments/index.ts +3 -1
- package/runtime-src/publish/review-context.ts +93 -0
- package/runtime-src/publish/schema-review.ts +192 -0
- package/runtime-src/publish-admission.ts +109 -0
- package/runtime-src/reverse-engineer/index.ts +122 -23
- package/runtime-src/runtime/local-server.ts +4 -15
- package/runtime-src/runtime/paths.ts +4 -0
- package/runtime-src/single-binary.ts +2 -0
- package/runtime-src/types/skill.ts +93 -0
- package/runtime-src/version.ts +41 -5
- package/runtime-src/workflow/publish.ts +23 -3
- package/scripts/postinstall.mjs +19 -4
- package/scripts/release-assets.mjs +4 -0
- package/scripts/verify-release-assets.mjs +8 -4
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +5 -5
package/dist/cli.js
CHANGED
|
@@ -22,12 +22,12 @@ var __promiseAll = (args) => Promise.all(args);
|
|
|
22
22
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
23
23
|
|
|
24
24
|
// ../../src/build-info.generated.ts
|
|
25
|
-
var BUILD_GIT_SHA = "
|
|
25
|
+
var BUILD_RELEASE_VERSION = "3.0.0", BUILD_GIT_SHA = "0f2fd9a7fbd4", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4wLjAiLCJnaXRfc2hhIjoiMGYyZmQ5YTdmYmQ0IiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AwZjJmZDlhN2ZiZDQiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA0VDExOjU3OjQ1LjMwNloifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
|
|
26
26
|
|
|
27
27
|
// ../../src/version.ts
|
|
28
28
|
import { createHash } from "crypto";
|
|
29
29
|
import { existsSync, readFileSync, readdirSync } from "fs";
|
|
30
|
-
import { dirname, join } from "path";
|
|
30
|
+
import { dirname, join, parse } from "path";
|
|
31
31
|
import { fileURLToPath } from "url";
|
|
32
32
|
function collectTsFiles(dir) {
|
|
33
33
|
const results = [];
|
|
@@ -89,20 +89,47 @@ function computeCodeHash() {
|
|
|
89
89
|
function getGitSha() {
|
|
90
90
|
return BUILD_GIT_SHA?.trim() || "unknown";
|
|
91
91
|
}
|
|
92
|
-
function
|
|
92
|
+
function decodeBase64UrlJson(value) {
|
|
93
93
|
try {
|
|
94
|
-
|
|
95
|
-
|
|
94
|
+
if (!value)
|
|
95
|
+
return null;
|
|
96
|
+
return JSON.parse(Buffer.from(value, "base64url").toString("utf-8"));
|
|
96
97
|
} catch {
|
|
97
|
-
return
|
|
98
|
+
return null;
|
|
98
99
|
}
|
|
99
100
|
}
|
|
100
|
-
|
|
101
|
+
function getEmbeddedReleaseVersion() {
|
|
102
|
+
if (BUILD_RELEASE_VERSION?.trim())
|
|
103
|
+
return BUILD_RELEASE_VERSION.trim();
|
|
104
|
+
const manifest = decodeBase64UrlJson(BUILD_RELEASE_MANIFEST_BASE64?.trim() || "");
|
|
105
|
+
return typeof manifest?.release_version === "string" && manifest.release_version.trim() ? manifest.release_version.trim() : null;
|
|
106
|
+
}
|
|
107
|
+
function getPackageVersionForModuleDir(moduleDir) {
|
|
108
|
+
let dir = moduleDir;
|
|
109
|
+
const root = parse(dir).root;
|
|
110
|
+
while (true) {
|
|
111
|
+
try {
|
|
112
|
+
const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf-8"));
|
|
113
|
+
return typeof pkg.version === "string" ? pkg.version : "unknown";
|
|
114
|
+
} catch {}
|
|
115
|
+
if (dir === root)
|
|
116
|
+
return "unknown";
|
|
117
|
+
dir = dirname(dir);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function getPackageVersion() {
|
|
121
|
+
const packageVersion = getPackageVersionForModuleDir(MODULE_DIR);
|
|
122
|
+
if (packageVersion !== "unknown")
|
|
123
|
+
return packageVersion;
|
|
124
|
+
return getEmbeddedReleaseVersion() ?? "unknown";
|
|
125
|
+
}
|
|
126
|
+
var MODULE_DIR, CODE_HASH, GIT_SHA, PACKAGE_VERSION, DEFAULT_BACKEND_URL, TRACE_VERSION, RELEASE_MANIFEST_BASE64, RELEASE_MANIFEST_SIGNATURE;
|
|
101
127
|
var init_version = __esm(() => {
|
|
102
128
|
MODULE_DIR = dirname(fileURLToPath(import.meta.url));
|
|
103
129
|
CODE_HASH = BUILD_CODE_HASH?.trim() || computeCodeHash();
|
|
104
130
|
GIT_SHA = getGitSha();
|
|
105
131
|
PACKAGE_VERSION = getPackageVersion();
|
|
132
|
+
DEFAULT_BACKEND_URL = BUILD_DEFAULT_BACKEND_URL?.trim() || "https://beta-api.unbrowse.ai";
|
|
106
133
|
TRACE_VERSION = `${CODE_HASH}@${GIT_SHA}`;
|
|
107
134
|
RELEASE_MANIFEST_BASE64 = BUILD_RELEASE_MANIFEST_BASE64?.trim() || "";
|
|
108
135
|
RELEASE_MANIFEST_SIGNATURE = BUILD_RELEASE_MANIFEST_SIGNATURE?.trim() || "";
|
|
@@ -362,7 +389,7 @@ var init_client = __esm(() => {
|
|
|
362
389
|
init_cascade();
|
|
363
390
|
init_wallet();
|
|
364
391
|
init_telemetry_attribution();
|
|
365
|
-
API_URL2 = process.env.UNBROWSE_BACKEND_URL ||
|
|
392
|
+
API_URL2 = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
|
|
366
393
|
PROFILE_NAME2 = sanitizeProfileName2(process.env.UNBROWSE_PROFILE ?? "");
|
|
367
394
|
recentLocalSkills2 = new Map;
|
|
368
395
|
LOCAL_ONLY2 = process.env.UNBROWSE_LOCAL_ONLY === "1";
|
|
@@ -408,11 +435,12 @@ function readProbabilityEnv(name, fallback) {
|
|
|
408
435
|
const parsed = Number.parseFloat(process.env[name] ?? "");
|
|
409
436
|
return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : fallback;
|
|
410
437
|
}
|
|
411
|
-
var DEFAULT_PUBLISH_ENDPOINT_LIMIT, MIN_PUBLISH_RELIABILITY;
|
|
438
|
+
var DEFAULT_PUBLISH_ENDPOINT_LIMIT, MIN_PUBLISH_RELIABILITY, CLOSURE_EDGE_KINDS;
|
|
412
439
|
var init_publish_admission = __esm(() => {
|
|
413
440
|
init_domain();
|
|
414
441
|
DEFAULT_PUBLISH_ENDPOINT_LIMIT = readPositiveIntEnv("UNBROWSE_PUBLISH_ENDPOINT_LIMIT", 12);
|
|
415
442
|
MIN_PUBLISH_RELIABILITY = readProbabilityEnv("UNBROWSE_PUBLISH_MIN_RELIABILITY", 0.2);
|
|
443
|
+
CLOSURE_EDGE_KINDS = new Set(["dependency", "auth", "parent_child", "pagination"]);
|
|
416
444
|
});
|
|
417
445
|
|
|
418
446
|
// ../../src/logger.ts
|
|
@@ -820,6 +848,11 @@ var init_settings = __esm(() => {
|
|
|
820
848
|
init_domain();
|
|
821
849
|
});
|
|
822
850
|
|
|
851
|
+
// ../../src/publish/schema-review.ts
|
|
852
|
+
var init_schema_review = __esm(() => {
|
|
853
|
+
init_sanitize();
|
|
854
|
+
});
|
|
855
|
+
|
|
823
856
|
// ../../src/indexer/index.ts
|
|
824
857
|
import { join as join6 } from "node:path";
|
|
825
858
|
var SKILL_SNAPSHOT_DIR, indexInFlight, pendingIndexJobs;
|
|
@@ -833,6 +866,8 @@ var init_indexer = __esm(async () => {
|
|
|
833
866
|
init_artifact();
|
|
834
867
|
init_publish();
|
|
835
868
|
init_settings();
|
|
869
|
+
init_graph();
|
|
870
|
+
init_schema_review();
|
|
836
871
|
await init_orchestrator();
|
|
837
872
|
SKILL_SNAPSHOT_DIR = process.env.UNBROWSE_SKILL_SNAPSHOT_DIR ?? join6(process.env.HOME ?? "/tmp", ".unbrowse", "skill-snapshots");
|
|
838
873
|
indexInFlight = new Map;
|
|
@@ -842,7 +877,8 @@ var init_indexer = __esm(async () => {
|
|
|
842
877
|
// ../../src/payments/index.ts
|
|
843
878
|
var PRICING_API_URL;
|
|
844
879
|
var init_payments = __esm(() => {
|
|
845
|
-
|
|
880
|
+
init_version();
|
|
881
|
+
PRICING_API_URL = process.env.UNBROWSE_BACKEND_URL ?? DEFAULT_BACKEND_URL;
|
|
846
882
|
});
|
|
847
883
|
|
|
848
884
|
// ../../src/execution/robots.ts
|
|
@@ -945,8 +981,9 @@ var init_planner = __esm(() => {
|
|
|
945
981
|
// ../../src/client/graph-client.ts
|
|
946
982
|
var API_URL3, GRAPH_TIMEOUT_MS;
|
|
947
983
|
var init_graph_client = __esm(() => {
|
|
984
|
+
init_version();
|
|
948
985
|
init_client();
|
|
949
|
-
API_URL3 = process.env.UNBROWSE_BACKEND_URL ??
|
|
986
|
+
API_URL3 = process.env.UNBROWSE_BACKEND_URL ?? DEFAULT_BACKEND_URL;
|
|
950
987
|
GRAPH_TIMEOUT_MS = parseInt(process.env.UNBROWSE_GRAPH_TIMEOUT_MS ?? "4000", 10);
|
|
951
988
|
});
|
|
952
989
|
|
|
@@ -963,6 +1000,7 @@ var init_dag_feedback = __esm(() => {
|
|
|
963
1000
|
init_client();
|
|
964
1001
|
init_graph_client();
|
|
965
1002
|
init_graph();
|
|
1003
|
+
init_version();
|
|
966
1004
|
SESSION_ID = nanoid7();
|
|
967
1005
|
lastWriteAt = new Map;
|
|
968
1006
|
pendingTimers = new Map;
|
|
@@ -988,11 +1026,6 @@ var init_prefetch = __esm(async () => {
|
|
|
988
1026
|
await init_execution();
|
|
989
1027
|
});
|
|
990
1028
|
|
|
991
|
-
// ../../src/orchestrator/first-pass-action.ts
|
|
992
|
-
var init_first_pass_action = __esm(() => {
|
|
993
|
-
init_client2();
|
|
994
|
-
});
|
|
995
|
-
|
|
996
1029
|
// ../../src/orchestrator/timing-economics.ts
|
|
997
1030
|
var TOKEN_COST_PER_MILLION_USD = 3, TOKEN_COST_UC, SAVINGS_SOURCES;
|
|
998
1031
|
var init_timing_economics = __esm(() => {
|
|
@@ -1029,7 +1062,6 @@ var init_orchestrator = __esm(async () => {
|
|
|
1029
1062
|
init_search_forms();
|
|
1030
1063
|
init_trace_store();
|
|
1031
1064
|
init_passive_publish();
|
|
1032
|
-
init_first_pass_action();
|
|
1033
1065
|
init_timing_economics();
|
|
1034
1066
|
init_payments();
|
|
1035
1067
|
init_wallet();
|
|
@@ -1180,10 +1212,20 @@ import { join as join3 } from "path";
|
|
|
1180
1212
|
import { homedir as homedir2, hostname } from "os";
|
|
1181
1213
|
import { randomBytes, createHash as createHash2 } from "crypto";
|
|
1182
1214
|
import { createInterface } from "readline";
|
|
1183
|
-
var API_URL = process.env.UNBROWSE_BACKEND_URL ||
|
|
1215
|
+
var API_URL = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
|
|
1184
1216
|
var PROFILE_NAME = sanitizeProfileName(process.env.UNBROWSE_PROFILE ?? "");
|
|
1185
1217
|
var recentLocalSkills = new Map;
|
|
1186
1218
|
var LOCAL_ONLY = process.env.UNBROWSE_LOCAL_ONLY === "1";
|
|
1219
|
+
function buildReleaseAttestationHeaders(manifestBase64, signature) {
|
|
1220
|
+
const manifest = manifestBase64.trim();
|
|
1221
|
+
const sig = signature.trim();
|
|
1222
|
+
if (!manifest || !sig)
|
|
1223
|
+
return {};
|
|
1224
|
+
return {
|
|
1225
|
+
"X-Unbrowse-Release-Manifest": manifest,
|
|
1226
|
+
"X-Unbrowse-Release-Signature": sig
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1187
1229
|
function decodeBase64Json(value) {
|
|
1188
1230
|
try {
|
|
1189
1231
|
if (typeof globalThis !== "undefined" && typeof globalThis.atob === "function") {
|
|
@@ -1467,6 +1509,7 @@ async function findUsableApiKey() {
|
|
|
1467
1509
|
}
|
|
1468
1510
|
async function apiRequest(method, path, body, opts) {
|
|
1469
1511
|
const key = opts?.noAuth ? "" : getApiKey();
|
|
1512
|
+
const releaseAttestationHeaders = buildReleaseAttestationHeaders(RELEASE_MANIFEST_BASE64, RELEASE_MANIFEST_SIGNATURE);
|
|
1470
1513
|
const controller = new AbortController;
|
|
1471
1514
|
const timer = setTimeout(() => controller.abort(), opts?.timeoutMs ?? API_TIMEOUT_MS);
|
|
1472
1515
|
let res;
|
|
@@ -1479,8 +1522,7 @@ async function apiRequest(method, path, body, opts) {
|
|
|
1479
1522
|
"X-Unbrowse-Trace-Version": TRACE_VERSION,
|
|
1480
1523
|
"X-Unbrowse-Code-Hash": CODE_HASH,
|
|
1481
1524
|
"X-Unbrowse-Git-Sha": GIT_SHA,
|
|
1482
|
-
...
|
|
1483
|
-
...RELEASE_MANIFEST_SIGNATURE ? { "X-Unbrowse-Release-Signature": RELEASE_MANIFEST_SIGNATURE } : {},
|
|
1525
|
+
...releaseAttestationHeaders,
|
|
1484
1526
|
...key ? { Authorization: `Bearer ${key}` } : {}
|
|
1485
1527
|
},
|
|
1486
1528
|
body: body ? JSON.stringify(body) : undefined,
|
|
@@ -1944,15 +1986,6 @@ function deriveListenEnv(baseUrl) {
|
|
|
1944
1986
|
const port = url.port || (url.protocol === "https:" ? "443" : "80");
|
|
1945
1987
|
return { HOST: host, PORT: port, UNBROWSE_URL: baseUrl };
|
|
1946
1988
|
}
|
|
1947
|
-
function getVersion(metaUrl) {
|
|
1948
|
-
try {
|
|
1949
|
-
const root = getPackageRoot(metaUrl);
|
|
1950
|
-
const pkg = JSON.parse(readFileSync4(path2.join(root, "package.json"), "utf-8"));
|
|
1951
|
-
return pkg.version ?? "unknown";
|
|
1952
|
-
} catch {
|
|
1953
|
-
return "unknown";
|
|
1954
|
-
}
|
|
1955
|
-
}
|
|
1956
1989
|
function isCompiledBinary() {
|
|
1957
1990
|
return !!(process.versions.bun && !process.argv[1]?.match(/\.(ts|js|mjs)$/));
|
|
1958
1991
|
}
|
|
@@ -1998,7 +2031,7 @@ function spawnServer(baseUrl, metaUrl, pidFile, restartCount = 0) {
|
|
|
1998
2031
|
base_url: baseUrl,
|
|
1999
2032
|
started_at: new Date().toISOString(),
|
|
2000
2033
|
entrypoint: spawnSpec.recordedEntrypoint,
|
|
2001
|
-
version:
|
|
2034
|
+
version: PACKAGE_VERSION,
|
|
2002
2035
|
code_hash: CODE_HASH,
|
|
2003
2036
|
restart_count: restartCount
|
|
2004
2037
|
};
|
|
@@ -2007,7 +2040,7 @@ function spawnServer(baseUrl, metaUrl, pidFile, restartCount = 0) {
|
|
|
2007
2040
|
}
|
|
2008
2041
|
var supervisor = new LocalSupervisor;
|
|
2009
2042
|
async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
|
|
2010
|
-
const installedVersion =
|
|
2043
|
+
const installedVersion = PACKAGE_VERSION;
|
|
2011
2044
|
const initialHealth = await fetchServerHealth(baseUrl);
|
|
2012
2045
|
if (initialHealth) {
|
|
2013
2046
|
const runningVersion = initialHealth.package_version;
|
|
@@ -2076,7 +2109,7 @@ function checkServerVersion(baseUrl, metaUrl, healthOverride) {
|
|
|
2076
2109
|
const state = readPidState(pidFile);
|
|
2077
2110
|
if (!state)
|
|
2078
2111
|
return null;
|
|
2079
|
-
const installed =
|
|
2112
|
+
const installed = PACKAGE_VERSION;
|
|
2080
2113
|
const running = healthOverride?.runningVersion ?? state.version ?? "unknown";
|
|
2081
2114
|
const runningCodeHash = healthOverride?.runningCodeHash ?? state.code_hash;
|
|
2082
2115
|
return {
|
|
@@ -2118,6 +2151,9 @@ function resolveSiblingEntrypoint2(metaUrl, basename) {
|
|
|
2118
2151
|
const file = fileURLToPath3(metaUrl);
|
|
2119
2152
|
return path3.join(path3.dirname(file), `${basename}${path3.extname(file) || ".js"}`);
|
|
2120
2153
|
}
|
|
2154
|
+
function isBundledVirtualEntrypoint(entrypoint) {
|
|
2155
|
+
return entrypoint.startsWith("/$bunfs/");
|
|
2156
|
+
}
|
|
2121
2157
|
function runtimeArgsForEntrypoint2(metaUrl, entrypoint) {
|
|
2122
2158
|
if (path3.extname(entrypoint) !== ".ts")
|
|
2123
2159
|
return [entrypoint];
|
|
@@ -2155,6 +2191,8 @@ init_sanitize();
|
|
|
2155
2191
|
init_artifact();
|
|
2156
2192
|
init_publish();
|
|
2157
2193
|
init_settings();
|
|
2194
|
+
init_graph();
|
|
2195
|
+
init_schema_review();
|
|
2158
2196
|
import { join as join8 } from "node:path";
|
|
2159
2197
|
var SKILL_SNAPSHOT_DIR3 = process.env.UNBROWSE_SKILL_SNAPSHOT_DIR ?? join8(process.env.HOME ?? "/tmp", ".unbrowse", "skill-snapshots");
|
|
2160
2198
|
var indexInFlight2 = new Map;
|
|
@@ -2656,9 +2694,9 @@ function loadInstallSource2(metaUrl) {
|
|
|
2656
2694
|
return readJsonFile2(getInstallSourcePath2()) ?? resolveInstallSource2(metaUrl);
|
|
2657
2695
|
}
|
|
2658
2696
|
function compareSemver(a, b) {
|
|
2659
|
-
const
|
|
2660
|
-
const left =
|
|
2661
|
-
const right =
|
|
2697
|
+
const parse2 = (value) => value.split("-", 1)[0].split(".").map((part) => Number.parseInt(part, 10) || 0);
|
|
2698
|
+
const left = parse2(a);
|
|
2699
|
+
const right = parse2(b);
|
|
2662
2700
|
const max = Math.max(left.length, right.length);
|
|
2663
2701
|
for (let i = 0;i < max; i++) {
|
|
2664
2702
|
const diff = (left[i] ?? 0) - (right[i] ?? 0);
|
|
@@ -2807,15 +2845,9 @@ function resolveResultError(result) {
|
|
|
2807
2845
|
function resolveLoginUrl(result, fallbackUrl) {
|
|
2808
2846
|
return result.result?.login_url ?? fallbackUrl ?? "";
|
|
2809
2847
|
}
|
|
2810
|
-
function hasIndexingFallback(result) {
|
|
2811
|
-
return result.result?.indexing_fallback_available === true;
|
|
2812
|
-
}
|
|
2813
2848
|
function isResolveSuccessResult(result) {
|
|
2814
|
-
const resultObj = result.result;
|
|
2815
2849
|
if (resolveResultError(result))
|
|
2816
2850
|
return false;
|
|
2817
|
-
if (resultObj?.status === "browse_session_open")
|
|
2818
|
-
return false;
|
|
2819
2851
|
return !!result.result || Array.isArray(result.available_endpoints);
|
|
2820
2852
|
}
|
|
2821
2853
|
async function withPendingNotice(promise, message, delayMs = 3000) {
|
|
@@ -2997,51 +3029,15 @@ async function cmdResolve(flags) {
|
|
|
2997
3029
|
body.force_capture = true;
|
|
2998
3030
|
body.projection = { raw: true };
|
|
2999
3031
|
const startedAt = Date.now();
|
|
3000
|
-
async function resolveOnce(message = "Still working.
|
|
3032
|
+
async function resolveOnce(message = "Still working. Searching cached routes...") {
|
|
3001
3033
|
return withPendingNotice(api2("POST", "/v1/intent/resolve", body), message);
|
|
3002
3034
|
}
|
|
3003
3035
|
let result = await resolveOnce();
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
if (resultError === "payment_required" && hasIndexingFallback(result) && url && !attemptedForceCapture) {
|
|
3010
|
-
attemptedForceCapture = true;
|
|
3011
|
-
body.force_capture = true;
|
|
3012
|
-
info("Marketplace search is paid here. Falling back to free live capture on the exact URL...");
|
|
3013
|
-
result = await resolveOnce("Running free live capture...");
|
|
3014
|
-
continue;
|
|
3015
|
-
}
|
|
3016
|
-
if (resultError === "auth_required") {
|
|
3017
|
-
const loginUrl = resolveLoginUrl(result, url);
|
|
3018
|
-
if (!loginUrl)
|
|
3019
|
-
break;
|
|
3020
|
-
if (!attemptedCookieImport) {
|
|
3021
|
-
attemptedCookieImport = true;
|
|
3022
|
-
info("Site requires authentication. Trying browser cookie import first...");
|
|
3023
|
-
const stealResult = await api2("POST", "/v1/auth/steal", { url: loginUrl });
|
|
3024
|
-
const cookiesStored = typeof stealResult.cookies_stored === "number" ? stealResult.cookies_stored : Number(stealResult.cookies_stored ?? 0);
|
|
3025
|
-
if (stealResult.success === true && cookiesStored > 0) {
|
|
3026
|
-
info(`Imported ${cookiesStored} browser cookies. Retrying...`);
|
|
3027
|
-
result = await resolveOnce("Retrying after browser cookie import...");
|
|
3028
|
-
continue;
|
|
3029
|
-
}
|
|
3030
|
-
}
|
|
3031
|
-
if (!attemptedInteractiveLogin) {
|
|
3032
|
-
attemptedInteractiveLogin = true;
|
|
3033
|
-
info("Site requires authentication. Opening browser for login...");
|
|
3034
|
-
const loginResult = await api2("POST", "/v1/auth/login", { url: loginUrl });
|
|
3035
|
-
if (loginResult.error || loginResult.success === false) {
|
|
3036
|
-
const message = typeof loginResult.error === "string" ? loginResult.error : "interactive login did not produce a reusable session";
|
|
3037
|
-
throw new Error(`Login failed: ${message}. Run: unbrowse login --url "${loginUrl}"`);
|
|
3038
|
-
}
|
|
3039
|
-
info("Login complete. Retrying...");
|
|
3040
|
-
result = await resolveOnce("Retrying after login...");
|
|
3041
|
-
continue;
|
|
3042
|
-
}
|
|
3043
|
-
}
|
|
3044
|
-
break;
|
|
3036
|
+
const resultError = resolveResultError(result);
|
|
3037
|
+
if (resultError === "auth_required") {
|
|
3038
|
+
const loginUrl = resolveLoginUrl(result, url);
|
|
3039
|
+
if (loginUrl)
|
|
3040
|
+
info(`Authentication required. Run: unbrowse login --url "${loginUrl}"`);
|
|
3045
3041
|
}
|
|
3046
3042
|
if (explicitEndpointId && result.available_endpoints) {
|
|
3047
3043
|
const skillId = resolveSkillId();
|
|
@@ -3063,26 +3059,6 @@ async function cmdResolve(flags) {
|
|
|
3063
3059
|
result = await withPendingNotice(api2("POST", `/v1/skills/${skillId}/execute`, execBody(bestEndpoint.endpoint_id)), "Executing best endpoint...");
|
|
3064
3060
|
}
|
|
3065
3061
|
}
|
|
3066
|
-
const resultObj = result.result;
|
|
3067
|
-
if (resultObj?.status === "browse_session_open") {
|
|
3068
|
-
info(`No cached API. Browser session open on ${resultObj.domain ?? resultObj.url}.`);
|
|
3069
|
-
info(`Preferred flow: snap -> click/fill/eval -> submit -> sync -> close.`);
|
|
3070
|
-
info(`Use these commands to get your data:`);
|
|
3071
|
-
const commands = resultObj.commands ?? [
|
|
3072
|
-
resultObj.session_id ? `unbrowse snap --session ${resultObj.session_id} --filter interactive` : "unbrowse snap --filter interactive",
|
|
3073
|
-
resultObj.session_id ? `unbrowse click --session ${resultObj.session_id} <ref>` : "unbrowse click <ref>",
|
|
3074
|
-
resultObj.session_id ? `unbrowse fill --session ${resultObj.session_id} <ref> <value>` : "unbrowse fill <ref> <value>",
|
|
3075
|
-
resultObj.session_id ? `unbrowse submit --session ${resultObj.session_id} --wait-for "/next-step"` : 'unbrowse submit --wait-for "/next-step"',
|
|
3076
|
-
resultObj.session_id ? `unbrowse sync --session ${resultObj.session_id}` : "unbrowse sync",
|
|
3077
|
-
resultObj.session_id ? `unbrowse close --session ${resultObj.session_id}` : "unbrowse close"
|
|
3078
|
-
];
|
|
3079
|
-
for (const cmd of commands)
|
|
3080
|
-
info(` ${cmd}`);
|
|
3081
|
-
info(`For JS-heavy forms: prefer real date/time clicks first, inspect hidden inputs with eval when needed, then submit.`);
|
|
3082
|
-
info(`All traffic is being passively captured. Run "unbrowse close" when done.`);
|
|
3083
|
-
output(slimTrace(result), !!flags.pretty);
|
|
3084
|
-
return;
|
|
3085
|
-
}
|
|
3086
3062
|
if (Date.now() - startedAt > 3000 && result.source === "live-capture") {
|
|
3087
3063
|
info("Live capture finished. Future runs against this site should be much faster.");
|
|
3088
3064
|
}
|
|
@@ -3293,7 +3269,7 @@ async function cmdExecute(flags) {
|
|
|
3293
3269
|
...result.next_actions ? { next_actions: result.next_actions } : {}
|
|
3294
3270
|
};
|
|
3295
3271
|
if (trace?.skill_id && trace?.endpoint_id && limited.length > 0) {
|
|
3296
|
-
out._review_hint = `After presenting results,
|
|
3272
|
+
out._review_hint = `After presenting results, review this endpoint's contract: description plus request/response schema notes where needed. Example: unbrowse review --skill ${trace.skill_id} --endpoints '[{"endpoint_id":"${trace.endpoint_id}","description":"DESCRIBE WHAT THIS RETURNS AND ANY IMPORTANT CONSTRAINTS","action_kind":"ACTION","resource_kind":"RESOURCE","parameter_reviews":[{"location":"query","name":"q","description":"Search query text","type":"string","required":true}],"response_reviews":[{"path":"items[].url","description":"Canonical result URL","type":"string"}]}]'`;
|
|
3297
3273
|
}
|
|
3298
3274
|
output(out, !!flags.pretty);
|
|
3299
3275
|
return;
|
|
@@ -3357,7 +3333,7 @@ async function cmdReview(flags) {
|
|
|
3357
3333
|
die("--skill is required");
|
|
3358
3334
|
const endpointsJson = flags.endpoints;
|
|
3359
3335
|
if (!endpointsJson)
|
|
3360
|
-
die("--endpoints is required (JSON array of {endpoint_id, description?, action_kind?, resource_kind?})");
|
|
3336
|
+
die("--endpoints is required (JSON array of {endpoint_id, description?, action_kind?, resource_kind?, parameter_reviews?, response_reviews?})");
|
|
3361
3337
|
const endpoints = JSON.parse(endpointsJson);
|
|
3362
3338
|
if (!Array.isArray(endpoints) || endpoints.length === 0)
|
|
3363
3339
|
die("--endpoints must be a non-empty JSON array");
|
|
@@ -3395,6 +3371,17 @@ async function cmdPublish(flags) {
|
|
|
3395
3371
|
}), !!flags.pretty);
|
|
3396
3372
|
}
|
|
3397
3373
|
}
|
|
3374
|
+
async function cmdPublishBundle(flags) {
|
|
3375
|
+
const presetPath = flags.preset;
|
|
3376
|
+
if (!presetPath)
|
|
3377
|
+
die("--preset is required");
|
|
3378
|
+
const hosts = typeof flags.hosts === "string" ? flags.hosts.split(",").map((host) => host.trim()).filter(Boolean) : undefined;
|
|
3379
|
+
output(await api2("POST", "/v1/foundry/publish-bundle", {
|
|
3380
|
+
preset_path: presetPath,
|
|
3381
|
+
...typeof flags["site-url"] === "string" ? { site_url: flags["site-url"] } : {},
|
|
3382
|
+
...hosts?.length ? { hosts } : {}
|
|
3383
|
+
}), !!flags.pretty);
|
|
3384
|
+
}
|
|
3398
3385
|
async function cmdSettings(flags) {
|
|
3399
3386
|
const body = {};
|
|
3400
3387
|
if (typeof flags["auto-publish"] === "string") {
|
|
@@ -3442,61 +3429,7 @@ async function cmdCleanupStale(flags) {
|
|
|
3442
3429
|
output(await withPendingNotice(api2("POST", "/v1/stale/cleanup", body), "Cleaning stale endpoints..."), !!flags.pretty);
|
|
3443
3430
|
}
|
|
3444
3431
|
async function cmdSearch(flags) {
|
|
3445
|
-
|
|
3446
|
-
if (!intent)
|
|
3447
|
-
die("--intent is required");
|
|
3448
|
-
const domain = flags.domain;
|
|
3449
|
-
const path9 = domain ? "/v1/search/domain" : "/v1/search";
|
|
3450
|
-
const body = { intent, k: Number(flags.k) || 5 };
|
|
3451
|
-
if (domain)
|
|
3452
|
-
body.domain = domain;
|
|
3453
|
-
const hostType = detectTelemetryHostType();
|
|
3454
|
-
await ensureCliInstallTracked(hostType);
|
|
3455
|
-
await recordFunnelTelemetryEvent("cli_invoked", {
|
|
3456
|
-
source: "cli",
|
|
3457
|
-
hostType,
|
|
3458
|
-
properties: { command: "search" }
|
|
3459
|
-
});
|
|
3460
|
-
await recordFunnelTelemetryEvent("search_started", {
|
|
3461
|
-
source: "cli",
|
|
3462
|
-
hostType,
|
|
3463
|
-
properties: {
|
|
3464
|
-
command: "search",
|
|
3465
|
-
intent,
|
|
3466
|
-
domain: domain ?? null,
|
|
3467
|
-
k: body.k
|
|
3468
|
-
}
|
|
3469
|
-
});
|
|
3470
|
-
try {
|
|
3471
|
-
const result = await api2("POST", path9, body);
|
|
3472
|
-
const results = Array.isArray(result.results) ? result.results : [];
|
|
3473
|
-
await recordFunnelTelemetryEvent("search_completed", {
|
|
3474
|
-
source: "cli",
|
|
3475
|
-
hostType,
|
|
3476
|
-
properties: {
|
|
3477
|
-
command: "search",
|
|
3478
|
-
intent,
|
|
3479
|
-
domain: domain ?? null,
|
|
3480
|
-
k: body.k,
|
|
3481
|
-
result_count: results.length
|
|
3482
|
-
}
|
|
3483
|
-
});
|
|
3484
|
-
output(result, !!flags.pretty);
|
|
3485
|
-
} catch (error) {
|
|
3486
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
3487
|
-
await recordFunnelTelemetryEvent("search_failed", {
|
|
3488
|
-
source: "cli",
|
|
3489
|
-
hostType,
|
|
3490
|
-
properties: {
|
|
3491
|
-
command: "search",
|
|
3492
|
-
intent,
|
|
3493
|
-
domain: domain ?? null,
|
|
3494
|
-
failure_stage: "search",
|
|
3495
|
-
failure_reason: message
|
|
3496
|
-
}
|
|
3497
|
-
});
|
|
3498
|
-
throw error;
|
|
3499
|
-
}
|
|
3432
|
+
await cmdResolve(flags);
|
|
3500
3433
|
}
|
|
3501
3434
|
async function cmdSessions(flags) {
|
|
3502
3435
|
const domain = flags.domain;
|
|
@@ -3595,20 +3528,20 @@ var CLI_REFERENCE = {
|
|
|
3595
3528
|
{ name: "mcp", usage: "[--no-auto-start]", desc: "Run the stdio MCP server" },
|
|
3596
3529
|
{ name: "setup", usage: "[--opencode auto|global|project|off] [--no-start]", desc: "Bootstrap browser deps + Open Code command" },
|
|
3597
3530
|
{ name: "upgrade", usage: "", desc: "Check latest release and print the right upgrade command" },
|
|
3598
|
-
{ name: "resolve", usage: '--intent "..." --url "..." [opts]', desc: "
|
|
3531
|
+
{ name: "resolve", usage: '--intent "..." [--domain "..."] [--url "..."] [opts]', desc: "Search cached indexed/published routes and optionally execute the top trusted endpoint" },
|
|
3599
3532
|
{ name: "execute", usage: "--skill ID --endpoint ID [opts]", desc: "Execute a specific endpoint" },
|
|
3600
3533
|
{ name: "feedback", usage: "--skill ID --endpoint ID --rating N", desc: "Submit feedback (mandatory after resolve)" },
|
|
3601
|
-
{ name: "review", usage: "--skill ID --endpoints '[...]'", desc: "Push reviewed descriptions/metadata back to skill" },
|
|
3534
|
+
{ name: "review", usage: "--skill ID --endpoints '[...]'", desc: "Push reviewed descriptions/schema metadata back to a captured skill before publish" },
|
|
3602
3535
|
{ name: "index", usage: "--skill ID", desc: "Recompute local graph/contracts/export from cached skill state only" },
|
|
3603
|
-
{ name: "publish", usage: "--skill ID [--confirm-publish] [--endpoints '[...]']", desc: "Re-index locally, then publish/share from cached skill state
|
|
3536
|
+
{ name: "publish", usage: "--skill ID [--confirm-publish] [--endpoints '[...]']", desc: "Re-index locally, inspect publish-review metadata, then publish/share from cached skill state" },
|
|
3537
|
+
{ name: "publish-bundle", usage: "--preset path [--hosts codex,claude,openclaw] [--site-url https://www.unbrowse.ai]", desc: "Derive foundry bundle/share/host artifacts from one preset and write the public share manifest" },
|
|
3604
3538
|
{ name: "settings", usage: "[--auto-publish on|off] [--publish-blacklist domains] [--publish-promptlist domains]", desc: "Show or update local capture/publish policy settings" },
|
|
3605
3539
|
{ name: "login", usage: '--url "..."', desc: "Interactive browser login" },
|
|
3606
3540
|
{ name: "skills", usage: "", desc: "List all skills" },
|
|
3607
3541
|
{ name: "skill", usage: "<id>", desc: "Get skill details" },
|
|
3608
3542
|
{ name: "cleanup-stale", usage: "[--skill ID] [--domain host] [--limit N]", desc: "Verify skills and evict stale cached endpoints" },
|
|
3609
|
-
{ name: "search", usage: '--intent "..." [--domain "..."]', desc: "Search marketplace" },
|
|
3610
3543
|
{ name: "sessions", usage: '--domain "..." [--limit N]', desc: "Debug session logs" },
|
|
3611
|
-
{ name: "go", usage: "<url> [--session id]", desc: "Open a
|
|
3544
|
+
{ name: "go", usage: "<url> [--session id]", desc: "Open a fresh Kuri browser tab, or reuse explicit --session" },
|
|
3612
3545
|
{ name: "submit", usage: "[--session id] [--form-selector sel] [--submit-selector sel] [--wait-for hint] [--assist-site-state]", desc: "Submit current form. Thin browser-native proxy by default; site-state assist and same-origin rehydrate are explicit opt-ins" },
|
|
3613
3546
|
{ name: "snap", usage: "[--session id] [--filter interactive]", desc: "A11y snapshot with @eN refs" },
|
|
3614
3547
|
{ name: "click", usage: "[--session id] <ref>", desc: "Click element by ref (e.g. e5)" },
|
|
@@ -3624,8 +3557,8 @@ var CLI_REFERENCE = {
|
|
|
3624
3557
|
{ name: "eval", usage: "[--session id] <expression>", desc: "Evaluate JavaScript" },
|
|
3625
3558
|
{ name: "back", usage: "[--session id]", desc: "Navigate back" },
|
|
3626
3559
|
{ name: "forward", usage: "[--session id]", desc: "Navigate forward" },
|
|
3627
|
-
{ name: "sync", usage: "[--session id]", desc: "Checkpoint current capture, keep tab open, queue background index + publish" },
|
|
3628
|
-
{ name: "close", usage: "[--session id]", desc: "Checkpoint capture, queue background index + publish,
|
|
3560
|
+
{ name: "sync", usage: "[--session id]", desc: "Checkpoint current capture, keep tab open, queue background index + publish, then inspect via skill/publish review" },
|
|
3561
|
+
{ name: "close", usage: "[--session id]", desc: "Checkpoint capture, queue background index + publish, close browse session, then inspect via skill/publish review" }
|
|
3629
3562
|
],
|
|
3630
3563
|
globalFlags: [
|
|
3631
3564
|
{ flag: "--pretty", desc: "Indented JSON output" },
|
|
@@ -3635,19 +3568,19 @@ var CLI_REFERENCE = {
|
|
|
3635
3568
|
{ flag: "--opencode auto|global|project|off", desc: "setup: install /unbrowse command for Open Code" }
|
|
3636
3569
|
],
|
|
3637
3570
|
resolveExecuteFlags: [
|
|
3571
|
+
{ flag: "--execute", desc: "Auto-execute the top trusted endpoint from resolve" },
|
|
3638
3572
|
{ flag: "--schema", desc: "Show response schema + extraction hints only (no data)" },
|
|
3639
3573
|
{ flag: '--path "data.items[]"', desc: "Drill into result before extract/output" },
|
|
3640
3574
|
{ flag: '--extract "field1,alias:deep.path.to.val"', desc: "Pick specific fields (no piping needed)" },
|
|
3641
3575
|
{ flag: "--limit N", desc: "Cap array output to N items" },
|
|
3642
3576
|
{ flag: "--endpoint-id ID", desc: "Pick a specific endpoint" },
|
|
3643
3577
|
{ flag: "--dry-run", desc: "Preview mutations" },
|
|
3644
|
-
{ flag: "--force-capture", desc: "Bypass caches, re-capture" },
|
|
3645
3578
|
{ flag: "--params '{...}'", desc: "Extra params as JSON" }
|
|
3646
3579
|
],
|
|
3647
3580
|
examples: [
|
|
3648
3581
|
"unbrowse setup",
|
|
3649
3582
|
"unbrowse mcp",
|
|
3650
|
-
'unbrowse resolve --intent "top stories" --url "https://news.ycombinator.com" --execute',
|
|
3583
|
+
'unbrowse resolve --intent "top stories" --domain "news.ycombinator.com" --url "https://news.ycombinator.com" --execute',
|
|
3651
3584
|
'unbrowse resolve --intent "get timeline" --url "https://x.com"',
|
|
3652
3585
|
'unbrowse go "https://www.mandai.com/en/ticketing/admission-and-rides/parks-selection.html"',
|
|
3653
3586
|
"unbrowse snap --filter interactive",
|
|
@@ -3657,10 +3590,11 @@ var CLI_REFERENCE = {
|
|
|
3657
3590
|
"unbrowse execute --skill abc --endpoint def --schema --pretty",
|
|
3658
3591
|
'unbrowse execute --skill abc --endpoint def --path "data.items[]" --extract "name,url" --limit 10 --pretty',
|
|
3659
3592
|
"unbrowse feedback --skill abc --endpoint def --rating 5",
|
|
3660
|
-
`unbrowse review --skill abc --endpoints '[{"endpoint_id":"def","description":"..."}]'`,
|
|
3593
|
+
`unbrowse review --skill abc --endpoints '[{"endpoint_id":"def","description":"...","parameter_reviews":[{"location":"query","name":"q","description":"Search query","type":"string","required":true}],"response_reviews":[{"path":"items[].title","description":"Listing title","type":"string"}]}]'`,
|
|
3661
3594
|
"unbrowse index --skill abc --pretty",
|
|
3662
3595
|
"unbrowse publish --skill abc --pretty",
|
|
3663
3596
|
"unbrowse publish --skill abc --confirm-publish --pretty",
|
|
3597
|
+
"unbrowse publish-bundle --preset skills/x-account-operator/foundry-preset.json --pretty",
|
|
3664
3598
|
'unbrowse settings --auto-publish off --publish-blacklist "linkedin.com,x.com" --publish-promptlist "github.com" --pretty',
|
|
3665
3599
|
`unbrowse publish --skill abc --endpoints '[{"endpoint_id":"def","description":"Search court judgments by keywords","action_kind":"search","resource_kind":"judgment"}]'`
|
|
3666
3600
|
]
|
|
@@ -3688,7 +3622,7 @@ function printHelp() {
|
|
|
3688
3622
|
for (const e of r.examples) {
|
|
3689
3623
|
lines.push(` ${e}`);
|
|
3690
3624
|
}
|
|
3691
|
-
lines.push("", "Browser workflow:", " 1. go -> open the live tab you want to work in", " 2. snap -> inspect refs and confirm the page state", " 3. click/fill/eval -> set real page state", " 4. submit -> prefer DOM submit; keep traversal browser-native; opt into same-origin rehydrate only for explicit replay/recovery debugging", " 5. sync -> checkpoint the current step and queue background index + publish", " 6. close -> final checkpoint + queue background index + publish, then close the session");
|
|
3625
|
+
lines.push("", "Browser workflow:", " 1. go -> open the live tab you want to work in", " 2. snap -> inspect refs and confirm the page state", " 3. click/fill/eval -> set real page state", " 4. submit -> prefer DOM submit; keep traversal browser-native; opt into same-origin rehydrate only for explicit replay/recovery debugging", " 5. sync -> checkpoint the current step and queue background index + publish", " 6. close -> final checkpoint + queue background index + publish, then close the session", " 7. skill/publish --pretty -> inspect the fresh captured endpoints and review context", " 8. review/publish -> annotate and share the contract; resolve is for later reuse, not first-pass capture validation");
|
|
3692
3626
|
lines.push("", "JS-heavy forms:", " Prefer real calendar/time clicks before submit.", " If the UI is flaky, inspect hidden inputs/cookies with eval, then submit the real form.");
|
|
3693
3627
|
lines.push("");
|
|
3694
3628
|
process.stderr.write(lines.join(`
|
|
@@ -3745,7 +3679,8 @@ async function cmdUpgrade(flags) {
|
|
|
3745
3679
|
}
|
|
3746
3680
|
async function cmdMcp(flags) {
|
|
3747
3681
|
const entrypoint = resolveSiblingEntrypoint2(import.meta.url, "mcp");
|
|
3748
|
-
const
|
|
3682
|
+
const childArgs = isBundledVirtualEntrypoint(entrypoint) ? ["mcp-serve", ...flags["no-auto-start"] ? ["--no-auto-start"] : []] : [...runtimeArgsForEntrypoint2(import.meta.url, entrypoint), ...flags["no-auto-start"] ? ["--no-auto-start"] : []];
|
|
3683
|
+
const child = spawn3(process.execPath, childArgs, {
|
|
3749
3684
|
cwd: process.cwd(),
|
|
3750
3685
|
stdio: "inherit",
|
|
3751
3686
|
env: {
|
|
@@ -4097,6 +4032,7 @@ async function main() {
|
|
|
4097
4032
|
"review",
|
|
4098
4033
|
"index",
|
|
4099
4034
|
"publish",
|
|
4035
|
+
"publish-bundle",
|
|
4100
4036
|
"settings",
|
|
4101
4037
|
"login",
|
|
4102
4038
|
"skills",
|
|
@@ -4169,6 +4105,8 @@ async function main() {
|
|
|
4169
4105
|
return cmdIndex(flags);
|
|
4170
4106
|
case "publish":
|
|
4171
4107
|
return cmdPublish(flags);
|
|
4108
|
+
case "publish-bundle":
|
|
4109
|
+
return cmdPublishBundle(flags);
|
|
4172
4110
|
case "settings":
|
|
4173
4111
|
return cmdSettings(flags);
|
|
4174
4112
|
case "login":
|