unbrowse 3.6.0-preview.0 → 3.7.0-preview.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3 -21
- package/dist/mcp.js +4 -4
- package/dist/server.js +90 -40
- package/package.json +3 -1
package/dist/cli.js
CHANGED
|
@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
|
|
|
31
31
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
32
32
|
|
|
33
33
|
// ../../src/build-info.generated.ts
|
|
34
|
-
var BUILD_RELEASE_VERSION = "3.
|
|
34
|
+
var BUILD_RELEASE_VERSION = "3.7.0-preview.1", BUILD_GIT_SHA = "b3ecde042c51", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
|
|
35
35
|
|
|
36
36
|
// ../../src/version.ts
|
|
37
37
|
import { createHash } from "crypto";
|
|
@@ -2630,18 +2630,12 @@ async function autonomousLogin(loginUrl, domain) {
|
|
|
2630
2630
|
return { success: false, method: "failed", domain: targetDomain, cookies_stored: 0, error: "No email provider configured (set AGENTMAIL_API_KEY or configure Gmail via gcloud)", duration_ms: elapsed() };
|
|
2631
2631
|
}
|
|
2632
2632
|
log("autonomous-login", `starting for ${targetDomain} — url: ${loginUrl}`);
|
|
2633
|
-
const prevHeadless = process.env.HEADLESS;
|
|
2634
|
-
process.env.HEADLESS = "false";
|
|
2635
2633
|
let tabId;
|
|
2636
2634
|
try {
|
|
2637
2635
|
await start();
|
|
2638
2636
|
tabId = await getDefaultTab();
|
|
2639
2637
|
await networkEnable(tabId);
|
|
2640
2638
|
} catch (err) {
|
|
2641
|
-
if (prevHeadless !== undefined)
|
|
2642
|
-
process.env.HEADLESS = prevHeadless;
|
|
2643
|
-
else
|
|
2644
|
-
delete process.env.HEADLESS;
|
|
2645
2639
|
return { success: false, method: "failed", domain: targetDomain, cookies_stored: 0, error: `Kuri start failed: ${err}`, duration_ms: elapsed() };
|
|
2646
2640
|
}
|
|
2647
2641
|
try {
|
|
@@ -2730,12 +2724,7 @@ async function autonomousLogin(loginUrl, domain) {
|
|
|
2730
2724
|
};
|
|
2731
2725
|
}
|
|
2732
2726
|
return { success: false, method: "failed", domain: targetDomain, email: agentEmail, cookies_stored: 0, error: "login flow completed but no auth cookies detected", duration_ms: elapsed() };
|
|
2733
|
-
} finally {
|
|
2734
|
-
if (prevHeadless !== undefined)
|
|
2735
|
-
process.env.HEADLESS = prevHeadless;
|
|
2736
|
-
else
|
|
2737
|
-
delete process.env.HEADLESS;
|
|
2738
|
-
}
|
|
2727
|
+
} finally {}
|
|
2739
2728
|
}
|
|
2740
2729
|
var OTP_FILL_TIMEOUT_MS = 120000, POST_SUBMIT_SETTLE_MS = 3000;
|
|
2741
2730
|
var init_autonomous_login = __esm(async () => {
|
|
@@ -3374,8 +3363,6 @@ import fs2 from "node:fs";
|
|
|
3374
3363
|
import path9 from "node:path";
|
|
3375
3364
|
async function bootstrapAgentMailKey() {
|
|
3376
3365
|
log("bootstrap-agentmail", "starting — opening console.agentmail.to");
|
|
3377
|
-
const prevHeadless = process.env.HEADLESS;
|
|
3378
|
-
process.env.HEADLESS = "false";
|
|
3379
3366
|
try {
|
|
3380
3367
|
await start();
|
|
3381
3368
|
const tabId = await getDefaultTab();
|
|
@@ -3496,12 +3483,7 @@ async function bootstrapAgentMailKey() {
|
|
|
3496
3483
|
success: false,
|
|
3497
3484
|
error: "Reached dashboard but could not extract API key. Create one manually at https://console.agentmail.to/dashboard/api-keys"
|
|
3498
3485
|
};
|
|
3499
|
-
} finally {
|
|
3500
|
-
if (prevHeadless !== undefined)
|
|
3501
|
-
process.env.HEADLESS = prevHeadless;
|
|
3502
|
-
else
|
|
3503
|
-
delete process.env.HEADLESS;
|
|
3504
|
-
}
|
|
3486
|
+
} finally {}
|
|
3505
3487
|
}
|
|
3506
3488
|
function persistApiKeyToShell(apiKey) {
|
|
3507
3489
|
const shell = process.env.SHELL ?? "/bin/zsh";
|
package/dist/mcp.js
CHANGED
|
@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
|
|
|
225
225
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
226
226
|
|
|
227
227
|
// ../../src/build-info.generated.ts
|
|
228
|
-
var BUILD_RELEASE_VERSION = "3.
|
|
229
|
-
var BUILD_GIT_SHA = "
|
|
228
|
+
var BUILD_RELEASE_VERSION = "3.7.0-preview.1";
|
|
229
|
+
var BUILD_GIT_SHA = "b3ecde042c51";
|
|
230
230
|
var BUILD_CODE_HASH = "5d9ebf619c61";
|
|
231
|
-
var BUILD_RELEASE_MANIFEST_BASE64 = "
|
|
232
|
-
var BUILD_RELEASE_MANIFEST_SIGNATURE = "
|
|
231
|
+
var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0";
|
|
232
|
+
var BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ";
|
|
233
233
|
var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
|
|
234
234
|
|
|
235
235
|
// ../../src/version.ts
|
package/dist/server.js
CHANGED
|
@@ -3827,6 +3827,13 @@ __export(exports_reverse_engineer, {
|
|
|
3827
3827
|
extractAuthHeaders: () => extractAuthHeaders
|
|
3828
3828
|
});
|
|
3829
3829
|
import { nanoid as nanoid2 } from "nanoid";
|
|
3830
|
+
import { createHash } from "node:crypto";
|
|
3831
|
+
function stableEndpointId(method, urlTemplate) {
|
|
3832
|
+
if (!method || !urlTemplate)
|
|
3833
|
+
return nanoid2();
|
|
3834
|
+
const hash = createHash("sha256").update(`${method}:${urlTemplate}`).digest("base64url");
|
|
3835
|
+
return hash.slice(0, 21);
|
|
3836
|
+
}
|
|
3830
3837
|
function extractGraphQLOperationName(url, requestBody) {
|
|
3831
3838
|
const urlMatch = url.match(/\/graphql\/[^/]+\/(\w+)/);
|
|
3832
3839
|
if (urlMatch)
|
|
@@ -4447,10 +4454,11 @@ function extractEndpoints(requests, wsMessages, context) {
|
|
|
4447
4454
|
});
|
|
4448
4455
|
const csrfPlan = inferCsrfPlan(req, parsedRequestBody);
|
|
4449
4456
|
const endpointGraphqlOp = /graphql/i.test(req.url) ? extractGraphQLOperationName(req.url, req.request_body) : undefined;
|
|
4457
|
+
const computedUrlTemplate = qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate;
|
|
4450
4458
|
let endpoint = {
|
|
4451
|
-
endpoint_id:
|
|
4459
|
+
endpoint_id: stableEndpointId(req.method, computedUrlTemplate),
|
|
4452
4460
|
method: req.method,
|
|
4453
|
-
url_template:
|
|
4461
|
+
url_template: computedUrlTemplate,
|
|
4454
4462
|
description: buildEndpointDescription(req, sampleRequest, sampleResponse),
|
|
4455
4463
|
headers_template: sanitizeHeaders(req.request_headers),
|
|
4456
4464
|
query: sanitizedQParams,
|
|
@@ -4541,7 +4549,7 @@ function extractEndpoints(requests, wsMessages, context) {
|
|
|
4541
4549
|
response_schema = inferSchema(jsonSamples);
|
|
4542
4550
|
}
|
|
4543
4551
|
const endpoint = {
|
|
4544
|
-
endpoint_id:
|
|
4552
|
+
endpoint_id: stableEndpointId("WS", wsUrl),
|
|
4545
4553
|
method: "WS",
|
|
4546
4554
|
url_template: wsUrl,
|
|
4547
4555
|
idempotency: "safe",
|
|
@@ -7096,10 +7104,10 @@ var init_capture = __esm(async () => {
|
|
|
7096
7104
|
});
|
|
7097
7105
|
|
|
7098
7106
|
// ../../src/build-info.generated.ts
|
|
7099
|
-
var BUILD_RELEASE_VERSION = "3.
|
|
7107
|
+
var BUILD_RELEASE_VERSION = "3.7.0-preview.1", BUILD_GIT_SHA = "b3ecde042c51", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
|
|
7100
7108
|
|
|
7101
7109
|
// ../../src/version.ts
|
|
7102
|
-
import { createHash } from "crypto";
|
|
7110
|
+
import { createHash as createHash2 } from "crypto";
|
|
7103
7111
|
import { existsSync as existsSync4, readFileSync as readFileSync2, readdirSync } from "fs";
|
|
7104
7112
|
import { dirname, join as join3, parse } from "path";
|
|
7105
7113
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
@@ -7116,7 +7124,7 @@ function collectTsFiles(dir) {
|
|
|
7116
7124
|
return results;
|
|
7117
7125
|
}
|
|
7118
7126
|
function hashFiles(srcDir, files) {
|
|
7119
|
-
const hash =
|
|
7127
|
+
const hash = createHash2("sha256");
|
|
7120
7128
|
for (const file of files) {
|
|
7121
7129
|
hash.update(file.slice(srcDir.length));
|
|
7122
7130
|
hash.update(readFileSync2(file, "utf-8"));
|
|
@@ -7156,7 +7164,7 @@ function computeCodeHash() {
|
|
|
7156
7164
|
} catch {}
|
|
7157
7165
|
const pkgVersion = getPackageVersion();
|
|
7158
7166
|
if (pkgVersion !== "unknown") {
|
|
7159
|
-
return
|
|
7167
|
+
return createHash2("sha256").update(`package:${pkgVersion}`).digest("hex").slice(0, 12);
|
|
7160
7168
|
}
|
|
7161
7169
|
return "compiled";
|
|
7162
7170
|
}
|
|
@@ -7210,13 +7218,13 @@ var init_version = __esm(() => {
|
|
|
7210
7218
|
});
|
|
7211
7219
|
|
|
7212
7220
|
// ../../src/payments/cascade.ts
|
|
7213
|
-
import { createHash as
|
|
7221
|
+
import { createHash as createHash3 } from "crypto";
|
|
7214
7222
|
import bs58 from "bs58";
|
|
7215
7223
|
function payableContributors(skill) {
|
|
7216
7224
|
return (skill.contributors ?? []).filter((c) => !!c.wallet_address?.trim());
|
|
7217
7225
|
}
|
|
7218
7226
|
function cascadeLabel(skillId) {
|
|
7219
|
-
const digest =
|
|
7227
|
+
const digest = createHash3("sha256").update(skillId).digest("hex");
|
|
7220
7228
|
return `ubr-${digest.slice(0, 23)}`;
|
|
7221
7229
|
}
|
|
7222
7230
|
function decodeSecretKey(raw) {
|
|
@@ -7610,7 +7618,7 @@ __export(exports_client2, {
|
|
|
7610
7618
|
import { readFileSync as readFileSync4, writeFileSync as writeFileSync3, existsSync as existsSync7, mkdirSync as mkdirSync4, readdirSync as readdirSync2, unlinkSync } from "fs";
|
|
7611
7619
|
import { join as join6 } from "path";
|
|
7612
7620
|
import { homedir as homedir4, hostname, release as osRelease } from "os";
|
|
7613
|
-
import { randomBytes as randomBytes2, createHash as
|
|
7621
|
+
import { randomBytes as randomBytes2, createHash as createHash4 } from "crypto";
|
|
7614
7622
|
import { createInterface } from "readline";
|
|
7615
7623
|
import { execSync } from "child_process";
|
|
7616
7624
|
function buildReleaseAttestationHeaders(manifestBase64, signature) {
|
|
@@ -7887,7 +7895,7 @@ function getApiKey() {
|
|
|
7887
7895
|
function hashApiKey(key) {
|
|
7888
7896
|
if (!key || key === "local-only")
|
|
7889
7897
|
return "";
|
|
7890
|
-
return
|
|
7898
|
+
return createHash4("sha256").update(key).digest("hex");
|
|
7891
7899
|
}
|
|
7892
7900
|
function getAgentId() {
|
|
7893
7901
|
const config = loadConfig();
|
|
@@ -9069,7 +9077,7 @@ var init_publish_admission = __esm(() => {
|
|
|
9069
9077
|
});
|
|
9070
9078
|
|
|
9071
9079
|
// ../../src/telemetry.ts
|
|
9072
|
-
import { createHash as
|
|
9080
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
9073
9081
|
import { existsSync as existsSync8, mkdirSync as mkdirSync5, writeFileSync as writeFileSync4 } from "node:fs";
|
|
9074
9082
|
import { join as join7 } from "node:path";
|
|
9075
9083
|
function getTraceDir() {
|
|
@@ -9079,7 +9087,7 @@ function isTracingEnabled() {
|
|
|
9079
9087
|
return process.env.UNBROWSE_DISABLE_TRACES !== "1";
|
|
9080
9088
|
}
|
|
9081
9089
|
function hashValue(value) {
|
|
9082
|
-
return
|
|
9090
|
+
return createHash5("sha256").update(value).digest("hex").slice(0, 16);
|
|
9083
9091
|
}
|
|
9084
9092
|
function isSensitiveName(name) {
|
|
9085
9093
|
return SENSITIVE_PATTERNS.some((re) => re.test(name));
|
|
@@ -9097,7 +9105,7 @@ function anonymizeUrl(url) {
|
|
|
9097
9105
|
}
|
|
9098
9106
|
function hashResponseBody(result) {
|
|
9099
9107
|
const str = typeof result === "string" ? result : JSON.stringify(result ?? "");
|
|
9100
|
-
return
|
|
9108
|
+
return createHash5("sha256").update(str).digest("hex").slice(0, 32);
|
|
9101
9109
|
}
|
|
9102
9110
|
function classifyFailure(error) {
|
|
9103
9111
|
if (!error)
|
|
@@ -10893,18 +10901,12 @@ async function autonomousLogin(loginUrl, domain) {
|
|
|
10893
10901
|
return { success: false, method: "failed", domain: targetDomain, cookies_stored: 0, error: "No email provider configured (set AGENTMAIL_API_KEY or configure Gmail via gcloud)", duration_ms: elapsed() };
|
|
10894
10902
|
}
|
|
10895
10903
|
log("autonomous-login", `starting for ${targetDomain} — url: ${loginUrl}`);
|
|
10896
|
-
const prevHeadless = process.env.HEADLESS;
|
|
10897
|
-
process.env.HEADLESS = "false";
|
|
10898
10904
|
let tabId;
|
|
10899
10905
|
try {
|
|
10900
10906
|
await start();
|
|
10901
10907
|
tabId = await getDefaultTab();
|
|
10902
10908
|
await networkEnable(tabId);
|
|
10903
10909
|
} catch (err) {
|
|
10904
|
-
if (prevHeadless !== undefined)
|
|
10905
|
-
process.env.HEADLESS = prevHeadless;
|
|
10906
|
-
else
|
|
10907
|
-
delete process.env.HEADLESS;
|
|
10908
10910
|
return { success: false, method: "failed", domain: targetDomain, cookies_stored: 0, error: `Kuri start failed: ${err}`, duration_ms: elapsed() };
|
|
10909
10911
|
}
|
|
10910
10912
|
try {
|
|
@@ -10993,12 +10995,7 @@ async function autonomousLogin(loginUrl, domain) {
|
|
|
10993
10995
|
};
|
|
10994
10996
|
}
|
|
10995
10997
|
return { success: false, method: "failed", domain: targetDomain, email: agentEmail, cookies_stored: 0, error: "login flow completed but no auth cookies detected", duration_ms: elapsed() };
|
|
10996
|
-
} finally {
|
|
10997
|
-
if (prevHeadless !== undefined)
|
|
10998
|
-
process.env.HEADLESS = prevHeadless;
|
|
10999
|
-
else
|
|
11000
|
-
delete process.env.HEADLESS;
|
|
11001
|
-
}
|
|
10998
|
+
} finally {}
|
|
11002
10999
|
}
|
|
11003
11000
|
var OTP_FILL_TIMEOUT_MS = 120000, POST_SUBMIT_SETTLE_MS = 3000;
|
|
11004
11001
|
var init_autonomous_login = __esm(async () => {
|
|
@@ -15506,6 +15503,12 @@ __export(exports_execution, {
|
|
|
15506
15503
|
buildCanonicalDocumentEndpoint: () => buildCanonicalDocumentEndpoint
|
|
15507
15504
|
});
|
|
15508
15505
|
import { nanoid as nanoid6 } from "nanoid";
|
|
15506
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
15507
|
+
function stableEndpointId2(method, urlTemplate) {
|
|
15508
|
+
if (!method || !urlTemplate)
|
|
15509
|
+
return nanoid6();
|
|
15510
|
+
return createHash6("sha256").update(`${method}:${urlTemplate}`).digest("base64url").slice(0, 21);
|
|
15511
|
+
}
|
|
15509
15512
|
function stampTrace(trace) {
|
|
15510
15513
|
trace.trace_version = TRACE_VERSION;
|
|
15511
15514
|
return trace;
|
|
@@ -16049,10 +16052,11 @@ function buildPageArtifactCapture(url, intent, html, authRequired = false) {
|
|
|
16049
16052
|
const searchForms = detectSearchForms(html);
|
|
16050
16053
|
const validSearchForm = searchForms.find((spec) => isStructuredSearchForm(spec));
|
|
16051
16054
|
const response_schema = inferSchema([extracted.data]);
|
|
16055
|
+
const computedTemplate = templatizeQueryParams(url);
|
|
16052
16056
|
const endpoint = {
|
|
16053
|
-
endpoint_id:
|
|
16057
|
+
endpoint_id: stableEndpointId2("GET", computedTemplate),
|
|
16054
16058
|
method: "GET",
|
|
16055
|
-
url_template:
|
|
16059
|
+
url_template: computedTemplate,
|
|
16056
16060
|
idempotency: "safe",
|
|
16057
16061
|
verification_status: "verified",
|
|
16058
16062
|
reliability_score: extracted.confidence,
|
|
@@ -16100,8 +16104,9 @@ function buildCanonicalDocumentEndpoint(url, intent, authRequired = false) {
|
|
|
16100
16104
|
const replayTemplate = deriveStructuredDataReplayTemplate(url);
|
|
16101
16105
|
if (replayUrl === url && replayTemplate === url)
|
|
16102
16106
|
return;
|
|
16107
|
+
const canonicalId = createHash6("sha1").update(replayTemplate !== url ? replayTemplate : replayUrl).digest("base64url").slice(0, 21);
|
|
16103
16108
|
const endpoint = {
|
|
16104
|
-
endpoint_id:
|
|
16109
|
+
endpoint_id: canonicalId,
|
|
16105
16110
|
method: "GET",
|
|
16106
16111
|
url_template: replayTemplate !== url ? replayTemplate : replayUrl,
|
|
16107
16112
|
idempotency: "safe",
|
|
@@ -16604,7 +16609,7 @@ async function executeBrowserCapture(skill, params, options) {
|
|
|
16604
16609
|
epUrl = `${route.url}?${qStr}`;
|
|
16605
16610
|
}
|
|
16606
16611
|
endpoints.push({
|
|
16607
|
-
endpoint_id:
|
|
16612
|
+
endpoint_id: stableEndpointId2("GET", epUrl),
|
|
16608
16613
|
method: "GET",
|
|
16609
16614
|
url_template: epUrl,
|
|
16610
16615
|
query: epQuery,
|
|
@@ -19126,10 +19131,10 @@ var init_timing_economics = __esm(() => {
|
|
|
19126
19131
|
});
|
|
19127
19132
|
|
|
19128
19133
|
// ../../src/routing-telemetry.ts
|
|
19129
|
-
import { createHash as
|
|
19134
|
+
import { createHash as createHash7 } from "node:crypto";
|
|
19130
19135
|
import { nanoid as nanoid8 } from "nanoid";
|
|
19131
19136
|
function stableHash(value) {
|
|
19132
|
-
return
|
|
19137
|
+
return createHash7("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 24);
|
|
19133
19138
|
}
|
|
19134
19139
|
function sanitizeScalar(value) {
|
|
19135
19140
|
if (value == null)
|
|
@@ -19483,7 +19488,7 @@ function applyVerificationResults(skill, verification) {
|
|
|
19483
19488
|
import { nanoid as nanoid9 } from "nanoid";
|
|
19484
19489
|
import { existsSync as existsSync13, writeFileSync as writeFileSync8, readFileSync as readFileSync8, mkdirSync as mkdirSync9, readdirSync as readdirSync6 } from "node:fs";
|
|
19485
19490
|
import { dirname as dirname2, join as join12 } from "node:path";
|
|
19486
|
-
import { createHash as
|
|
19491
|
+
import { createHash as createHash8 } from "node:crypto";
|
|
19487
19492
|
function summarizeSchema(schema, maxDepth = 3) {
|
|
19488
19493
|
function walk(s, depth) {
|
|
19489
19494
|
if (depth <= 0)
|
|
@@ -19611,7 +19616,7 @@ function scopedResolveCacheKeys(scope, key) {
|
|
|
19611
19616
|
return scope === "global" ? [scopedCacheKey("global", key)] : [scopedCacheKey(scope, key), scopedCacheKey("global", key)];
|
|
19612
19617
|
}
|
|
19613
19618
|
function snapshotPathForCacheKey(cacheKey) {
|
|
19614
|
-
const digest =
|
|
19619
|
+
const digest = createHash8("sha1").update(cacheKey).digest("hex");
|
|
19615
19620
|
return join12(SKILL_SNAPSHOT_DIR, `${digest}.json`);
|
|
19616
19621
|
}
|
|
19617
19622
|
function writeSkillSnapshot(cacheKey, skill) {
|
|
@@ -19933,10 +19938,17 @@ function withContextReplayEndpoint(skill, intent, contextUrl) {
|
|
|
19933
19938
|
if (skill.endpoints.some((endpoint) => endpoint.method === canonical.method && endpoint.url_template === canonical.url_template)) {
|
|
19934
19939
|
return skill;
|
|
19935
19940
|
}
|
|
19936
|
-
|
|
19941
|
+
const augmented = {
|
|
19937
19942
|
...skill,
|
|
19938
19943
|
endpoints: [canonical, ...skill.endpoints]
|
|
19939
19944
|
};
|
|
19945
|
+
try {
|
|
19946
|
+
const existing = findExistingSkillForDomain(skill.domain);
|
|
19947
|
+
if (!existing || !existing.endpoints.some((ep) => ep.endpoint_id === canonical.endpoint_id)) {
|
|
19948
|
+
cachePublishedSkill(augmented);
|
|
19949
|
+
}
|
|
19950
|
+
} catch {}
|
|
19951
|
+
return augmented;
|
|
19940
19952
|
}
|
|
19941
19953
|
function isSearchLikeIntent(intent, contextUrl) {
|
|
19942
19954
|
if (/\b(search|find|lookup|browse|discover)\b/i.test(intent ?? ""))
|
|
@@ -19955,7 +19967,7 @@ function buildLocalCanonicalReplaySkill(intent, contextUrl) {
|
|
|
19955
19967
|
const domain = new URL(contextUrl).hostname.replace(/^www\./, "");
|
|
19956
19968
|
const now = new Date().toISOString();
|
|
19957
19969
|
const skill = {
|
|
19958
|
-
skill_id: `canonical-${
|
|
19970
|
+
skill_id: `canonical-${createHash8("sha1").update(contextUrl).digest("hex").slice(0, 12)}`,
|
|
19959
19971
|
version: "1.0.0",
|
|
19960
19972
|
schema_version: "1",
|
|
19961
19973
|
name: `Canonical replay for ${domain}`,
|
|
@@ -22157,7 +22169,18 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
22157
22169
|
continue;
|
|
22158
22170
|
if (targetRegDomain && getRegistrableDomain(skill.domain) !== targetRegDomain)
|
|
22159
22171
|
continue;
|
|
22160
|
-
|
|
22172
|
+
let endpointId = extractEndpointId(c.metadata) ?? undefined;
|
|
22173
|
+
if (endpointId && !skill.endpoints.some((ep) => ep.endpoint_id === endpointId)) {
|
|
22174
|
+
const vecUrl = c.metadata?.url_template;
|
|
22175
|
+
const urlMatch = vecUrl ? skill.endpoints.find((ep) => ep.url_template === vecUrl) : undefined;
|
|
22176
|
+
if (urlMatch) {
|
|
22177
|
+
console.log(`[marketplace] vecdb endpoint ${endpointId} stale → recovered via URL match: ${urlMatch.endpoint_id}`);
|
|
22178
|
+
endpointId = urlMatch.endpoint_id;
|
|
22179
|
+
} else {
|
|
22180
|
+
console.log(`[marketplace] vecdb endpoint ${endpointId} not found on skill ${skillId} — dropping to skill-level match`);
|
|
22181
|
+
endpointId = undefined;
|
|
22182
|
+
}
|
|
22183
|
+
}
|
|
22161
22184
|
ranked.push({
|
|
22162
22185
|
candidate: c,
|
|
22163
22186
|
skill,
|
|
@@ -24198,7 +24221,13 @@ var init_browse_session = __esm(() => {
|
|
|
24198
24221
|
|
|
24199
24222
|
// ../../src/api/browse-index.ts
|
|
24200
24223
|
import { nanoid as nanoid10 } from "nanoid";
|
|
24224
|
+
import { createHash as createHash9 } from "node:crypto";
|
|
24201
24225
|
import { readFileSync as readFileSync12 } from "node:fs";
|
|
24226
|
+
function stableEndpointId3(method, urlTemplate) {
|
|
24227
|
+
if (!method || !urlTemplate)
|
|
24228
|
+
return nanoid10();
|
|
24229
|
+
return createHash9("sha256").update(`${method}:${urlTemplate}`).digest("base64url").slice(0, 21);
|
|
24230
|
+
}
|
|
24202
24231
|
function normalizeBrowseUrl(url, baseUrl) {
|
|
24203
24232
|
if (!url)
|
|
24204
24233
|
return url;
|
|
@@ -24376,7 +24405,7 @@ async function cacheBrowseRequests(params) {
|
|
|
24376
24405
|
return { domain, indexed: false, mode: "none", skill: null };
|
|
24377
24406
|
const urlTemplate = templatizeQueryParams2(sessionUrl);
|
|
24378
24407
|
const endpoint = {
|
|
24379
|
-
endpoint_id:
|
|
24408
|
+
endpoint_id: stableEndpointId3("GET", urlTemplate),
|
|
24380
24409
|
method: "GET",
|
|
24381
24410
|
url_template: urlTemplate,
|
|
24382
24411
|
idempotency: "safe",
|
|
@@ -26682,7 +26711,28 @@ async function registerRoutes(app) {
|
|
|
26682
26711
|
...context_url && typeof params?.url !== "string" ? { url: context_url } : {}
|
|
26683
26712
|
};
|
|
26684
26713
|
try {
|
|
26685
|
-
|
|
26714
|
+
let execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
|
|
26715
|
+
if (!execResult.trace.success && execResult.result?.error === "endpoint_not_found" && typeof execParams.endpoint_id === "string") {
|
|
26716
|
+
let recovered = false;
|
|
26717
|
+
const freshSkill = await getSkill2(skill_id, clientScope);
|
|
26718
|
+
if (freshSkill && freshSkill.endpoints.some((e) => e.endpoint_id === execParams.endpoint_id)) {
|
|
26719
|
+
console.log(`[exec] endpoint ${execParams.endpoint_id} found in fresh marketplace skill — retrying`);
|
|
26720
|
+
skill = freshSkill;
|
|
26721
|
+
recovered = true;
|
|
26722
|
+
}
|
|
26723
|
+
if (!recovered && context_url) {
|
|
26724
|
+
const { buildCanonicalDocumentEndpoint: buildCanonicalDocumentEndpoint2 } = await init_execution().then(() => exports_execution);
|
|
26725
|
+
const canonical = buildCanonicalDocumentEndpoint2(context_url, intent ?? "", false);
|
|
26726
|
+
if (canonical && canonical.endpoint_id === execParams.endpoint_id) {
|
|
26727
|
+
console.log(`[exec] endpoint ${execParams.endpoint_id} is a canonical replay — injecting into skill`);
|
|
26728
|
+
skill = { ...skill, endpoints: [canonical, ...skill.endpoints] };
|
|
26729
|
+
recovered = true;
|
|
26730
|
+
}
|
|
26731
|
+
}
|
|
26732
|
+
if (recovered) {
|
|
26733
|
+
execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
|
|
26734
|
+
}
|
|
26735
|
+
}
|
|
26686
26736
|
saveTrace(execResult.trace);
|
|
26687
26737
|
if (execResult.trace.endpoint_id) {
|
|
26688
26738
|
recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});
|
|
@@ -27588,7 +27638,7 @@ var init_routes = __esm(async () => {
|
|
|
27588
27638
|
]);
|
|
27589
27639
|
BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
|
|
27590
27640
|
TRACES_DIR = process.env.TRACES_DIR ?? join17(process.cwd(), "traces");
|
|
27591
|
-
BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "
|
|
27641
|
+
BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "1"));
|
|
27592
27642
|
BROWSE_BROKER_BASE_PORT = Number(process.env.KURI_PORT ?? "7700");
|
|
27593
27643
|
browseSessions = new Map;
|
|
27594
27644
|
STATS_CACHE_TTL = 5 * 60 * 1000;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "unbrowse",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.7.0-preview.1",
|
|
4
4
|
"description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -29,6 +29,8 @@
|
|
|
29
29
|
"@fastify/rate-limit": "^10.3.0",
|
|
30
30
|
"@cascade-fyi/splits-sdk": "^0.11.1",
|
|
31
31
|
"@solana/kit": "^6.6.0",
|
|
32
|
+
"@x402/fetch": "^2.9.0",
|
|
33
|
+
"agentmail": "^0.4.18",
|
|
32
34
|
"bs58": "^6.0.0",
|
|
33
35
|
"cheerio": "^1.2.0",
|
|
34
36
|
"dotenv": "^17.3.1",
|