unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +79 -33
- package/dist/index.js +2 -6
- package/dist/mcp.js +73 -22
- package/dist/server.js +25994 -0
- package/package.json +1 -2
- package/vendor/kuri/manifest.json +1 -1
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -343
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1833
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1929
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1734
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3407
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/execution/token-resolver.ts +0 -135
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1532
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/impact-log.ts +0 -227
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -111
- package/runtime-src/mcp.ts +0 -1911
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/reverse-engineer/token-sources.ts +0 -379
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -931
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
package/runtime-src/settings.ts
DELETED
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
import { homedir } from "node:os";
|
|
4
|
-
import { getRegistrableDomain } from "./domain.js";
|
|
5
|
-
|
|
6
|
-
type RawConfig = Record<string, unknown>;
|
|
7
|
-
|
|
8
|
-
export interface CapturePipelineSettings {
|
|
9
|
-
auto_publish_checkpoints: boolean;
|
|
10
|
-
publish_domain_blacklist: string[];
|
|
11
|
-
publish_domain_promptlist: string[];
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export interface CheckpointPublishDecision {
|
|
15
|
-
publishQueued: boolean;
|
|
16
|
-
mode: "auto" | "disabled" | "blacklisted" | "prompt";
|
|
17
|
-
reason: string;
|
|
18
|
-
matchedDomain?: string;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export interface ExplicitPublishDecision {
|
|
22
|
-
allowed: boolean;
|
|
23
|
-
mode: "allowed" | "blacklisted" | "prompt";
|
|
24
|
-
reason: string;
|
|
25
|
-
matchedDomain?: string;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
const DEFAULT_CAPTURE_PIPELINE_SETTINGS: CapturePipelineSettings = {
|
|
29
|
-
auto_publish_checkpoints: true,
|
|
30
|
-
publish_domain_blacklist: [],
|
|
31
|
-
publish_domain_promptlist: [],
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
function sanitizeProfileName(value: string): string {
|
|
35
|
-
return value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
function getConfigDir(): string {
|
|
39
|
-
if (process.env.UNBROWSE_CONFIG_DIR) return process.env.UNBROWSE_CONFIG_DIR;
|
|
40
|
-
const profile = sanitizeProfileName(process.env.UNBROWSE_PROFILE ?? "");
|
|
41
|
-
return profile
|
|
42
|
-
? join(homedir(), ".unbrowse", "profiles", profile)
|
|
43
|
-
: join(homedir(), ".unbrowse");
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export function getUnbrowseConfigPath(): string {
|
|
47
|
-
return join(getConfigDir(), "config.json");
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function loadRawConfig(): RawConfig {
|
|
51
|
-
try {
|
|
52
|
-
const configPath = getUnbrowseConfigPath();
|
|
53
|
-
if (existsSync(configPath)) {
|
|
54
|
-
const parsed = JSON.parse(readFileSync(configPath, "utf-8")) as unknown;
|
|
55
|
-
if (parsed && typeof parsed === "object") return parsed as RawConfig;
|
|
56
|
-
}
|
|
57
|
-
} catch {
|
|
58
|
-
// ignore broken local config and fall back to defaults
|
|
59
|
-
}
|
|
60
|
-
return {};
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function saveRawConfig(config: RawConfig): void {
|
|
64
|
-
const configDir = getConfigDir();
|
|
65
|
-
if (!existsSync(configDir)) mkdirSync(configDir, { recursive: true });
|
|
66
|
-
writeFileSync(getUnbrowseConfigPath(), JSON.stringify(config, null, 2), { mode: 0o600 });
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function normalizeDomainRule(value: string): string {
|
|
70
|
-
let normalized = value.trim().toLowerCase();
|
|
71
|
-
if (!normalized) return "";
|
|
72
|
-
normalized = normalized.replace(/^\*\./, "");
|
|
73
|
-
if (normalized.startsWith("http://") || normalized.startsWith("https://")) {
|
|
74
|
-
try {
|
|
75
|
-
normalized = new URL(normalized).hostname.toLowerCase();
|
|
76
|
-
} catch {
|
|
77
|
-
normalized = normalized.replace(/^https?:\/\//, "");
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
normalized = normalized.split("/")[0] ?? normalized;
|
|
81
|
-
normalized = normalized.replace(/:\d+$/, "");
|
|
82
|
-
normalized = normalized.replace(/^www\./, "");
|
|
83
|
-
return normalized;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function normalizeDomainList(values: unknown): string[] {
|
|
87
|
-
if (!Array.isArray(values)) return [];
|
|
88
|
-
const normalized = values
|
|
89
|
-
.filter((value): value is string => typeof value === "string")
|
|
90
|
-
.map(normalizeDomainRule)
|
|
91
|
-
.filter(Boolean);
|
|
92
|
-
return Array.from(new Set(normalized)).sort();
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
function matchesDomainRule(domain: string, rule: string): boolean {
|
|
96
|
-
const host = normalizeDomainRule(domain);
|
|
97
|
-
const target = normalizeDomainRule(rule);
|
|
98
|
-
if (!host || !target) return false;
|
|
99
|
-
if (host === target || host.endsWith(`.${target}`)) return true;
|
|
100
|
-
try {
|
|
101
|
-
return getRegistrableDomain(host) === target;
|
|
102
|
-
} catch {
|
|
103
|
-
return false;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
function firstMatchingRule(domain: string, rules: string[]): string | undefined {
|
|
108
|
-
return rules.find((rule) => matchesDomainRule(domain, rule));
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
export function getCapturePipelineSettings(): CapturePipelineSettings {
|
|
112
|
-
const raw = loadRawConfig();
|
|
113
|
-
const capturePipeline = raw.capture_pipeline && typeof raw.capture_pipeline === "object"
|
|
114
|
-
? raw.capture_pipeline as Record<string, unknown>
|
|
115
|
-
: {};
|
|
116
|
-
|
|
117
|
-
return {
|
|
118
|
-
auto_publish_checkpoints: capturePipeline.auto_publish_checkpoints !== false,
|
|
119
|
-
publish_domain_blacklist: normalizeDomainList(capturePipeline.publish_domain_blacklist),
|
|
120
|
-
publish_domain_promptlist: normalizeDomainList(capturePipeline.publish_domain_promptlist),
|
|
121
|
-
};
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
export function updateCapturePipelineSettings(update: {
|
|
125
|
-
auto_publish_checkpoints?: boolean;
|
|
126
|
-
publish_domain_blacklist?: string[];
|
|
127
|
-
publish_domain_promptlist?: string[];
|
|
128
|
-
clear_publish_domain_blacklist?: boolean;
|
|
129
|
-
clear_publish_domain_promptlist?: boolean;
|
|
130
|
-
}): CapturePipelineSettings {
|
|
131
|
-
const raw = loadRawConfig();
|
|
132
|
-
const existing = getCapturePipelineSettings();
|
|
133
|
-
const next: CapturePipelineSettings = {
|
|
134
|
-
auto_publish_checkpoints: typeof update.auto_publish_checkpoints === "boolean"
|
|
135
|
-
? update.auto_publish_checkpoints
|
|
136
|
-
: existing.auto_publish_checkpoints,
|
|
137
|
-
publish_domain_blacklist: update.clear_publish_domain_blacklist
|
|
138
|
-
? []
|
|
139
|
-
: update.publish_domain_blacklist
|
|
140
|
-
? normalizeDomainList(update.publish_domain_blacklist)
|
|
141
|
-
: existing.publish_domain_blacklist,
|
|
142
|
-
publish_domain_promptlist: update.clear_publish_domain_promptlist
|
|
143
|
-
? []
|
|
144
|
-
: update.publish_domain_promptlist
|
|
145
|
-
? normalizeDomainList(update.publish_domain_promptlist)
|
|
146
|
-
: existing.publish_domain_promptlist,
|
|
147
|
-
};
|
|
148
|
-
|
|
149
|
-
saveRawConfig({
|
|
150
|
-
...raw,
|
|
151
|
-
capture_pipeline: next,
|
|
152
|
-
});
|
|
153
|
-
|
|
154
|
-
return next;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
export function decideCheckpointPublish(domain: string): CheckpointPublishDecision {
|
|
158
|
-
const settings = getCapturePipelineSettings();
|
|
159
|
-
if (!settings.auto_publish_checkpoints) {
|
|
160
|
-
return {
|
|
161
|
-
publishQueued: false,
|
|
162
|
-
mode: "disabled",
|
|
163
|
-
reason: "Auto-publish after sync/close is disabled in local settings.",
|
|
164
|
-
};
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
const blacklisted = firstMatchingRule(domain, settings.publish_domain_blacklist);
|
|
168
|
-
if (blacklisted) {
|
|
169
|
-
return {
|
|
170
|
-
publishQueued: false,
|
|
171
|
-
mode: "blacklisted",
|
|
172
|
-
reason: `Auto-publish skipped because ${domain} matches blacklist entry ${blacklisted}.`,
|
|
173
|
-
matchedDomain: blacklisted,
|
|
174
|
-
};
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
const prompted = firstMatchingRule(domain, settings.publish_domain_promptlist);
|
|
178
|
-
if (prompted) {
|
|
179
|
-
return {
|
|
180
|
-
publishQueued: false,
|
|
181
|
-
mode: "prompt",
|
|
182
|
-
reason: `Auto-publish paused because ${domain} matches prompt-list entry ${prompted}.`,
|
|
183
|
-
matchedDomain: prompted,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
return {
|
|
188
|
-
publishQueued: true,
|
|
189
|
-
mode: "auto",
|
|
190
|
-
reason: "Background publish is allowed for this checkpoint.",
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
export function decideExplicitPublish(domain: string, confirmPublish: boolean): ExplicitPublishDecision {
|
|
195
|
-
const settings = getCapturePipelineSettings();
|
|
196
|
-
const blacklisted = firstMatchingRule(domain, settings.publish_domain_blacklist);
|
|
197
|
-
if (blacklisted && !confirmPublish) {
|
|
198
|
-
return {
|
|
199
|
-
allowed: false,
|
|
200
|
-
mode: "blacklisted",
|
|
201
|
-
reason: `Explicit publish requires confirmation because ${domain} matches blacklist entry ${blacklisted}.`,
|
|
202
|
-
matchedDomain: blacklisted,
|
|
203
|
-
};
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
const prompted = firstMatchingRule(domain, settings.publish_domain_promptlist);
|
|
207
|
-
if (prompted && !confirmPublish) {
|
|
208
|
-
return {
|
|
209
|
-
allowed: false,
|
|
210
|
-
mode: "prompt",
|
|
211
|
-
reason: `Explicit publish requires confirmation because ${domain} matches prompt-list entry ${prompted}.`,
|
|
212
|
-
matchedDomain: prompted,
|
|
213
|
-
};
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
return {
|
|
217
|
-
allowed: true,
|
|
218
|
-
mode: "allowed",
|
|
219
|
-
reason: "Explicit publish allowed.",
|
|
220
|
-
};
|
|
221
|
-
}
|
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bun
|
|
2
|
-
/**
|
|
3
|
-
* Unbrowse single-binary entrypoint.
|
|
4
|
-
*
|
|
5
|
-
* Modes:
|
|
6
|
-
* unbrowse serve → start the API server inline
|
|
7
|
-
* unbrowse [command] → CLI mode, auto-starts server if not running
|
|
8
|
-
*
|
|
9
|
-
* Kuri is embedded as a file asset. On first run it's extracted to
|
|
10
|
-
* ~/.unbrowse/bin/kuri and reused from there.
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
import { existsSync, mkdirSync, writeFileSync, chmodSync, readFileSync } from "node:fs";
|
|
14
|
-
import { join, dirname } from "node:path";
|
|
15
|
-
import { homedir, platform, arch } from "node:os";
|
|
16
|
-
import { execSync } from "node:child_process";
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
// Embed the platform-appropriate kuri binary at compile time.
|
|
20
|
-
// Bun's compiled binaries can read files from the virtual /$bunfs/ filesystem
|
|
21
|
-
// when they are imported/required at bundle time.
|
|
22
|
-
import kuriDarwinArm64Path from "../packages/skill/vendor/kuri/darwin-arm64/kuri" with { type: "file" };
|
|
23
|
-
import kuriDarwinX64Path from "../packages/skill/vendor/kuri/darwin-x64/kuri" with { type: "file" };
|
|
24
|
-
import kuriLinuxArm64Path from "../packages/skill/vendor/kuri/linux-arm64/kuri" with { type: "file" };
|
|
25
|
-
import kuriLinuxX64Path from "../packages/skill/vendor/kuri/linux-x64/kuri" with { type: "file" };
|
|
26
|
-
|
|
27
|
-
const UNBROWSE_HOME = join(homedir(), ".unbrowse");
|
|
28
|
-
const KURI_CACHE = join(UNBROWSE_HOME, "bin", "kuri");
|
|
29
|
-
const EMBEDDED_KURI_BY_TARGET: Record<string, string> = {
|
|
30
|
-
"darwin-arm64": kuriDarwinArm64Path,
|
|
31
|
-
"darwin-x64": kuriDarwinX64Path,
|
|
32
|
-
"linux-arm64": kuriLinuxArm64Path,
|
|
33
|
-
"linux-x64": kuriLinuxX64Path,
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
function currentKuriTarget(): string {
|
|
37
|
-
return `${platform()}-${arch()}`;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function extractEmbeddedKuri(): string | null {
|
|
41
|
-
const kuriBinaryPath = EMBEDDED_KURI_BY_TARGET[currentKuriTarget()];
|
|
42
|
-
if (!kuriBinaryPath) {
|
|
43
|
-
console.warn(`[unbrowse] no embedded kuri available for ${currentKuriTarget()}`);
|
|
44
|
-
return null;
|
|
45
|
-
}
|
|
46
|
-
try {
|
|
47
|
-
const data = readFileSync(kuriBinaryPath);
|
|
48
|
-
mkdirSync(dirname(KURI_CACHE), { recursive: true });
|
|
49
|
-
writeFileSync(KURI_CACHE, data);
|
|
50
|
-
chmodSync(KURI_CACHE, 0o755);
|
|
51
|
-
console.log(`[unbrowse] extracted kuri to ${KURI_CACHE}`);
|
|
52
|
-
return KURI_CACHE;
|
|
53
|
-
} catch (err) {
|
|
54
|
-
console.warn(`[unbrowse] kuri extraction failed: ${(err as Error).message}`);
|
|
55
|
-
return null;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
function findKuriBinary(): string | null {
|
|
59
|
-
// 1. Explicit env var
|
|
60
|
-
if (process.env.KURI_BIN && existsSync(process.env.KURI_BIN)) {
|
|
61
|
-
return process.env.KURI_BIN;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// 2. Cached in ~/.unbrowse/bin/ (previously extracted)
|
|
65
|
-
if (existsSync(KURI_CACHE)) {
|
|
66
|
-
return KURI_CACHE;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// 3. Extract from embedded asset
|
|
70
|
-
const extracted = extractEmbeddedKuri();
|
|
71
|
-
if (extracted) return extracted;
|
|
72
|
-
|
|
73
|
-
// 4. Alongside this binary
|
|
74
|
-
const alongside = join(dirname(process.execPath), "kuri");
|
|
75
|
-
if (existsSync(alongside)) {
|
|
76
|
-
return alongside;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// 5. System PATH
|
|
80
|
-
try {
|
|
81
|
-
const which = execSync("which kuri 2>/dev/null", { encoding: "utf-8" }).trim();
|
|
82
|
-
if (which && existsSync(which)) return which;
|
|
83
|
-
} catch {}
|
|
84
|
-
|
|
85
|
-
return null;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
function ensureKuri(): string {
|
|
89
|
-
const kuri = findKuriBinary();
|
|
90
|
-
if (!kuri) {
|
|
91
|
-
console.error("[unbrowse] kuri binary not found.");
|
|
92
|
-
console.error("[unbrowse] Place kuri next to the unbrowse binary, or set KURI_BIN env var.");
|
|
93
|
-
console.error("[unbrowse] Or run: unbrowse setup");
|
|
94
|
-
process.exit(1);
|
|
95
|
-
}
|
|
96
|
-
process.env.KURI_BIN = kuri;
|
|
97
|
-
return kuri;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
async function main() {
|
|
101
|
-
const args = process.argv.slice(2);
|
|
102
|
-
|
|
103
|
-
// Ensure kuri is available
|
|
104
|
-
ensureKuri();
|
|
105
|
-
|
|
106
|
-
if (args[0] === "serve") {
|
|
107
|
-
// Server mode — run inline
|
|
108
|
-
const { startUnbrowseServer, installServerExitCleanup } = await import("./server.js");
|
|
109
|
-
const { config: loadEnv } = await import("dotenv");
|
|
110
|
-
|
|
111
|
-
loadEnv({ quiet: true });
|
|
112
|
-
loadEnv({ path: ".env.runtime", quiet: true });
|
|
113
|
-
|
|
114
|
-
const pidFile = process.env.UNBROWSE_PID_FILE;
|
|
115
|
-
installServerExitCleanup(pidFile);
|
|
116
|
-
|
|
117
|
-
const server = await startUnbrowseServer({
|
|
118
|
-
pidFile,
|
|
119
|
-
scheduleVerification: true,
|
|
120
|
-
});
|
|
121
|
-
|
|
122
|
-
const shutdown = async (signal: string) => {
|
|
123
|
-
console.log(`[shutdown] ${signal}`);
|
|
124
|
-
await server.close();
|
|
125
|
-
process.exit(0);
|
|
126
|
-
};
|
|
127
|
-
|
|
128
|
-
process.on("SIGTERM", () => shutdown("SIGTERM").catch(() => process.exit(1)));
|
|
129
|
-
process.on("SIGINT", () => shutdown("SIGINT").catch(() => process.exit(1)));
|
|
130
|
-
|
|
131
|
-
console.log(`unbrowse running on http://${server.host}:${server.port}`);
|
|
132
|
-
} else if (args[0] === "mcp-serve") {
|
|
133
|
-
await import("./mcp.js");
|
|
134
|
-
} else {
|
|
135
|
-
// CLI mode
|
|
136
|
-
await import("./cli.js");
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
main().catch((err) => {
|
|
141
|
-
console.error(err);
|
|
142
|
-
process.exit(1);
|
|
143
|
-
});
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import { getRegistrableDomain } from "./domain.js";
|
|
2
|
-
import type { EndpointDescriptor, EndpointPolicyDescriptor } from "./types/index.js";
|
|
3
|
-
|
|
4
|
-
const MUTATION_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
|
|
5
|
-
|
|
6
|
-
type DomainPolicy = {
|
|
7
|
-
domain: string;
|
|
8
|
-
reason: string;
|
|
9
|
-
require_for_mutations: boolean;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
const DOMAIN_POLICIES: DomainPolicy[] = [
|
|
13
|
-
{
|
|
14
|
-
domain: "x.com",
|
|
15
|
-
reason: "X write endpoints require explicit user confirmation because automated posting or other mutations may violate third-party site terms or automation rules.",
|
|
16
|
-
require_for_mutations: true,
|
|
17
|
-
},
|
|
18
|
-
];
|
|
19
|
-
|
|
20
|
-
function resolveEndpointDomain(endpoint: EndpointDescriptor): string | null {
|
|
21
|
-
try {
|
|
22
|
-
const parsed = new URL(endpoint.url_template);
|
|
23
|
-
return getRegistrableDomain(parsed.hostname);
|
|
24
|
-
} catch {
|
|
25
|
-
return null;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export function getEndpointPolicy(endpoint: EndpointDescriptor): EndpointPolicyDescriptor | undefined {
|
|
30
|
-
if (endpoint.policy?.requires_third_party_terms_confirmation) return endpoint.policy;
|
|
31
|
-
|
|
32
|
-
const domain = resolveEndpointDomain(endpoint);
|
|
33
|
-
if (!domain) return undefined;
|
|
34
|
-
|
|
35
|
-
const matched = DOMAIN_POLICIES.find((policy) => policy.domain === domain);
|
|
36
|
-
if (!matched) return undefined;
|
|
37
|
-
if (matched.require_for_mutations && !MUTATION_METHODS.has(endpoint.method)) return undefined;
|
|
38
|
-
|
|
39
|
-
return {
|
|
40
|
-
requires_third_party_terms_confirmation: true,
|
|
41
|
-
policy_domain: matched.domain,
|
|
42
|
-
reason: matched.reason,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export function endpointRequiresThirdPartyTermsConfirmation(endpoint: EndpointDescriptor): boolean {
|
|
47
|
-
return !!getEndpointPolicy(endpoint)?.requires_third_party_terms_confirmation;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export function annotateEndpointPolicy<T extends EndpointDescriptor>(endpoint: T): T {
|
|
51
|
-
const policy = getEndpointPolicy(endpoint);
|
|
52
|
-
if (!policy || endpoint.policy) return endpoint;
|
|
53
|
-
return { ...endpoint, policy } as T;
|
|
54
|
-
}
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
import { listSkills } from "./marketplace/index.js";
|
|
2
|
-
import { verifySkill } from "./verification/index.js";
|
|
3
|
-
import { selectVerificationCandidates } from "./verification/candidates.js";
|
|
4
|
-
import type { SkillManifest } from "./types/index.js";
|
|
5
|
-
import {
|
|
6
|
-
applyVerificationResults,
|
|
7
|
-
countStaleEndpoints,
|
|
8
|
-
type LocalCacheCleanupSummary,
|
|
9
|
-
} from "./stale-cleanup.js";
|
|
10
|
-
import { pruneLocalCacheEntriesForSkill } from "./orchestrator/index.js";
|
|
11
|
-
|
|
12
|
-
const VERIFY_TIMEOUT_MS = Math.max(5_000, Number(process.env.UNBROWSE_STALE_VERIFY_TIMEOUT_MS ?? 15_000));
|
|
13
|
-
const CLEANUP_INTERVAL_MS = Math.max(30 * 60 * 1000, Number(process.env.UNBROWSE_STALE_CLEANUP_INTERVAL_MS ?? 6 * 60 * 60 * 1000));
|
|
14
|
-
const CLEANUP_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_STALE_CLEANUP_BATCH_SIZE ?? 25));
|
|
15
|
-
const CLEANUP_VERIFY_ENDPOINT_LIMIT = Math.max(1, Number(process.env.UNBROWSE_STALE_VERIFY_ENDPOINT_LIMIT ?? 2));
|
|
16
|
-
|
|
17
|
-
export type SkillCleanupSummary = {
|
|
18
|
-
skill_id: string;
|
|
19
|
-
domain: string;
|
|
20
|
-
lifecycle: SkillManifest["lifecycle"];
|
|
21
|
-
verified: boolean;
|
|
22
|
-
stale_before: number;
|
|
23
|
-
stale_after: number;
|
|
24
|
-
cache_cleanup: LocalCacheCleanupSummary;
|
|
25
|
-
error?: string;
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
export async function cleanupStaleSkills(options?: {
|
|
29
|
-
skill_id?: string;
|
|
30
|
-
domain?: string;
|
|
31
|
-
limit?: number;
|
|
32
|
-
offset?: number;
|
|
33
|
-
}): Promise<{
|
|
34
|
-
ok: true;
|
|
35
|
-
scanned_skills: number;
|
|
36
|
-
verified_skills: number;
|
|
37
|
-
stale_before: number;
|
|
38
|
-
stale_after: number;
|
|
39
|
-
cache_cleanup: LocalCacheCleanupSummary;
|
|
40
|
-
skills: SkillCleanupSummary[];
|
|
41
|
-
}> {
|
|
42
|
-
const normalizedDomain = options?.domain?.trim().toLowerCase();
|
|
43
|
-
let skills = await listSkills();
|
|
44
|
-
if (options?.skill_id) skills = skills.filter((skill) => skill.skill_id === options.skill_id);
|
|
45
|
-
if (normalizedDomain) {
|
|
46
|
-
skills = skills.filter((skill) => {
|
|
47
|
-
const domain = skill.domain.trim().toLowerCase();
|
|
48
|
-
return domain === normalizedDomain || domain.replace(/^www\./, "") === normalizedDomain.replace(/^www\./, "");
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
if (options?.offset && Number.isFinite(options.offset) && options.offset > 0) {
|
|
52
|
-
skills = skills.slice(options.offset);
|
|
53
|
-
}
|
|
54
|
-
if (options?.limit && Number.isFinite(options.limit) && options.limit > 0) {
|
|
55
|
-
skills = skills.slice(0, options.limit);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
const perSkill: SkillCleanupSummary[] = [];
|
|
59
|
-
const cacheTotals: LocalCacheCleanupSummary = {
|
|
60
|
-
route_cache_entries_removed: 0,
|
|
61
|
-
domain_cache_entries_removed: 0,
|
|
62
|
-
route_result_entries_removed: 0,
|
|
63
|
-
captured_domain_entries_removed: 0,
|
|
64
|
-
};
|
|
65
|
-
let staleBefore = 0;
|
|
66
|
-
let staleAfter = 0;
|
|
67
|
-
let verifiedSkills = 0;
|
|
68
|
-
|
|
69
|
-
for (const skill of skills) {
|
|
70
|
-
const before = countStaleEndpoints(skill);
|
|
71
|
-
staleBefore += before;
|
|
72
|
-
let refreshed = skill;
|
|
73
|
-
let verified = false;
|
|
74
|
-
let error: string | undefined;
|
|
75
|
-
|
|
76
|
-
if (skill.lifecycle === "active" && skill.endpoints.some((endpoint) => endpoint.method === "GET")) {
|
|
77
|
-
try {
|
|
78
|
-
const endpoints = selectVerificationCandidates(skill, {
|
|
79
|
-
staleOnly: true,
|
|
80
|
-
limit: CLEANUP_VERIFY_ENDPOINT_LIMIT,
|
|
81
|
-
});
|
|
82
|
-
const verification = await Promise.race([
|
|
83
|
-
verifySkill(skill, { endpoints }),
|
|
84
|
-
new Promise<never>((_, reject) => {
|
|
85
|
-
setTimeout(() => reject(new Error(`verification timed out after ${VERIFY_TIMEOUT_MS}ms`)), VERIFY_TIMEOUT_MS);
|
|
86
|
-
}),
|
|
87
|
-
]);
|
|
88
|
-
refreshed = applyVerificationResults(skill, verification);
|
|
89
|
-
verified = true;
|
|
90
|
-
verifiedSkills++;
|
|
91
|
-
} catch (err) {
|
|
92
|
-
error = (err as Error).message;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
const cacheCleanup = pruneLocalCacheEntriesForSkill(refreshed);
|
|
97
|
-
cacheTotals.route_cache_entries_removed += cacheCleanup.route_cache_entries_removed;
|
|
98
|
-
cacheTotals.domain_cache_entries_removed += cacheCleanup.domain_cache_entries_removed;
|
|
99
|
-
cacheTotals.route_result_entries_removed += cacheCleanup.route_result_entries_removed;
|
|
100
|
-
cacheTotals.captured_domain_entries_removed += cacheCleanup.captured_domain_entries_removed;
|
|
101
|
-
|
|
102
|
-
const after = countStaleEndpoints(refreshed);
|
|
103
|
-
staleAfter += after;
|
|
104
|
-
perSkill.push({
|
|
105
|
-
skill_id: refreshed.skill_id,
|
|
106
|
-
domain: refreshed.domain,
|
|
107
|
-
lifecycle: refreshed.lifecycle,
|
|
108
|
-
verified,
|
|
109
|
-
stale_before: before,
|
|
110
|
-
stale_after: after,
|
|
111
|
-
cache_cleanup: cacheCleanup,
|
|
112
|
-
...(error ? { error } : {}),
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
return {
|
|
117
|
-
ok: true,
|
|
118
|
-
scanned_skills: skills.length,
|
|
119
|
-
verified_skills: verifiedSkills,
|
|
120
|
-
stale_before: staleBefore,
|
|
121
|
-
stale_after: staleAfter,
|
|
122
|
-
cache_cleanup: cacheTotals,
|
|
123
|
-
skills: perSkill,
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
export function schedulePeriodicStaleCleanup(): ReturnType<typeof setInterval> {
|
|
128
|
-
let offset = 0;
|
|
129
|
-
return setInterval(async () => {
|
|
130
|
-
try {
|
|
131
|
-
const result = await cleanupStaleSkills({
|
|
132
|
-
limit: CLEANUP_BATCH_SIZE,
|
|
133
|
-
offset,
|
|
134
|
-
});
|
|
135
|
-
if (result.scanned_skills < CLEANUP_BATCH_SIZE) offset = 0;
|
|
136
|
-
else offset += result.scanned_skills;
|
|
137
|
-
console.log(
|
|
138
|
-
`[stale-cleanup] scanned=${result.scanned_skills} verified=${result.verified_skills} stale=${result.stale_before}->${result.stale_after} removed=${result.cache_cleanup.route_cache_entries_removed + result.cache_cleanup.domain_cache_entries_removed + result.cache_cleanup.route_result_entries_removed + result.cache_cleanup.captured_domain_entries_removed}`,
|
|
139
|
-
);
|
|
140
|
-
} catch (err) {
|
|
141
|
-
console.warn(`[stale-cleanup] sweep failed: ${(err as Error).message}`);
|
|
142
|
-
}
|
|
143
|
-
}, CLEANUP_INTERVAL_MS);
|
|
144
|
-
}
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
import type { SkillManifest, VerificationStatus } from "./types/index.js";
|
|
2
|
-
|
|
3
|
-
export type SkillRouteCacheEntryLike = {
|
|
4
|
-
skillId: string;
|
|
5
|
-
domain: string;
|
|
6
|
-
endpointId?: string;
|
|
7
|
-
localSkillPath?: string;
|
|
8
|
-
ts: number;
|
|
9
|
-
};
|
|
10
|
-
|
|
11
|
-
export type DomainSkillCacheEntryLike = {
|
|
12
|
-
skillId: string;
|
|
13
|
-
endpointId?: string;
|
|
14
|
-
localSkillPath?: string;
|
|
15
|
-
ts: number;
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
export type CapturedDomainCacheEntryLike = {
|
|
19
|
-
skill: SkillManifest;
|
|
20
|
-
endpointId?: string;
|
|
21
|
-
expires: number;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export type RouteResultCacheEntryLike = {
|
|
25
|
-
skill: SkillManifest;
|
|
26
|
-
endpointId?: string;
|
|
27
|
-
result: unknown;
|
|
28
|
-
trace: unknown;
|
|
29
|
-
expires: number;
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
export type LocalCacheState = {
|
|
33
|
-
capturedDomainCache: Map<string, CapturedDomainCacheEntryLike>;
|
|
34
|
-
skillRouteCache: Map<string, SkillRouteCacheEntryLike>;
|
|
35
|
-
routeResultCache: Map<string, RouteResultCacheEntryLike>;
|
|
36
|
-
domainSkillCache: Map<string, DomainSkillCacheEntryLike>;
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
export type LocalCacheCleanupSummary = {
|
|
40
|
-
route_cache_entries_removed: number;
|
|
41
|
-
domain_cache_entries_removed: number;
|
|
42
|
-
route_result_entries_removed: number;
|
|
43
|
-
captured_domain_entries_removed: number;
|
|
44
|
-
};
|
|
45
|
-
|
|
46
|
-
export function isEndpointStale(
|
|
47
|
-
endpoint: Pick<SkillManifest["endpoints"][number], "verification_status" | "reliability_score"> | undefined,
|
|
48
|
-
): boolean {
|
|
49
|
-
if (!endpoint) return true;
|
|
50
|
-
if (endpoint.verification_status === "disabled" || endpoint.verification_status === "failed") return true;
|
|
51
|
-
return typeof endpoint.reliability_score === "number" && endpoint.reliability_score < 0.2;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export function countStaleEndpoints(skill: SkillManifest): number {
|
|
55
|
-
return skill.endpoints.filter((endpoint) => isEndpointStale(endpoint)).length;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
function isSkillEntryStale(
|
|
59
|
-
skill: SkillManifest,
|
|
60
|
-
endpointId?: string,
|
|
61
|
-
): boolean {
|
|
62
|
-
if (skill.lifecycle !== "active") return true;
|
|
63
|
-
if (!endpointId) return false;
|
|
64
|
-
return isEndpointStale(skill.endpoints.find((endpoint) => endpoint.endpoint_id === endpointId));
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export function pruneLocalCacheStateForSkill(
|
|
68
|
-
skill: SkillManifest,
|
|
69
|
-
caches: LocalCacheState,
|
|
70
|
-
): LocalCacheCleanupSummary {
|
|
71
|
-
let routeCacheRemoved = 0;
|
|
72
|
-
let domainCacheRemoved = 0;
|
|
73
|
-
let routeResultRemoved = 0;
|
|
74
|
-
let capturedRemoved = 0;
|
|
75
|
-
|
|
76
|
-
for (const [key, entry] of caches.skillRouteCache) {
|
|
77
|
-
if (entry.skillId !== skill.skill_id) continue;
|
|
78
|
-
if (!isSkillEntryStale(skill, entry.endpointId)) continue;
|
|
79
|
-
caches.skillRouteCache.delete(key);
|
|
80
|
-
routeCacheRemoved++;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
for (const [key, entry] of caches.domainSkillCache) {
|
|
84
|
-
if (entry.skillId !== skill.skill_id) continue;
|
|
85
|
-
if (!isSkillEntryStale(skill, entry.endpointId)) continue;
|
|
86
|
-
caches.domainSkillCache.delete(key);
|
|
87
|
-
domainCacheRemoved++;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
for (const [key, entry] of caches.routeResultCache) {
|
|
91
|
-
if (entry.skill.skill_id !== skill.skill_id) continue;
|
|
92
|
-
if (!isSkillEntryStale(skill, entry.endpointId)) continue;
|
|
93
|
-
caches.routeResultCache.delete(key);
|
|
94
|
-
routeResultRemoved++;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
for (const [key, entry] of caches.capturedDomainCache) {
|
|
98
|
-
if (entry.skill.skill_id !== skill.skill_id) continue;
|
|
99
|
-
if (!isSkillEntryStale(skill, entry.endpointId)) continue;
|
|
100
|
-
caches.capturedDomainCache.delete(key);
|
|
101
|
-
capturedRemoved++;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
return {
|
|
105
|
-
route_cache_entries_removed: routeCacheRemoved,
|
|
106
|
-
domain_cache_entries_removed: domainCacheRemoved,
|
|
107
|
-
route_result_entries_removed: routeResultRemoved,
|
|
108
|
-
captured_domain_entries_removed: capturedRemoved,
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
export function applyVerificationResults(
|
|
113
|
-
skill: SkillManifest,
|
|
114
|
-
verification: Record<string, VerificationStatus>,
|
|
115
|
-
): SkillManifest {
|
|
116
|
-
return {
|
|
117
|
-
...skill,
|
|
118
|
-
endpoints: skill.endpoints.map((endpoint) => {
|
|
119
|
-
const nextStatus = verification[endpoint.endpoint_id];
|
|
120
|
-
if (!nextStatus) return endpoint;
|
|
121
|
-
const recoveredReliability =
|
|
122
|
-
endpoint.verification_status === "disabled" && nextStatus === "verified"
|
|
123
|
-
? Math.max(endpoint.reliability_score ?? 0, 0.5)
|
|
124
|
-
: endpoint.reliability_score;
|
|
125
|
-
return {
|
|
126
|
-
...endpoint,
|
|
127
|
-
verification_status: nextStatus,
|
|
128
|
-
reliability_score: recoveredReliability,
|
|
129
|
-
last_verified_at: new Date().toISOString(),
|
|
130
|
-
};
|
|
131
|
-
}),
|
|
132
|
-
};
|
|
133
|
-
}
|