unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,221 +0,0 @@
1
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
- import { join } from "node:path";
3
- import { homedir } from "node:os";
4
- import { getRegistrableDomain } from "./domain.js";
5
-
6
- type RawConfig = Record<string, unknown>;
7
-
8
- export interface CapturePipelineSettings {
9
- auto_publish_checkpoints: boolean;
10
- publish_domain_blacklist: string[];
11
- publish_domain_promptlist: string[];
12
- }
13
-
14
- export interface CheckpointPublishDecision {
15
- publishQueued: boolean;
16
- mode: "auto" | "disabled" | "blacklisted" | "prompt";
17
- reason: string;
18
- matchedDomain?: string;
19
- }
20
-
21
- export interface ExplicitPublishDecision {
22
- allowed: boolean;
23
- mode: "allowed" | "blacklisted" | "prompt";
24
- reason: string;
25
- matchedDomain?: string;
26
- }
27
-
28
- const DEFAULT_CAPTURE_PIPELINE_SETTINGS: CapturePipelineSettings = {
29
- auto_publish_checkpoints: true,
30
- publish_domain_blacklist: [],
31
- publish_domain_promptlist: [],
32
- };
33
-
34
- function sanitizeProfileName(value: string): string {
35
- return value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
36
- }
37
-
38
- function getConfigDir(): string {
39
- if (process.env.UNBROWSE_CONFIG_DIR) return process.env.UNBROWSE_CONFIG_DIR;
40
- const profile = sanitizeProfileName(process.env.UNBROWSE_PROFILE ?? "");
41
- return profile
42
- ? join(homedir(), ".unbrowse", "profiles", profile)
43
- : join(homedir(), ".unbrowse");
44
- }
45
-
46
- export function getUnbrowseConfigPath(): string {
47
- return join(getConfigDir(), "config.json");
48
- }
49
-
50
- function loadRawConfig(): RawConfig {
51
- try {
52
- const configPath = getUnbrowseConfigPath();
53
- if (existsSync(configPath)) {
54
- const parsed = JSON.parse(readFileSync(configPath, "utf-8")) as unknown;
55
- if (parsed && typeof parsed === "object") return parsed as RawConfig;
56
- }
57
- } catch {
58
- // ignore broken local config and fall back to defaults
59
- }
60
- return {};
61
- }
62
-
63
- function saveRawConfig(config: RawConfig): void {
64
- const configDir = getConfigDir();
65
- if (!existsSync(configDir)) mkdirSync(configDir, { recursive: true });
66
- writeFileSync(getUnbrowseConfigPath(), JSON.stringify(config, null, 2), { mode: 0o600 });
67
- }
68
-
69
- function normalizeDomainRule(value: string): string {
70
- let normalized = value.trim().toLowerCase();
71
- if (!normalized) return "";
72
- normalized = normalized.replace(/^\*\./, "");
73
- if (normalized.startsWith("http://") || normalized.startsWith("https://")) {
74
- try {
75
- normalized = new URL(normalized).hostname.toLowerCase();
76
- } catch {
77
- normalized = normalized.replace(/^https?:\/\//, "");
78
- }
79
- }
80
- normalized = normalized.split("/")[0] ?? normalized;
81
- normalized = normalized.replace(/:\d+$/, "");
82
- normalized = normalized.replace(/^www\./, "");
83
- return normalized;
84
- }
85
-
86
- function normalizeDomainList(values: unknown): string[] {
87
- if (!Array.isArray(values)) return [];
88
- const normalized = values
89
- .filter((value): value is string => typeof value === "string")
90
- .map(normalizeDomainRule)
91
- .filter(Boolean);
92
- return Array.from(new Set(normalized)).sort();
93
- }
94
-
95
- function matchesDomainRule(domain: string, rule: string): boolean {
96
- const host = normalizeDomainRule(domain);
97
- const target = normalizeDomainRule(rule);
98
- if (!host || !target) return false;
99
- if (host === target || host.endsWith(`.${target}`)) return true;
100
- try {
101
- return getRegistrableDomain(host) === target;
102
- } catch {
103
- return false;
104
- }
105
- }
106
-
107
- function firstMatchingRule(domain: string, rules: string[]): string | undefined {
108
- return rules.find((rule) => matchesDomainRule(domain, rule));
109
- }
110
-
111
- export function getCapturePipelineSettings(): CapturePipelineSettings {
112
- const raw = loadRawConfig();
113
- const capturePipeline = raw.capture_pipeline && typeof raw.capture_pipeline === "object"
114
- ? raw.capture_pipeline as Record<string, unknown>
115
- : {};
116
-
117
- return {
118
- auto_publish_checkpoints: capturePipeline.auto_publish_checkpoints !== false,
119
- publish_domain_blacklist: normalizeDomainList(capturePipeline.publish_domain_blacklist),
120
- publish_domain_promptlist: normalizeDomainList(capturePipeline.publish_domain_promptlist),
121
- };
122
- }
123
-
124
- export function updateCapturePipelineSettings(update: {
125
- auto_publish_checkpoints?: boolean;
126
- publish_domain_blacklist?: string[];
127
- publish_domain_promptlist?: string[];
128
- clear_publish_domain_blacklist?: boolean;
129
- clear_publish_domain_promptlist?: boolean;
130
- }): CapturePipelineSettings {
131
- const raw = loadRawConfig();
132
- const existing = getCapturePipelineSettings();
133
- const next: CapturePipelineSettings = {
134
- auto_publish_checkpoints: typeof update.auto_publish_checkpoints === "boolean"
135
- ? update.auto_publish_checkpoints
136
- : existing.auto_publish_checkpoints,
137
- publish_domain_blacklist: update.clear_publish_domain_blacklist
138
- ? []
139
- : update.publish_domain_blacklist
140
- ? normalizeDomainList(update.publish_domain_blacklist)
141
- : existing.publish_domain_blacklist,
142
- publish_domain_promptlist: update.clear_publish_domain_promptlist
143
- ? []
144
- : update.publish_domain_promptlist
145
- ? normalizeDomainList(update.publish_domain_promptlist)
146
- : existing.publish_domain_promptlist,
147
- };
148
-
149
- saveRawConfig({
150
- ...raw,
151
- capture_pipeline: next,
152
- });
153
-
154
- return next;
155
- }
156
-
157
- export function decideCheckpointPublish(domain: string): CheckpointPublishDecision {
158
- const settings = getCapturePipelineSettings();
159
- if (!settings.auto_publish_checkpoints) {
160
- return {
161
- publishQueued: false,
162
- mode: "disabled",
163
- reason: "Auto-publish after sync/close is disabled in local settings.",
164
- };
165
- }
166
-
167
- const blacklisted = firstMatchingRule(domain, settings.publish_domain_blacklist);
168
- if (blacklisted) {
169
- return {
170
- publishQueued: false,
171
- mode: "blacklisted",
172
- reason: `Auto-publish skipped because ${domain} matches blacklist entry ${blacklisted}.`,
173
- matchedDomain: blacklisted,
174
- };
175
- }
176
-
177
- const prompted = firstMatchingRule(domain, settings.publish_domain_promptlist);
178
- if (prompted) {
179
- return {
180
- publishQueued: false,
181
- mode: "prompt",
182
- reason: `Auto-publish paused because ${domain} matches prompt-list entry ${prompted}.`,
183
- matchedDomain: prompted,
184
- };
185
- }
186
-
187
- return {
188
- publishQueued: true,
189
- mode: "auto",
190
- reason: "Background publish is allowed for this checkpoint.",
191
- };
192
- }
193
-
194
- export function decideExplicitPublish(domain: string, confirmPublish: boolean): ExplicitPublishDecision {
195
- const settings = getCapturePipelineSettings();
196
- const blacklisted = firstMatchingRule(domain, settings.publish_domain_blacklist);
197
- if (blacklisted && !confirmPublish) {
198
- return {
199
- allowed: false,
200
- mode: "blacklisted",
201
- reason: `Explicit publish requires confirmation because ${domain} matches blacklist entry ${blacklisted}.`,
202
- matchedDomain: blacklisted,
203
- };
204
- }
205
-
206
- const prompted = firstMatchingRule(domain, settings.publish_domain_promptlist);
207
- if (prompted && !confirmPublish) {
208
- return {
209
- allowed: false,
210
- mode: "prompt",
211
- reason: `Explicit publish requires confirmation because ${domain} matches prompt-list entry ${prompted}.`,
212
- matchedDomain: prompted,
213
- };
214
- }
215
-
216
- return {
217
- allowed: true,
218
- mode: "allowed",
219
- reason: "Explicit publish allowed.",
220
- };
221
- }
@@ -1,143 +0,0 @@
1
- #!/usr/bin/env bun
2
- /**
3
- * Unbrowse single-binary entrypoint.
4
- *
5
- * Modes:
6
- * unbrowse serve → start the API server inline
7
- * unbrowse [command] → CLI mode, auto-starts server if not running
8
- *
9
- * Kuri is embedded as a file asset. On first run it's extracted to
10
- * ~/.unbrowse/bin/kuri and reused from there.
11
- */
12
-
13
- import { existsSync, mkdirSync, writeFileSync, chmodSync, readFileSync } from "node:fs";
14
- import { join, dirname } from "node:path";
15
- import { homedir, platform, arch } from "node:os";
16
- import { execSync } from "node:child_process";
17
-
18
-
19
- // Embed the platform-appropriate kuri binary at compile time.
20
- // Bun's compiled binaries can read files from the virtual /$bunfs/ filesystem
21
- // when they are imported/required at bundle time.
22
- import kuriDarwinArm64Path from "../packages/skill/vendor/kuri/darwin-arm64/kuri" with { type: "file" };
23
- import kuriDarwinX64Path from "../packages/skill/vendor/kuri/darwin-x64/kuri" with { type: "file" };
24
- import kuriLinuxArm64Path from "../packages/skill/vendor/kuri/linux-arm64/kuri" with { type: "file" };
25
- import kuriLinuxX64Path from "../packages/skill/vendor/kuri/linux-x64/kuri" with { type: "file" };
26
-
27
- const UNBROWSE_HOME = join(homedir(), ".unbrowse");
28
- const KURI_CACHE = join(UNBROWSE_HOME, "bin", "kuri");
29
- const EMBEDDED_KURI_BY_TARGET: Record<string, string> = {
30
- "darwin-arm64": kuriDarwinArm64Path,
31
- "darwin-x64": kuriDarwinX64Path,
32
- "linux-arm64": kuriLinuxArm64Path,
33
- "linux-x64": kuriLinuxX64Path,
34
- };
35
-
36
- function currentKuriTarget(): string {
37
- return `${platform()}-${arch()}`;
38
- }
39
-
40
- function extractEmbeddedKuri(): string | null {
41
- const kuriBinaryPath = EMBEDDED_KURI_BY_TARGET[currentKuriTarget()];
42
- if (!kuriBinaryPath) {
43
- console.warn(`[unbrowse] no embedded kuri available for ${currentKuriTarget()}`);
44
- return null;
45
- }
46
- try {
47
- const data = readFileSync(kuriBinaryPath);
48
- mkdirSync(dirname(KURI_CACHE), { recursive: true });
49
- writeFileSync(KURI_CACHE, data);
50
- chmodSync(KURI_CACHE, 0o755);
51
- console.log(`[unbrowse] extracted kuri to ${KURI_CACHE}`);
52
- return KURI_CACHE;
53
- } catch (err) {
54
- console.warn(`[unbrowse] kuri extraction failed: ${(err as Error).message}`);
55
- return null;
56
- }
57
- }
58
- function findKuriBinary(): string | null {
59
- // 1. Explicit env var
60
- if (process.env.KURI_BIN && existsSync(process.env.KURI_BIN)) {
61
- return process.env.KURI_BIN;
62
- }
63
-
64
- // 2. Cached in ~/.unbrowse/bin/ (previously extracted)
65
- if (existsSync(KURI_CACHE)) {
66
- return KURI_CACHE;
67
- }
68
-
69
- // 3. Extract from embedded asset
70
- const extracted = extractEmbeddedKuri();
71
- if (extracted) return extracted;
72
-
73
- // 4. Alongside this binary
74
- const alongside = join(dirname(process.execPath), "kuri");
75
- if (existsSync(alongside)) {
76
- return alongside;
77
- }
78
-
79
- // 5. System PATH
80
- try {
81
- const which = execSync("which kuri 2>/dev/null", { encoding: "utf-8" }).trim();
82
- if (which && existsSync(which)) return which;
83
- } catch {}
84
-
85
- return null;
86
- }
87
-
88
- function ensureKuri(): string {
89
- const kuri = findKuriBinary();
90
- if (!kuri) {
91
- console.error("[unbrowse] kuri binary not found.");
92
- console.error("[unbrowse] Place kuri next to the unbrowse binary, or set KURI_BIN env var.");
93
- console.error("[unbrowse] Or run: unbrowse setup");
94
- process.exit(1);
95
- }
96
- process.env.KURI_BIN = kuri;
97
- return kuri;
98
- }
99
-
100
- async function main() {
101
- const args = process.argv.slice(2);
102
-
103
- // Ensure kuri is available
104
- ensureKuri();
105
-
106
- if (args[0] === "serve") {
107
- // Server mode — run inline
108
- const { startUnbrowseServer, installServerExitCleanup } = await import("./server.js");
109
- const { config: loadEnv } = await import("dotenv");
110
-
111
- loadEnv({ quiet: true });
112
- loadEnv({ path: ".env.runtime", quiet: true });
113
-
114
- const pidFile = process.env.UNBROWSE_PID_FILE;
115
- installServerExitCleanup(pidFile);
116
-
117
- const server = await startUnbrowseServer({
118
- pidFile,
119
- scheduleVerification: true,
120
- });
121
-
122
- const shutdown = async (signal: string) => {
123
- console.log(`[shutdown] ${signal}`);
124
- await server.close();
125
- process.exit(0);
126
- };
127
-
128
- process.on("SIGTERM", () => shutdown("SIGTERM").catch(() => process.exit(1)));
129
- process.on("SIGINT", () => shutdown("SIGINT").catch(() => process.exit(1)));
130
-
131
- console.log(`unbrowse running on http://${server.host}:${server.port}`);
132
- } else if (args[0] === "mcp-serve") {
133
- await import("./mcp.js");
134
- } else {
135
- // CLI mode
136
- await import("./cli.js");
137
- }
138
- }
139
-
140
- main().catch((err) => {
141
- console.error(err);
142
- process.exit(1);
143
- });
@@ -1,54 +0,0 @@
1
- import { getRegistrableDomain } from "./domain.js";
2
- import type { EndpointDescriptor, EndpointPolicyDescriptor } from "./types/index.js";
3
-
4
- const MUTATION_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
5
-
6
- type DomainPolicy = {
7
- domain: string;
8
- reason: string;
9
- require_for_mutations: boolean;
10
- };
11
-
12
- const DOMAIN_POLICIES: DomainPolicy[] = [
13
- {
14
- domain: "x.com",
15
- reason: "X write endpoints require explicit user confirmation because automated posting or other mutations may violate third-party site terms or automation rules.",
16
- require_for_mutations: true,
17
- },
18
- ];
19
-
20
- function resolveEndpointDomain(endpoint: EndpointDescriptor): string | null {
21
- try {
22
- const parsed = new URL(endpoint.url_template);
23
- return getRegistrableDomain(parsed.hostname);
24
- } catch {
25
- return null;
26
- }
27
- }
28
-
29
- export function getEndpointPolicy(endpoint: EndpointDescriptor): EndpointPolicyDescriptor | undefined {
30
- if (endpoint.policy?.requires_third_party_terms_confirmation) return endpoint.policy;
31
-
32
- const domain = resolveEndpointDomain(endpoint);
33
- if (!domain) return undefined;
34
-
35
- const matched = DOMAIN_POLICIES.find((policy) => policy.domain === domain);
36
- if (!matched) return undefined;
37
- if (matched.require_for_mutations && !MUTATION_METHODS.has(endpoint.method)) return undefined;
38
-
39
- return {
40
- requires_third_party_terms_confirmation: true,
41
- policy_domain: matched.domain,
42
- reason: matched.reason,
43
- };
44
- }
45
-
46
- export function endpointRequiresThirdPartyTermsConfirmation(endpoint: EndpointDescriptor): boolean {
47
- return !!getEndpointPolicy(endpoint)?.requires_third_party_terms_confirmation;
48
- }
49
-
50
- export function annotateEndpointPolicy<T extends EndpointDescriptor>(endpoint: T): T {
51
- const policy = getEndpointPolicy(endpoint);
52
- if (!policy || endpoint.policy) return endpoint;
53
- return { ...endpoint, policy } as T;
54
- }
@@ -1,144 +0,0 @@
1
- import { listSkills } from "./marketplace/index.js";
2
- import { verifySkill } from "./verification/index.js";
3
- import { selectVerificationCandidates } from "./verification/candidates.js";
4
- import type { SkillManifest } from "./types/index.js";
5
- import {
6
- applyVerificationResults,
7
- countStaleEndpoints,
8
- type LocalCacheCleanupSummary,
9
- } from "./stale-cleanup.js";
10
- import { pruneLocalCacheEntriesForSkill } from "./orchestrator/index.js";
11
-
12
- const VERIFY_TIMEOUT_MS = Math.max(5_000, Number(process.env.UNBROWSE_STALE_VERIFY_TIMEOUT_MS ?? 15_000));
13
- const CLEANUP_INTERVAL_MS = Math.max(30 * 60 * 1000, Number(process.env.UNBROWSE_STALE_CLEANUP_INTERVAL_MS ?? 6 * 60 * 60 * 1000));
14
- const CLEANUP_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_STALE_CLEANUP_BATCH_SIZE ?? 25));
15
- const CLEANUP_VERIFY_ENDPOINT_LIMIT = Math.max(1, Number(process.env.UNBROWSE_STALE_VERIFY_ENDPOINT_LIMIT ?? 2));
16
-
17
- export type SkillCleanupSummary = {
18
- skill_id: string;
19
- domain: string;
20
- lifecycle: SkillManifest["lifecycle"];
21
- verified: boolean;
22
- stale_before: number;
23
- stale_after: number;
24
- cache_cleanup: LocalCacheCleanupSummary;
25
- error?: string;
26
- };
27
-
28
- export async function cleanupStaleSkills(options?: {
29
- skill_id?: string;
30
- domain?: string;
31
- limit?: number;
32
- offset?: number;
33
- }): Promise<{
34
- ok: true;
35
- scanned_skills: number;
36
- verified_skills: number;
37
- stale_before: number;
38
- stale_after: number;
39
- cache_cleanup: LocalCacheCleanupSummary;
40
- skills: SkillCleanupSummary[];
41
- }> {
42
- const normalizedDomain = options?.domain?.trim().toLowerCase();
43
- let skills = await listSkills();
44
- if (options?.skill_id) skills = skills.filter((skill) => skill.skill_id === options.skill_id);
45
- if (normalizedDomain) {
46
- skills = skills.filter((skill) => {
47
- const domain = skill.domain.trim().toLowerCase();
48
- return domain === normalizedDomain || domain.replace(/^www\./, "") === normalizedDomain.replace(/^www\./, "");
49
- });
50
- }
51
- if (options?.offset && Number.isFinite(options.offset) && options.offset > 0) {
52
- skills = skills.slice(options.offset);
53
- }
54
- if (options?.limit && Number.isFinite(options.limit) && options.limit > 0) {
55
- skills = skills.slice(0, options.limit);
56
- }
57
-
58
- const perSkill: SkillCleanupSummary[] = [];
59
- const cacheTotals: LocalCacheCleanupSummary = {
60
- route_cache_entries_removed: 0,
61
- domain_cache_entries_removed: 0,
62
- route_result_entries_removed: 0,
63
- captured_domain_entries_removed: 0,
64
- };
65
- let staleBefore = 0;
66
- let staleAfter = 0;
67
- let verifiedSkills = 0;
68
-
69
- for (const skill of skills) {
70
- const before = countStaleEndpoints(skill);
71
- staleBefore += before;
72
- let refreshed = skill;
73
- let verified = false;
74
- let error: string | undefined;
75
-
76
- if (skill.lifecycle === "active" && skill.endpoints.some((endpoint) => endpoint.method === "GET")) {
77
- try {
78
- const endpoints = selectVerificationCandidates(skill, {
79
- staleOnly: true,
80
- limit: CLEANUP_VERIFY_ENDPOINT_LIMIT,
81
- });
82
- const verification = await Promise.race([
83
- verifySkill(skill, { endpoints }),
84
- new Promise<never>((_, reject) => {
85
- setTimeout(() => reject(new Error(`verification timed out after ${VERIFY_TIMEOUT_MS}ms`)), VERIFY_TIMEOUT_MS);
86
- }),
87
- ]);
88
- refreshed = applyVerificationResults(skill, verification);
89
- verified = true;
90
- verifiedSkills++;
91
- } catch (err) {
92
- error = (err as Error).message;
93
- }
94
- }
95
-
96
- const cacheCleanup = pruneLocalCacheEntriesForSkill(refreshed);
97
- cacheTotals.route_cache_entries_removed += cacheCleanup.route_cache_entries_removed;
98
- cacheTotals.domain_cache_entries_removed += cacheCleanup.domain_cache_entries_removed;
99
- cacheTotals.route_result_entries_removed += cacheCleanup.route_result_entries_removed;
100
- cacheTotals.captured_domain_entries_removed += cacheCleanup.captured_domain_entries_removed;
101
-
102
- const after = countStaleEndpoints(refreshed);
103
- staleAfter += after;
104
- perSkill.push({
105
- skill_id: refreshed.skill_id,
106
- domain: refreshed.domain,
107
- lifecycle: refreshed.lifecycle,
108
- verified,
109
- stale_before: before,
110
- stale_after: after,
111
- cache_cleanup: cacheCleanup,
112
- ...(error ? { error } : {}),
113
- });
114
- }
115
-
116
- return {
117
- ok: true,
118
- scanned_skills: skills.length,
119
- verified_skills: verifiedSkills,
120
- stale_before: staleBefore,
121
- stale_after: staleAfter,
122
- cache_cleanup: cacheTotals,
123
- skills: perSkill,
124
- };
125
- }
126
-
127
- export function schedulePeriodicStaleCleanup(): ReturnType<typeof setInterval> {
128
- let offset = 0;
129
- return setInterval(async () => {
130
- try {
131
- const result = await cleanupStaleSkills({
132
- limit: CLEANUP_BATCH_SIZE,
133
- offset,
134
- });
135
- if (result.scanned_skills < CLEANUP_BATCH_SIZE) offset = 0;
136
- else offset += result.scanned_skills;
137
- console.log(
138
- `[stale-cleanup] scanned=${result.scanned_skills} verified=${result.verified_skills} stale=${result.stale_before}->${result.stale_after} removed=${result.cache_cleanup.route_cache_entries_removed + result.cache_cleanup.domain_cache_entries_removed + result.cache_cleanup.route_result_entries_removed + result.cache_cleanup.captured_domain_entries_removed}`,
139
- );
140
- } catch (err) {
141
- console.warn(`[stale-cleanup] sweep failed: ${(err as Error).message}`);
142
- }
143
- }, CLEANUP_INTERVAL_MS);
144
- }
@@ -1,133 +0,0 @@
1
- import type { SkillManifest, VerificationStatus } from "./types/index.js";
2
-
3
- export type SkillRouteCacheEntryLike = {
4
- skillId: string;
5
- domain: string;
6
- endpointId?: string;
7
- localSkillPath?: string;
8
- ts: number;
9
- };
10
-
11
- export type DomainSkillCacheEntryLike = {
12
- skillId: string;
13
- endpointId?: string;
14
- localSkillPath?: string;
15
- ts: number;
16
- };
17
-
18
- export type CapturedDomainCacheEntryLike = {
19
- skill: SkillManifest;
20
- endpointId?: string;
21
- expires: number;
22
- };
23
-
24
- export type RouteResultCacheEntryLike = {
25
- skill: SkillManifest;
26
- endpointId?: string;
27
- result: unknown;
28
- trace: unknown;
29
- expires: number;
30
- };
31
-
32
- export type LocalCacheState = {
33
- capturedDomainCache: Map<string, CapturedDomainCacheEntryLike>;
34
- skillRouteCache: Map<string, SkillRouteCacheEntryLike>;
35
- routeResultCache: Map<string, RouteResultCacheEntryLike>;
36
- domainSkillCache: Map<string, DomainSkillCacheEntryLike>;
37
- };
38
-
39
- export type LocalCacheCleanupSummary = {
40
- route_cache_entries_removed: number;
41
- domain_cache_entries_removed: number;
42
- route_result_entries_removed: number;
43
- captured_domain_entries_removed: number;
44
- };
45
-
46
- export function isEndpointStale(
47
- endpoint: Pick<SkillManifest["endpoints"][number], "verification_status" | "reliability_score"> | undefined,
48
- ): boolean {
49
- if (!endpoint) return true;
50
- if (endpoint.verification_status === "disabled" || endpoint.verification_status === "failed") return true;
51
- return typeof endpoint.reliability_score === "number" && endpoint.reliability_score < 0.2;
52
- }
53
-
54
- export function countStaleEndpoints(skill: SkillManifest): number {
55
- return skill.endpoints.filter((endpoint) => isEndpointStale(endpoint)).length;
56
- }
57
-
58
- function isSkillEntryStale(
59
- skill: SkillManifest,
60
- endpointId?: string,
61
- ): boolean {
62
- if (skill.lifecycle !== "active") return true;
63
- if (!endpointId) return false;
64
- return isEndpointStale(skill.endpoints.find((endpoint) => endpoint.endpoint_id === endpointId));
65
- }
66
-
67
- export function pruneLocalCacheStateForSkill(
68
- skill: SkillManifest,
69
- caches: LocalCacheState,
70
- ): LocalCacheCleanupSummary {
71
- let routeCacheRemoved = 0;
72
- let domainCacheRemoved = 0;
73
- let routeResultRemoved = 0;
74
- let capturedRemoved = 0;
75
-
76
- for (const [key, entry] of caches.skillRouteCache) {
77
- if (entry.skillId !== skill.skill_id) continue;
78
- if (!isSkillEntryStale(skill, entry.endpointId)) continue;
79
- caches.skillRouteCache.delete(key);
80
- routeCacheRemoved++;
81
- }
82
-
83
- for (const [key, entry] of caches.domainSkillCache) {
84
- if (entry.skillId !== skill.skill_id) continue;
85
- if (!isSkillEntryStale(skill, entry.endpointId)) continue;
86
- caches.domainSkillCache.delete(key);
87
- domainCacheRemoved++;
88
- }
89
-
90
- for (const [key, entry] of caches.routeResultCache) {
91
- if (entry.skill.skill_id !== skill.skill_id) continue;
92
- if (!isSkillEntryStale(skill, entry.endpointId)) continue;
93
- caches.routeResultCache.delete(key);
94
- routeResultRemoved++;
95
- }
96
-
97
- for (const [key, entry] of caches.capturedDomainCache) {
98
- if (entry.skill.skill_id !== skill.skill_id) continue;
99
- if (!isSkillEntryStale(skill, entry.endpointId)) continue;
100
- caches.capturedDomainCache.delete(key);
101
- capturedRemoved++;
102
- }
103
-
104
- return {
105
- route_cache_entries_removed: routeCacheRemoved,
106
- domain_cache_entries_removed: domainCacheRemoved,
107
- route_result_entries_removed: routeResultRemoved,
108
- captured_domain_entries_removed: capturedRemoved,
109
- };
110
- }
111
-
112
- export function applyVerificationResults(
113
- skill: SkillManifest,
114
- verification: Record<string, VerificationStatus>,
115
- ): SkillManifest {
116
- return {
117
- ...skill,
118
- endpoints: skill.endpoints.map((endpoint) => {
119
- const nextStatus = verification[endpoint.endpoint_id];
120
- if (!nextStatus) return endpoint;
121
- const recoveredReliability =
122
- endpoint.verification_status === "disabled" && nextStatus === "verified"
123
- ? Math.max(endpoint.reliability_score ?? 0, 0.5)
124
- : endpoint.reliability_score;
125
- return {
126
- ...endpoint,
127
- verification_status: nextStatus,
128
- reliability_score: recoveredReliability,
129
- last_verified_at: new Date().toISOString(),
130
- };
131
- }),
132
- };
133
- }