unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +79 -33
- package/dist/index.js +2 -6
- package/dist/mcp.js +73 -22
- package/dist/server.js +25994 -0
- package/package.json +1 -2
- package/vendor/kuri/manifest.json +1 -1
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -343
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1833
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1929
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1734
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3407
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/execution/token-resolver.ts +0 -135
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1532
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/impact-log.ts +0 -227
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -111
- package/runtime-src/mcp.ts +0 -1911
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/reverse-engineer/token-sources.ts +0 -379
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -931
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
const RETRYABLE_STATUSES = new Set([500, 502, 503, 504, 429]);
|
|
2
|
-
const MAX_RETRIES = 2;
|
|
3
|
-
const BASE_DELAY_MS = 1000;
|
|
4
|
-
const MAX_DELAY_MS = 10000;
|
|
5
|
-
|
|
6
|
-
export interface RetryOptions {
|
|
7
|
-
maxRetries?: number;
|
|
8
|
-
baseDelay?: number;
|
|
9
|
-
maxDelay?: number;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* Wrap an async function with exponential backoff retry.
|
|
14
|
-
* Only retries if isRetryable returns true.
|
|
15
|
-
*/
|
|
16
|
-
export async function withRetry<T>(
|
|
17
|
-
fn: () => Promise<T>,
|
|
18
|
-
isRetryable: (result: T) => boolean,
|
|
19
|
-
opts?: RetryOptions
|
|
20
|
-
): Promise<T> {
|
|
21
|
-
const maxRetries = opts?.maxRetries ?? MAX_RETRIES;
|
|
22
|
-
const baseDelay = opts?.baseDelay ?? BASE_DELAY_MS;
|
|
23
|
-
const maxDelay = opts?.maxDelay ?? MAX_DELAY_MS;
|
|
24
|
-
|
|
25
|
-
let lastResult: T | undefined;
|
|
26
|
-
|
|
27
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
28
|
-
const result = await fn();
|
|
29
|
-
lastResult = result;
|
|
30
|
-
|
|
31
|
-
if (!isRetryable(result) || attempt === maxRetries) {
|
|
32
|
-
return result;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
// Exponential backoff with jitter
|
|
36
|
-
const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
|
|
37
|
-
const jitter = delay * 0.5 * Math.random();
|
|
38
|
-
await new Promise((r) => setTimeout(r, delay + jitter));
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
return lastResult!;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
export function isRetryableStatus(status: number): boolean {
|
|
45
|
-
return RETRYABLE_STATUSES.has(status);
|
|
46
|
-
}
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* robots.txt compliance check for route execution.
|
|
3
|
-
*
|
|
4
|
-
* Fetches and caches robots.txt per domain (24 h TTL), then checks
|
|
5
|
-
* whether a given path is allowed for the Unbrowse user-agent.
|
|
6
|
-
*
|
|
7
|
-
* RFC 9309-compliant: the function checks rules for "unbrowse" first,
|
|
8
|
-
* then falls back to the wildcard agent "*". Paths under Disallow
|
|
9
|
-
* directives are blocked; an absent or unparseable robots.txt is
|
|
10
|
-
* treated as fully permissive.
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
const USER_AGENT = "unbrowse";
|
|
14
|
-
const TTL_MS = 24 * 60 * 60 * 1000; // 24 h
|
|
15
|
-
|
|
16
|
-
interface CacheEntry {
|
|
17
|
-
rules: AgentRules[];
|
|
18
|
-
fetchedAt: number;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
interface AgentRules {
|
|
22
|
-
agents: string[]; // lowercase user-agent names; ["*"] for wildcard
|
|
23
|
-
disallow: string[];
|
|
24
|
-
allow: string[];
|
|
25
|
-
crawlDelay?: number;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// In-process cache — survives for the lifetime of the process.
|
|
29
|
-
const cache = new Map<string, CacheEntry>();
|
|
30
|
-
|
|
31
|
-
/** Exported for tests only — clear the in-process cache. */
|
|
32
|
-
export function clearRobotsCache(): void {
|
|
33
|
-
cache.clear();
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/** Parse a raw robots.txt string into per-agent rule blocks. */
|
|
37
|
-
export function parseRobotsTxt(text: string): AgentRules[] {
|
|
38
|
-
const groups: AgentRules[] = [];
|
|
39
|
-
let current: AgentRules | null = null;
|
|
40
|
-
|
|
41
|
-
for (const rawLine of text.split(/\r?\n/)) {
|
|
42
|
-
const line = rawLine.replace(/#.*$/, "").trim();
|
|
43
|
-
if (!line) {
|
|
44
|
-
// Blank line ends the current group.
|
|
45
|
-
current = null;
|
|
46
|
-
continue;
|
|
47
|
-
}
|
|
48
|
-
const colon = line.indexOf(":");
|
|
49
|
-
if (colon === -1) continue;
|
|
50
|
-
const field = line.slice(0, colon).trim().toLowerCase();
|
|
51
|
-
const value = line.slice(colon + 1).trim();
|
|
52
|
-
|
|
53
|
-
if (field === "user-agent") {
|
|
54
|
-
if (!current) {
|
|
55
|
-
current = { agents: [], disallow: [], allow: [] };
|
|
56
|
-
groups.push(current);
|
|
57
|
-
}
|
|
58
|
-
current.agents.push(value.toLowerCase());
|
|
59
|
-
} else if (field === "disallow") {
|
|
60
|
-
if (current && value) current.disallow.push(value);
|
|
61
|
-
} else if (field === "allow") {
|
|
62
|
-
if (current && value) current.allow.push(value);
|
|
63
|
-
} else if (field === "crawl-delay") {
|
|
64
|
-
if (current) current.crawlDelay = parseFloat(value);
|
|
65
|
-
} else {
|
|
66
|
-
// Unknown directive — reset so a subsequent User-agent starts fresh.
|
|
67
|
-
current = null;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
return groups;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/** Return the best-matching rule block for a given user-agent string. */
|
|
75
|
-
function selectRules(groups: AgentRules[], agent: string): AgentRules | null {
|
|
76
|
-
const lower = agent.toLowerCase();
|
|
77
|
-
// Exact match first.
|
|
78
|
-
const exact = groups.find((g) => g.agents.some((a) => a === lower));
|
|
79
|
-
if (exact) return exact;
|
|
80
|
-
// Wildcard fallback.
|
|
81
|
-
return groups.find((g) => g.agents.includes("*")) ?? null;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/** True when `path` starts with `prefix` (robots.txt path matching). */
|
|
85
|
-
function pathMatches(path: string, prefix: string): boolean {
|
|
86
|
-
// The "$" anchor means the pattern must match the entire path.
|
|
87
|
-
if (prefix.endsWith("$")) {
|
|
88
|
-
return path === prefix.slice(0, -1);
|
|
89
|
-
}
|
|
90
|
-
return path.startsWith(prefix);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/** Return the length of the longest matching Allow/Disallow prefix. */
|
|
94
|
-
function longestMatch(path: string, patterns: string[]): number {
|
|
95
|
-
let best = -1;
|
|
96
|
-
for (const p of patterns) {
|
|
97
|
-
const base = p.endsWith("$") ? p.slice(0, -1) : p;
|
|
98
|
-
if (pathMatches(path, p) && base.length > best) best = base.length;
|
|
99
|
-
}
|
|
100
|
-
return best;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* Fetch (with caching) and parse robots.txt for a given origin.
|
|
105
|
-
* Never throws — returns [] (fully permissive) on any error.
|
|
106
|
-
*/
|
|
107
|
-
async function fetchRules(origin: string): Promise<AgentRules[]> {
|
|
108
|
-
const now = Date.now();
|
|
109
|
-
const cached = cache.get(origin);
|
|
110
|
-
if (cached && now - cached.fetchedAt < TTL_MS) return cached.rules;
|
|
111
|
-
|
|
112
|
-
try {
|
|
113
|
-
const res = await fetch(`${origin}/robots.txt`, {
|
|
114
|
-
headers: { "user-agent": USER_AGENT },
|
|
115
|
-
redirect: "follow",
|
|
116
|
-
signal: AbortSignal.timeout(5000),
|
|
117
|
-
});
|
|
118
|
-
if (!res.ok) {
|
|
119
|
-
// 4xx → treat as no restrictions; 5xx → same (permissive on error).
|
|
120
|
-
cache.set(origin, { rules: [], fetchedAt: now });
|
|
121
|
-
return [];
|
|
122
|
-
}
|
|
123
|
-
const text = await res.text();
|
|
124
|
-
const rules = parseRobotsTxt(text);
|
|
125
|
-
cache.set(origin, { rules, fetchedAt: now });
|
|
126
|
-
return rules;
|
|
127
|
-
} catch {
|
|
128
|
-
// Network error / timeout → permissive.
|
|
129
|
-
cache.set(origin, { rules: [], fetchedAt: now });
|
|
130
|
-
return [];
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
/**
|
|
135
|
-
* Check whether the Unbrowse user-agent is allowed to access `url`.
|
|
136
|
-
*
|
|
137
|
-
* Returns `true` (allowed) when:
|
|
138
|
-
* - robots.txt cannot be fetched or parsed
|
|
139
|
-
* - no matching rule exists
|
|
140
|
-
* - the most-specific matching rule is an Allow directive
|
|
141
|
-
*
|
|
142
|
-
* Returns `false` (blocked) only when the most-specific matching rule
|
|
143
|
-
* is a Disallow directive with a longer (or equal) prefix than any
|
|
144
|
-
* matching Allow directive.
|
|
145
|
-
*/
|
|
146
|
-
export async function isAllowedByRobots(url: string): Promise<boolean> {
|
|
147
|
-
let origin: string;
|
|
148
|
-
let pathname: string;
|
|
149
|
-
try {
|
|
150
|
-
const parsed = new URL(url);
|
|
151
|
-
origin = parsed.origin;
|
|
152
|
-
pathname = parsed.pathname || "/";
|
|
153
|
-
} catch {
|
|
154
|
-
return true; // unparseable URL — let execution proceed
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
const groups = await fetchRules(origin);
|
|
158
|
-
const rules = selectRules(groups, USER_AGENT);
|
|
159
|
-
if (!rules) return true; // no matching rules — fully allowed
|
|
160
|
-
|
|
161
|
-
const allowLen = longestMatch(pathname, rules.allow);
|
|
162
|
-
const disallowLen = longestMatch(pathname, rules.disallow);
|
|
163
|
-
|
|
164
|
-
if (disallowLen < 0) return true; // no disallow matched
|
|
165
|
-
if (allowLen >= disallowLen) return true; // allow wins on tie or longer match
|
|
166
|
-
return false;
|
|
167
|
-
}
|
|
@@ -1,188 +0,0 @@
|
|
|
1
|
-
export interface SearchFormField {
|
|
2
|
-
name: string;
|
|
3
|
-
type: "text" | "select" | "radio" | "checkbox" | "date" | "hidden";
|
|
4
|
-
selector: string;
|
|
5
|
-
options?: string[];
|
|
6
|
-
required: boolean;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export interface SearchFormSpec {
|
|
10
|
-
form_selector: string;
|
|
11
|
-
submit_selector: string;
|
|
12
|
-
fields: SearchFormField[];
|
|
13
|
-
result_selector?: string;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export function isStructuredSearchForm(spec: SearchFormSpec): boolean {
|
|
17
|
-
return spec.fields.length > 0 && !!spec.submit_selector;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
// ---------------------------------------------------------------------------
|
|
21
|
-
// HTML detection — parse raw HTML to discover search forms
|
|
22
|
-
// ---------------------------------------------------------------------------
|
|
23
|
-
|
|
24
|
-
const SEARCH_FIELD_NAMES = new Set([
|
|
25
|
-
"q", "query", "search", "keyword", "keywords", "term", "terms",
|
|
26
|
-
"find", "lookup", "filter", "s", "text", "input",
|
|
27
|
-
]);
|
|
28
|
-
|
|
29
|
-
const LOGIN_FIELD_NAMES = new Set([
|
|
30
|
-
"password", "passwd", "pass", "pwd", "confirm_password",
|
|
31
|
-
"username", "email", "login", "user",
|
|
32
|
-
]);
|
|
33
|
-
|
|
34
|
-
const SUPPORTED_INPUT_TYPES = new Set([
|
|
35
|
-
"text", "search", "hidden", "date", "number", "tel", "email",
|
|
36
|
-
]);
|
|
37
|
-
|
|
38
|
-
function formSelectorFromElement(
|
|
39
|
-
attribs: Record<string, string>,
|
|
40
|
-
index: number,
|
|
41
|
-
): string {
|
|
42
|
-
const id = attribs.id;
|
|
43
|
-
if (id) return `form#${id}`;
|
|
44
|
-
const name = attribs.name;
|
|
45
|
-
if (name) return `form[name="${name}"]`;
|
|
46
|
-
const action = attribs.action;
|
|
47
|
-
if (action) return `form[action="${action}"]`;
|
|
48
|
-
return `form:nth-of-type(${index + 1})`;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function inputSelectorFromElement(
|
|
52
|
-
attribs: Record<string, string>,
|
|
53
|
-
tagName: string,
|
|
54
|
-
): string {
|
|
55
|
-
const id = attribs.id;
|
|
56
|
-
if (id) return `#${id}`;
|
|
57
|
-
const name = attribs.name;
|
|
58
|
-
if (name) return `${tagName}[name="${name}"]`;
|
|
59
|
-
return tagName;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function mapInputType(
|
|
63
|
-
typeAttr: string | undefined,
|
|
64
|
-
tagName: string,
|
|
65
|
-
): SearchFormField["type"] | null {
|
|
66
|
-
if (tagName === "select") return "select";
|
|
67
|
-
if (tagName === "textarea") return "text";
|
|
68
|
-
const t = (typeAttr ?? "text").toLowerCase();
|
|
69
|
-
if (t === "radio") return "radio";
|
|
70
|
-
if (t === "checkbox") return "checkbox";
|
|
71
|
-
if (t === "date") return "date";
|
|
72
|
-
if (t === "hidden") return "hidden";
|
|
73
|
-
if (t === "submit" || t === "button" || t === "image" || t === "reset") return null;
|
|
74
|
-
if (t === "password" || t === "file") return null;
|
|
75
|
-
if (SUPPORTED_INPUT_TYPES.has(t)) return "text";
|
|
76
|
-
return "text";
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function parseAttrs(attrStr: string): Record<string, string> {
|
|
80
|
-
const attrs: Record<string, string> = {};
|
|
81
|
-
const attrRegex = /(\w[\w-]*)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|(\S+)))?/g;
|
|
82
|
-
let m: RegExpExecArray | null;
|
|
83
|
-
while ((m = attrRegex.exec(attrStr)) !== null) {
|
|
84
|
-
attrs[m[1]] = m[2] ?? m[3] ?? m[4] ?? "";
|
|
85
|
-
}
|
|
86
|
-
return attrs;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Detect structured search forms from raw HTML.
|
|
91
|
-
* Returns a SearchFormSpec for each form that looks like a search/filter form
|
|
92
|
-
* (has at least one search-like field and a submit mechanism).
|
|
93
|
-
* Login/password forms are excluded.
|
|
94
|
-
*/
|
|
95
|
-
export function detectSearchForms(html: string): SearchFormSpec[] {
|
|
96
|
-
const results: SearchFormSpec[] = [];
|
|
97
|
-
const formRegex = /<form([^>]*)>([\s\S]*?)<\/form>/gi;
|
|
98
|
-
let formMatch: RegExpExecArray | null;
|
|
99
|
-
let formIndex = 0;
|
|
100
|
-
|
|
101
|
-
while ((formMatch = formRegex.exec(html)) !== null) {
|
|
102
|
-
const formAttrs = formMatch[1];
|
|
103
|
-
const formBody = formMatch[2];
|
|
104
|
-
|
|
105
|
-
const formElAttrs = parseAttrs(formAttrs);
|
|
106
|
-
|
|
107
|
-
// Find all input/select/textarea elements
|
|
108
|
-
const fieldRegex = /<(input|select|textarea)([^>]*)\/?>/gi;
|
|
109
|
-
let fieldMatch: RegExpExecArray | null;
|
|
110
|
-
const fields: SearchFormField[] = [];
|
|
111
|
-
const seenNames = new Set<string>();
|
|
112
|
-
let hasLoginField = false;
|
|
113
|
-
let hasSearchLikeField = false;
|
|
114
|
-
|
|
115
|
-
while ((fieldMatch = fieldRegex.exec(formBody)) !== null) {
|
|
116
|
-
const tagName = fieldMatch[1].toLowerCase();
|
|
117
|
-
const fieldAttrs = parseAttrs(fieldMatch[2]);
|
|
118
|
-
const name = fieldAttrs.name ?? "";
|
|
119
|
-
const typeAttr = fieldAttrs.type;
|
|
120
|
-
|
|
121
|
-
// Check for login-form indicators
|
|
122
|
-
if (LOGIN_FIELD_NAMES.has(name.toLowerCase()) || typeAttr === "password") {
|
|
123
|
-
hasLoginField = true;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Check for search-like fields
|
|
127
|
-
if (SEARCH_FIELD_NAMES.has(name.toLowerCase())) {
|
|
128
|
-
hasSearchLikeField = true;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
const mappedType = mapInputType(typeAttr, tagName);
|
|
132
|
-
if (!mappedType) continue;
|
|
133
|
-
if (!name && mappedType !== "text") continue;
|
|
134
|
-
if (seenNames.has(name) && mappedType !== "radio") continue;
|
|
135
|
-
if (name) seenNames.add(name);
|
|
136
|
-
|
|
137
|
-
// Collect select options
|
|
138
|
-
let options: string[] | undefined;
|
|
139
|
-
if (tagName === "select") {
|
|
140
|
-
const optRegex = /<option[^>]*value="([^"]*)"[^>]*>/gi;
|
|
141
|
-
let optMatch: RegExpExecArray | null;
|
|
142
|
-
options = [];
|
|
143
|
-
while ((optMatch = optRegex.exec(formBody)) !== null) {
|
|
144
|
-
options.push(optMatch[1]);
|
|
145
|
-
}
|
|
146
|
-
if (options.length === 0) options = undefined;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
fields.push({
|
|
150
|
-
name: name || `unnamed_${fields.length}`,
|
|
151
|
-
type: mappedType,
|
|
152
|
-
selector: inputSelectorFromElement(fieldAttrs, tagName),
|
|
153
|
-
...(options ? { options } : {}),
|
|
154
|
-
required: fieldAttrs.required !== undefined,
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
// Detect submit mechanism
|
|
159
|
-
let submitSelector = "";
|
|
160
|
-
if (/<button[^>]*type\s*=\s*"submit"/i.test(formBody)) {
|
|
161
|
-
submitSelector = "button[type=submit]";
|
|
162
|
-
} else if (/<input[^>]*type\s*=\s*"submit"/i.test(formBody)) {
|
|
163
|
-
submitSelector = 'input[type="submit"]';
|
|
164
|
-
} else if (/<button/i.test(formBody)) {
|
|
165
|
-
submitSelector = "button";
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
// Skip login forms, require at least one meaningful field
|
|
169
|
-
const nonHiddenFields = fields.filter((f) => f.type !== "hidden");
|
|
170
|
-
if (
|
|
171
|
-
!hasLoginField &&
|
|
172
|
-
nonHiddenFields.length > 0 &&
|
|
173
|
-
submitSelector &&
|
|
174
|
-
(hasSearchLikeField || nonHiddenFields.length >= 1)
|
|
175
|
-
) {
|
|
176
|
-
const formSelector = formSelectorFromElement(formElAttrs, formIndex);
|
|
177
|
-
results.push({
|
|
178
|
-
form_selector: formSelector,
|
|
179
|
-
submit_selector: submitSelector,
|
|
180
|
-
fields,
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
formIndex++;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
return results;
|
|
188
|
-
}
|
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Token resolver — resolves auth_tokens bindings at execute time.
|
|
3
|
-
*
|
|
4
|
-
* Given an endpoint with auth_tokens, loads the trigger page via Kuri,
|
|
5
|
-
* extracts fresh token values from HTML/JS sources, and returns headers
|
|
6
|
-
* ready to inject into the outgoing request.
|
|
7
|
-
*
|
|
8
|
-
* This closes the gap for sites that rotate CSRF tokens per page-load,
|
|
9
|
-
* keep bearer tokens in JS bundles, or hydrate tokens into inline scripts.
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
import type { EndpointDescriptor, AuthTokenBinding } from "../types/index.js";
|
|
13
|
-
import { extractTokenFromHtml, extractTokenFromBundle } from "../reverse-engineer/token-sources.js";
|
|
14
|
-
import * as kuri from "../kuri/client.js";
|
|
15
|
-
|
|
16
|
-
const RESOLVE_TIMEOUT_MS = 12000;
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Resolve auth_tokens bindings by loading the trigger page and scraping
|
|
20
|
-
* token values from their known source locations.
|
|
21
|
-
*
|
|
22
|
-
* Returns a map of header-name → resolved-value for all successfully
|
|
23
|
-
* resolved header-type bindings. Returns empty map if no bindings or
|
|
24
|
-
* all fail.
|
|
25
|
-
*/
|
|
26
|
-
export async function resolveAuthTokens(
|
|
27
|
-
endpoint: EndpointDescriptor,
|
|
28
|
-
cookies: Array<{ name: string; value: string; domain: string }>,
|
|
29
|
-
existingAuthHeaders: Record<string, string>,
|
|
30
|
-
): Promise<Record<string, string>> {
|
|
31
|
-
const bindings = endpoint.auth_tokens;
|
|
32
|
-
if (!bindings || bindings.length === 0) return {};
|
|
33
|
-
|
|
34
|
-
const triggerUrl = endpoint.trigger_url;
|
|
35
|
-
if (!triggerUrl) return {};
|
|
36
|
-
|
|
37
|
-
// Resolve ALL header bindings — DAG sources are authoritative over vault cache
|
|
38
|
-
const headerBindings = bindings.filter((b) => b.param_location === "header");
|
|
39
|
-
if (headerBindings.length === 0) return {};
|
|
40
|
-
|
|
41
|
-
const resolved: Record<string, string> = {};
|
|
42
|
-
|
|
43
|
-
try {
|
|
44
|
-
// Open a tab, inject cookies, navigate to trigger page
|
|
45
|
-
const tabId = await openResolverTab(triggerUrl, cookies);
|
|
46
|
-
if (!tabId) return {};
|
|
47
|
-
|
|
48
|
-
try {
|
|
49
|
-
await waitForLoad(tabId);
|
|
50
|
-
|
|
51
|
-
const html = await kuri.getPageHtml(tabId).catch(() => "");
|
|
52
|
-
if (typeof html !== "string" || !html.startsWith("<")) {
|
|
53
|
-
return {};
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
for (const binding of headerBindings) {
|
|
57
|
-
const value = await resolveBinding(binding, html, cookies);
|
|
58
|
-
if (value) {
|
|
59
|
-
resolved[binding.param_name] = binding.param_name.toLowerCase() === "authorization"
|
|
60
|
-
? (value.startsWith("Bearer ") ? value : `Bearer ${value}`)
|
|
61
|
-
: value;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
} finally {
|
|
65
|
-
await kuri.closeTab(tabId).catch(() => {});
|
|
66
|
-
}
|
|
67
|
-
} catch {
|
|
68
|
-
// Tab open/nav failed — return whatever we resolved so far
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
return resolved;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async function resolveBinding(
|
|
75
|
-
binding: AuthTokenBinding,
|
|
76
|
-
html: string,
|
|
77
|
-
cookies: Array<{ name: string; value: string; domain: string }>,
|
|
78
|
-
): Promise<string | undefined> {
|
|
79
|
-
for (const source of binding.sources) {
|
|
80
|
-
let value: string | undefined;
|
|
81
|
-
|
|
82
|
-
if (source.kind === "cookie" && source.cookie_names?.length) {
|
|
83
|
-
// Resolve from cookies — CSRF tokens typically live here
|
|
84
|
-
for (const name of source.cookie_names) {
|
|
85
|
-
const cookie = cookies.find((c) => c.name === name);
|
|
86
|
-
if (cookie?.value) { value = cookie.value; break; }
|
|
87
|
-
}
|
|
88
|
-
} else if (source.kind === "html-meta" || source.kind === "html-inline-script") {
|
|
89
|
-
value = extractTokenFromHtml(source, html);
|
|
90
|
-
} else if (source.kind === "js-bundle" && source.bundle_url_pattern) {
|
|
91
|
-
try {
|
|
92
|
-
const resp = await fetch(source.bundle_url_pattern);
|
|
93
|
-
if (resp.ok) {
|
|
94
|
-
const body = await resp.text();
|
|
95
|
-
value = extractTokenFromBundle(source, body);
|
|
96
|
-
}
|
|
97
|
-
} catch { /* fetch failed — try next source */ }
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
if (value && value.length >= 8) return value;
|
|
101
|
-
}
|
|
102
|
-
return undefined;
|
|
103
|
-
}
|
|
104
|
-
async function openResolverTab(
|
|
105
|
-
url: string,
|
|
106
|
-
cookies: Array<{ name: string; value: string; domain: string }>,
|
|
107
|
-
): Promise<string | undefined> {
|
|
108
|
-
try {
|
|
109
|
-
const tab = await kuri.newTab(url);
|
|
110
|
-
const tabId = typeof tab === "string" ? tab : (tab as { tab_id?: string })?.tab_id;
|
|
111
|
-
if (!tabId) return undefined;
|
|
112
|
-
|
|
113
|
-
if (cookies.length > 0) {
|
|
114
|
-
for (const c of cookies) {
|
|
115
|
-
await kuri.setCookie(tabId, c.name, c.value, c.domain).catch(() => {});
|
|
116
|
-
}
|
|
117
|
-
await kuri.navigate(tabId, url).catch(() => {});
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
return tabId;
|
|
121
|
-
} catch {
|
|
122
|
-
return undefined;
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
async function waitForLoad(tabId: string): Promise<void> {
|
|
127
|
-
const start = Date.now();
|
|
128
|
-
while (Date.now() - start < RESOLVE_TIMEOUT_MS) {
|
|
129
|
-
try {
|
|
130
|
-
const state = await kuri.evaluate(tabId, "document.readyState");
|
|
131
|
-
if (state === "complete" || state === "interactive") return;
|
|
132
|
-
} catch { /* page not ready */ }
|
|
133
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
134
|
-
}
|
|
135
|
-
}
|