smart-web-mcp 0.44.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +342 -0
- package/README.md +33 -568
- package/dist/assessment.d.ts +10 -3
- package/dist/assessment.js +21 -6
- package/dist/assessment.js.map +1 -1
- package/dist/body-export.d.ts +10 -0
- package/dist/body-export.js +28 -0
- package/dist/body-export.js.map +1 -0
- package/dist/browser-session.d.ts +32 -1
- package/dist/browser-session.js +373 -3
- package/dist/browser-session.js.map +1 -1
- package/dist/cli-fetch-options.d.ts +7 -0
- package/dist/cli-fetch-options.js +41 -0
- package/dist/cli-fetch-options.js.map +1 -0
- package/dist/cli.js +28 -11
- package/dist/cli.js.map +1 -1
- package/dist/comment-export.d.ts +10 -0
- package/dist/comment-export.js +39 -0
- package/dist/comment-export.js.map +1 -0
- package/dist/composition.d.ts +1 -22
- package/dist/composition.js +0 -16
- package/dist/composition.js.map +1 -1
- package/dist/extraction/budget.d.ts +3 -0
- package/dist/extraction/budget.js +88 -7
- package/dist/extraction/budget.js.map +1 -1
- package/dist/extraction/document.js +134 -9
- package/dist/extraction/document.js.map +1 -1
- package/dist/korea/coupang-filters.d.ts +5 -0
- package/dist/korea/coupang-filters.js +87 -0
- package/dist/korea/coupang-filters.js.map +1 -0
- package/dist/korea/coupang.d.ts +27 -6
- package/dist/korea/coupang.js +202 -82
- package/dist/korea/coupang.js.map +1 -1
- package/dist/korea/proxy-client.d.ts +1 -0
- package/dist/korea/proxy-client.js +6 -2
- package/dist/korea/proxy-client.js.map +1 -1
- package/dist/korea/smartvertical.d.ts +1 -0
- package/dist/korea/smartvertical.js +39 -25
- package/dist/korea/smartvertical.js.map +1 -1
- package/dist/lib.d.ts +4 -2
- package/dist/lib.js +2 -1
- package/dist/lib.js.map +1 -1
- package/dist/mcp-server.js +30 -15
- package/dist/mcp-server.js.map +1 -1
- package/dist/operator-browser-fixture.d.ts +7 -0
- package/dist/operator-browser-fixture.js +19 -0
- package/dist/operator-browser-fixture.js.map +1 -0
- package/dist/operator-canary-cli.d.ts +6 -0
- package/dist/operator-canary-cli.js +190 -0
- package/dist/operator-canary-cli.js.map +1 -0
- package/dist/operator-canary.d.ts +74 -0
- package/dist/operator-canary.js +135 -0
- package/dist/operator-canary.js.map +1 -0
- package/dist/operator-cdp.d.ts +15 -0
- package/dist/operator-cdp.js +195 -0
- package/dist/operator-cdp.js.map +1 -0
- package/dist/operator-frame.d.ts +3 -0
- package/dist/operator-frame.js +55 -0
- package/dist/operator-frame.js.map +1 -0
- package/dist/settings.d.ts +3 -0
- package/dist/settings.js +1 -0
- package/dist/settings.js.map +1 -1
- package/dist/shared.d.ts +33 -8
- package/dist/shared.js +15 -9
- package/dist/shared.js.map +1 -1
- package/dist/smartbrowser.d.ts +169 -0
- package/dist/smartbrowser.js +288 -0
- package/dist/smartbrowser.js.map +1 -0
- package/dist/smartcrawl.js +16 -21
- package/dist/smartcrawl.js.map +1 -1
- package/dist/smartfetch/academic-fallback.js +3 -5
- package/dist/smartfetch/academic-fallback.js.map +1 -1
- package/dist/smartfetch/archive-fallback.js +2 -5
- package/dist/smartfetch/archive-fallback.js.map +1 -1
- package/dist/smartfetch/assets.d.ts +1 -1
- package/dist/smartfetch/assets.js +28 -1
- package/dist/smartfetch/assets.js.map +1 -1
- package/dist/smartfetch/jina-reader.d.ts +1 -0
- package/dist/smartfetch/jina-reader.js +17 -3
- package/dist/smartfetch/jina-reader.js.map +1 -1
- package/dist/smartfetch/pipeline.js +15 -25
- package/dist/smartfetch/pipeline.js.map +1 -1
- package/dist/smartfetch/provider-policy.d.ts +3 -0
- package/dist/smartfetch/provider-policy.js +45 -2
- package/dist/smartfetch/provider-policy.js.map +1 -1
- package/dist/smartfetch/provider-types.d.ts +8 -0
- package/dist/smartfetch/providers/article.js +76 -10
- package/dist/smartfetch/providers/article.js.map +1 -1
- package/dist/smartfetch/providers/blind-post.d.ts +2 -0
- package/dist/smartfetch/providers/blind-post.js +167 -0
- package/dist/smartfetch/providers/blind-post.js.map +1 -0
- package/dist/smartfetch/providers/clien.d.ts +2 -0
- package/dist/smartfetch/providers/clien.js +157 -0
- package/dist/smartfetch/providers/clien.js.map +1 -0
- package/dist/smartfetch/providers/commerce.d.ts +15 -0
- package/dist/smartfetch/providers/commerce.js +123 -7
- package/dist/smartfetch/providers/commerce.js.map +1 -1
- package/dist/smartfetch/providers/dcinside.js +313 -20
- package/dist/smartfetch/providers/dcinside.js.map +1 -1
- package/dist/smartfetch/providers/hackernews-html.d.ts +21 -0
- package/dist/smartfetch/providers/hackernews-html.js +148 -0
- package/dist/smartfetch/providers/hackernews-html.js.map +1 -0
- package/dist/smartfetch/providers/hackernews.js +76 -14
- package/dist/smartfetch/providers/hackernews.js.map +1 -1
- package/dist/smartfetch/providers/index.js +7 -1
- package/dist/smartfetch/providers/index.js.map +1 -1
- package/dist/smartfetch/providers/linkedin.js +2 -1
- package/dist/smartfetch/providers/linkedin.js.map +1 -1
- package/dist/smartfetch/providers/naver-blog.d.ts +2 -0
- package/dist/smartfetch/providers/naver-blog.js +87 -45
- package/dist/smartfetch/providers/naver-blog.js.map +1 -1
- package/dist/smartfetch/providers/naver-cafe-native.d.ts +7 -0
- package/dist/smartfetch/providers/naver-cafe-native.js +188 -0
- package/dist/smartfetch/providers/naver-cafe-native.js.map +1 -0
- package/dist/smartfetch/providers/naver-cafe.js +165 -4
- package/dist/smartfetch/providers/naver-cafe.js.map +1 -1
- package/dist/smartfetch/providers/naver-map-search.js +1 -1
- package/dist/smartfetch/providers/naver-map-search.js.map +1 -1
- package/dist/smartfetch/providers/ppomppu.d.ts +2 -0
- package/dist/smartfetch/providers/ppomppu.js +289 -0
- package/dist/smartfetch/providers/ppomppu.js.map +1 -0
- package/dist/smartfetch/providers/reddit.js +24 -3
- package/dist/smartfetch/providers/reddit.js.map +1 -1
- package/dist/smartfetch/providers/threads.js +1 -1
- package/dist/smartfetch/providers/threads.js.map +1 -1
- package/dist/smartfetch/providers/wikipedia.js +138 -0
- package/dist/smartfetch/providers/wikipedia.js.map +1 -1
- package/dist/smartfetch.d.ts +7 -1
- package/dist/smartfetch.js +353 -76
- package/dist/smartfetch.js.map +1 -1
- package/dist/smartsearch-brave.d.ts +2 -0
- package/dist/smartsearch-brave.js +43 -0
- package/dist/smartsearch-brave.js.map +1 -0
- package/dist/smartsearch-browser.d.ts +5 -0
- package/dist/smartsearch-browser.js +116 -0
- package/dist/smartsearch-browser.js.map +1 -0
- package/dist/smartsearch-ddg.d.ts +2 -0
- package/dist/smartsearch-ddg.js +46 -0
- package/dist/smartsearch-ddg.js.map +1 -0
- package/dist/smartsearch-node.d.ts +6 -0
- package/dist/smartsearch-node.js +21 -0
- package/dist/smartsearch-node.js.map +1 -0
- package/dist/smartsearch-postgresql.d.ts +7 -0
- package/dist/smartsearch-postgresql.js +44 -0
- package/dist/smartsearch-postgresql.js.map +1 -0
- package/dist/smartsearch-python.d.ts +8 -0
- package/dist/smartsearch-python.js +87 -0
- package/dist/smartsearch-python.js.map +1 -0
- package/dist/smartsearch-quality.d.ts +6 -0
- package/dist/smartsearch-quality.js +47 -0
- package/dist/smartsearch-quality.js.map +1 -0
- package/dist/smartsearch-youtube.d.ts +7 -0
- package/dist/smartsearch-youtube.js +76 -0
- package/dist/smartsearch-youtube.js.map +1 -0
- package/dist/smartsearch.d.ts +2 -0
- package/dist/smartsearch.js +506 -84
- package/dist/smartsearch.js.map +1 -1
- package/package.json +8 -5
- package/python/requirements-undetected-chromedriver.txt +2 -2
package/dist/assessment.d.ts
CHANGED
|
@@ -5,7 +5,6 @@ export declare function createHandoffSuggestion(input: {
|
|
|
5
5
|
mode?: RetrievalHandoffSuggestion["mode"];
|
|
6
6
|
url: string;
|
|
7
7
|
goal: string;
|
|
8
|
-
captcha_type?: CaptchaType;
|
|
9
8
|
}): RetrievalHandoffSuggestion;
|
|
10
9
|
export declare function createAssessment(input?: {
|
|
11
10
|
confidence?: RetrievalConfidence;
|
|
@@ -14,15 +13,23 @@ export declare function createAssessment(input?: {
|
|
|
14
13
|
interactive_flow_likely?: boolean;
|
|
15
14
|
signals?: string[];
|
|
16
15
|
recommended_handoff?: RetrievalHandoffSuggestion | null;
|
|
16
|
+
continuation?: RetrievalAssessment["continuation"];
|
|
17
17
|
}): RetrievalAssessment;
|
|
18
18
|
export declare function firstMeaningfulBlockedReason(items: Array<ToolError | string | null | undefined>): string;
|
|
19
19
|
export declare function hasTerminalSafetyBlock(items: Array<ToolError | string | null | undefined>): boolean;
|
|
20
20
|
export declare function detectAuthLikeText(value: string): boolean;
|
|
21
21
|
export declare function detectInteractiveLikeText(value: string): boolean;
|
|
22
|
+
/** Describes unfinished work only. Never authorizes or dispatches an action. */
|
|
23
|
+
export declare function assessBrowserContinuation(input: {
|
|
24
|
+
url: string;
|
|
25
|
+
incomplete: boolean;
|
|
26
|
+
content: string;
|
|
27
|
+
interactive: boolean;
|
|
28
|
+
errors?: Array<ToolError | undefined>;
|
|
29
|
+
}): RetrievalAssessment["continuation"];
|
|
22
30
|
/**
|
|
23
31
|
* Detect CAPTCHA widget type from page text/HTML content.
|
|
24
|
-
*
|
|
25
|
-
* infers the likely CAPTCHA type from known signatures in the fetched HTML
|
|
32
|
+
* Infers the likely CAPTCHA type from known signatures in the fetched HTML
|
|
26
33
|
* (iframe src patterns, class names, data-sitekey attributes).
|
|
27
34
|
*/
|
|
28
35
|
export declare function detectCaptchaTypeFromContent(html: string): CaptchaType | null;
|
package/dist/assessment.js
CHANGED
|
@@ -8,7 +8,6 @@ export function createHandoffSuggestion(input) {
|
|
|
8
8
|
mode: input.mode || "read",
|
|
9
9
|
url: String(input.url || "").trim(),
|
|
10
10
|
goal: String(input.goal || "").trim(),
|
|
11
|
-
...(input.captcha_type ? { captcha_type: input.captcha_type } : {}),
|
|
12
11
|
};
|
|
13
12
|
}
|
|
14
13
|
export function createAssessment(input = {}) {
|
|
@@ -19,6 +18,7 @@ export function createAssessment(input = {}) {
|
|
|
19
18
|
interactive_flow_likely: Boolean(input.interactive_flow_likely),
|
|
20
19
|
signals: normalizeSignals(input.signals || []),
|
|
21
20
|
recommended_handoff: input.recommended_handoff || null,
|
|
21
|
+
continuation: input.continuation || null,
|
|
22
22
|
};
|
|
23
23
|
}
|
|
24
24
|
export function firstMeaningfulBlockedReason(items) {
|
|
@@ -64,10 +64,27 @@ export function detectInteractiveLikeText(value) {
|
|
|
64
64
|
return false;
|
|
65
65
|
return /(enable javascript|checking your browser|just a moment|please wait|open the app|continue in browser|use the search box|select a filter|load more|show more)/i.test(text);
|
|
66
66
|
}
|
|
67
|
+
/** Describes unfinished work only. Never authorizes or dispatches an action. */
|
|
68
|
+
export function assessBrowserContinuation(input) {
|
|
69
|
+
if (!input.incomplete)
|
|
70
|
+
return null;
|
|
71
|
+
const captcha = detectCaptchaTypeFromContent(input.content);
|
|
72
|
+
const explicitParticipation = input.errors?.find((error) => error?.code === "operator_browser_user_required");
|
|
73
|
+
const userRequired = explicitParticipation || input.errors?.some((error) => error?.code === "operator_browser_auth_required") || Boolean(captcha) || /(?:sign in|log in|login) (?:required|to continue)|로그인(?:이 필요| 후)|consent required|동의가 필요|(?:enter|provide) (?:your |the )?(?:password|one[- ]time code)|verify you are human|complete (?:the |this )?captcha|captcha (?:is )?required|password\/MFA\/CAPTCHA\/consent/i.test(input.content);
|
|
74
|
+
return {
|
|
75
|
+
status: userRequired ? "user_required" : input.interactive ? "action_required" : "unavailable",
|
|
76
|
+
url: input.url,
|
|
77
|
+
reason: userRequired
|
|
78
|
+
? explicitParticipation?.message || "Password, MFA, CAPTCHA or consent requires user participation; no automated solving or further action is authorized."
|
|
79
|
+
: input.interactive
|
|
80
|
+
? "Requested content still needs observed browser interaction; this assessment did not execute an action."
|
|
81
|
+
: "Available acquisition did not verify the requested content; no further backend task was dispatched.",
|
|
82
|
+
...(captcha ? { captcha_type: captcha } : {}),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
67
85
|
/**
|
|
68
86
|
* Detect CAPTCHA widget type from page text/HTML content.
|
|
69
|
-
*
|
|
70
|
-
* infers the likely CAPTCHA type from known signatures in the fetched HTML
|
|
87
|
+
* Infers the likely CAPTCHA type from known signatures in the fetched HTML
|
|
71
88
|
* (iframe src patterns, class names, data-sitekey attributes).
|
|
72
89
|
*/
|
|
73
90
|
export function detectCaptchaTypeFromContent(html) {
|
|
@@ -96,9 +113,7 @@ export function detectCaptchaTypeFromContent(html) {
|
|
|
96
113
|
return "hcaptcha";
|
|
97
114
|
if (/data-sitekey[^>]*class="[^"]*h-captcha/i.test(html))
|
|
98
115
|
return "hcaptcha";
|
|
99
|
-
//
|
|
100
|
-
if (/\bcaptcha\b/i.test(html))
|
|
101
|
-
return "recaptcha";
|
|
116
|
+
// An untyped challenge or a mere mention does not identify its vendor.
|
|
102
117
|
return null;
|
|
103
118
|
}
|
|
104
119
|
//# sourceMappingURL=assessment.js.map
|
package/dist/assessment.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"assessment.js","sourceRoot":"","sources":["../src/assessment.ts"],"names":[],"mappings":"AAEA,SAAS,gBAAgB,CAAC,OAAiB;IACzC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAC9F,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,
|
|
1
|
+
{"version":3,"file":"assessment.js","sourceRoot":"","sources":["../src/assessment.ts"],"names":[],"mappings":"AAEA,SAAS,gBAAgB,CAAC,OAAiB;IACzC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAC9F,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAMvC;IACC,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,MAAM;QAC1B,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACnC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;KACtC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAK,GAQlC,EAAE;IACJ,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,QAAQ;QACxC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACzD,oBAAoB,EAAE,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC;QACzD,uBAAuB,EAAE,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC;QAC/D,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;QAC9C,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,IAAI,IAAI;QACtD,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;KACzC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAAmD;IAC9F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YACxB,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,6EAA6E,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAA;YACxK,SAAQ;QACV,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAA;QAC/D,IAAI,yDAAyD,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAA;IACtI,CAAC;IACD,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAmD;IACxF,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAA;QACnD,OAAO,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI;YAClC,sBAAsB;YACtB,aAAa;YACb,kBAAkB;YAClB,YAAY;YACZ,oBAAoB;YACpB,oBAAoB;SACrB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvB,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAA;IACvB,OAAO,gLAAgL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACpM,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAa;IACrD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAA;IACvB,OAAO,8JAA8J,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAClL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,yBAAyB,CAAC,KAAyH;IACjK,IAAI,CAAC,KAAK,CAAC,UAAU;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,OAAO,GAAG,4BAA4B,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC3D,MAAM,qBAAqB,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,IAAI,KAAK,gCAAgC,CAAC,CAAA;IAC7G,MAAM,YAAY,GAAG,qBAAqB,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,IAAI,KAAK,gCAAgC,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,wQAAwQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IACjb,OAAO;QACL,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa;QAC9F,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,YAAY;YAClB,CAAC,CAAC,qBAAqB,EAAE,OAAO,IAAI,sHAAsH;YAC1J,CAAC,CAAC,KAAK,CAAC,WAAW;gBACjB,CAAC,CAAC,wGAAwG;gBAC1G,CAAC,CAAC,qGAAqG;QAC3G,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CAAC,IAAY;IACvD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,kEAAkE;IAClE,IAAI,2DAA2D,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IAC9F,IAAI,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IACpE,IAAI,4CAA4C,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IAC/E,2CAA2C;IAC3C,IAAI,yCAAyC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IAC5E,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IACnE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IACnE,IAAI,2CAA2C,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAA;IAC9E,4CAA4C;IAC5C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAC3D,IAAI,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAChE,IAAI,yCAAyC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAC3E,uEAAuE;IACvE,OAAO,IAAI,CAAA;AACb,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SmartfetchOutput } from "./shared.js";
|
|
2
|
+
/** Export normalized body before inline projection, never raw DOM or session state. */
|
|
3
|
+
export declare function exportBody(output: SmartfetchOutput): {
|
|
4
|
+
path: string;
|
|
5
|
+
format: "text";
|
|
6
|
+
bytes: number;
|
|
7
|
+
sha256: string;
|
|
8
|
+
chars: number;
|
|
9
|
+
complete: boolean;
|
|
10
|
+
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { resolveSmartWebTempRoot } from "./runtime-temp.js";
|
|
5
|
+
/** Export normalized body before inline projection, never raw DOM or session state. */
|
|
6
|
+
export function exportBody(output) {
|
|
7
|
+
const text = output.post?.text;
|
|
8
|
+
if (typeof text !== "string" || !text.trim())
|
|
9
|
+
throw new Error("No readable body to export");
|
|
10
|
+
const bytes = Buffer.from(text);
|
|
11
|
+
if (bytes.length > 1024 * 1024)
|
|
12
|
+
throw new Error("Body export exceeds the 1 MiB bound");
|
|
13
|
+
const root = resolveSmartWebTempRoot();
|
|
14
|
+
mkdirSync(root, { recursive: true });
|
|
15
|
+
const dir = mkdtempSync(join(root, "smartfetch-body-"));
|
|
16
|
+
const path = join(dir, "body.txt");
|
|
17
|
+
try {
|
|
18
|
+
writeFileSync(path, bytes, { mode: 0o600, flag: "wx" });
|
|
19
|
+
const coverage = output.post?.body_coverage;
|
|
20
|
+
const complete = output.partial === false && (!coverage || (typeof coverage === "object" && "complete" in coverage && coverage.complete === true && "collected_chars" in coverage && coverage.collected_chars === text.length));
|
|
21
|
+
return { path, format: "text", bytes: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex"), chars: text.length, complete };
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
rmSync(dir, { recursive: true, force: true });
|
|
25
|
+
throw error;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=body-export.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"body-export.js","sourceRoot":"","sources":["../src/body-export.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAA;AAG3D,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,MAAwB;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,IAAI,CAAA;IAC9B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAC3F,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;IACtF,MAAM,IAAI,GAAG,uBAAuB,EAAE,CAAA;IACtC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACpC,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAA;IACvD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;IAClC,IAAI,CAAC;QACH,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,aAAa,CAAA;QAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,UAAU,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,IAAI,IAAI,iBAAiB,IAAI,QAAQ,IAAI,QAAQ,CAAC,eAAe,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/N,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAe,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAA;IACvJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAC7C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import type { Browser, BrowserContext, CDPSession, Page } from "playwright";
|
|
1
2
|
import type { ToolError } from "./shared.js";
|
|
2
3
|
export declare const SMART_WEB_PROFILE_NAME_PATTERN: RegExp;
|
|
3
4
|
export declare const SMART_WEB_PROFILE_NAME_DESCRIPTION = "1-64 characters: letters, numbers, dots, underscores, or hyphens; must start with a letter or number and cannot end with a dot";
|
|
4
5
|
type BrowserWaitUntil = "domcontentloaded" | "networkidle";
|
|
5
|
-
type BrowserMethod = "playwright_stealth" | "undetected_chromedriver";
|
|
6
|
+
type BrowserMethod = "playwright_stealth" | "undetected_chromedriver" | "local_browser_cdp";
|
|
6
7
|
export type BrowserAnchor = {
|
|
7
8
|
href: string;
|
|
8
9
|
text: string;
|
|
@@ -40,6 +41,8 @@ export type BrowserPageResult = BrowserPageSnapshot | BrowserPageFailure;
|
|
|
40
41
|
export type BrowserSessionOptions = {
|
|
41
42
|
url: string;
|
|
42
43
|
timeoutMs: number;
|
|
44
|
+
/** Explicitly approved local automation browser. Never discovered or launched. */
|
|
45
|
+
cdpUrl?: string;
|
|
43
46
|
allowPrivateHosts?: boolean;
|
|
44
47
|
waitUntil?: BrowserWaitUntil;
|
|
45
48
|
extraHeaders?: Record<string, string>;
|
|
@@ -55,17 +58,45 @@ export type BrowserSessionOptions = {
|
|
|
55
58
|
* useful for SPA pages where `networkidle` fires before JS has rendered
|
|
56
59
|
* the target content. */
|
|
57
60
|
waitForSelector?: string;
|
|
61
|
+
/** Trusted local action runner; only supported on the approved CDP lane. */
|
|
62
|
+
operatorTask?: (page: Page, remaining: () => number, session: CDPSession) => Promise<void>;
|
|
63
|
+
/** Install task-local client adapters before the owned page is created. */
|
|
64
|
+
operatorPrepareContext?: (context: BrowserContext) => Promise<void>;
|
|
65
|
+
/** Explicit scoped-frame batches allow only same-origin HTTP(S) children. */
|
|
66
|
+
operatorSameOriginFrames?: boolean;
|
|
67
|
+
/** Capture an explicit attachment without altering the shared download policy. */
|
|
68
|
+
operatorDownload?: (download: {
|
|
69
|
+
id: number;
|
|
70
|
+
url: string;
|
|
71
|
+
bytes: Buffer;
|
|
72
|
+
contentType: string;
|
|
73
|
+
}) => Promise<void>;
|
|
74
|
+
operatorDownloadTarget?: () => {
|
|
75
|
+
url: string;
|
|
76
|
+
id: number;
|
|
77
|
+
frameId?: string;
|
|
78
|
+
} | undefined;
|
|
79
|
+
signal?: AbortSignal;
|
|
58
80
|
};
|
|
59
81
|
type TimedCache = {
|
|
60
82
|
value: any;
|
|
61
83
|
expiresAt: number;
|
|
62
84
|
};
|
|
85
|
+
type OperatorBrowserConnector = (url: string, options: {
|
|
86
|
+
timeout: number;
|
|
87
|
+
noDefaults: true;
|
|
88
|
+
}) => Promise<Browser>;
|
|
89
|
+
export declare function TEST_setOperatorBrowserConnector(connector?: OperatorBrowserConnector): void;
|
|
63
90
|
export declare function TEST_setChromiumCache(value: TimedCache | undefined): void;
|
|
64
91
|
export declare function TEST_setUndetectedChromeRunner(value: ((options: BrowserSessionOptions) => Promise<BrowserPageResult>) | undefined): void;
|
|
65
92
|
export declare function TEST_setPlaywrightInstaller(value: (() => Promise<ToolError | null>) | undefined): void;
|
|
66
93
|
export declare function TEST_resetPlaywrightStartupWarning(): void;
|
|
67
94
|
/** Override the system Chrome path for tests. Pass undefined to restore auto-detection. */
|
|
68
95
|
export declare function TEST_setSystemChromePath(value: string | undefined): void;
|
|
96
|
+
export declare class BrowserSessionToolError extends Error {
|
|
97
|
+
readonly toolError: ToolError;
|
|
98
|
+
constructor(toolError: ToolError);
|
|
99
|
+
}
|
|
69
100
|
export declare function looksBotBlockedContent(value: string): boolean;
|
|
70
101
|
export declare function validateProfileName(name: string): string;
|
|
71
102
|
/** Resolve the user-data-dir for a named profile (e.g. "work", "personal"). */
|
package/dist/browser-session.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { chmodSync, existsSync, lstatSync, mkdirSync, readdirSync } from "node:fs";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
|
+
import { STATUS_CODES } from "node:http";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
6
7
|
import { fileURLToPath } from "node:url";
|
|
7
8
|
import { promisify } from "node:util";
|
|
9
|
+
import { connectOwnedBrowser, ownedBrowserCleanupIncomplete } from "./operator-cdp.js";
|
|
8
10
|
import { asString, configEnabled, dedupeUrls, resolveValidatedUrl, validateOutboundUrl } from "./shared.js";
|
|
9
11
|
import { sanitizeSmartWebTempEnv } from "./runtime-temp.js";
|
|
10
12
|
import { configText } from "./settings.js";
|
|
@@ -23,6 +25,10 @@ let playwrightInstallerOverride = undefined;
|
|
|
23
25
|
let playwrightInstallPromise = undefined;
|
|
24
26
|
let playwrightStartupWarningShown = false;
|
|
25
27
|
let systemChromePathOverride = undefined;
|
|
28
|
+
let operatorBrowserConnector = connectOwnedBrowser;
|
|
29
|
+
export function TEST_setOperatorBrowserConnector(connector) {
|
|
30
|
+
operatorBrowserConnector = connector || connectOwnedBrowser;
|
|
31
|
+
}
|
|
26
32
|
export function TEST_setChromiumCache(value) {
|
|
27
33
|
chromiumCache = value;
|
|
28
34
|
}
|
|
@@ -41,7 +47,7 @@ export function TEST_resetPlaywrightStartupWarning() {
|
|
|
41
47
|
export function TEST_setSystemChromePath(value) {
|
|
42
48
|
systemChromePathOverride = value;
|
|
43
49
|
}
|
|
44
|
-
class BrowserSessionToolError extends Error {
|
|
50
|
+
export class BrowserSessionToolError extends Error {
|
|
45
51
|
toolError;
|
|
46
52
|
constructor(toolError) {
|
|
47
53
|
super(toolError.message);
|
|
@@ -647,8 +653,18 @@ async function extractAnchors(page, maxAnchors) {
|
|
|
647
653
|
return nodes.map((node) => {
|
|
648
654
|
const element = node;
|
|
649
655
|
const closest = element.closest("article, li, tr, section, main, .topic-list-item, .topic-body, .topic-post, .athing, .titleline, [role='article'], [data-pressable-container='true'], .feed-item, .timeline-item, .post, .storylink");
|
|
650
|
-
const
|
|
651
|
-
|
|
656
|
+
const productName = /(^|\.)coupang\.com$/.test(element.hostname) && /\/vp\/products\/\d+/.test(element.pathname)
|
|
657
|
+
? element.querySelector('[class*="productName"]') : null;
|
|
658
|
+
const text = (productName?.innerText || element.innerText || element.getAttribute("aria-label") || element.getAttribute("title") || "").replace(/\s+/g, " ").trim();
|
|
659
|
+
let contextSource = closest?.innerText || element.innerText || element.textContent || "";
|
|
660
|
+
if (productName) {
|
|
661
|
+
// A product card's first visible amount may be a crossed-out list price.
|
|
662
|
+
// Keep current/conditional labels, never borrow the neighbouring card.
|
|
663
|
+
const card = element.cloneNode(true);
|
|
664
|
+
card.querySelectorAll("del, s").forEach((node) => node.remove());
|
|
665
|
+
card.querySelectorAll("div, p, li, br").forEach((node) => node.after(document.createTextNode(" ")));
|
|
666
|
+
contextSource = card.textContent || "";
|
|
667
|
+
}
|
|
652
668
|
return {
|
|
653
669
|
href: element.href || "",
|
|
654
670
|
text,
|
|
@@ -721,8 +737,362 @@ function shouldPromoteUndetectedChrome(playwrightResult) {
|
|
|
721
737
|
function preferUndetectedChromeResult(playwrightResult, ucResult) {
|
|
722
738
|
return ucResult.ok;
|
|
723
739
|
}
|
|
740
|
+
function assertReadableBrowserContent(title, text) {
|
|
741
|
+
if (/^\s*(?:please\s+)?(?:verify you are human|complete (?:the |this )?captcha|captcha (?:is )?required)/i.test(text)
|
|
742
|
+
|| (text.length < 1000 && /verify you are human|complete (?:the |this )?captcha|captcha (?:is )?required/i.test(text))) {
|
|
743
|
+
throw new BrowserSessionToolError({ category: "block", code: "operator_browser_user_required", message: "Approved local browser requires CAPTCHA user participation; no content was verified" });
|
|
744
|
+
}
|
|
745
|
+
if (looksBotBlockedContent(title) || looksBotBlockedContent(text.slice(0, 4000)) || /사용\s*권한(?:이 없| 제한)|접근.*권한이 없습니다/.test(text.slice(0, 4000))) {
|
|
746
|
+
throw new BrowserSessionToolError({ category: "block", code: "operator_browser_blocked", message: "Approved local browser returned a challenge or denied page; no content was verified" });
|
|
747
|
+
}
|
|
748
|
+
if (text.length < 1000 && /로그인이 필요|로그인 후|sign in to continue|log in to continue/i.test(text)) {
|
|
749
|
+
throw new BrowserSessionToolError({ category: "block", code: "operator_browser_auth_required", message: "Approved local browser requires login or user participation; no content was verified" });
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
async function readGuardedResponse(session, requestId, remaining) {
|
|
753
|
+
const { stream } = await session.send("Fetch.takeResponseBodyAsStream", { requestId });
|
|
754
|
+
const chunks = [];
|
|
755
|
+
let bytes = 0;
|
|
756
|
+
try {
|
|
757
|
+
for (;;) {
|
|
758
|
+
remaining();
|
|
759
|
+
const chunk = await session.send("IO.read", { handle: stream, size: 64 * 1024 });
|
|
760
|
+
const buffer = Buffer.from(chunk.data, chunk.base64Encoded ? "base64" : "utf8");
|
|
761
|
+
bytes += buffer.length;
|
|
762
|
+
if (bytes > 8 * 1024 * 1024)
|
|
763
|
+
throw new Error("Operator response exceeds the 8 MiB guarded byte limit");
|
|
764
|
+
chunks.push(buffer);
|
|
765
|
+
if (chunk.eof)
|
|
766
|
+
break;
|
|
767
|
+
}
|
|
768
|
+
return Buffer.concat(chunks, bytes);
|
|
769
|
+
}
|
|
770
|
+
finally {
|
|
771
|
+
void session.send("IO.close", { handle: stream }).catch(() => { });
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
async function runConnectedBrowserSession(options) {
|
|
775
|
+
let browser;
|
|
776
|
+
let page;
|
|
777
|
+
let session;
|
|
778
|
+
const deadline = Date.now() + options.timeoutMs;
|
|
779
|
+
let expired = false;
|
|
780
|
+
let navigationError;
|
|
781
|
+
let phase = "validate_url";
|
|
782
|
+
let interruptionPhase;
|
|
783
|
+
const interruptionMessage = (cancelled) => `Operator browser task ${cancelled ? "cancelled" : "deadline exhausted"} (phase=${interruptionPhase ?? phase})`;
|
|
784
|
+
const interruptionError = () => {
|
|
785
|
+
interruptionPhase ??= phase;
|
|
786
|
+
return new BrowserSessionToolError({
|
|
787
|
+
category: "timeout", code: options.signal?.aborted ? "operator_browser_cancelled" : "operator_browser_timeout",
|
|
788
|
+
message: interruptionMessage(options.signal?.aborted === true),
|
|
789
|
+
});
|
|
790
|
+
};
|
|
791
|
+
const remaining = () => {
|
|
792
|
+
if (navigationError)
|
|
793
|
+
throw new BrowserSessionToolError(navigationError);
|
|
794
|
+
if (expired || options.signal?.aborted || Date.now() >= deadline)
|
|
795
|
+
throw interruptionError();
|
|
796
|
+
return Math.max(1, deadline - Date.now());
|
|
797
|
+
};
|
|
798
|
+
let interrupt;
|
|
799
|
+
const interrupted = new Promise((_resolve, reject) => { interrupt = reject; });
|
|
800
|
+
void interrupted.catch(() => { });
|
|
801
|
+
const bounded = (promise) => Promise.race([promise, interrupted]);
|
|
802
|
+
const cancel = () => {
|
|
803
|
+
expired = true;
|
|
804
|
+
interrupt(interruptionError());
|
|
805
|
+
void page?.close().catch(() => { });
|
|
806
|
+
};
|
|
807
|
+
const timer = setTimeout(cancel, options.timeoutMs);
|
|
808
|
+
options.signal?.addEventListener("abort", cancel, { once: true });
|
|
809
|
+
try {
|
|
810
|
+
remaining();
|
|
811
|
+
const endpoint = new URL(options.cdpUrl || "");
|
|
812
|
+
if (endpoint.protocol !== "http:" || !["127.0.0.1", "[::1]"].includes(endpoint.hostname)
|
|
813
|
+
|| endpoint.username || endpoint.password || endpoint.pathname !== "/" || endpoint.search || endpoint.hash) {
|
|
814
|
+
throw new BrowserSessionToolError({ category: "block", code: "operator_cdp_endpoint_invalid", message: "Operator browser requires an explicit loopback HTTP CDP endpoint without credentials or path" });
|
|
815
|
+
}
|
|
816
|
+
await bounded(assertSafeBrowserFinalUrl(options.url, options));
|
|
817
|
+
// Plain Playwright is only a CDP client here. Do not install stealth plugins,
|
|
818
|
+
// context routes, headers or init scripts into somebody else's session.
|
|
819
|
+
phase = "connect";
|
|
820
|
+
browser = await bounded(operatorBrowserConnector(endpoint.origin, { timeout: remaining(), noDefaults: true }).then((connected) => {
|
|
821
|
+
if (expired || options.signal?.aborted)
|
|
822
|
+
void connected.close().catch(() => { });
|
|
823
|
+
return connected;
|
|
824
|
+
}));
|
|
825
|
+
remaining();
|
|
826
|
+
const context = browser.contexts()[0];
|
|
827
|
+
if (!context)
|
|
828
|
+
throw new Error("Approved browser has no existing context");
|
|
829
|
+
if (options.operatorPrepareContext)
|
|
830
|
+
await bounded(options.operatorPrepareContext(context));
|
|
831
|
+
phase = "create_tab";
|
|
832
|
+
page = await bounded(context.newPage().then((created) => {
|
|
833
|
+
if (expired || options.signal?.aborted)
|
|
834
|
+
void created.close().catch(() => { });
|
|
835
|
+
return created;
|
|
836
|
+
}));
|
|
837
|
+
if (expired)
|
|
838
|
+
throw new Error("Operator browser timeout");
|
|
839
|
+
// Guards belong to this tab, never the shared context. CDP Fetch pauses each
|
|
840
|
+
// redirect hop too (Playwright route only sees the first redirected request).
|
|
841
|
+
phase = "install_guards";
|
|
842
|
+
session = await bounded(context.newCDPSession(page).then((created) => {
|
|
843
|
+
if (expired || options.signal?.aborted)
|
|
844
|
+
void created.detach().catch(() => { });
|
|
845
|
+
return created;
|
|
846
|
+
}));
|
|
847
|
+
const ownedSession = session;
|
|
848
|
+
const mainFrameId = options.operatorTask ? (await bounded(session.send("Page.getFrameTree"))).frameTree.frame.id : undefined;
|
|
849
|
+
const downloadRequests = new Set();
|
|
850
|
+
let activeDownloadId;
|
|
851
|
+
const guardedRequests = new Map();
|
|
852
|
+
const pendingGuards = new Set();
|
|
853
|
+
const settleGuards = async () => {
|
|
854
|
+
while (pendingGuards.size) {
|
|
855
|
+
if (navigationError)
|
|
856
|
+
throw new BrowserSessionToolError(navigationError);
|
|
857
|
+
remaining();
|
|
858
|
+
await bounded(Promise.all([...pendingGuards]));
|
|
859
|
+
}
|
|
860
|
+
if (navigationError)
|
|
861
|
+
throw new BrowserSessionToolError(navigationError);
|
|
862
|
+
};
|
|
863
|
+
session.on("Network.loadingFailed", event => {
|
|
864
|
+
if (event.canceled !== true || event.errorText !== "net::ERR_ABORTED")
|
|
865
|
+
return;
|
|
866
|
+
for (const guard of guardedRequests.get(event.requestId) || [])
|
|
867
|
+
guard.cancelled = true;
|
|
868
|
+
});
|
|
869
|
+
const stopForeignFrameNavigation = (frameId, url) => {
|
|
870
|
+
if (!options.operatorSameOriginFrames || frameId === mainFrameId)
|
|
871
|
+
return false;
|
|
872
|
+
if (/^https?:/.test(url) && new URL(url).origin === new URL(page.url()).origin)
|
|
873
|
+
return false;
|
|
874
|
+
navigationError = { category: "block", code: "operator_frame_navigation_blocked", message: "Frame navigation violates the same-origin HTTP(S) action scope" };
|
|
875
|
+
expired = true;
|
|
876
|
+
interrupt(new BrowserSessionToolError(navigationError));
|
|
877
|
+
void page?.close().catch(() => { });
|
|
878
|
+
return true;
|
|
879
|
+
};
|
|
880
|
+
if (options.operatorSameOriginFrames) {
|
|
881
|
+
// Native CSP blocks transmission; the event reports the blocked intent
|
|
882
|
+
// without waiting for a locator in Chromium's opaque error document.
|
|
883
|
+
session.on("Page.frameRequestedNavigation", event => stopForeignFrameNavigation(event.frameId, event.url));
|
|
884
|
+
await bounded(session.send("Page.enable"));
|
|
885
|
+
}
|
|
886
|
+
session.on("Fetch.requestPaused", (event) => {
|
|
887
|
+
const mainGet = event.resourceType === "Document" && event.frameId === mainFrameId && event.request.method === "GET";
|
|
888
|
+
const requestStage = event.responseStatusCode === undefined && event.responseErrorReason === undefined;
|
|
889
|
+
const guard = { cancelled: false, replaced: false,
|
|
890
|
+
...(mainGet && event.responseErrorReason === "Aborted" && event.responseStatusCode === undefined
|
|
891
|
+
? { abortedNavigation: { requestId: event.requestId, url: event.request.url } } : {}),
|
|
892
|
+
};
|
|
893
|
+
if (event.networkId) {
|
|
894
|
+
const guards = guardedRequests.get(event.networkId) || new Set();
|
|
895
|
+
// Chromium can replace an aborted navigation with a new interception
|
|
896
|
+
// while retaining its native Network ID. This is not loadingFailed and
|
|
897
|
+
// not permission to swallow a response error without a replacement.
|
|
898
|
+
if (mainGet && requestStage)
|
|
899
|
+
for (const previous of guards) {
|
|
900
|
+
if (previous.abortedNavigation && previous.abortedNavigation.requestId !== event.requestId
|
|
901
|
+
&& previous.abortedNavigation.url === event.request.url)
|
|
902
|
+
previous.replaced = true;
|
|
903
|
+
}
|
|
904
|
+
guards.add(guard);
|
|
905
|
+
guardedRequests.set(event.networkId, guards);
|
|
906
|
+
}
|
|
907
|
+
const work = (async () => {
|
|
908
|
+
if (event.resourceType === "Document" && event.responseStatusCode === undefined
|
|
909
|
+
&& stopForeignFrameNavigation(event.frameId, event.request.url)) {
|
|
910
|
+
await ownedSession.send("Fetch.failRequest", { requestId: event.requestId, errorReason: "BlockedByClient" }).catch(() => { });
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
const target = options.operatorDownloadTarget?.();
|
|
914
|
+
if (target?.id !== activeDownloadId) {
|
|
915
|
+
downloadRequests.clear();
|
|
916
|
+
activeDownloadId = target?.id;
|
|
917
|
+
}
|
|
918
|
+
const downloadFrameId = target?.frameId ?? mainFrameId;
|
|
919
|
+
if (event.responseStatusCode === undefined && target && event.resourceType === "Document" && event.frameId === downloadFrameId
|
|
920
|
+
&& (event.request.url === target.url || Boolean(event.redirectedRequestId && downloadRequests.has(event.redirectedRequestId)))) {
|
|
921
|
+
downloadRequests.add(event.requestId);
|
|
922
|
+
}
|
|
923
|
+
if (options.operatorTask && event.responseStatusCode !== undefined && event.responseStatusCode >= 200
|
|
924
|
+
&& ![204, 205, 304].includes(event.responseStatusCode) && (event.responseStatusCode < 300 || event.responseStatusCode >= 400)) {
|
|
925
|
+
// Native Chromium sandbox, scoped to this task's documents only.
|
|
926
|
+
// Do not let actions create unguarded popups or navigate another tab.
|
|
927
|
+
const disposition = event.responseHeaders?.find((header) => header.name.toLowerCase() === "content-disposition")?.value || "";
|
|
928
|
+
const attachment = Boolean(options.operatorDownload && /^attachment(?:;|$)/i.test(disposition.trim()));
|
|
929
|
+
if (attachment && (event.responseStatusCode !== 200 || event.responseHeaders?.some((header) => header.name.toLowerCase() === "content-range")))
|
|
930
|
+
throw new Error("Only complete HTTP 200 attachments without Content-Range are supported");
|
|
931
|
+
if (attachment && (!target || event.frameId !== downloadFrameId || !downloadRequests.has(event.requestId)))
|
|
932
|
+
throw new Error("Attachment does not match the explicit armed download target and redirect chain");
|
|
933
|
+
const body = await readGuardedResponse(ownedSession, event.requestId, remaining);
|
|
934
|
+
if (attachment && options.operatorDownload && target) {
|
|
935
|
+
remaining();
|
|
936
|
+
if (options.operatorDownloadTarget?.()?.id !== target.id)
|
|
937
|
+
throw new Error("Download step changed before its response completed");
|
|
938
|
+
// Consume only this owned tab's response. Do not toggle Browser-wide
|
|
939
|
+
// download behavior or write to the shared browser's Downloads dir.
|
|
940
|
+
await ownedSession.send("Fetch.fulfillRequest", { requestId: event.requestId, responseCode: 204, responseHeaders: [], body: "" });
|
|
941
|
+
remaining();
|
|
942
|
+
await options.operatorDownload({ id: target.id, url: event.request.url, bytes: body, contentType: event.responseHeaders?.find((header) => header.name.toLowerCase() === "content-type")?.value.slice(0, 200) || "application/octet-stream" });
|
|
943
|
+
return;
|
|
944
|
+
}
|
|
945
|
+
await ownedSession.send("Fetch.fulfillRequest", {
|
|
946
|
+
requestId: event.requestId, responseCode: event.responseStatusCode,
|
|
947
|
+
// Chromium rejects empty phrases for extension codes (e.g. 299).
|
|
948
|
+
// Keep the actual numeric status; a neutral phrase is not success.
|
|
949
|
+
responsePhrase: event.responseStatusText || STATUS_CODES[event.responseStatusCode] || "Unknown",
|
|
950
|
+
responseHeaders: [...(event.responseHeaders || []).filter((header) => !/^(content-length|content-encoding|transfer-encoding)$/i.test(header.name)), { name: "Content-Security-Policy", value: `sandbox allow-scripts allow-same-origin allow-forms${options.operatorSameOriginFrames ? "; frame-src 'self'" : ""}` }],
|
|
951
|
+
body: body.toString("base64"),
|
|
952
|
+
});
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
const validation = await validateBrowserNavigationUrl(event.request.url, options);
|
|
956
|
+
if (!validation.ok) {
|
|
957
|
+
if (event.resourceType === "Document")
|
|
958
|
+
navigationError = browserNavigationBlockedError(validation.reason, validation.message);
|
|
959
|
+
await ownedSession.send("Fetch.failRequest", { requestId: event.requestId, errorReason: "BlockedByClient" });
|
|
960
|
+
}
|
|
961
|
+
else {
|
|
962
|
+
try {
|
|
963
|
+
await ownedSession.send("Fetch.continueRequest", {
|
|
964
|
+
requestId: event.requestId,
|
|
965
|
+
...(options.operatorTask && event.resourceType === "Document" ? { interceptResponse: true } : {}),
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
catch (error) {
|
|
969
|
+
// Both exceptions require native session-local evidence and exact
|
|
970
|
+
// stale-ID failure after validation. Replacement guards still run
|
|
971
|
+
// normally and must settle before any action/evidence publication.
|
|
972
|
+
const retired = (requestStage && guard.cancelled) || guard.replaced;
|
|
973
|
+
if (!retired || !(error instanceof Error)
|
|
974
|
+
|| !/Protocol error \(Fetch\.continueRequest\): Invalid InterceptionId\.?$/.test(error.message))
|
|
975
|
+
throw error;
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
})().catch((error) => {
|
|
979
|
+
navigationError = error instanceof BrowserSessionToolError ? error.toolError : { category: "block", code: "operator_request_guard_failed", message: error instanceof Error ? error.message : "Operator request guard failed" };
|
|
980
|
+
expired = true;
|
|
981
|
+
interrupt(new BrowserSessionToolError(navigationError));
|
|
982
|
+
void ownedSession.send("Fetch.failRequest", { requestId: event.requestId, errorReason: "BlockedByClient" }).catch(() => { });
|
|
983
|
+
}).finally(() => {
|
|
984
|
+
pendingGuards.delete(work);
|
|
985
|
+
if (!event.networkId)
|
|
986
|
+
return;
|
|
987
|
+
const guards = guardedRequests.get(event.networkId);
|
|
988
|
+
guards?.delete(guard);
|
|
989
|
+
if (guards?.size === 0)
|
|
990
|
+
guardedRequests.delete(event.networkId);
|
|
991
|
+
});
|
|
992
|
+
pendingGuards.add(work);
|
|
993
|
+
});
|
|
994
|
+
await bounded(session.send("Network.enable"));
|
|
995
|
+
await bounded(session.send("Network.setBypassServiceWorker", { bypass: true }));
|
|
996
|
+
if (options.operatorTask)
|
|
997
|
+
await bounded(session.send("Page.setBypassCSP", { enabled: false }));
|
|
998
|
+
await bounded(session.send("Fetch.enable", { patterns: [{ urlPattern: "*", requestStage: "Request" }] }));
|
|
999
|
+
phase = "navigate";
|
|
1000
|
+
try {
|
|
1001
|
+
const response = await bounded(page.goto(options.url, { waitUntil: "domcontentloaded", timeout: remaining() }));
|
|
1002
|
+
const status = response?.status();
|
|
1003
|
+
if (status !== undefined && status >= 400) {
|
|
1004
|
+
// Read only the already-loaded denial document, without readiness waits
|
|
1005
|
+
// or actions. Preserve observed participation instead of hiding it as HTTP.
|
|
1006
|
+
const text = await bounded(page.evaluate(() => document.body?.innerText || ""));
|
|
1007
|
+
const title = await bounded(page.title());
|
|
1008
|
+
assertReadableBrowserContent(title, text);
|
|
1009
|
+
throw new BrowserSessionToolError({
|
|
1010
|
+
category: status < 500 ? "block" : "unavailable", code: "operator_browser_http_error",
|
|
1011
|
+
message: `Approved local browser returned HTTP ${status}; requested content was not verified`,
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
catch (error) {
|
|
1016
|
+
if (navigationError)
|
|
1017
|
+
throw new BrowserSessionToolError(navigationError);
|
|
1018
|
+
throw error;
|
|
1019
|
+
}
|
|
1020
|
+
phase = options.operatorTask ? "actions" : "wait_for_content";
|
|
1021
|
+
if (options.operatorTask) {
|
|
1022
|
+
await settleGuards();
|
|
1023
|
+
await bounded(options.operatorTask(page, remaining, session));
|
|
1024
|
+
}
|
|
1025
|
+
else if (options.waitForSelector) {
|
|
1026
|
+
await bounded(page.waitForSelector(options.waitForSelector, { timeout: remaining() }));
|
|
1027
|
+
}
|
|
1028
|
+
else {
|
|
1029
|
+
await bounded(page.waitForFunction(() => {
|
|
1030
|
+
const body = document.body?.innerText || "";
|
|
1031
|
+
if (/access denied|사용\s*권한|접근.*권한|verify you are human|captcha|로그인이 필요|sign in to continue/i.test(body))
|
|
1032
|
+
return true;
|
|
1033
|
+
if (/(^|\.)coupang\.com$/.test(location.hostname) && location.pathname === "/np/search") {
|
|
1034
|
+
return Array.from(document.querySelectorAll('a[href*="/vp/products/"]')).some((a) => /[\d,]+원/.test(a.innerText));
|
|
1035
|
+
}
|
|
1036
|
+
return body.trim().length >= 80;
|
|
1037
|
+
}, undefined, { timeout: remaining() }));
|
|
1038
|
+
}
|
|
1039
|
+
await settleGuards();
|
|
1040
|
+
phase = "extract";
|
|
1041
|
+
await bounded(assertSafeBrowserFinalUrl(page.url(), options));
|
|
1042
|
+
const text = await bounded(page.evaluate(() => document.body?.innerText || ""));
|
|
1043
|
+
const title = await bounded(page.title());
|
|
1044
|
+
assertReadableBrowserContent(title, text);
|
|
1045
|
+
const anchors = await bounded(extractAnchors(page, Math.max(50, Math.min(2000, options.maxAnchors || 400))));
|
|
1046
|
+
const primary = await bounded(extractPrimaryContent(page));
|
|
1047
|
+
const html = await bounded(page.content());
|
|
1048
|
+
await settleGuards();
|
|
1049
|
+
if (expired)
|
|
1050
|
+
throw new Error("Operator browser timeout");
|
|
1051
|
+
return {
|
|
1052
|
+
ok: true, method: "local_browser_cdp", url: options.url, final_url: page.url(), title,
|
|
1053
|
+
html, text, content_html: primary.html, content_text: primary.text,
|
|
1054
|
+
headings: primary.headings, anchors, links: dedupeUrls(anchors.map((anchor) => anchor.href)),
|
|
1055
|
+
scroll_steps: 0, load_more_clicks: 0,
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1058
|
+
catch (error) {
|
|
1059
|
+
const timedOut = expired || /timeout/i.test(String(error));
|
|
1060
|
+
const code = options.signal?.aborted ? "operator_browser_cancelled" : timedOut ? "operator_browser_timeout" : "operator_browser_failed";
|
|
1061
|
+
const toolError = browserSessionToolError(error, code, timedOut ? "timeout" : "unavailable");
|
|
1062
|
+
// Playwright call logs can echo signed URLs, selectors or fill values. Keep
|
|
1063
|
+
// interruption codes and a closed phase, never their raw diagnostic text.
|
|
1064
|
+
const safeError = ["operator_browser_timeout", "operator_browser_cancelled"].includes(toolError.code)
|
|
1065
|
+
? { ...toolError, message: interruptionMessage(toolError.code === "operator_browser_cancelled") }
|
|
1066
|
+
: toolError;
|
|
1067
|
+
return { ok: false, method: "local_browser_cdp", error: safeError };
|
|
1068
|
+
}
|
|
1069
|
+
finally {
|
|
1070
|
+
clearTimeout(timer);
|
|
1071
|
+
options.signal?.removeEventListener("abort", cancel);
|
|
1072
|
+
// Close only the page created above. Browser.close() on connectOverCDP
|
|
1073
|
+
// disconnects this client; it does not terminate the external browser.
|
|
1074
|
+
let cleanupTimer;
|
|
1075
|
+
const cleaned = await Promise.race([
|
|
1076
|
+
(async () => {
|
|
1077
|
+
await page?.close().catch(() => { });
|
|
1078
|
+
await session?.detach().catch(() => { });
|
|
1079
|
+
await browser?.close().catch(() => { });
|
|
1080
|
+
return !ownedBrowserCleanupIncomplete(browser);
|
|
1081
|
+
})(),
|
|
1082
|
+
new Promise((resolve) => { cleanupTimer = setTimeout(() => resolve(false), 1000); }),
|
|
1083
|
+
]);
|
|
1084
|
+
clearTimeout(cleanupTimer);
|
|
1085
|
+
if (!cleaned) {
|
|
1086
|
+
void session?.detach().catch(() => { });
|
|
1087
|
+
void browser?.close().catch(() => { });
|
|
1088
|
+
return { ok: false, method: "local_browser_cdp", error: { category: "timeout", code: "operator_cleanup_incomplete", message: "Task cleanup could not be confirmed within its bounded cleanup window; do not assume its owned tab was closed" } };
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
724
1092
|
export async function runBrowserSession(options) {
|
|
725
1093
|
sanitizeSmartWebTempEnv();
|
|
1094
|
+
if (options.cdpUrl && !options.profileDir)
|
|
1095
|
+
return runConnectedBrowserSession(options);
|
|
726
1096
|
const resolveOptions = {
|
|
727
1097
|
timeoutMs: options.timeoutMs,
|
|
728
1098
|
};
|