whopper 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyzer/apply.js +2 -2
- package/dist/analyzer/apply.js.map +1 -1
- package/dist/analyzer/match.d.ts +1 -0
- package/dist/analyzer/match.d.ts.map +1 -1
- package/dist/analyzer/match.js +1 -0
- package/dist/analyzer/match.js.map +1 -1
- package/dist/browser/active_scan.d.ts +5 -0
- package/dist/browser/active_scan.d.ts.map +1 -0
- package/dist/browser/active_scan.js +73 -0
- package/dist/browser/active_scan.js.map +1 -0
- package/dist/browser/active_scan.test.d.ts +2 -0
- package/dist/browser/active_scan.test.d.ts.map +1 -0
- package/dist/browser/active_scan.test.js +207 -0
- package/dist/browser/active_scan.test.js.map +1 -0
- package/dist/browser/index.d.ts.map +1 -1
- package/dist/browser/index.js +148 -36
- package/dist/browser/index.js.map +1 -1
- package/dist/browser/index.test.js +141 -3
- package/dist/browser/index.test.js.map +1 -1
- package/dist/browser/types.d.ts +1 -0
- package/dist/browser/types.d.ts.map +1 -1
- package/dist/commands/active_scan_runner.d.ts +5 -0
- package/dist/commands/active_scan_runner.d.ts.map +1 -0
- package/dist/commands/active_scan_runner.js +28 -0
- package/dist/commands/active_scan_runner.js.map +1 -0
- package/dist/commands/active_scan_runner.test.d.ts +2 -0
- package/dist/commands/active_scan_runner.test.d.ts.map +1 -0
- package/dist/commands/active_scan_runner.test.js +80 -0
- package/dist/commands/active_scan_runner.test.js.map +1 -0
- package/dist/commands/detect.d.ts.map +1 -1
- package/dist/commands/detect.js +7 -0
- package/dist/commands/detect.js.map +1 -1
- package/dist/commands/detect.test.js +12 -0
- package/dist/commands/detect.test.js.map +1 -1
- package/dist/signatures/_types.d.ts +6 -0
- package/dist/signatures/_types.d.ts.map +1 -1
- package/dist/signatures/signatures.test.js +24 -2
- package/dist/signatures/signatures.test.js.map +1 -1
- package/dist/signatures/technologies/magento.d.ts.map +1 -1
- package/dist/signatures/technologies/magento.js +6 -0
- package/dist/signatures/technologies/magento.js.map +1 -1
- package/dist/signatures/technologies/magento.test.d.ts +2 -0
- package/dist/signatures/technologies/magento.test.d.ts.map +1 -0
- package/dist/signatures/technologies/magento.test.js +28 -0
- package/dist/signatures/technologies/magento.test.js.map +1 -0
- package/package.json +1 -1
package/dist/analyzer/apply.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { matchString } from "./match.js";
|
|
1
|
+
import { matchString, truncateBodyForEvidence } from "./match.js";
|
|
2
2
|
function isFirstPartyResponse(response) {
|
|
3
3
|
return response.isFirstParty ?? true;
|
|
4
4
|
}
|
|
@@ -113,7 +113,7 @@ export const applySignature = (context, signature) => {
|
|
|
113
113
|
}
|
|
114
114
|
evidences.push({
|
|
115
115
|
type: "body",
|
|
116
|
-
value:
|
|
116
|
+
value: truncateBodyForEvidence(body),
|
|
117
117
|
version: result.version,
|
|
118
118
|
confidence: rule.confidence,
|
|
119
119
|
host: response.host,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"apply.js","sourceRoot":"","sources":["../../src/analyzer/apply.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"apply.js","sourceRoot":"","sources":["../../src/analyzer/apply.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAElE,SAAS,oBAAoB,CAAC,QAAkB;IAC9C,OAAO,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAkC;IAC5D,OAAO,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;AACrC,CAAC;AAED,SAAS,qBAAqB,CAAC,WAA+B;IAC5D,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,gBAAgB,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IACnD,OAAO,CACL,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC;QAClC,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC;QACvC,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC;QACvC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,CACjC,CAAC;AACJ,CAAC;AAED,SAAS,kCAAkC,CAAC,WAAmB;IAC7D,MAAM,gBAAgB,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IACnD,OAAO,CACL,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC;QACrC,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC;QACvC,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC,CACxC,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,SAAoB;IACxC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzE,OAAO,UAAU,IAAI,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxD,CAAC;AAED,SAAS,cAAc,CAAC,SAAoB;IAC1C,OAAO,SAAS,CAAC,OAAO,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAkB,EAAE,OAAgB;IAC9D,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACrD,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,WAAW,CAAC,CAAC,CAAC,kCAAkC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC/E,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,OAAgB,EAChB,SAAoB,EACG,EAAE;IACzB,IAAI,SAAS,GAAe,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;IAC5B,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,CAAC;IACvC,MAAM,mBAAmB,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC3E,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAErE,aAAa;IACb,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC;QACf,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9B,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;gBACpC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;oBAC5D,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;gBACzB,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBACvC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,SAAS,EAAE,QAAQ,CAAC,GAAG;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;YAC3E,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAE,CAAC;YACjD,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,GAAG,MAAM,KAAK,WAAW,EAAE;gBAClC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,SAAS,EAAE,QAAQ,CAAC,GAAG;aACxB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,eAAe;IACf,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;gBACpC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC3C,SAAS;gBACX,CAAC;gBAED,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,SAAS;gBACX,CAAC;gBACD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,uBAAuB,CAAC,IAAI,CAAC;oBACpC,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,SAAS,EAAE,QAAQ,CAAC,GAAG;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzD,MAAM,eAAe,GAAG,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,GAAG,CAAC,CAAC;YACzD,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3E,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,EAAE;gBACxC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;YAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,GAAG,IAAI,KAAK,MAAM,EAAE;gBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,IAAI,EAAE,2BAA2B,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,2BAA2B,CAAC,KAAK,CACvD,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,CACrC,CAAC;QACF,MAAM,oBAAoB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACxE,IAAI,CAAC,UAAU,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACzC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,SAAS;SACV,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC"}
|
package/dist/analyzer/match.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../../src/analyzer/match.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAErD,KAAK,WAAW,GAAG;IACjB,GAAG,EAAE,OAAO,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,MAAM,EAAE,OAAO,KAAK,KAAG,WAQzD,CAAC"}
|
|
1
|
+
{"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../../src/analyzer/match.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAErD,KAAK,WAAW,GAAG;IACjB,GAAG,EAAE,OAAO,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,MAAM,EAAE,OAAO,KAAK,KAAG,WAQzD,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAAI,MAAM,MAAM,KAAG,MACvB,CAAC"}
|
package/dist/analyzer/match.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"match.js","sourceRoot":"","sources":["../../src/analyzer/match.ts"],"names":[],"mappings":"AAOA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,KAAY,EAAe,EAAE;IACtE,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAC5C,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"match.js","sourceRoot":"","sources":["../../src/analyzer/match.ts"],"names":[],"mappings":"AAOA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,KAAY,EAAe,EAAE;IACtE,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC9D,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { APIRequestContext } from "playwright";
|
|
2
|
+
import type { Response } from "./types.js";
|
|
3
|
+
export declare function isRelativePath(path: string): boolean;
|
|
4
|
+
export declare function fetchActiveRule(baseUrl: string, path: string, request: APIRequestContext, timeoutMs: number): Promise<Response | null>;
|
|
5
|
+
//# sourceMappingURL=active_scan.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active_scan.d.ts","sourceRoot":"","sources":["../../src/browser/active_scan.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD;AAQD,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,iBAAiB,EAC1B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAkE1B"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { logger } from "../logger/index.js";
|
|
2
|
+
import { getHostFromUrl, isFirstPartyHost, isSameHost } from "./utils.js";
|
|
3
|
+
export function isRelativePath(path) {
|
|
4
|
+
return !/^[a-z][a-z0-9+.-]*:/i.test(path) && !path.startsWith("//");
|
|
5
|
+
}
|
|
6
|
+
const MAX_REDIRECT_HOPS = 3;
|
|
7
|
+
function isRedirectStatus(status) {
|
|
8
|
+
return status >= 300 && status < 400;
|
|
9
|
+
}
|
|
10
|
+
export async function fetchActiveRule(baseUrl, path, request, timeoutMs) {
|
|
11
|
+
if (!isRelativePath(path)) {
|
|
12
|
+
logger.warn(`Active scan path must be relative: ${path}`);
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
let url;
|
|
16
|
+
try {
|
|
17
|
+
url = new URL(path, baseUrl).toString();
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
logger.warn(`Invalid active scan baseUrl or path: baseUrl=${baseUrl}, path=${path}`);
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
const pageHost = getHostFromUrl(baseUrl) ?? "";
|
|
24
|
+
try {
|
|
25
|
+
for (let hop = 0; hop <= MAX_REDIRECT_HOPS; hop++) {
|
|
26
|
+
logger.info(`Active scan request: ${url}`);
|
|
27
|
+
const res = await request.get(url, {
|
|
28
|
+
timeout: timeoutMs,
|
|
29
|
+
maxRedirects: 0,
|
|
30
|
+
});
|
|
31
|
+
const status = res.status();
|
|
32
|
+
const location = res.headers()["location"];
|
|
33
|
+
if (isRedirectStatus(status) && location) {
|
|
34
|
+
if (hop === MAX_REDIRECT_HOPS) {
|
|
35
|
+
logger.warn(`Active scan exceeded redirect limit: ${url}`);
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
let nextUrl;
|
|
39
|
+
try {
|
|
40
|
+
nextUrl = new URL(location, url).toString();
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
logger.warn(`Active scan invalid redirect location: ${location}`);
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
const nextHost = getHostFromUrl(nextUrl) ?? "";
|
|
47
|
+
if (!nextHost || !isSameHost(pageHost, nextHost)) {
|
|
48
|
+
logger.warn(`Active scan redirect to non-same-host blocked: ${nextUrl}`);
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
url = nextUrl;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const host = getHostFromUrl(url) ?? "";
|
|
55
|
+
const body = await res.text().catch(() => "");
|
|
56
|
+
return {
|
|
57
|
+
url,
|
|
58
|
+
host,
|
|
59
|
+
isFirstParty: host ? isFirstPartyHost(pageHost, host) : false,
|
|
60
|
+
status,
|
|
61
|
+
headers: res.headers(),
|
|
62
|
+
body,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
catch (e) {
|
|
68
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
69
|
+
logger.warn(`Active scan fetch failed (${url}): ${message.split("\n")[0]}`);
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=active_scan.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active_scan.js","sourceRoot":"","sources":["../../src/browser/active_scan.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,IAAY,EACZ,OAA0B,EAC1B,SAAiB;IAEjB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,IAAI,CAAC,gDAAgD,OAAO,UAAU,IAAI,EAAE,CAAC,CAAC;QACrF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/C,IAAI,CAAC;QACH,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,iBAAiB,EAAE,GAAG,EAAE,EAAE,CAAC;YAClD,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAC;YAC3C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;gBACjC,OAAO,EAAE,SAAS;gBAClB,YAAY,EAAE,CAAC;aAChB,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC3C,IAAI,gBAAgB,CAAC,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,iBAAiB,EAAE,CAAC;oBAC9B,MAAM,CAAC,IAAI,CAAC,wCAAwC,GAAG,EAAE,CAAC,CAAC;oBAC3D,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC9C,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,CAAC,IAAI,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;oBAClE,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC/C,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;oBACjD,MAAM,CAAC,IAAI,CACT,kDAAkD,OAAO,EAAE,CAC5D,CAAC;oBACF,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,GAAG,GAAG,OAAO,CAAC;gBACd,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,OAAO;gBACL,GAAG;gBACH,IAAI;gBACJ,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC7D,MAAM;gBACN,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;gBACtB,IAAI;aACL,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,CACT,6BAA6B,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAC/D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active_scan.test.d.ts","sourceRoot":"","sources":["../../src/browser/active_scan.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
2
|
+
import { fetchActiveRule } from "./active_scan.js";
|
|
3
|
+
const mockRequest = (impl) => {
|
|
4
|
+
return {
|
|
5
|
+
get: vi.fn(async (url) => impl(url)),
|
|
6
|
+
};
|
|
7
|
+
};
|
|
8
|
+
describe("fetchActiveRule", () => {
|
|
9
|
+
beforeEach(() => {
|
|
10
|
+
vi.restoreAllMocks();
|
|
11
|
+
});
|
|
12
|
+
it("fetches a relative path resolved against the base URL", async () => {
|
|
13
|
+
const request = mockRequest((url) => {
|
|
14
|
+
expect(url).toBe("https://example.com/magento_version");
|
|
15
|
+
return {
|
|
16
|
+
status: () => 200,
|
|
17
|
+
text: async () => "Magento/2.4 (Community)",
|
|
18
|
+
headers: () => ({ "content-type": "text/plain" }),
|
|
19
|
+
};
|
|
20
|
+
});
|
|
21
|
+
const res = await fetchActiveRule("https://example.com/some/page", "/magento_version", request, 5000);
|
|
22
|
+
expect(res?.url).toBe("https://example.com/magento_version");
|
|
23
|
+
expect(res?.status).toBe(200);
|
|
24
|
+
expect(res?.body).toBe("Magento/2.4 (Community)");
|
|
25
|
+
expect(res?.host).toBe("example.com");
|
|
26
|
+
expect(res?.isFirstParty).toBe(true);
|
|
27
|
+
});
|
|
28
|
+
it("rejects absolute URLs", async () => {
|
|
29
|
+
const request = mockRequest(() => {
|
|
30
|
+
throw new Error("should not be called");
|
|
31
|
+
});
|
|
32
|
+
const res = await fetchActiveRule("https://example.com/", "https://evil.example/x", request, 5000);
|
|
33
|
+
expect(res).toBeNull();
|
|
34
|
+
});
|
|
35
|
+
it("rejects protocol-relative paths", async () => {
|
|
36
|
+
const request = mockRequest(() => {
|
|
37
|
+
throw new Error("should not be called");
|
|
38
|
+
});
|
|
39
|
+
const res = await fetchActiveRule("https://example.com/", "//evil.example/x", request, 5000);
|
|
40
|
+
expect(res).toBeNull();
|
|
41
|
+
});
|
|
42
|
+
it("returns null when the request throws", async () => {
|
|
43
|
+
const request = mockRequest(() => {
|
|
44
|
+
throw new Error("network down");
|
|
45
|
+
});
|
|
46
|
+
const res = await fetchActiveRule("https://example.com/", "/magento_version", request, 5000);
|
|
47
|
+
expect(res).toBeNull();
|
|
48
|
+
});
|
|
49
|
+
describe("URL resolution with /path (host root)", () => {
|
|
50
|
+
const cases = [
|
|
51
|
+
{
|
|
52
|
+
base: "https://example.com/",
|
|
53
|
+
expected: "https://example.com/magento_version",
|
|
54
|
+
note: "root",
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
base: "https://www.example.com/en/",
|
|
58
|
+
expected: "https://www.example.com/magento_version",
|
|
59
|
+
note: "redirected to subpath ignores subpath",
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
base: "https://example.com/shop/catalog/item",
|
|
63
|
+
expected: "https://example.com/magento_version",
|
|
64
|
+
note: "deep path ignored",
|
|
65
|
+
},
|
|
66
|
+
];
|
|
67
|
+
for (const { base, expected, note } of cases) {
|
|
68
|
+
it(`resolves '/magento_version' on ${note} (${base})`, async () => {
|
|
69
|
+
let called = "";
|
|
70
|
+
const request = mockRequest((url) => {
|
|
71
|
+
called = url;
|
|
72
|
+
return {
|
|
73
|
+
status: () => 200,
|
|
74
|
+
text: async () => "Magento/2.4",
|
|
75
|
+
headers: () => ({}),
|
|
76
|
+
};
|
|
77
|
+
});
|
|
78
|
+
await fetchActiveRule(base, "/magento_version", request, 5000);
|
|
79
|
+
expect(called).toBe(expected);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
describe("URL resolution with ./path", () => {
|
|
84
|
+
const cases = [
|
|
85
|
+
{
|
|
86
|
+
base: "https://example.com/",
|
|
87
|
+
expected: "https://example.com/magento_version",
|
|
88
|
+
note: "root",
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
base: "https://example.com/shop/",
|
|
92
|
+
expected: "https://example.com/shop/magento_version",
|
|
93
|
+
note: "subpath with trailing slash",
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
base: "https://example.com/shop",
|
|
97
|
+
expected: "https://example.com/magento_version",
|
|
98
|
+
note: "subpath without trailing slash (treated as file)",
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
base: "https://example.com/index.html",
|
|
102
|
+
expected: "https://example.com/magento_version",
|
|
103
|
+
note: "file at root",
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
base: "https://example.com/shop/index.html",
|
|
107
|
+
expected: "https://example.com/shop/magento_version",
|
|
108
|
+
note: "file under subpath",
|
|
109
|
+
},
|
|
110
|
+
];
|
|
111
|
+
for (const { base, expected, note } of cases) {
|
|
112
|
+
it(`resolves './magento_version' on ${note} (${base})`, async () => {
|
|
113
|
+
let called = "";
|
|
114
|
+
const request = mockRequest((url) => {
|
|
115
|
+
called = url;
|
|
116
|
+
return {
|
|
117
|
+
status: () => 200,
|
|
118
|
+
text: async () => "Magento/2.4",
|
|
119
|
+
headers: () => ({}),
|
|
120
|
+
};
|
|
121
|
+
});
|
|
122
|
+
await fetchActiveRule(base, "./magento_version", request, 5000);
|
|
123
|
+
expect(called).toBe(expected);
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
it("returns null when baseUrl is not a valid absolute URL", async () => {
|
|
128
|
+
const request = mockRequest(() => {
|
|
129
|
+
throw new Error("should not be called");
|
|
130
|
+
});
|
|
131
|
+
const res = await fetchActiveRule("not-a-url", "./magento_version", request, 5000);
|
|
132
|
+
expect(res).toBeNull();
|
|
133
|
+
});
|
|
134
|
+
it("follows a same-host redirect and returns the final response", async () => {
|
|
135
|
+
const calls = [];
|
|
136
|
+
const request = mockRequest((url) => {
|
|
137
|
+
calls.push(url);
|
|
138
|
+
if (url === "https://example.com/magento_version") {
|
|
139
|
+
return {
|
|
140
|
+
status: () => 302,
|
|
141
|
+
text: async () => "",
|
|
142
|
+
headers: () => ({ location: "/en/magento_version" }),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
status: () => 200,
|
|
147
|
+
text: async () => "Magento/2.4.6",
|
|
148
|
+
headers: () => ({}),
|
|
149
|
+
};
|
|
150
|
+
});
|
|
151
|
+
const res = await fetchActiveRule("https://example.com/", "/magento_version", request, 5000);
|
|
152
|
+
expect(calls).toEqual([
|
|
153
|
+
"https://example.com/magento_version",
|
|
154
|
+
"https://example.com/en/magento_version",
|
|
155
|
+
]);
|
|
156
|
+
expect(res?.url).toBe("https://example.com/en/magento_version");
|
|
157
|
+
expect(res?.status).toBe(200);
|
|
158
|
+
expect(res?.body).toBe("Magento/2.4.6");
|
|
159
|
+
});
|
|
160
|
+
it("blocks redirects to a sibling subdomain even if same registrable domain", async () => {
|
|
161
|
+
const request = mockRequest((url) => {
|
|
162
|
+
if (url === "https://app.example.com/magento_version") {
|
|
163
|
+
return {
|
|
164
|
+
status: () => 302,
|
|
165
|
+
text: async () => "",
|
|
166
|
+
headers: () => ({ location: "https://cdn.example.com/magento_version" }),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
throw new Error("should not follow cross-host redirect");
|
|
170
|
+
});
|
|
171
|
+
const res = await fetchActiveRule("https://app.example.com/", "/magento_version", request, 5000);
|
|
172
|
+
expect(res).toBeNull();
|
|
173
|
+
});
|
|
174
|
+
it("blocks redirects to a third-party host", async () => {
|
|
175
|
+
const request = mockRequest((url) => {
|
|
176
|
+
if (url === "https://example.com/magento_version") {
|
|
177
|
+
return {
|
|
178
|
+
status: () => 302,
|
|
179
|
+
text: async () => "",
|
|
180
|
+
headers: () => ({ location: "https://evil.example/magento_version" }),
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
throw new Error("should not follow cross-host redirect");
|
|
184
|
+
});
|
|
185
|
+
const res = await fetchActiveRule("https://example.com/", "/magento_version", request, 5000);
|
|
186
|
+
expect(res).toBeNull();
|
|
187
|
+
});
|
|
188
|
+
it("stops after exceeding the redirect hop limit", async () => {
|
|
189
|
+
const request = mockRequest(() => ({
|
|
190
|
+
status: () => 302,
|
|
191
|
+
text: async () => "",
|
|
192
|
+
headers: () => ({ location: "/loop" }),
|
|
193
|
+
}));
|
|
194
|
+
const res = await fetchActiveRule("https://example.com/", "/magento_version", request, 5000);
|
|
195
|
+
expect(res).toBeNull();
|
|
196
|
+
});
|
|
197
|
+
it("returns response even on non-200 status (caller decides)", async () => {
|
|
198
|
+
const request = mockRequest(() => ({
|
|
199
|
+
status: () => 404,
|
|
200
|
+
text: async () => "",
|
|
201
|
+
headers: () => ({}),
|
|
202
|
+
}));
|
|
203
|
+
const res = await fetchActiveRule("https://example.com/", "/magento_version", request, 5000);
|
|
204
|
+
expect(res?.status).toBe(404);
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
//# sourceMappingURL=active_scan.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active_scan.test.js","sourceRoot":"","sources":["../../src/browser/active_scan.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAQnD,MAAM,WAAW,GAAG,CAAC,IAA2D,EAAE,EAAE;IAClF,OAAO;QACL,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;KACb,CAAC;AACpC,CAAC,CAAC;AAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACxD,OAAO;gBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;gBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,yBAAyB;gBAC3C,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;aAClD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,+BAA+B,EAC/B,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAClD,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,wBAAwB,EACxB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;QACrD,MAAM,KAAK,GAA4D;YACrE;gBACE,IAAI,EAAE,sBAAsB;gBAC5B,QAAQ,EAAE,qCAAqC;gBAC/C,IAAI,EAAE,MAAM;aACb;YACD;gBACE,IAAI,EAAE,6BAA6B;gBACnC,QAAQ,EAAE,yCAAyC;gBACnD,IAAI,EAAE,uCAAuC;aAC9C;YACD;gBACE,IAAI,EAAE,uCAAuC;gBAC7C,QAAQ,EAAE,qCAAqC;gBAC/C,IAAI,EAAE,mBAAmB;aAC1B;SACF,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;YAC7C,EAAE,CAAC,kCAAkC,IAAI,KAAK,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE;gBAChE,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE;oBAClC,MAAM,GAAG,GAAG,CAAC;oBACb,OAAO;wBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;wBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,aAAa;wBAC/B,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;qBACpB,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,MAAM,eAAe,CAAC,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,MAAM,KAAK,GAA4D;YACrE;gBACE,IAAI,EAAE,sBAAsB;gBAC5B,QAAQ,EAAE,qCAAqC;gBAC/C,IAAI,EAAE,MAAM;aACb;YACD;gBACE,IAAI,EAAE,2BAA2B;gBACjC,QAAQ,EAAE,0CAA0C;gBACpD,IAAI,EAAE,6BAA6B;aACpC;YACD;gBACE,IAAI,EAAE,0BAA0B;gBAChC,QAAQ,EAAE,qCAAqC;gBAC/C,IAAI,EAAE,kDAAkD;aACzD;YACD;gBACE,IAAI,EAAE,gCAAgC;gBACtC,QAAQ,EAAE,qCAAqC;gBAC/C,IAAI,EAAE,cAAc;aACrB;YACD;gBACE,IAAI,EAAE,qCAAqC;gBAC3C,QAAQ,EAAE,0CAA0C;gBACpD,IAAI,EAAE,oBAAoB;aAC3B;SACF,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;YAC7C,EAAE,CAAC,mCAAmC,IAAI,KAAK,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE;gBACjE,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE;oBAClC,MAAM,GAAG,GAAG,CAAC;oBACb,OAAO;wBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;wBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,aAAa;wBAC/B,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;qBACpB,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,MAAM,eAAe,CAAC,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;gBAChE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,WAAW,EACX,mBAAmB,EACnB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE;YAClC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChB,IAAI,GAAG,KAAK,qCAAqC,EAAE,CAAC;gBAClD,OAAO;oBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;oBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;oBACpB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE,CAAC;iBACrD,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;gBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,eAAe;gBACjC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;aACpB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;YACpB,qCAAqC;YACrC,wCAAwC;SACzC,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QAChE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;QACvF,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE;YAClC,IAAI,GAAG,KAAK,yCAAyC,EAAE,CAAC;gBACtD,OAAO;oBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;oBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;oBACpB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,yCAAyC,EAAE,CAAC;iBACzE,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,0BAA0B,EAC1B,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE;YAClC,IAAI,GAAG,KAAK,qCAAqC,EAAE,CAAC;gBAClD,OAAO;oBACL,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;oBACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;oBACpB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,sCAAsC,EAAE,CAAC;iBACtE,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YACjC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;YACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;YACpB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YACjC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;YACjB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;YACpB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;SACpB,CAAC,CAAC,CAAC;QAEJ,MAAM,GAAG,GAAG,MAAM,eAAe,CAC/B,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/browser/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAsB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/browser/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAsB,MAAM,YAAY,CAAC;AA0C/E,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,uBAAuB,EAAE,MAAM,EAAE,EACjC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CA6YlB"}
|
package/dist/browser/index.js
CHANGED
|
@@ -3,6 +3,15 @@ import { chromium } from "playwright";
|
|
|
3
3
|
import { logger } from "../logger/index.js";
|
|
4
4
|
import { extractJsVariables, getHostFromUrl, isFirstPartyHost, isSameHost, sleep, } from "./utils.js";
|
|
5
5
|
const MAX_REDIRECT_HOPS = 20;
|
|
6
|
+
// Quiet period (ms) used for the custom network-idle decision.
|
|
7
|
+
// Playwright's built-in `networkidle` has a fixed 500ms threshold, which
|
|
8
|
+
// is too short: it fires before script-driven secondary requests (e.g. a
|
|
9
|
+
// CSS file loaded from within an async script) have a chance to start,
|
|
10
|
+
// causing technology-detection evidence to be dropped. We use a longer
|
|
11
|
+
// threshold evaluated by our own tracking.
|
|
12
|
+
const NETWORK_IDLE_THRESHOLD_MS = 2000;
|
|
13
|
+
// Polling interval (ms) for the idle-decision loop.
|
|
14
|
+
const NETWORK_IDLE_POLL_INTERVAL_MS = 200;
|
|
6
15
|
function colorizeStatusCode(statusCode) {
|
|
7
16
|
const code = String(statusCode);
|
|
8
17
|
if (statusCode >= 100 && statusCode < 200) {
|
|
@@ -23,7 +32,7 @@ function colorizeStatusCode(statusCode) {
|
|
|
23
32
|
return chalk.gray(code);
|
|
24
33
|
}
|
|
25
34
|
export async function openPage(url, timeoutMs, javascriptVariableNames, options = {}) {
|
|
26
|
-
const { userAgent, locale, extraHTTPHeaders, blockCrossDomainRedirect = false, } = options;
|
|
35
|
+
const { userAgent, locale, extraHTTPHeaders, blockCrossDomainRedirect = false, networkIdleThresholdMs = NETWORK_IDLE_THRESHOLD_MS, } = options;
|
|
27
36
|
const pageHost = getHostFromUrl(url);
|
|
28
37
|
if (!pageHost) {
|
|
29
38
|
throw new Error(`Invalid URL: ${url}`);
|
|
@@ -68,8 +77,7 @@ export async function openPage(url, timeoutMs, javascriptVariableNames, options
|
|
|
68
77
|
await route.continue();
|
|
69
78
|
return;
|
|
70
79
|
}
|
|
71
|
-
if (blockCrossDomainRedirect &&
|
|
72
|
-
!isSameHost(pageHost, targetHost)) {
|
|
80
|
+
if (blockCrossDomainRedirect && !isSameHost(pageHost, targetHost)) {
|
|
73
81
|
logger.warn(`Blocked cross-domain redirect: ${targetUrl}`);
|
|
74
82
|
blockedByRedirectPolicy = true;
|
|
75
83
|
urls.push({ url: targetUrl, error: "Blocked cross-domain redirect" });
|
|
@@ -104,7 +112,9 @@ export async function openPage(url, timeoutMs, javascriptVariableNames, options
|
|
|
104
112
|
// URLs captured here are added to preInspectedUrls so that the
|
|
105
113
|
// page.on("response") listener can skip them and avoid duplicates.
|
|
106
114
|
const firstResponse = response;
|
|
107
|
-
for (let hop = 0; hop < MAX_REDIRECT_HOPS &&
|
|
115
|
+
for (let hop = 0; hop < MAX_REDIRECT_HOPS &&
|
|
116
|
+
response.status() >= 300 &&
|
|
117
|
+
response.status() < 400; hop++) {
|
|
108
118
|
// Capture intermediate 3xx responses so their headers (e.g. server,
|
|
109
119
|
// x-powered-by) are available for analysis even though the browser
|
|
110
120
|
// never sees these responses directly.
|
|
@@ -161,44 +171,146 @@ export async function openPage(url, timeoutMs, javascriptVariableNames, options
|
|
|
161
171
|
});
|
|
162
172
|
const urls = [];
|
|
163
173
|
const responses = [];
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
174
|
+
// Timestamp of the most recent network activity (request sent or
|
|
175
|
+
// response received). Used by the custom idle-decision loop below.
|
|
176
|
+
let lastNetworkActivityAt = Date.now();
|
|
177
|
+
// Number of requests currently in flight. We include this in the idle
|
|
178
|
+
// condition so that a request whose response takes longer than the
|
|
179
|
+
// quiet-period threshold cannot cause the loop to exit before the
|
|
180
|
+
// response is captured.
|
|
181
|
+
let inFlightRequestCount = 0;
|
|
182
|
+
// Pending response-handler promises. The response handler awaits
|
|
183
|
+
// `response.text()`, which can still be in progress when the idle loop
|
|
184
|
+
// decides to break; awaiting this set after the loop prevents those
|
|
185
|
+
// in-progress body reads from being dropped.
|
|
186
|
+
const pendingResponseWork = new Set();
|
|
187
|
+
const onRequest = () => {
|
|
188
|
+
inFlightRequestCount++;
|
|
189
|
+
lastNetworkActivityAt = Date.now();
|
|
190
|
+
};
|
|
191
|
+
const onRequestFinished = () => {
|
|
192
|
+
inFlightRequestCount = Math.max(0, inFlightRequestCount - 1);
|
|
193
|
+
lastNetworkActivityAt = Date.now();
|
|
194
|
+
};
|
|
195
|
+
const onRequestFailed = () => {
|
|
196
|
+
inFlightRequestCount = Math.max(0, inFlightRequestCount - 1);
|
|
197
|
+
lastNetworkActivityAt = Date.now();
|
|
198
|
+
};
|
|
199
|
+
const onResponse = (response) => {
|
|
200
|
+
lastNetworkActivityAt = Date.now();
|
|
201
|
+
const work = (async () => {
|
|
202
|
+
const responseUrl = response.url();
|
|
203
|
+
const statusCode = response.status();
|
|
204
|
+
// Skip responses already captured by the pre-inspection loop to
|
|
205
|
+
// avoid duplicates.
|
|
206
|
+
if (preInspectedUrls.has(responseUrl) &&
|
|
207
|
+
statusCode >= 300 &&
|
|
208
|
+
statusCode < 400) {
|
|
209
|
+
logger.debug(`Skipping already captured response [${colorizeStatusCode(statusCode)}] ${responseUrl}`);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const responseHost = getHostFromUrl(responseUrl) ?? "";
|
|
213
|
+
logger.debug(`Received response [${colorizeStatusCode(statusCode)}] ${responseUrl}`);
|
|
214
|
+
const request = response.request();
|
|
215
|
+
if (request.isNavigationRequest() &&
|
|
216
|
+
request.frame() === page.mainFrame()) {
|
|
217
|
+
urls.push({ url: responseUrl, status: statusCode });
|
|
218
|
+
}
|
|
219
|
+
const res = {
|
|
220
|
+
url: responseUrl,
|
|
221
|
+
host: responseHost,
|
|
222
|
+
isFirstParty: responseHost
|
|
223
|
+
? isFirstPartyHost(pageHost, responseHost)
|
|
224
|
+
: false,
|
|
225
|
+
status: statusCode,
|
|
226
|
+
headers: response.headers(),
|
|
227
|
+
};
|
|
228
|
+
const body = await response.text().catch(() => null);
|
|
229
|
+
if (body) {
|
|
230
|
+
res.body = body;
|
|
231
|
+
}
|
|
232
|
+
responses.push(res);
|
|
233
|
+
})();
|
|
234
|
+
pendingResponseWork.add(work);
|
|
235
|
+
work.finally(() => {
|
|
236
|
+
pendingResponseWork.delete(work);
|
|
237
|
+
});
|
|
238
|
+
};
|
|
239
|
+
page.on("request", onRequest);
|
|
240
|
+
page.on("requestfinished", onRequestFinished);
|
|
241
|
+
page.on("requestfailed", onRequestFailed);
|
|
242
|
+
page.on("response", onResponse);
|
|
194
243
|
let timeoutOccurred = false;
|
|
195
|
-
const
|
|
244
|
+
const navigationStartedAt = Date.now();
|
|
245
|
+
// Playwright's built-in `networkidle` is flaky (500ms fixed threshold),
|
|
246
|
+
// so resolve goto at the `load` event (onload fired) and wait for
|
|
247
|
+
// secondary requests triggered by deferred scripts using our own
|
|
248
|
+
// idle-decision loop below.
|
|
249
|
+
const goto = page.goto(url, { waitUntil: "load" });
|
|
196
250
|
const result = await Promise.race([
|
|
197
251
|
goto.then(() => "loaded").catch((e) => e.message),
|
|
198
252
|
sleep(timeoutMs).then(() => "timeout"),
|
|
199
253
|
]);
|
|
254
|
+
// After onload, wait until all in-flight requests have finished and a
|
|
255
|
+
// quiet period of `networkIdleThresholdMs` has elapsed (or the overall
|
|
256
|
+
// timeout is reached) so that secondary requests triggered by deferred
|
|
257
|
+
// scripts are captured. Including in-flight count in the condition
|
|
258
|
+
// prevents the loop from exiting while a slow response
|
|
259
|
+
// (> networkIdleThresholdMs) is still pending.
|
|
260
|
+
//
|
|
261
|
+
// Tracks whether the loop exited because the network actually went
|
|
262
|
+
// idle, or because the overall timeout budget was exhausted. In the
|
|
263
|
+
// latter case we must return `timeoutOccurred: true` so callers can
|
|
264
|
+
// distinguish a full capture from a partial one.
|
|
265
|
+
let idleWaitTimedOut = false;
|
|
200
266
|
if (result === "loaded") {
|
|
201
|
-
|
|
267
|
+
while (true) {
|
|
268
|
+
const now = Date.now();
|
|
269
|
+
const elapsed = now - navigationStartedAt;
|
|
270
|
+
if (elapsed >= timeoutMs) {
|
|
271
|
+
idleWaitTimedOut = true;
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
274
|
+
const idleFor = now - lastNetworkActivityAt;
|
|
275
|
+
if (inFlightRequestCount === 0 && idleFor >= networkIdleThresholdMs) {
|
|
276
|
+
break;
|
|
277
|
+
}
|
|
278
|
+
const remainingBudget = timeoutMs - elapsed;
|
|
279
|
+
await sleep(Math.min(NETWORK_IDLE_POLL_INTERVAL_MS, remainingBudget));
|
|
280
|
+
}
|
|
281
|
+
// If any response handlers are still running at loop exit (e.g.
|
|
282
|
+
// waiting on `response.text()`), wait for them to finish so their
|
|
283
|
+
// captures are not dropped. However, do not wait beyond the overall
|
|
284
|
+
// timeout: race against the remaining budget so that when the budget
|
|
285
|
+
// is exhausted we drop any still-pending in-flight captures rather
|
|
286
|
+
// than letting the process block indefinitely on slow body reads.
|
|
287
|
+
if (pendingResponseWork.size > 0) {
|
|
288
|
+
const remainingBudget = Math.max(0, timeoutMs - (Date.now() - navigationStartedAt));
|
|
289
|
+
if (remainingBudget > 0) {
|
|
290
|
+
await Promise.race([
|
|
291
|
+
Promise.allSettled(Array.from(pendingResponseWork)),
|
|
292
|
+
sleep(remainingBudget),
|
|
293
|
+
]);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
// Detach network listeners so that the `responses` snapshot is frozen
|
|
298
|
+
// at this point. Subsequent activity from cookie/JS extraction (or any
|
|
299
|
+
// delayed in-flight responses) must not mutate the captured set, since
|
|
300
|
+
// those entries would not be awaited and could race with the return.
|
|
301
|
+
page.off("request", onRequest);
|
|
302
|
+
page.off("requestfinished", onRequestFinished);
|
|
303
|
+
page.off("requestfailed", onRequestFailed);
|
|
304
|
+
page.off("response", onResponse);
|
|
305
|
+
logger.info(`${responses.length} responses captured`);
|
|
306
|
+
if (result === "loaded") {
|
|
307
|
+
if (idleWaitTimedOut) {
|
|
308
|
+
timeoutOccurred = true;
|
|
309
|
+
logger.warn(`Timeout of ${timeoutMs}ms exceeded while waiting for network idle after load on ${url}`);
|
|
310
|
+
}
|
|
311
|
+
else {
|
|
312
|
+
logger.info("Page loaded successfully");
|
|
313
|
+
}
|
|
202
314
|
}
|
|
203
315
|
else if (result === "timeout") {
|
|
204
316
|
timeoutOccurred = true;
|