webpeel 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -5
- package/dist/cli.js +1299 -85
- package/dist/cli.js.map +1 -1
- package/dist/core/application-tracker.d.ts +85 -0
- package/dist/core/application-tracker.d.ts.map +1 -0
- package/dist/core/application-tracker.js +184 -0
- package/dist/core/application-tracker.js.map +1 -0
- package/dist/core/apply.d.ts +163 -0
- package/dist/core/apply.d.ts.map +1 -0
- package/dist/core/apply.js +817 -0
- package/dist/core/apply.js.map +1 -0
- package/dist/core/branding.d.ts +1 -1
- package/dist/core/branding.d.ts.map +1 -1
- package/dist/core/budget.d.ts +43 -0
- package/dist/core/budget.d.ts.map +1 -0
- package/dist/core/budget.js +325 -0
- package/dist/core/budget.js.map +1 -0
- package/dist/core/challenge-detection.d.ts +27 -0
- package/dist/core/challenge-detection.d.ts.map +1 -0
- package/dist/core/challenge-detection.js +436 -0
- package/dist/core/challenge-detection.js.map +1 -0
- package/dist/core/change-tracking.d.ts.map +1 -1
- package/dist/core/change-tracking.js +10 -1
- package/dist/core/change-tracking.js.map +1 -1
- package/dist/core/crawler.d.ts.map +1 -1
- package/dist/core/crawler.js +17 -4
- package/dist/core/crawler.js.map +1 -1
- package/dist/core/diff.d.ts +62 -0
- package/dist/core/diff.d.ts.map +1 -0
- package/dist/core/diff.js +289 -0
- package/dist/core/diff.js.map +1 -0
- package/dist/core/extract-listings.d.ts +39 -0
- package/dist/core/extract-listings.d.ts.map +1 -0
- package/dist/core/extract-listings.js +331 -0
- package/dist/core/extract-listings.js.map +1 -0
- package/dist/core/extract.d.ts.map +1 -1
- package/dist/core/extract.js +15 -2
- package/dist/core/extract.js.map +1 -1
- package/dist/core/fetcher.d.ts +29 -3
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +158 -20
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/human.d.ts +176 -0
- package/dist/core/human.d.ts.map +1 -0
- package/dist/core/human.js +681 -0
- package/dist/core/human.js.map +1 -0
- package/dist/core/jobs.d.ts +12 -2
- package/dist/core/jobs.d.ts.map +1 -1
- package/dist/core/jobs.js +124 -2
- package/dist/core/jobs.js.map +1 -1
- package/dist/core/map.d.ts.map +1 -1
- package/dist/core/map.js +14 -2
- package/dist/core/map.js.map +1 -1
- package/dist/core/paginate.d.ts +32 -0
- package/dist/core/paginate.d.ts.map +1 -0
- package/dist/core/paginate.js +107 -0
- package/dist/core/paginate.js.map +1 -0
- package/dist/core/rate-governor.d.ts +81 -0
- package/dist/core/rate-governor.d.ts.map +1 -0
- package/dist/core/rate-governor.js +238 -0
- package/dist/core/rate-governor.js.map +1 -0
- package/dist/core/search-provider.d.ts +5 -0
- package/dist/core/search-provider.d.ts.map +1 -1
- package/dist/core/search-provider.js +81 -2
- package/dist/core/search-provider.js.map +1 -1
- package/dist/core/site-search.d.ts +45 -0
- package/dist/core/site-search.d.ts.map +1 -0
- package/dist/core/site-search.js +253 -0
- package/dist/core/site-search.js.map +1 -0
- package/dist/core/strategies.d.ts +8 -0
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +185 -45
- package/dist/core/strategies.js.map +1 -1
- package/dist/core/strategy-hooks.d.ts +6 -0
- package/dist/core/strategy-hooks.d.ts.map +1 -1
- package/dist/core/strategy-hooks.js.map +1 -1
- package/dist/core/table-format.d.ts +31 -0
- package/dist/core/table-format.d.ts.map +1 -0
- package/dist/core/table-format.js +147 -0
- package/dist/core/table-format.js.map +1 -0
- package/dist/core/user-agents.d.ts +58 -0
- package/dist/core/user-agents.d.ts.map +1 -0
- package/dist/core/user-agents.js +159 -0
- package/dist/core/user-agents.js.map +1 -0
- package/dist/core/watch.d.ts +100 -0
- package/dist/core/watch.d.ts.map +1 -0
- package/dist/core/watch.js +368 -0
- package/dist/core/watch.js.map +1 -0
- package/dist/index.d.ts +13 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +41 -4
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +3 -0
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +73 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/llms.txt +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Human Behavior Engine for WebPeel
|
|
3
|
+
*
|
|
4
|
+
* Simulates realistic human interaction patterns to avoid bot detection.
|
|
5
|
+
* All functions introduce natural variance and imperfection that mirrors
|
|
6
|
+
* how real users actually interact with web pages.
|
|
7
|
+
*
|
|
8
|
+
* Key techniques:
|
|
9
|
+
* - Gaussian-distributed delays (not flat uniform random)
|
|
10
|
+
* - Bézier curve mouse movement (not teleport or linear)
|
|
11
|
+
* - Realistic typing with occasional typo+correction
|
|
12
|
+
* - Variable scroll speed with pauses
|
|
13
|
+
* - Site-specific warmup sequences
|
|
14
|
+
*/
|
|
15
|
+
const DEFAULT_CONFIG = {
|
|
16
|
+
typingSpeed: [45, 120],
|
|
17
|
+
typoChance: 0.03,
|
|
18
|
+
mouseSpeed: 1,
|
|
19
|
+
minThinkTime: 500,
|
|
20
|
+
maxThinkTime: 3000,
|
|
21
|
+
};
|
|
22
|
+
function mergeConfig(config) {
|
|
23
|
+
return { ...DEFAULT_CONFIG, ...config };
|
|
24
|
+
}
|
|
25
|
+
// ── Core Utilities ────────────────────────────────────────────────────────────
|
|
26
|
+
/**
|
|
27
|
+
* Random number between min and max (inclusive, uniform distribution).
|
|
28
|
+
*/
|
|
29
|
+
function rand(min, max) {
|
|
30
|
+
return min + Math.random() * (max - min);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Gaussian-distributed random number using Box-Muller transform.
|
|
34
|
+
* More natural than uniform distribution for human timing simulation.
|
|
35
|
+
*/
|
|
36
|
+
function gaussianRand(mean, stddev) {
|
|
37
|
+
// Box-Muller transform
|
|
38
|
+
let u = 0;
|
|
39
|
+
let v = 0;
|
|
40
|
+
while (u === 0)
|
|
41
|
+
u = Math.random();
|
|
42
|
+
while (v === 0)
|
|
43
|
+
v = Math.random();
|
|
44
|
+
const normal = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
|
45
|
+
return mean + stddev * normal;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Delay with gaussian-distributed timing centered between minMs and maxMs.
|
|
49
|
+
* Clamps to [minMs, maxMs] so the result is always in-range.
|
|
50
|
+
*/
|
|
51
|
+
export async function humanDelay(minMs, maxMs) {
|
|
52
|
+
const mean = (minMs + maxMs) / 2;
|
|
53
|
+
const stddev = (maxMs - minMs) / 6; // 99.7% of values within [min, max]
|
|
54
|
+
const delay = Math.max(minMs, Math.min(maxMs, gaussianRand(mean, stddev)));
|
|
55
|
+
await new Promise(resolve => setTimeout(resolve, Math.round(delay)));
|
|
56
|
+
}
|
|
57
|
+
// ── QWERTY keyboard neighbor map ──────────────────────────────────────────────
|
|
58
|
+
/**
|
|
59
|
+
* Adjacent keys on a QWERTY layout for realistic typo simulation.
|
|
60
|
+
* Only lowercase letters and common punctuation are mapped.
|
|
61
|
+
*/
|
|
62
|
+
const KEYBOARD_NEIGHBORS = {
|
|
63
|
+
'a': ['s', 'q', 'w', 'z'],
|
|
64
|
+
'b': ['v', 'g', 'h', 'n'],
|
|
65
|
+
'c': ['x', 'd', 'f', 'v'],
|
|
66
|
+
'd': ['s', 'e', 'r', 'f', 'c', 'x'],
|
|
67
|
+
'e': ['w', 's', 'd', 'r'],
|
|
68
|
+
'f': ['d', 'r', 't', 'g', 'v', 'c'],
|
|
69
|
+
'g': ['f', 't', 'y', 'h', 'b', 'v'],
|
|
70
|
+
'h': ['g', 'y', 'u', 'j', 'n', 'b'],
|
|
71
|
+
'i': ['u', 'o', 'k', 'j'],
|
|
72
|
+
'j': ['h', 'u', 'i', 'k', 'm', 'n'],
|
|
73
|
+
'k': ['j', 'i', 'o', 'l', 'm'],
|
|
74
|
+
'l': ['k', 'o', 'p', ';'],
|
|
75
|
+
'm': ['n', 'j', 'k', ','],
|
|
76
|
+
'n': ['b', 'h', 'j', 'm'],
|
|
77
|
+
'o': ['i', 'p', 'l', 'k'],
|
|
78
|
+
'p': ['o', 'l', ';', '['],
|
|
79
|
+
'q': ['w', 'a'],
|
|
80
|
+
'r': ['e', 'd', 'f', 't'],
|
|
81
|
+
's': ['a', 'w', 'e', 'd', 'x', 'z'],
|
|
82
|
+
't': ['r', 'f', 'g', 'y'],
|
|
83
|
+
'u': ['y', 'h', 'j', 'i'],
|
|
84
|
+
'v': ['c', 'f', 'g', 'b'],
|
|
85
|
+
'w': ['q', 'a', 's', 'e'],
|
|
86
|
+
'x': ['z', 's', 'd', 'c'],
|
|
87
|
+
'y': ['t', 'g', 'h', 'u'],
|
|
88
|
+
'z': ['a', 's', 'x'],
|
|
89
|
+
'0': ['9', '-', 'o', 'p'],
|
|
90
|
+
'1': ['2', 'q'],
|
|
91
|
+
'2': ['1', '3', 'w', 'q'],
|
|
92
|
+
'3': ['2', '4', 'e', 'w'],
|
|
93
|
+
'4': ['3', '5', 'r', 'e'],
|
|
94
|
+
'5': ['4', '6', 't', 'r'],
|
|
95
|
+
'6': ['5', '7', 'y', 't'],
|
|
96
|
+
'7': ['6', '8', 'u', 'y'],
|
|
97
|
+
'8': ['7', '9', 'i', 'u'],
|
|
98
|
+
'9': ['8', '0', 'o', 'i'],
|
|
99
|
+
'.': [',', '/', 'l'],
|
|
100
|
+
',': ['m', '.', 'k', 'l'],
|
|
101
|
+
'-': ['0', '=', 'p', '['],
|
|
102
|
+
';': ['l', 'p', "'", '/'],
|
|
103
|
+
};
|
|
104
|
+
/**
|
|
105
|
+
* Returns a random adjacent key for typo simulation.
|
|
106
|
+
* Falls back to a random letter if the char has no mapping.
|
|
107
|
+
*/
|
|
108
|
+
function nearbyKey(char) {
|
|
109
|
+
const lower = char.toLowerCase();
|
|
110
|
+
const neighbors = KEYBOARD_NEIGHBORS[lower];
|
|
111
|
+
if (neighbors && neighbors.length > 0) {
|
|
112
|
+
return neighbors[Math.floor(Math.random() * neighbors.length)];
|
|
113
|
+
}
|
|
114
|
+
// Fallback: random lowercase letter
|
|
115
|
+
return String.fromCharCode(97 + Math.floor(Math.random() * 26));
|
|
116
|
+
}
|
|
117
|
+
// ── Bézier curve mouse movement ───────────────────────────────────────────────
|
|
118
|
+
/**
|
|
119
|
+
* Calculate a point on a cubic Bézier curve at parameter t (0-1).
|
|
120
|
+
*/
|
|
121
|
+
function bezierPoint(t, p0, p1, p2, p3) {
|
|
122
|
+
const u = 1 - t;
|
|
123
|
+
return u * u * u * p0 + 3 * u * u * t * p1 + 3 * u * t * t * p2 + t * t * t * p3;
|
|
124
|
+
}
|
|
125
|
+
/** Tracks the last known mouse position across calls */
|
|
126
|
+
let lastMouseX = 0;
|
|
127
|
+
let lastMouseY = 0;
|
|
128
|
+
/**
|
|
129
|
+
* Move mouse along a cubic Bézier curve from current position to (targetX, targetY).
|
|
130
|
+
* Generates 15-30 intermediate points with variable speed for natural movement.
|
|
131
|
+
*/
|
|
132
|
+
async function moveMouse(page, targetX, targetY, speedFactor = 1) {
|
|
133
|
+
const startX = lastMouseX;
|
|
134
|
+
const startY = lastMouseY;
|
|
135
|
+
// Generate two random control points that create a natural arc
|
|
136
|
+
const midX = (startX + targetX) / 2;
|
|
137
|
+
const midY = (startY + targetY) / 2;
|
|
138
|
+
const spread = Math.max(50, Math.sqrt((targetX - startX) ** 2 + (targetY - startY) ** 2) * 0.3);
|
|
139
|
+
const cp1x = midX + rand(-spread, spread);
|
|
140
|
+
const cp1y = midY + rand(-spread, spread);
|
|
141
|
+
const cp2x = midX + rand(-spread, spread);
|
|
142
|
+
const cp2y = midY + rand(-spread, spread);
|
|
143
|
+
// Number of steps scales with distance — more steps = smoother curve
|
|
144
|
+
const distance = Math.sqrt((targetX - startX) ** 2 + (targetY - startY) ** 2);
|
|
145
|
+
const steps = Math.round(rand(15, 30) * Math.min(1, distance / 300));
|
|
146
|
+
const effectiveSteps = Math.max(8, steps);
|
|
147
|
+
for (let i = 1; i <= effectiveSteps; i++) {
|
|
148
|
+
const t = i / effectiveSteps;
|
|
149
|
+
// Add slight jitter to simulate hand tremor (very small)
|
|
150
|
+
const jitter = 1.5;
|
|
151
|
+
const x = bezierPoint(t, startX, cp1x, cp2x, targetX) + rand(-jitter, jitter);
|
|
152
|
+
const y = bezierPoint(t, startY, cp1y, cp2y, targetY) + rand(-jitter, jitter);
|
|
153
|
+
await page.mouse.move(Math.round(x), Math.round(y));
|
|
154
|
+
// Variable speed: slower near start/end, faster in the middle (ease-in-out)
|
|
155
|
+
const ease = Math.sin(t * Math.PI); // 0 → 1 → 0 across the curve
|
|
156
|
+
const baseDelay = rand(8, 25) / speedFactor;
|
|
157
|
+
const stepDelay = baseDelay * (1 - ease * 0.5); // 50% faster at peak speed
|
|
158
|
+
await new Promise(resolve => setTimeout(resolve, Math.round(stepDelay)));
|
|
159
|
+
}
|
|
160
|
+
lastMouseX = targetX;
|
|
161
|
+
lastMouseY = targetY;
|
|
162
|
+
}
|
|
163
|
+
// ── Typing ────────────────────────────────────────────────────────────────────
|
|
164
|
+
/**
|
|
165
|
+
* Type text with human-like timing and occasional typos.
|
|
166
|
+
*
|
|
167
|
+
* Behavior:
|
|
168
|
+
* - Each character has a gaussian-distributed delay based on typingSpeed config
|
|
169
|
+
* - Speed varies: faster mid-word, slower at word boundaries
|
|
170
|
+
* - Occasionally pauses 200-500ms (simulating thinking/reading ahead)
|
|
171
|
+
* - Rarely makes a typo, notices it (100-300ms), backspaces, then corrects
|
|
172
|
+
*
|
|
173
|
+
* @param page Playwright page
|
|
174
|
+
* @param selector CSS selector for the input element
|
|
175
|
+
* @param text Text to type
|
|
176
|
+
* @param config Optional human behavior config overrides
|
|
177
|
+
*/
|
|
178
|
+
export async function humanType(page, selector, text, config) {
|
|
179
|
+
const cfg = mergeConfig(config);
|
|
180
|
+
const [minSpeed, maxSpeed] = cfg.typingSpeed;
|
|
181
|
+
// Click the element first to focus it
|
|
182
|
+
await page.click(selector);
|
|
183
|
+
await humanDelay(80, 200);
|
|
184
|
+
let typoInserted = false;
|
|
185
|
+
const typoTargetIdx = Math.random() < cfg.typoChance
|
|
186
|
+
? Math.floor(rand(2, text.length - 2))
|
|
187
|
+
: -1;
|
|
188
|
+
for (let i = 0; i < text.length; i++) {
|
|
189
|
+
const char = text[i];
|
|
190
|
+
// Word boundary: slower at spaces and punctuation
|
|
191
|
+
const isWordBoundary = char === ' ' || char === '.' || char === ',' || char === '!' || char === '?';
|
|
192
|
+
const speedMultiplier = isWordBoundary ? 1.4 : 1.0;
|
|
193
|
+
// Gaussian delay for this character
|
|
194
|
+
const charDelay = gaussianRand((minSpeed + maxSpeed) / 2, (maxSpeed - minSpeed) / 4) * speedMultiplier;
|
|
195
|
+
const clampedDelay = Math.max(minSpeed * 0.5, Math.min(maxSpeed * 2, charDelay));
|
|
196
|
+
await new Promise(resolve => setTimeout(resolve, Math.round(clampedDelay)));
|
|
197
|
+
// Occasional thinking pause (after spaces or at natural breakpoints)
|
|
198
|
+
if (char === ' ' && Math.random() < 0.07) {
|
|
199
|
+
await humanDelay(200, 500);
|
|
200
|
+
}
|
|
201
|
+
// Typo simulation: insert a wrong character then correct it
|
|
202
|
+
if (!typoInserted && i === typoTargetIdx && text.length > 4) {
|
|
203
|
+
typoInserted = true;
|
|
204
|
+
// Type the wrong character
|
|
205
|
+
const wrongChar = nearbyKey(char);
|
|
206
|
+
await page.keyboard.type(wrongChar);
|
|
207
|
+
// Pause — "noticing" the typo
|
|
208
|
+
await humanDelay(100, 350);
|
|
209
|
+
// Backspace to remove the typo
|
|
210
|
+
await page.keyboard.press('Backspace');
|
|
211
|
+
await humanDelay(30, 100);
|
|
212
|
+
// Type the correct character
|
|
213
|
+
await page.keyboard.type(char);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
await page.keyboard.type(char);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Clear a field and type new text — like a human triple-clicking to select all,
|
|
222
|
+
* then typing the replacement. Useful when the field already has content.
|
|
223
|
+
*
|
|
224
|
+
* @param page Playwright page
|
|
225
|
+
* @param selector CSS selector for the input element
|
|
226
|
+
* @param text Replacement text
|
|
227
|
+
* @param config Optional human behavior config overrides
|
|
228
|
+
*/
|
|
229
|
+
export async function humanClearAndType(page, selector, text, config) {
|
|
230
|
+
const cfg = mergeConfig(config);
|
|
231
|
+
// Think before clearing
|
|
232
|
+
await humanDelay(cfg.minThinkTime / 4, cfg.minThinkTime);
|
|
233
|
+
// Triple-click to select all existing content
|
|
234
|
+
await page.click(selector, { clickCount: 3 });
|
|
235
|
+
await humanDelay(50, 150);
|
|
236
|
+
// Type the new text (will replace selection)
|
|
237
|
+
await humanType(page, selector, text, config);
|
|
238
|
+
}
|
|
239
|
+
// ── Clicking ──────────────────────────────────────────────────────────────────
|
|
240
|
+
/**
|
|
241
|
+
* Click an element with human-like behavior:
|
|
242
|
+
* 1. Brief thinking pause
|
|
243
|
+
* 2. Bézier curve mouse movement to the element
|
|
244
|
+
* 3. Small random offset from center (humans rarely click the exact center)
|
|
245
|
+
* 4. Brief hover before clicking
|
|
246
|
+
* 5. Click
|
|
247
|
+
*
|
|
248
|
+
* @param page Playwright page
|
|
249
|
+
* @param selector CSS selector for the target element
|
|
250
|
+
* @param config Optional human behavior config overrides
|
|
251
|
+
*/
|
|
252
|
+
export async function humanClick(page, selector, config) {
|
|
253
|
+
const cfg = mergeConfig(config);
|
|
254
|
+
// Think before acting
|
|
255
|
+
await humanDelay(cfg.minThinkTime / 2, cfg.minThinkTime);
|
|
256
|
+
// Get element bounding box
|
|
257
|
+
const element = await page.waitForSelector(selector, { timeout: 10000 });
|
|
258
|
+
if (!element) {
|
|
259
|
+
throw new Error(`humanClick: element not found for selector "${selector}"`);
|
|
260
|
+
}
|
|
261
|
+
const box = await element.boundingBox();
|
|
262
|
+
if (!box) {
|
|
263
|
+
throw new Error(`humanClick: element has no bounding box for selector "${selector}"`);
|
|
264
|
+
}
|
|
265
|
+
// Random offset from center (within 30% of element size)
|
|
266
|
+
const offsetX = rand(-box.width * 0.25, box.width * 0.25);
|
|
267
|
+
const offsetY = rand(-box.height * 0.25, box.height * 0.25);
|
|
268
|
+
const targetX = Math.round(box.x + box.width / 2 + offsetX);
|
|
269
|
+
const targetY = Math.round(box.y + box.height / 2 + offsetY);
|
|
270
|
+
// Move mouse along a Bézier curve
|
|
271
|
+
await moveMouse(page, targetX, targetY, cfg.mouseSpeed);
|
|
272
|
+
// Brief hover — humans don't click instantly on arrival
|
|
273
|
+
await humanDelay(50, 200);
|
|
274
|
+
// Click
|
|
275
|
+
await page.mouse.click(targetX, targetY);
|
|
276
|
+
lastMouseX = targetX;
|
|
277
|
+
lastMouseY = targetY;
|
|
278
|
+
}
|
|
279
|
+
// ── Scrolling ─────────────────────────────────────────────────────────────────
|
|
280
|
+
/**
|
|
281
|
+
* Scroll the page like a human.
|
|
282
|
+
*
|
|
283
|
+
* Behavior:
|
|
284
|
+
* - Variable scroll speed (natural: faster through boring areas, slower near content)
|
|
285
|
+
* - Occasional reading pauses mid-scroll
|
|
286
|
+
* - Small back-scrolls (re-reading behavior)
|
|
287
|
+
* - Uses page.mouse.wheel() for proper scroll events (not scrollTo)
|
|
288
|
+
*
|
|
289
|
+
* @param page Playwright page
|
|
290
|
+
* @param options Scroll direction, amount, and duration
|
|
291
|
+
*/
|
|
292
|
+
export async function humanScroll(page, options = {}) {
|
|
293
|
+
const { direction = 'down', amount = Math.round(rand(300, 800)), duration = Math.round(rand(1000, 3000)), } = options;
|
|
294
|
+
const sign = direction === 'down' ? 1 : -1;
|
|
295
|
+
const totalPixels = amount * sign;
|
|
296
|
+
// Break the scroll into chunks of varying size (simulates natural hand movement)
|
|
297
|
+
const numChunks = Math.round(rand(4, 12));
|
|
298
|
+
const startTime = Date.now();
|
|
299
|
+
let scrolled = 0;
|
|
300
|
+
let targetX = Math.round(rand(300, 900));
|
|
301
|
+
let targetY = Math.round(rand(200, 600));
|
|
302
|
+
// Move mouse to a natural scroll position first
|
|
303
|
+
await moveMouse(page, targetX, targetY, 1.5);
|
|
304
|
+
for (let chunk = 0; chunk < numChunks; chunk++) {
|
|
305
|
+
const remaining = totalPixels - scrolled;
|
|
306
|
+
if (Math.abs(remaining) < 10)
|
|
307
|
+
break;
|
|
308
|
+
// Each chunk is a random fraction of remaining pixels
|
|
309
|
+
const isLastChunk = chunk === numChunks - 1;
|
|
310
|
+
const chunkFraction = isLastChunk ? 1 : rand(0.1, 0.35);
|
|
311
|
+
let chunkPixels = Math.round(remaining * chunkFraction);
|
|
312
|
+
// Small back-scroll (re-reading), ~15% chance
|
|
313
|
+
const isBackScroll = !isLastChunk && Math.random() < 0.15;
|
|
314
|
+
if (isBackScroll) {
|
|
315
|
+
chunkPixels = Math.round(rand(30, 100)) * -sign;
|
|
316
|
+
}
|
|
317
|
+
// Small horizontal mouse drift during scroll (natural)
|
|
318
|
+
const drift = rand(-20, 20);
|
|
319
|
+
targetX = Math.max(100, Math.min(1500, targetX + drift));
|
|
320
|
+
await page.mouse.move(Math.round(targetX), Math.round(targetY));
|
|
321
|
+
// Apply the scroll event
|
|
322
|
+
await page.mouse.wheel(0, chunkPixels);
|
|
323
|
+
scrolled += chunkPixels;
|
|
324
|
+
// Variable delay between chunks based on remaining duration
|
|
325
|
+
const elapsed = Date.now() - startTime;
|
|
326
|
+
const remainingDuration = duration - elapsed;
|
|
327
|
+
const avgChunkDelay = remainingDuration / (numChunks - chunk);
|
|
328
|
+
const chunkDelay = gaussianRand(avgChunkDelay, avgChunkDelay * 0.3);
|
|
329
|
+
// Pause to "read" — longer pause on some chunks
|
|
330
|
+
const isReadingPause = Math.random() < 0.25 && !isLastChunk;
|
|
331
|
+
const actualDelay = isReadingPause
|
|
332
|
+
? chunkDelay + rand(500, 1500)
|
|
333
|
+
: Math.max(50, chunkDelay);
|
|
334
|
+
await new Promise(resolve => setTimeout(resolve, Math.round(actualDelay)));
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Scroll the page until a specific element is visible, with natural scrolling.
|
|
339
|
+
*
|
|
340
|
+
* @param page Playwright page
|
|
341
|
+
* @param selector CSS selector for the target element
|
|
342
|
+
* @param config Optional human behavior config overrides
|
|
343
|
+
*/
|
|
344
|
+
export async function humanScrollToElement(page, selector, config) {
|
|
345
|
+
void mergeConfig(config); // config reserved for future speed/behavior options
|
|
346
|
+
// Check if element is already visible
|
|
347
|
+
const isVisible = await page.isVisible(selector);
|
|
348
|
+
if (isVisible)
|
|
349
|
+
return;
|
|
350
|
+
// Scroll down in increments until the element comes into view
|
|
351
|
+
let attempts = 0;
|
|
352
|
+
const maxAttempts = 10;
|
|
353
|
+
while (attempts < maxAttempts) {
|
|
354
|
+
await humanScroll(page, { direction: 'down', amount: Math.round(rand(200, 500)) });
|
|
355
|
+
const nowVisible = await page.isVisible(selector);
|
|
356
|
+
if (nowVisible)
|
|
357
|
+
break;
|
|
358
|
+
attempts++;
|
|
359
|
+
}
|
|
360
|
+
// Final scroll to center the element in view — string-based eval avoids DOM lib requirement
|
|
361
|
+
await page.evaluate((sel) => globalThis
|
|
362
|
+
.document
|
|
363
|
+
.querySelector(sel)
|
|
364
|
+
?.scrollIntoView({ behavior: 'smooth', block: 'center' }), selector);
|
|
365
|
+
await humanDelay(300, 700);
|
|
366
|
+
}
|
|
367
|
+
// ── Session Warmup ────────────────────────────────────────────────────────────
|
|
368
|
+
/**
|
|
369
|
+
* Browse a site naturally for warmup before performing target actions.
|
|
370
|
+
* Establishes a realistic browsing pattern that helps avoid bot detection.
|
|
371
|
+
*
|
|
372
|
+
* Site-specific behavior:
|
|
373
|
+
* - **LinkedIn**: Scroll feed, maybe hover posts, pause to "read"
|
|
374
|
+
* - **Upwork**: Browse job listings, scroll, hover over results
|
|
375
|
+
* - **Indeed**: Scroll job results, maybe hover over a few listings
|
|
376
|
+
* - **Generic**: Slow scroll, hover over links, maybe click one internal link
|
|
377
|
+
*
|
|
378
|
+
* @param page Playwright page
|
|
379
|
+
* @param site Site identifier for tailored warmup behavior
|
|
380
|
+
* @param durationMs Approximate warmup duration in ms (default: 30000-60000)
|
|
381
|
+
*/
|
|
382
|
+
export async function warmupSession(page, site = 'generic', durationMs) {
|
|
383
|
+
const duration = durationMs ?? Math.round(rand(30000, 60000));
|
|
384
|
+
const startTime = Date.now();
|
|
385
|
+
const elapsed = () => Date.now() - startTime;
|
|
386
|
+
const shouldContinue = () => elapsed() < duration;
|
|
387
|
+
switch (site) {
|
|
388
|
+
case 'linkedin':
|
|
389
|
+
await warmupLinkedIn(page, duration);
|
|
390
|
+
break;
|
|
391
|
+
case 'upwork':
|
|
392
|
+
await warmupUpwork(page, duration);
|
|
393
|
+
break;
|
|
394
|
+
case 'indeed':
|
|
395
|
+
await warmupIndeed(page, duration);
|
|
396
|
+
break;
|
|
397
|
+
default:
|
|
398
|
+
await warmupGeneric(page, duration, elapsed, shouldContinue);
|
|
399
|
+
break;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
/** LinkedIn-specific warmup: scroll feed, hover posts, reading pauses */
|
|
403
|
+
async function warmupLinkedIn(page, duration) {
|
|
404
|
+
const screenHeights = rand(2, 4);
|
|
405
|
+
// Slow scroll through the feed (2-4 screen heights)
|
|
406
|
+
for (let i = 0; i < screenHeights; i++) {
|
|
407
|
+
if (Date.now() >= duration)
|
|
408
|
+
break;
|
|
409
|
+
await humanScroll(page, {
|
|
410
|
+
direction: 'down',
|
|
411
|
+
amount: Math.round(rand(600, 900)),
|
|
412
|
+
duration: Math.round(rand(3000, 6000)),
|
|
413
|
+
});
|
|
414
|
+
// Pause to "read" a post
|
|
415
|
+
await humanDelay(2000, 6000);
|
|
416
|
+
// Occasionally hover over a post card (50% chance)
|
|
417
|
+
if (Math.random() < 0.5) {
|
|
418
|
+
try {
|
|
419
|
+
const posts = await page.$$('.feed-shared-update-v2, .occludable-update');
|
|
420
|
+
if (posts.length > 0) {
|
|
421
|
+
const post = posts[Math.floor(Math.random() * Math.min(posts.length, 3))];
|
|
422
|
+
const box = post ? await post.boundingBox() : null;
|
|
423
|
+
if (box) {
|
|
424
|
+
await moveMouse(page, Math.round(box.x + rand(50, box.width - 50)), Math.round(box.y + rand(20, box.height / 2)), 0.7);
|
|
425
|
+
await humanDelay(500, 2000);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
catch { /* continue warmup even if hover fails */ }
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
/** Upwork-specific warmup: browse job listings */
|
|
434
|
+
async function warmupUpwork(page, duration) {
|
|
435
|
+
const scrollRounds = Math.round(rand(2, 3));
|
|
436
|
+
for (let i = 0; i < scrollRounds && Date.now() < duration; i++) {
|
|
437
|
+
await humanScroll(page, {
|
|
438
|
+
direction: 'down',
|
|
439
|
+
amount: Math.round(rand(400, 700)),
|
|
440
|
+
duration: Math.round(rand(2000, 5000)),
|
|
441
|
+
});
|
|
442
|
+
await humanDelay(1500, 4000);
|
|
443
|
+
// Occasionally hover over a job tile
|
|
444
|
+
if (Math.random() < 0.4) {
|
|
445
|
+
try {
|
|
446
|
+
const tiles = await page.$$('[data-test="job-tile-header"], .job-tile');
|
|
447
|
+
if (tiles.length > 0) {
|
|
448
|
+
const tile = tiles[Math.floor(Math.random() * Math.min(tiles.length, 4))];
|
|
449
|
+
const box = tile ? await tile.boundingBox() : null;
|
|
450
|
+
if (box) {
|
|
451
|
+
await moveMouse(page, Math.round(box.x + rand(20, box.width - 20)), Math.round(box.y + rand(10, box.height / 2)), 0.8);
|
|
452
|
+
await humanDelay(800, 2500);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
catch { /* continue */ }
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
/** Indeed-specific warmup: scroll job results */
|
|
461
|
+
async function warmupIndeed(page, duration) {
|
|
462
|
+
const scrollRounds = Math.round(rand(2, 4));
|
|
463
|
+
for (let i = 0; i < scrollRounds && Date.now() < duration; i++) {
|
|
464
|
+
await humanScroll(page, {
|
|
465
|
+
direction: 'down',
|
|
466
|
+
amount: Math.round(rand(350, 650)),
|
|
467
|
+
duration: Math.round(rand(2000, 4500)),
|
|
468
|
+
});
|
|
469
|
+
await humanDelay(1000, 3500);
|
|
470
|
+
// Occasionally hover over a job card
|
|
471
|
+
if (Math.random() < 0.45) {
|
|
472
|
+
try {
|
|
473
|
+
const cards = await page.$$('.job_seen_beacon, .jobsearch-SerpJobCard');
|
|
474
|
+
if (cards.length > 0) {
|
|
475
|
+
const card = cards[Math.floor(Math.random() * Math.min(cards.length, 5))];
|
|
476
|
+
const box = card ? await card.boundingBox() : null;
|
|
477
|
+
if (box) {
|
|
478
|
+
await moveMouse(page, Math.round(box.x + rand(20, box.width - 20)), Math.round(box.y + rand(5, box.height / 2)), 0.9);
|
|
479
|
+
await humanDelay(600, 2000);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
catch { /* continue */ }
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
/** Generic site warmup: scroll, hover links, maybe click one internal link */
|
|
488
|
+
async function warmupGeneric(page, _duration, elapsed, shouldContinue) {
|
|
489
|
+
// Phase 1: Slow scroll down
|
|
490
|
+
const scrollRounds = Math.round(rand(2, 4));
|
|
491
|
+
for (let i = 0; i < scrollRounds && shouldContinue(); i++) {
|
|
492
|
+
await humanScroll(page, {
|
|
493
|
+
direction: 'down',
|
|
494
|
+
amount: Math.round(rand(300, 700)),
|
|
495
|
+
duration: Math.round(rand(2000, 4000)),
|
|
496
|
+
});
|
|
497
|
+
await humanDelay(800, 2500);
|
|
498
|
+
}
|
|
499
|
+
if (!shouldContinue())
|
|
500
|
+
return;
|
|
501
|
+
// Phase 2: Hover over a few links
|
|
502
|
+
const hoverCount = Math.round(rand(1, 4));
|
|
503
|
+
for (let i = 0; i < hoverCount && shouldContinue(); i++) {
|
|
504
|
+
try {
|
|
505
|
+
const links = await page.$$('a[href]');
|
|
506
|
+
const visibleLinks = links.slice(0, 10); // only top 10
|
|
507
|
+
if (visibleLinks.length > 0) {
|
|
508
|
+
const link = visibleLinks[Math.floor(Math.random() * visibleLinks.length)];
|
|
509
|
+
const box = link ? await link.boundingBox() : null;
|
|
510
|
+
if (box && box.width > 0 && box.height > 0) {
|
|
511
|
+
await moveMouse(page, Math.round(box.x + box.width / 2), Math.round(box.y + box.height / 2), 0.8);
|
|
512
|
+
await humanDelay(300, 1500);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
catch { /* continue */ }
|
|
517
|
+
}
|
|
518
|
+
if (!shouldContinue())
|
|
519
|
+
return;
|
|
520
|
+
// Phase 3: Maybe click an internal link and go back (30% chance)
|
|
521
|
+
if (Math.random() < 0.3) {
|
|
522
|
+
try {
|
|
523
|
+
const currentUrl = page.url();
|
|
524
|
+
const currentOrigin = new URL(currentUrl).origin;
|
|
525
|
+
const links = await page.$$('a[href]');
|
|
526
|
+
for (const link of links.slice(0, 15)) {
|
|
527
|
+
const href = await link.getAttribute('href');
|
|
528
|
+
if (!href)
|
|
529
|
+
continue;
|
|
530
|
+
// Only click internal links
|
|
531
|
+
try {
|
|
532
|
+
const linkUrl = new URL(href, currentUrl);
|
|
533
|
+
if (linkUrl.origin === currentOrigin && linkUrl.href !== currentUrl) {
|
|
534
|
+
await humanClick(page, 'a[href="' + href + '"]');
|
|
535
|
+
await humanDelay(2000, 5000);
|
|
536
|
+
// Go back
|
|
537
|
+
await page.goBack({ waitUntil: 'domcontentloaded' });
|
|
538
|
+
await humanDelay(500, 1500);
|
|
539
|
+
break;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
catch { /* invalid URL — skip */ }
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
catch { /* continue */ }
|
|
546
|
+
}
|
|
547
|
+
// Final reading pause
|
|
548
|
+
const remainingMs = _duration - elapsed();
|
|
549
|
+
if (remainingMs > 1000) {
|
|
550
|
+
await humanDelay(Math.min(remainingMs * 0.3, 1000), Math.min(remainingMs * 0.5, 3000));
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
// ── Form Interaction ──────────────────────────────────────────────────────────
|
|
554
|
+
/**
|
|
555
|
+
* Select an option from a <select> dropdown with human-like behavior:
|
|
556
|
+
* 1. Click the dropdown
|
|
557
|
+
* 2. Brief pause (deciding)
|
|
558
|
+
* 3. Select the target option
|
|
559
|
+
*
|
|
560
|
+
* @param page Playwright page
|
|
561
|
+
* @param selector CSS selector for the <select> element
|
|
562
|
+
* @param value Option value or label text to select
|
|
563
|
+
* @param config Optional human behavior config overrides
|
|
564
|
+
*/
|
|
565
|
+
export async function humanSelect(page, selector, value, config) {
|
|
566
|
+
const cfg = mergeConfig(config);
|
|
567
|
+
// Think before opening the dropdown
|
|
568
|
+
await humanDelay(cfg.minThinkTime / 3, cfg.minThinkTime / 2);
|
|
569
|
+
// Click to open the dropdown
|
|
570
|
+
await humanClick(page, selector, config);
|
|
571
|
+
await humanDelay(200, 600);
|
|
572
|
+
// Select the value
|
|
573
|
+
await page.selectOption(selector, value);
|
|
574
|
+
await humanDelay(100, 300);
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* Upload a file via a file input with natural delays.
|
|
578
|
+
* Simulates the realistic timing of a user who opened a file picker and
|
|
579
|
+
* navigated to the file.
|
|
580
|
+
*
|
|
581
|
+
* @param page Playwright page
|
|
582
|
+
* @param selector CSS selector for the file input element
|
|
583
|
+
* @param filePath Absolute path to the file to upload
|
|
584
|
+
*/
|
|
585
|
+
export async function humanUploadFile(page, selector, filePath) {
|
|
586
|
+
// "Thinking" about where the file is
|
|
587
|
+
await humanDelay(500, 2000);
|
|
588
|
+
await page.setInputFiles(selector, filePath);
|
|
589
|
+
// Brief pause after the file is selected (user confirming the selection)
|
|
590
|
+
await humanDelay(300, 1000);
|
|
591
|
+
}
|
|
592
|
+
/**
|
|
593
|
+
* Click a checkbox or radio button with human-like behavior.
|
|
594
|
+
* Moves the mouse naturally before clicking.
|
|
595
|
+
*
|
|
596
|
+
* @param page Playwright page
|
|
597
|
+
* @param selector CSS selector for the checkbox or radio
|
|
598
|
+
* @param config Optional human behavior config overrides
|
|
599
|
+
*/
|
|
600
|
+
export async function humanToggle(page, selector, config) {
|
|
601
|
+
await humanClick(page, selector, config);
|
|
602
|
+
// Brief pause after toggling — humans verify the state visually
|
|
603
|
+
const cfg = mergeConfig(config);
|
|
604
|
+
await humanDelay(cfg.minThinkTime / 4, cfg.minThinkTime / 2);
|
|
605
|
+
}
|
|
606
|
+
// ── Public wrappers for programmatic use ──────────────────────────────────────
|
|
607
|
+
/**
|
|
608
|
+
* Move mouse along a natural Bézier curve to a position.
|
|
609
|
+
* Public wrapper around the internal moveMouse function.
|
|
610
|
+
*
|
|
611
|
+
* @param page Playwright page
|
|
612
|
+
* @param targetX Target X coordinate
|
|
613
|
+
* @param targetY Target Y coordinate
|
|
614
|
+
* @param options Speed options
|
|
615
|
+
*/
|
|
616
|
+
export async function humanMouseMove(page, targetX, targetY, options) {
|
|
617
|
+
const speedFactor = options?.duration
|
|
618
|
+
? 600 / ((options.duration[0] + options.duration[1]) / 2)
|
|
619
|
+
: 1;
|
|
620
|
+
await moveMouse(page, targetX, targetY, speedFactor);
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* Read a page like a human — scroll through content with natural pauses.
|
|
624
|
+
* Simulates someone actually reading the page before taking action.
|
|
625
|
+
*
|
|
626
|
+
* @param page Playwright page
|
|
627
|
+
* @param durationMs Approximate time to spend "reading" (default: 5000-15000 random)
|
|
628
|
+
*/
|
|
629
|
+
export async function humanRead(page, durationMs) {
|
|
630
|
+
const readTime = durationMs ?? Math.round(rand(5000, 15000));
|
|
631
|
+
const startTime = Date.now();
|
|
632
|
+
const elapsed = () => Date.now() - startTime;
|
|
633
|
+
while (elapsed() < readTime) {
|
|
634
|
+
const remaining = readTime - elapsed();
|
|
635
|
+
if (remaining < 500)
|
|
636
|
+
break;
|
|
637
|
+
// Scroll a chunk (reading speed: one screen section at a time)
|
|
638
|
+
const chunkAmount = Math.round(rand(180, 350));
|
|
639
|
+
const chunkDuration = Math.round(rand(800, 2000));
|
|
640
|
+
await humanScroll(page, {
|
|
641
|
+
direction: 'down',
|
|
642
|
+
amount: chunkAmount,
|
|
643
|
+
duration: Math.min(chunkDuration, remaining - 200),
|
|
644
|
+
});
|
|
645
|
+
if (elapsed() >= readTime)
|
|
646
|
+
break;
|
|
647
|
+
// Pause to "read" the revealed content
|
|
648
|
+
const pauseTime = Math.min(Math.round(rand(1000, 3500)), readTime - elapsed());
|
|
649
|
+
if (pauseTime > 100) {
|
|
650
|
+
await humanDelay(pauseTime * 0.7, pauseTime);
|
|
651
|
+
}
|
|
652
|
+
// Occasionally scroll back up slightly (re-reading)
|
|
653
|
+
if (Math.random() < 0.2 && elapsed() < readTime - 2000) {
|
|
654
|
+
await humanScroll(page, {
|
|
655
|
+
direction: 'up',
|
|
656
|
+
amount: Math.round(rand(60, 120)),
|
|
657
|
+
duration: Math.round(rand(400, 800)),
|
|
658
|
+
});
|
|
659
|
+
await humanDelay(300, 800);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Browse a site naturally for warmup before performing the target action.
|
|
665
|
+
* Thin wrapper around `warmupSession` with a simplified site-detection interface.
|
|
666
|
+
*
|
|
667
|
+
* @param page Playwright page
|
|
668
|
+
* @param durationMs How long to browse (default: 30000-60000 random)
|
|
669
|
+
*/
|
|
670
|
+
export async function warmupBrowse(page, durationMs) {
|
|
671
|
+
const url = page.url();
|
|
672
|
+
let site = 'generic';
|
|
673
|
+
if (url.includes('linkedin.com'))
|
|
674
|
+
site = 'linkedin';
|
|
675
|
+
else if (url.includes('indeed.com'))
|
|
676
|
+
site = 'indeed';
|
|
677
|
+
else if (url.includes('upwork.com'))
|
|
678
|
+
site = 'upwork';
|
|
679
|
+
await warmupSession(page, site, durationMs);
|
|
680
|
+
}
|
|
681
|
+
//# sourceMappingURL=human.js.map
|