pi-chrome-search 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/package.json +4 -3
- package/src/config/defaults.ts +20 -1
- package/src/config/doc.ts +3 -0
- package/src/config/schema.ts +2 -0
- package/src/engines/avito.ts +552 -0
- package/src/engines/google.ts +87 -2
- package/src/engines/index.ts +12 -3
- package/src/engines/types.ts +65 -6
- package/src/engines/yandex.ts +29 -3
- package/src/extract/compact.ts +61 -5
- package/src/extract/facts.ts +39 -0
- package/src/runtime/doctor.ts +26 -3
- package/src/runtime/limiter.ts +5 -5
- package/src/runtime/panel.ts +2 -2
- package/src/tools/web_search.ts +39 -7
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Avito adapter (SPEC S6.4, §15 D11/D12) — a *special* engine: marketplace
|
|
3
|
+
* listings for one site, reachable only by naming it (`engines: "avito"`).
|
|
4
|
+
*
|
|
5
|
+
* What the live SERP actually gives us (captured 2026-11-15, `all` region,
|
|
6
|
+
* fixture `tests/fixtures/avito-serp.html.gz`):
|
|
7
|
+
*
|
|
8
|
+
* - Every listing is `[data-marker="item"]` and carries `data-item-id`. Titles
|
|
9
|
+
* are `a[data-marker="item-title"]` with a relative `href`; the card is
|
|
10
|
+
* server-rendered, so a settled SERP has the items in the first HTML byte —
|
|
11
|
+
* no AI block, no client-side hydration to wait for.
|
|
12
|
+
* - Price is visible text (`[data-marker="item-price-value"]`, "23 990 ₽") and
|
|
13
|
+
* also machine-readable (`meta[itemprop="price"]` + `priceCurrency`). The
|
|
14
|
+
* visible string wins: it is what the user sees, formatted by Avito itself.
|
|
15
|
+
* - Location (`item-location`) and listing age (`item-date`, on some cards)
|
|
16
|
+
* are card facts, not prose. They go into `SerpItem.facts` (D13) instead of
|
|
17
|
+
* being glued into the snippet, because a price is not a sentence.
|
|
18
|
+
* - `Продвинуто` is a paid boost of the *same* listing, not an ad slot: it is
|
|
19
|
+
* neither `isAd` (that would drop the item, which the page does not do) nor a
|
|
20
|
+
* fact (it would sit on most cards).
|
|
21
|
+
* - A query with no exact match is answered with *similar* listings under the
|
|
22
|
+
* visible heading "Похожие объявления рядом и в других городах" and a quoted
|
|
23
|
+
* `h1` («запрос»: объявления). That state is reported as `RawSerp.notice`
|
|
24
|
+
* (S7.6): the items are real, but they do not answer the query.
|
|
25
|
+
*
|
|
26
|
+
* Extraction is a self-contained in-page function (same `toString()` rule as the
|
|
27
|
+
* other adapters): live code sends `(${extractFromDocument})(document, arg)`
|
|
28
|
+
* through `Runtime.evaluate`, tests run the *same* source through `runInPage` in
|
|
29
|
+
* a `node:vm` context.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import type { CdpPage } from "../cdp/types.ts";
|
|
33
|
+
import { waitForMarker } from "../cdp/humanize.ts";
|
|
34
|
+
import { asFacts } from "../extract/facts.ts";
|
|
35
|
+
import { normalizeUrl } from "../extract/urls.ts";
|
|
36
|
+
import { fail } from "../runtime/errors.ts";
|
|
37
|
+
import type { Clock } from "../runtime/types.ts";
|
|
38
|
+
import type {
|
|
39
|
+
Engine,
|
|
40
|
+
EngineInteraction,
|
|
41
|
+
Query,
|
|
42
|
+
RawSerp,
|
|
43
|
+
SerpItem,
|
|
44
|
+
Wall,
|
|
45
|
+
} from "./types.ts";
|
|
46
|
+
|
|
47
|
+
/** A server-rendered page that still hydrates ~1 MB of JS; 12 s is generous. */
|
|
48
|
+
export const AVITO_SETTLE_DEADLINE_MS = 12_000;
|
|
49
|
+
const SETTLE_POLL_MS = 200;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Readiness gate. `item` cards are the results; `page-title` is the result
|
|
53
|
+
* header, present when the list is empty or when Avito answers with "similar
|
|
54
|
+
* listings"; the home page renders neither, so a typed warm-up cannot settle on
|
|
55
|
+
* it (verified on the capture of `https://www.avito.ru/`).
|
|
56
|
+
*
|
|
57
|
+
* The captcha widget is part of the gate for the same reason Google's `#rso`
|
|
58
|
+
* lives next to `#captcha-form` and Yandex's `.OrganicTitle` next to
|
|
59
|
+
* `.SmartCaptcha`: a wall page has no results node (measured on the firewall
|
|
60
|
+
* markup in `avito-wall`), so without it the settle would burn the whole 12 s
|
|
61
|
+
* deadline *before* `detectWall()` could report the wall it is looking at. Settling on a wall is safe because the wall probe runs
|
|
62
|
+
* immediately after the gate and before extraction (`web_search`,
|
|
63
|
+
* `web_doctor`), and it is what makes the report immediate (D5).
|
|
64
|
+
*/
|
|
65
|
+
export const AVITO_WALL_SELECTOR =
|
|
66
|
+
'#h-captcha, .h-captcha, #geetest_captcha, .form-captcha-image, #captcha, form[action*="checkcaptcha"]';
|
|
67
|
+
export const AVITO_READY_SELECTOR = `[data-marker="item"], [data-marker="page-title"], ${AVITO_WALL_SELECTOR}`;
|
|
68
|
+
export const AVITO_ITEM_SELECTOR = '[data-marker="item"]';
|
|
69
|
+
|
|
70
|
+
const BOX_SELECTOR = 'input[data-marker="search-form/suggest/input"]';
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Typed-search profile. The box is `maxlength="100"` (captured), so a longer
|
|
74
|
+
* query is never typed — a truncated query would search for something the user
|
|
75
|
+
* did not ask for (the caller falls back to the pinned SERP URL instead).
|
|
76
|
+
*/
|
|
77
|
+
const TYPE_PROFILE = Object.freeze({
|
|
78
|
+
box: BOX_SELECTOR,
|
|
79
|
+
maxChars: 100,
|
|
80
|
+
keyDelayMs: Object.freeze([30, 70] as const),
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Region slugs are path segments (`/kaluga`, `/moskva`, `/all`). The SERP URL is
|
|
85
|
+
* built from a slug, so anything that is not one falls back to `all` instead of
|
|
86
|
+
* becoming an encoded path that 404s (S7.5: no key that cannot work).
|
|
87
|
+
*/
|
|
88
|
+
export const AVITO_DEFAULT_LOCATION = "all";
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The home page is parameterized by region (`https://www.avito.ru/<location>/`),
|
|
92
|
+
* and `all` redirects to the root (`https://www.avito.ru/`), which is the
|
|
93
|
+
* "во всех регионах" home. Both hosts are legitimate landings for the warm-up.
|
|
94
|
+
*/
|
|
95
|
+
function homeUrlFor(q: Query): string {
|
|
96
|
+
return `https://www.avito.ru/${locationSlug(q.region)}/`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const INTERACTION: EngineInteraction = Object.freeze({
|
|
100
|
+
homeUrl: "https://www.avito.ru/all/",
|
|
101
|
+
homeHosts: Object.freeze(["avito.ru"]),
|
|
102
|
+
boxSelectors: Object.freeze([BOX_SELECTOR, 'input[data-marker*="search-form"]']),
|
|
103
|
+
marker: "",
|
|
104
|
+
selector: AVITO_READY_SELECTOR,
|
|
105
|
+
minCount: 1,
|
|
106
|
+
typeProfile: TYPE_PROFILE,
|
|
107
|
+
homeUrlFor,
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Wall markers, text-first so class-name churn does not blind us, with the same
|
|
112
|
+
* corroboration rule as the other adapters: a phrase alone is not a wall (a
|
|
113
|
+
* listing title may quote it, and a no-match SERP quotes the query), but a
|
|
114
|
+
* phrase plus zero `item` cards is.
|
|
115
|
+
*
|
|
116
|
+
* What the fixture `tests/fixtures/avito-wall.html.gz` is: Avito's own firewall
|
|
117
|
+
* markup (the IP wall with an hCaptcha checkbox). It was obtained **out of band**
|
|
118
|
+
* and pasted into a tab, so the markup is evidence and the page *URL* is not —
|
|
119
|
+
* the tab happened to hold a search URL, which says nothing about where Avito
|
|
120
|
+
* serves this page. Consequences, both honest:
|
|
121
|
+
* - the `/captcha` and `/blocked` URL markers stay, but only as a cheap extra: a
|
|
122
|
+
* wall on such a path is caught, and nothing here claims Avito redirects;
|
|
123
|
+
* - the sentence "Доступ ограничен" lives in `<title>`, never in `innerText`,
|
|
124
|
+
* which is why the title is scanned separately from the body;
|
|
125
|
+
* - the body says "Страница не загрузилась из-за проблем с IP" / "поставьте
|
|
126
|
+
* галочку и нажмите «Продолжить»", and the widget markup
|
|
127
|
+
* (`#h-captcha`, `.h-captcha`, `.form-captcha-image`) gives `captchaNodes = 1`.
|
|
128
|
+
* That markup carries neither `item` nor `page-title`, so it can never settle.
|
|
129
|
+
*/
|
|
130
|
+
const CAPTCHA_URL_MARKERS: readonly string[] = Object.freeze(["/captcha", "/blocked"]);
|
|
131
|
+
/** Only on the engine's own page title; a SERP title is `<query> — объявления`. */
|
|
132
|
+
const CAPTCHA_TITLE_MARKERS: readonly string[] = Object.freeze(["Доступ ограничен", "проблема с IP"]);
|
|
133
|
+
const CAPTCHA_TEXT_MARKERS: readonly string[] = Object.freeze([
|
|
134
|
+
"Страница не загрузилась из-за проблем с",
|
|
135
|
+
"поставьте галочку",
|
|
136
|
+
"Подтвердите, что вы не робот",
|
|
137
|
+
"проверка на робота",
|
|
138
|
+
"Ограниченный доступ",
|
|
139
|
+
"Request blocked",
|
|
140
|
+
"Access denied",
|
|
141
|
+
]);
|
|
142
|
+
const ITEM_COUNT_EXPRESSION = `document.querySelectorAll(${JSON.stringify(AVITO_ITEM_SELECTOR)}).length`;
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
// In-page extraction (self-contained: no imports, no closures)
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
export interface AvitoExtractedItem {
|
|
148
|
+
title: string;
|
|
149
|
+
url: string;
|
|
150
|
+
snippet: string;
|
|
151
|
+
facts: string[];
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export interface AvitoExtraction {
|
|
155
|
+
results: AvitoExtractedItem[];
|
|
156
|
+
/** True when the page carried more cards than were returned. */
|
|
157
|
+
pageCarried: boolean;
|
|
158
|
+
/** True when Avito answered with "similar listings" instead of matches. */
|
|
159
|
+
similar: boolean;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export interface AvitoExtractionArg {
|
|
163
|
+
n: number;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* One-off in-page extractor. Everything it needs lives inside the body, because
|
|
168
|
+
* only its `toString()` crosses the CDP boundary.
|
|
169
|
+
*/
|
|
170
|
+
export function extractFromDocument(doc: Document, arg: AvitoExtractionArg): AvitoExtraction {
|
|
171
|
+
const WHITESPACE = /[\s\u00a0\u202f]+/g;
|
|
172
|
+
/**
|
|
173
|
+
* Tracking parameters Avito appends to item links (`?context=<blob>&cd=1`,
|
|
174
|
+
* `?slocation=…`). They are ranking/session state, never part of the listing,
|
|
175
|
+
* so the link the user clicks is the clean one (S7.4).
|
|
176
|
+
*/
|
|
177
|
+
const TRACKING = ["context", "cd", "slocation", "searchfrom"];
|
|
178
|
+
const normalize = (value: string | null | undefined): string =>
|
|
179
|
+
(value === null || value === undefined ? "" : value).replace(WHITESPACE, " ").trim();
|
|
180
|
+
|
|
181
|
+
const base = doc.baseURI !== "" ? doc.baseURI : doc.URL;
|
|
182
|
+
|
|
183
|
+
/** http(s) only, Avito tracking parameters removed, fragment dropped. */
|
|
184
|
+
const toHttpUrl = (raw: string): string | null => {
|
|
185
|
+
const trimmed = raw.trim();
|
|
186
|
+
if (trimmed === "") return null;
|
|
187
|
+
let parsed: URL;
|
|
188
|
+
try {
|
|
189
|
+
parsed = new URL(trimmed, base === "" ? undefined : base);
|
|
190
|
+
} catch {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
194
|
+
const kept: Array<[string, string]> = [];
|
|
195
|
+
parsed.searchParams.forEach((value: string, key: string) => {
|
|
196
|
+
if (TRACKING.indexOf(key.toLowerCase()) !== -1) return;
|
|
197
|
+
kept.push([key, value]);
|
|
198
|
+
});
|
|
199
|
+
parsed.search = "";
|
|
200
|
+
for (const [key, value] of kept) parsed.searchParams.append(key, value);
|
|
201
|
+
parsed.hash = "";
|
|
202
|
+
return parsed.href;
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
/** Card text of one marker, or "" when the card does not show it. */
|
|
206
|
+
const cardText = (card: Element, selector: string): string => {
|
|
207
|
+
const node = card.querySelector(selector);
|
|
208
|
+
return node === null ? "" : normalize(node.textContent);
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Leading text of an element, stopping at the first child element. Avito puts
|
|
213
|
+
* the date as a bare text node followed by the seller link ("Вчера <a>…</a>"),
|
|
214
|
+
* so `textContent` would report the seller name as the date.
|
|
215
|
+
*/
|
|
216
|
+
const leadingText = (node: Element): string => {
|
|
217
|
+
let out = "";
|
|
218
|
+
for (const child of Array.from(node.childNodes)) {
|
|
219
|
+
if (child.nodeType === 1) break;
|
|
220
|
+
if (child.nodeType === 3) out += child.nodeValue === null ? "" : child.nodeValue;
|
|
221
|
+
}
|
|
222
|
+
return normalize(out);
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Visible price first (Avito formats "23 990 ₽" itself), machine-readable meta
|
|
227
|
+
* as the fallback. A card with neither contributes no price fact.
|
|
228
|
+
*/
|
|
229
|
+
const priceText = (card: Element): string => {
|
|
230
|
+
const visible = cardText(card, '[data-marker="item-price-value"]');
|
|
231
|
+
if (visible !== "") return visible;
|
|
232
|
+
const holder = card.querySelector('[data-marker="item-price"]');
|
|
233
|
+
if (holder === null) return "";
|
|
234
|
+
const amount = normalize(holder.querySelector('meta[itemprop="price"]')?.getAttribute("content"));
|
|
235
|
+
const currency = normalize(holder.querySelector('meta[itemprop="priceCurrency"]')?.getAttribute("content"));
|
|
236
|
+
if (amount === "") return "";
|
|
237
|
+
return currency === "" ? amount : `${amount} ${currency}`;
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
const limit =
|
|
241
|
+
typeof arg.n === "number" && Number.isFinite(arg.n) ? Math.max(0, Math.floor(arg.n)) : 0;
|
|
242
|
+
|
|
243
|
+
const bodyText = normalize(doc.body === null ? "" : doc.body.textContent);
|
|
244
|
+
if (bodyText === "") return { results: [], pageCarried: false, similar: false };
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* "No exact match" state: the result header quotes the query («…»: объявления)
|
|
248
|
+
* *and* the page says "Похожие объявления рядом и в других городах". Both are
|
|
249
|
+
* required — on a matching SERP the header is a catalog name ("Apple iPhone
|
|
250
|
+
* 13") and the phrase is absent (verified on both captures), so a phrase query
|
|
251
|
+
* that does find real listings is not mislabelled.
|
|
252
|
+
*/
|
|
253
|
+
const titleNode = doc.querySelector('[data-marker="page-title/text"]');
|
|
254
|
+
const titleText = titleNode === null ? "" : normalize(titleNode.textContent);
|
|
255
|
+
const quotedTitle = titleText.charAt(0) === "«" && titleText.indexOf("»") !== -1;
|
|
256
|
+
const similar = quotedTitle && bodyText.toLowerCase().indexOf("похожие объявления рядом") !== -1;
|
|
257
|
+
|
|
258
|
+
const results: AvitoExtractedItem[] = [];
|
|
259
|
+
const seenUrls: { [url: string]: boolean } = {};
|
|
260
|
+
let cards = 0;
|
|
261
|
+
|
|
262
|
+
for (const card of Array.from(doc.querySelectorAll('[data-marker="item"]'))) {
|
|
263
|
+
const anchor = card.querySelector('a[data-marker="item-title"][href]');
|
|
264
|
+
const href = anchor === null ? null : anchor.getAttribute("href");
|
|
265
|
+
if (href === null || href === "") continue;
|
|
266
|
+
const url = toHttpUrl(href);
|
|
267
|
+
if (url === null) continue;
|
|
268
|
+
const title = normalize(anchor === null ? "" : anchor.textContent);
|
|
269
|
+
if (title === "") continue;
|
|
270
|
+
cards += 1;
|
|
271
|
+
if (seenUrls[url] === true) continue;
|
|
272
|
+
seenUrls[url] = true;
|
|
273
|
+
if (results.length >= limit) continue;
|
|
274
|
+
|
|
275
|
+
const facts: string[] = [];
|
|
276
|
+
const price = priceText(card);
|
|
277
|
+
if (price !== "") facts.push(price);
|
|
278
|
+
// "Саратов· выезжает по городу" — the travel note is a separate badge glued
|
|
279
|
+
// on with a middot; keep the place, not the service blurb.
|
|
280
|
+
const location = cardText(card, '[data-marker="item-location"]').split("·")[0] ?? "";
|
|
281
|
+
if (location !== "") facts.push(location.trim());
|
|
282
|
+
const dateNode = card.querySelector('[data-marker="item-date"]');
|
|
283
|
+
const date = dateNode === null ? "" : leadingText(dateNode);
|
|
284
|
+
if (date !== "") facts.push(date);
|
|
285
|
+
|
|
286
|
+
results.push({
|
|
287
|
+
title,
|
|
288
|
+
url,
|
|
289
|
+
// `item-specific-params` ("Новый, 6.1″", "128 ГБ") is the closest thing a
|
|
290
|
+
// listing card has to a snippet; Avito ships no description in the list.
|
|
291
|
+
snippet: cardText(card, '[data-marker="item-specific-params"]'),
|
|
292
|
+
facts,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
return { results, pageCarried: cards > results.length, similar };
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ---------------------------------------------------------------------------
|
|
300
|
+
// Selector canaries (self-contained, same toString() rule)
|
|
301
|
+
// ---------------------------------------------------------------------------
|
|
302
|
+
|
|
303
|
+
export function canaryCounts(doc: Document): Record<string, number> {
|
|
304
|
+
const SELECTORS = [
|
|
305
|
+
'[data-marker="item"]',
|
|
306
|
+
'[data-marker="item-title"]',
|
|
307
|
+
'[data-marker="item-price-value"]',
|
|
308
|
+
'[data-marker="item-specific-params"]',
|
|
309
|
+
'[data-marker="item-location"]',
|
|
310
|
+
'[data-marker="item-date"]',
|
|
311
|
+
'[data-marker="page-title"]',
|
|
312
|
+
'[data-marker="pagination-button/nextPage"]',
|
|
313
|
+
];
|
|
314
|
+
const counts: Record<string, number> = {};
|
|
315
|
+
for (const selector of SELECTORS) {
|
|
316
|
+
counts[selector] = doc.querySelectorAll(selector).length;
|
|
317
|
+
}
|
|
318
|
+
return counts;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* 1 when the document carries a captcha widget, 0 otherwise. A JS scan instead
|
|
323
|
+
* of a CSS selector because the class test must be case-insensitive and
|
|
324
|
+
* `[attr*="captcha" i]` is not portable across the DOM implementations the
|
|
325
|
+
* offline harness uses.
|
|
326
|
+
*/
|
|
327
|
+
export function captchaNodeCount(doc: Document): number {
|
|
328
|
+
// Self-contained (the function is stringified into the page), so the ids are
|
|
329
|
+
// repeated here instead of importing `AVITO_WALL_SELECTOR`; the class half of
|
|
330
|
+
// that selector is covered by the scan below.
|
|
331
|
+
if (doc.querySelector('#h-captcha, #geetest_captcha, #captcha, iframe[src*="captcha"], form[action*="checkcaptcha"]') !== null) {
|
|
332
|
+
return 1;
|
|
333
|
+
}
|
|
334
|
+
for (const node of Array.from(doc.querySelectorAll("[class]"))) {
|
|
335
|
+
const className = node.getAttribute("class") ?? "";
|
|
336
|
+
if (className.toLowerCase().indexOf("captcha") !== -1) return 1;
|
|
337
|
+
}
|
|
338
|
+
return 0;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ---------------------------------------------------------------------------
|
|
342
|
+
// Result validation (TS side: the page is untrusted even when we wrote the code)
|
|
343
|
+
// ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
346
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function parseFailure(message: string): Error {
|
|
350
|
+
return fail("E_PARSE", "extract", {
|
|
351
|
+
engine: "avito",
|
|
352
|
+
message,
|
|
353
|
+
hint: "the Avito page returned an unexpected shape; run /web-doctor and inspect the artifact",
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function asExtraction(value: unknown): AvitoExtraction {
|
|
358
|
+
if (!isRecord(value)) throw parseFailure("in-page extractor returned no object");
|
|
359
|
+
const rawResults = value.results;
|
|
360
|
+
if (!Array.isArray(rawResults)) throw parseFailure("in-page extractor returned no results array");
|
|
361
|
+
|
|
362
|
+
const results: AvitoExtractedItem[] = [];
|
|
363
|
+
const seenUrls = new Set<string>();
|
|
364
|
+
for (const rawItem of rawResults) {
|
|
365
|
+
if (!isRecord(rawItem)) throw parseFailure("in-page extractor returned a malformed result");
|
|
366
|
+
const { title, url, snippet } = rawItem;
|
|
367
|
+
if (typeof title !== "string" || typeof url !== "string" || typeof snippet !== "string") {
|
|
368
|
+
throw parseFailure("in-page extractor returned a result with non-string fields");
|
|
369
|
+
}
|
|
370
|
+
if (title === "") continue;
|
|
371
|
+
// Second, TS-side hygiene pass: the in-page copy cannot import `urls.ts`.
|
|
372
|
+
const normalized = normalizeUrl(url);
|
|
373
|
+
if (normalized === null || seenUrls.has(normalized)) continue;
|
|
374
|
+
seenUrls.add(normalized);
|
|
375
|
+
results.push({
|
|
376
|
+
title,
|
|
377
|
+
url: normalized,
|
|
378
|
+
snippet,
|
|
379
|
+
facts: asFacts(rawItem.facts),
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
return {
|
|
384
|
+
results,
|
|
385
|
+
pageCarried: value.pageCarried === true,
|
|
386
|
+
similar: value.similar === true,
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function asCanaries(value: unknown): Record<string, number> {
|
|
391
|
+
const counts: Record<string, number> = {};
|
|
392
|
+
if (!isRecord(value)) return counts;
|
|
393
|
+
for (const [selector, count] of Object.entries(value)) {
|
|
394
|
+
if (typeof count === "number" && Number.isFinite(count)) counts[selector] = count;
|
|
395
|
+
}
|
|
396
|
+
return counts;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// ---------------------------------------------------------------------------
|
|
400
|
+
// Settle / wall
|
|
401
|
+
// ---------------------------------------------------------------------------
|
|
402
|
+
|
|
403
|
+
export interface AvitoSettleOptions {
|
|
404
|
+
signal?: AbortSignal;
|
|
405
|
+
/** Test/live knob; defaults to 12 s. */
|
|
406
|
+
deadlineMs?: number;
|
|
407
|
+
/** Test/live knob; defaults to 200 ms plus humanize jitter. */
|
|
408
|
+
pollMs?: number;
|
|
409
|
+
clock?: Clock;
|
|
410
|
+
rng?: () => number;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Bounded settle on the structural gate; abort/timeout use the shared taxonomy. */
|
|
414
|
+
export async function settleAvito(page: CdpPage, options: AvitoSettleOptions = {}): Promise<void> {
|
|
415
|
+
await waitForMarker(page, {
|
|
416
|
+
marker: INTERACTION.marker,
|
|
417
|
+
selector: INTERACTION.selector,
|
|
418
|
+
minCount: INTERACTION.minCount ?? 1,
|
|
419
|
+
deadlineMs: options.deadlineMs ?? AVITO_SETTLE_DEADLINE_MS,
|
|
420
|
+
pollMs: options.pollMs ?? SETTLE_POLL_MS,
|
|
421
|
+
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
422
|
+
...(options.rng === undefined ? {} : { rng: options.rng }),
|
|
423
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
async function itemCount(page: CdpPage): Promise<number> {
|
|
428
|
+
const count = await page.evaluate<unknown>(ITEM_COUNT_EXPRESSION);
|
|
429
|
+
return typeof count === "number" && Number.isFinite(count) ? count : 0;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
async function captchaNodes(page: CdpPage): Promise<number> {
|
|
433
|
+
const count = await page.evaluate<unknown>(`(${captchaNodeCount.toString()})(document)`);
|
|
434
|
+
return typeof count === "number" && Number.isFinite(count) ? count : 0;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** Markers present in a lowercased haystack, in list order. */
|
|
438
|
+
function markersIn(haystack: string, markers: readonly string[]): string[] {
|
|
439
|
+
const found: string[] = [];
|
|
440
|
+
for (const marker of markers) {
|
|
441
|
+
if (haystack.indexOf(marker.toLowerCase()) !== -1) found.push(marker);
|
|
442
|
+
}
|
|
443
|
+
return found;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* Wall detection. Nobody clicks anything here: the tool surfaces the window, the
|
|
448
|
+
* human solves it and retries (D5).
|
|
449
|
+
*/
|
|
450
|
+
export async function detectAvitoWall(page: CdpPage): Promise<Wall | null> {
|
|
451
|
+
const url = page.url();
|
|
452
|
+
const urlMarkers = markersIn(url.toLowerCase(), CAPTCHA_URL_MARKERS);
|
|
453
|
+
if (urlMarkers.length > 0) return { kind: "captcha", url, markers: urlMarkers };
|
|
454
|
+
|
|
455
|
+
// Title and body are scanned separately: Avito's firewall names itself
|
|
456
|
+
// "Доступ ограничен: проблема с IP" in `<title>` only (captured). A failing
|
|
457
|
+
// `title()` is a signal we lose, not a wall we invent.
|
|
458
|
+
const title = await page.title().catch((): string => "");
|
|
459
|
+
const text = await page.innerText();
|
|
460
|
+
const markers = [
|
|
461
|
+
...markersIn(title.toLowerCase(), CAPTCHA_TITLE_MARKERS),
|
|
462
|
+
...markersIn(text.toLowerCase(), CAPTCHA_TEXT_MARKERS),
|
|
463
|
+
];
|
|
464
|
+
|
|
465
|
+
// A captcha widget is evidence on its own: its label lives in an iframe the
|
|
466
|
+
// page-text scan cannot see.
|
|
467
|
+
if ((await captchaNodes(page)) > 0) {
|
|
468
|
+
return { kind: "captcha", url, markers: markers.length > 0 ? markers : ["captcha-node"] };
|
|
469
|
+
}
|
|
470
|
+
// A phrase can be quoted by a listing or by the query itself; a wall has no cards.
|
|
471
|
+
if (markers.length > 0 && (await itemCount(page)) === 0) {
|
|
472
|
+
return { kind: "blocked", url, markers };
|
|
473
|
+
}
|
|
474
|
+
return null;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// ---------------------------------------------------------------------------
|
|
478
|
+
// URL builder + engine
|
|
479
|
+
// ---------------------------------------------------------------------------
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Region path segment. `all` is Avito's all-Russia slug (`rossiya` redirects to
|
|
483
|
+
* it), and the config value is that slug, not a display name: a slug is what the
|
|
484
|
+
* URL needs, and a wrong one is a 404 the caller cannot debug.
|
|
485
|
+
*/
|
|
486
|
+
export function locationSlug(region: string | undefined): string {
|
|
487
|
+
const trimmed = typeof region === "string" ? region.trim() : "";
|
|
488
|
+
if (trimmed === "") return AVITO_DEFAULT_LOCATION;
|
|
489
|
+
return /^[a-z0-9_-]+$/i.test(trimmed) ? trimmed.toLowerCase() : AVITO_DEFAULT_LOCATION;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Pure: `https://www.avito.ru/<location>?q=<query>[&p=<page>]`.
|
|
494
|
+
*
|
|
495
|
+
* Verified live (2026-11-15): Avito resolves `?q=` into its catalog, `p` is the
|
|
496
|
+
* 1-based page number (the page-2 link carries `p=2`), and the search box on the
|
|
497
|
+
* resulting page holds the query. There is no `lr`-style region parameter: the
|
|
498
|
+
* region is the first path segment.
|
|
499
|
+
*/
|
|
500
|
+
function buildUrl(q: Query): string {
|
|
501
|
+
const parts = [`q=${encodeURIComponent(q.text)}`];
|
|
502
|
+
const page = typeof q.page === "number" && Number.isFinite(q.page) ? Math.floor(q.page) : 1;
|
|
503
|
+
if (page > 1) parts.push(`p=${page}`);
|
|
504
|
+
return `https://www.avito.ru/${locationSlug(q.region)}?${parts.join("&")}`;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
export const avitoEngine: Engine = {
|
|
508
|
+
id: "avito",
|
|
509
|
+
interaction: INTERACTION,
|
|
510
|
+
buildUrl,
|
|
511
|
+
|
|
512
|
+
async settle(page: CdpPage, signal?: AbortSignal): Promise<void> {
|
|
513
|
+
await settleAvito(page, signal === undefined ? {} : { signal });
|
|
514
|
+
},
|
|
515
|
+
|
|
516
|
+
detectWall: detectAvitoWall,
|
|
517
|
+
|
|
518
|
+
async extract(page: CdpPage, q: Query): Promise<RawSerp> {
|
|
519
|
+
const startedAt = Date.now();
|
|
520
|
+
const arg: AvitoExtractionArg = { n: q.n };
|
|
521
|
+
const raw = await page.evaluate<unknown>(
|
|
522
|
+
`(${extractFromDocument.toString()})(document, ${JSON.stringify(arg)})`,
|
|
523
|
+
);
|
|
524
|
+
const parsed = asExtraction(raw);
|
|
525
|
+
const results: SerpItem[] = parsed.results.map((item) => ({
|
|
526
|
+
title: item.title,
|
|
527
|
+
url: item.url,
|
|
528
|
+
snippet: item.snippet,
|
|
529
|
+
...(item.facts.length === 0 ? {} : { facts: item.facts }),
|
|
530
|
+
}));
|
|
531
|
+
const timings = { extractMs: Math.max(0, Date.now() - startedAt) };
|
|
532
|
+
return {
|
|
533
|
+
engine: "avito",
|
|
534
|
+
query: q.text,
|
|
535
|
+
results,
|
|
536
|
+
pageCarried: parsed.pageCarried,
|
|
537
|
+
...(parsed.similar ? { notice: SIMILAR_NOTICE } : {}),
|
|
538
|
+
timings,
|
|
539
|
+
};
|
|
540
|
+
},
|
|
541
|
+
|
|
542
|
+
async canaries(page: CdpPage): Promise<Record<string, number>> {
|
|
543
|
+
const raw = await page.evaluate<unknown>(`(${canaryCounts.toString()})(document)`);
|
|
544
|
+
return asCanaries(raw);
|
|
545
|
+
},
|
|
546
|
+
};
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Adapter-authored (never page text): the caller must not read a page of
|
|
550
|
+
* recommendations as an answer to its query (S7.6).
|
|
551
|
+
*/
|
|
552
|
+
export const SIMILAR_NOTICE = "no exact match: Avito returned similar listings";
|
package/src/engines/google.ts
CHANGED
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import type { CdpPage } from "../cdp/types.ts";
|
|
31
31
|
import { waitForMarker } from "../cdp/humanize.ts";
|
|
32
|
+
import { asFacts } from "../extract/facts.ts";
|
|
32
33
|
import { isHttpUrl } from "../extract/urls.ts";
|
|
33
34
|
import { systemClock } from "../runtime/clock.ts";
|
|
34
35
|
import type { Clock } from "../runtime/types.ts";
|
|
@@ -598,6 +599,85 @@ export function extractFromDocument(doc: Document, arg: ExtractArgument): Extrac
|
|
|
598
599
|
return { results: [], aiProbe: { state: measured.state, chars: measured.text.length }, pageCarried: false };
|
|
599
600
|
}
|
|
600
601
|
|
|
602
|
+
/**
|
|
603
|
+
* Card facts of a shopping-like result (SPEC S7.7, `google-facts` fixture):
|
|
604
|
+
* Google prints one "rich" row under the snippet — `От 4 057,00 ₽ · 4,9(302)`
|
|
605
|
+
* — and nothing in the DOM says which number is a price and which is a
|
|
606
|
+
* rating, so both are read from what the page itself labels:
|
|
607
|
+
*
|
|
608
|
+
* - the rating is the row's `aria-label` ("Оценка: 4,9 из 5, (302) отзыва
|
|
609
|
+
* пользователей"), copied verbatim: a rating Google computes is Google's
|
|
610
|
+
* statement, not ours;
|
|
611
|
+
* - the price is a leaf whose *entire* text is one price. The full-text rule
|
|
612
|
+
* is what keeps "41 246 ₽" inside a sentence out of `facts`.
|
|
613
|
+
*
|
|
614
|
+
* Both rules are locale- and currency-agnostic on purpose (`lang` is a config
|
|
615
|
+
* key, so a German or Polish SERP is a supported input, not a surprise):
|
|
616
|
+
* - a price is recognised by shape — one currency token (symbol, ISO code or
|
|
617
|
+
* a short suffix like `zł`/`Kč`/`kr`), digits, separators, and at most a
|
|
618
|
+
* four-letter prefix ("От", "From", "ab"). Anything wordier is prose. A
|
|
619
|
+
* locale whose prefix is longer loses the price fact instead of getting a
|
|
620
|
+
* wrong one;
|
|
621
|
+
* - a rating is recognised by its invariant shape, `<score> … 5`, in any
|
|
622
|
+
* language ("4,9 из 5", "4.9 out of 5", "4,9 von 5", "4,9 sur 5"), and is
|
|
623
|
+
* then copied whole, so the reader gets the page's own words.
|
|
624
|
+
*/
|
|
625
|
+
function cardFacts(scope: Element | null): string[] {
|
|
626
|
+
if (scope === null) return [];
|
|
627
|
+
const facts: string[] = [];
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* Symbols and ISO codes seen on Google's own currency list, plus the
|
|
631
|
+
* word-suffix currencies whose symbol follows the amount. Deliberately a
|
|
632
|
+
* closed list: an unknown currency means "no price fact", never a guess.
|
|
633
|
+
*/
|
|
634
|
+
const CURRENCY =
|
|
635
|
+
/[$€£¥₹₽₩₪₺₴₸₾₼₫₱฿]|\b(?:USD|EUR|RUB|GBP|UAH|BYN|KZT|PLN|CZK|SEK|NOK|DKK|CHF|JPY|CNY|INR|TRY|GEL|AMD|AZN|UZS|KGS|TJS|MDL|RSD|BGN|HUF|RON|ILS|AED|SAR|BRL|KRW|THB|VND|PHP|IDR|MYR|SGD|HKD|TWD|MXN|CAD|AUD|NZD|ARS|CLP|COP|PEN)\b|zł|Kč|kr|TL|RMB|руб|грн|сум|тенге|лари|драм|ман|円|元|圓/iu;
|
|
636
|
+
const CURRENCY_ALL =
|
|
637
|
+
/[$€£¥₹₽₩₪₺₴₸₾₼₫₱฿]|(?:USD|EUR|RUB|GBP|UAH|BYN|KZT|PLN|CZK|SEK|NOK|DKK|CHF|JPY|CNY|INR|TRY|GEL|AMD|AZN|UZS|KGS|TJS|MDL|RSD|BGN|HUF|RON|ILS|AED|SAR|BRL|KRW|THB|VND|PHP|IDR|MYR|SGD|HKD|TWD|MXN|CAD|AUD|NZD|ARS|CLP|COP|PEN)|zł|Kč|kr|TL|RMB|руб|грн|сум|тенге|лари|драм|ман|円|元|圓/giu;
|
|
638
|
+
/** Letters a price may carry besides the currency: "От", "From", "ab". */
|
|
639
|
+
const PRICE_PREFIX = /^[A-Za-zА-Яа-яЁё]{0,4}$/;
|
|
640
|
+
|
|
641
|
+
function looksLikePrice(text: string): boolean {
|
|
642
|
+
if (text === "" || text.length > 26) return false;
|
|
643
|
+
if (!/[0-9]/.test(text)) return false;
|
|
644
|
+
if (!CURRENCY.test(text)) return false;
|
|
645
|
+
const rest = text
|
|
646
|
+
.replace(CURRENCY_ALL, " ")
|
|
647
|
+
// Digits, grouping/decimals, ranges ("–") and noise like "~".
|
|
648
|
+
.replace(/[0-9\s.,'\u00a0\u202f\u2013\u2014~≈-]/g, " ")
|
|
649
|
+
.trim();
|
|
650
|
+
return PRICE_PREFIX.test(rest);
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
for (const node of Array.from(scope.querySelectorAll("span, b"))) {
|
|
654
|
+
if (node.children.length > 0) continue;
|
|
655
|
+
const text = normalize(node.textContent);
|
|
656
|
+
if (!looksLikePrice(text)) continue;
|
|
657
|
+
facts.push(text);
|
|
658
|
+
break;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* `<score> … 5` — the scale is the only language-independent part of a
|
|
663
|
+
* rating label; the words around it differ per locale and are kept as-is.
|
|
664
|
+
* A label longer than a fact (60 chars, the same ceiling `asFacts` applies)
|
|
665
|
+
* is a paragraph that happens to contain numbers, not a card label.
|
|
666
|
+
*
|
|
667
|
+
* Both orders exist: "4,9 из 5" (ru/de/pl) and "5 üzerinden 4,9" /
|
|
668
|
+
* "5 段階中 4.9" (tr/ja). The reverse form is anchored to the start of the
|
|
669
|
+
* label, so a stray "5" inside a sentence cannot qualify.
|
|
670
|
+
*/
|
|
671
|
+
const RATING = /[0-9][.,][0-9][^0-9]{1,14}5(?![0-9])|^5[^0-9]{1,14}[0-9][.,][0-9]/;
|
|
672
|
+
for (const node of Array.from(scope.querySelectorAll("[aria-label]"))) {
|
|
673
|
+
const label = normalize(node.getAttribute("aria-label"));
|
|
674
|
+
if (label === "" || label.length > 60 || !RATING.test(label)) continue;
|
|
675
|
+
facts.push(label);
|
|
676
|
+
break;
|
|
677
|
+
}
|
|
678
|
+
return facts;
|
|
679
|
+
}
|
|
680
|
+
|
|
601
681
|
const results: SerpItem[] = [];
|
|
602
682
|
const seenUrls = new Set<string>();
|
|
603
683
|
let organics = 0;
|
|
@@ -621,8 +701,11 @@ export function extractFromDocument(doc: Document, arg: ExtractArgument): Extrac
|
|
|
621
701
|
const snippetNode = container === null ? null : container.querySelector(".VwiC3b, [data-sncf]");
|
|
622
702
|
const snippet = clampText(snippetNode === null ? "" : visibleText(snippetNode), MAX_SNIPPET_CHARS);
|
|
623
703
|
|
|
704
|
+
const facts = cardFacts(container);
|
|
624
705
|
organics += 1;
|
|
625
|
-
if (results.length < limit)
|
|
706
|
+
if (results.length < limit) {
|
|
707
|
+
results.push({ title, url: href, snippet, ...(facts.length === 0 ? {} : { facts }) });
|
|
708
|
+
}
|
|
626
709
|
}
|
|
627
710
|
|
|
628
711
|
let ai: ExtractedAi | undefined;
|
|
@@ -923,7 +1006,9 @@ function asSerpItem(value: unknown): SerpItem | null {
|
|
|
923
1006
|
const url = typeof value.url === "string" ? value.url : "";
|
|
924
1007
|
const snippet = typeof value.snippet === "string" ? value.snippet.trim() : "";
|
|
925
1008
|
if (title === "" || !isHttpUrl(url)) return null;
|
|
926
|
-
|
|
1009
|
+
// `facts` is page output like any other field: capped in one place (S7.7).
|
|
1010
|
+
const facts = asFacts(value.facts);
|
|
1011
|
+
return { title, url, snippet, ...(facts.length === 0 ? {} : { facts }) };
|
|
927
1012
|
}
|
|
928
1013
|
|
|
929
1014
|
function asAiState(value: unknown): "answer" | "lead" | undefined {
|