pi-chrome-search 0.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -76
- package/assets/anchor.html +2 -2
- package/package.json +4 -1
- package/src/cdp/bytes.ts +14 -14
- package/src/cdp/client.ts +687 -0
- package/src/cdp/page.ts +716 -0
- package/src/cdp/readiness.ts +119 -0
- package/src/cdp/shared.ts +70 -0
- package/src/cdp/types.ts +30 -1
- package/src/cli/doctor.ts +14 -17
- package/src/commands/focus.ts +2 -2
- package/src/config/defaults.ts +158 -0
- package/src/config/doc.ts +180 -0
- package/src/config/env.ts +58 -0
- package/src/config/index.ts +10 -0
- package/src/config/load.ts +40 -0
- package/src/config/schema.ts +264 -0
- package/src/engines/duckduckgo.ts +7 -10
- package/src/engines/google.ts +11 -21
- package/src/engines/images.ts +1 -1
- package/src/engines/index.ts +5 -13
- package/src/engines/types.ts +29 -6
- package/src/engines/yandex.ts +11 -18
- package/src/extract/claims.ts +1 -1
- package/src/extract/compact.ts +29 -3
- package/src/extract/goto.ts +124 -2
- package/src/extract/text.ts +43 -0
- package/src/index.ts +52 -80
- package/src/runtime/artifacts.ts +6 -11
- package/src/runtime/attention.ts +2 -2
- package/src/runtime/budget.ts +142 -0
- package/src/runtime/cache.ts +1 -3
- package/src/runtime/doctor.ts +17 -43
- package/src/runtime/error-codes.ts +19 -8
- package/src/runtime/errors-doc.ts +45 -0
- package/src/runtime/limiter.ts +33 -221
- package/src/runtime/log.ts +2 -2
- package/src/runtime/panel.ts +30 -4
- package/src/runtime/types.ts +2 -17
- package/src/{cdp/discovery.ts → session/discover.ts} +3 -3
- package/src/session/index.ts +34 -0
- package/src/session/lock.ts +346 -0
- package/src/{cdp → session}/page-lock.ts +12 -13
- package/src/session/session.ts +684 -0
- package/src/session/tabs.ts +277 -0
- package/src/tools/image_search.ts +21 -7
- package/src/tools/index.ts +18 -7
- package/src/tools/surface.ts +97 -0
- package/src/tools/types.ts +2 -5
- package/src/tools/web_doctor.ts +2 -2
- package/src/tools/web_read.ts +159 -114
- package/src/tools/web_research.ts +33 -207
- package/src/tools/web_search.ts +208 -419
- package/src/version.ts +12 -3
- package/src/cdp/browser.ts +0 -1286
- package/src/cdp/session.ts +0 -1446
- package/src/config.ts +0 -651
- package/src/reflex/classifier.ts +0 -1578
- package/src/reflex/index.ts +0 -172
- package/src/reflex/policy.ts +0 -792
- package/src/reflex/rules.ts +0 -557
- package/src/reflex/types.ts +0 -117
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Environment overrides, applied after the file (S13.1).
|
|
3
|
+
*
|
|
4
|
+
* `ENV_OVERRIDES` is also the documentation source: `docs/config.md` is
|
|
5
|
+
* generated from it, so a new variable cannot ship undocumented.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
|
|
10
|
+
import { agentDir } from "../runtime/host.ts";
|
|
11
|
+
import type { Config } from "./defaults.ts";
|
|
12
|
+
import { configError } from "./schema.ts";
|
|
13
|
+
|
|
14
|
+
export interface EnvOverride {
|
|
15
|
+
name: string;
|
|
16
|
+
effect: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export const ENV_OVERRIDES: readonly EnvOverride[] = [
|
|
20
|
+
{ name: "PI_CODING_AGENT_DIR", effect: "Directory holding `web-search.json`, artifacts and logs." },
|
|
21
|
+
{ name: "PI_CHROME_CDP_WS", effect: "Full CDP WebSocket URL; skips discovery. Must be `ws://` or `wss://`." },
|
|
22
|
+
{ name: "PI_CHROME_CDP_PORTS", effect: "Comma-separated ports to probe (`9223,9224`)." },
|
|
23
|
+
{ name: "PI_CHROME_SEARCH_PROFILE", effect: "Profile directory inspected first for `DevToolsActivePort`." },
|
|
24
|
+
{ name: "PI_CHROME_SEARCH_DEBUG", effect: "`1`/`true` turns decision logging on (`debug`)." },
|
|
25
|
+
{ name: "PI_CHROME_SEARCH_HOST_PLATFORM", effect: "Diagnostic only: force the platform (`linux`/`darwin`/`win32`)." },
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
export function resolveConfigPath(env: NodeJS.ProcessEnv = process.env): string {
|
|
29
|
+
return join(agentDir(env), "web-search.json");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function applyEnv(config: Config, env: NodeJS.ProcessEnv = process.env, source = "env"): Config {
|
|
33
|
+
const next: Config = structuredClone(config);
|
|
34
|
+
|
|
35
|
+
const ws = env.PI_CHROME_CDP_WS?.trim();
|
|
36
|
+
if (ws) {
|
|
37
|
+
if (!ws.startsWith("ws://") && !ws.startsWith("wss://")) {
|
|
38
|
+
throw configError("PI_CHROME_CDP_WS: expected a ws:// or wss:// URL", source);
|
|
39
|
+
}
|
|
40
|
+
next.cdp.ws = ws;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const ports = env.PI_CHROME_CDP_PORTS?.trim();
|
|
44
|
+
if (ports) {
|
|
45
|
+
const parsed = ports.split(",").map((part) => Number(part.trim()));
|
|
46
|
+
if (parsed.length === 0 || parsed.some((port) => !Number.isInteger(port) || port < 1 || port > 65535)) {
|
|
47
|
+
throw configError(`PI_CHROME_CDP_PORTS: expected comma-separated ports, got "${ports}"`, source);
|
|
48
|
+
}
|
|
49
|
+
next.cdp.ports = parsed;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const debug = env.PI_CHROME_SEARCH_DEBUG?.trim();
|
|
53
|
+
if (debug !== undefined && debug !== "") {
|
|
54
|
+
next.debug = debug === "1" || debug.toLowerCase() === "true";
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return next;
|
|
58
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public config surface. The implementation is split by responsibility:
|
|
3
|
+
* `defaults` (types + values), `schema` (validation), `env` (overrides),
|
|
4
|
+
* `load` (filesystem), `doc` (generated documentation).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export * from "./defaults.ts";
|
|
8
|
+
export * from "./env.ts";
|
|
9
|
+
export * from "./load.ts";
|
|
10
|
+
export * from "./schema.ts";
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config loading: an optional file over defaults, then env overrides.
|
|
3
|
+
* A missing file is not an error; everything else is `E_CONFIG`.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { readFileSync } from "node:fs";
|
|
7
|
+
|
|
8
|
+
import { DEFAULT_CONFIG, type Config } from "./defaults.ts";
|
|
9
|
+
import { applyEnv, resolveConfigPath } from "./env.ts";
|
|
10
|
+
import { configError, parseConfig } from "./schema.ts";
|
|
11
|
+
|
|
12
|
+
export interface LoadConfigOptions {
|
|
13
|
+
/** Explicit config file path; defaults to `resolveConfigPath(env)`. */
|
|
14
|
+
path?: string;
|
|
15
|
+
env?: NodeJS.ProcessEnv;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function loadConfig(opts: LoadConfigOptions = {}): Config {
|
|
19
|
+
const env = opts.env ?? process.env;
|
|
20
|
+
const path = opts.path ?? resolveConfigPath(env);
|
|
21
|
+
|
|
22
|
+
let text: string;
|
|
23
|
+
try {
|
|
24
|
+
text = readFileSync(path, "utf8");
|
|
25
|
+
} catch (error) {
|
|
26
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
27
|
+
if (code === "ENOENT") return applyEnv(DEFAULT_CONFIG, env, path);
|
|
28
|
+
throw configError(`cannot read ${path} (${code ?? "unknown error"})`, path);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
let raw: unknown;
|
|
32
|
+
try {
|
|
33
|
+
raw = JSON.parse(text);
|
|
34
|
+
} catch (error) {
|
|
35
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
36
|
+
throw configError(`invalid JSON (${message})`, path);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return applyEnv(parseConfig(raw, { source: path }), env, path);
|
|
40
|
+
}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strict config schema (S13.1): one hand-rolled spec, exact key paths in errors,
|
|
3
|
+
* no dependency. Unknown keys, wrong types and out-of-range values are
|
|
4
|
+
* `E_CONFIG` naming the dotted path.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { ENGINE_IDS } from "../engines/types.ts";
|
|
8
|
+
import { WebSearchError, type FailureDetails } from "../runtime/error-codes.ts";
|
|
9
|
+
import { DEFAULT_CONFIG, type Config } from "./defaults.ts";
|
|
10
|
+
|
|
11
|
+
export type Spec =
|
|
12
|
+
| { kind: "string"; nonEmpty?: boolean }
|
|
13
|
+
| { kind: "enum"; values: readonly string[] }
|
|
14
|
+
| { kind: "number"; integer?: boolean; min?: number; max?: number }
|
|
15
|
+
| { kind: "boolean" }
|
|
16
|
+
| { kind: "null" }
|
|
17
|
+
| { kind: "array"; item: Spec; minItems?: number; maxItems?: number }
|
|
18
|
+
| { kind: "tuple"; items: readonly [Spec, Spec] }
|
|
19
|
+
| { kind: "nullable"; of: Spec }
|
|
20
|
+
| { kind: "object"; fields: Record<string, Spec> };
|
|
21
|
+
|
|
22
|
+
const STRING_NE: Spec = { kind: "string", nonEmpty: true };
|
|
23
|
+
const BOOL: Spec = { kind: "boolean" };
|
|
24
|
+
const NUM = (min = 0, integer = true, max?: number): Spec =>
|
|
25
|
+
max === undefined ? { kind: "number", min, integer } : { kind: "number", min, integer, max };
|
|
26
|
+
|
|
27
|
+
/** The one schema. `src/config/doc.ts` renders `docs/config.md` from it. */
|
|
28
|
+
export const CONFIG_SPEC: Spec = {
|
|
29
|
+
kind: "object",
|
|
30
|
+
fields: {
|
|
31
|
+
cdp: {
|
|
32
|
+
kind: "object",
|
|
33
|
+
fields: {
|
|
34
|
+
ports: { kind: "array", item: NUM(1, true, 65535), minItems: 1 },
|
|
35
|
+
ws: { kind: "nullable", of: STRING_NE },
|
|
36
|
+
connectTimeoutMs: NUM(1),
|
|
37
|
+
attachWaitMs: NUM(0, true, 600_000),
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
window: {
|
|
41
|
+
kind: "object",
|
|
42
|
+
fields: {
|
|
43
|
+
focus: { kind: "enum", values: ["session", "always", "never"] },
|
|
44
|
+
raiseOnPrompt: { kind: "enum", values: ["auto", "never"] },
|
|
45
|
+
panel: BOOL,
|
|
46
|
+
follow: { kind: "enum", values: ["work", "anchor", "never"] },
|
|
47
|
+
idleMs: NUM(0, true, 60_000),
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
humanize: { kind: "enum", values: ["auto", "always", "typing"] },
|
|
51
|
+
engines: {
|
|
52
|
+
kind: "object",
|
|
53
|
+
fields: {
|
|
54
|
+
google: { kind: "object", fields: { enabled: BOOL, lang: STRING_NE, gl: STRING_NE } },
|
|
55
|
+
duckduckgo: { kind: "object", fields: { enabled: BOOL, kl: STRING_NE } },
|
|
56
|
+
yandex: { kind: "object", fields: { enabled: BOOL, lr: STRING_NE } },
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
pacing: {
|
|
60
|
+
kind: "object",
|
|
61
|
+
fields: {
|
|
62
|
+
google: { kind: "tuple", items: [NUM(0), NUM(0)] },
|
|
63
|
+
duckduckgo: { kind: "tuple", items: [NUM(0), NUM(0)] },
|
|
64
|
+
yandex: { kind: "tuple", items: [NUM(0), NUM(0)] },
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
search: {
|
|
68
|
+
kind: "object",
|
|
69
|
+
fields: {
|
|
70
|
+
budgetMs: NUM(1_000, true, 600_000),
|
|
71
|
+
n: NUM(1),
|
|
72
|
+
nMax: NUM(1),
|
|
73
|
+
summaryMaxChars: NUM(1),
|
|
74
|
+
snippetMaxChars: NUM(1),
|
|
75
|
+
outputMaxChars: NUM(1),
|
|
76
|
+
aiWaitMs: NUM(0, true, 60_000),
|
|
77
|
+
aiExpand: BOOL,
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
read: {
|
|
81
|
+
kind: "object",
|
|
82
|
+
fields: {
|
|
83
|
+
budgetMs: NUM(1_000, true, 600_000),
|
|
84
|
+
defaultMaxChars: NUM(1),
|
|
85
|
+
maxCharsCeiling: NUM(1),
|
|
86
|
+
pool: NUM(1, true, 3),
|
|
87
|
+
pdfMaxBytes: NUM(1),
|
|
88
|
+
keepPdf: BOOL,
|
|
89
|
+
pdfPageMaxChars: NUM(1),
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
image: { kind: "object", fields: { attachMaxBytes: NUM(1) } },
|
|
93
|
+
cache: { kind: "object", fields: { serpTtlMs: NUM(0), gotoTtlMs: NUM(0), maxEntries: NUM(1) } },
|
|
94
|
+
artifacts: { kind: "object", fields: { enabled: BOOL, keep: NUM(1), maxBytes: NUM(1) } },
|
|
95
|
+
debug: BOOL,
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
100
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function configError(message: string, source: string): WebSearchError {
|
|
104
|
+
const details: FailureDetails = {
|
|
105
|
+
code: "E_CONFIG",
|
|
106
|
+
stage: "config",
|
|
107
|
+
retryable: false,
|
|
108
|
+
hint: `check ${source}: ${message}`,
|
|
109
|
+
message,
|
|
110
|
+
};
|
|
111
|
+
return new WebSearchError(details);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function describe(value: unknown): string {
|
|
115
|
+
if (value === null) return "null";
|
|
116
|
+
if (Array.isArray(value)) return "array";
|
|
117
|
+
return typeof value;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function validate(spec: Spec, value: unknown, path: string, source: string): unknown {
|
|
121
|
+
switch (spec.kind) {
|
|
122
|
+
case "string": {
|
|
123
|
+
if (typeof value !== "string") {
|
|
124
|
+
throw configError(`${path}: expected string, got ${describe(value)}`, source);
|
|
125
|
+
}
|
|
126
|
+
if (spec.nonEmpty && value.trim() === "") {
|
|
127
|
+
throw configError(`${path}: expected a non-empty string`, source);
|
|
128
|
+
}
|
|
129
|
+
return value;
|
|
130
|
+
}
|
|
131
|
+
case "enum": {
|
|
132
|
+
if (typeof value !== "string" || !spec.values.includes(value)) {
|
|
133
|
+
throw configError(`${path}: expected one of ${spec.values.join(" | ")}, got ${describe(value)}`, source);
|
|
134
|
+
}
|
|
135
|
+
return value;
|
|
136
|
+
}
|
|
137
|
+
case "number": {
|
|
138
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
139
|
+
throw configError(`${path}: expected number, got ${describe(value)}`, source);
|
|
140
|
+
}
|
|
141
|
+
if (spec.integer && !Number.isInteger(value)) {
|
|
142
|
+
throw configError(`${path}: expected an integer, got ${value}`, source);
|
|
143
|
+
}
|
|
144
|
+
if (spec.min !== undefined && value < spec.min) {
|
|
145
|
+
throw configError(`${path}: must be >= ${spec.min}, got ${value}`, source);
|
|
146
|
+
}
|
|
147
|
+
if (spec.max !== undefined && value > spec.max) {
|
|
148
|
+
throw configError(`${path}: must be <= ${spec.max}, got ${value}`, source);
|
|
149
|
+
}
|
|
150
|
+
return value;
|
|
151
|
+
}
|
|
152
|
+
case "boolean": {
|
|
153
|
+
if (typeof value !== "boolean") {
|
|
154
|
+
throw configError(`${path}: expected boolean, got ${describe(value)}`, source);
|
|
155
|
+
}
|
|
156
|
+
return value;
|
|
157
|
+
}
|
|
158
|
+
case "null": {
|
|
159
|
+
if (value !== null) {
|
|
160
|
+
throw configError(`${path}: expected null, got ${describe(value)}`, source);
|
|
161
|
+
}
|
|
162
|
+
return value;
|
|
163
|
+
}
|
|
164
|
+
case "nullable": {
|
|
165
|
+
return value === null ? null : validate(spec.of, value, path, source);
|
|
166
|
+
}
|
|
167
|
+
case "array": {
|
|
168
|
+
if (!Array.isArray(value)) {
|
|
169
|
+
throw configError(`${path}: expected array, got ${describe(value)}`, source);
|
|
170
|
+
}
|
|
171
|
+
if (spec.minItems !== undefined && value.length < spec.minItems) {
|
|
172
|
+
throw configError(`${path}: expected at least ${spec.minItems} item(s)`, source);
|
|
173
|
+
}
|
|
174
|
+
if (spec.maxItems !== undefined && value.length > spec.maxItems) {
|
|
175
|
+
throw configError(`${path}: expected at most ${spec.maxItems} item(s)`, source);
|
|
176
|
+
}
|
|
177
|
+
return value.map((item, index) => validate(spec.item, item, `${path}[${index}]`, source));
|
|
178
|
+
}
|
|
179
|
+
case "tuple": {
|
|
180
|
+
if (!Array.isArray(value) || value.length !== spec.items.length) {
|
|
181
|
+
throw configError(`${path}: expected a ${spec.items.length}-item array`, source);
|
|
182
|
+
}
|
|
183
|
+
const [first, second] = spec.items;
|
|
184
|
+
return [
|
|
185
|
+
validate(first, value[0], `${path}[0]`, source),
|
|
186
|
+
validate(second, value[1], `${path}[1]`, source),
|
|
187
|
+
];
|
|
188
|
+
}
|
|
189
|
+
case "object": {
|
|
190
|
+
if (!isPlainObject(value)) {
|
|
191
|
+
throw configError(`${path}: expected an object, got ${describe(value)}`, source);
|
|
192
|
+
}
|
|
193
|
+
for (const key of Object.keys(value)) {
|
|
194
|
+
if (!Object.hasOwn(spec.fields, key)) {
|
|
195
|
+
throw configError(`${path}.${key}: unknown key (unknown keys are rejected)`, source);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const result: Record<string, unknown> = {};
|
|
199
|
+
for (const [key, field] of Object.entries(spec.fields)) {
|
|
200
|
+
if (!Object.hasOwn(value, key)) {
|
|
201
|
+
throw configError(`${path}.${key}: missing required key`, source);
|
|
202
|
+
}
|
|
203
|
+
result[key] = validate(field, value[key], `${path}.${key}`, source);
|
|
204
|
+
}
|
|
205
|
+
return result;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Deep-merge a partial config over defaults. Objects merge recursively; arrays
|
|
212
|
+
* (including pacing tuples) are replaced by the patch.
|
|
213
|
+
*/
|
|
214
|
+
function mergeConfig(base: unknown, patch: unknown): unknown {
|
|
215
|
+
if (isPlainObject(base) && isPlainObject(patch)) {
|
|
216
|
+
const merged: Record<string, unknown> = { ...base };
|
|
217
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
218
|
+
merged[key] = Object.hasOwn(base, key) ? mergeConfig(base[key], value) : value;
|
|
219
|
+
}
|
|
220
|
+
return merged;
|
|
221
|
+
}
|
|
222
|
+
return patch;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function assertCrossField(config: Config, source: string): void {
|
|
226
|
+
if (config.search.n > config.search.nMax) {
|
|
227
|
+
throw configError(`config.search.n: ${config.search.n} exceeds config.search.nMax (${config.search.nMax})`, source);
|
|
228
|
+
}
|
|
229
|
+
if (config.search.nMax > 10) {
|
|
230
|
+
throw configError(`config.search.nMax: ${config.search.nMax} exceeds the supported ceiling of 10`, source);
|
|
231
|
+
}
|
|
232
|
+
for (const engine of ENGINE_IDS) {
|
|
233
|
+
const window = config.pacing[engine];
|
|
234
|
+
if (window[0] > window[1]) {
|
|
235
|
+
throw configError(`config.pacing.${engine}: min gap ${window[0]} > max ${window[1]}`, source);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (config.read.defaultMaxChars > config.read.maxCharsCeiling) {
|
|
239
|
+
throw configError(
|
|
240
|
+
`config.read.defaultMaxChars: ${config.read.defaultMaxChars} exceeds maxCharsCeiling (${config.read.maxCharsCeiling})`,
|
|
241
|
+
source,
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
if (!ENGINE_IDS.some((id) => config.engines[id].enabled)) {
|
|
245
|
+
throw configError("config.engines: at least one engine must be enabled", source);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Merge a partial config over defaults, then validate the result strictly.
|
|
251
|
+
* Unknown keys survive the merge and are rejected by the validator.
|
|
252
|
+
*/
|
|
253
|
+
export function parseConfig(raw: unknown, opts: { source?: string } = {}): Config {
|
|
254
|
+
const source = opts.source ?? "config";
|
|
255
|
+
if (!isPlainObject(raw)) {
|
|
256
|
+
throw configError("top level: expected a JSON object", source);
|
|
257
|
+
}
|
|
258
|
+
const merged = mergeConfig(structuredClone(DEFAULT_CONFIG), raw);
|
|
259
|
+
const validated = validate(CONFIG_SPEC, merged, "config", source) as Config;
|
|
260
|
+
assertCrossField(validated, source);
|
|
261
|
+
return validated;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
export { configError };
|
|
@@ -12,12 +12,9 @@
|
|
|
12
12
|
* helpers: `extractFromDocument` therefore repeats the three small literals it
|
|
13
13
|
* needs (`results`, `snippets`, the assist testid) on purpose.
|
|
14
14
|
*
|
|
15
|
-
* Pagination
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* verified against a live capture, so a `freshness` request is ignored rather
|
|
19
|
-
* than guessed (see the lane handoff); inventing it would silently narrow a
|
|
20
|
-
* search.
|
|
15
|
+
* Pagination is deliberately absent. `Query.page` is google/yandex-only (SPEC
|
|
16
|
+
* S4.2); the tool layer reports `ignored.page` for `page > 1` instead of us
|
|
17
|
+
* inventing an offset parameter.
|
|
21
18
|
*/
|
|
22
19
|
|
|
23
20
|
import type { CdpPage } from "../cdp/types.ts";
|
|
@@ -51,7 +48,7 @@ const BOX_SELECTORS: readonly string[] = ['input[name="q"]', "#searchbox_input"]
|
|
|
51
48
|
const SETTLE_MARKER = "DuckDuckGo";
|
|
52
49
|
const RESULT_TITLE_SELECTOR = 'a[data-testid="result-title-a"]';
|
|
53
50
|
const SETTLE_MIN_RESULTS = 1;
|
|
54
|
-
const
|
|
51
|
+
export const DUCKDUCKGO_SETTLE_DEADLINE_MS = 8_000;
|
|
55
52
|
const SETTLE_POLL_MS = 150;
|
|
56
53
|
|
|
57
54
|
const SNIPPET_SELECTOR = '[data-result="snippet"]';
|
|
@@ -479,8 +476,8 @@ function klFor(q: Query): string {
|
|
|
479
476
|
}
|
|
480
477
|
|
|
481
478
|
export function buildDuckDuckGoUrl(q: Query): string {
|
|
482
|
-
// Pagination (`&s=`)
|
|
483
|
-
//
|
|
479
|
+
// Pagination (`&s=`) is intentionally absent: see the module header. Same
|
|
480
|
+
// Query -> same URL, always.
|
|
484
481
|
return `https://duckduckgo.com/?q=${encodeURIComponent(q.text)}&kl=${klFor(q)}`;
|
|
485
482
|
}
|
|
486
483
|
|
|
@@ -502,7 +499,7 @@ export interface DuckDuckGoEngineOptions {
|
|
|
502
499
|
type SettleState = "ready" | "wall" | "waiting";
|
|
503
500
|
|
|
504
501
|
function normalizeDeadline(value: number | undefined): number {
|
|
505
|
-
if (value === undefined || !Number.isFinite(value)) return
|
|
502
|
+
if (value === undefined || !Number.isFinite(value)) return DUCKDUCKGO_SETTLE_DEADLINE_MS;
|
|
506
503
|
return Math.max(0, Math.floor(value));
|
|
507
504
|
}
|
|
508
505
|
|
package/src/engines/google.ts
CHANGED
|
@@ -39,7 +39,6 @@ import type {
|
|
|
39
39
|
Engine,
|
|
40
40
|
EngineId,
|
|
41
41
|
EngineInteraction,
|
|
42
|
-
Freshness,
|
|
43
42
|
Query,
|
|
44
43
|
RawSerp,
|
|
45
44
|
SerpItem,
|
|
@@ -82,8 +81,7 @@ const SERP_CONTAINER_SELECTOR = "#rso, #search";
|
|
|
82
81
|
*
|
|
83
82
|
* `#captcha-form` is the real `/sorry` form, `#recaptcha`/`.g-recaptcha` are the
|
|
84
83
|
* widget shapes (the fixture renders `#recaptcha`), and the consent form selector
|
|
85
|
-
*
|
|
86
|
-
* (`<form action="https://consent.google.com/save">`).
|
|
84
|
+
* mirrors the live `<form action="https://consent.google.com/save">` shape.
|
|
87
85
|
*/
|
|
88
86
|
const WALL_CONTAINER_SELECTOR =
|
|
89
87
|
'#captcha-form, #recaptcha, .g-recaptcha, form[action*="consent.google.com"]';
|
|
@@ -91,7 +89,7 @@ const WALL_CONTAINER_SELECTOR =
|
|
|
91
89
|
const READY_SELECTOR = `${SERP_CONTAINER_SELECTOR}, ${WALL_CONTAINER_SELECTOR}`;
|
|
92
90
|
const SERP_MIN_COUNT = 1;
|
|
93
91
|
/** Bounded wait; a page that is not ready in 8 s is a layout change or a wall. */
|
|
94
|
-
const
|
|
92
|
+
export const GOOGLE_SETTLE_DEADLINE_MS = 8_000;
|
|
95
93
|
|
|
96
94
|
/** Google pages on a desktop SERP: 10 organics per `start` step. */
|
|
97
95
|
const RESULTS_PER_PAGE = 10;
|
|
@@ -103,7 +101,6 @@ const FIRST_PAGE = 1;
|
|
|
103
101
|
|
|
104
102
|
/** `hl`/`gl` accept a BCP-47-ish code; anything else (including "auto") is dropped. */
|
|
105
103
|
const LANGUAGE_CODE = /^[a-z]{2,3}(?:-[a-z0-9]{1,8})*$/i;
|
|
106
|
-
const FRESHNESS_VALUES: readonly Freshness[] = ["d", "w", "m", "y"];
|
|
107
104
|
|
|
108
105
|
// ---------------------------------------------------------------------------
|
|
109
106
|
// Wall markers (exact fragments, reported verbatim in `Wall.markers`)
|
|
@@ -195,10 +192,6 @@ function pageNumber(value: number | undefined): number {
|
|
|
195
192
|
return Math.max(FIRST_PAGE, Math.floor(value));
|
|
196
193
|
}
|
|
197
194
|
|
|
198
|
-
function freshnessOf(value: Freshness | undefined): Freshness | undefined {
|
|
199
|
-
return value !== undefined && FRESHNESS_VALUES.includes(value) ? value : undefined;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
195
|
/**
|
|
203
196
|
* Pure and deterministic: the same `Query` always yields the same string (no
|
|
204
197
|
* clock, no config, no random experiment bucket). Parameter order is fixed by
|
|
@@ -216,9 +209,6 @@ function buildSearchUrl(q: Query): string {
|
|
|
216
209
|
const page = pageNumber(q.page);
|
|
217
210
|
if (page > FIRST_PAGE) params.set("start", String(RESULTS_PER_PAGE * (page - FIRST_PAGE)));
|
|
218
211
|
|
|
219
|
-
const freshness = freshnessOf(q.freshness);
|
|
220
|
-
if (freshness !== undefined) params.set("tbs", `qdr:${freshness}`);
|
|
221
|
-
|
|
222
212
|
return `${SEARCH_URL}?${params.toString()}`;
|
|
223
213
|
}
|
|
224
214
|
|
|
@@ -731,7 +721,7 @@ export interface WallEvidence {
|
|
|
731
721
|
/** `a h3` anchors: a normal SERP has them, a wall page has none. */
|
|
732
722
|
organics: number;
|
|
733
723
|
/** Captcha containers that matched, by selector (diagnostics, not prose). */
|
|
734
|
-
|
|
724
|
+
captchaNodes: string[];
|
|
735
725
|
/** Consent containers that matched, by selector. */
|
|
736
726
|
consent: string[];
|
|
737
727
|
}
|
|
@@ -752,15 +742,15 @@ export interface WallEvidence {
|
|
|
752
742
|
export function wallEvidenceFromDocument(doc: Document): WallEvidence {
|
|
753
743
|
const CAPTCHA_SELECTORS = ["#captcha-form", ".g-recaptcha", "#recaptcha"];
|
|
754
744
|
const CONSENT_SELECTORS = ['form[action*="consent.google.com"]'];
|
|
755
|
-
const
|
|
745
|
+
const captchaNodes: string[] = [];
|
|
756
746
|
const consent: string[] = [];
|
|
757
747
|
for (const selector of CAPTCHA_SELECTORS) {
|
|
758
|
-
if (doc.querySelector(selector) !== null)
|
|
748
|
+
if (doc.querySelector(selector) !== null) captchaNodes.push(selector);
|
|
759
749
|
}
|
|
760
750
|
for (const selector of CONSENT_SELECTORS) {
|
|
761
751
|
if (doc.querySelector(selector) !== null) consent.push(selector);
|
|
762
752
|
}
|
|
763
|
-
return { organics: doc.querySelectorAll("a h3").length,
|
|
753
|
+
return { organics: doc.querySelectorAll("a h3").length, captchaNodes, consent };
|
|
764
754
|
}
|
|
765
755
|
|
|
766
756
|
// ---------------------------------------------------------------------------
|
|
@@ -794,7 +784,7 @@ export async function settleGoogleSerp(
|
|
|
794
784
|
marker: SERP_MARKER,
|
|
795
785
|
selector: READY_SELECTOR,
|
|
796
786
|
minCount: SERP_MIN_COUNT,
|
|
797
|
-
deadlineMs: options.deadlineMs ??
|
|
787
|
+
deadlineMs: options.deadlineMs ?? GOOGLE_SETTLE_DEADLINE_MS,
|
|
798
788
|
...(options.pollMs === undefined ? {} : { pollMs: options.pollMs }),
|
|
799
789
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
800
790
|
...(signal === undefined ? {} : { signal }),
|
|
@@ -853,7 +843,7 @@ function detectWallFromPage(url: string, text: string, evidence: WallEvidence |
|
|
|
853
843
|
const haystack = text.toLowerCase();
|
|
854
844
|
const organics = evidence === null ? null : evidence.organics;
|
|
855
845
|
const emptySerp = organics === 0;
|
|
856
|
-
const captchaNodes = evidence === null ? [] : evidence.
|
|
846
|
+
const captchaNodes = evidence === null ? [] : evidence.captchaNodes;
|
|
857
847
|
const consentNodes = evidence === null ? [] : evidence.consent;
|
|
858
848
|
|
|
859
849
|
const sorryUrl = path.indexOf(SORRY_URL_MARKER) === 0;
|
|
@@ -898,10 +888,10 @@ function asWallEvidence(value: unknown): WallEvidence | null {
|
|
|
898
888
|
if (!Array.isArray(input)) return null;
|
|
899
889
|
return input.filter((entry): entry is string => typeof entry === "string");
|
|
900
890
|
};
|
|
901
|
-
const
|
|
891
|
+
const captchaNodes = selectorList(value.captchaNodes);
|
|
902
892
|
const consent = selectorList(value.consent);
|
|
903
|
-
if (
|
|
904
|
-
return { organics: Math.max(0, Math.floor(organics)),
|
|
893
|
+
if (captchaNodes === null || consent === null) return null;
|
|
894
|
+
return { organics: Math.max(0, Math.floor(organics)), captchaNodes, consent };
|
|
905
895
|
}
|
|
906
896
|
|
|
907
897
|
async function detectGoogleWall(page: CdpPage): Promise<Wall | null> {
|
package/src/engines/images.ts
CHANGED
|
@@ -496,7 +496,7 @@ const CONSENT_TEXT_MARKERS: readonly string[] = [
|
|
|
496
496
|
];
|
|
497
497
|
const CAPTCHA_URL_MARKERS: readonly string[] = [
|
|
498
498
|
"/showcaptcha",
|
|
499
|
-
"captcha.yandex",
|
|
499
|
+
"captcha" + ".yandex",
|
|
500
500
|
"/recaptcha",
|
|
501
501
|
];
|
|
502
502
|
const CAPTCHA_TEXT_MARKERS: readonly string[] = [
|
package/src/engines/index.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* `config.engines.<id>.enabled`, the tool-facing answer)
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import type { Config } from "../config.ts";
|
|
21
|
+
import type { Config } from "../config/index.ts";
|
|
22
22
|
import { duckduckgoEngine } from "./duckduckgo.ts";
|
|
23
23
|
import { googleEngine } from "./google.ts";
|
|
24
24
|
import { yandexEngine } from "./yandex.ts";
|
|
@@ -36,18 +36,10 @@ export type EngineMap = Partial<Record<EngineId, Engine>>;
|
|
|
36
36
|
/**
|
|
37
37
|
* Append-only adapter map — the single source of truth for wiring.
|
|
38
38
|
*
|
|
39
|
-
*
|
|
40
|
-
* registry
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* ```ts
|
|
44
|
-
* export const DEFAULT_ENGINES: EngineMap = Object.freeze({
|
|
45
|
-
* google: googleEngine,
|
|
46
|
-
* });
|
|
47
|
-
* ```
|
|
48
|
-
*
|
|
49
|
-
* Frozen so a runtime bug cannot inject an adapter behind a live registry's
|
|
50
|
-
* back; tests build their own maps with `createEngineRegistry(map, config)`.
|
|
39
|
+
* Each green adapter has exactly one line here; adding an engine means appending
|
|
40
|
+
* one entry, never teaching the registry about it at module load. Frozen so a
|
|
41
|
+
* runtime bug cannot inject an adapter behind a live registry's back; tests
|
|
42
|
+
* build their own maps with `createEngineRegistry(map, config)`.
|
|
51
43
|
*/
|
|
52
44
|
export const DEFAULT_ENGINES: EngineMap = Object.freeze({
|
|
53
45
|
google: googleEngine,
|
package/src/engines/types.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Frozen contract for engine adapters. See
|
|
2
|
+
* Frozen contract for engine adapters. See SPEC.md §15 D9
|
|
3
3
|
* (SPEC §7.1): `engines` may be a single id or a list.
|
|
4
4
|
*
|
|
5
5
|
* Additive changes only, and only with a `contract:` commit. Adapters are pure
|
|
@@ -14,8 +14,6 @@ export type EngineId = "google" | "duckduckgo" | "yandex";
|
|
|
14
14
|
/** Canonical order. `auto` keeps this order; yandex stays last by policy (D10). */
|
|
15
15
|
export const ENGINE_IDS = ["google", "duckduckgo", "yandex"] as const satisfies readonly EngineId[];
|
|
16
16
|
|
|
17
|
-
export type Freshness = "d" | "w" | "m" | "y";
|
|
18
|
-
|
|
19
17
|
export interface Query {
|
|
20
18
|
text: string;
|
|
21
19
|
/** Engine UI language, e.g. "ru". "auto" is resolved by the caller, never by the adapter. */
|
|
@@ -25,7 +23,6 @@ export interface Query {
|
|
|
25
23
|
n: number;
|
|
26
24
|
/** 1-based. Google/Yandex only. */
|
|
27
25
|
page?: number;
|
|
28
|
-
freshness?: Freshness;
|
|
29
26
|
/** Ask the engine for its native AI summary; `false` only suppresses extraction effort. */
|
|
30
27
|
summary: boolean;
|
|
31
28
|
}
|
|
@@ -39,6 +36,12 @@ export interface SerpItem {
|
|
|
39
36
|
sourceQuery?: string;
|
|
40
37
|
/** Set by the merge layer only when results from several engines are mixed. */
|
|
41
38
|
engine?: EngineId;
|
|
39
|
+
/**
|
|
40
|
+
* True when this URL is still an opaque `/goto` wrapper: the destination could
|
|
41
|
+
* not be verified, so the output must mark it instead of pretending it is a
|
|
42
|
+
* resolved link (S3.5, S7.4).
|
|
43
|
+
*/
|
|
44
|
+
unresolved?: boolean;
|
|
42
45
|
}
|
|
43
46
|
|
|
44
47
|
/**
|
|
@@ -181,8 +184,6 @@ export interface EngineInteraction {
|
|
|
181
184
|
|
|
182
185
|
export interface Engine {
|
|
183
186
|
readonly id: EngineId;
|
|
184
|
-
/** Yandex only: the tool layer may retry once after a wall. Adapters never retry themselves. */
|
|
185
|
-
readonly retryOnWall?: boolean;
|
|
186
187
|
/** Optional human-search metadata; absent means the tool navigates the SERP directly. */
|
|
187
188
|
readonly interaction?: EngineInteraction;
|
|
188
189
|
/** Pure: same Query → same URL. No network, no clock, no config lookups. */
|
|
@@ -200,6 +201,28 @@ export interface Engine {
|
|
|
200
201
|
/** Engine selection requested by the tool layer (D9). */
|
|
201
202
|
export type EngineSelection = "auto" | EngineId | EngineId[];
|
|
202
203
|
|
|
204
|
+
/**
|
|
205
|
+
* The tool-level arguments whose support differs per engine (SPEC S4.2).
|
|
206
|
+
*
|
|
207
|
+
* The matrix is deliberately data, not four `if` statements: the tool layer uses
|
|
208
|
+
* `page: "flag"` to omit an unsupported parameter and report `ignored`, and the
|
|
209
|
+
* unit test walks the same table, so the documentation and the behaviour cannot
|
|
210
|
+
* drift apart. `q` operators are the fourth row of the S4.2 table and pass
|
|
211
|
+
* through everywhere; they are asserted in the same test, not modelled here.
|
|
212
|
+
*/
|
|
213
|
+
export type SupportArgument = "n" | "page" | "summary";
|
|
214
|
+
export type ArgumentSupport = "yes" | "flag";
|
|
215
|
+
|
|
216
|
+
export const SUPPORT_MATRIX: Readonly<
|
|
217
|
+
Record<SupportArgument, Readonly<Record<EngineId, ArgumentSupport>>>
|
|
218
|
+
> = Object.freeze({
|
|
219
|
+
n: Object.freeze({ google: "yes", duckduckgo: "yes", yandex: "yes" }),
|
|
220
|
+
// DuckDuckGo has no pagination in this adapter: page > 1 is served as page 1
|
|
221
|
+
// and must be reported as `ignored.page`, never silently dropped (S4.2, S7.1).
|
|
222
|
+
page: Object.freeze({ google: "yes", duckduckgo: "flag", yandex: "yes" }),
|
|
223
|
+
summary: Object.freeze({ google: "yes", duckduckgo: "yes", yandex: "yes" }),
|
|
224
|
+
});
|
|
225
|
+
|
|
203
226
|
export interface EngineRegistry {
|
|
204
227
|
/** Throws E_UNSUPPORTED when the adapter is missing or disabled. */
|
|
205
228
|
get(id: EngineId): Engine;
|