web-access-cli 0.1.0

package/dist/providers/xcrawl.js

@@ -0,0 +1,170 @@
+import { redactText, WebAccessError } from "../core/errors.js";
+import { buildEndpoint } from "../transport/http.js";
+import { assertOk, freshnessStartDate, normalizeHits, parseJsonResponse, providerHeaders, ref, requireBaseUrl, requireCredential, searchQueryWithDomains, } from "./common.js";
+function isRecord(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function stringField(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+function responseMessage(value) {
+    for (const candidate of [
+        value.message,
+        value.error,
+        isRecord(value.data) ? value.data.message : undefined,
+        isRecord(value.data) ? value.data.error : undefined,
+    ]) {
+        if (typeof candidate === "string" && candidate.trim())
+            return candidate.trim().slice(0, 500);
+        if (isRecord(candidate)) {
+            const nested = stringField(candidate.message) ?? stringField(candidate.error);
+            if (nested)
+                return nested.slice(0, 500);
+        }
+    }
+    return "上游任务失败";
+}
+function assertCompleted(parsed, request) {
+    if (parsed.status === "completed")
+        return;
+    if (parsed.status === "failed") {
+        const message = redactText(responseMessage(parsed), [
+            request.instance.apiKey,
+        ]);
+        throw new WebAccessError("provider_error", `XCrawl 任务失败: ${message}`, {
+            provider: ref(request.instance),
+            retryable: true,
+            raw: parsed,
+        });
+    }
+    throw new WebAccessError("invalid_response", "XCrawl 返回未知任务状态", {
+        provider: ref(request.instance),
+        retryable: true,
+        raw: parsed,
+    });
+}
+function queryWithFilters(request) {
+    const parts = [
+        searchQueryWithDomains(request.query, request.includeDomains, request.excludeDomains),
+    ];
+    if (request.instance.searchFilterMode === "best_effort" && request.freshness)
+        parts.push(`after:${freshnessStartDate(request.freshness)}`);
+    return parts.join(" ");
+}
+function requestHeaders(request) {
+    const key = requireCredential(request.instance);
+    const headers = providerHeaders(request.instance, {
+        "Content-Type": "application/json",
+        Accept: "application/json",
+    });
+    for (const name of Object.keys(headers))
+        if (name.toLowerCase() === "authorization")
+            delete headers[name];
+    headers.Authorization = `Bearer ${key}`;
+    return headers;
+}
+function normalizedMarkdown(value) {
+    return value
+        .replace(/\r\n/g, "\n")
+        .replace(/[ \t]+\n/g, "\n")
+        .trim();
+}
+function httpUrl(value) {
+    if (typeof value !== "string")
+        return undefined;
+    try {
+        const parsed = new URL(value);
+        return parsed.protocol === "http:" || parsed.protocol === "https:"
+            ? parsed.toString()
+            : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+const xcrawl = {
+    type: "xcrawl",
+    capabilities: ["search", "extract"],
+    isConfigured: (instance) => !!instance.apiKey && !!instance.baseUrl,
+    async search(request) {
+        if (request.freshness &&
+            (request.instance.searchFilterMode ?? "strict") === "strict")
+            throw new WebAccessError("provider_unavailable", "XCrawl 严格模式不支持 freshness", { provider: ref(request.instance), retryable: true });
+        const response = await request.transport.request(buildEndpoint(requireBaseUrl(request.instance), "v1/search"), {
+            method: "POST",
+            headers: requestHeaders(request),
+            body: JSON.stringify({
+                query: queryWithFilters(request),
+                limit: request.includeDomains.length || request.excludeDomains.length
+                    ? 20
+                    : request.limit,
+            }),
+            signal: request.signal,
+            maxResponseBytes: request.maxResponseBytes,
+        });
+        assertOk(response, request.instance);
+        const parsed = parseJsonResponse(response, request.instance);
+        assertCompleted(parsed, request);
+        if (!isRecord(parsed.data) || !Array.isArray(parsed.data.data))
+            throw new WebAccessError("invalid_response", "XCrawl Search 返回缺少 data.data", {
+                provider: ref(request.instance),
+                retryable: true,
+                raw: parsed,
+            });
+        return {
+            data: {
+                results: normalizeHits(parsed.data.data, request.limit, request.includeDomains, request.excludeDomains),
+            },
+            raw: parsed,
+        };
+    },
+    async extract(request) {
+        const response = await request.transport.request(buildEndpoint(requireBaseUrl(request.instance), "v1/scrape"), {
+            method: "POST",
+            headers: requestHeaders(request),
+            body: JSON.stringify({
+                url: request.url,
+                mode: "sync",
+                output: { formats: ["markdown"] },
+            }),
+            signal: request.signal,
+            maxResponseBytes: request.maxResponseBytes,
+        });
+        assertOk(response, request.instance);
+        const parsed = parseJsonResponse(response, request.instance);
+        assertCompleted(parsed, request);
+        if (!isRecord(parsed.data))
+            throw new WebAccessError("invalid_response", "XCrawl Scrape 返回缺少 data", {
+                provider: ref(request.instance),
+                retryable: true,
+                raw: parsed,
+            });
+        const content = stringField(parsed.data.markdown);
+        if (!content)
+            throw new WebAccessError("no_usable_content", "XCrawl Scrape 没有返回 Markdown 正文", {
+                provider: ref(request.instance),
+                retryable: true,
+                raw: parsed,
+            });
+        const markdown = normalizedMarkdown(content);
+        const metadata = isRecord(parsed.data.metadata)
+            ? parsed.data.metadata
+            : undefined;
+        const title = stringField(metadata?.title) ??
+            /^#\s+(.+)$/m.exec(markdown)?.[1]?.trim() ??
+            "";
+        return {
+            data: {
+                document: {
+                    sourceUrl: httpUrl(metadata?.final_url) ?? httpUrl(parsed.url) ?? request.url,
+                    title,
+                    content: markdown,
+                    contentType: "text/markdown",
+                },
+            },
+            raw: parsed,
+        };
+    },
+};
+export const XCRAWL_ADAPTER = xcrawl;
+//# sourceMappingURL=xcrawl.js.map

package/dist/providers/xcrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"xcrawl.js","sourceRoot":"","sources":["../../src/providers/xcrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAS/D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EACN,QAAQ,EACR,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,GAAG,EACH,cAAc,EACd,iBAAiB,EACjB,sBAAsB,GACtB,MAAM,aAAa,CAAC;AAErB,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,CAAC;AAED,SAAS,eAAe,CAAC,KAA8B;IACtD,KAAK,MAAM,SAAS,IAAI;QACvB,KAAK,CAAC,OAAO;QACb,KAAK,CAAC,KAAK;QACX,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACrD,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KACnD,EAAE,CAAC;QACH,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE;YACpD,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACvC,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACzB,MAAM,MAAM,GACX,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChE,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CACvB,MAA+B,EAC/B,OAAqD;IAErD,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;QAAE,OAAO;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE;YACnD,OAAO,CAAC,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,OAAO,EAAE,EAAE;YACrE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC/B,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,MAAM;SACX,CAAC,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,cAAc,CAAC,kBAAkB,EAAE,iBAAiB,EAAE;QAC/D,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/B,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,MAAM;KACX,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAA6B;IACtD,MAAM,KAAK,GAAG;QACb,sBAAsB,CACrB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,cAAc,CACtB;KACD,CAAC;IACF,IAAI,OAAO,CAAC,QAAQ,CAAC,gBAAgB,KAAK,aAAa,IAAI,OAAO,CAAC,SAAS;QAC3E,KAAK,CAAC,IAAI,CAAC,SAAS,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAC9D,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,cAAc,CACtB,OAAqD;IAErD,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE;QACjD,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;KAC1B,CAAC,CAAC;IACH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QACtC,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,eAAe;YAAE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,OAAO,CAAC,aAAa,GAAG,UAAU,GAAG,EAAE,CAAC;IACxC,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,KAAK;SACV,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,IAAI,EAAE,CAAC;AACV,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;YACjE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;YACnB,CAAC,CAAC,SAAS,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,MAAM,MAAM,GAAoB;IAC/B,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;IACnC,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO;IACnE,KAAK,CAAC,MAAM,CAAC,OAAO;QACnB,IACC,OAAO,CAAC,SAAS;YACjB,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,KAAK,QAAQ;YAE5D,MAAM,IAAI,cAAc,CACvB,sBAAsB,EACtB,0BAA0B,EAC1B,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CACpD,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAC/C,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,EAC5D;YACC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC;YAChC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACpB,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC;gBAChC,KAAK,EACJ,OAAO,CAAC,cAAc,CAAC,MAAM,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM;oBAC7D,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,OAAO,CAAC,KAAK;aACjB,CAAC;YACF,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CACD,CAAC;QACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7D,MAAM,IAAI,cAAc,CACvB,kBAAkB,EAClB,8BAA8B,EAC9B;gBACC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC/B,SAAS,EAAE,IAAI;gBACf,GAAG,EAAE,MAAM;aACX,CACD,CAAC;QACH,OAAO;YACN,IAAI,EAAE;gBACL,OAAO,EAAE,aAAa,CACrB,MAAM,CAAC,IAAI,CAAC,IAAI,EAChB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,cAAc,CACtB;aACD;YACD,GAAG,EAAE,MAAM;SACX,CAAC;IACH,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,OAAO;QACpB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAC/C,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,EAC5D;YACC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC;YAChC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACpB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE;aACjC,CAAC;YACF,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CACD,CAAC;QACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;YACzB,MAAM,IAAI,cAAc,CACvB,kBAAkB,EAClB,yBAAyB,EACzB;gBACC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC/B,SAAS,EAAE,IAAI;gBACf,GAAG,EAAE,MAAM;aACX,CACD,CAAC;QACH,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO;YACX,MAAM,IAAI,cAAc,CACvB,mBAAmB,EACnB,gCAAgC,EAChC;gBACC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC/B,SAAS,EAAE,IAAI;gBACf,GAAG,EAAE,MAAM;aACX,CACD,CAAC;QACH,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC9C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ;YACtB,CAAC,CAAC,SAAS,CAAC;QACb,MAAM,KAAK,GACV,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;YACzC,EAAE,CAAC;QACJ,OAAO;YACN,IAAI,EAAE;gBACL,QAAQ,EAAE;oBACT,SAAS,EACR,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG;oBACnE,KAAK;oBACL,OAAO,EAAE,QAAQ;oBACjB,WAAW,EAAE,eAAe;iBAC5B;aACD;YACD,GAAG,EAAE,MAAM;SACX,CAAC;IACH,CAAC;CACD,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC"}

package/dist/transport/http.js

@@ -0,0 +1,175 @@
+import { EnvHttpProxyAgent, fetch as undiciFetch, } from "undici";
+import { WebAccessError } from "../core/errors.js";
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const SENSITIVE_HEADERS = new Set([
+    "authorization",
+    "cookie",
+    "proxy-authorization",
+    "x-api-key",
+    "x-subscription-token",
+    "x-auth-token",
+    "x-access-token",
+]);
+function isAbortLike(error, signal) {
+    if (signal.aborted)
+        return true;
+    if (!(error instanceof Error))
+        return false;
+    return (error.name === "AbortError" ||
+        error.name === "TimeoutError" ||
+        /aborted|timed out|timeout/i.test(error.message));
+}
+function stripSensitiveHeaders(headers) {
+    return Object.fromEntries(Object.entries(headers).filter(([name]) => !SENSITIVE_HEADERS.has(name.toLowerCase())));
+}
+function decodeBody(buffer, headers) {
+    const contentType = headers.get("content-type") ?? "";
+    const charset = contentType.match(/charset\s*=\s*["']?([^;"'\s]+)/i)?.[1] ?? "utf-8";
+    try {
+        return new TextDecoder(charset).decode(buffer);
+    }
+    catch {
+        return new TextDecoder("utf-8").decode(buffer);
+    }
+}
+async function readBody(response, maxBytes, signal) {
+    const reader = response.body?.getReader();
+    if (!reader) {
+        const buffer = new Uint8Array(await response.arrayBuffer());
+        if (buffer.byteLength > maxBytes)
+            throw new WebAccessError("response_too_large", `响应超过 ${maxBytes} 字节`, { retryable: true });
+        return decodeBody(buffer, response.headers);
+    }
+    const chunks = [];
+    let size = 0;
+    try {
+        while (true) {
+            if (signal.aborted)
+                throw new WebAccessError("aborted", "请求已取消");
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            size += value.byteLength;
+            if (size > maxBytes) {
+                await reader.cancel();
+                throw new WebAccessError("response_too_large", `响应超过 ${maxBytes} 字节`, { retryable: true });
+            }
+            chunks.push(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    const buffer = new Uint8Array(size);
+    let offset = 0;
+    for (const chunk of chunks) {
+        buffer.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return decodeBody(buffer, response.headers);
+}
+export function jsonBody(text, provider) {
+    try {
+        const value = JSON.parse(text);
+        if (!value || typeof value !== "object" || Array.isArray(value)) {
+            throw new Error("响应 JSON 根节点不是对象");
+        }
+        return value;
+    }
+    catch (error) {
+        throw new WebAccessError("invalid_response", `${provider} 返回了无效 JSON: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+export function buildEndpoint(baseUrl, path) {
+    const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
+    return new URL(path.replace(/^\//, ""), base).toString();
+}
+export class DefaultHttpTransport {
+    dispatcher = new EnvHttpProxyAgent();
+    async request(rawUrl, options) {
+        let current = typeof rawUrl === "string" ? new URL(rawUrl) : new URL(rawUrl.toString());
+        if (current.protocol !== "http:" && current.protocol !== "https:")
+            throw new WebAccessError("invalid_input", "只能请求 HTTP(S) URL");
+        let method = (options.method ?? "GET").toUpperCase();
+        let body = options.body;
+        let headers = { ...(options.headers ?? {}) };
+        const maxRedirects = options.maxRedirects ?? 5;
+        for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount++) {
+            let response;
+            try {
+                response = await undiciFetch(current, {
+                    method,
+                    ...(body !== undefined ? { body } : {}),
+                    headers,
+                    signal: options.signal,
+                    dispatcher: this.dispatcher,
+                });
+            }
+            catch (error) {
+                if (isAbortLike(error, options.signal)) {
+                    throw new WebAccessError(options.signal.aborted ? "aborted" : "timeout", "HTTP 请求超时或已取消", { retryable: options.signal.aborted === false });
+                }
+                throw new WebAccessError("network_error", error instanceof Error ? error.message : String(error), { retryable: true });
+            }
+            if (!REDIRECT_STATUSES.has(response.status)) {
+                const text = await readBody(response, options.maxResponseBytes, options.signal);
+                return {
+                    status: response.status,
+                    statusText: response.statusText,
+                    url: response.url || current.toString(),
+                    headers: response.headers,
+                    body: text,
+                };
+            }
+            const location = response.headers.get("location");
+            if (!location) {
+                const text = await readBody(response, options.maxResponseBytes, options.signal);
+                return {
+                    status: response.status,
+                    statusText: response.statusText,
+                    url: response.url || current.toString(),
+                    headers: response.headers,
+                    body: text,
+                };
+            }
+            await response.body?.cancel();
+            if (redirectCount === maxRedirects) {
+                throw new WebAccessError("provider_error", `重定向次数超过 ${maxRedirects}`, { retryable: true, httpStatus: response.status });
+            }
+            const next = new URL(location, current);
+            if (next.protocol !== "http:" && next.protocol !== "https:")
+                throw new WebAccessError("provider_error", "重定向目标不是 HTTP(S) URL", { retryable: false });
+            if (next.origin !== current.origin)
+                headers = stripSensitiveHeaders(headers);
+            if (response.status === 303 ||
+                ((response.status === 301 || response.status === 302) &&
+                    method === "POST")) {
+                method = "GET";
+                body = undefined;
+                delete headers["Content-Type"];
+                delete headers["content-type"];
+            }
+            current = next;
+        }
+        throw new WebAccessError("provider_error", "HTTP 重定向失败", {
+            retryable: true,
+        });
+    }
+}
+export function createAbortSignal(parent, timeoutMs) {
+    const timeout = AbortSignal.timeout(Math.max(1, timeoutMs));
+    return parent ? AbortSignal.any([parent, timeout]) : timeout;
+}
+export function mergeHeaders(...sources) {
+    const result = {};
+    for (const source of sources) {
+        if (!source)
+            continue;
+        for (const [name, value] of Object.entries(source))
+            result[name] = value;
+    }
+    return result;
+}
+//# sourceMappingURL=http.js.map

package/dist/transport/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/transport/http.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,iBAAiB,EAEjB,KAAK,IAAI,WAAW,GACpB,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC7D,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACjC,eAAe;IACf,QAAQ;IACR,qBAAqB;IACrB,WAAW;IACX,sBAAsB;IACtB,cAAc;IACd,gBAAgB;CAChB,CAAC,CAAC;AA0BH,SAAS,WAAW,CAAC,KAAc,EAAE,MAAmB;IACvD,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,CACN,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,cAAc;QAC7B,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAChD,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC7B,OAA+B;IAE/B,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAC7B,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CACtD,CACD,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAkB,EAAE,OAAsB;IAC7D,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,OAAO,GACZ,WAAW,CAAC,KAAK,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC;IACtE,IAAI,CAAC;QACJ,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC;AACF,CAAC;AAED,KAAK,UAAU,QAAQ,CACtB,QAIC,EACD,QAAgB,EAChB,MAAmB;IAEnB,MAAM,MAAM,GACX,QAAQ,CAAC,IAOT,EAAE,SAAS,EAAE,CAAC;IACf,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,UAAU,GAAG,QAAQ;YAC/B,MAAM,IAAI,cAAc,CACvB,oBAAoB,EACpB,QAAQ,QAAQ,KAAK,EACrB,EAAE,SAAS,EAAE,IAAI,EAAE,CACnB,CAAC;QACH,OAAO,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC;QACJ,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACjE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;YACzB,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;gBACrB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,IAAI,cAAc,CACvB,oBAAoB,EACpB,QAAQ,QAAQ,KAAK,EACrB,EAAE,SAAS,EAAE,IAAI,EAAE,CACnB,CAAC;YACH,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,WAAW,EAAE,CAAC;IACtB,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;IAC5B,CAAC;IACD,OAAO,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,QAAQ,CACvB,IAAY,EACZ,QAAgB;IAEhB,IAAI,CAAC;QACJ,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,KAAgC,CAAC;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,IAAI,cAAc,CACvB,kBAAkB,EAClB,GAAG,QAAQ,gBAAgB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACnF,CAAC;IACH,CAAC;AACF,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,IAAY;IAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC;IAC7D,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,OAAO,oBAAoB;IACf,UAAU,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAEtD,KAAK,CAAC,OAAO,CACZ,MAAoB,EACpB,OAA2B;QAE3B,IAAI,OAAO,GACV,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3E,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ;YAChE,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAC;QAC/D,IAAI,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACxB,IAAI,OAAO,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;QAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC;QAE/C,KACC,IAAI,aAAa,GAAG,CAAC,EACrB,aAAa,IAAI,YAAY,EAC7B,aAAa,EAAE,EACd,CAAC;YACF,IAAI,QAAiD,CAAC;YACtD,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE;oBACrC,MAAM;oBACN,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACvC,OAAO;oBACP,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,UAAU,EAAE,IAAI,CAAC,UAAU;iBAClB,CAAC,CAAC;YACb,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACxC,MAAM,IAAI,cAAc,CACvB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAC9C,eAAe,EACf,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,EAAE,CAC/C,CAAC;gBACH,CAAC;gBACD,MAAM,IAAI,cAAc,CACvB,eAAe,EACf,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EACtD,EAAE,SAAS,EAAE,IAAI,EAAE,CACnB,CAAC;YACH,CAAC;YAED,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAC1B,QAAQ,EACR,OAAO,CAAC,gBAAgB,EACxB,OAAO,CAAC,MAAM,CACd,CAAC;gBACF,OAAO;oBACN,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,QAAQ,EAAE;oBACvC,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,IAAI,EAAE,IAAI;iBACV,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,MAAM,QAAQ,CAC1B,QAAQ,EACR,OAAO,CAAC,gBAAgB,EACxB,OAAO,CAAC,MAAM,CACd,CAAC;gBACF,OAAO;oBACN,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,QAAQ,EAAE;oBACvC,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,IAAI,EAAE,IAAI;iBACV,CAAC;YACH,CAAC;YACD,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC9B,IAAI,aAAa,KAAK,YAAY,EAAE,CAAC;gBACpC,MAAM,IAAI,cAAc,CACvB,gBAAgB,EAChB,WAAW,YAAY,EAAE,EACzB,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,CAChD,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAC1D,MAAM,IAAI,cAAc,CACvB,gBAAgB,EAChB,qBAAqB,EACrB,EAAE,SAAS,EAAE,KAAK,EAAE,CACpB,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;gBACjC,OAAO,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;YAC1C,IACC,QAAQ,CAAC,MAAM,KAAK,GAAG;gBACvB,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;oBACpD,MAAM,KAAK,MAAM,CAAC,EAClB,CAAC;gBACF,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC;gBACjB,OAAO,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC/B,OAAO,OAAO,CAAC,cAAc,CAAC,CAAC;YAChC,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,YAAY,EAAE;YACxD,SAAS,EAAE,IAAI;SACf,CAAC,CAAC;IACJ,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,MAA+B,EAC/B,SAAiB;IAEjB,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC5D,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,YAAY,CAC3B,GAAG,OAAkD;IAErD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC"}

package/dist/version.js

@@ -0,0 +1,2 @@
+export const VERSION = "0.1.0";
+//# sourceMappingURL=version.js.map

package/dist/version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}

package/package.json

@@ -0,0 +1,61 @@
+{
+	"name": "web-access-cli",
+	"version": "0.1.0",
+	"description": "Agent-neutral web search and content extraction CLI",
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/Juexe/web-access-cli.git"
+	},
+	"homepage": "https://github.com/Juexe/web-access-cli#readme",
+	"bugs": {
+		"url": "https://github.com/Juexe/web-access-cli/issues"
+	},
+	"keywords": [
+		"cli",
+		"web-search",
+		"content-extraction",
+		"agent"
+	],
+	"type": "module",
+	"bin": {
+		"web-access": "./dist/cli.js"
+	},
+	"files": [
+		"dist",
+		"schemas",
+		"README.md",
+		"README_CN.md",
+		"LICENSE",
+		"THIRD_PARTY_NOTICES.md"
+	],
+	"engines": {
+		"node": ">=22.19.0"
+	},
+	"packageManager": "pnpm@11.16.0",
+	"scripts": {
+		"build": "tsc -p tsconfig.build.json && node scripts/write-schemas.mjs",
+		"typecheck": "tsc -p tsconfig.json --noEmit",
+		"test": "tsx --test test/**/*.test.ts",
+		"lint": "biome check .",
+		"format": "biome format --write .",
+		"check": "pnpm lint && pnpm typecheck && pnpm test && pnpm build",
+		"pack:check": "pnpm build && pnpm pack --pack-destination .artifacts"
+	},
+	"dependencies": {
+		"@mozilla/readability": "^0.6.0",
+		"commander": "^15.0.0",
+		"linkedom": "^0.18.12",
+		"open": "^11.0.0",
+		"turndown": "^7.2.0",
+		"typebox": "^1.3.11",
+		"undici": "^8.10.0"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "^2.5.7",
+		"@types/node": "^24.0.0",
+		"@types/turndown": "^5.0.6",
+		"tsx": "^4.20.0",
+		"typescript": "^5.9.0"
+	}
+}

package/schemas/config.schema.json

@@ -0,0 +1,210 @@
+{
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string",
+      "format": "uri-reference"
+    },
+    "providers": {
+      "type": "array",
+      "items": {
+        "anyOf": [
+          {
+            "type": "object",
+            "required": [
+              "id",
+              "type"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9_-]{0,63}$"
+              },
+              "apiKey": {
+                "type": "string"
+              },
+              "apiKeyEnv": {
+                "type": "string",
+                "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
+              },
+              "baseUrl": {
+                "type": "string",
+                "format": "uri"
+              },
+              "baseUrlEnv": {
+                "type": "string",
+                "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
+              },
+              "headers": {
+                "type": "object",
+                "patternProperties": {
+                  "^[^\\r\\n]+$": {
+                    "type": "string",
+                    "pattern": "^[^\\r\\n]*$"
+                  }
+                },
+                "additionalProperties": false
+              },
+              "type": {
+                "anyOf": [
+                  {
+                    "type": "string",
+                    "const": "tavily"
+                  },
+                  {
+                    "type": "string",
+                    "const": "exa"
+                  },
+                  {
+                    "type": "string",
+                    "const": "brave"
+                  },
+                  {
+                    "type": "string",
+                    "const": "searxng"
+                  },
+                  {
+                    "type": "string",
+                    "const": "firecrawl"
+                  },
+                  {
+                    "type": "string",
+                    "const": "jina"
+                  },
+                  {
+                    "type": "string",
+                    "const": "http"
+                  }
+                ]
+              }
+            },
+            "additionalProperties": false
+          },
+          {
+            "type": "object",
+            "required": [
+              "id",
+              "type"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9_-]{0,63}$"
+              },
+              "apiKey": {
+                "type": "string"
+              },
+              "apiKeyEnv": {
+                "type": "string",
+                "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
+              },
+              "baseUrl": {
+                "type": "string",
+                "format": "uri"
+              },
+              "baseUrlEnv": {
+                "type": "string",
+                "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
+              },
+              "headers": {
+                "type": "object",
+                "patternProperties": {
+                  "^[^\\r\\n]+$": {
+                    "type": "string",
+                    "pattern": "^[^\\r\\n]*$"
+                  }
+                },
+                "additionalProperties": false
+              },
+              "type": {
+                "anyOf": [
+                  {
+                    "type": "string",
+                    "const": "anysearch"
+                  },
+                  {
+                    "type": "string",
+                    "const": "xcrawl"
+                  }
+                ]
+              },
+              "searchFilterMode": {
+                "anyOf": [
+                  {
+                    "type": "string",
+                    "const": "strict"
+                  },
+                  {
+                    "type": "string",
+                    "const": "best_effort"
+                  }
+                ]
+              }
+            },
+            "additionalProperties": false
+          }
+        ]
+      }
+    },
+    "search": {
+      "type": "object",
+      "properties": {
+        "providers": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9_-]{0,63}$"
+          }
+        },
+        "limit": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 20
+        },
+        "timeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "attemptTimeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "maxResponseBytes": {
+          "type": "integer",
+          "minimum": 1
+        }
+      },
+      "additionalProperties": false
+    },
+    "extract": {
+      "type": "object",
+      "properties": {
+        "providers": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9_-]{0,63}$"
+          }
+        },
+        "timeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "attemptTimeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "maxResponseBytes": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "minContentCharacters": {
+          "type": "integer",
+          "minimum": 1
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/extractRequest.schema.json

@@ -0,0 +1,21 @@
+{
+  "type": "object",
+  "required": [
+    "url",
+    "provider"
+  ],
+  "properties": {
+    "url": {
+      "type": "string",
+      "format": "uri"
+    },
+    "provider": {
+      "type": "string",
+      "minLength": 1
+    },
+    "timeoutMs": {
+      "type": "integer",
+      "minimum": 1
+    }
+  }
+}