web-access-cli 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -15
- package/README_CN.md +24 -15
- package/dist/cli.js +17 -0
- package/dist/cli.js.map +1 -1
- package/dist/config/config.js +35 -46
- package/dist/config/config.js.map +1 -1
- package/dist/config/file.js +30 -0
- package/dist/config/file.js.map +1 -1
- package/dist/config/init.js +59 -0
- package/dist/config/init.js.map +1 -0
- package/dist/config/provider-order.js +14 -7
- package/dist/config/provider-order.js.map +1 -1
- package/dist/core/diagnostics.js +13 -7
- package/dist/core/diagnostics.js.map +1 -1
- package/dist/core/router.js +228 -94
- package/dist/core/router.js.map +1 -1
- package/dist/core/schema.js +18 -2
- package/dist/core/schema.js.map +1 -1
- package/dist/core/types.js +1 -0
- package/dist/core/types.js.map +1 -1
- package/dist/providers/extract.js +37 -0
- package/dist/providers/extract.js.map +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
- package/schemas/config.schema.json +14 -2
- package/schemas/output.schema.json +76 -0
package/README.md
CHANGED
|
@@ -16,7 +16,7 @@ The CLI is the primary product interface. It is not tied to Pi, Claude Code, Cod
|
|
|
16
16
|
| `search` | Tavily, Exa, Bocha, Brave, SearXNG, AnySearch, XCrawl, DeepSeek, xAI x_search, xAI web_search | `rank`, `title`, `url`, `snippet` |
|
|
17
17
|
| `extract` | Firecrawl v2, Jina Reader, Exa Contents, AnySearch, XCrawl, HTTP | Markdown `Document` |
|
|
18
18
|
|
|
19
|
-
A provider type describes an implementation, while a provider instance is a configurable instance of that type. One type can have multiple instances, such as `exa_team` and `exa_personal`.
|
|
19
|
+
A provider type describes an implementation, while a provider instance is a configurable instance of that type. One type can have multiple instances, such as `exa_team` and `exa_personal`. Each capability has a primary `providers` route and a `providers_fallback` route. Both arrays determine enabled instances and their current `auto` order; the CLI only reorders members within their existing group.
|
|
20
20
|
|
|
21
21
|
## Installation
|
|
22
22
|
|
|
@@ -89,9 +89,11 @@ web-access doctor
|
|
|
89
89
|
|
|
90
90
|
web-access config edit
|
|
91
91
|
web-access --config "/path/to/config.json" config edit
|
|
92
|
+
web-access config init
|
|
93
|
+
web-access --config "/path/to/config.json" config init --json
|
|
92
94
|
```
|
|
93
95
|
|
|
94
|
-
The CLI provides the `search` and `extract` capability commands, the `providers` and `doctor` diagnostic commands, and the `config edit` configuration
|
|
96
|
+
The CLI provides the `search` and `extract` capability commands, the `providers` and `doctor` diagnostic commands, and the `config init` and `config edit` configuration commands. The current version does not provide batch/all, answer, a dedicated PDF path, a Node.js SDK, or a generic MCP integration. The repository includes an optional [web-access-cli Agent Skill](skills/web-access-cli/SKILL.md) for source-checkout use; it is not included in the npm package.
|
|
95
97
|
|
|
96
98
|
### Global options
|
|
97
99
|
|
|
@@ -99,7 +101,7 @@ The CLI provides the `search` and `extract` capability commands, the `providers`
|
|
|
99
101
|
- `--json`: Output a compact schema v2 JSON envelope. Without it, successful commands write human-readable Markdown; all failures always write JSON.
|
|
100
102
|
- `--help`, `--version`: Print standard CLI help or version text.
|
|
101
103
|
|
|
102
|
-
Successful commands default to Markdown for human readers. `search` writes a YAML front matter block followed by a numbered result list; `extract` writes YAML front matter followed by the normalized Markdown body; `providers`, `doctor`, and `config edit` write a heading followed by pretty-printed JSON data. Add `--json` when a schema v2 envelope is required. Extract `partial` results and all failures, including invalid input and runtime errors, always remain JSON envelopes. Diagnostic messages are never mixed into stdout.
|
|
104
|
+
Successful commands default to Markdown for human readers. `search` writes a YAML front matter block followed by a numbered result list; `extract` writes YAML front matter followed by the normalized Markdown body; `providers`, `doctor`, and `config edit` write a heading followed by pretty-printed JSON data; `config init` writes the created absolute path on one line. Add `--json` when a schema v2 envelope is required. Extract `partial` results and all failures, including invalid input and runtime errors, always remain JSON envelopes. Diagnostic messages are never mixed into stdout.
|
|
103
105
|
|
|
104
106
|
## Configuration
|
|
105
107
|
|
|
@@ -109,13 +111,13 @@ The default path on every platform is `~/.config/web-access-cli/config.json`, wh
|
|
|
109
111
|
|
|
110
112
|
`web-access config edit` creates parent directories and a complete default configuration when the file is missing, then opens it with the command in `VISUAL`, or `EDITOR` when `VISUAL` is empty. The value may contain a quoted executable path and arguments; it is parsed without shell evaluation and the configuration path is appended as a separate argument. An existing file is opened byte-for-byte as-is, even when it is temporarily invalid JSON; it is never overwritten or reformatted. The command waits only for the editor process to start, not for it to close. When neither variable is set, the command returns `open_failed`; a configuration file that was created successfully remains available for manual editing.
|
|
111
113
|
|
|
112
|
-
`config
|
|
114
|
+
`config init` creates a new configuration from standard credential environment variables and refuses to overwrite an existing file. It keeps all built-in instance definitions, writes detected Search providers to the primary route, leaves Search fallback empty, writes detected remote Extract providers to the primary route, and places `http` in the Extract fallback route. After an `auto` capability call succeeds or encounters fallback-eligible failures, the CLI atomically reorders the matching `providers` or `providers_fallback` array in place. Members never move between groups. The legacy `_providers` and `_providers_fallback` fields are rejected as unknown configuration fields.
|
|
113
115
|
|
|
114
116
|
The complete JSON Schema is available at [schemas/config.schema.json](schemas/config.schema.json). Example:
|
|
115
117
|
|
|
116
118
|
```json
|
|
117
119
|
{
|
|
118
|
-
"$schema": "https://unpkg.com/web-access-cli@0.
|
|
120
|
+
"$schema": "https://unpkg.com/web-access-cli@0.5.0/schemas/config.schema.json",
|
|
119
121
|
"providers": [
|
|
120
122
|
{
|
|
121
123
|
"id": "exa_team",
|
|
@@ -138,13 +140,15 @@ The complete JSON Schema is available at [schemas/config.schema.json](schemas/co
|
|
|
138
140
|
],
|
|
139
141
|
"search": {
|
|
140
142
|
"providers": ["searx_local", "exa_team", "brave"],
|
|
143
|
+
"providers_fallback": ["deepseek"],
|
|
141
144
|
"limit": 5,
|
|
142
145
|
"timeoutMs": 120000,
|
|
143
146
|
"attemptTimeoutMs": 60000,
|
|
144
147
|
"maxResponseBytes": 5242880
|
|
145
148
|
},
|
|
146
149
|
"extract": {
|
|
147
|
-
"providers": ["firecrawl_local", "jina", "exa_team"
|
|
150
|
+
"providers": ["firecrawl_local", "jina", "exa_team"],
|
|
151
|
+
"providers_fallback": ["http"],
|
|
148
152
|
"timeoutMs": 120000,
|
|
149
153
|
"attemptTimeoutMs": 45000,
|
|
150
154
|
"maxResponseBytes": 5242880,
|
|
@@ -153,16 +157,18 @@ The complete JSON Schema is available at [schemas/config.schema.json](schemas/co
|
|
|
153
157
|
}
|
|
154
158
|
```
|
|
155
159
|
|
|
156
|
-
The built-in instances are `tavily`, `exa`, `bocha`, `brave`, `searxng`, `firecrawl`, `jina`, `http`, `anysearch`, `xcrawl`, `deepseek`, `xai_x_search`, and `xai_web_search`. A configuration entry with the same ID overrides fields on the built-in instance. A custom ID creates another instance of the selected type. An instance is enabled
|
|
160
|
+
The built-in instances are `tavily`, `exa`, `bocha`, `brave`, `searxng`, `firecrawl`, `jina`, `http`, `anysearch`, `xcrawl`, `deepseek`, `xai_x_search`, and `xai_web_search`. A configuration entry with the same ID overrides fields on the built-in instance. A custom ID creates another instance of the selected type. An instance is enabled when it appears in either corresponding route. Bocha defaults to `https://api.bocha.cn` and requires an API key; AnySearch defaults to `https://api.anysearch.com` and supports anonymous calls; XCrawl defaults to `https://run.xcrawl.com` and requires an API key; DeepSeek defaults to `https://api.deepseek.com/anthropic/v1` and requires an API key.
|
|
157
161
|
|
|
158
162
|
Default routes:
|
|
159
163
|
|
|
160
|
-
- Search: `tavily -> exa -> bocha -> brave -> searxng -> anysearch -> xcrawl
|
|
161
|
-
-
|
|
164
|
+
- Search primary: `tavily -> exa -> bocha -> brave -> searxng -> anysearch -> xcrawl`
|
|
165
|
+
- Search fallback: `deepseek -> xai_x_search -> xai_web_search`
|
|
166
|
+
- Extract primary: `firecrawl -> jina -> exa -> anysearch -> xcrawl`
|
|
167
|
+
- Extract fallback: `http`
|
|
162
168
|
|
|
163
|
-
The default routes
|
|
169
|
+
The default routes split built-in providers into a primary group and a fallback group. Custom instance IDs are merged into the instance list but must still be added to a route explicitly. An omitted capability route uses the defaults above; an explicit Search primary empty array is invalid, while Extract may be explicitly disabled with an empty primary route. In `auto` mode, incompletely configured instances are skipped. AnySearch and XCrawl accept `searchFilterMode`: `strict` (default; freshness skips the provider) or `best_effort` (rewrites freshness into a query fragment). Domain constraints are rewritten into the query and strictly re-applied locally. XCrawl Extract always uses synchronous Scrape with Markdown output; Map, Crawl, and asynchronous jobs are outside the current CLI capabilities.
|
|
164
170
|
|
|
165
|
-
DeepSeek Search performs a full Anthropic-compatible Messages model turn with the native `web_search_20250305` server tool, so it can have higher latency and cost than a dedicated search endpoint. It is
|
|
171
|
+
DeepSeek Search performs a full Anthropic-compatible Messages model turn with the native `web_search_20250305` server tool, so it can have higher latency and cost than a dedicated search endpoint. It is in the default Search fallback group and is reached only after every primary provider is unavailable, returns a final non-2xx HTTP response, or otherwise fails recoverably. The adapter accepts URLs only from structured `web_search_tool_result` blocks, joins citation excerpts by URL, and never extracts URLs from model prose. Domain constraints are rewritten into the query and strictly re-applied locally; `freshness` is unsupported and skips DeepSeek with a recoverable error. Redirects are rejected without contacting the `Location` target. Provider-private `encrypted_content` payloads are omitted from `raw` because they are opaque, cannot be displayed or decoded by the CLI, and add no diagnostic value.
|
|
166
172
|
|
|
167
173
|
XAI hosted Search reads an external OAuth JSON file from `XAI_AUTH_JSON` on every call and uses only its `access_token`; an external CLIProxyAPI process owns refresh and write-back. Instance fields `authJson`/`authJsonEnv` and `model`/`modelEnv` override the path and model. `xai_web_search` accepts at most five allow or exclude domains (mutually exclusive); `xai_x_search` rejects `freshness`. Model-backed search can take longer than ordinary API searches; Search defaults allow up to 60 seconds per attempt and 120 seconds total.
|
|
168
174
|
|
|
@@ -189,15 +195,17 @@ Custom instances use `apiKeyEnv` and `baseUrlEnv` to name their environment vari
|
|
|
189
195
|
|
|
190
196
|
## Execution and fallback
|
|
191
197
|
|
|
192
|
-
`--provider auto` follows the
|
|
198
|
+
`--provider auto` follows the configured `providers` order first and enters `providers_fallback` only after the complete primary group fails with a fallback-eligible result:
|
|
193
199
|
|
|
194
200
|
1. An instance with incomplete configuration is recorded as a failed attempt and skipped.
|
|
195
201
|
2. A final provider HTTP response outside 2xx, including authentication, request, rate-limit, and server errors, causes the router to try the next instance.
|
|
196
202
|
3. Other recoverable errors, including network errors, timeouts, oversized responses, and missing usable content, also cause the router to continue.
|
|
197
203
|
4. Non-recoverable failures without a final non-2xx response, including invalid input and unsupported content detected in a successful response, stop execution immediately.
|
|
198
|
-
5. If every instance fails, the command returns `provider_exhausted`. The best short content produced during extraction is preserved in `partial`.
|
|
204
|
+
5. If every instance in both groups fails, the command returns `provider_exhausted`. The best short content produced during extraction is preserved in `partial`.
|
|
199
205
|
|
|
200
|
-
|
|
206
|
+
Before those routes run, `extract --provider auto` checks one Markdown variant when an HTTP instance is enabled in either group. It appends `.md` to file-like paths or uses `index.md` for directory paths, requests `text/markdown` and `text/plain`, and accepts only a sufficiently long non-HTML text response. A miss or request failure is transparent and falls through to the configured routes. A hit is reported as the enabled HTTP instance; it does not change provider order. Explicit non-HTTP selections are unchanged, while an explicit HTTP selection checks the Markdown variant before fetching the original URL.
|
|
207
|
+
|
|
208
|
+
After each `auto` call, a successful instance moves to the front of its group, untried instances remain in the middle, and instances with the fallback-eligible failures above move stably to the back. Relative order within each group is preserved; no provider crosses between primary and fallback. The order stays unchanged when every instance in a group fails. Search and Extract learn independently; explicit instances, non-fallback failures, and user cancellation never update the order. Writes use atomic replacement, and the last writer wins across concurrent processes.
|
|
201
209
|
|
|
202
210
|
An attempt keeps the provider's original error code, HTTP status, and `retryable` value. `retryable` describes whether the original operation can be retried against the same provider; it is not the sole condition for switching to the next provider in an automatic route.
|
|
203
211
|
|
|
@@ -252,7 +260,7 @@ Exit codes:
|
|
|
252
260
|
- `1`: Runtime, provider, doctor, or editor launch failure
|
|
253
261
|
- `130`: User cancellation
|
|
254
262
|
|
|
255
|
-
`providers` lists each instance's type, capabilities, route status, credential source, and base URL source. Its `searchRoute` and `extractRoute` fields show the effective
|
|
263
|
+
`providers` lists each instance's type, capabilities, route status, credential source, and base URL source. Its `searchRoute`/`searchFallbackRoute` and `extractRoute`/`extractFallbackRoute` fields show the two effective orders for the next `auto` call. `doctor` performs local configuration checks only and does not call remote APIs. If an enabled route contains an unconfigured instance, the command returns `doctor_failed` with exit code `1`.
|
|
256
264
|
|
|
257
265
|
## HTTP and security boundaries
|
|
258
266
|
|
|
@@ -260,6 +268,7 @@ Exit codes:
|
|
|
260
268
|
- Follows up to five redirects. Cross-origin redirects remove authentication, Cookie, and common token headers.
|
|
261
269
|
- Enforces a streaming byte limit on every response instead of buffering an unbounded response first.
|
|
262
270
|
- HTTP Extract uses LinkeDOM, Mozilla Readability, and Turndown, with fallback parsing for Next.js RSC payloads.
|
|
271
|
+
- HTTP Extract prefers an enabled source Markdown variant (`.md` or directory `index.md`) before provider routes when that variant returns usable Markdown.
|
|
263
272
|
- Returns `unsupported_content` for PDF, image, audio, video, zip, and general binary content.
|
|
264
273
|
- By design, the CLI does not block localhost, private network addresses, or cloud metadata URLs. For untrusted input, callers must enforce their own URL allowlist, network isolation, or outbound proxy policy.
|
|
265
274
|
|
package/README_CN.md
CHANGED
|
@@ -16,7 +16,7 @@ CLI 是主要产品形态,不绑定 Pi、Claude Code、Codex、Cursor、OpenCo
|
|
|
16
16
|
| `search` | Tavily、Exa、Bocha、Brave、SearXNG、AnySearch、XCrawl、DeepSeek、xAI x_search、xAI web_search | `rank`、`title`、`url`、`snippet` |
|
|
17
17
|
| `extract` | Firecrawl v2、Jina Reader、Exa Contents、AnySearch、XCrawl、HTTP | Markdown `Document` |
|
|
18
18
|
|
|
19
|
-
Provider Type 描述实现类型;Provider Instance 是一份可配置实例。一个 Type 可以有多个 Instance,例如 `exa_team` 和 `exa_personal
|
|
19
|
+
Provider Type 描述实现类型;Provider Instance 是一份可配置实例。一个 Type 可以有多个 Instance,例如 `exa_team` 和 `exa_personal`。每个能力都有主 `providers` Route 和兜底 `providers_fallback` Route;两组数组同时决定启用状态和 `auto` 顺序,CLI 只在原分组内部调整成员顺序。
|
|
20
20
|
|
|
21
21
|
## 安装
|
|
22
22
|
|
|
@@ -89,9 +89,11 @@ web-access doctor
|
|
|
89
89
|
|
|
90
90
|
web-access config edit
|
|
91
91
|
web-access --config "/path/to/config.json" config edit
|
|
92
|
+
web-access config init
|
|
93
|
+
web-access --config "/path/to/config.json" config init --json
|
|
92
94
|
```
|
|
93
95
|
|
|
94
|
-
CLI 提供 `search`、`extract` 两个能力命令,`providers`、`doctor` 两个诊断命令,以及 `config edit` 配置维护命令。当前版本不提供 batch/all、answer、PDF 专线、Node SDK 或通用 MCP 集成。仓库提供可选的 [web-access-cli Agent Skill](skills/web-access-cli/SKILL.md),仅供源码仓库使用,不包含在 npm 发布包中。
|
|
96
|
+
CLI 提供 `search`、`extract` 两个能力命令,`providers`、`doctor` 两个诊断命令,以及 `config init`、`config edit` 配置维护命令。当前版本不提供 batch/all、answer、PDF 专线、Node SDK 或通用 MCP 集成。仓库提供可选的 [web-access-cli Agent Skill](skills/web-access-cli/SKILL.md),仅供源码仓库使用,不包含在 npm 发布包中。
|
|
95
97
|
|
|
96
98
|
### 通用选项
|
|
97
99
|
|
|
@@ -99,7 +101,7 @@ CLI 提供 `search`、`extract` 两个能力命令,`providers`、`doctor` 两
|
|
|
99
101
|
- `--json`:输出紧凑 schema v2 JSON envelope;不指定时,成功命令输出面向人的 Markdown,失败始终输出 JSON。
|
|
100
102
|
- `--help`、`--version`:输出常规 CLI 帮助或版本文本。
|
|
101
103
|
|
|
102
|
-
成功命令默认输出面向人的 Markdown。`search` 输出 YAML front matter 和编号结果列表;`extract` 输出 YAML front matter 和规范化后的 Markdown 正文;`providers`、`doctor` 和 `config edit` 输出标题和格式化 JSON
|
|
104
|
+
成功命令默认输出面向人的 Markdown。`search` 输出 YAML front matter 和编号结果列表;`extract` 输出 YAML front matter 和规范化后的 Markdown 正文;`providers`、`doctor` 和 `config edit` 输出标题和格式化 JSON 数据;`config init` 输出新配置的绝对路径。需要 schema v2 envelope 时显式追加 `--json`。提取失败(包括 `partial`)以及所有失败路径始终输出 JSON envelope。诊断信息不会混入 stdout。
|
|
103
105
|
|
|
104
106
|
## 配置
|
|
105
107
|
|
|
@@ -109,13 +111,13 @@ CLI 提供 `search`、`extract` 两个能力命令,`providers`、`doctor` 两
|
|
|
109
111
|
|
|
110
112
|
`web-access config edit` 会在配置文件缺失时创建父目录和完整默认配置,再优先使用 `VISUAL`、其次使用 `EDITOR` 打开;变量值可以包含带引号的可执行文件路径和参数,解析时不会执行 shell,并将配置路径作为独立的最后一个参数传入。已有文件会原样打开,即使内容暂时不是有效 JSON 也不会被覆盖或格式化。命令只等待编辑器进程启动,不等待编辑器关闭。两个变量都未设置、格式无效或启动失败时返回 `open_failed`;如果配置刚刚创建成功,文件仍会保留以便手动编辑。
|
|
111
113
|
|
|
112
|
-
`config
|
|
114
|
+
`config init` 按标准凭据环境变量创建新配置,已存在文件时拒绝覆盖。它保留全部内置 Instance 定义,将检测到的 Search provider 写入主 Route,Search fallback 留空,将检测到的远程 Extract provider 写入主 Route,并把 `http` 放入 Extract fallback。能力命令使用 `auto` 成功或发生可回退失败后,会原子重排对应的 `providers` 或 `providers_fallback` 数组;成员不会跨组移动。旧的 `_providers` 和 `_providers_fallback` 字段会被严格配置校验拒绝。
|
|
113
115
|
|
|
114
116
|
完整 JSON Schema 位于 [schemas/config.schema.json](schemas/config.schema.json)。示例:
|
|
115
117
|
|
|
116
118
|
```json
|
|
117
119
|
{
|
|
118
|
-
"$schema": "https://unpkg.com/web-access-cli@0.
|
|
120
|
+
"$schema": "https://unpkg.com/web-access-cli@0.5.0/schemas/config.schema.json",
|
|
119
121
|
"providers": [
|
|
120
122
|
{
|
|
121
123
|
"id": "exa_team",
|
|
@@ -138,13 +140,15 @@ CLI 提供 `search`、`extract` 两个能力命令,`providers`、`doctor` 两
|
|
|
138
140
|
],
|
|
139
141
|
"search": {
|
|
140
142
|
"providers": ["searx_local", "exa_team", "brave"],
|
|
143
|
+
"providers_fallback": ["deepseek"],
|
|
141
144
|
"limit": 5,
|
|
142
145
|
"timeoutMs": 120000,
|
|
143
146
|
"attemptTimeoutMs": 60000,
|
|
144
147
|
"maxResponseBytes": 5242880
|
|
145
148
|
},
|
|
146
149
|
"extract": {
|
|
147
|
-
"providers": ["firecrawl_local", "jina", "exa_team"
|
|
150
|
+
"providers": ["firecrawl_local", "jina", "exa_team"],
|
|
151
|
+
"providers_fallback": ["http"],
|
|
148
152
|
"timeoutMs": 120000,
|
|
149
153
|
"attemptTimeoutMs": 45000,
|
|
150
154
|
"maxResponseBytes": 5242880,
|
|
@@ -153,16 +157,18 @@ CLI 提供 `search`、`extract` 两个能力命令,`providers`、`doctor` 两
|
|
|
153
157
|
}
|
|
154
158
|
```
|
|
155
159
|
|
|
156
|
-
内置 Instance 为 `tavily`、`exa`、`bocha`、`brave`、`searxng`、`firecrawl`、`jina`、`http`、`anysearch`、`xcrawl`、`deepseek`、`xai_x_search`、`xai_web_search`。配置同 ID 时会覆盖内置实例的字段;自定义 ID 可以创建同 Type
|
|
160
|
+
内置 Instance 为 `tavily`、`exa`、`bocha`、`brave`、`searxng`、`firecrawl`、`jina`、`http`、`anysearch`、`xcrawl`、`deepseek`、`xai_x_search`、`xai_web_search`。配置同 ID 时会覆盖内置实例的字段;自定义 ID 可以创建同 Type 的额外实例。出现在对应主 Route 或 fallback Route 中的实例才启用。Bocha 默认 base URL 为 `https://api.bocha.cn`,必须配置 API key;AnySearch 默认 base URL 为 `https://api.anysearch.com`,支持匿名调用;XCrawl 默认 base URL 为 `https://run.xcrawl.com`,必须配置 API key;DeepSeek 默认 base URL 为 `https://api.deepseek.com/anthropic/v1`,必须配置 API key。
|
|
157
161
|
|
|
158
162
|
默认 Route:
|
|
159
163
|
|
|
160
|
-
- Search
|
|
161
|
-
-
|
|
164
|
+
- Search 主轮:`tavily -> exa -> bocha -> brave -> searxng -> anysearch -> xcrawl`
|
|
165
|
+
- Search fallback:`deepseek -> xai_x_search -> xai_web_search`
|
|
166
|
+
- Extract 主轮:`firecrawl -> jina -> exa -> anysearch -> xcrawl`
|
|
167
|
+
- Extract fallback:`http`
|
|
162
168
|
|
|
163
|
-
默认 Route
|
|
169
|
+
默认 Route 将内置 provider 分为主轮和 fallback。自定义 ID 会合并到 Instance 列表,但仍需显式加入 Route;省略能力 Route 时使用上述默认分组,Search 主 Route 显式空数组会报错,Extract 主 Route 可以显式为空以禁用 Extract。`auto` 会跳过未完成配置的 Instance。AnySearch 与 XCrawl 可设置 `searchFilterMode`:`strict`(默认,遇到 freshness 时跳过)或 `best_effort`(将日期改写为查询片段)。域名条件会改写查询并在本地再次严格过滤。XCrawl Extract 固定使用同步 Scrape 的 Markdown 输出;Map、Crawl 和异步任务不属于当前 CLI 能力。
|
|
164
170
|
|
|
165
|
-
DeepSeek Search 通过 Anthropic-compatible Messages API 调用原生 `web_search_20250305` server tool,一次搜索是完整模型轮次,因此延迟和成本可能高于专用搜索 endpoint。它位于默认
|
|
171
|
+
DeepSeek Search 通过 Anthropic-compatible Messages API 调用原生 `web_search_20250305` server tool,一次搜索是完整模型轮次,因此延迟和成本可能高于专用搜索 endpoint。它位于默认 Search fallback,仅在全部主轮 Provider 未配置、返回最终非 2xx HTTP 响应或发生其他可恢复失败后触发。Adapter 只接受 `web_search_tool_result` 中的结构化 URL,按 URL 合并 citation 摘要,绝不从模型 prose 中猜测 URL。域名条件会改写查询并在本地再次严格过滤;DeepSeek 不支持 `freshness`,遇到该参数时以可恢复错误跳过;重定向会被严格拒绝,且不会访问 `Location` 目标。Provider 私有的 `encrypted_content` 是 CLI 无法展示或解码的 opaque payload,不具备诊断价值,因此会从 `raw` 中移除。
|
|
166
172
|
|
|
167
173
|
XAI hosted Search 使用 `XAI_AUTH_JSON` 指向外部 OAuth JSON 文件,每次调用只读取 `access_token`,由外部 CLIProxyAPI 负责刷新和回写。可用 `authJson`/`authJsonEnv` 与 `model`/`modelEnv` 为实例覆盖路径和模型。`xai_web_search` 支持最多 5 个 allow 或 exclude 域名(两者不能同时使用);`xai_x_search` 不接受 `freshness`。模型搜索可能需要更长时间;Search 默认单次超时 60 秒、总超时 120 秒。
|
|
168
174
|
|
|
@@ -189,15 +195,17 @@ XAI hosted Search 使用 `XAI_AUTH_JSON` 指向外部 OAuth JSON 文件,每次
|
|
|
189
195
|
|
|
190
196
|
## 执行与回退
|
|
191
197
|
|
|
192
|
-
`--provider auto`
|
|
198
|
+
`--provider auto` 先按 `providers` 顺序执行;只有主轮全部失败且结果可回退时才进入 `providers_fallback`:
|
|
193
199
|
|
|
194
200
|
1. 未完成配置的实例会记录为失败 attempt 并跳过。
|
|
195
201
|
2. Provider 返回的最终 HTTP 响应不在 2xx 范围时,包括鉴权、请求、限流和服务器错误,Router 会尝试下一个实例。
|
|
196
202
|
3. 网络错误、超时、响应过大、无可用正文等其他可恢复错误也会继续 Route。
|
|
197
203
|
4. 没有最终非 2xx 响应的不可恢复错误,例如无效输入或在成功响应中检测到不支持的内容,会立即停止。
|
|
198
|
-
5.
|
|
204
|
+
5. 两组实例全部失败时返回 `provider_exhausted`。提取过程中产生的最佳短正文会保留在 `partial`。
|
|
199
205
|
|
|
200
|
-
|
|
206
|
+
执行这些 Route 前,`extract --provider auto` 会在任一组启用了 HTTP Instance 时先探测一个源站 Markdown 变体:文件型路径追加 `.md`,目录路径使用 `index.md`,请求 `text/markdown`/`text/plain`,只接受足够长且非 HTML 的文本响应。未命中或请求失败时不产生额外 attempt,并继续原有 Route;命中时结果归属已启用的 HTTP Instance,且不改变 Provider 顺序。显式非 HTTP 选择不变;显式 HTTP 会先探测 Markdown,未命中后再请求原 URL。
|
|
207
|
+
|
|
208
|
+
每次 `auto` 调用完成后,成功 Instance 移到所在组队头,未尝试 Instance 保持在中间,发生上述可回退失败的 Instance 稳定移到所在组队尾;顺序学习不会跨组移动 provider。所有 Instance 都失败时顺序不变。Search 与 Extract 独立学习;显式 Instance、不可回退错误和用户取消不会更新顺序。写入采用原子替换,并发进程由最后写入者生效。
|
|
201
209
|
|
|
202
210
|
Attempt 会保留 Provider 的原始错误码、HTTP status 和 `retryable` 值。`retryable` 表示原始操作是否适合对同一 Provider 重试,不再是自动 Route 切换到下一 Provider 的唯一条件。
|
|
203
211
|
|
|
@@ -252,7 +260,7 @@ title: "Example title"
|
|
|
252
260
|
- `1`:运行时/provider/doctor/编辑器启动失败
|
|
253
261
|
- `130`:用户取消
|
|
254
262
|
|
|
255
|
-
`providers` 会列出每个 Instance 的 Type、能力、Route 启用状态、凭据来源和 base URL 来源;`searchRoute` 与 `extractRoute` 是下一次 `auto`
|
|
263
|
+
`providers` 会列出每个 Instance 的 Type、能力、Route 启用状态、凭据来源和 base URL 来源;`searchRoute`/`searchFallbackRoute` 与 `extractRoute`/`extractFallbackRoute` 是下一次 `auto` 使用的两组有效顺序。`doctor` 只做本地配置检查,不主动调用远端 API;当已启用 Route 中存在未配置 Instance 时,命令返回 `doctor_failed` 和退出码 `1`。
|
|
256
264
|
|
|
257
265
|
## HTTP 与安全边界
|
|
258
266
|
|
|
@@ -260,6 +268,7 @@ title: "Example title"
|
|
|
260
268
|
- 最多跟随 5 次重定向;跨 origin 重定向会移除鉴权、Cookie 和常见 token headers。
|
|
261
269
|
- 所有响应都执行流式字节上限,避免先完整缓冲超大响应。
|
|
262
270
|
- HTTP Extract 使用 LinkeDOM、Mozilla Readability、Turndown,并对 Next.js RSC payload 做后备解析。
|
|
271
|
+
- HTTP Extract 在 route 已启用 HTTP Instance 时优先读取可用的源站 Markdown 变体(`.md` 或目录 `index.md`)。
|
|
263
272
|
- PDF、图片、音频、视频、zip 和通用二进制内容会返回 `unsupported_content`。
|
|
264
273
|
- 按项目设计,CLI 不阻止 localhost、私网地址或云元数据 URL。调用者必须在不可信输入场景中自行实施 URL allowlist、网络隔离或出站代理策略。
|
|
265
274
|
|
package/dist/cli.js
CHANGED
|
@@ -5,6 +5,7 @@ import { Command, CommanderError, InvalidArgumentError, Option, } from "commande
|
|
|
5
5
|
import { formatMarkdown } from "./cli-output.js";
|
|
6
6
|
import { loadConfig } from "./config/config.js";
|
|
7
7
|
import { executeConfigEdit } from "./config/edit.js";
|
|
8
|
+
import { executeConfigInit } from "./config/init.js";
|
|
8
9
|
import { createProviderOrderWriter } from "./config/provider-order.js";
|
|
9
10
|
import { executeDoctor, executeProviders } from "./core/diagnostics.js";
|
|
10
11
|
import { asWebAccessError, WebAccessError } from "./core/errors.js";
|
|
@@ -51,6 +52,13 @@ function writeEnvelope(envelope) {
|
|
|
51
52
|
process.stdout.write(`${JSON.stringify(envelope)}\n`);
|
|
52
53
|
}
|
|
53
54
|
function writeOutput(envelope, mode) {
|
|
55
|
+
if (mode === "path" &&
|
|
56
|
+
envelope.ok &&
|
|
57
|
+
"command" in envelope &&
|
|
58
|
+
envelope.command === "config.init") {
|
|
59
|
+
process.stdout.write(`${envelope.data.path}\n`);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
54
62
|
if (mode === "markdown" && envelope.ok) {
|
|
55
63
|
process.stdout.write(formatMarkdown(envelope));
|
|
56
64
|
return;
|
|
@@ -60,6 +68,7 @@ function writeOutput(envelope, mode) {
|
|
|
60
68
|
export function createProgram(run, dependencies = {}) {
|
|
61
69
|
const program = new Command();
|
|
62
70
|
const runConfigEdit = dependencies.executeConfigEdit ?? executeConfigEdit;
|
|
71
|
+
const runConfigInit = dependencies.executeConfigInit ?? executeConfigInit;
|
|
63
72
|
program
|
|
64
73
|
.name("web-access")
|
|
65
74
|
.description("Agent-neutral 的网页搜索与内容提取 CLI")
|
|
@@ -149,6 +158,14 @@ export function createProgram(run, dependencies = {}) {
|
|
|
149
158
|
? "json"
|
|
150
159
|
: "markdown"));
|
|
151
160
|
const config = program.command("config").description("管理配置文件");
|
|
161
|
+
config
|
|
162
|
+
.command("init")
|
|
163
|
+
.description("按当前标准凭据环境变量创建配置")
|
|
164
|
+
.option("--json", "输出 JSON envelope", false)
|
|
165
|
+
.action((options) => {
|
|
166
|
+
const globals = program.optsWithGlobals();
|
|
167
|
+
run(() => runConfigInit({ explicitPath: globals.config }), options.json || globals.json ? "json" : "path");
|
|
168
|
+
});
|
|
152
169
|
config
|
|
153
170
|
.command("edit")
|
|
154
171
|
.description("创建默认配置文件并用 VISUAL 或 EDITOR 打开")
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EACN,OAAO,EACP,cAAc,EACd,oBAAoB,EACpB,MAAM,GACN,MAAM,WAAW,CAAC;AACnB,OAAO,EAAsB,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAQhF,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EACN,OAAO,EACP,cAAc,EACd,oBAAoB,EACpB,MAAM,GACN,MAAM,WAAW,CAAC;AACnB,OAAO,EAAsB,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAQhF,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAYvC,SAAS,OAAO,CAAC,KAAa,EAAE,QAAkB;IACjD,OAAO,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAC9C,MAAM,IAAI,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC7B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACJ,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QACxD,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,uBAAuB,CAAC,CAAC;IACpE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,QAAQ,CAAC,QAAwB;IACzC,IAAI,QAAQ,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC;IAC1B,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IAClD,IACC;QACC,eAAe;QACf,cAAc;QACd,kBAAkB;QAClB,mBAAmB;KACnB,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAE/B,OAAO,CAAC,CAAC;IACV,OAAO,CAAC,CAAC;AACV,CAAC;AAED,SAAS,aAAa,CAAC,QAAwB;IAC9C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,WAAW,CAAC,QAAwB,EAAE,IAAmB;IACjE,IACC,IAAI,KAAK,MAAM;QACf,QAAQ,CAAC,EAAE;QACX,SAAS,IAAI,QAAQ;QACrB,QAAQ,CAAC,OAAO,KAAK,aAAa,EACjC,CAAC;QACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAChD,OAAO;IACR,CAAC;IACD,IAAI,IAAI,KAAK,UAAU,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC/C,OAAO;IACR,CAAC;IACD,aAAa,CAAC,QAAQ,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAC5B,GAGS,EACT,YAAY,GAAoB,EAAE;IAElC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAG,YAAY,CAAC,iBAAiB,IAAI,iBAAiB,CAAC;IAC1E,MAAM,aAAa,GAAG,YAAY,CAAC,iBAAiB,IAAI,iBAAiB,CAAC;IAC1E,OAAO;SACL,IAAI,CAAC,YAAY,CAAC;SAClB,WAAW,CAAC,8BAA8B,CAAC;SAC3C,OAAO,CAAC,OAAO,CAAC;SAChB,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC;SACzC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,wBAAwB,EAAE,CAAC;IAC7B,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,YAAY,EAAE,CAAC;IAEvB,OAAO;SACL,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,MAAM,CAAC;SACnB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;SAC5B,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,EAAE,MAAM,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,YAAY,EAAE,OAAO,CAAC;SACrD,SAAS,CACT,IAAI,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;QAClD,KAAK;QACL,OAAO;QACP,MAAM;KACN,CAAC,CACF;SACA,MAAM,CAAC,2BAA2B,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;SAC7D,MAAM,CAAC,2BAA2B,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;SAC5D,MAAM,CAAC,0BAA0B,EAAE,QAAQ,EAAE,OAAO,CAAC;SACrD,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,MAAM,CACN,CACC,KAAa,EACb,OAQC,EACA,EAAE;QACH,GAAG,CACF,KAAK,IAAI,EAAE;YACV,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAiB,CAAC;YAC9C,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAkB;gBAC9B,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACnB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE;gBACxC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,cAAc,EAAE,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC;gBACvD,cAAc,EAAE,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC;gBACvD,SAAS,EAAE,OAAO,CAAC,OAAO;aAC1B,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,KAAK;gBACjB,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,KAAK,GAAG,EAAE;gBACrB,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;YAC5D,OAAO,aAAa,CAAC,OAAO,EAAE;gBAC7B,MAAM;gBACN,MAAM,EAAE,aAAa,CAAC,MAAM;gBAC5B,oBAAoB,EAAE,yBAAyB,CAAC,MAAM,CAAC;aACvD,CAAC,CAAC;QACJ,CAAC,EACD,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAiB,CAAC,IAAI;YACjD,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,UAAU,CACb,CAAC;IACH,CAAC,CACD,CAAC;IAEH,OAAO;SACL,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,wCAAwC,CAAC;SACrD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;SAChC,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,EAAE,MAAM,CAAC;SACpE,MAAM,CAAC,0BAA0B,EAAE,QAAQ,EAAE,OAAO,CAAC;SACrD,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,MAAM,CACN,CACC,GAAW,EACX,OAIC,EACA,EAAE;QACH,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAiB,CAAC;QAC9C,MAAM,UAAU,GACf,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QACpD,GAAG,CAAC,KAAK,IAAI,EAAE;YACd,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAmB;gBAC/B,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC;gBACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE;gBACxC,SAAS,EAAE,OAAO,CAAC,OAAO;aAC1B,CAAC;YACF,OAAO,cAAc,CAAC,OAAO,EAAE;gBAC9B,MAAM;gBACN,MAAM,EAAE,aAAa,CAAC,MAAM;gBAC5B,oBAAoB,EAAE,yBAAyB,CAAC,MAAM,CAAC;aACvD,CAAC,CAAC;QACJ,CAAC,EAAE,UAAU,CAAC,CAAC;IAChB,CAAC,CACD,CAAC;IAEH,OAAO;SACL,OAAO,CAAC,WAAW,CAAC;SACpB,WAAW,CAAC,kCAAkC,CAAC;SAC/C,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,MAAM,CAAC,CAAC,OAA0B,EAAE,EAAE,CACtC,GAAG,CACF,GAAG,EAAE,CACJ,gBAAgB,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAiB,CAAC,MAAM,CAAC,CAAC,EACnE,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAiB,CAAC,IAAI;QACjD,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,UAAU,CACb,CACD,CAAC;IAEH,OAAO;SACL,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,0BAA0B,CAAC;SACvC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,MAAM,CAAC,CAAC,OAA0B,EAAE,EAAE,CACtC,GAAG,CACF,GAAG,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAiB,CAAC,MAAM,CAAC,CAAC,EACrE,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAiB,CAAC,IAAI;QACjD,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,UAAU,CACb,CACD,CAAC;IAEH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,iBAAiB,CAAC;SAC9B,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,MAAM,CAAC,CAAC,OAA0B,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,EAAiB,CAAC;QACzD,GAAG,CACF,GAAG,EAAE,CAAC,aAAa,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,EACrD,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAC9C,CAAC;IACH,CAAC,CAAC,CAAC;IACJ,MAAM;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,+BAA+B,CAAC;SAC5C,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC3C,MAAM,CAAC,CAAC,OAA0B,EAAE,EAAE,CACtC,GAAG,CACF,GAAG,EAAE,CACJ,aAAa,CAAC;QACb,YAAY,EAAE,OAAO,CAAC,IAAI,EAAiB,CAAC,MAAM;KAClD,CAAC,EACH,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAiB,CAAC,IAAI;QACjD,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,UAAU,CACb,CACD,CAAC;IAEH,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,IAAc;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAyB,CAAC;QAC/C,OAAO,MAAyB,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,OAAO,QAAQ,CAAC,QAAQ,CAAC,OAA0B,CAAC;QACnD,CAAC,CAAE,OAA2B;QAC9B,CAAC,CAAC,IAAI,CAAC;AACT,CAAC;AAED,MAAM,aAAa,GAAG,IAAI,eAAe,EAAE,CAAC;AAC5C,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;AAEpD,KAAK,UAAU,IAAI;IAClB,IAAI,OAA4C,CAAC;IACjD,IAAI,iBAAiB,GAAkB,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,EAAE,EAAE;QACrD,iBAAiB,GAAG,IAAI,CAAC;QACzB,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,aAAa,CAC7B,IAAI,cAAc,CAAC,eAAe,EAAE,QAAQ,CAAC,CAC7C,CAAC;QACF,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACR,CAAC;IACD,IAAI,CAAC;QACJ,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC;QAC/B,WAAW,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACzC,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACjB,IACC,MAAM,YAAY,cAAc;YAChC,MAAM,CAAC,IAAI,KAAK,yBAAyB;YAEzC,OAAO;QACR,IAAI,MAAM,YAAY,cAAc,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB;YAC1E,OAAO;QACR,MAAM,UAAU,GACf,MAAM,YAAY,cAAc;YAC/B,CAAC,CAAC,IAAI,cAAc,CAClB,eAAe,EACf,MAAM,CAAC,IAAI,KAAK,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ;gBAC/D,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,MAAM,CAAC,OAAO,CACjB;YACF,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,UAAU,EAAE,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1E,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;AACF,CAAC;AAED,MAAM,UAAU,YAAY,CAC3B,SAAiB,EACjB,SAA6B;IAE7B,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7B,IAAI,CAAC;QACJ,OAAO,SAAS,KAAK,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AACF,CAAC;AAED,IAAI,YAAY,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,MAAM,IAAI,EAAE,CAAC;AACd,CAAC"}
|
package/dist/config/config.js
CHANGED
|
@@ -12,7 +12,10 @@ export const DEFAULT_SEARCH_PROVIDERS = [
|
|
|
12
12
|
"searxng",
|
|
13
13
|
"anysearch",
|
|
14
14
|
"xcrawl",
|
|
15
|
+
];
|
|
16
|
+
export const DEFAULT_SEARCH_FALLBACK_PROVIDERS = [
|
|
15
17
|
"deepseek",
|
|
18
|
+
"xai_x_search",
|
|
16
19
|
"xai_web_search",
|
|
17
20
|
];
|
|
18
21
|
export const DEFAULT_EXTRACT_PROVIDERS = [
|
|
@@ -21,11 +24,12 @@ export const DEFAULT_EXTRACT_PROVIDERS = [
|
|
|
21
24
|
"exa",
|
|
22
25
|
"anysearch",
|
|
23
26
|
"xcrawl",
|
|
24
|
-
"http",
|
|
25
27
|
];
|
|
28
|
+
export const DEFAULT_EXTRACT_FALLBACK_PROVIDERS = ["http"];
|
|
26
29
|
export const DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
|
|
27
30
|
const DEFAULT_SEARCH_CONFIG = {
|
|
28
31
|
providers: [...DEFAULT_SEARCH_PROVIDERS],
|
|
32
|
+
providers_fallback: [...DEFAULT_SEARCH_FALLBACK_PROVIDERS],
|
|
29
33
|
limit: 5,
|
|
30
34
|
timeoutMs: 120_000,
|
|
31
35
|
attemptTimeoutMs: 60_000,
|
|
@@ -33,6 +37,7 @@ const DEFAULT_SEARCH_CONFIG = {
|
|
|
33
37
|
};
|
|
34
38
|
const DEFAULT_EXTRACT_CONFIG = {
|
|
35
39
|
providers: [...DEFAULT_EXTRACT_PROVIDERS],
|
|
40
|
+
providers_fallback: [...DEFAULT_EXTRACT_FALLBACK_PROVIDERS],
|
|
36
41
|
timeoutMs: 120_000,
|
|
37
42
|
attemptTimeoutMs: 45_000,
|
|
38
43
|
maxResponseBytes: DEFAULT_MAX_RESPONSE_BYTES,
|
|
@@ -122,7 +127,7 @@ const ALLOWED_INSTANCE_KEYS = new Set([
|
|
|
122
127
|
]);
|
|
123
128
|
const ALLOWED_SEARCH_KEYS = new Set([
|
|
124
129
|
"providers",
|
|
125
|
-
"
|
|
130
|
+
"providers_fallback",
|
|
126
131
|
"limit",
|
|
127
132
|
"timeoutMs",
|
|
128
133
|
"attemptTimeoutMs",
|
|
@@ -130,7 +135,7 @@ const ALLOWED_SEARCH_KEYS = new Set([
|
|
|
130
135
|
]);
|
|
131
136
|
const ALLOWED_EXTRACT_KEYS = new Set([
|
|
132
137
|
"providers",
|
|
133
|
-
"
|
|
138
|
+
"providers_fallback",
|
|
134
139
|
"timeoutMs",
|
|
135
140
|
"attemptTimeoutMs",
|
|
136
141
|
"maxResponseBytes",
|
|
@@ -291,22 +296,6 @@ function parseRoute(value, path) {
|
|
|
291
296
|
}
|
|
292
297
|
return result;
|
|
293
298
|
}
|
|
294
|
-
function parseInternalRoute(value) {
|
|
295
|
-
if (value === undefined)
|
|
296
|
-
return undefined;
|
|
297
|
-
if (!Array.isArray(value))
|
|
298
|
-
return undefined;
|
|
299
|
-
const result = [];
|
|
300
|
-
for (const item of value) {
|
|
301
|
-
if (typeof item !== "string")
|
|
302
|
-
return undefined;
|
|
303
|
-
const id = item.trim();
|
|
304
|
-
if (!/^[a-z][a-z0-9_-]{0,63}$/.test(id) || id === "auto")
|
|
305
|
-
return undefined;
|
|
306
|
-
result.push(id);
|
|
307
|
-
}
|
|
308
|
-
return result;
|
|
309
|
-
}
|
|
310
299
|
function parseSearch(value) {
|
|
311
300
|
if (value === undefined)
|
|
312
301
|
return {};
|
|
@@ -314,7 +303,7 @@ function parseSearch(value) {
|
|
|
314
303
|
assertKnownKeys(value, ALLOWED_SEARCH_KEYS, "search");
|
|
315
304
|
return {
|
|
316
305
|
providers: parseRoute(value.providers, "search.providers"),
|
|
317
|
-
|
|
306
|
+
providers_fallback: parseRoute(value.providers_fallback, "search.providers_fallback"),
|
|
318
307
|
limit: parseLimit(value.limit),
|
|
319
308
|
timeoutMs: parsePositiveInt(value.timeoutMs, "search.timeoutMs", DEFAULT_SEARCH_CONFIG.timeoutMs),
|
|
320
309
|
attemptTimeoutMs: parsePositiveInt(value.attemptTimeoutMs, "search.attemptTimeoutMs", DEFAULT_SEARCH_CONFIG.attemptTimeoutMs),
|
|
@@ -328,7 +317,7 @@ function parseExtract(value) {
|
|
|
328
317
|
assertKnownKeys(value, ALLOWED_EXTRACT_KEYS, "extract");
|
|
329
318
|
return {
|
|
330
319
|
providers: parseRoute(value.providers, "extract.providers"),
|
|
331
|
-
|
|
320
|
+
providers_fallback: parseRoute(value.providers_fallback, "extract.providers_fallback"),
|
|
332
321
|
timeoutMs: parsePositiveInt(value.timeoutMs, "extract.timeoutMs", DEFAULT_EXTRACT_CONFIG.timeoutMs),
|
|
333
322
|
attemptTimeoutMs: parsePositiveInt(value.attemptTimeoutMs, "extract.attemptTimeoutMs", DEFAULT_EXTRACT_CONFIG.attemptTimeoutMs),
|
|
334
323
|
maxResponseBytes: parsePositiveInt(value.maxResponseBytes, "extract.maxResponseBytes", DEFAULT_EXTRACT_CONFIG.maxResponseBytes),
|
|
@@ -354,6 +343,8 @@ function parseRawConfig(value) {
|
|
|
354
343
|
instances,
|
|
355
344
|
search: parseSearch(value.search),
|
|
356
345
|
extract: parseExtract(value.extract),
|
|
346
|
+
searchSpecified: value.search !== undefined,
|
|
347
|
+
extractSpecified: value.extract !== undefined,
|
|
357
348
|
};
|
|
358
349
|
}
|
|
359
350
|
function mergeInstances(overrides) {
|
|
@@ -374,13 +365,12 @@ function mergeInstances(overrides) {
|
|
|
374
365
|
}
|
|
375
366
|
return merged;
|
|
376
367
|
}
|
|
377
|
-
function validateRoutes(instances, route, capability) {
|
|
368
|
+
function validateRoutes(instances, route, capability, defaultRoute, requireNonEmpty = false) {
|
|
378
369
|
const allowed = capability === "search" ? SEARCH_TYPES : EXTRACT_TYPES;
|
|
379
370
|
const byId = new Map(instances.map((instance) => [instance.id, instance]));
|
|
380
|
-
const chosen = route
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
: [...DEFAULT_EXTRACT_PROVIDERS]);
|
|
371
|
+
const chosen = route ? [...route] : [...defaultRoute];
|
|
372
|
+
if (requireNonEmpty && chosen.length === 0)
|
|
373
|
+
throw new ConfigError(`${capability}.providers 不能为空`);
|
|
384
374
|
for (const id of chosen) {
|
|
385
375
|
const instance = byId.get(id);
|
|
386
376
|
if (!instance)
|
|
@@ -390,16 +380,6 @@ function validateRoutes(instances, route, capability) {
|
|
|
390
380
|
}
|
|
391
381
|
return chosen;
|
|
392
382
|
}
|
|
393
|
-
function validInternalRoute(configured, internal) {
|
|
394
|
-
if (!internal || internal.length !== configured.length)
|
|
395
|
-
return undefined;
|
|
396
|
-
if (new Set(internal).size !== internal.length)
|
|
397
|
-
return undefined;
|
|
398
|
-
const configuredIds = new Set(configured);
|
|
399
|
-
if (internal.some((id) => !configuredIds.has(id)))
|
|
400
|
-
return undefined;
|
|
401
|
-
return internal;
|
|
402
|
-
}
|
|
403
383
|
export function getDefaultConfigPath() {
|
|
404
384
|
return join(homedir(), ".config", "web-access-cli", "config.json");
|
|
405
385
|
}
|
|
@@ -515,10 +495,12 @@ export function createDefaultAppConfig() {
|
|
|
515
495
|
search: {
|
|
516
496
|
...DEFAULT_SEARCH_CONFIG,
|
|
517
497
|
providers: [...DEFAULT_SEARCH_CONFIG.providers],
|
|
498
|
+
providers_fallback: [...DEFAULT_SEARCH_CONFIG.providers_fallback],
|
|
518
499
|
},
|
|
519
500
|
extract: {
|
|
520
501
|
...DEFAULT_EXTRACT_CONFIG,
|
|
521
502
|
providers: [...DEFAULT_EXTRACT_CONFIG.providers],
|
|
503
|
+
providers_fallback: [...DEFAULT_EXTRACT_CONFIG.providers_fallback],
|
|
522
504
|
},
|
|
523
505
|
};
|
|
524
506
|
}
|
|
@@ -534,13 +516,15 @@ function buildLoadedConfig(path, exists, parsed, env) {
|
|
|
534
516
|
const defaults = createDefaultAppConfig();
|
|
535
517
|
const raw = parseRawConfig(parsed);
|
|
536
518
|
const instanceConfigs = mergeInstances(raw.instances);
|
|
537
|
-
const searchProviders = validateRoutes(instanceConfigs, raw.search.providers, "search");
|
|
538
|
-
const
|
|
539
|
-
|
|
540
|
-
const
|
|
519
|
+
const searchProviders = validateRoutes(instanceConfigs, raw.search.providers, "search", DEFAULT_SEARCH_PROVIDERS, true);
|
|
520
|
+
const searchFallbackProviders = validateRoutes(instanceConfigs, raw.search.providers_fallback ??
|
|
521
|
+
(raw.searchSpecified ? [] : DEFAULT_SEARCH_FALLBACK_PROVIDERS), "search", DEFAULT_SEARCH_FALLBACK_PROVIDERS);
|
|
522
|
+
const extractProviders = validateRoutes(instanceConfigs, raw.extract.providers, "extract", DEFAULT_EXTRACT_PROVIDERS);
|
|
523
|
+
const extractFallbackProviders = validateRoutes(instanceConfigs, raw.extract.providers_fallback ??
|
|
524
|
+
(raw.extractSpecified ? [] : DEFAULT_EXTRACT_FALLBACK_PROVIDERS), "extract", DEFAULT_EXTRACT_FALLBACK_PROVIDERS);
|
|
541
525
|
const search = {
|
|
542
526
|
providers: searchProviders,
|
|
543
|
-
|
|
527
|
+
providers_fallback: searchFallbackProviders,
|
|
544
528
|
limit: raw.search.limit ?? defaults.search.limit,
|
|
545
529
|
timeoutMs: raw.search.timeoutMs ?? defaults.search.timeoutMs,
|
|
546
530
|
attemptTimeoutMs: raw.search.attemptTimeoutMs ?? defaults.search.attemptTimeoutMs,
|
|
@@ -548,9 +532,7 @@ function buildLoadedConfig(path, exists, parsed, env) {
|
|
|
548
532
|
};
|
|
549
533
|
const extract = {
|
|
550
534
|
providers: extractProviders,
|
|
551
|
-
|
|
552
|
-
? { _providers: extractInternalProviders }
|
|
553
|
-
: {}),
|
|
535
|
+
providers_fallback: extractFallbackProviders,
|
|
554
536
|
timeoutMs: raw.extract.timeoutMs ?? defaults.extract.timeoutMs,
|
|
555
537
|
attemptTimeoutMs: raw.extract.attemptTimeoutMs ?? defaults.extract.attemptTimeoutMs,
|
|
556
538
|
maxResponseBytes: raw.extract.maxResponseBytes ?? defaults.extract.maxResponseBytes,
|
|
@@ -588,9 +570,16 @@ export function getRoute(config, capability) {
|
|
|
588
570
|
? config.search.providers
|
|
589
571
|
: config.extract.providers;
|
|
590
572
|
}
|
|
573
|
+
export function getFallbackRoute(config, capability) {
|
|
574
|
+
const primary = getRoute(config, capability);
|
|
575
|
+
const fallback = capability === "search"
|
|
576
|
+
? config.search.providers_fallback
|
|
577
|
+
: config.extract.providers_fallback;
|
|
578
|
+
const primaryIds = new Set(primary);
|
|
579
|
+
return fallback.filter((id) => !primaryIds.has(id));
|
|
580
|
+
}
|
|
591
581
|
export function getEffectiveRoute(config, capability) {
|
|
592
|
-
|
|
593
|
-
return capabilityConfig._providers ?? capabilityConfig.providers;
|
|
582
|
+
return getRoute(config, capability);
|
|
594
583
|
}
|
|
595
584
|
export function getCapabilityConfig(config, capability) {
|
|
596
585
|
return capability === "search" ? config.search : config.extract;
|