pi-better-web-search 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,32 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.1.6
4
+
5
+ - Keep oversized results as valid JSON, sharing the character budget across
6
+ search results and fetched pages rather than cutting off the serialized tail.
7
+ Report affected paths and mark shortened strings; retain unchanged provider
8
+ data when it fits, and compact whitespace before dropping data.
9
+ - Fix premature truncation immediately below the configured output limit.
10
+
11
+ ## 0.1.5
12
+
13
+ - Rank models inside a provider family by capability tier first (cheapest tier
14
+ wins) and version number second, newest first. OpenAI tiers are
15
+ `luna → sol → astra` and Claude tiers are `haiku → sonnet → opus → fable`;
16
+ other families have no named tiers and sort purely by version. This replaces
17
+ the previous regex-based "likely search-capable" ordering, so legacy models
18
+ no longer win a lexicographic tie-break and large tiers are not called while
19
+ a cheaper sibling is available.
20
+ - Add a deeper fallback pass: after one candidate per family, alternate models
21
+ from other families are tried round-robin, so a provider account that rejects
22
+ a specific model (legacy id, account tier, regional availability) can still
23
+ serve a sibling model. Only the current model's provider id is excluded; a
24
+ different provider id in the same family is still tried.
25
+ - Classify failures for both search and fetch: provider-wide errors
26
+ (401/403/429, 5xx, insufficient balance, network failure, timeout, malformed
27
+ request header) skip the rest of that provider, while model-specific errors
28
+ keep trying sibling models.
29
+
3
30
  ## 0.1.4
4
31
 
5
32
  - Return `Failed: This tool is not available in this conversation.` when all
package/README.md CHANGED
@@ -8,7 +8,7 @@ provider supports it, automatically falls back across authenticated providers,
8
8
  and returns provider data as-is.
9
9
 
10
10
  [![CI](https://github.com/zhexusun10/pi-better-web-search/actions/workflows/ci.yml/badge.svg)](https://github.com/zhexusun10/pi-better-web-search/actions/workflows/ci.yml)
11
- [![npm](https://img.shields.io/badge/npm-v0.1.1-blue)](https://www.npmjs.com/package/pi-better-web-search)
11
+ [![npm](https://img.shields.io/npm/v/pi-better-web-search)](https://www.npmjs.com/package/pi-better-web-search)
12
12
  [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
13
13
  [![Pi 1.3](https://img.shields.io/badge/Pi-1.3-blue.svg)](https://pi.dev)
14
14
  [![Node](https://img.shields.io/badge/node-%3E%3D22.19-339933.svg)](https://nodejs.org)
@@ -58,8 +58,8 @@ Requirements: **pi 1.3+** and **Node.js 22.19+**.
58
58
  provider supports URL analysis. `urls` can be used alone or with `query`.
59
59
  - **Raw results.** Provider data is returned in a JSON wrapper without
60
60
  deduplication, URL normalization, junk filtering, re-ranking, summarization,
61
- citation-marker insertion, or appended Sources sections. Only a final
62
- character guard truncates oversized output.
61
+ citation-marker insertion, or appended Sources sections. A final character
62
+ guard applies structured truncation only to oversized output.
63
63
  - **Correct semantics.** The tool description is exactly
64
64
  `Search the web or analyze specified URLs.` The schema makes clear that
65
65
  `urls` does not require `query`.
@@ -142,10 +142,17 @@ For a search:
142
142
  when the cache fingerprint is current.
143
143
  3. The remaining provider families are tried in this fixed order:
144
144
  **DeepSeek → Ollama → Google → OpenAI → xAI → Anthropic**.
145
- 4. Within a family, a likely search-capable authenticated model is selected
146
- first, with a deterministic tie-break. Remaining attempt slots are filled
147
- with other authenticated provider interfaces, but a failed provider id is
148
- never retried.
145
+ 4. Within a family, models are ordered by capability tier first (cheapest
146
+ available tier wins) and then by version number, newest first. OpenAI tiers
147
+ are `luna → sol → astra` and Claude tiers are
148
+ `haiku → sonnet → opus → fable`; other families have no named tiers and are
149
+ ordered purely by version. Ties are deterministic.
150
+ 5. Alternate models from other providers are tried round-robin, so a provider
151
+ that rejects one specific model can still serve a sibling. Only the current
152
+ model's provider id is excluded; a different provider id in the same family
153
+ is still tried. A provider-wide failure (auth, balance, network, timeout,
154
+ malformed request) skips the rest of that provider; a model-specific
155
+ failure keeps trying sibling models.
149
156
 
150
157
  URL fetching uses the same idea, but considers only providers that implement
151
158
  URL fetching.
@@ -243,18 +250,36 @@ this:
243
250
  - Citation markers are not inserted, and no "Sources" or "Additional results"
244
251
  sections are appended.
245
252
  - Redirects are not resolved and sources are not derived.
246
- - Unknown provider fields are preserved.
247
-
248
- The only transformation is a final output-size guard. The model-facing JSON is
249
- truncated to **64,000 characters** by default and marked with `[Truncated]`;
250
- `details.truncated` is set to `true`. Change the limit with:
253
+ - Unknown provider fields are preserved when the result fits the size guard.
254
+
255
+ A final output-size guard limits successful model-facing results to **64,000
256
+ Unicode code points** by default. Normal-sized results retain their original
257
+ fields and formatting. If indentation alone exceeds the limit, compact JSON is
258
+ returned without dropping data or setting the truncation flag.
259
+
260
+ When even compact JSON is too large, the guard allocates space across sibling
261
+ fields and array entries, redistributing unused space. It tries to preserve
262
+ identifying fields such as URLs, titles and errors, while shortening long
263
+ strings or omitting fields and trailing array entries when needed. The result
264
+ remains valid JSON. Shortened strings end in `…[truncated]` when space permits,
265
+ and the model-facing `_truncation` object reports `truncated`, `originalChars`
266
+ (the original indented JSON size), and `affectedPaths`. `moreAffectedPaths`
267
+ indicates that the path list is incomplete. Array paths can indicate omitted
268
+ entries; with very small budgets, not every URL or field can be preserved.
269
+ `details.truncated` is also set to `true` when data is actually removed.
270
+
271
+ Change the limit **before starting pi**:
251
272
 
252
273
  ```bash
253
274
  export PI_BETTER_WEB_SEARCH_MAX_CHARS=100000
254
275
  ```
255
276
 
256
- Truncation is Unicode code-point safe. The guard is intentionally applied after
257
- the raw JSON is assembled so normal-sized provider payloads are not altered.
277
+ In PowerShell, use `$env:PI_BETTER_WEB_SEARCH_MAX_CHARS = "100000"`. The limit
278
+ is in code points, **not tokens**, and does not adapt to the current model's
279
+ context window. Extremely small limits below 19 characters cannot fit a
280
+ model-visible truncation flag; the result is still valid JSON and
281
+ `details.truncated` remains true. There is no continuation/pagination cache:
282
+ retry individual URLs or raise the limit if more content is needed.
258
283
 
259
284
  ## Exposure rules
260
285
 
@@ -46,7 +46,7 @@ pi install ./path/to/pi-better-web-search
46
46
  `urls`。`urls` 可以单独使用,也可以和 `query` 一起使用。
47
47
  - **原始结果。** provider 数据放在 JSON 外层中返回,不去重、不规范化 URL、
48
48
  不过滤垃圾 URL、不重排、不总结、不插入 citation 标记、不追加 Sources。
49
- 只做最终字符数限制和截断。
49
+ 只有超出最终字符数限制时才进行结构化截断。
50
50
  - **正确语义。** 工具描述是 `Search the web or analyze specified URLs.`;
51
51
  schema 明确说明 `urls` 不依赖 `query`。
52
52
  - **不碰系统提示词。** 不添加 prompt snippet、guideline、loadout hook、
@@ -120,8 +120,14 @@ Provider 识别顺序:
120
120
  2. 缓存指纹有效时,使用记住的搜索 provider。
121
121
  3. 其余 provider family 按固定顺序尝试:
122
122
  **DeepSeek → Ollama → Google → OpenAI → xAI → Anthropic**。
123
- 4. 同一 family 内优先选择看起来支持搜索的已认证模型。剩余槽位用其他已认证
124
- provider 接口填充;同一个 provider id 不会用不同模型重复尝试。
123
+ 4. 同一 family 内先按能力等级排序(便宜的等级优先,未知等级排最后):
124
+ OpenAI 为 `luna → sol → astra`,Claude 为 `haiku → sonnet → opus → fable`;
125
+ 其他 family 没有命名等级,直接按版本号从新到旧排序。同级内版本号大的优先。
126
+ 5. 除当前模型的 provider id 外,其他 provider 的备用模型会按 round-robin 继续尝试,
127
+ 这样即使某个账号拒绝某个具体模型,也能换用同 provider 的其他模型。
128
+ 同一 family 下不同 provider id(如 `openai` 与 `openai-codex`)仍会尝试。
129
+ provider 级失败(鉴权、余额、网络、超时、请求头异常)会跳过该 provider 的
130
+ 其余模型;模型级失败则继续尝试同 provider 的其他模型。
125
131
 
126
132
  URL 抓取使用同样的思路,但只考虑实现了 URL 抓取的 provider。
127
133
 
@@ -209,17 +215,32 @@ Provider 数据放在 JSON 外层中返回,不做清理。示例:
209
215
  - 结果不重排、不总结。
210
216
  - 不插入 citation 标记,不追加 "Sources" 或 "Additional results"。
211
217
  - 不解析 redirect,不派生 sources。
212
- - provider 的未知字段保留。
218
+ - 未超出大小限制时,provider 的未知字段也会保留。
213
219
 
214
- 唯一的转换是最终输出大小保护。模型可见的 JSON 默认截断到
215
- **64,000 字符**,末尾标记 `[Truncated]`,`details.truncated` 为 `true`。
216
- 可通过环境变量调整:
220
+ 最终大小保护默认限制成功结果不超过 **64,000 个 Unicode code point**。
221
+ 未超限时保留原始字段及格式;如果只是缩进导致超限,会先压缩 JSON,
222
+ 不丢失数据,也不标记为截断。
223
+
224
+ 压缩后仍超限时,按字段和数组条目分配预算,未用完的额度会重新分配;
225
+ 尽量保留 URL、标题和错误等标识信息。必要时会缩短字符串或省略字段、
226
+ 数组末尾的条目,但返回值始终是合法 JSON。空间允许时,缩短的字符串以
227
+ `…[truncated]` 结尾;模型可见的 `_truncation` 对象包含 `truncated`、
228
+ `originalChars`(原始缩进 JSON 的字符数)和 `affectedPaths`。
229
+ `moreAffectedPaths` 表示路径列表不完整,数组路径也可能表示部分条目被省略。
230
+ 预算极小时无法保证保留所有 URL 或字段。确实丢失数据时,
231
+ `details.truncated` 也会设为 `true`。
232
+
233
+ 可在**启动 pi 之前**通过环境变量调整:
217
234
 
218
235
  ```bash
219
236
  export PI_BETTER_WEB_SEARCH_MAX_CHARS=100000
220
237
  ```
221
238
 
222
- 截断按 Unicode code point 处理,不会拆开 surrogate pair。
239
+ PowerShell 使用 `$env:PI_BETTER_WEB_SEARCH_MAX_CHARS = "100000"`。
240
+ 上限是 code point,**不是 token**,不会根据当前模型的上下文窗口自动调整。
241
+ 极小上限(少于 19 字符)无法在模型可见内容中容纳截断标记,此时仍返回
242
+ 合法 JSON,并在 `details.truncated` 中报告。当前没有续读或分页缓存;
243
+ 需要更多内容时可单独重新抓取 URL 或提高上限。
223
244
 
224
245
  ## 暴露规则
225
246
 
@@ -38,7 +38,7 @@ tool execute
38
38
  ├─ run search candidates until one succeeds
39
39
  ├─ run fetch candidates per URL until each URL succeeds or candidates end
40
40
  ├─ write search/fetch preferences + fingerprint on success
41
- └─ format raw JSON, truncate only if the character guard is exceeded
41
+ └─ format raw JSON; compact or structurally truncate if the guard is exceeded
42
42
  ```
43
43
 
44
44
  ## Exposure
@@ -72,15 +72,22 @@ Search candidates:
72
72
  2. The remembered search provider when the cache fingerprint is current.
73
73
  3. One candidate per remaining family in this order:
74
74
  **DeepSeek → Ollama → Google → OpenAI → xAI → Anthropic**.
75
- 4. Remaining authenticated provider interfaces, up to 12 attempts.
75
+ 4. Alternate models from every provider except the current model's provider
76
+ id, round-robin across families, up to 12 attempts. A different provider id
77
+ in the same family (for example `openai` vs `openai-codex`) is still tried.
76
78
 
77
79
  Fetch candidates use the same shape but include only adapters with
78
80
  `supportsUrls` and a `fetch` implementation. The fetch cap is 6 attempts.
79
81
 
80
- Within a family, likely search-capable model IDs rank first (for example
81
- `gpt-5`/`gpt-4.1`/`gpt-4o` for OpenAI, `gemini` for Google, `claude` for
82
- Anthropic, `grok` for xAI). Ties are deterministic. A provider id that was
83
- already selected is not retried with another model.
82
+ Within a family, models are ordered by capability tier first and version
83
+ second. Named tiers rank cheapest first and unknown ids rank last: OpenAI uses
84
+ `luna → sol → astra` and Claude uses `haiku → sonnet → opus → fable`. Other
85
+ families have no named tiers. Version numbers are compared as tuples with the
86
+ newest first (`gpt-7.0` before `gpt-6.1` before `gpt-4.1`, `gemini-3.8` before
87
+ `gemini-2.5`). Ties fall back to the deterministic `provider/id@baseUrl` order.
88
+ A provider id is only retried with another model when the first failure was
89
+ model-specific; provider-wide failures (auth, balance, network, timeout,
90
+ malformed request) skip the rest of that provider.
84
91
 
85
92
  The current model is preferred when it supports search. The cache is a fallback,
86
93
  not an override. When the current model is unsupported, the cached provider is
@@ -153,9 +160,16 @@ The result is a JSON wrapper:
153
160
 
154
161
  Fields inside `search` and `fetched[].data` are provider data. The extension does
155
162
  not deduplicate, normalize, filter, re-rank, summarize, insert citation markers,
156
- or append source sections. The only transformation is the final character
157
- guard: the serialized JSON is truncated at 64,000 characters by default
158
- (`PI_BETTER_WEB_SEARCH_MAX_CHARS`) and marked with `[Truncated]`.
163
+ or append source sections. The final guard keeps successful model-facing output
164
+ within 64,000 Unicode code points by default (`PI_BETTER_WEB_SEARCH_MAX_CHARS`).
165
+ It preserves ordinary formatted JSON, or uses compact JSON if that alone fits.
166
+ Otherwise it budgets sibling fields and array entries independently, preferring
167
+ identifiers when space is tight. Strings can be shortened, and fields or trailing
168
+ array entries omitted; the output remains valid JSON. `_truncation` records the
169
+ original indented JSON length and affected paths, while `details.truncated` marks
170
+ actual data loss. For very small limits below 19 characters, only the details
171
+ flag fits. This is a character guard, not a token or model-context limit; it does
172
+ not provide pagination or retain the complete provider payload for later use.
159
173
 
160
174
  ## Provider adapter contract
161
175
 
@@ -30,7 +30,7 @@ local `pi-web-search` reference checkout, `.env` files, and `DESIGN.md`.
30
30
  npm run build
31
31
  ```
32
32
 
33
- This produces `dist/pi-better-web-search-0.1.4.tgz`. The tarball is not
33
+ This produces `dist/pi-better-web-search-0.1.6.tgz`. The tarball is not
34
34
  committed.
35
35
 
36
36
  ## Publish
@@ -42,7 +42,7 @@ npm publish --access public
42
42
  Alternatively, publish the verified tarball directly:
43
43
 
44
44
  ```bash
45
- npm publish dist/pi-better-web-search-0.1.4.tgz --access public
45
+ npm publish dist/pi-better-web-search-0.1.6.tgz --access public
46
46
  ```
47
47
 
48
48
  `prepublishOnly` reruns the checks, tests, and package verification. npm may
@@ -51,10 +51,10 @@ require browser or OTP authentication.
51
51
  ## Tag and release
52
52
 
53
53
  ```bash
54
- git tag -a v0.1.4 -m "v0.1.4"
55
- git push origin v0.1.4
56
- gh release create v0.1.4 dist/pi-better-web-search-0.1.4.tgz \
57
- --title "v0.1.4" \
54
+ git tag -a v0.1.6 -m "v0.1.6"
55
+ git push origin v0.1.6
56
+ gh release create v0.1.6 dist/pi-better-web-search-0.1.6.tgz \
57
+ --title "v0.1.6" \
58
58
  --notes-file /path/to/release-notes.md
59
59
  ```
60
60
 
package/docs/testing.md CHANGED
@@ -12,12 +12,12 @@ npm run verify:package
12
12
  ```
13
13
 
14
14
  - `npm run check` runs `tsc --noEmit`.
15
- - `npm test` runs the 116 unit and integration tests with Node's test runner.
15
+ - `npm test` runs the unit and integration tests with Node's test runner.
16
16
  - `npm run verify:package` runs `npm pack --dry-run --json` and checks that the
17
17
  tarball contains runtime sources, docs, assets, and metadata, while excluding
18
18
  tests, scripts, CI files, the local reference checkout, and `DESIGN.md`.
19
19
  - `npm run build` runs the checks, tests, package verification, and packs
20
- `dist/pi-better-web-search-0.1.0.tgz`.
20
+ `dist/pi-better-web-search-0.1.6.tgz`.
21
21
 
22
22
  Tests import the TypeScript sources directly. The suite mocks `globalThis.fetch`
23
23
  and uses temporary cache directories. Coverage includes:
@@ -30,7 +30,8 @@ and uses temporary cache directories. Coverage includes:
30
30
  - search and fetch fallback, including per-URL fallback;
31
31
  - provider request shapes and SSE parsing;
32
32
  - HTTP/stream errors and abort propagation;
33
- - character-guard truncation;
33
+ - structured character-guard truncation, valid JSON, Unicode and escaping,
34
+ boundary lengths, and fair allocation across search results and URLs;
34
35
  - one-tool registration and no system-prompt hooks.
35
36
 
36
37
  ## Live DeepSeek test
package/lib/candidates.ts CHANGED
@@ -52,15 +52,53 @@ export const MAX_FETCH_CANDIDATES = FETCH_ATTEMPT_CAP;
52
52
  export const SEARCH_CAP = SEARCH_ATTEMPT_CAP;
53
53
  export const FETCH_CAP = FETCH_ATTEMPT_CAP;
54
54
 
55
- const PREFERRED_MODEL_PATTERNS: Record<ProviderFamily, RegExp> = {
56
- deepseek: /deepseek/i,
57
- ollama: /gpt-oss|qwen|llama|deepseek|mistral/i,
58
- google: /gemini/i,
59
- openai: /gpt-5|gpt-4\.1|gpt-4o|o[134]/i,
60
- xai: /grok/i,
61
- anthropic: /claude/i,
55
+ /**
56
+ * Cheapest-first capability tiers per family, ordered small to large. The
57
+ * first matching tier wins and ids that match no tier rank last, so fallback
58
+ * does not burn a large model when a small one is available. Version numbers
59
+ * are compared separately, newest first. Only families whose provider uses
60
+ * tier names need an entry; everywhere else every model shares tier 0.
61
+ */
62
+ function tierMatcher(name: string): RegExp {
63
+ return new RegExp(`(?:^|[^a-z0-9])${name}(?:$|[^a-z0-9])`, "i");
64
+ }
65
+
66
+ const MODEL_TIERS: Partial<Record<ProviderFamily, readonly RegExp[]>> = {
67
+ openai: ["luna", "sol", "astra"].map(tierMatcher),
68
+ anthropic: ["haiku", "sonnet", "opus", "fable"].map(tierMatcher),
62
69
  };
63
70
 
71
+ function modelTier(candidate: Candidate): number {
72
+ const patterns = MODEL_TIERS[candidate.family];
73
+ if (!patterns) return 0;
74
+ for (let index = 0; index < patterns.length; index += 1) {
75
+ if (patterns[index].test(candidate.model.id)) return index;
76
+ }
77
+ return patterns.length;
78
+ }
79
+
80
+ /**
81
+ * Parse the first version-like run in a model id into a numeric tuple.
82
+ * `gpt-6.1` -> [6, 1], `claude-opus-4-5` -> [4, 5], `gpt-oss:120b` -> [120].
83
+ * Ids without a number return an empty tuple and rank below versioned ones.
84
+ */
85
+ function versionTuple(id: string): number[] {
86
+ const match = id.match(/\d+(?:[.-]\d+)*/);
87
+ if (!match) return [];
88
+ return match[0].split(/[.-]/).map((part) => Number.parseInt(part, 10));
89
+ }
90
+
91
+ /** Compare version tuples so that a larger, newer version sorts first. */
92
+ function compareVersionsDesc(left: number[], right: number[]): number {
93
+ const length = Math.max(left.length, right.length);
94
+ for (let index = 0; index < length; index += 1) {
95
+ const a = left[index] ?? 0;
96
+ const b = right[index] ?? 0;
97
+ if (a !== b) return b - a;
98
+ }
99
+ return 0;
100
+ }
101
+
64
102
  function isCandidate(value: Candidate | Model<Api>): value is Candidate {
65
103
  return "model" in value && "family" in value && "adapter" in value;
66
104
  }
@@ -140,15 +178,12 @@ function compareText(left: string, right: string): number {
140
178
  return left < right ? -1 : left > right ? 1 : 0;
141
179
  }
142
180
 
143
- function modelRank(candidate: Candidate): number {
144
- return PREFERRED_MODEL_PATTERNS[candidate.family].test(candidate.model.id) ? 0 : 1;
145
- }
146
-
147
181
  /** Rank candidates inside one family independently of registry order. */
148
182
  export function rankFamilyCandidates(candidates: Candidate[]): Candidate[] {
149
183
  return [...candidates].sort(
150
184
  (left, right) =>
151
- modelRank(left) - modelRank(right)
185
+ modelTier(left) - modelTier(right)
186
+ || compareVersionsDesc(versionTuple(left.model.id), versionTuple(right.model.id))
152
187
  || compareText(candidateKey(left), candidateKey(right))
153
188
  || compareText(left.model.baseUrl ?? "", right.model.baseUrl ?? ""),
154
189
  );
@@ -193,6 +228,16 @@ function addUnique(
193
228
  return true;
194
229
  }
195
230
 
231
+ function usableFamilyCandidates(
232
+ resolution: ResolutionContext,
233
+ family: ProviderFamily,
234
+ isUsable: (candidate: Candidate) => boolean,
235
+ ): Candidate[] {
236
+ return rankFamilyCandidates(
237
+ resolution.available.filter((candidate) => candidate.family === family && isUsable(candidate)),
238
+ );
239
+ }
240
+
196
241
  function buildCandidates(
197
242
  resolution: ResolutionContext,
198
243
  isUsable: (candidate: Candidate) => boolean,
@@ -223,23 +268,44 @@ function buildCandidates(
223
268
  if (selected.length >= cap) break;
224
269
  if (representedFamilies.has(family)) continue;
225
270
 
226
- const candidates = rankFamilyCandidates(
227
- resolution.available.filter((candidate) => candidate.family === family && isUsable(candidate)),
228
- );
271
+ const candidates = usableFamilyCandidates(resolution, family, isUsable);
229
272
  if (candidates[0] && addUnique(selected, candidates[0], seen, cap)) {
230
273
  representedFamilies.add(family);
231
274
  }
232
275
  }
233
276
 
234
- // Fill any remaining attempts with other available provider interfaces. A
235
- // provider id is never retried with a different model, and the fixed family
236
- // priority above still determines which interface comes first.
237
- for (const candidate of resolution.available) {
238
- if (selected.length >= cap) break;
239
- if (!isUsable(candidate)) continue;
240
- if (seen.has(candidateKey(candidate))) continue;
241
- if (selected.some((entry) => entry.model.provider === candidate.model.provider)) continue;
242
- addUnique(selected, candidate, seen, cap);
277
+ // Deeper fallback: alternate models from providers other than the current
278
+ // model's provider. Providers frequently reject one specific model (legacy
279
+ // IDs, account tiers, regional availability) while serving a sibling, so a
280
+ // single interface per family is too fragile. Round-robin across families so
281
+ // every family gets a second chance before any family gets a third, keeping
282
+ // the broad cross-provider coverage that the one-per-family pass provides.
283
+ //
284
+ // Only the current model's provider id is excluded: a different provider id
285
+ // in the same family (for example `openai` vs `openai-codex`, or `anthropic`
286
+ // vs `opencode-go`) has its own credentials and endpoint and is still worth
287
+ // trying. This mirrors the original "never repeat a provider id" rule while
288
+ // still allowing sibling models of a fallback provider.
289
+ const currentProvider = resolution.current?.model.provider;
290
+ const queues = new Map<ProviderFamily, Candidate[]>();
291
+ for (const family of PROVIDER_FAMILIES) {
292
+ const queue = usableFamilyCandidates(resolution, family, isUsable).filter(
293
+ (candidate) =>
294
+ candidate.model.provider !== currentProvider
295
+ && !seen.has(candidateKey(candidate)),
296
+ );
297
+ if (queue.length > 0) queues.set(family, queue);
298
+ }
299
+
300
+ let progressed = true;
301
+ while (selected.length < cap && progressed) {
302
+ progressed = false;
303
+ for (const family of PROVIDER_FAMILIES) {
304
+ if (selected.length >= cap) break;
305
+ const queue = queues.get(family);
306
+ const next = queue?.shift();
307
+ if (next && addUnique(selected, next, seen, cap)) progressed = true;
308
+ }
243
309
  }
244
310
 
245
311
  return selected;
package/lib/format.ts CHANGED
@@ -1,9 +1,10 @@
1
1
  import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
2
+ import { formatJsonWithinLimit } from "./truncate-json.ts";
2
3
 
3
- const configuredOutputLimit = Number(process.env.PI_BETTER_WEB_SEARCH_MAX_CHARS);
4
+ const configuredOutputLimit = Math.floor(Number(process.env.PI_BETTER_WEB_SEARCH_MAX_CHARS));
4
5
  export const MAX_OUTPUT_CHARS =
5
- Number.isFinite(configuredOutputLimit) && configuredOutputLimit > 0
6
- ? Math.floor(configuredOutputLimit)
6
+ Number.isFinite(configuredOutputLimit) && configuredOutputLimit >= 1
7
+ ? configuredOutputLimit
7
8
  : 64_000;
8
9
 
9
10
  export interface WebSearchOutput {
@@ -41,8 +42,6 @@ export interface WebSearchDetails {
41
42
  error?: string;
42
43
  }
43
44
 
44
- const TRUNCATION_MARKER = "\n\n[Truncated]";
45
-
46
45
  /**
47
46
  * Truncate by Unicode code point so a surrogate pair is never split. The
48
47
  * returned `text` is at most `maxChars` code points.
@@ -61,11 +60,9 @@ export function formatSuccess(
61
60
  output: WebSearchOutput,
62
61
  details: WebSearchDetails,
63
62
  ): AgentToolResult<WebSearchDetails> {
64
- const json = JSON.stringify(output, null, 2);
65
- const marker = TRUNCATION_MARKER.slice(0, Math.max(0, MAX_OUTPUT_CHARS));
66
- const { text, truncated } = truncateText(json, Math.max(0, MAX_OUTPUT_CHARS - marker.length));
63
+ const { text, truncated } = formatJsonWithinLimit(output, MAX_OUTPUT_CHARS);
67
64
  return {
68
- content: [{ type: "text", text: truncated ? text + marker : text }],
65
+ content: [{ type: "text", text }],
69
66
  details: truncated ? { ...details, truncated: true } : details,
70
67
  };
71
68
  }
package/lib/search.ts CHANGED
@@ -173,6 +173,33 @@ function attemptKey(candidate: Candidate): ProviderRef {
173
173
  return { provider: candidate.model.provider, model: candidate.model.id };
174
174
  }
175
175
 
176
+ /**
177
+ * Provider-wide failures cannot be fixed by trying another model from the same
178
+ * provider: the credential, balance, network path, or request shape is broken.
179
+ * Model-specific failures (an unsupported or missing model, or a rejected
180
+ * legacy model) can be, so the orchestrator keeps trying sibling models for
181
+ * those. The classifier is intentionally conservative: anything it does not
182
+ * recognize is treated as model-specific, which only costs an extra attempt.
183
+ */
184
+ const PROVIDER_WIDE_FAILURE_PATTERNS: readonly RegExp[] = [
185
+ // HTTP status codes as adapters format them: `(401 Unauthorized)`,
186
+ // `(500 Internal Server Error)`, `HTTP 429`, `status code 403`.
187
+ /(?:\(|HTTP\s+|status(?:\s+code)?\s*)(?:401|403|429)\b/i,
188
+ /(?:\(|HTTP\s+|status(?:\s+code)?\s*)5\d{2}\b/i,
189
+ /\bunauthoriz(?:ed|ation)?\b/i,
190
+ /\bforbidden\b/i,
191
+ /\binsufficient\b[^.]*\b(?:balance|quota|credit|credits|funds|billing)\b/i,
192
+ /\brate.?limit/i,
193
+ /\b(?:ECONNREFUSED|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH)\b/,
194
+ /\bfetch failed\b/i,
195
+ /\btimed?\s?out\b/i,
196
+ /\bbytestring\b/i,
197
+ ];
198
+
199
+ export function isProviderWideFailure(message: string): boolean {
200
+ return PROVIDER_WIDE_FAILURE_PATTERNS.some((pattern) => pattern.test(message));
201
+ }
202
+
176
203
  function resultCount(result: { results?: unknown }): number | undefined {
177
204
  return Array.isArray(result.results) ? result.results.length : undefined;
178
205
  }
@@ -222,8 +249,14 @@ async function runSearch(
222
249
  const candidates = getSearchCandidates(resolution);
223
250
  if (candidates.length === 0) return undefined;
224
251
 
252
+ // Providers whose failure was not model-specific are not retried with a
253
+ // different model, so a dead credential or network path does not burn the
254
+ // whole attempt budget on one interface.
255
+ const failedProviders = new Set<string>();
256
+
225
257
  for (const candidate of candidates) {
226
258
  if (externalSignal?.aborted) throw abortReason(externalSignal);
259
+ if (failedProviders.has(candidate.model.provider)) continue;
227
260
 
228
261
  const timeout = timeoutMs();
229
262
  const signal = attemptSignal(externalSignal, timeout);
@@ -238,12 +271,14 @@ async function runSearch(
238
271
  return { candidate, result: result ?? {} };
239
272
  } catch (error) {
240
273
  if (externalSignal?.aborted) throw abortReason(externalSignal);
274
+ const message = errorMessage(error);
241
275
  attempts.push({
242
276
  phase: "search",
243
277
  provider: candidate.model.provider,
244
278
  model: candidate.model.id,
245
- message: errorMessage(error),
279
+ message,
246
280
  });
281
+ if (isProviderWideFailure(message)) failedProviders.add(candidate.model.provider);
247
282
  }
248
283
  }
249
284
 
@@ -309,9 +344,14 @@ async function runFetch(
309
344
  let preferred: Candidate | undefined;
310
345
  let preferredSuccessful = 0;
311
346
 
347
+ // As with search, a provider-wide failure is not retried with another model
348
+ // from the same provider id.
349
+ const failedProviders = new Set<string>();
350
+
312
351
  for (const candidate of candidates) {
313
352
  if (remaining.length === 0) break;
314
353
  if (externalSignal?.aborted) throw abortReason(externalSignal);
354
+ if (failedProviders.has(candidate.model.provider)) continue;
315
355
 
316
356
  const timeout = timeoutMs();
317
357
  const signal = attemptSignal(externalSignal, timeout);
@@ -356,6 +396,7 @@ async function runFetch(
356
396
  model: candidate.model.id,
357
397
  message,
358
398
  });
399
+ if (isProviderWideFailure(message)) failedProviders.add(candidate.model.provider);
359
400
  for (const index of remaining) {
360
401
  states[index].error = message;
361
402
  }
@@ -0,0 +1,299 @@
1
+ type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
2
+
3
+ interface ChangeTracker {
4
+ paths: string[];
5
+ seen: Set<string>;
6
+ more: boolean;
7
+ }
8
+
9
+ interface TruncationInfo {
10
+ truncated: true;
11
+ originalChars: number;
12
+ affectedPaths: string[];
13
+ moreAffectedPaths: boolean;
14
+ }
15
+
16
+ interface Slot {
17
+ index: number;
18
+ key?: string;
19
+ value: JsonValue;
20
+ minimum: number;
21
+ full: number;
22
+ allocation: number;
23
+ }
24
+
25
+ const MAX_RECORDED_PATHS = 64;
26
+ const MAX_PATH_LENGTH = 160;
27
+ const MAX_PATH_BUDGET = 2_048;
28
+ const STRING_MARKER = "…[truncated]";
29
+ const IDENTITY_KEYS = ["url", "uri", "title", "name", "id", "error", "status", "query", "answer", "content", "text"];
30
+ const IMPORTANT_KEYS = [...IDENTITY_KEYS, "search", "fetched", "results", "data"];
31
+ const KEY_PRIORITY = new Map(IMPORTANT_KEYS.map((key, index) => [key, index]));
32
+
33
+ /** Count Unicode code points, rather than UTF-16 code units. */
34
+ function charCount(text: string): number {
35
+ let count = 0;
36
+ for (const _character of text) count += 1;
37
+ return count;
38
+ }
39
+
40
+ function recordChange(tracker: ChangeTracker, path: string): void {
41
+ if (tracker.seen.has(path)) return;
42
+ if (tracker.paths.length >= MAX_RECORDED_PATHS) {
43
+ tracker.more = true;
44
+ return;
45
+ }
46
+ tracker.seen.add(path);
47
+ tracker.paths.push(path);
48
+ }
49
+
50
+ function propertyPath(parent: string, key: string): string {
51
+ // A very long or unusual provider field name is represented by its parent,
52
+ // rather than allowing an unbounded field name to consume the path budget.
53
+ if (parent.length + key.length > MAX_PATH_LENGTH) return parent;
54
+ if (/^[A-Za-z_$][\w$]*$/.test(key)) return `${parent}.${key}`;
55
+ return `${parent}[${JSON.stringify(key)}]`;
56
+ }
57
+
58
+ function indexPath(parent: string, index: number): string {
59
+ const path = `${parent}[${index}]`;
60
+ return path.length <= MAX_PATH_LENGTH ? path : parent;
61
+ }
62
+
63
+ /** Encoded length of one code point inside a JSON string (without quotes). */
64
+ function encodedCharacterLength(character: string): number {
65
+ if (character.length === 2) return 1; // a surrogate pair is one code point
66
+ const code = character.charCodeAt(0);
67
+ if (character === '"' || character === "\\") return 2;
68
+ if (code === 8 || code === 9 || code === 10 || code === 12 || code === 13) return 2;
69
+ if (code < 32 || (code >= 0xd800 && code <= 0xdfff)) return 6;
70
+ return 1;
71
+ }
72
+
73
+ function fitString(value: string, budget: number): string {
74
+ // A JSON string needs two quotes. Include a marker whenever it fits; paths in
75
+ // _truncation also identify shortened strings when the budget is very small.
76
+ const marker = budget >= charCount(STRING_MARKER) + 2 ? STRING_MARKER : budget >= 3 ? "…" : "";
77
+ let remaining = budget - 2 - charCount(marker);
78
+ const prefix: string[] = [];
79
+ for (const character of value) {
80
+ const size = encodedCharacterLength(character);
81
+ if (size > remaining) break;
82
+ prefix.push(character);
83
+ remaining -= size;
84
+ }
85
+ return prefix.join("") + marker;
86
+ }
87
+
88
+ /** Distribute the available space fairly, giving unused space to larger values. */
89
+ function allocate(slots: Slot[], available: number): void {
90
+ let remaining = available;
91
+ for (const slot of slots) {
92
+ slot.allocation = slot.minimum;
93
+ remaining -= slot.minimum;
94
+ }
95
+
96
+ const pending = slots
97
+ .filter((slot) => slot.full > slot.minimum)
98
+ .sort((a, b) => (a.full - a.minimum) - (b.full - b.minimum) || a.index - b.index);
99
+
100
+ for (let i = 0; i < pending.length && remaining > 0; i += 1) {
101
+ const slot = pending[i];
102
+ const count = pending.length - i;
103
+ const share = Math.floor(remaining / count);
104
+ const demand = slot.full - slot.minimum;
105
+ if (demand <= share) {
106
+ slot.allocation += demand;
107
+ remaining -= demand;
108
+ continue;
109
+ }
110
+
111
+ const remainder = remaining % count;
112
+ for (let j = i; j < pending.length; j += 1) {
113
+ pending[j].allocation += share + (j - i < remainder ? 1 : 0);
114
+ }
115
+ break;
116
+ }
117
+ }
118
+
119
+ function createFitter(tracker: ChangeTracker) {
120
+ const sizes = new WeakMap<object, number>();
121
+
122
+ const sizeOf = (value: JsonValue): number => {
123
+ if (value !== null && typeof value === "object") {
124
+ const cached = sizes.get(value);
125
+ if (cached !== undefined) return cached;
126
+ const size = charCount(JSON.stringify(value));
127
+ sizes.set(value, size);
128
+ return size;
129
+ }
130
+ return charCount(JSON.stringify(value));
131
+ };
132
+
133
+ const minimumSize = (value: JsonValue): number => {
134
+ if (typeof value === "string" || Array.isArray(value)) return 2; // "" or []
135
+ if (value !== null && typeof value === "object") {
136
+ // An array of hundreds of empty objects is not useful. Reserve enough
137
+ // for one identifying or text field when the item has one; if the
138
+ // budget cannot accommodate all entries, report an omitted suffix.
139
+ for (const key of IDENTITY_KEYS) {
140
+ if (!Object.hasOwn(value, key)) continue;
141
+ const child = value[key];
142
+ if (child !== null && typeof child === "object") continue;
143
+ const childMinimum = typeof child === "string" ? Math.min(sizeOf(child), 128) : sizeOf(child);
144
+ return 2 + charCount(JSON.stringify(key)) + 1 + childMinimum;
145
+ }
146
+ return 2;
147
+ }
148
+ return sizeOf(value);
149
+ };
150
+
151
+ const fit = (value: JsonValue, budget: number, path: string): JsonValue => {
152
+ if (sizeOf(value) <= budget) return value;
153
+
154
+ if (typeof value === "string") {
155
+ recordChange(tracker, path);
156
+ return fitString(value, budget);
157
+ }
158
+
159
+ if (Array.isArray(value)) {
160
+ const slots: Slot[] = [];
161
+ let fixed = 2; // brackets and commas
162
+ let minimums = 0;
163
+ for (let index = 0; index < value.length; index += 1) {
164
+ const child = value[index];
165
+ const minimum = minimumSize(child);
166
+ const punctuation = slots.length > 0 ? 1 : 0;
167
+ if (fixed + punctuation + minimums + minimum > budget) {
168
+ // Preserve array order and report that one or more entries were omitted.
169
+ recordChange(tracker, path);
170
+ break;
171
+ }
172
+ fixed += punctuation;
173
+ minimums += minimum;
174
+ slots.push({ index, value: child, minimum, full: sizeOf(child), allocation: minimum });
175
+ }
176
+ allocate(slots, budget - fixed);
177
+ return slots.map((slot) => fit(slot.value, slot.allocation, indexPath(path, slot.index)));
178
+ }
179
+
180
+ if (value !== null && typeof value === "object") {
181
+ const entries = Object.entries(value);
182
+ // Prefer identifiers and the search/fetch containers when even the
183
+ // structural overhead does not fit. Restore the original field order in
184
+ // the result; no provider data is re-ranked.
185
+ const candidates = entries.map(([key, child], index) => ({ key, child, index }));
186
+ candidates.sort((a, b) =>
187
+ (KEY_PRIORITY.get(a.key) ?? IMPORTANT_KEYS.length) - (KEY_PRIORITY.get(b.key) ?? IMPORTANT_KEYS.length)
188
+ || a.index - b.index,
189
+ );
190
+ const slots: Slot[] = [];
191
+ let fixed = 2; // braces, keys, colons and commas
192
+ let minimums = 0;
193
+ for (const { key, child, index } of candidates) {
194
+ const minimum = minimumSize(child);
195
+ const punctuation = (slots.length > 0 ? 1 : 0) + charCount(JSON.stringify(key)) + 1;
196
+ if (fixed + punctuation + minimums + minimum > budget) {
197
+ recordChange(tracker, propertyPath(path, key));
198
+ continue;
199
+ }
200
+ fixed += punctuation;
201
+ minimums += minimum;
202
+ slots.push({ index, key, value: child, minimum, full: sizeOf(child), allocation: minimum });
203
+ }
204
+ slots.sort((a, b) => a.index - b.index);
205
+ allocate(slots, budget - fixed);
206
+ const result: { [key: string]: JsonValue } = Object.create(null);
207
+ for (const slot of slots) {
208
+ const key = slot.key!;
209
+ result[key] = fit(slot.value, slot.allocation, propertyPath(path, key));
210
+ }
211
+ return result;
212
+ }
213
+
214
+ // Parents reserve the complete serialized size for non-string scalars.
215
+ throw new Error("Insufficient JSON budget for scalar value");
216
+ };
217
+
218
+ return fit;
219
+ }
220
+
221
+ function minimalTruncatedJson(limit: number): string {
222
+ for (const text of ['{"_truncation":{"truncated":true}}', '{"_truncated":true}', "{}", "0"]) {
223
+ if (text.length <= limit) return text;
224
+ }
225
+ throw new RangeError("The JSON output limit must be at least one character");
226
+ }
227
+
228
+ /**
229
+ * Preserve ordinary output exactly. If compact JSON still exceeds the limit,
230
+ * fit its fields and array entries independently and report affected paths.
231
+ * The returned text is valid JSON and never exceeds `maxChars` code points.
232
+ */
233
+ export function formatJsonWithinLimit(
234
+ output: object,
235
+ maxChars: number,
236
+ ): { text: string; truncated: boolean } {
237
+ if (!Number.isFinite(maxChars) || maxChars < 1) {
238
+ throw new RangeError("The JSON output limit must be at least one character");
239
+ }
240
+ const limit = Math.floor(maxChars);
241
+ const pretty = JSON.stringify(output, null, 2);
242
+ const originalChars = charCount(pretty);
243
+ if (originalChars <= limit) return { text: pretty, truncated: false };
244
+
245
+ // Work from the already serialized value, so JSON's handling of undefined,
246
+ // dates and toJSON is identical even if an input object is not plain data.
247
+ const parsed = JSON.parse(pretty) as { [key: string]: JsonValue };
248
+ const compact = JSON.stringify(parsed);
249
+ if (charCount(compact) <= limit) return { text: compact, truncated: false };
250
+
251
+ const info: TruncationInfo = {
252
+ truncated: true,
253
+ originalChars,
254
+ affectedPaths: [],
255
+ moreAffectedPaths: false,
256
+ };
257
+ // Account for the metadata entry and a comma joining it to the root object.
258
+ const overhead = charCount(JSON.stringify({ _truncation: info })) - 2 + 1;
259
+ if (limit - overhead < 2) return { text: minimalTruncatedJson(limit), truncated: true };
260
+
261
+ const build = (reservedPaths: number) => {
262
+ const tracker: ChangeTracker = { paths: [], seen: new Set(), more: false };
263
+ const fit = createFitter(tracker);
264
+ const fitted = fit(parsed, limit - overhead - reservedPaths, "$") as { [key: string]: JsonValue };
265
+ return { fitted, tracker };
266
+ };
267
+
268
+ let result = build(0);
269
+ // Reserve just enough room for the first pass's affected paths, capped so
270
+ // reporting never consumes a disproportionate share of the result budget.
271
+ const desiredPathSpace = Math.min(
272
+ MAX_PATH_BUDGET,
273
+ Math.max(64, Math.floor(limit * 0.05)),
274
+ charCount(JSON.stringify(result.tracker.paths)) - 2,
275
+ );
276
+ if (desiredPathSpace > 0 && limit - overhead - desiredPathSpace >= 2) {
277
+ result = build(desiredPathSpace);
278
+ }
279
+
280
+ const combined: { [key: string]: unknown } = Object.create(null);
281
+ combined._truncation = info;
282
+ for (const [key, value] of Object.entries(result.fitted)) {
283
+ if (key !== "_truncation") combined[key] = value;
284
+ }
285
+
286
+ let remaining = limit - charCount(JSON.stringify(combined));
287
+ for (const path of result.tracker.paths) {
288
+ const cost = charCount(JSON.stringify(path)) + (info.affectedPaths.length > 0 ? 1 : 0);
289
+ if (cost > remaining) {
290
+ info.moreAffectedPaths = true;
291
+ break;
292
+ }
293
+ info.affectedPaths.push(path);
294
+ remaining -= cost;
295
+ }
296
+ if (result.tracker.more) info.moreAffectedPaths = true;
297
+
298
+ return { text: JSON.stringify(combined), truncated: true };
299
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-better-web-search",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "description": "A better provider-native web_search tool for pi: raw results, automatic provider fallback, URL analysis, and native DeepSeek Search/Fetch.",
5
5
  "type": "module",
6
6
  "license": "MIT",