pi-better-web-search 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +20 -14
- package/docs/README.zh-CN.md +19 -13
- package/docs/architecture.md +14 -8
- package/docs/publishing.md +6 -6
- package/docs/testing.md +4 -3
- package/lib/format.ts +1 -1
- package/lib/truncate-json.ts +111 -133
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.1.7
|
|
4
|
+
|
|
5
|
+
- Keep recognized URL/URI/link and title/name/ID fields intact when truncating;
|
|
6
|
+
omit entries that cannot fit rather than returning partial links. Preserve
|
|
7
|
+
provider order and drop incomplete URL tokens at text boundaries.
|
|
8
|
+
- Raise the default model-facing output limit from 64,000 to 96,000 Unicode
|
|
9
|
+
code points; the environment override remains available.
|
|
10
|
+
- Replace model-facing truncation metadata and per-string markers with a single
|
|
11
|
+
`[truncated]` line at the end of oversized results. Keep the existing fair
|
|
12
|
+
allocation across search results and fetched pages.
|
|
13
|
+
|
|
3
14
|
## 0.1.6
|
|
4
15
|
|
|
5
16
|
- Keep oversized results as valid JSON, sharing the character budget across
|
package/README.md
CHANGED
|
@@ -252,20 +252,24 @@ this:
|
|
|
252
252
|
- Redirects are not resolved and sources are not derived.
|
|
253
253
|
- Unknown provider fields are preserved when the result fits the size guard.
|
|
254
254
|
|
|
255
|
-
A final output-size guard limits successful model-facing results to **
|
|
255
|
+
A final output-size guard limits successful model-facing results to **96,000
|
|
256
256
|
Unicode code points** by default. Normal-sized results retain their original
|
|
257
257
|
fields and formatting. If indentation alone exceeds the limit, compact JSON is
|
|
258
258
|
returned without dropping data or setting the truncation flag.
|
|
259
259
|
|
|
260
260
|
When even compact JSON is too large, the guard allocates space across sibling
|
|
261
|
-
fields and array entries, redistributing unused space.
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
entries;
|
|
261
|
+
fields and array entries, redistributing unused space. Recognized link and
|
|
262
|
+
identity fields (`url`, `uri`, `href`, `link`, `title`, `name`, `id`, and common
|
|
263
|
+
aliases) stay complete; if they cannot fit, the associated entry or branch is
|
|
264
|
+
omitted instead. Standalone URLs and entries in link/citation arrays also stay
|
|
265
|
+
whole. When a text cut would split a recognizable embedded URL token (such as
|
|
266
|
+
`https://`, `mailto:`, or `www.`), the incomplete token is omitted. Relative
|
|
267
|
+
links or bare domains in unrecognized fields cannot always be identified. Other long strings may be shortened, and fields or trailing array
|
|
268
|
+
entries omitted; results are not re-ranked or summarized.
|
|
269
|
+
|
|
270
|
+
The JSON portion remains valid, followed by a single `[truncated]` line. No
|
|
271
|
+
truncation metadata or per-string markers are added to the model-facing JSON.
|
|
272
|
+
Because of the trailing line, the complete text is not itself valid JSON.
|
|
269
273
|
`details.truncated` is also set to `true` when data is actually removed.
|
|
270
274
|
|
|
271
275
|
Change the limit **before starting pi**:
|
|
@@ -276,10 +280,12 @@ export PI_BETTER_WEB_SEARCH_MAX_CHARS=100000
|
|
|
276
280
|
|
|
277
281
|
In PowerShell, use `$env:PI_BETTER_WEB_SEARCH_MAX_CHARS = "100000"`. The limit
|
|
278
282
|
is in code points, **not tokens**, and does not adapt to the current model's
|
|
279
|
-
context window.
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
+
context window. The marker counts toward the limit. If truncation is needed and
|
|
284
|
+
the limit is 11–13 characters, only `[truncated]` is returned; below 11
|
|
285
|
+
characters the marker does not fit and only a minimal JSON value is returned.
|
|
286
|
+
`details.truncated` remains true in both cases. There is no continuation or
|
|
287
|
+
pagination cache: retry individual URLs or raise the limit if more content is
|
|
288
|
+
needed.
|
|
283
289
|
|
|
284
290
|
## Exposure rules
|
|
285
291
|
|
|
@@ -384,7 +390,7 @@ npm run build
|
|
|
384
390
|
```
|
|
385
391
|
|
|
386
392
|
`npm run build` runs the checks and tests, verifies the npm file list, and packs
|
|
387
|
-
`dist/pi-better-web-search-0.1.
|
|
393
|
+
`dist/pi-better-web-search-0.1.7.tgz`. Nothing is published automatically.
|
|
388
394
|
|
|
389
395
|
The test suite uses plain `.mjs` files that import the `.ts` sources directly.
|
|
390
396
|
It mocks `globalThis.fetch`, uses a temporary cache file, and exercises
|
package/docs/README.zh-CN.md
CHANGED
|
@@ -217,18 +217,23 @@ Provider 数据放在 JSON 外层中返回,不做清理。示例:
|
|
|
217
217
|
- 不解析 redirect,不派生 sources。
|
|
218
218
|
- 未超出大小限制时,provider 的未知字段也会保留。
|
|
219
219
|
|
|
220
|
-
最终大小保护默认限制成功结果不超过 **
|
|
220
|
+
最终大小保护默认限制成功结果不超过 **96,000 个 Unicode code point**。
|
|
221
221
|
未超限时保留原始字段及格式;如果只是缩进导致超限,会先压缩 JSON,
|
|
222
222
|
不丢失数据,也不标记为截断。
|
|
223
223
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
`
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
224
|
+
压缩后仍超限时,按字段和数组条目分配预算,未用完的额度会重新分配。
|
|
225
|
+
已识别的链接及标识字段(包括 `url`、`uri`、`href`、`link`、`title`、
|
|
226
|
+
`name`、`id` 和常见别名)保持完整;放不下时省略对应条目或分支,而不是
|
|
227
|
+
返回半截值。独立 URL 和链接、引用数组中的条目也保持完整;正文截断点
|
|
228
|
+
若落在可识别的 URL(如 `https://`、`mailto:` 或 `www.`)中间,会省略该
|
|
229
|
+
未完整显示的 URL。未知字段中的相对链接或裸域名无法保证都能识别。
|
|
230
|
+
其他长字符串可能被缩短,字段或数组末尾的条目可能被省略;
|
|
231
|
+
不对结果重排或总结。
|
|
232
|
+
|
|
233
|
+
JSON 部分仍然合法,末尾仅追加一行 `[truncated]`。不会向模型可见的 JSON
|
|
234
|
+
中添加截断元数据,也不会逐个标记缩短的字符串。由于末尾追加了这一行,
|
|
235
|
+
整个文本不再是合法 JSON。确实丢失数据时,`details.truncated` 也会设为
|
|
236
|
+
`true`。
|
|
232
237
|
|
|
233
238
|
可在**启动 pi 之前**通过环境变量调整:
|
|
234
239
|
|
|
@@ -238,9 +243,10 @@ export PI_BETTER_WEB_SEARCH_MAX_CHARS=100000
|
|
|
238
243
|
|
|
239
244
|
PowerShell 使用 `$env:PI_BETTER_WEB_SEARCH_MAX_CHARS = "100000"`。
|
|
240
245
|
上限是 code point,**不是 token**,不会根据当前模型的上下文窗口自动调整。
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
246
|
+
标记也计入上限。需要截断且上限为 11–13 字符时,只返回 `[truncated]`;
|
|
247
|
+
少于 11 字符时放不下标记,只返回一个最小 JSON 值。这两种情况仍会通过
|
|
248
|
+
`details.truncated` 报告截断。当前没有续读或分页缓存;需要更多内容时可
|
|
249
|
+
单独重新抓取 URL 或提高上限。
|
|
244
250
|
|
|
245
251
|
## 暴露规则
|
|
246
252
|
|
|
@@ -323,7 +329,7 @@ npm run build
|
|
|
323
329
|
```
|
|
324
330
|
|
|
325
331
|
`npm run build` 会运行检查、测试、包内容验证,并生成
|
|
326
|
-
`dist/pi-better-web-search-0.1.
|
|
332
|
+
`dist/pi-better-web-search-0.1.7.tgz`。不会自动发布。
|
|
327
333
|
|
|
328
334
|
测试是 `.mjs` 文件,直接导入 `.ts` 源码,mock `globalThis.fetch`,使用临时缓存
|
|
329
335
|
目录。详见 [测试](testing.md) 和 [发布](publishing.md)。
|
package/docs/architecture.md
CHANGED
|
@@ -161,15 +161,21 @@ The result is a JSON wrapper:
|
|
|
161
161
|
Fields inside `search` and `fetched[].data` are provider data. The extension does
|
|
162
162
|
not deduplicate, normalize, filter, re-rank, summarize, insert citation markers,
|
|
163
163
|
or append source sections. The final guard keeps successful model-facing output
|
|
164
|
-
within
|
|
164
|
+
within 96,000 Unicode code points by default (`PI_BETTER_WEB_SEARCH_MAX_CHARS`).
|
|
165
165
|
It preserves ordinary formatted JSON, or uses compact JSON if that alone fits.
|
|
166
|
-
Otherwise it budgets sibling fields and array entries independently
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
166
|
+
Otherwise it budgets sibling fields and array entries independently. Recognized
|
|
167
|
+
link and identity fields stay whole; entries or branches that cannot fit them
|
|
168
|
+
are omitted instead, without re-ranking. Standalone URLs and link/citation array
|
|
169
|
+
entries remain intact, and a recognizable URL crossing a text cut is omitted
|
|
170
|
+
rather than partially emitted. Unknown relative links or bare domains may not be
|
|
171
|
+
identifiable. Other strings can be shortened, and fields or trailing array entries
|
|
172
|
+
omitted. The JSON portion
|
|
173
|
+
stays valid, followed by one `[truncated]` line; the complete text is therefore
|
|
174
|
+
not valid JSON. No truncation metadata is injected into the model-facing JSON,
|
|
175
|
+
while `details.truncated` marks actual data loss. When truncation is needed with a limit of 11–13 characters,
|
|
176
|
+
only the marker is returned; below 11, a minimal JSON value is returned instead.
|
|
177
|
+
This is a character guard, not a token or model-context limit; it does not
|
|
178
|
+
provide pagination or retain the complete provider payload for later use.
|
|
173
179
|
|
|
174
180
|
## Provider adapter contract
|
|
175
181
|
|
package/docs/publishing.md
CHANGED
|
@@ -30,7 +30,7 @@ local `pi-web-search` reference checkout, `.env` files, and `DESIGN.md`.
|
|
|
30
30
|
npm run build
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
-
This produces `dist/pi-better-web-search-0.1.
|
|
33
|
+
This produces `dist/pi-better-web-search-0.1.7.tgz`. The tarball is not
|
|
34
34
|
committed.
|
|
35
35
|
|
|
36
36
|
## Publish
|
|
@@ -42,7 +42,7 @@ npm publish --access public
|
|
|
42
42
|
Alternatively, publish the verified tarball directly:
|
|
43
43
|
|
|
44
44
|
```bash
|
|
45
|
-
npm publish dist/pi-better-web-search-0.1.
|
|
45
|
+
npm publish dist/pi-better-web-search-0.1.7.tgz --access public
|
|
46
46
|
```
|
|
47
47
|
|
|
48
48
|
`prepublishOnly` reruns the checks, tests, and package verification. npm may
|
|
@@ -51,10 +51,10 @@ require browser or OTP authentication.
|
|
|
51
51
|
## Tag and release
|
|
52
52
|
|
|
53
53
|
```bash
|
|
54
|
-
git tag -a v0.1.
|
|
55
|
-
git push origin v0.1.
|
|
56
|
-
gh release create v0.1.
|
|
57
|
-
--title "v0.1.
|
|
54
|
+
git tag -a v0.1.7 -m "v0.1.7"
|
|
55
|
+
git push origin v0.1.7
|
|
56
|
+
gh release create v0.1.7 dist/pi-better-web-search-0.1.7.tgz \
|
|
57
|
+
--title "v0.1.7" \
|
|
58
58
|
--notes-file /path/to/release-notes.md
|
|
59
59
|
```
|
|
60
60
|
|
package/docs/testing.md
CHANGED
|
@@ -17,7 +17,7 @@ npm run verify:package
|
|
|
17
17
|
tarball contains runtime sources, docs, assets, and metadata, while excluding
|
|
18
18
|
tests, scripts, CI files, the local reference checkout, and `DESIGN.md`.
|
|
19
19
|
- `npm run build` runs the checks, tests, package verification, and packs
|
|
20
|
-
`dist/pi-better-web-search-0.1.
|
|
20
|
+
`dist/pi-better-web-search-0.1.7.tgz`.
|
|
21
21
|
|
|
22
22
|
Tests import the TypeScript sources directly. The suite mocks `globalThis.fetch`
|
|
23
23
|
and uses temporary cache directories. Coverage includes:
|
|
@@ -30,8 +30,9 @@ and uses temporary cache directories. Coverage includes:
|
|
|
30
30
|
- search and fetch fallback, including per-URL fallback;
|
|
31
31
|
- provider request shapes and SSE parsing;
|
|
32
32
|
- HTTP/stream errors and abort propagation;
|
|
33
|
-
- structured character-guard truncation,
|
|
34
|
-
|
|
33
|
+
- structured character-guard truncation, a single trailing marker, valid JSON
|
|
34
|
+
before the marker, Unicode and escaping, boundary lengths, fair allocation,
|
|
35
|
+
and intact links and identifiers across search results and fetched pages;
|
|
35
36
|
- one-tool registration and no system-prompt hooks.
|
|
36
37
|
|
|
37
38
|
## Live DeepSeek test
|
package/lib/format.ts
CHANGED
|
@@ -5,7 +5,7 @@ const configuredOutputLimit = Math.floor(Number(process.env.PI_BETTER_WEB_SEARCH
|
|
|
5
5
|
export const MAX_OUTPUT_CHARS =
|
|
6
6
|
Number.isFinite(configuredOutputLimit) && configuredOutputLimit >= 1
|
|
7
7
|
? configuredOutputLimit
|
|
8
|
-
:
|
|
8
|
+
: 96_000;
|
|
9
9
|
|
|
10
10
|
export interface WebSearchOutput {
|
|
11
11
|
query?: string;
|
package/lib/truncate-json.ts
CHANGED
|
@@ -1,18 +1,5 @@
|
|
|
1
1
|
type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
|
|
2
2
|
|
|
3
|
-
interface ChangeTracker {
|
|
4
|
-
paths: string[];
|
|
5
|
-
seen: Set<string>;
|
|
6
|
-
more: boolean;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
interface TruncationInfo {
|
|
10
|
-
truncated: true;
|
|
11
|
-
originalChars: number;
|
|
12
|
-
affectedPaths: string[];
|
|
13
|
-
moreAffectedPaths: boolean;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
3
|
interface Slot {
|
|
17
4
|
index: number;
|
|
18
5
|
key?: string;
|
|
@@ -22,42 +9,33 @@ interface Slot {
|
|
|
22
9
|
allocation: number;
|
|
23
10
|
}
|
|
24
11
|
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
const MAX_PATH_BUDGET = 2_048;
|
|
28
|
-
const STRING_MARKER = "…[truncated]";
|
|
29
|
-
const IDENTITY_KEYS = ["url", "uri", "title", "name", "id", "error", "status", "query", "answer", "content", "text"];
|
|
12
|
+
const TRUNCATION_MARKER = "[truncated]";
|
|
13
|
+
const IDENTITY_KEYS = ["url", "uri", "href", "link", "title", "name", "id", "error", "status", "query", "answer", "content", "text"];
|
|
30
14
|
const IMPORTANT_KEYS = [...IDENTITY_KEYS, "search", "fetched", "results", "data"];
|
|
31
15
|
const KEY_PRIORITY = new Map(IMPORTANT_KEYS.map((key, index) => [key, index]));
|
|
32
16
|
|
|
33
|
-
/**
|
|
34
|
-
function
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
17
|
+
/** Link and identity fields are kept intact or their containing entry is omitted. */
|
|
18
|
+
function isProtectedKey(key: string): boolean {
|
|
19
|
+
return /^(?:url|uri|href|link|permalink|website|title|name|id)$/i.test(key)
|
|
20
|
+
|| /[_-](?:url|uri|href|link|permalink|website|title|name|id)$/i.test(key)
|
|
21
|
+
|| /(?:URL|URI|HREF|LINK|TITLE|NAME|Url|Uri|Href|Link|Title|Name|Id|ID)$/.test(key);
|
|
38
22
|
}
|
|
39
23
|
|
|
40
|
-
function
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
tracker.seen.add(path);
|
|
47
|
-
tracker.paths.push(path);
|
|
24
|
+
function isLinkCollectionKey(key: string): boolean {
|
|
25
|
+
return /^(?:urls?|uris?|hrefs?|links?|permalinks?|websites?|titles?|names?|ids?|sources?|citations?)$/i.test(key)
|
|
26
|
+
|| /[_-](?:urls|uris|hrefs|links|titles|names|ids)$/i.test(key)
|
|
27
|
+
|| /(?:URLs|URIs|Urls|Uris|Hrefs|Links|Titles|Names|Ids|IDs)$/.test(key);
|
|
48
28
|
}
|
|
49
29
|
|
|
50
|
-
function
|
|
51
|
-
|
|
52
|
-
// rather than allowing an unbounded field name to consume the path budget.
|
|
53
|
-
if (parent.length + key.length > MAX_PATH_LENGTH) return parent;
|
|
54
|
-
if (/^[A-Za-z_$][\w$]*$/.test(key)) return `${parent}.${key}`;
|
|
55
|
-
return `${parent}[${JSON.stringify(key)}]`;
|
|
30
|
+
function isStandaloneUrl(value: string): boolean {
|
|
31
|
+
return /^(?:[a-z][a-z0-9+.-]{0,31}:\/\/|mailto:|urn:|doi:|tel:|www\.)\S+$/i.test(value);
|
|
56
32
|
}
|
|
57
33
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
34
|
+
/** Count Unicode code points, rather than UTF-16 code units. */
|
|
35
|
+
function charCount(text: string): number {
|
|
36
|
+
let count = 0;
|
|
37
|
+
for (const _character of text) count += 1;
|
|
38
|
+
return count;
|
|
61
39
|
}
|
|
62
40
|
|
|
63
41
|
/** Encoded length of one code point inside a JSON string (without quotes). */
|
|
@@ -71,10 +49,8 @@ function encodedCharacterLength(character: string): number {
|
|
|
71
49
|
}
|
|
72
50
|
|
|
73
51
|
function fitString(value: string, budget: number): string {
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
const marker = budget >= charCount(STRING_MARKER) + 2 ? STRING_MARKER : budget >= 3 ? "…" : "";
|
|
77
|
-
let remaining = budget - 2 - charCount(marker);
|
|
52
|
+
// Reserve the quotes; the single truncation marker is added after the JSON.
|
|
53
|
+
let remaining = budget - 2;
|
|
78
54
|
const prefix: string[] = [];
|
|
79
55
|
for (const character of value) {
|
|
80
56
|
const size = encodedCharacterLength(character);
|
|
@@ -82,7 +58,19 @@ function fitString(value: string, budget: number): string {
|
|
|
82
58
|
prefix.push(character);
|
|
83
59
|
remaining -= size;
|
|
84
60
|
}
|
|
85
|
-
|
|
61
|
+
const visible = prefix.join("");
|
|
62
|
+
if (visible.length === value.length) return visible;
|
|
63
|
+
|
|
64
|
+
// A cut inside a URL embedded in prose must not leave a plausible but broken
|
|
65
|
+
// link. Drop the incomplete URL token instead of changing its address.
|
|
66
|
+
const urlStart = /\b(?:[a-z][a-z0-9+.-]{0,31}:\/\/|mailto:|urn:|doi:|tel:|www\.)/gi;
|
|
67
|
+
for (let match = urlStart.exec(value); match && match.index < visible.length; match = urlStart.exec(value)) {
|
|
68
|
+
let end = match.index + match[0].length;
|
|
69
|
+
while (end < value.length && !/\s/.test(value[end])) end += 1;
|
|
70
|
+
if (end > visible.length) return visible.slice(0, match.index);
|
|
71
|
+
urlStart.lastIndex = end;
|
|
72
|
+
}
|
|
73
|
+
return visible;
|
|
86
74
|
}
|
|
87
75
|
|
|
88
76
|
/** Distribute the available space fairly, giving unused space to larger values. */
|
|
@@ -116,8 +104,9 @@ function allocate(slots: Slot[], available: number): void {
|
|
|
116
104
|
}
|
|
117
105
|
}
|
|
118
106
|
|
|
119
|
-
function createFitter(
|
|
107
|
+
function createFitter() {
|
|
120
108
|
const sizes = new WeakMap<object, number>();
|
|
109
|
+
const requiredSizes = new WeakMap<object, number>();
|
|
121
110
|
|
|
122
111
|
const sizeOf = (value: JsonValue): number => {
|
|
123
112
|
if (value !== null && typeof value === "object") {
|
|
@@ -130,31 +119,61 @@ function createFitter(tracker: ChangeTracker) {
|
|
|
130
119
|
return charCount(JSON.stringify(value));
|
|
131
120
|
};
|
|
132
121
|
|
|
133
|
-
|
|
134
|
-
if (
|
|
122
|
+
function requiredPropertySize(key: string, value: JsonValue): number | undefined {
|
|
123
|
+
if (isProtectedKey(key) && !Array.isArray(value)) return sizeOf(value);
|
|
135
124
|
if (value !== null && typeof value === "object") {
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
125
|
+
if (Array.isArray(value)) return undefined;
|
|
126
|
+
const nested = requiredSize(value);
|
|
127
|
+
return nested > 0 ? nested : undefined;
|
|
128
|
+
}
|
|
129
|
+
if (typeof value === "string" && isStandaloneUrl(value)) return sizeOf(value);
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Reserve every direct or nested identity field before allocating space to
|
|
134
|
+
// optional content. Arrays can still omit a suffix of complete entries.
|
|
135
|
+
function requiredSize(value: { [key: string]: JsonValue }): number {
|
|
136
|
+
const cached = requiredSizes.get(value);
|
|
137
|
+
if (cached !== undefined) return cached;
|
|
138
|
+
let size = 2;
|
|
139
|
+
let count = 0;
|
|
140
|
+
for (const [key, child] of Object.entries(value)) {
|
|
141
|
+
const required = requiredPropertySize(key, child);
|
|
142
|
+
if (required === undefined) continue;
|
|
143
|
+
size += (count > 0 ? 1 : 0) + charCount(JSON.stringify(key)) + 1 + required;
|
|
144
|
+
count += 1;
|
|
145
|
+
}
|
|
146
|
+
const result = count > 0 ? size : 0;
|
|
147
|
+
requiredSizes.set(value, result);
|
|
148
|
+
return result;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const minimumSize = (value: JsonValue, key = "", linkCollection = false): number => {
|
|
152
|
+
if (Array.isArray(value)) return 2; // []
|
|
153
|
+
if (isProtectedKey(key)) return sizeOf(value);
|
|
154
|
+
if (typeof value === "string") {
|
|
155
|
+
return linkCollection || isStandaloneUrl(value) ? sizeOf(value) : 2;
|
|
156
|
+
}
|
|
157
|
+
if (value !== null && typeof value === "object") {
|
|
158
|
+
const required = requiredSize(value);
|
|
159
|
+
if (required > 0) return required;
|
|
160
|
+
// Avoid arrays of empty objects when a useful text field is available.
|
|
161
|
+
for (const identity of IDENTITY_KEYS) {
|
|
162
|
+
if (!Object.hasOwn(value, identity)) continue;
|
|
163
|
+
const child = value[identity];
|
|
142
164
|
if (child !== null && typeof child === "object") continue;
|
|
143
165
|
const childMinimum = typeof child === "string" ? Math.min(sizeOf(child), 128) : sizeOf(child);
|
|
144
|
-
return 2 + charCount(JSON.stringify(
|
|
166
|
+
return 2 + charCount(JSON.stringify(identity)) + 1 + childMinimum;
|
|
145
167
|
}
|
|
146
168
|
return 2;
|
|
147
169
|
}
|
|
148
170
|
return sizeOf(value);
|
|
149
171
|
};
|
|
150
172
|
|
|
151
|
-
const fit = (value: JsonValue, budget: number,
|
|
173
|
+
const fit = (value: JsonValue, budget: number, linkCollection = false, root = false): JsonValue => {
|
|
152
174
|
if (sizeOf(value) <= budget) return value;
|
|
153
175
|
|
|
154
|
-
if (typeof value === "string")
|
|
155
|
-
recordChange(tracker, path);
|
|
156
|
-
return fitString(value, budget);
|
|
157
|
-
}
|
|
176
|
+
if (typeof value === "string") return fitString(value, budget);
|
|
158
177
|
|
|
159
178
|
if (Array.isArray(value)) {
|
|
160
179
|
const slots: Slot[] = [];
|
|
@@ -162,11 +181,10 @@ function createFitter(tracker: ChangeTracker) {
|
|
|
162
181
|
let minimums = 0;
|
|
163
182
|
for (let index = 0; index < value.length; index += 1) {
|
|
164
183
|
const child = value[index];
|
|
165
|
-
const minimum = minimumSize(child);
|
|
184
|
+
const minimum = minimumSize(child, "", linkCollection);
|
|
166
185
|
const punctuation = slots.length > 0 ? 1 : 0;
|
|
167
186
|
if (fixed + punctuation + minimums + minimum > budget) {
|
|
168
|
-
// Preserve array order
|
|
169
|
-
recordChange(tracker, path);
|
|
187
|
+
// Preserve array order when some entries must be omitted.
|
|
170
188
|
break;
|
|
171
189
|
}
|
|
172
190
|
fixed += punctuation;
|
|
@@ -174,29 +192,30 @@ function createFitter(tracker: ChangeTracker) {
|
|
|
174
192
|
slots.push({ index, value: child, minimum, full: sizeOf(child), allocation: minimum });
|
|
175
193
|
}
|
|
176
194
|
allocate(slots, budget - fixed);
|
|
177
|
-
return slots.map((slot) => fit(slot.value, slot.allocation,
|
|
195
|
+
return slots.map((slot) => fit(slot.value, slot.allocation, linkCollection));
|
|
178
196
|
}
|
|
179
197
|
|
|
180
198
|
if (value !== null && typeof value === "object") {
|
|
199
|
+
// An object cannot be shown with only a fragment of a required identity.
|
|
200
|
+
if (!root && requiredSize(value) > budget) return Object.create(null) as JsonValue;
|
|
181
201
|
const entries = Object.entries(value);
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
|
|
185
|
-
|
|
202
|
+
// Reserve required identifiers first, but restore the original field
|
|
203
|
+
// order in the result; no provider data or array entry is re-ranked.
|
|
204
|
+
const candidates = entries.map(([key, child], index) => ({
|
|
205
|
+
key, child, index, required: requiredPropertySize(key, child) !== undefined,
|
|
206
|
+
}));
|
|
186
207
|
candidates.sort((a, b) =>
|
|
187
|
-
(
|
|
208
|
+
Number(b.required) - Number(a.required)
|
|
209
|
+
|| (KEY_PRIORITY.get(a.key) ?? IMPORTANT_KEYS.length) - (KEY_PRIORITY.get(b.key) ?? IMPORTANT_KEYS.length)
|
|
188
210
|
|| a.index - b.index,
|
|
189
211
|
);
|
|
190
212
|
const slots: Slot[] = [];
|
|
191
213
|
let fixed = 2; // braces, keys, colons and commas
|
|
192
214
|
let minimums = 0;
|
|
193
215
|
for (const { key, child, index } of candidates) {
|
|
194
|
-
const minimum = minimumSize(child);
|
|
216
|
+
const minimum = minimumSize(child, key);
|
|
195
217
|
const punctuation = (slots.length > 0 ? 1 : 0) + charCount(JSON.stringify(key)) + 1;
|
|
196
|
-
if (fixed + punctuation + minimums + minimum > budget)
|
|
197
|
-
recordChange(tracker, propertyPath(path, key));
|
|
198
|
-
continue;
|
|
199
|
-
}
|
|
218
|
+
if (fixed + punctuation + minimums + minimum > budget) continue;
|
|
200
219
|
fixed += punctuation;
|
|
201
220
|
minimums += minimum;
|
|
202
221
|
slots.push({ index, key, value: child, minimum, full: sizeOf(child), allocation: minimum });
|
|
@@ -206,7 +225,7 @@ function createFitter(tracker: ChangeTracker) {
|
|
|
206
225
|
const result: { [key: string]: JsonValue } = Object.create(null);
|
|
207
226
|
for (const slot of slots) {
|
|
208
227
|
const key = slot.key!;
|
|
209
|
-
result[key] = fit(slot.value, slot.allocation,
|
|
228
|
+
result[key] = fit(slot.value, slot.allocation, isLinkCollectionKey(key));
|
|
210
229
|
}
|
|
211
230
|
return result;
|
|
212
231
|
}
|
|
@@ -218,17 +237,11 @@ function createFitter(tracker: ChangeTracker) {
|
|
|
218
237
|
return fit;
|
|
219
238
|
}
|
|
220
239
|
|
|
221
|
-
function minimalTruncatedJson(limit: number): string {
|
|
222
|
-
for (const text of ['{"_truncation":{"truncated":true}}', '{"_truncated":true}', "{}", "0"]) {
|
|
223
|
-
if (text.length <= limit) return text;
|
|
224
|
-
}
|
|
225
|
-
throw new RangeError("The JSON output limit must be at least one character");
|
|
226
|
-
}
|
|
227
|
-
|
|
228
240
|
/**
|
|
229
241
|
* Preserve ordinary output exactly. If compact JSON still exceeds the limit,
|
|
230
|
-
* fit its fields and array entries independently and
|
|
231
|
-
* The
|
|
242
|
+
* fit its fields and array entries independently and append one marker line.
|
|
243
|
+
* The JSON portion stays valid and the entire text fits within `maxChars` code
|
|
244
|
+
* points. Tiny limits may only fit the marker or a minimal JSON value.
|
|
232
245
|
*/
|
|
233
246
|
export function formatJsonWithinLimit(
|
|
234
247
|
output: object,
|
|
@@ -239,61 +252,26 @@ export function formatJsonWithinLimit(
|
|
|
239
252
|
}
|
|
240
253
|
const limit = Math.floor(maxChars);
|
|
241
254
|
const pretty = JSON.stringify(output, null, 2);
|
|
242
|
-
|
|
243
|
-
if (originalChars <= limit) return { text: pretty, truncated: false };
|
|
255
|
+
if (charCount(pretty) <= limit) return { text: pretty, truncated: false };
|
|
244
256
|
|
|
245
257
|
// Work from the already serialized value, so JSON's handling of undefined,
|
|
246
258
|
// dates and toJSON is identical even if an input object is not plain data.
|
|
247
|
-
const parsed = JSON.parse(pretty) as
|
|
259
|
+
const parsed = JSON.parse(pretty) as JsonValue;
|
|
248
260
|
const compact = JSON.stringify(parsed);
|
|
249
261
|
if (charCount(compact) <= limit) return { text: compact, truncated: false };
|
|
250
262
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
};
|
|
257
|
-
// Account for the metadata entry and a comma joining it to the root object.
|
|
258
|
-
const overhead = charCount(JSON.stringify({ _truncation: info })) - 2 + 1;
|
|
259
|
-
if (limit - overhead < 2) return { text: minimalTruncatedJson(limit), truncated: true };
|
|
260
|
-
|
|
261
|
-
const build = (reservedPaths: number) => {
|
|
262
|
-
const tracker: ChangeTracker = { paths: [], seen: new Set(), more: false };
|
|
263
|
-
const fit = createFitter(tracker);
|
|
264
|
-
const fitted = fit(parsed, limit - overhead - reservedPaths, "$") as { [key: string]: JsonValue };
|
|
265
|
-
return { fitted, tracker };
|
|
266
|
-
};
|
|
267
|
-
|
|
268
|
-
let result = build(0);
|
|
269
|
-
// Reserve just enough room for the first pass's affected paths, capped so
|
|
270
|
-
// reporting never consumes a disproportionate share of the result budget.
|
|
271
|
-
const desiredPathSpace = Math.min(
|
|
272
|
-
MAX_PATH_BUDGET,
|
|
273
|
-
Math.max(64, Math.floor(limit * 0.05)),
|
|
274
|
-
charCount(JSON.stringify(result.tracker.paths)) - 2,
|
|
275
|
-
);
|
|
276
|
-
if (desiredPathSpace > 0 && limit - overhead - desiredPathSpace >= 2) {
|
|
277
|
-
result = build(desiredPathSpace);
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
const combined: { [key: string]: unknown } = Object.create(null);
|
|
281
|
-
combined._truncation = info;
|
|
282
|
-
for (const [key, value] of Object.entries(result.fitted)) {
|
|
283
|
-
if (key !== "_truncation") combined[key] = value;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
let remaining = limit - charCount(JSON.stringify(combined));
|
|
287
|
-
for (const path of result.tracker.paths) {
|
|
288
|
-
const cost = charCount(JSON.stringify(path)) + (info.affectedPaths.length > 0 ? 1 : 0);
|
|
289
|
-
if (cost > remaining) {
|
|
290
|
-
info.moreAffectedPaths = true;
|
|
291
|
-
break;
|
|
292
|
-
}
|
|
293
|
-
info.affectedPaths.push(path);
|
|
294
|
-
remaining -= cost;
|
|
263
|
+
// Below 14 characters there is not enough room for both an empty JSON object
|
|
264
|
+
// and the marker on a new line. Prefer a complete marker whenever it fits.
|
|
265
|
+
if (limit < TRUNCATION_MARKER.length + 3) {
|
|
266
|
+
const text = limit >= TRUNCATION_MARKER.length ? TRUNCATION_MARKER : limit >= 2 ? "{}" : "0";
|
|
267
|
+
return { text, truncated: true };
|
|
295
268
|
}
|
|
296
|
-
if (result.tracker.more) info.moreAffectedPaths = true;
|
|
297
269
|
|
|
298
|
-
|
|
270
|
+
const budget = limit - TRUNCATION_MARKER.length - 1; // newline + marker
|
|
271
|
+
const fit = createFitter();
|
|
272
|
+
// A toJSON method can return a scalar that cannot be shortened to fit.
|
|
273
|
+
const fitted = parsed === null || typeof parsed === "boolean" || typeof parsed === "number"
|
|
274
|
+
? {}
|
|
275
|
+
: fit(parsed, budget, false, true);
|
|
276
|
+
return { text: `${JSON.stringify(fitted)}\n${TRUNCATION_MARKER}`, truncated: true };
|
|
299
277
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-better-web-search",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "A better provider-native web_search tool for pi: raw results, automatic provider fallback, URL analysis, and native DeepSeek Search/Fetch.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|