pi-image-slim 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +252 -2
- package/README.zh-CN.md +208 -0
- package/caption.ts +104 -0
- package/config.ts +124 -0
- package/image.ts +170 -0
- package/index.ts +392 -0
- package/package.json +79 -4
- package/protocol.ts +99 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 wiidede
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,253 @@
|
|
|
1
|
-
#
|
|
1
|
+
# pi-image-slim
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[中文文档](./README.zh-CN.md)
|
|
4
|
+
|
|
5
|
+
A [Pi](https://pi.dev) extension that shrinks screenshots and stops old ones from being
|
|
6
|
+
re-uploaded.
|
|
7
|
+
|
|
8
|
+
## The problem
|
|
9
|
+
|
|
10
|
+
Pi sends the whole conversation with every request. Prompt caching makes that cheap for the
|
|
11
|
+
server, but not for your uplink — an image that entered the conversation is uploaded again on
|
|
12
|
+
**every single turn**, forever.
|
|
13
|
+
|
|
14
|
+
Read 30 MB of screenshots and you upload 30 MB per turn. On a 500 KB/s connection that is a
|
|
15
|
+
minute of uploading before the model even starts thinking.
|
|
16
|
+
|
|
17
|
+
## The fix
|
|
18
|
+
|
|
19
|
+
Two things happen, at two different moments. Each can be turned off on its own.
|
|
20
|
+
|
|
21
|
+
**When an image arrives**, it is compressed and described:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
your screenshot → 6.4x smaller WebP + one-line description
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
**Once it falls out of the window**, it is replaced by text:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
[image removed from history to save upload bandwidth. Re-read /proj/title.png if you need to see it again.]
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
So each image is uploaded a small, bounded number of times, and what stays behind is a path
|
|
34
|
+
plus a description — enough for the model to know what the picture was, and to fetch it again
|
|
35
|
+
if it really needs to.
|
|
36
|
+
|
|
37
|
+
If you would rather keep every image in view, set `"strip": false`. You still get the 6.4x
|
|
38
|
+
compression, and the model never loses sight of anything.
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pi install npm:pi-image-slim
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Or try it without touching your settings:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pi -e npm:pi-image-slim
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Measured
|
|
53
|
+
|
|
54
|
+
Four real screenshots from Pi's own docs, compressed to 1280 px / WebP / quality 75:
|
|
55
|
+
|
|
56
|
+
| File | Before | After | |
|
|
57
|
+
| --- | --- | --- | --- |
|
|
58
|
+
| `tree-view.png` | 275 KB | 68 KB | 4.0x |
|
|
59
|
+
| `interactive-mode.png` | 506 KB | 84 KB | 6.0x |
|
|
60
|
+
| `exy.png` | 1475 KB | 179 KB | 8.2x |
|
|
61
|
+
| `doom-extension.png` | 167 KB | 44 KB | 3.7x |
|
|
62
|
+
| **total** | **2.4 MB** | **377 KB** | **6.4x** |
|
|
63
|
+
|
|
64
|
+
That saves bandwidth on the turn the image is sent, **and** disk space in the session file.
|
|
65
|
+
Stripping saves bandwidth on every turn after that.
|
|
66
|
+
|
|
67
|
+
## Who this is for
|
|
68
|
+
|
|
69
|
+
- You read screenshots, mockups, or browser captures into Pi
|
|
70
|
+
- You are on a slow, metered, or proxied connection
|
|
71
|
+
- Requests feel slow while the model seems idle
|
|
72
|
+
|
|
73
|
+
If you paste one image and never look back, you do not need this.
|
|
74
|
+
|
|
75
|
+
## How it works
|
|
76
|
+
|
|
77
|
+
Two hooks, nothing more.
|
|
78
|
+
|
|
79
|
+
**`tool_result`** — runs when a tool returns an image.
|
|
80
|
+
|
|
81
|
+
1. Re-encodes PNG/JPEG as WebP with the longest edge capped at `maxEdge` (`compress`).
|
|
82
|
+
2. Asks the current model to describe the stored bytes in one line (`caption`).
|
|
83
|
+
3. Writes both into the conversation as plain text:
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
Image file: /proj/docs/mockups/title.png
|
|
87
|
+
Image description: A dark settings panel with three toggles, the middle one off.
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
When one call returns several images, they are numbered instead, and no path is claimed —
|
|
91
|
+
Pi reports a single `path` for the whole call, so pointing every placeholder at it would send
|
|
92
|
+
the model to a file holding only one of the pictures:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
Image batch: 2 images from one call, no single file to re-read.
|
|
96
|
+
Image description: [1] Header area: dark nav bar, a sign-in button on the right.
|
|
97
|
+
Image description: [2] Footer area: grey link list, copyright at the bottom.
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Because this is ordinary transcript text, it survives `/reload`, restarts, and compaction.
|
|
101
|
+
The description is generated once, though its tokens are paid on every later request.
|
|
102
|
+
|
|
103
|
+
**`before_provider_request`** — runs on the finished request body, just before it goes out.
|
|
104
|
+
|
|
105
|
+
Every image outside the newest `keepImages` image-bearing messages becomes a placeholder
|
|
106
|
+
(`strip`). Nothing else is touched. This pass never decodes anything: it swaps object pointers, so a 30 MB body costs
|
|
107
|
+
about a millisecond.
|
|
108
|
+
|
|
109
|
+
### What the model sees
|
|
110
|
+
|
|
111
|
+
| | Sent |
|
|
112
|
+
| --- | --- |
|
|
113
|
+
| The newest `keepImages` messages that carry an image | full image |
|
|
114
|
+
| Anything older | placeholder + path |
|
|
115
|
+
| The description note | always |
|
|
116
|
+
|
|
117
|
+
### A window of messages, not of images
|
|
118
|
+
|
|
119
|
+
The last two image-bearing messages are sent in full; everything older is a placeholder.
|
|
120
|
+
|
|
121
|
+
- Ten screenshots from **one** tool call share one message, so all ten arrive in full and the
|
|
122
|
+
model can compare them. The window counts messages, not pictures.
|
|
123
|
+
- Each new screenshot pushes the oldest one out. The image is still there in the description
|
|
124
|
+
note, so the model knows what it was.
|
|
125
|
+
- The model needs another look → it `read`s the path, and that one image is uploaded again.
|
|
126
|
+
|
|
127
|
+
The window is why this also works inside a single long task. While the agent works on its own
|
|
128
|
+
there is no new user message, so a boundary based on "the last thing you typed" would never
|
|
129
|
+
move and every screenshot would be re-uploaded on every step. Measured over a 30-step loop at
|
|
130
|
+
68 KB per screenshot: **31 MB with a turn boundary, 4.1 MB with a window of 2** (ideal is 2 MB).
|
|
131
|
+
|
|
132
|
+
Set `"keepImages": 1` for the strictest version — each image is then uploaded exactly once.
|
|
133
|
+
|
|
134
|
+
### Why WebP
|
|
135
|
+
|
|
136
|
+
Measured on real screenshots it beats JPEG by 30-45% at the same visual quality, and roughly
|
|
137
|
+
ties on photos. AVIF compresses further but Anthropic rejects it with a 400, and Pi rewrites
|
|
138
|
+
unknown formats to PNG — which makes them bigger. Compression is best-effort: an unsupported
|
|
139
|
+
format, a failure, or a result that is not actually smaller all leave the original alone.
|
|
140
|
+
|
|
141
|
+
## Configuration
|
|
142
|
+
|
|
143
|
+
A JSON file, not environment variables. Two locations, most specific last:
|
|
144
|
+
|
|
145
|
+
| Path | Scope |
|
|
146
|
+
| --- | --- |
|
|
147
|
+
| `<agent-dir>/pi-image-slim.json` | every project (defaults to `~/.pi/agent`) |
|
|
148
|
+
| `.pi/pi-image-slim.json` | this project only |
|
|
149
|
+
|
|
150
|
+
```json
|
|
151
|
+
{
|
|
152
|
+
"maxEdge": 1280,
|
|
153
|
+
"quality": 75,
|
|
154
|
+
"strip": true,
|
|
155
|
+
"keepImages": 2
|
|
156
|
+
}
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
| Key | Default | Meaning |
|
|
160
|
+
| --- | --- | --- |
|
|
161
|
+
| `compress` | `true` | Re-encode PNG/JPEG as WebP |
|
|
162
|
+
| `caption` | `true` | Describe each image before the pixels go away |
|
|
163
|
+
| `strip` | `true` | Replace old images with a placeholder |
|
|
164
|
+
| `keepImages` | `2` | How many recent image messages stay intact (1-20) |
|
|
165
|
+
| `maxEdge` | `1280` | Ceiling for the **longer** side |
|
|
166
|
+
| `quality` | `75` | WebP quality, 0-100 |
|
|
167
|
+
| `captionTimeoutMs` | `20000` | Fall back to a bare path after this long; `0` = no limit |
|
|
168
|
+
|
|
169
|
+
Every key is optional. Unknown keys and bad values are ignored, and a malformed file falls
|
|
170
|
+
back to defaults instead of breaking tool results. Run `/reload` after editing.
|
|
171
|
+
|
|
172
|
+
### Pick your level
|
|
173
|
+
|
|
174
|
+
| `compress` | `strip` | What happens |
|
|
175
|
+
| --- | --- | --- |
|
|
176
|
+
| `true` | `true` | **Default.** Smaller images, old ones become text. |
|
|
177
|
+
| `true` | `false` | Images shrink ~6x, but the model keeps seeing all of them. |
|
|
178
|
+
| `false` | `true` | Full-size images, old ones become text. |
|
|
179
|
+
|
|
180
|
+
`"strip": false` changes nothing about the conversation — the model can always look back at
|
|
181
|
+
any image — while still cutting upload by about 84%. That is the safest setting if you are
|
|
182
|
+
unsure.
|
|
183
|
+
|
|
184
|
+
`caption` only does something while `strip` is on: an image that never leaves context does
|
|
185
|
+
not need a description, so the extra round trip is skipped automatically. `keepImages` below 1
|
|
186
|
+
is ignored, because the model has to be able to see the image it just asked for.
|
|
187
|
+
|
|
188
|
+
**Raise `maxEdge` if you work with small text** — 1568 is a common choice. Keep it at 2000 or
|
|
189
|
+
below: above that, Pi's own image normalisation re-encodes the result and throws the WebP
|
|
190
|
+
away.
|
|
191
|
+
|
|
192
|
+
**`maxEdge` measures the longer side**, so a 1200×4000 phone screenshot shrinks by its height
|
|
193
|
+
just like a wide desktop capture shrinks by its width.
|
|
194
|
+
|
|
195
|
+
## Disable
|
|
196
|
+
|
|
197
|
+
There is no `enabled` key and no environment variable. To turn it off, disable the extension
|
|
198
|
+
in your Pi config or uninstall it — one way instead of three.
|
|
199
|
+
|
|
200
|
+
## Compatibility
|
|
201
|
+
|
|
202
|
+
The placeholder shape is chosen from the model's API, so switching providers does not produce
|
|
203
|
+
a 400:
|
|
204
|
+
|
|
205
|
+
| API | Part |
|
|
206
|
+
| --- | --- |
|
|
207
|
+
| `openai-completions`, `anthropic-messages`, `mistral-conversations`, `pi-messages` | `{ type: "text", text }` |
|
|
208
|
+
| `openai-responses`, `azure-openai-responses`, `openai-codex-responses` | `{ type: "input_text", text }` |
|
|
209
|
+
| `google-generative-ai`, `google-vertex`, `bedrock-converse-stream` | `{ text }` |
|
|
210
|
+
|
|
211
|
+
Images are detected in all of them, including images nested inside `tool_result`.
|
|
212
|
+
|
|
213
|
+
## Caveats
|
|
214
|
+
|
|
215
|
+
- **Compression lowers resolution permanently.** Re-`read`ing the file will not bring back a
|
|
216
|
+
detail — Pi applies the same resize every time. Raise `maxEdge` for small text.
|
|
217
|
+
- **A description can be wrong.** A confident but incorrect one is worse than none, because
|
|
218
|
+
the model will reason from it. Descriptions come from the same model that already saw the
|
|
219
|
+
image, and nothing is sent anywhere new, but they are not verified. Set
|
|
220
|
+
`"caption": false` to turn them off.
|
|
221
|
+
- **A description is written once and never retried.** If that request times out, the
|
|
222
|
+
placeholder stays path-only. Raise `captionTimeoutMs` to make that less likely.
|
|
223
|
+
- **Reading an image costs one extra model round trip**, because the description is generated
|
|
224
|
+
before the tool result is committed. Images are handled one at a time, so ten images means
|
|
225
|
+
ten round trips.
|
|
226
|
+
- **Prompt cache misses once**, on the turn after an image was read. The `tools` and `system`
|
|
227
|
+
layers still hit; only `messages` is invalidated.
|
|
228
|
+
- **A batch of images cannot be re-read.** When one call returns several images, Pi reports one
|
|
229
|
+
`path`, so none of them can claim it. The descriptions survive; the pixels do not.
|
|
230
|
+
- **The model may answer from the description instead of looking again.** For pixel-level
|
|
231
|
+
work, ask it to re-`read` the file.
|
|
232
|
+
- **Old images are invisible to the model within a long task.** With a window of 2, a
|
|
233
|
+
screenshot taken ten steps ago is a placeholder by the time the model wants to re-check it.
|
|
234
|
+
The description is still there; the pixels are one `read` away. Raise `keepImages` if you
|
|
235
|
+
work across many screenshots at once.
|
|
236
|
+
- **Old sessions are not retrofitted.** Only images read while the extension is active get
|
|
237
|
+
compressed and described.
|
|
238
|
+
- **Session files still keep the compressed image**, not just the text. Stripping only
|
|
239
|
+
affects what goes over the wire.
|
|
240
|
+
|
|
241
|
+
## Development
|
|
242
|
+
|
|
243
|
+
```bash
|
|
244
|
+
git clone https://github.com/wiidede/pi-image-slim
|
|
245
|
+
cd pi-image-slim
|
|
246
|
+
pnpm install
|
|
247
|
+
pnpm check # lint + typecheck + tests
|
|
248
|
+
pnpm test # 59 tests, no network, no fixtures
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
## License
|
|
252
|
+
|
|
253
|
+
MIT
|
package/README.zh-CN.md
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# pi-image-slim
|
|
2
|
+
|
|
3
|
+
[English](./README.md)
|
|
4
|
+
|
|
5
|
+
一个 [Pi](https://pi.dev) 扩展:压缩截图,并让旧图不再反复重传。
|
|
6
|
+
|
|
7
|
+
## 问题
|
|
8
|
+
|
|
9
|
+
Pi 每次请求都会把整个对话发一遍。prompt cache 让服务端算起来便宜,但省不了你的上行带宽——一张图一旦进了对话,之后**每一轮**都会被重新上传,永远如此。
|
|
10
|
+
|
|
11
|
+
读了 30 MB 截图,就是每轮上传 30 MB。在 500 KB/s 的链路上,模型还没开始思考,光上传就要一分钟。
|
|
12
|
+
|
|
13
|
+
## 做法
|
|
14
|
+
|
|
15
|
+
两个时刻,各做一件事,两件都能单独关掉。
|
|
16
|
+
|
|
17
|
+
**图片进来时**,压缩并描述:
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
你的截图 → 小 6.4 倍的 WebP + 一句描述
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
**等它被挤出窗口后**,它被替换成文字:
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
[image removed from history to save upload bandwidth. Re-read /proj/title.png if you need to see it again.]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
于是每张图的上传次数是有界的、很少的几次,留下的是路径加描述——够模型知道那张图是什么,也够它真需要时再取回来。
|
|
30
|
+
|
|
31
|
+
如果你更希望所有图片一直可见,设 `"strip": false`。照样能得到 6.4 倍的压缩,而且模型什么都不会丢。
|
|
32
|
+
|
|
33
|
+
## 安装
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pi install npm:pi-image-slim
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
或者不改配置先试试:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pi -e npm:pi-image-slim
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## 实测
|
|
46
|
+
|
|
47
|
+
Pi 官方文档里的四张真实截图,压到 1280 px / WebP / 质量 75:
|
|
48
|
+
|
|
49
|
+
| 文件 | 压缩前 | 压缩后 | |
|
|
50
|
+
| --- | --- | --- | --- |
|
|
51
|
+
| `tree-view.png` | 275 KB | 68 KB | 4.0x |
|
|
52
|
+
| `interactive-mode.png` | 506 KB | 84 KB | 6.0x |
|
|
53
|
+
| `exy.png` | 1475 KB | 179 KB | 8.2x |
|
|
54
|
+
| `doom-extension.png` | 167 KB | 44 KB | 3.7x |
|
|
55
|
+
| **合计** | **2.4 MB** | **377 KB** | **6.4x** |
|
|
56
|
+
|
|
57
|
+
这既省了图片被发送那一轮的带宽,**也**省了会话文件的磁盘空间。剥离则省掉之后每一轮的带宽。
|
|
58
|
+
|
|
59
|
+
## 适合谁
|
|
60
|
+
|
|
61
|
+
- 你会把截图、设计稿、浏览器抓图读进 Pi
|
|
62
|
+
- 你在用慢速、计费或需要代理的网络
|
|
63
|
+
- 你感觉请求很慢,但模型看起来在发呆
|
|
64
|
+
|
|
65
|
+
如果你只是偶尔贴一张图就再也不看,那不需要装这个。
|
|
66
|
+
|
|
67
|
+
## 它怎么工作
|
|
68
|
+
|
|
69
|
+
两个钩子,没有别的。
|
|
70
|
+
|
|
71
|
+
**`tool_result`** —— 工具返回图片时触发。
|
|
72
|
+
|
|
73
|
+
1. 把 PNG/JPEG 重新编码成 WebP,最长边限制在 `maxEdge`(`compress`)。
|
|
74
|
+
2. 让当前模型对"真正会被存下来的那份字节"写一句描述(`caption`)。
|
|
75
|
+
3. 两者都作为普通文本写进对话:
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
Image file: /proj/docs/mockups/title.png
|
|
79
|
+
Image description: A dark settings panel with three toggles, the middle one off.
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
一次调用返回多张图时,描述会带序号,而且**不写路径**——Pi 对整次调用只报一个 `path`,把它写给每张图会让占位符指向一个只含其中一张的文件:
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
Image batch: 2 images from one call, no single file to re-read.
|
|
86
|
+
Image description: [1] Header 区域:深色导航栏,右侧一个登录按钮。
|
|
87
|
+
Image description: [2] Footer 区域:灰色链接列表,版权信息在底部。
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
因为这就是普通的对话记录,它能挺过 `/reload`、进程重启和上下文压缩。描述只生成一次,但它的 token 每轮都要重付。
|
|
91
|
+
|
|
92
|
+
**`before_provider_request`** —— 在请求体组装完、还没发出去的一刻触发。
|
|
93
|
+
|
|
94
|
+
最近 `keepImages` 条带图消息之外的图片都变成占位符,其它什么都不动。这一遍**不解码任何数据**,只是替换对象指针,30 MB 的请求体大约一毫秒。
|
|
95
|
+
|
|
96
|
+
### 模型每轮看到什么
|
|
97
|
+
|
|
98
|
+
| | 发送内容 |
|
|
99
|
+
| --- | --- |
|
|
100
|
+
| 最近 `keepImages` 条带图消息 | 完整图片 |
|
|
101
|
+
| 更早的 | 占位符 + 路径 |
|
|
102
|
+
| 那条描述 note | 始终保留 |
|
|
103
|
+
|
|
104
|
+
### 窗口按消息算,不按图片算
|
|
105
|
+
|
|
106
|
+
最近两条带图消息完整发送,更早的变成占位符。
|
|
107
|
+
|
|
108
|
+
- **一次**工具调用返回的十张截图共享同一条消息,所以十张都完整送达,模型可以互相对比。窗口数的是消息,不是张数。
|
|
109
|
+
- 每来一张新截图,最老的那条被挤出去。图虽然没了,描述还在,模型仍知道它是什么。
|
|
110
|
+
- 模型要再看 → 它对路径 `read` 一次,那一张图会被重新上传。
|
|
111
|
+
|
|
112
|
+
这个窗口也是它能在一个长任务里生效的原因。agent 自己干活时不会有新的用户消息,所以"以你最后一次发言为界"这条规则永远不会移动,每一步都会把所有截图重传一遍。实测 30 步、每步一张 68 KB 截图:**按轮划分是 31 MB,窗口为 2 是 4.1 MB**(理想值 2 MB)。
|
|
113
|
+
|
|
114
|
+
想要最严格就把 `"keepImages"` 设成 `1`——那样每张图确实只上传一次。
|
|
115
|
+
|
|
116
|
+
### 为什么是 WebP
|
|
117
|
+
|
|
118
|
+
在真实截图上实测,同视觉质量下比 JPEG 小 30–45%,照片类内容则大致持平。AVIF 能再小些,但 Anthropic 直接 400 拒绝,而 Pi 会把不认识的格式重写成 PNG——反而更大。压缩是尽力而为的:不支持的格式、失败、或者结果没变小,都保持原字节不动。
|
|
119
|
+
|
|
120
|
+
## 配置
|
|
121
|
+
|
|
122
|
+
用 JSON 文件,不用环境变量。两个位置,越靠后优先级越高:
|
|
123
|
+
|
|
124
|
+
| 路径 | 作用范围 |
|
|
125
|
+
| --- | --- |
|
|
126
|
+
| `<agent-dir>/pi-image-slim.json` | 所有项目(默认为 `~/.pi/agent`) |
|
|
127
|
+
| `.pi/pi-image-slim.json` | 仅当前项目 |
|
|
128
|
+
|
|
129
|
+
```json
|
|
130
|
+
{
|
|
131
|
+
"maxEdge": 1280,
|
|
132
|
+
"quality": 75,
|
|
133
|
+
"strip": true,
|
|
134
|
+
"keepImages": 2
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
| 键 | 默认 | 含义 |
|
|
139
|
+
| --- | --- | --- |
|
|
140
|
+
| `compress` | `true` | 把 PNG/JPEG 重新编码成 WebP |
|
|
141
|
+
| `caption` | `true` | 像素丢掉之前先写一句描述 |
|
|
142
|
+
| `strip` | `true` | 把旧图换成占位符 |
|
|
143
|
+
| `keepImages` | `2` | 保留最近几条带图消息(1–20) |
|
|
144
|
+
| `maxEdge` | `1280` | **较长边**的上限 |
|
|
145
|
+
| `quality` | `75` | WebP 质量,0–100 |
|
|
146
|
+
| `captionTimeoutMs` | `20000` | 超过这么久就退回纯路径;`0` 表示不限时 |
|
|
147
|
+
|
|
148
|
+
所有键都可选。未知的键和不合法的值会被忽略,文件格式坏了也只是退回默认值,不会弄坏工具结果。改完记得 `/reload`。
|
|
149
|
+
|
|
150
|
+
### 选一个档位
|
|
151
|
+
|
|
152
|
+
| `compress` | `strip` | 效果 |
|
|
153
|
+
| --- | --- | --- |
|
|
154
|
+
| `true` | `true` | **默认。** 图片变小,旧的变成文字。 |
|
|
155
|
+
| `true` | `false` | 图片小 6 倍左右,但模型始终能看到全部图片。 |
|
|
156
|
+
| `false` | `true` | 原图尺寸,旧的变成文字。 |
|
|
157
|
+
|
|
158
|
+
`"strip": false` **完全不影响对话内容**——模型随时能回头看任何一张图——但上行带宽仍然能省大约 84%。拿不定主意时,选这个最稳。
|
|
159
|
+
|
|
160
|
+
`caption` 只在 `strip` 开着时才有意义:永远不会离开上下文的图,不需要描述,所以那次额外的模型往返会自动跳过。`keepImages` 小于 1 会被忽略——模型总得能看见它刚要求的那张图。
|
|
161
|
+
|
|
162
|
+
**如果常看小字,把 `maxEdge` 调大** —— 1568 是常见选择。但别超过 2000:超过之后 Pi 自己的图片归一化会把结果重新编码,我们生成的 WebP 就白做了。
|
|
163
|
+
|
|
164
|
+
**`maxEdge` 量的是较长边**,所以 1200×4000 的手机竖屏截图按高度缩,和宽屏截图按宽度缩是一个道理。
|
|
165
|
+
|
|
166
|
+
## 关闭
|
|
167
|
+
|
|
168
|
+
没有 `enabled` 键,也没有环境变量。想关掉就在 Pi 的配置里禁用这个扩展,或者直接卸载——只有一种方式,而不是三种。
|
|
169
|
+
|
|
170
|
+
## 兼容性
|
|
171
|
+
|
|
172
|
+
占位符的形状按模型的 API 生成,所以换供应商不会 400:
|
|
173
|
+
|
|
174
|
+
| API | 形状 |
|
|
175
|
+
| --- | --- |
|
|
176
|
+
| `openai-completions`、`anthropic-messages`、`mistral-conversations`、`pi-messages` | `{ type: "text", text }` |
|
|
177
|
+
| `openai-responses`、`azure-openai-responses`、`openai-codex-responses` | `{ type: "input_text", text }` |
|
|
178
|
+
| `google-generative-ai`、`google-vertex`、`bedrock-converse-stream` | `{ text }` |
|
|
179
|
+
|
|
180
|
+
这些协议里的图片都能认出来,包括嵌在 `tool_result` 里的。
|
|
181
|
+
|
|
182
|
+
## 注意事项
|
|
183
|
+
|
|
184
|
+
- **压缩会永久降低分辨率。** 重新 `read` 也找不回细节——Pi 每次读都会应用同样的缩放。需要看清小字就调大 `maxEdge`。
|
|
185
|
+
- **描述可能是错的。** 一句自信但错误的描述比没有更糟,因为模型会照着它推理。描述由本来就要看这张图的模型当场生成,也没发给任何新的端点,但**没有被验证过**。设 `"caption": false` 可以关掉。
|
|
186
|
+
- **批量返回的图无法重新读取。** 一次调用返回多张图时,Pi 只报一个 `path`,所以它们谁也用不上。描述还在,像素没了。
|
|
187
|
+
- **长任务里旧的图对模型是不可见的。** 窗口为 2 时,十步之前的截图早就变成占位符了。描述还在,像素只差一次 `read`。如果你经常要同时对照很多张图,把 `keepImages` 调大。
|
|
188
|
+
- **描述只生成一次,不会重试。** 如果那次请求超时,占位符就永远是纯路径。调大 `captionTimeoutMs` 可以降低概率。
|
|
189
|
+
- **读一张图会多一次模型往返**,因为描述要在工具结果落盘前生成。图片是一张一张处理的,读十张就是十次往返。
|
|
190
|
+
- **prompt cache 会 miss 一次**,发生在图片被读进来的下一轮。`tools` 和 `system` 层仍然命中,只有 `messages` 失效。
|
|
191
|
+
- **模型可能照描述回答,而不是重新看图。** 做像素级的工作时,直接让它再 `read` 一次那个文件。
|
|
192
|
+
- **轮次边界的判断依赖 Pi 的序列化细节**(那条合成的 `Attached image(s) from tool result:` 消息)。如果未来的 Pi 改了形状,扩展会安全降级:找不到边界,就什么都不剥离。
|
|
193
|
+
- **旧会话不会被追溯处理。** 只有扩展启用期间读进来的图才会被压缩和描述。
|
|
194
|
+
- **会话文件里仍然保留压缩后的图片**,不只是文字。剥离只影响线上传输的部分。
|
|
195
|
+
|
|
196
|
+
## 开发
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
git clone https://github.com/wiidede/pi-image-slim
|
|
200
|
+
cd pi-image-slim
|
|
201
|
+
pnpm install
|
|
202
|
+
pnpm check # lint + typecheck + 测试
|
|
203
|
+
pnpm test # 59 个测试,不联网,不依赖磁盘文件
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
MIT
|
package/caption.ts
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Image descriptions — the "compression" half that keeps meaning instead of pixels.
|
|
3
|
+
*
|
|
4
|
+
* Once an image leaves the current turn its bytes are dropped and only a
|
|
5
|
+
* placeholder remains. A path is an address, not information: the model cannot
|
|
6
|
+
* answer anything about the picture without spending a tool round trip (and
|
|
7
|
+
* re-uploading the image) to `read` it again.
|
|
8
|
+
*
|
|
9
|
+
* So each image is described once, in the turn it was read, by the same model
|
|
10
|
+
* that is about to look at it. Nothing leaves for a second endpoint, and a
|
|
11
|
+
* description costs a few dozen tokens where the image cost thousands.
|
|
12
|
+
*
|
|
13
|
+
* There is deliberately no cache and no store here. The caller writes the
|
|
14
|
+
* description into the transcript as message text, which is where it needs to be
|
|
15
|
+
* anyway: a side table keyed by image hash would be lost on `/reload` and would
|
|
16
|
+
* silently degrade every existing placeholder back to a bare path.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const CAPTION_PROMPT = [
|
|
20
|
+
'Describe the image for a future reader who can no longer see it.',
|
|
21
|
+
'One or two sentences, under 40 words. Name the kind of image (screenshot, photo, diagram, chart),',
|
|
22
|
+
'the main subject, and any text, labels, numbers, colours, or UI state that someone would likely ask about later.',
|
|
23
|
+
'Do not speculate, do not describe how it was produced, and do not mention that it is an image.',
|
|
24
|
+
].join(' ')
|
|
25
|
+
|
|
26
|
+
/** The subset of `ExtensionContext` a caption needs, so this stays testable. */
|
|
27
|
+
export interface CaptionContext {
|
|
28
|
+
model?: { input?: readonly string[] } | undefined
|
|
29
|
+
signal?: AbortSignal | undefined
|
|
30
|
+
modelRegistry: {
|
|
31
|
+
complete: (
|
|
32
|
+
model: never,
|
|
33
|
+
context: never,
|
|
34
|
+
options?: { signal?: AbortSignal },
|
|
35
|
+
) => Promise<{ content: readonly { type: string, text?: string }[] }>
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface CaptionRequest {
|
|
40
|
+
model: unknown
|
|
41
|
+
base64: string
|
|
42
|
+
mimeType: string
|
|
43
|
+
/** Abort the request after this many milliseconds. `0` means no limit. */
|
|
44
|
+
timeoutMs?: number
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Combine the caller's signal with a timeout, so an aborted session and an
|
|
49
|
+
* over-budget request both cancel the model call.
|
|
50
|
+
*/
|
|
51
|
+
function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal | undefined {
|
|
52
|
+
if (!timeoutMs || timeoutMs <= 0)
|
|
53
|
+
return signal
|
|
54
|
+
const timeout = AbortSignal.timeout(timeoutMs)
|
|
55
|
+
if (!signal)
|
|
56
|
+
return timeout
|
|
57
|
+
return AbortSignal.any([signal, timeout])
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Ask the current model to describe one image.
|
|
62
|
+
* Never rejects: a missing description only makes the placeholder less useful.
|
|
63
|
+
*/
|
|
64
|
+
export async function captionImage(ctx: CaptionContext, request: CaptionRequest): Promise<string | undefined> {
|
|
65
|
+
// A model without image input cannot describe one.
|
|
66
|
+
if (!ctx.model?.input?.includes('image'))
|
|
67
|
+
return undefined
|
|
68
|
+
|
|
69
|
+
try {
|
|
70
|
+
const response = await ctx.modelRegistry.complete(
|
|
71
|
+
request.model as never,
|
|
72
|
+
{
|
|
73
|
+
systemPrompt: CAPTION_PROMPT,
|
|
74
|
+
messages: [
|
|
75
|
+
{
|
|
76
|
+
role: 'user',
|
|
77
|
+
content: [
|
|
78
|
+
{ type: 'image', data: request.base64, mimeType: request.mimeType },
|
|
79
|
+
{ type: 'text', text: 'Describe this image.' },
|
|
80
|
+
],
|
|
81
|
+
timestamp: Date.now(),
|
|
82
|
+
},
|
|
83
|
+
],
|
|
84
|
+
} as never,
|
|
85
|
+
{
|
|
86
|
+
signal: withTimeout(ctx.signal, request.timeoutMs ?? 0),
|
|
87
|
+
} as never,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
const text = response.content
|
|
91
|
+
.filter(part => part.type === 'text' && typeof part.text === 'string')
|
|
92
|
+
.map(part => part.text!.trim())
|
|
93
|
+
.filter(Boolean)
|
|
94
|
+
.join(' ')
|
|
95
|
+
.replace(/\s+/g, ' ')
|
|
96
|
+
|
|
97
|
+
return text || undefined
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
// Timeout, abort, provider error, malformed response — all mean the same
|
|
101
|
+
// thing to a caller: fall back to a bare path placeholder.
|
|
102
|
+
return undefined
|
|
103
|
+
}
|
|
104
|
+
}
|