pi-l1-cache 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +160 -0
- package/package.json +47 -0
- package/src/index.test.ts +136 -0
- package/src/index.ts +320 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Tobias Weiß
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# pi-l1-cache
|
|
2
|
+
|
|
3
|
+
> **L1 In-Memory Cache Extension for pi** — CPU/RAM optimized, production-ready
|
|
4
|
+
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
[](https://pi.dev/packages)
|
|
7
|
+
|
|
8
|
+
A high-performance, production-ready **L1 (in-memory) cache** extension for the [pi coding agent](https://pi.dev). Designed for stoic Unix simplicity: one file, one purpose, zero dependencies.
|
|
9
|
+
|
|
10
|
+
## Features
|
|
11
|
+
|
|
12
|
+
| Feature | Description |
|
|
13
|
+
|---------|-------------|
|
|
14
|
+
| **~0.1ms lookup** | FNV-1a fast hash — ~100× faster than SHA256 |
|
|
15
|
+
| **Memory cap** | Hard limit (20MB default) prevents RAM bloat |
|
|
16
|
+
| **Auto-eviction** | LRU-style cleanup when limits reached |
|
|
17
|
+
| **CPU-aware** | Auto-disables when CPU > 95% |
|
|
18
|
+
| **TTL-based** | 1-hour default expiry for cached entries |
|
|
19
|
+
| **Atomic cleanup** | Periodic expired entry removal |
|
|
20
|
+
|
|
21
|
+
## Architecture
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
pi → [L1: RAM Map] → [L2: Redis via LiteLLM] → Provider
|
|
25
|
+
<0.5ms ~150ms 1-3s
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# From npm (recommended)
|
|
32
|
+
pi install npm:pi-l1-cache
|
|
33
|
+
|
|
34
|
+
# From GitHub
|
|
35
|
+
pi install git:github.com/tobias-weiss-ai-xr/pi-l1-cache@main
|
|
36
|
+
|
|
37
|
+
# From local clone
|
|
38
|
+
pi install /path/to/pi-l1-cache
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Then **restart pi** — the extension auto-loads.
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
# Show cache stats
|
|
47
|
+
/l1-cache
|
|
48
|
+
|
|
49
|
+
# Show detailed stats
|
|
50
|
+
/l1-cache stats
|
|
51
|
+
|
|
52
|
+
# Clear cache
|
|
53
|
+
/l1-cache clear
|
|
54
|
+
|
|
55
|
+
# Enable cache (if disabled)
|
|
56
|
+
/l1-cache enable
|
|
57
|
+
|
|
58
|
+
# Disable cache (if enabled)
|
|
59
|
+
/l1-cache disable
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
### Stats Output
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
L1 cache status: ENABLED
|
|
66
|
+
Entries: 47 / 200
|
|
67
|
+
Memory: 4.2MB / 20MB
|
|
68
|
+
TTL: 3600s | CPU threshold: 95%
|
|
69
|
+
Hits: 23 | Misses: 70 | Evictions: 5
|
|
70
|
+
Hit rate: 24.7%
|
|
71
|
+
Init CPU: 45.2% (ok)
|
|
72
|
+
Last cleanup: 2025-08-22T10:30:00.000Z
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Configuration
|
|
76
|
+
|
|
77
|
+
### Environment Variables
|
|
78
|
+
|
|
79
|
+
Change defaults without modifying source code:
|
|
80
|
+
|
|
81
|
+
| Variable | Default | Description |
|
|
82
|
+
|----------|---------|-------------|
|
|
83
|
+
| `L1_CACHE_ENABLED` | `true` | Master enable/disable |
|
|
84
|
+
| `L1_CACHE_MAX_ENTRIES` | `200` | Maximum number of cache entries |
|
|
85
|
+
| `L1_CACHE_MAX_MB` | `20` | Maximum memory in MB |
|
|
86
|
+
| `L1_CACHE_TTL` | `3600` | TTL in seconds (1 hour) |
|
|
87
|
+
| `L1_CACHE_LOG` | `false` | Enable debug logging |
|
|
88
|
+
|
|
89
|
+
Example:
|
|
90
|
+
```bash
|
|
91
|
+
# Disable cache
|
|
92
|
+
L1_CACHE_ENABLED=false pi
|
|
93
|
+
|
|
94
|
+
# Use 50MB cache with 30-minute TTL
|
|
95
|
+
L1_CACHE_MAX_MB=50 L1_CACHE_TTL=1800 pi
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Default Settings
|
|
99
|
+
|
|
100
|
+
Edit `src/index.ts` (lines 30-38) to change compiled-in defaults:
|
|
101
|
+
|
|
102
|
+
```typescript
|
|
103
|
+
const DEFAULTS: Settings = {
|
|
104
|
+
enabled: true,
|
|
105
|
+
maxEntries: 200,
|
|
106
|
+
maxMemoryBytes: 20 * 1024 * 1024, // 20MB
|
|
107
|
+
ttlSeconds: 3600, // 1 hour
|
|
108
|
+
cpuThreshold: 95, // disable if CPU > 95%
|
|
109
|
+
logStats: false,
|
|
110
|
+
}
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Design Philosophy
|
|
114
|
+
|
|
115
|
+
### Stoic Unix Principles
|
|
116
|
+
- **One thing, done well** — Caching, and only caching
|
|
117
|
+
- **Do not rely on external services** — Pure in-memory, no Redis
|
|
118
|
+
- **Graceful degradation** — Works even on constrained systems
|
|
119
|
+
- **Zero dependencies** — Single TypeScript file
|
|
120
|
+
|
|
121
|
+
### Performance Optimizations
|
|
122
|
+
1. **Fast string hashing** (FNV-1a) instead of SHA256
|
|
123
|
+
2. **L1 only** — avoid disk I/O in hot path
|
|
124
|
+
3. **Batch eviction** — remove 10-20% at a time, not one-by-one
|
|
125
|
+
4. **Periodic cleanup** — async, non-blocking garbage collection
|
|
126
|
+
5. **Single CPU check** — at startup only, not per-request
|
|
127
|
+
|
|
128
|
+
## Testing
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# Run all tests
|
|
132
|
+
npm test
|
|
133
|
+
|
|
134
|
+
# Watch mode
|
|
135
|
+
npm run test:watch
|
|
136
|
+
|
|
137
|
+
# Check TypeScript
|
|
138
|
+
npx tsc --noEmit
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
17 tests covering:
|
|
142
|
+
- Hash consistency & collision resistance
|
|
143
|
+
- Size estimation for various data types
|
|
144
|
+
- LRU eviction behavior (entry count + memory)
|
|
145
|
+
- State management & reset
|
|
146
|
+
- Settings override
|
|
147
|
+
|
|
148
|
+
## Related Projects
|
|
149
|
+
|
|
150
|
+
- **[opencode-saia-plugin](https://github.com/tobias-weiss-ai-xr/opencode-saia-plugin)** — SAIA provider for OpenCode
|
|
151
|
+
- **[zot-saia-plugin](https://github.com/tobias-weiss-ai-xr/zot-saia-plugin)** — SAIA provider for zot CLI
|
|
152
|
+
- **[pi-saia-plugin](https://github.com/tobias-weiss-ai-xr/pi-saia-plugin)** — SAIA provider for pi coding agent
|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
MIT — see [LICENSE](LICENSE)
|
|
157
|
+
|
|
158
|
+
## Maintainer
|
|
159
|
+
|
|
160
|
+
[Tobias Weiß](https://github.com/tobias-weiss-ai-xr) — weissto@hrz.uni-marburg.de
|
package/package.json
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pi-l1-cache",
|
|
3
|
+
"version": "1.2.0",
|
|
4
|
+
"description": "L1 in-memory cache extension for pi — CPU/RAM optimized, production-ready",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./src/index.ts",
|
|
7
|
+
"engines": {
|
|
8
|
+
"node": ">=18.0.0"
|
|
9
|
+
},
|
|
10
|
+
"keywords": [
|
|
11
|
+
"pi",
|
|
12
|
+
"pi-coding-agent",
|
|
13
|
+
"pi-package",
|
|
14
|
+
"extension",
|
|
15
|
+
"cache",
|
|
16
|
+
"performance",
|
|
17
|
+
"L1",
|
|
18
|
+
"memory",
|
|
19
|
+
"caching"
|
|
20
|
+
],
|
|
21
|
+
"files": [
|
|
22
|
+
"src",
|
|
23
|
+
"README.md",
|
|
24
|
+
"LICENSE"
|
|
25
|
+
],
|
|
26
|
+
"author": "Tobias Weiß <weissto@hrz.uni-marburg.de>",
|
|
27
|
+
"license": "MIT",
|
|
28
|
+
"homepage": "https://github.com/tobias-weiss-ai-xr/pi-l1-cache",
|
|
29
|
+
"repository": {
|
|
30
|
+
"type": "git",
|
|
31
|
+
"url": "git+https://github.com/tobias-weiss-ai-xr/pi-l1-cache.git"
|
|
32
|
+
},
|
|
33
|
+
"bugs": {
|
|
34
|
+
"url": "https://github.com/tobias-weiss-ai-xr/pi-l1-cache/issues"
|
|
35
|
+
},
|
|
36
|
+
"pi": {
|
|
37
|
+
"extensions": ["./src/index.ts"],
|
|
38
|
+
"minPiVersion": "0.4.0"
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"test": "node --test src/*.test.ts",
|
|
42
|
+
"test:watch": "node --watch --test src/*.test.ts"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@types/node": "^20.0.0"
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { describe, it, beforeEach } from "node:test"
|
|
2
|
+
import assert from "node:assert/strict"
|
|
3
|
+
import { _testHooks } from "./index.ts"
|
|
4
|
+
|
|
5
|
+
const Test = _testHooks()
|
|
6
|
+
|
|
7
|
+
describe("pi-l1-cache", () => {
|
|
8
|
+
describe("fastHash", () => {
|
|
9
|
+
it("returns consistent hash for same input", () => {
|
|
10
|
+
const input = "test-string-123"
|
|
11
|
+
assert.equal(Test.fastHash(input), Test.fastHash(input))
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
it("returns different hash for different input", () => {
|
|
15
|
+
assert.notEqual(Test.fastHash("hello"), Test.fastHash("world"))
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
it("handles empty string", () => {
|
|
19
|
+
const empty = Test.fastHash("")
|
|
20
|
+
assert.ok(empty.length > 0)
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
it("handles unicode", () => {
|
|
24
|
+
const emoji = Test.fastHash("🎉 pi-l1-cache 🎉")
|
|
25
|
+
assert.ok(emoji.length > 0)
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
it("is deterministic", () => {
|
|
29
|
+
const obj = JSON.stringify({ a: 1, b: 2 })
|
|
30
|
+
assert.equal(Test.fastHash(obj), Test.fastHash(obj))
|
|
31
|
+
})
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
describe("estimateSize", () => {
|
|
35
|
+
it("estimates size of simple strings", () => {
|
|
36
|
+
const size = Test.estimateSize("hello world")
|
|
37
|
+
assert.ok(size > 0)
|
|
38
|
+
assert.ok(size < 100)
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
it("estimates size of objects", () => {
|
|
42
|
+
const obj = { a: 1, b: "test", c: [1, 2, 3] }
|
|
43
|
+
const size = Test.estimateSize(obj)
|
|
44
|
+
assert.ok(size > 0)
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
it("estimates size of arrays", () => {
|
|
48
|
+
const arr = new Array(100).fill("x")
|
|
49
|
+
const size = Test.estimateSize(arr)
|
|
50
|
+
assert.ok(size > 100)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
it("returns fallback for non-serializable", () => {
|
|
54
|
+
const circular: any = { a: 1 }
|
|
55
|
+
circular.self = circular
|
|
56
|
+
const size = Test.estimateSize(circular)
|
|
57
|
+
assert.equal(size, 1024)
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
it("handles null and undefined", () => {
|
|
61
|
+
assert.ok(Test.estimateSize(null) > 0)
|
|
62
|
+
assert.ok(Test.estimateSize(undefined) > 0)
|
|
63
|
+
})
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
describe("eviction logic", () => {
|
|
67
|
+
beforeEach(() => {
|
|
68
|
+
Test.reset()
|
|
69
|
+
Test.setSettings({ maxEntries: 10, maxMemoryBytes: 10000 })
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it("evicts oldest entries first", () => {
|
|
73
|
+
for (let i = 0; i < 15; i++) {
|
|
74
|
+
Test._setCache(`key-${i}`, { response: {}, timestamp: i * 1000, sizeBytes: 100 })
|
|
75
|
+
}
|
|
76
|
+
Test.evictIfNeeded()
|
|
77
|
+
assert.ok(Test._getCacheSize() <= 10)
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
it("evicts by memory when full", () => {
|
|
81
|
+
Test.setSettings({ maxEntries: 1000, maxMemoryBytes: 1000 })
|
|
82
|
+
Test.reset()
|
|
83
|
+
for (let i = 0; i < 5; i++) {
|
|
84
|
+
Test._setCache(`key-${i}`, { response: new Array(100).fill("x"), timestamp: Date.now(), sizeBytes: 200 })
|
|
85
|
+
}
|
|
86
|
+
Test.evictIfNeeded()
|
|
87
|
+
assert.ok(Test._getTotalMemory() > 0)
|
|
88
|
+
assert.ok(Test._getTotalMemory() <= 1500)
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
it("evictIfNeeded enforces limits", () => {
|
|
92
|
+
for (let i = 0; i < 20; i++) {
|
|
93
|
+
Test._setCache(`key-${i}`, { response: {}, timestamp: i, sizeBytes: 100 })
|
|
94
|
+
}
|
|
95
|
+
Test.evictIfNeeded()
|
|
96
|
+
const after = Test._getCacheSize()
|
|
97
|
+
assert.ok(after <= 10)
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
it("evictOldest removes specified count", () => {
|
|
101
|
+
for (let i = 0; i < 10; i++) {
|
|
102
|
+
Test._setCache(`key-${i}`, { response: {}, timestamp: i * 1000, sizeBytes: 100 })
|
|
103
|
+
}
|
|
104
|
+
Test.evictOldest(3)
|
|
105
|
+
assert.equal(Test._getCacheSize(), 7)
|
|
106
|
+
})
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
describe("state management", () => {
|
|
110
|
+
beforeEach(() => {
|
|
111
|
+
Test.reset()
|
|
112
|
+
Test.setSettings({ maxEntries: 100, maxMemoryBytes: 1000000, ttlSeconds: 3600 })
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
it("returns correct state after adding entries", () => {
|
|
116
|
+
Test._setCache("k1", { response: {}, timestamp: Date.now(), sizeBytes: 100 })
|
|
117
|
+
Test._setCache("k2", { response: {}, timestamp: Date.now(), sizeBytes: 100 })
|
|
118
|
+
assert.equal(Test._getCacheSize(), 2)
|
|
119
|
+
assert.ok(Test._getTotalMemory() > 0)
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
it("reset clears all state", () => {
|
|
123
|
+
Test._setCache("k1", { response: {}, timestamp: Date.now(), sizeBytes: 100 })
|
|
124
|
+
Test.reset()
|
|
125
|
+
assert.equal(Test._getCacheSize(), 0)
|
|
126
|
+
assert.equal(Test._getTotalMemory(), 0)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it("setSettings works", () => {
|
|
130
|
+
Test.setSettings({ maxEntries: 999, maxMemoryBytes: 999999 })
|
|
131
|
+
const state = Test.getState()
|
|
132
|
+
assert.equal(state.settings.maxEntries, 999)
|
|
133
|
+
assert.equal(state.settings.maxMemoryBytes, 999999)
|
|
134
|
+
})
|
|
135
|
+
})
|
|
136
|
+
})
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
// L1 Cache Extension — in-memory response cache for pi
|
|
2
|
+
//
|
|
3
|
+
// Stoic Unix principle: One thing, done well.
|
|
4
|
+
// Optimized for minimal CPU/RAM overhead.
|
|
5
|
+
//
|
|
6
|
+
// Architecture:
|
|
7
|
+
// pi → [L1: RAM Map] → [L2: Redis via LiteLLM] → Provider
|
|
8
|
+
// Lookup: ~0.1ms (vs 150ms Redis, 1-3s API)
|
|
9
|
+
//
|
|
10
|
+
// Design goals:
|
|
11
|
+
// - Fast string hash (FNV-1a) — ~100x faster than SHA256
|
|
12
|
+
// - Hard memory cap — never lets RAM bloat
|
|
13
|
+
// - TTL + LRU eviction
|
|
14
|
+
// - CPU-aware graceful degradation
|
|
15
|
+
// - Zero dependencies, single file
|
|
16
|
+
|
|
17
|
+
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"
|
|
18
|
+
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Types
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
interface CacheEntry {
|
|
24
|
+
response: unknown
|
|
25
|
+
timestamp: number
|
|
26
|
+
sizeBytes: number
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface Settings {
|
|
30
|
+
enabled: boolean
|
|
31
|
+
maxEntries: number
|
|
32
|
+
maxMemoryBytes: number
|
|
33
|
+
ttlSeconds: number
|
|
34
|
+
cpuThreshold: number
|
|
35
|
+
logStats: boolean
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface Stats {
|
|
39
|
+
hits: number
|
|
40
|
+
misses: number
|
|
41
|
+
evictions: number
|
|
42
|
+
cpuSkips: number
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
// Configuration
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
|
|
49
|
+
const DEFAULTS: Settings = {
|
|
50
|
+
enabled: true,
|
|
51
|
+
maxEntries: 200,
|
|
52
|
+
maxMemoryBytes: 20 * 1024 * 1024, // 20MB
|
|
53
|
+
ttlSeconds: 3600,
|
|
54
|
+
cpuThreshold: 95,
|
|
55
|
+
logStats: false,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Users can override via environment variables (highest priority)
|
|
59
|
+
function envSettings(): Partial<Settings> {
|
|
60
|
+
const out: Partial<Settings> = {}
|
|
61
|
+
if (process.env.L1_CACHE_ENABLED !== undefined) out.enabled = process.env.L1_CACHE_ENABLED !== "false"
|
|
62
|
+
if (process.env.L1_CACHE_MAX_ENTRIES) out.maxEntries = parseInt(process.env.L1_CACHE_MAX_ENTRIES, 10) || DEFAULTS.maxEntries
|
|
63
|
+
if (process.env.L1_CACHE_MAX_MB) out.maxMemoryBytes = (parseInt(process.env.L1_CACHE_MAX_MB, 10) || 20) * 1024 * 1024
|
|
64
|
+
if (process.env.L1_CACHE_TTL) out.ttlSeconds = parseInt(process.env.L1_CACHE_TTL, 10) || DEFAULTS.ttlSeconds
|
|
65
|
+
if (process.env.L1_CACHE_LOG) out.logStats = process.env.L1_CACHE_LOG === "true"
|
|
66
|
+
return out
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const settings: Settings = { ...DEFAULTS, ...envSettings() }
|
|
70
|
+
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
// State
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
let cache = new Map<string, CacheEntry>()
|
|
76
|
+
let totalMemory = 0
|
|
77
|
+
let initialCpuLoad = 0
|
|
78
|
+
let initialCpuStatus: "ok" | "disabled" | "error" = "ok"
|
|
79
|
+
let lastCleanup = Date.now()
|
|
80
|
+
const stats: Stats = { hits: 0, misses: 0, evictions: 0, cpuSkips: 0 }
|
|
81
|
+
|
|
82
|
+
// ---------------------------------------------------------------------------
|
|
83
|
+
// Core helpers
|
|
84
|
+
// ---------------------------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
/** Fast string hash (FNV-1a) — ~100x faster than SHA256, enough for cache keys */
|
|
87
|
+
function fastHash(str: string): string {
|
|
88
|
+
let hash = 2166136261
|
|
89
|
+
for (let i = 0; i < str.length; i++) {
|
|
90
|
+
hash ^= str.charCodeAt(i)
|
|
91
|
+
hash = Math.imul(hash, 16777619)
|
|
92
|
+
}
|
|
93
|
+
return hash.toString(16)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Estimate in-memory size of an arbitrary object (UTF-16 overhead ×2) */
|
|
97
|
+
function estimateSize(obj: unknown): number {
|
|
98
|
+
try {
|
|
99
|
+
const json = JSON.stringify(obj)
|
|
100
|
+
if (json === undefined) return 64 // undefined/unserializable primitive
|
|
101
|
+
return json.length * 2
|
|
102
|
+
} catch {
|
|
103
|
+
return 1024 // fallback for non-serializable
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function log(...args: unknown[]) {
|
|
108
|
+
if (settings.logStats) console.log("[l1-cache]", ...args)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
// Eviction
|
|
113
|
+
// ---------------------------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
function evictOldest(count: number) {
|
|
116
|
+
if (count <= 0) return
|
|
117
|
+
const sorted = Array.from(cache.entries()).sort((a, b) => a[1].timestamp - b[1].timestamp)
|
|
118
|
+
for (let i = 0; i < Math.min(count, sorted.length); i++) {
|
|
119
|
+
const [key, entry] = sorted[i]
|
|
120
|
+
totalMemory -= entry.sizeBytes
|
|
121
|
+
cache.delete(key)
|
|
122
|
+
stats.evictions++
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Enforce size and memory caps using LRU-style (oldest-first) eviction */
|
|
127
|
+
function evictIfNeeded() {
|
|
128
|
+
while (cache.size > settings.maxEntries) {
|
|
129
|
+
evictOldest(Math.max(1, Math.ceil(settings.maxEntries * 0.1)))
|
|
130
|
+
}
|
|
131
|
+
while (totalMemory > settings.maxMemoryBytes) {
|
|
132
|
+
evictOldest(Math.max(1, Math.ceil(settings.maxEntries * 0.2)))
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Remove expired entries (called periodically + on access) */
|
|
137
|
+
function cleanupExpired() {
|
|
138
|
+
const now = Date.now()
|
|
139
|
+
let expired = 0
|
|
140
|
+
for (const [key, entry] of cache.entries()) {
|
|
141
|
+
if (now - entry.timestamp > settings.ttlSeconds * 1000) {
|
|
142
|
+
totalMemory -= entry.sizeBytes
|
|
143
|
+
cache.delete(key)
|
|
144
|
+
expired++
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (expired > 0) log(`expired ${expired} entries`)
|
|
148
|
+
lastCleanup = now
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// ---------------------------------------------------------------------------
|
|
152
|
+
// CPU check (once at startup — not per-request, to avoid overhead)
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
|
|
155
|
+
async function checkInitialCpuLoad(): Promise<number> {
|
|
156
|
+
try {
|
|
157
|
+
const { execSync } = await import("child_process")
|
|
158
|
+
if (process.platform === "win32") {
|
|
159
|
+
const out = execSync("wmic cpu get loadpercentage /value", { encoding: "utf8", timeout: 2000 })
|
|
160
|
+
const match = out.match(/LoadPercentage\s*:\s*(\d+)/)
|
|
161
|
+
return match ? parseInt(match[1], 10) : 0
|
|
162
|
+
}
|
|
163
|
+
const out = execSync("cat /proc/loadavg", { encoding: "utf8", timeout: 2000 })
|
|
164
|
+
const { cpus } = await import("os")
|
|
165
|
+
const cores = cpus().length
|
|
166
|
+
const load = parseFloat(out.split(" ")[0])
|
|
167
|
+
return Math.min(100, (load / cores) * 100)
|
|
168
|
+
} catch {
|
|
169
|
+
return 0
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ---------------------------------------------------------------------------
|
|
174
|
+
// Tests (self-check on load — see also src/index.test.ts)
|
|
175
|
+
// ---------------------------------------------------------------------------
|
|
176
|
+
|
|
177
|
+
export function _testHooks() {
|
|
178
|
+
return {
|
|
179
|
+
fastHash,
|
|
180
|
+
estimateSize,
|
|
181
|
+
evictIfNeeded,
|
|
182
|
+
cleanupExpired,
|
|
183
|
+
evictOldest,
|
|
184
|
+
getStats: () => ({ ...stats }),
|
|
185
|
+
getState: () => ({ size: cache.size, totalMemory, settings: { ...settings } }),
|
|
186
|
+
reset: () => {
|
|
187
|
+
cache = new Map()
|
|
188
|
+
totalMemory = 0
|
|
189
|
+
stats.hits = 0
|
|
190
|
+
stats.misses = 0
|
|
191
|
+
stats.evictions = 0
|
|
192
|
+
stats.cpuSkips = 0
|
|
193
|
+
},
|
|
194
|
+
_setCache: (key: string, entry: CacheEntry) => {
|
|
195
|
+
cache.set(key, entry)
|
|
196
|
+
totalMemory += entry.sizeBytes
|
|
197
|
+
},
|
|
198
|
+
_getCacheSize: () => cache.size,
|
|
199
|
+
_getTotalMemory: () => totalMemory,
|
|
200
|
+
setSettings: (patch: Partial<Settings>) => Object.assign(settings, patch),
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
// Main plugin
|
|
206
|
+
// ---------------------------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
export default async function (pi: ExtensionAPI) {
|
|
209
|
+
// Async init: check CPU once at startup
|
|
210
|
+
try {
|
|
211
|
+
initialCpuLoad = await checkInitialCpuLoad()
|
|
212
|
+
if (initialCpuLoad > settings.cpuThreshold) {
|
|
213
|
+
settings.enabled = false
|
|
214
|
+
initialCpuStatus = "disabled"
|
|
215
|
+
console.log(
|
|
216
|
+
`[l1-cache] disabled (CPU ${initialCpuLoad.toFixed(0)}% > ${settings.cpuThreshold}% threshold)`,
|
|
217
|
+
)
|
|
218
|
+
} else {
|
|
219
|
+
initialCpuStatus = "ok"
|
|
220
|
+
console.log(
|
|
221
|
+
`[l1-cache] enabled (max ${settings.maxEntries} entries, ${(settings.maxMemoryBytes / 1024 / 1024).toFixed(0)}MB, TTL ${settings.ttlSeconds}s)`,
|
|
222
|
+
)
|
|
223
|
+
}
|
|
224
|
+
} catch (err) {
|
|
225
|
+
initialCpuStatus = "error"
|
|
226
|
+
console.log(`[l1-cache] CPU check failed (${err}); continuing enabled`)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// Periodic cleanup (every 10 minutes)
|
|
230
|
+
const cleanupTimer = setInterval(cleanupExpired, 10 * 60 * 1000)
|
|
231
|
+
pi.on("session_shutdown", () => clearInterval(cleanupTimer))
|
|
232
|
+
|
|
233
|
+
// Interceptor: cache lookup before provider request
|
|
234
|
+
pi.on("before_provider_request", async (event, ctx) => {
|
|
235
|
+
if (!settings.enabled) return
|
|
236
|
+
|
|
237
|
+
const model = ctx.model?.id || "unknown"
|
|
238
|
+
const messages = event.messages || []
|
|
239
|
+
const params = event.parameters || {}
|
|
240
|
+
|
|
241
|
+
// Fast hash; no per-request CPU check to stay on the hot path
|
|
242
|
+
const key = fastHash(model + JSON.stringify(messages) + JSON.stringify(params))
|
|
243
|
+
const entry = cache.get(key)
|
|
244
|
+
|
|
245
|
+
if (entry && Date.now() - entry.timestamp <= settings.ttlSeconds * 1000) {
|
|
246
|
+
stats.hits++
|
|
247
|
+
return entry.response as never
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Miss — remember the key for after_provider_response
|
|
251
|
+
event._cacheKey = key
|
|
252
|
+
stats.misses++
|
|
253
|
+
})
|
|
254
|
+
|
|
255
|
+
// Interceptor: store response after provider finishes
|
|
256
|
+
pi.on("after_provider_response", async (event) => {
|
|
257
|
+
if (!settings.enabled) return
|
|
258
|
+
if (!event._cacheKey) return
|
|
259
|
+
|
|
260
|
+
const key = event._cacheKey as string
|
|
261
|
+
const response = event.response ?? event.choices ?? event
|
|
262
|
+
const size = estimateSize(response)
|
|
263
|
+
|
|
264
|
+
cache.set(key, { response, timestamp: Date.now(), sizeBytes: size })
|
|
265
|
+
totalMemory += size
|
|
266
|
+
evictIfNeeded()
|
|
267
|
+
})
|
|
268
|
+
|
|
269
|
+
// Commands
|
|
270
|
+
pi.registerCommand("l1-cache", {
|
|
271
|
+
description: "Show L1 cache stats, or use 'clear' to reset",
|
|
272
|
+
handler: async (args: string, ctx: ExtensionContext) => {
|
|
273
|
+
const arg = (args ?? "").trim()
|
|
274
|
+
|
|
275
|
+
if (arg === "clear") {
|
|
276
|
+
cache.clear()
|
|
277
|
+
totalMemory = 0
|
|
278
|
+
stats.hits = 0
|
|
279
|
+
stats.misses = 0
|
|
280
|
+
stats.evictions = 0
|
|
281
|
+
ctx.ui.notify("L1 cache cleared", "success")
|
|
282
|
+
return
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (arg === "stats" || arg === "") {
|
|
286
|
+
const hitRate =
|
|
287
|
+
stats.hits + stats.misses > 0 ? ((stats.hits / (stats.hits + stats.misses)) * 100).toFixed(1) : "0.0"
|
|
288
|
+
const lines = [
|
|
289
|
+
`L1 cache status: ${settings.enabled ? "ENABLED" : "disabled"}`,
|
|
290
|
+
`Entries: ${cache.size} / ${settings.maxEntries}`,
|
|
291
|
+
`Memory: ${(totalMemory / 1024 / 1024).toFixed(1)}MB / ${(settings.maxMemoryBytes / 1024 / 1024).toFixed(1)}MB`,
|
|
292
|
+
`TTL: ${settings.ttlSeconds}s | CPU threshold: ${settings.cpuThreshold}%`,
|
|
293
|
+
`Hits: ${stats.hits} | Misses: ${stats.misses} | Evictions: ${stats.evictions}`,
|
|
294
|
+
`Hit rate: ${hitRate}%`,
|
|
295
|
+
`Init CPU: ${initialCpuLoad.toFixed(1)}% (${initialCpuStatus})`,
|
|
296
|
+
`Last cleanup: ${new Date(lastCleanup).toISOString()}`,
|
|
297
|
+
]
|
|
298
|
+
ctx.ui.notify(lines.join("\n"), "info")
|
|
299
|
+
return ""
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
if (arg === "enable") {
|
|
303
|
+
settings.enabled = true
|
|
304
|
+
ctx.ui.notify("L1 cache enabled", "success")
|
|
305
|
+
return
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
if (arg === "disable") {
|
|
309
|
+
settings.enabled = false
|
|
310
|
+
ctx.ui.notify("L1 cache disabled", "success")
|
|
311
|
+
return
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
ctx.ui.notify(`Unknown command: /l1-cache ${arg}\nUsage: /l1-cache [stats|clear|enable|disable]`, "error")
|
|
315
|
+
return ""
|
|
316
|
+
},
|
|
317
|
+
})
|
|
318
|
+
|
|
319
|
+
console.log("[l1-cache] ready. /l1-cache for stats.")
|
|
320
|
+
}
|