pi-hermes-memory 0.3.3 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -9
- package/docs/0.4/PLAN.md +160 -0
- package/docs/0.4/TASKS.md +113 -0
- package/docs/ROADMAP.md +47 -29
- package/package.json +5 -1
- package/src/constants.ts +3 -3
- package/src/handlers/index-sessions.ts +64 -0
- package/src/index.ts +39 -0
- package/src/skills/learn-memory-tool/SKILL.md +125 -0
- package/src/store/db.ts +84 -0
- package/src/store/schema.ts +94 -0
- package/src/store/session-indexer.ts +153 -0
- package/src/store/session-parser.ts +214 -0
- package/src/store/session-search.ts +134 -0
- package/src/store/sqlite-memory-store.ts +215 -0
- package/src/tools/memory-search-tool.ts +74 -0
- package/src/tools/session-search-tool.ts +79 -0
- package/src/types.ts +7 -3
package/README.md
CHANGED
|
@@ -19,6 +19,9 @@ Your Pi agent normally forgets everything when you close a session. This extensi
|
|
|
19
19
|
| **Memory Aging** | Entries carry timestamps — consolidation knows which facts are stale and which are fresh |
|
|
20
20
|
| **Project Memory** | Per-project memory (`~/.pi/agent/<project>/MEMORY.md`) alongside your global memory |
|
|
21
21
|
| **Secret Detection** | API keys, tokens, SSH keys, and credential assignments are blocked from being persisted to memory |
|
|
22
|
+
| **Session History Search** | Search across all past conversations via SQLite FTS5 — "what did we discuss about auth?" |
|
|
23
|
+
| **Extended Memory Store** | Unlimited searchable memories beyond the core 5,000-char limit |
|
|
24
|
+
| **Learn Memory Tool** | `/learn-memory-tool` — a skill that teaches users how to use the memory system |
|
|
22
25
|
|
|
23
26
|
## How It Works
|
|
24
27
|
|
|
@@ -157,9 +160,34 @@ Run `tsc --noEmit` and confirm zero errors.
|
|
|
157
160
|
|
|
158
161
|
| Store | File | What goes here | Limit |
|
|
159
162
|
|---|---|---|---|
|
|
160
|
-
| **memory** | `MEMORY.md` | Agent's notes — env facts, project conventions, tool quirks, lessons learned |
|
|
161
|
-
| **user** | `USER.md` | User profile — name, preferences, communication style, habits |
|
|
163
|
+
| **memory** | `MEMORY.md` | Agent's notes — env facts, project conventions, tool quirks, lessons learned | 5,000 chars |
|
|
164
|
+
| **user** | `USER.md` | User profile — name, preferences, communication style, habits | 5,000 chars |
|
|
162
165
|
| **skills** | `skills/*.md` | Procedures — *how* to debug, deploy, test, or fix something | Unlimited |
|
|
166
|
+
| **extended** | `sessions.db` | Searchable memories beyond the core limit | Unlimited |
|
|
167
|
+
| **sessions** | `sessions.db` | Past conversation history (searchable via FTS5) | Unlimited |
|
|
168
|
+
|
|
169
|
+
### Session History Search
|
|
170
|
+
|
|
171
|
+
The extension indexes your Pi session history into a SQLite database with FTS5 full-text search. The agent can search across all past conversations using the `session_search` tool:
|
|
172
|
+
|
|
173
|
+
| Tool | What it does |
|
|
174
|
+
|---|---|---|
|
|
175
|
+
| `session_search` | Search past conversations — "what did we discuss about auth?" |
|
|
176
|
+
| `memory_search` | Search extended memory store — unlimited capacity, keyword-based |
|
|
177
|
+
|
|
178
|
+
Session history is indexed automatically on session shutdown. To bulk-import existing sessions:
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
/memory-index-sessions
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### Extended Memory Store
|
|
185
|
+
|
|
186
|
+
When the core memory (5,000 chars) isn't enough, the agent can store additional memories in the SQLite-backed extended store. These are searchable via `memory_search` but not automatically injected into the system prompt.
|
|
187
|
+
|
|
188
|
+
This is the **hybrid memory architecture**:
|
|
189
|
+
- **Core memory** (MEMORY.md/USER.md): Always injected, 5,000 chars each, human-readable
|
|
190
|
+
- **Extended memory** (SQLite): Unlimited, searchable on demand, agent-driven
|
|
163
191
|
|
|
164
192
|
### Correction Detection
|
|
165
193
|
|
|
@@ -211,6 +239,8 @@ This means skills build up naturally over time without you having to ask.
|
|
|
211
239
|
| `/memory-consolidate` | Manually trigger memory consolidation to free space |
|
|
212
240
|
| `/memory-interview` | Answer a few questions to pre-fill your user profile |
|
|
213
241
|
| `/memory-switch-project` | List all project memories and their entry counts |
|
|
242
|
+
| `/memory-index-sessions` | Import past Pi sessions into the search database |
|
|
243
|
+
| `/learn-memory-tool` | Skill that teaches users how to use the memory system |
|
|
214
244
|
|
|
215
245
|
### `/memory-insights` Output
|
|
216
246
|
|
|
@@ -254,9 +284,9 @@ Create `~/.pi/agent/hermes-memory-config.json`:
|
|
|
254
284
|
|
|
255
285
|
```json
|
|
256
286
|
{
|
|
257
|
-
"memoryCharLimit":
|
|
258
|
-
"userCharLimit":
|
|
259
|
-
"projectCharLimit":
|
|
287
|
+
"memoryCharLimit": 5000,
|
|
288
|
+
"userCharLimit": 5000,
|
|
289
|
+
"projectCharLimit": 5000,
|
|
260
290
|
"memoryDir": "~/.pi/agent/memory",
|
|
261
291
|
"nudgeInterval": 10,
|
|
262
292
|
"nudgeToolCalls": 15,
|
|
@@ -271,9 +301,9 @@ Create `~/.pi/agent/hermes-memory-config.json`:
|
|
|
271
301
|
|
|
272
302
|
| Setting | Default | Description |
|
|
273
303
|
|---|---|---|
|
|
274
|
-
| `memoryCharLimit` | `
|
|
275
|
-
| `userCharLimit` | `
|
|
276
|
-
| `projectCharLimit` | `
|
|
304
|
+
| `memoryCharLimit` | `5000` | Max characters in MEMORY.md |
|
|
305
|
+
| `userCharLimit` | `5000` | Max characters in USER.md |
|
|
306
|
+
| `projectCharLimit` | `5000` | Max characters in project-scoped MEMORY.md |
|
|
277
307
|
| `memoryDir` | `~/.pi/agent/memory` | Custom directory for memory files |
|
|
278
308
|
| `nudgeInterval` | `10` | Turns between auto-reviews |
|
|
279
309
|
| `nudgeToolCalls` | `15` | Tool calls between auto-reviews (OR with turns) |
|
|
@@ -290,6 +320,7 @@ Create `~/.pi/agent/hermes-memory-config.json`:
|
|
|
290
320
|
~/.pi/agent/memory/
|
|
291
321
|
├── MEMORY.md ← Agent's personal notes (env facts, patterns, lessons)
|
|
292
322
|
├── USER.md ← User profile (name, preferences, habits)
|
|
323
|
+
├── sessions.db ← SQLite database (session history + extended memory)
|
|
293
324
|
└── skills/
|
|
294
325
|
├── debug-typescript-errors.md
|
|
295
326
|
└── deploy-checklist.md
|
|
@@ -297,11 +328,13 @@ Create `~/.pi/agent/hermes-memory-config.json`:
|
|
|
297
328
|
|
|
298
329
|
These are plain markdown files. You can read and edit them directly if you want to curate what the agent remembers. Memory entries are separated by `§` (section sign). Skills use standard SKILL.md format with frontmatter.
|
|
299
330
|
|
|
331
|
+
The `sessions.db` SQLite database stores session history and extended memory entries. It's searchable via FTS5 full-text search.
|
|
332
|
+
|
|
300
333
|
## Known Limitations
|
|
301
334
|
|
|
302
335
|
- **`§` delimiter**: Memory entries are separated by `§` (section sign). If an entry naturally contains `§`, it will be split incorrectly on reload. This is rare in English text but possible. [Hermes uses the same delimiter.]
|
|
303
336
|
- **Background review cost**: Each review cycle costs one full LLM API call via a child `pi -p` process. Correction detection and skill auto-extraction add occasional extra calls.
|
|
304
|
-
- **
|
|
337
|
+
- **Session search requires indexing**: Past sessions must be indexed before they're searchable. Run `/memory-index-sessions` to bulk-import, or let the extension auto-index on session shutdown.
|
|
305
338
|
- **System prompts are invisible**: Pi's TUI does not display the system prompt. Memory injection works but you won't see it in the interface — verify by asking the agent a question that relies on stored memory.
|
|
306
339
|
- **Skills are agent-generated**: Skills are created by the agent based on its experience. They may not always be perfectly structured. You can edit or delete them in `~/.pi/agent/memory/skills/`.
|
|
307
340
|
|
package/docs/0.4/PLAN.md
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# v0.4 Plan: SQLite FTS5 Session Search + Hybrid Memory
|
|
2
|
+
|
|
3
|
+
## Problem
|
|
4
|
+
|
|
5
|
+
The current memory architecture has two scaling bottlenecks:
|
|
6
|
+
|
|
7
|
+
1. **Memory capacity**: MEMORY.md is capped at 2,200 chars. Power users accumulate knowledge faster than consolidation can manage. Important facts get pruned.
|
|
8
|
+
2. **No session history search**: Past conversations are stored as JSONL files in `~/.pi/agent/sessions/<project>/`, but there's no way to search them. When the agent needs context from a previous session, it's gone forever.
|
|
9
|
+
|
|
10
|
+
## Solution: Hybrid Memory Architecture
|
|
11
|
+
|
|
12
|
+
### Core memory (always injected, unchanged)
|
|
13
|
+
- `MEMORY.md` — 5,000 chars (up from 2,200)
|
|
14
|
+
- `USER.md` — 5,000 chars (up from 1,375)
|
|
15
|
+
- Still injected into every session via `<memory-context>` tags
|
|
16
|
+
- Still human-readable, still editable
|
|
17
|
+
|
|
18
|
+
### Extended memory (SQLite, searchable on demand)
|
|
19
|
+
- `~/.pi/agent/memory/sessions.db`
|
|
20
|
+
- `memories` table — unlimited entries, searchable via FTS5
|
|
21
|
+
- Agent uses `memory_search` tool to query when it needs context
|
|
22
|
+
- Not automatically injected — agent must explicitly search
|
|
23
|
+
|
|
24
|
+
### Session history (SQLite, searchable on demand)
|
|
25
|
+
- Same `sessions.db` file
|
|
26
|
+
- `sessions` + `messages` tables — all past conversations indexed
|
|
27
|
+
- `session_fts` FTS5 index — full-text search across all sessions
|
|
28
|
+
- Agent uses `session_search` tool to find relevant past context
|
|
29
|
+
|
|
30
|
+
## Architecture
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
Session starts
|
|
34
|
+
↓
|
|
35
|
+
┌─────────────────────────────────────────────────┐
|
|
36
|
+
│ System Prompt (always injected) │
|
|
37
|
+
│ ┌─────────────────────────────────────────────┐ │
|
|
38
|
+
│ │ <memory-context> │ │
|
|
39
|
+
│ │ MEMORY (your personal notes) [5,000 chars] │ │
|
|
40
|
+
│ │ ═══ END MEMORY ═══ │ │
|
|
41
|
+
│ │ </memory-context> │ │
|
|
42
|
+
│ │ <memory-context> │ │
|
|
43
|
+
│ │ USER PROFILE [5,000 chars] │ │
|
|
44
|
+
│ │ ═══ END MEMORY ═══ │ │
|
|
45
|
+
│ │ </memory-context> │ │
|
|
46
|
+
│ │ <memory-context> │ │
|
|
47
|
+
│ │ PROJECT MEMORY [5,000 chars] │ │
|
|
48
|
+
│ │ ═══ END MEMORY ═══ │ │
|
|
49
|
+
│ │ </memory-context> │ │
|
|
50
|
+
│ └─────────────────────────────────────────────┘ │
|
|
51
|
+
└─────────────────────────────────────────────────┘
|
|
52
|
+
|
|
53
|
+
Agent has access to tools:
|
|
54
|
+
memory_search("prisma migration")
|
|
55
|
+
→ Searches memories table (global + project)
|
|
56
|
+
→ Returns top-10 relevant entries
|
|
57
|
+
|
|
58
|
+
session_search("how we fixed the test hang")
|
|
59
|
+
→ Searches session history via FTS5
|
|
60
|
+
→ Returns relevant conversation snippets
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Data Model
|
|
64
|
+
|
|
65
|
+
```sql
|
|
66
|
+
-- Session metadata
|
|
67
|
+
CREATE TABLE sessions (
|
|
68
|
+
id TEXT PRIMARY KEY, -- UUID from JSONL
|
|
69
|
+
project TEXT NOT NULL, -- decoded cwd path
|
|
70
|
+
started_at TEXT NOT NULL, -- ISO timestamp
|
|
71
|
+
ended_at TEXT, -- ISO timestamp (null if still running)
|
|
72
|
+
message_count INTEGER DEFAULT 0
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
-- All messages from all sessions
|
|
76
|
+
CREATE TABLE messages (
|
|
77
|
+
id TEXT PRIMARY KEY, -- message ID from JSONL
|
|
78
|
+
session_id TEXT NOT NULL REFERENCES sessions(id),
|
|
79
|
+
role TEXT NOT NULL, -- 'user', 'assistant', 'system'
|
|
80
|
+
content TEXT NOT NULL, -- extracted text content
|
|
81
|
+
timestamp TEXT NOT NULL, -- ISO timestamp
|
|
82
|
+
tool_calls TEXT -- JSON array of tool call names (for assistant messages)
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
-- FTS5 index for full-text search across messages
|
|
86
|
+
CREATE VIRTUAL TABLE message_fts USING fts5(
|
|
87
|
+
content,
|
|
88
|
+
content='messages',
|
|
89
|
+
content_rowid='rowid'
|
|
90
|
+
);
|
|
91
|
+
|
|
92
|
+
-- Extended memory entries (beyond MEMORY.md limit)
|
|
93
|
+
CREATE TABLE memories (
|
|
94
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
95
|
+
project TEXT, -- NULL for global, project name for project-specific
|
|
96
|
+
target TEXT NOT NULL, -- 'memory' or 'user'
|
|
97
|
+
content TEXT NOT NULL,
|
|
98
|
+
created DATE NOT NULL,
|
|
99
|
+
last_referenced DATE NOT NULL
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
-- FTS5 index for memory search
|
|
103
|
+
CREATE VIRTUAL TABLE memory_fts USING fts5(
|
|
104
|
+
content,
|
|
105
|
+
content='memories',
|
|
106
|
+
content_rowid='id'
|
|
107
|
+
);
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Key Design Decisions
|
|
111
|
+
|
|
112
|
+
### 1. Session indexing is lazy (on session end)
|
|
113
|
+
- Don't parse JSONL files on every startup
|
|
114
|
+
- Index a session when `session_shutdown` fires
|
|
115
|
+
- Bulk import existing sessions via `/memory-index-sessions` command
|
|
116
|
+
|
|
117
|
+
### 2. FTS5 for both memories and sessions
|
|
118
|
+
- Keyword search is sufficient for v0.4
|
|
119
|
+
- Embeddings deferred to v0.5+ if search quality isn't enough
|
|
120
|
+
- `better-sqlite3` includes FTS5 by default
|
|
121
|
+
|
|
122
|
+
### 3. Single DB file
|
|
123
|
+
- `~/.pi/agent/memory/sessions.db` stores everything
|
|
124
|
+
- Memories + sessions + FTS indices in one file
|
|
125
|
+
- Simple backup (copy one file), simple cleanup
|
|
126
|
+
|
|
127
|
+
### 4. Agent-driven search
|
|
128
|
+
- `memory_search` and `session_search` are LLM tools
|
|
129
|
+
- Agent decides when to search (not automatic)
|
|
130
|
+
- Avoids injecting irrelevant context into every session
|
|
131
|
+
|
|
132
|
+
### 5. Char limit increase to 5,000
|
|
133
|
+
- MEMORY.md: 2,200 → 5,000 chars
|
|
134
|
+
- USER.md: 1,375 → 5,000 chars
|
|
135
|
+
- Project MEMORY.md: 2,200 → 5,000 chars
|
|
136
|
+
- More room for core memories before consolidation kicks in
|
|
137
|
+
|
|
138
|
+
## Dependencies
|
|
139
|
+
|
|
140
|
+
| Package | Purpose | Size |
|
|
141
|
+
|---|---|---|
|
|
142
|
+
| `better-sqlite3` | SQLite with FTS5 | ~1MB native addon |
|
|
143
|
+
|
|
144
|
+
## Risks
|
|
145
|
+
|
|
146
|
+
| Risk | Mitigation |
|
|
147
|
+
|---|---|
|
|
148
|
+
| `better-sqlite3` is a native C++ addon | Standard for dev tools; CI has build tools |
|
|
149
|
+
| FTS5 search quality | Start with keyword search, add embeddings later if needed |
|
|
150
|
+
| Session JSONL format changes | Parse defensively, skip unknown message types |
|
|
151
|
+
| Large session history (1000+ sessions) | FTS5 handles this well; add pagination to results |
|
|
152
|
+
| DB corruption | Atomic writes, WAL mode, backup before migrations |
|
|
153
|
+
|
|
154
|
+
## Success Criteria
|
|
155
|
+
|
|
156
|
+
1. `session_search("prisma migration")` returns relevant conversation snippets from past sessions
|
|
157
|
+
2. `memory_search("auth setup")` returns relevant entries from extended memory store
|
|
158
|
+
3. MEMORY.md limit raised to 5,000 chars without breaking existing functionality
|
|
159
|
+
4. Existing session files indexed without data loss
|
|
160
|
+
5. All tests pass, zero regressions
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# v0.4 Tasks: SQLite FTS5 Session Search + Hybrid Memory
|
|
2
|
+
|
|
3
|
+
## Status Legend
|
|
4
|
+
- `[ ]` Not started
|
|
5
|
+
- `[~]` In progress
|
|
6
|
+
- `[x]` Done
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## Epic 1: SQLite Foundation ✅
|
|
11
|
+
|
|
12
|
+
### Task 1.1: Install better-sqlite3 and create DB module
|
|
13
|
+
- [x] Install `better-sqlite3` + `@types/better-sqlite3`
|
|
14
|
+
- [x] Create `src/store/db.ts` — DatabaseManager class
|
|
15
|
+
- [x] Create `tests/store/db.test.ts` — 14 tests passing
|
|
16
|
+
|
|
17
|
+
### Task 1.2: Create schema and migrations
|
|
18
|
+
- [x] Define schema in `src/store/schema.ts`
|
|
19
|
+
- [x] Add triggers to keep FTS index in sync
|
|
20
|
+
- [x] Test: schema creates cleanly on fresh DB, idempotent on existing DB
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Epic 2: Session History Indexing ✅
|
|
25
|
+
|
|
26
|
+
### Task 2.1: JSONL parser
|
|
27
|
+
- [x] Create `src/store/session-parser.ts`
|
|
28
|
+
- [x] Create `tests/store/session-parser.test.ts` — 14 tests passing
|
|
29
|
+
|
|
30
|
+
### Task 2.2: Session indexer
|
|
31
|
+
- [x] Create `src/store/session-indexer.ts`
|
|
32
|
+
- [x] Create `tests/store/session-indexer.test.ts` — 12 tests passing
|
|
33
|
+
|
|
34
|
+
### Task 2.3: /memory-index-sessions command
|
|
35
|
+
- [x] Create `src/handlers/index-sessions.ts`
|
|
36
|
+
- [x] Wire into `src/index.ts`
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Epic 3: Session Search ✅
|
|
41
|
+
|
|
42
|
+
### Task 3.1: Session search store
|
|
43
|
+
- [x] Create `src/store/session-search.ts` — FTS5 search
|
|
44
|
+
- [x] Create `tests/store/session-search.test.ts` — 11 tests passing
|
|
45
|
+
|
|
46
|
+
### Task 3.2: session_search tool
|
|
47
|
+
- [x] Create `src/tools/session-search-tool.ts`
|
|
48
|
+
- [x] Register in `src/index.ts`
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Epic 4: Extended Memory Store ✅
|
|
53
|
+
|
|
54
|
+
### Task 4.1: SQLite memory store
|
|
55
|
+
- [x] Create `src/store/sqlite-memory-store.ts`
|
|
56
|
+
- [x] Create `tests/store/sqlite-memory-store.test.ts` — 19 tests passing
|
|
57
|
+
|
|
58
|
+
### Task 4.2: memory_search tool
|
|
59
|
+
- [x] Create `src/tools/memory-search-tool.ts`
|
|
60
|
+
- [x] Register in `src/index.ts`
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Epic 5: Char Limit Increase ✅
|
|
65
|
+
|
|
66
|
+
### Task 5.1: Update defaults
|
|
67
|
+
- [x] Update `src/constants.ts` — 5000 defaults
|
|
68
|
+
- [x] Update `src/types.ts` — updated comments
|
|
69
|
+
- [x] Update README configuration table
|
|
70
|
+
|
|
71
|
+
### Task 5.2: Update tests
|
|
72
|
+
- [x] Updated all tests that depend on char limits
|
|
73
|
+
- [x] Verified consolidation still works at new limits
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## Epic 6: Integration & Polish ✅
|
|
78
|
+
|
|
79
|
+
### Task 6.1: Wire everything into index.ts
|
|
80
|
+
- [x] Initialize DatabaseManager on extension load
|
|
81
|
+
- [x] Register `session_search` and `memory_search` tools
|
|
82
|
+
- [x] Register `/memory-index-sessions` command
|
|
83
|
+
- [x] Auto-index session on `session_shutdown` event
|
|
84
|
+
|
|
85
|
+
### Task 6.2: Add session indexing to background review
|
|
86
|
+
- [x] Auto-index on session_shutdown (indexes most recent session)
|
|
87
|
+
|
|
88
|
+
### Task 6.3: Update README
|
|
89
|
+
- [x] Added session history search and extended memory sections
|
|
90
|
+
- [x] Updated char limits: 2200/1375 → 5000
|
|
91
|
+
- [x] Updated configuration table and JSON example
|
|
92
|
+
- [x] Updated Where Data Lives with sessions.db
|
|
93
|
+
- [x] Updated Known Limitations
|
|
94
|
+
|
|
95
|
+
### Task 6.4: Version bump & release
|
|
96
|
+
- [x] Bump version to `0.4.0`
|
|
97
|
+
- [x] Run full test suite — 272 tests passing
|
|
98
|
+
- [x] Publish to npm
|
|
99
|
+
- [x] Create GitHub release
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Summary
|
|
104
|
+
|
|
105
|
+
| Epic | Files Created | Tests |
|
|
106
|
+
|---|---|---|
|
|
107
|
+
| 1. SQLite Foundation | db.ts, schema.ts | 14 |
|
|
108
|
+
| 2. Session Indexing | session-parser.ts, session-indexer.ts, index-sessions.ts | 26 |
|
|
109
|
+
| 3. Session Search | session-search.ts, session-search-tool.ts | 11 |
|
|
110
|
+
| 4. Extended Memory | sqlite-memory-store.ts, memory-search-tool.ts | 19 |
|
|
111
|
+
| 5. Char Limits | constants.ts, types.ts | — |
|
|
112
|
+
| 6. Integration | index.ts, README.md, learn-memory-tool skill | — |
|
|
113
|
+
| **Total** | **12 new files** | **272 tests** |
|
package/docs/ROADMAP.md
CHANGED
|
@@ -242,50 +242,68 @@ Project-scoped memory (`~/.pi/agent/<project>/MEMORY.md`) was added in the featu
|
|
|
242
242
|
|
|
243
243
|
---
|
|
244
244
|
|
|
245
|
-
## v0.4.0 —
|
|
245
|
+
## v0.4.0 — SQLite FTS5 Session Search + Hybrid Memory
|
|
246
246
|
|
|
247
|
-
**Goal**: SQLite backend with FTS5 full-text search over all past conversations.
|
|
247
|
+
**Goal**: SQLite backend with FTS5 full-text search over all past conversations. Extended memory store with unlimited capacity. Increased core memory limits.
|
|
248
248
|
|
|
249
|
-
|
|
249
|
+
**Why now**: Power users hit the 2,200-char limit and lose important knowledge. Past sessions are rich with context but unreachable. Hybrid memory solves both — core memories always injected, deep knowledge searchable on demand.
|
|
250
250
|
|
|
251
|
-
|
|
252
|
-
interface MemoryBackend {
|
|
253
|
-
// Write
|
|
254
|
-
add(target: "memory" | "user", entry: MemoryEntry): Promise<MemoryResult>;
|
|
255
|
-
replace(target: "memory" | "user", query: string, entry: MemoryEntry): Promise<MemoryResult>;
|
|
256
|
-
remove(target: "memory" | "user", query: string): Promise<MemoryResult>;
|
|
251
|
+
**Full plan**: `docs/0.4/PLAN.md` · **Tasks**: `docs/0.4/TASKS.md`
|
|
257
252
|
|
|
258
|
-
|
|
259
|
-
getAll(target: "memory" | "user"): Promise<MemoryEntry[]>;
|
|
260
|
-
search(query: string, limit?: number): Promise<MemoryEntry[]>;
|
|
253
|
+
### Architecture
|
|
261
254
|
|
|
262
|
-
// Lifecycle
|
|
263
|
-
formatForSystemPrompt(cwd?: string, prompt?: string): Promise<string>;
|
|
264
|
-
close(): Promise<void>;
|
|
265
|
-
}
|
|
266
255
|
```
|
|
256
|
+
Session starts
|
|
257
|
+
↓
|
|
258
|
+
┌─────────────────────────────────────────────────┐
|
|
259
|
+
│ System Prompt (always injected) │
|
|
260
|
+
│ • MEMORY.md — 5,000 chars (up from 2,200) │
|
|
261
|
+
│ • USER.md — 5,000 chars (up from 1,375) │
|
|
262
|
+
│ • Project MEMORY.md — 5,000 chars │
|
|
263
|
+
│ • Skills index │
|
|
264
|
+
└─────────────────────────────────────────────────┘
|
|
265
|
+
|
|
266
|
+
Agent has access to tools:
|
|
267
|
+
memory_search("prisma migration")
|
|
268
|
+
→ Searches SQLite memories table (global + project)
|
|
269
|
+
→ Returns top-10 relevant entries
|
|
270
|
+
|
|
271
|
+
session_search("how we fixed the test hang")
|
|
272
|
+
→ Searches session history via FTS5
|
|
273
|
+
→ Returns relevant conversation snippets
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
### Data Model
|
|
267
277
|
|
|
268
|
-
|
|
278
|
+
- `~/.pi/agent/memory/sessions.db` — single SQLite file
|
|
279
|
+
- `sessions` + `messages` tables — all past conversations indexed
|
|
280
|
+
- `message_fts` FTS5 virtual table — full-text search across messages
|
|
281
|
+
- `memories` table — extended memory entries (unlimited, searchable)
|
|
282
|
+
- `memory_fts` FTS5 virtual table — full-text search across memories
|
|
269
283
|
|
|
270
284
|
### Deliverables
|
|
271
285
|
|
|
272
|
-
- [ ] `
|
|
273
|
-
- [ ] `
|
|
274
|
-
- [ ] `
|
|
275
|
-
- [ ]
|
|
276
|
-
- [ ] `
|
|
277
|
-
- [ ]
|
|
278
|
-
- [ ]
|
|
286
|
+
- [ ] `better-sqlite3` dependency — SQLite with FTS5
|
|
287
|
+
- [ ] `src/store/db.ts` — DatabaseManager (lazy init, WAL mode, auto-create tables)
|
|
288
|
+
- [ ] `src/store/session-parser.ts` — JSONL parser for Pi session files
|
|
289
|
+
- [ ] `src/store/session-indexer.ts` — index sessions + messages into SQLite
|
|
290
|
+
- [ ] `src/store/session-search.ts` — FTS5 search across session history
|
|
291
|
+
- [ ] `src/store/sqlite-memory-store.ts` — extended memory store (unlimited, searchable)
|
|
292
|
+
- [ ] `session_search` tool — agent queries past conversations
|
|
293
|
+
- [ ] `memory_search` tool — agent queries extended memories
|
|
294
|
+
- [ ] `/memory-index-sessions` command — bulk import existing sessions
|
|
295
|
+
- [ ] Auto-index session on shutdown
|
|
296
|
+
- [ ] Char limits: MEMORY.md 2,200 → 5,000, USER.md 1,375 → 5,000, project 2,200 → 5,000
|
|
279
297
|
- [ ] Config: `sessionSearchEnabled: boolean` (default: true)
|
|
280
298
|
- [ ] Config: `sessionRetentionDays: number` (default: 90)
|
|
281
|
-
- [ ] Migration tool: markdown → sqlite one-time import
|
|
282
299
|
|
|
283
300
|
### What Does NOT Change
|
|
284
301
|
|
|
285
|
-
- Content scanner (guards all
|
|
286
|
-
-
|
|
302
|
+
- Content scanner (guards all writes)
|
|
303
|
+
- Memory tool interface (add/replace/remove actions)
|
|
287
304
|
- System prompt injection (frozen snapshot pattern)
|
|
288
|
-
-
|
|
305
|
+
- Skills system (unchanged)
|
|
306
|
+
- Background review, correction detection, auto-consolidation (unchanged)
|
|
289
307
|
|
|
290
308
|
---
|
|
291
309
|
|
|
@@ -387,7 +405,7 @@ gantt
|
|
|
387
405
|
Project memory polish :v03d, after v03c, 2d
|
|
388
406
|
|
|
389
407
|
section v0.4.0
|
|
390
|
-
|
|
408
|
+
SQLite FTS5 + session search + hybrid memory :v04a, after v03d, 10d
|
|
391
409
|
|
|
392
410
|
section v0.5.0
|
|
393
411
|
ExternalSync + Mem0 / Honcho :v05a, after v04b, 10d
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-hermes-memory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Your Pi agent remembers everything across sessions — your preferences, your stack, your corrections, and even how it solved problems. Zero-config install, works immediately. Persistent memory + procedural skills + auto-correction detection + security-first content scanning.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -43,8 +43,12 @@
|
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"@mariozechner/pi-ai": "^0.70.0",
|
|
45
45
|
"@mariozechner/pi-coding-agent": "^0.70.0",
|
|
46
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
46
47
|
"tsx": "^4.21.0",
|
|
47
48
|
"typebox": "^1.1.33",
|
|
48
49
|
"typescript": "^6.0.3"
|
|
50
|
+
},
|
|
51
|
+
"dependencies": {
|
|
52
|
+
"better-sqlite3": "^12.9.0"
|
|
49
53
|
}
|
|
50
54
|
}
|
package/src/constants.ts
CHANGED
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
export const ENTRY_DELIMITER = "\n§\n";
|
|
9
9
|
|
|
10
10
|
// ─── Character limits (not tokens — model-independent) ───
|
|
11
|
-
export const DEFAULT_MEMORY_CHAR_LIMIT =
|
|
12
|
-
export const DEFAULT_USER_CHAR_LIMIT =
|
|
11
|
+
export const DEFAULT_MEMORY_CHAR_LIMIT = 5000;
|
|
12
|
+
export const DEFAULT_USER_CHAR_LIMIT = 5000;
|
|
13
13
|
|
|
14
14
|
// ─── Learning loop defaults ───
|
|
15
|
-
export const DEFAULT_PROJECT_CHAR_LIMIT =
|
|
15
|
+
export const DEFAULT_PROJECT_CHAR_LIMIT = 5000;
|
|
16
16
|
|
|
17
17
|
export const DEFAULT_NUDGE_INTERVAL = 10;
|
|
18
18
|
export const DEFAULT_FLUSH_MIN_TURNS = 6;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
4
|
+
import { DatabaseManager } from '../store/db.js';
|
|
5
|
+
import { indexAllSessions, getSessionStats } from '../store/session-indexer.js';
|
|
6
|
+
|
|
7
|
+
const SESSIONS_DIR = path.join(os.homedir(), '.pi', 'agent', 'sessions');
|
|
8
|
+
|
|
9
|
+
export function registerIndexSessionsCommand(pi: ExtensionAPI): void {
|
|
10
|
+
pi.registerCommand("memory-index-sessions", {
|
|
11
|
+
description: "Import past Pi sessions into the search database",
|
|
12
|
+
handler: async (_args, ctx) => {
|
|
13
|
+
const sendUserMessage = (msg: string) => {
|
|
14
|
+
if (ctx && typeof ctx === 'object' && 'sendUserMessage' in ctx) {
|
|
15
|
+
(ctx as { sendUserMessage: (msg: string) => void }).sendUserMessage(msg);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
sendUserMessage('🔍 Indexing session history...');
|
|
20
|
+
|
|
21
|
+
try {
|
|
22
|
+
const memoryDir = path.join(os.homedir(), '.pi', 'agent', 'memory');
|
|
23
|
+
const dbManager = new DatabaseManager(memoryDir);
|
|
24
|
+
|
|
25
|
+
try {
|
|
26
|
+
const result = indexAllSessions(dbManager, SESSIONS_DIR);
|
|
27
|
+
const stats = getSessionStats(dbManager);
|
|
28
|
+
|
|
29
|
+
let output = `\n✅ Session indexing complete!\n\n`;
|
|
30
|
+
output += `📊 Results:\n`;
|
|
31
|
+
output += `• Sessions processed: ${result.sessionsProcessed}\n`;
|
|
32
|
+
output += `• Sessions indexed: ${result.sessionsIndexed}\n`;
|
|
33
|
+
output += `• Sessions skipped (already indexed): ${result.sessionsSkipped}\n`;
|
|
34
|
+
output += `• Messages indexed: ${result.messagesIndexed}\n`;
|
|
35
|
+
|
|
36
|
+
if (stats.projects.length > 0) {
|
|
37
|
+
output += `\n📁 Projects:\n`;
|
|
38
|
+
for (const p of stats.projects) {
|
|
39
|
+
output += `• ${p.project}: ${p.sessions} sessions, ${p.messages} messages\n`;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (result.errors.length > 0) {
|
|
44
|
+
output += `\n⚠️ Errors (${result.errors.length}):\n`;
|
|
45
|
+
for (const err of result.errors.slice(0, 5)) {
|
|
46
|
+
output += `• ${err}\n`;
|
|
47
|
+
}
|
|
48
|
+
if (result.errors.length > 5) {
|
|
49
|
+
output += `• ... and ${result.errors.length - 5} more\n`;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
output += `\n💡 Use the \`session_search\` tool to search across indexed sessions.`;
|
|
54
|
+
|
|
55
|
+
sendUserMessage(output);
|
|
56
|
+
} finally {
|
|
57
|
+
dbManager.close();
|
|
58
|
+
}
|
|
59
|
+
} catch (err) {
|
|
60
|
+
sendUserMessage(`❌ Session indexing failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -27,8 +27,13 @@ import * as os from "node:os";
|
|
|
27
27
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
28
28
|
import { MemoryStore } from "./store/memory-store.js";
|
|
29
29
|
import { SkillStore } from "./store/skill-store.js";
|
|
30
|
+
import { DatabaseManager } from "./store/db.js";
|
|
31
|
+
import { indexSession } from "./store/session-indexer.js";
|
|
32
|
+
import { parseSessionFile } from "./store/session-parser.js";
|
|
30
33
|
import { registerMemoryTool } from "./tools/memory-tool.js";
|
|
31
34
|
import { registerSkillTool } from "./tools/skill-tool.js";
|
|
35
|
+
import { registerSessionSearchTool } from "./tools/session-search-tool.js";
|
|
36
|
+
import { registerMemorySearchTool } from "./tools/memory-search-tool.js";
|
|
32
37
|
import { setupBackgroundReview } from "./handlers/background-review.js";
|
|
33
38
|
import { setupSessionFlush } from "./handlers/session-flush.js";
|
|
34
39
|
import { registerInsightsCommand } from "./handlers/insights.js";
|
|
@@ -38,6 +43,7 @@ import { setupSkillAutoTrigger } from "./handlers/skill-auto-trigger.js";
|
|
|
38
43
|
import { registerSkillsCommand } from "./handlers/skills-command.js";
|
|
39
44
|
import { registerInterviewCommand } from "./handlers/interview.js";
|
|
40
45
|
import { registerSwitchProjectCommand } from "./handlers/switch-project.js";
|
|
46
|
+
import { registerIndexSessionsCommand } from "./handlers/index-sessions.js";
|
|
41
47
|
import { loadConfig } from "./config.js";
|
|
42
48
|
import { detectProject } from "./project.js";
|
|
43
49
|
|
|
@@ -47,6 +53,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
47
53
|
const globalDir = config.memoryDir ?? path.join(os.homedir(), ".pi", "agent", "memory");
|
|
48
54
|
const store = new MemoryStore(config);
|
|
49
55
|
const skillStore = new SkillStore(path.join(globalDir, "skills"));
|
|
56
|
+
const dbManager = new DatabaseManager(globalDir);
|
|
50
57
|
|
|
51
58
|
// Detect project from cwd using shared helper
|
|
52
59
|
const project = detectProject();
|
|
@@ -111,4 +118,36 @@ export default function (pi: ExtensionAPI) {
|
|
|
111
118
|
registerSkillsCommand(pi, skillStore);
|
|
112
119
|
registerInterviewCommand(pi, store);
|
|
113
120
|
registerSwitchProjectCommand(pi);
|
|
121
|
+
|
|
122
|
+
// ── 11. SQLite session search + extended memory ──
|
|
123
|
+
registerSessionSearchTool(pi, dbManager);
|
|
124
|
+
registerMemorySearchTool(pi, dbManager);
|
|
125
|
+
registerIndexSessionsCommand(pi);
|
|
126
|
+
|
|
127
|
+
// ── 12. Auto-index session on shutdown ──
|
|
128
|
+
pi.on("session_shutdown", async (_event, _ctx) => {
|
|
129
|
+
try {
|
|
130
|
+
const fs = require("node:fs");
|
|
131
|
+
const sessionsDir = path.join(os.homedir(), ".pi", "agent", "sessions");
|
|
132
|
+
const cwd = process.cwd();
|
|
133
|
+
const encodedCwd = cwd.replace(/\//g, "-");
|
|
134
|
+
const sessionDir = path.join(sessionsDir, encodedCwd);
|
|
135
|
+
|
|
136
|
+
if (fs.existsSync(sessionDir)) {
|
|
137
|
+
// Find the most recent JSONL file (the one we just finished)
|
|
138
|
+
const files = fs.readdirSync(sessionDir)
|
|
139
|
+
.filter((f: string) => f.endsWith(".jsonl"))
|
|
140
|
+
.sort()
|
|
141
|
+
.reverse();
|
|
142
|
+
if (files.length > 0) {
|
|
143
|
+
const sessionData = parseSessionFile(path.join(sessionDir, files[0]));
|
|
144
|
+
if (sessionData) {
|
|
145
|
+
indexSession(dbManager, sessionData);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
} catch {
|
|
150
|
+
// Silent fail — don't block shutdown
|
|
151
|
+
}
|
|
152
|
+
});
|
|
114
153
|
}
|