somtum 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +663 -0
- package/README.md +459 -0
- package/dist/cli/config_cmd.d.ts +10 -0
- package/dist/cli/config_cmd.d.ts.map +1 -0
- package/dist/cli/config_cmd.js +96 -0
- package/dist/cli/config_cmd.js.map +1 -0
- package/dist/cli/doctor.d.ts +17 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +169 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/edit.d.ts +4 -0
- package/dist/cli/edit.d.ts.map +1 -0
- package/dist/cli/edit.js +63 -0
- package/dist/cli/edit.js.map +1 -0
- package/dist/cli/export.d.ts +19 -0
- package/dist/cli/export.d.ts.map +1 -0
- package/dist/cli/export.js +84 -0
- package/dist/cli/export.js.map +1 -0
- package/dist/cli/forget.d.ts +11 -0
- package/dist/cli/forget.d.ts.map +1 -0
- package/dist/cli/forget.js +32 -0
- package/dist/cli/forget.js.map +1 -0
- package/dist/cli/hook.d.ts +2 -0
- package/dist/cli/hook.d.ts.map +1 -0
- package/dist/cli/hook.js +66 -0
- package/dist/cli/hook.js.map +1 -0
- package/dist/cli/import.d.ts +18 -0
- package/dist/cli/import.d.ts.map +1 -0
- package/dist/cli/import.js +90 -0
- package/dist/cli/import.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +223 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/init.d.ts +24 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +144 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/purge.d.ts +16 -0
- package/dist/cli/purge.d.ts.map +1 -0
- package/dist/cli/purge.js +69 -0
- package/dist/cli/purge.js.map +1 -0
- package/dist/cli/rebuild.d.ts +15 -0
- package/dist/cli/rebuild.d.ts.map +1 -0
- package/dist/cli/rebuild.js +41 -0
- package/dist/cli/rebuild.js.map +1 -0
- package/dist/cli/reindex.d.ts +6 -0
- package/dist/cli/reindex.d.ts.map +1 -0
- package/dist/cli/reindex.js +47 -0
- package/dist/cli/reindex.js.map +1 -0
- package/dist/cli/search.d.ts +16 -0
- package/dist/cli/search.d.ts.map +1 -0
- package/dist/cli/search.js +62 -0
- package/dist/cli/search.js.map +1 -0
- package/dist/cli/show.d.ts +12 -0
- package/dist/cli/show.d.ts.map +1 -0
- package/dist/cli/show.js +49 -0
- package/dist/cli/show.js.map +1 -0
- package/dist/cli/stats.d.ts +27 -0
- package/dist/cli/stats.d.ts.map +1 -0
- package/dist/cli/stats.js +80 -0
- package/dist/cli/stats.js.map +1 -0
- package/dist/cli/sync.d.ts +18 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +138 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +52 -0
- package/dist/config.js.map +1 -0
- package/dist/core/cache.d.ts +29 -0
- package/dist/core/cache.d.ts.map +1 -0
- package/dist/core/cache.js +137 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/db.d.ts +12 -0
- package/dist/core/db.d.ts.map +1 -0
- package/dist/core/db.js +80 -0
- package/dist/core/db.js.map +1 -0
- package/dist/core/embeddings.d.ts +15 -0
- package/dist/core/embeddings.d.ts.map +1 -0
- package/dist/core/embeddings.js +74 -0
- package/dist/core/embeddings.js.map +1 -0
- package/dist/core/embeddings_bootstrap.d.ts +3 -0
- package/dist/core/embeddings_bootstrap.d.ts.map +1 -0
- package/dist/core/embeddings_bootstrap.js +16 -0
- package/dist/core/embeddings_bootstrap.js.map +1 -0
- package/dist/core/embeddings_xenova.d.ts +3 -0
- package/dist/core/embeddings_xenova.d.ts.map +1 -0
- package/dist/core/embeddings_xenova.js +28 -0
- package/dist/core/embeddings_xenova.js.map +1 -0
- package/dist/core/extractor.d.ts +27 -0
- package/dist/core/extractor.d.ts.map +1 -0
- package/dist/core/extractor.js +133 -0
- package/dist/core/extractor.js.map +1 -0
- package/dist/core/file_summary.d.ts +33 -0
- package/dist/core/file_summary.d.ts.map +1 -0
- package/dist/core/file_summary.js +127 -0
- package/dist/core/file_summary.js.map +1 -0
- package/dist/core/fingerprint.d.ts +13 -0
- package/dist/core/fingerprint.d.ts.map +1 -0
- package/dist/core/fingerprint.js +36 -0
- package/dist/core/fingerprint.js.map +1 -0
- package/dist/core/index_gen.d.ts +16 -0
- package/dist/core/index_gen.d.ts.map +1 -0
- package/dist/core/index_gen.js +173 -0
- package/dist/core/index_gen.js.map +1 -0
- package/dist/core/memory_files.d.ts +4 -0
- package/dist/core/memory_files.d.ts.map +1 -0
- package/dist/core/memory_files.js +36 -0
- package/dist/core/memory_files.js.map +1 -0
- package/dist/core/privacy.d.ts +9 -0
- package/dist/core/privacy.d.ts.map +1 -0
- package/dist/core/privacy.js +32 -0
- package/dist/core/privacy.js.map +1 -0
- package/dist/core/project_id.d.ts +3 -0
- package/dist/core/project_id.d.ts.map +1 -0
- package/dist/core/project_id.js +31 -0
- package/dist/core/project_id.js.map +1 -0
- package/dist/core/reindex.d.ts +10 -0
- package/dist/core/reindex.d.ts.map +1 -0
- package/dist/core/reindex.js +40 -0
- package/dist/core/reindex.js.map +1 -0
- package/dist/core/retrieval_stats.d.ts +22 -0
- package/dist/core/retrieval_stats.d.ts.map +1 -0
- package/dist/core/retrieval_stats.js +56 -0
- package/dist/core/retrieval_stats.js.map +1 -0
- package/dist/core/retriever/bm25.d.ts +15 -0
- package/dist/core/retriever/bm25.d.ts.map +1 -0
- package/dist/core/retriever/bm25.js +51 -0
- package/dist/core/retriever/bm25.js.map +1 -0
- package/dist/core/retriever/embeddings.d.ts +9 -0
- package/dist/core/retriever/embeddings.d.ts.map +1 -0
- package/dist/core/retriever/embeddings.js +34 -0
- package/dist/core/retriever/embeddings.js.map +1 -0
- package/dist/core/retriever/factory.d.ts +6 -0
- package/dist/core/retriever/factory.d.ts.map +1 -0
- package/dist/core/retriever/factory.js +53 -0
- package/dist/core/retriever/factory.js.map +1 -0
- package/dist/core/retriever/hybrid.d.ts +14 -0
- package/dist/core/retriever/hybrid.d.ts.map +1 -0
- package/dist/core/retriever/hybrid.js +52 -0
- package/dist/core/retriever/hybrid.js.map +1 -0
- package/dist/core/retriever/llm_index.d.ts +18 -0
- package/dist/core/retriever/llm_index.d.ts.map +1 -0
- package/dist/core/retriever/llm_index.js +90 -0
- package/dist/core/retriever/llm_index.js.map +1 -0
- package/dist/core/retriever/types.d.ts +16 -0
- package/dist/core/retriever/types.d.ts.map +1 -0
- package/dist/core/retriever/types.js +2 -0
- package/dist/core/retriever/types.js.map +1 -0
- package/dist/core/schema.d.ts +470 -0
- package/dist/core/schema.d.ts.map +1 -0
- package/dist/core/schema.js +153 -0
- package/dist/core/schema.js.map +1 -0
- package/dist/core/store.d.ts +35 -0
- package/dist/core/store.d.ts.map +1 -0
- package/dist/core/store.js +172 -0
- package/dist/core/store.js.map +1 -0
- package/dist/core/tokens.d.ts +3 -0
- package/dist/core/tokens.d.ts.map +1 -0
- package/dist/core/tokens.js +12 -0
- package/dist/core/tokens.js.map +1 -0
- package/dist/core/transcript.d.ts +16 -0
- package/dist/core/transcript.d.ts.map +1 -0
- package/dist/core/transcript.js +128 -0
- package/dist/core/transcript.js.map +1 -0
- package/dist/db/migrations/001_init.sql +90 -0
- package/dist/db/migrations/002_cache_files.sql +6 -0
- package/dist/db/migrations/003_retrieval_stats.sql +17 -0
- package/dist/hooks/post_session.d.ts +65 -0
- package/dist/hooks/post_session.d.ts.map +1 -0
- package/dist/hooks/post_session.js +249 -0
- package/dist/hooks/post_session.js.map +1 -0
- package/dist/hooks/pre_prompt.d.ts +55 -0
- package/dist/hooks/pre_prompt.d.ts.map +1 -0
- package/dist/hooks/pre_prompt.js +98 -0
- package/dist/hooks/pre_prompt.js.map +1 -0
- package/dist/hooks/pre_read.d.ts +56 -0
- package/dist/hooks/pre_read.d.ts.map +1 -0
- package/dist/hooks/pre_read.js +77 -0
- package/dist/hooks/pre_read.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/server.d.ts +20 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +128 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/tools.d.ts +72 -0
- package/dist/mcp/tools.d.ts.map +1 -0
- package/dist/mcp/tools.js +153 -0
- package/dist/mcp/tools.js.map +1 -0
- package/package.json +79 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { ulid } from 'ulid';
|
|
2
|
+
import { ObservationInputSchema, ObservationSchema, } from './schema.js';
|
|
3
|
+
import { redactAll } from './privacy.js';
|
|
4
|
+
function rowToObservation(row) {
|
|
5
|
+
return ObservationSchema.parse({
|
|
6
|
+
id: row.id,
|
|
7
|
+
project_id: row.project_id,
|
|
8
|
+
session_id: row.session_id,
|
|
9
|
+
kind: row.kind,
|
|
10
|
+
title: row.title,
|
|
11
|
+
body: row.body,
|
|
12
|
+
files: JSON.parse(row.files),
|
|
13
|
+
tags: JSON.parse(row.tags),
|
|
14
|
+
created_at: row.created_at,
|
|
15
|
+
tokens_saved: row.tokens_saved,
|
|
16
|
+
tokens_spent: row.tokens_spent,
|
|
17
|
+
superseded_by: row.superseded_by,
|
|
18
|
+
embedding: row.embedding,
|
|
19
|
+
deleted_at: row.deleted_at,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
export class MemoryStore {
|
|
23
|
+
db;
|
|
24
|
+
constructor(db) {
|
|
25
|
+
this.db = db;
|
|
26
|
+
}
|
|
27
|
+
insert(input, options = {}) {
|
|
28
|
+
const parsed = ObservationInputSchema.parse(input);
|
|
29
|
+
const patterns = options.redactPatterns ?? [];
|
|
30
|
+
const title = redactAll(parsed.title, patterns);
|
|
31
|
+
const body = redactAll(parsed.body, patterns);
|
|
32
|
+
const id = parsed.id ?? ulid();
|
|
33
|
+
const created_at = parsed.created_at ?? Date.now();
|
|
34
|
+
// Use an immediate transaction so writer contention fails fast
|
|
35
|
+
// instead of hanging on the serialization boundary.
|
|
36
|
+
const txn = this.db.transaction(() => {
|
|
37
|
+
this.db
|
|
38
|
+
.prepare(`INSERT INTO observations
|
|
39
|
+
(id, project_id, session_id, kind, title, body, files, tags,
|
|
40
|
+
created_at, tokens_saved, tokens_spent, superseded_by, embedding, deleted_at)
|
|
41
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL)`)
|
|
42
|
+
.run(id, parsed.project_id, parsed.session_id, parsed.kind, title, body, JSON.stringify(parsed.files), JSON.stringify(parsed.tags), created_at, parsed.tokens_saved, parsed.tokens_spent);
|
|
43
|
+
});
|
|
44
|
+
txn.immediate();
|
|
45
|
+
return this.get(id);
|
|
46
|
+
}
|
|
47
|
+
get(id) {
|
|
48
|
+
const row = this.db.prepare(`SELECT * FROM observations WHERE id = ?`).get(id);
|
|
49
|
+
return row ? rowToObservation(row) : null;
|
|
50
|
+
}
|
|
51
|
+
listByProject(projectId, options = {}) {
|
|
52
|
+
const where = options.includeDeleted
|
|
53
|
+
? `WHERE project_id = ?`
|
|
54
|
+
: `WHERE project_id = ? AND deleted_at IS NULL`;
|
|
55
|
+
const limit = options.limit ? `LIMIT ${Math.floor(options.limit)}` : '';
|
|
56
|
+
const rows = this.db
|
|
57
|
+
.prepare(`SELECT * FROM observations ${where} ORDER BY created_at DESC ${limit}`)
|
|
58
|
+
.all(projectId);
|
|
59
|
+
return rows.map(rowToObservation);
|
|
60
|
+
}
|
|
61
|
+
softDelete(id) {
|
|
62
|
+
const info = this.db
|
|
63
|
+
.prepare(`UPDATE observations SET deleted_at = ? WHERE id = ? AND deleted_at IS NULL`)
|
|
64
|
+
.run(Date.now(), id);
|
|
65
|
+
return info.changes > 0;
|
|
66
|
+
}
|
|
67
|
+
countByProject(projectId) {
|
|
68
|
+
const row = this.db
|
|
69
|
+
.prepare(`SELECT COUNT(*) AS n FROM observations WHERE project_id = ? AND deleted_at IS NULL`)
|
|
70
|
+
.get(projectId);
|
|
71
|
+
return row.n;
|
|
72
|
+
}
|
|
73
|
+
setEmbedding(id, embedding) {
|
|
74
|
+
this.db.prepare(`UPDATE observations SET embedding = ? WHERE id = ?`).run(embedding, id);
|
|
75
|
+
}
|
|
76
|
+
listMissingEmbeddings(projectId, limit = 1000) {
|
|
77
|
+
const rows = this.db
|
|
78
|
+
.prepare(`SELECT * FROM observations
|
|
79
|
+
WHERE project_id = ? AND deleted_at IS NULL AND embedding IS NULL
|
|
80
|
+
ORDER BY created_at ASC LIMIT ?`)
|
|
81
|
+
.all(projectId, Math.floor(limit));
|
|
82
|
+
return rows.map(rowToObservation);
|
|
83
|
+
}
|
|
84
|
+
listWithEmbeddings(projectId) {
|
|
85
|
+
const rows = this.db
|
|
86
|
+
.prepare(`SELECT id, embedding FROM observations
|
|
87
|
+
WHERE project_id = ? AND deleted_at IS NULL AND embedding IS NOT NULL`)
|
|
88
|
+
.all(projectId);
|
|
89
|
+
return rows;
|
|
90
|
+
}
|
|
91
|
+
countByKind(projectId) {
|
|
92
|
+
const rows = this.db
|
|
93
|
+
.prepare(`SELECT kind, COUNT(*) AS n FROM observations
|
|
94
|
+
WHERE project_id = ? AND deleted_at IS NULL
|
|
95
|
+
GROUP BY kind`)
|
|
96
|
+
.all(projectId);
|
|
97
|
+
const out = {
|
|
98
|
+
decision: 0,
|
|
99
|
+
learning: 0,
|
|
100
|
+
bugfix: 0,
|
|
101
|
+
file_summary: 0,
|
|
102
|
+
command: 0,
|
|
103
|
+
other: 0,
|
|
104
|
+
};
|
|
105
|
+
for (const r of rows)
|
|
106
|
+
out[r.kind] = r.n;
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
totalTokensSaved(projectId) {
|
|
110
|
+
const row = this.db
|
|
111
|
+
.prepare(`SELECT COALESCE(SUM(tokens_saved), 0) AS total FROM observations
|
|
112
|
+
WHERE project_id = ? AND deleted_at IS NULL`)
|
|
113
|
+
.get(projectId);
|
|
114
|
+
return row.total;
|
|
115
|
+
}
|
|
116
|
+
totalTokensSpent(projectId) {
|
|
117
|
+
const row = this.db
|
|
118
|
+
.prepare(`SELECT COALESCE(SUM(tokens_spent), 0) AS total FROM observations
|
|
119
|
+
WHERE project_id = ? AND deleted_at IS NULL`)
|
|
120
|
+
.get(projectId);
|
|
121
|
+
return row.total;
|
|
122
|
+
}
|
|
123
|
+
// Hard-delete rows that were soft-deleted before `olderThanMs`.
|
|
124
|
+
// Returns the number of rows removed.
|
|
125
|
+
purge(projectId, olderThanMs) {
|
|
126
|
+
const info = this.db
|
|
127
|
+
.prepare(`DELETE FROM observations
|
|
128
|
+
WHERE project_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`)
|
|
129
|
+
.run(projectId, olderThanMs);
|
|
130
|
+
return info.changes;
|
|
131
|
+
}
|
|
132
|
+
// Observations newer than `sinceMs`, most-recent first.
|
|
133
|
+
listRecent(projectId, sinceMs, limit = 100) {
|
|
134
|
+
const rows = this.db
|
|
135
|
+
.prepare(`SELECT * FROM observations
|
|
136
|
+
WHERE project_id = ? AND deleted_at IS NULL AND created_at >= ?
|
|
137
|
+
ORDER BY created_at DESC LIMIT ?`)
|
|
138
|
+
.all(projectId, sinceMs, Math.floor(limit));
|
|
139
|
+
return rows.map(rowToObservation);
|
|
140
|
+
}
|
|
141
|
+
// Latest N observations of a given kind.
|
|
142
|
+
listByKind(projectId, kind, limit = 50) {
|
|
143
|
+
const rows = this.db
|
|
144
|
+
.prepare(`SELECT * FROM observations
|
|
145
|
+
WHERE project_id = ? AND kind = ? AND deleted_at IS NULL
|
|
146
|
+
ORDER BY created_at DESC LIMIT ?`)
|
|
147
|
+
.all(projectId, kind, Math.floor(limit));
|
|
148
|
+
return rows.map(rowToObservation);
|
|
149
|
+
}
|
|
150
|
+
// Top N most-referenced files with observation ID lists.
|
|
151
|
+
// Uses SQLite json_each to unnest the files JSON array.
|
|
152
|
+
topFileReferences(projectId, limit = 20) {
|
|
153
|
+
// Two-phase: first count per file, then collect IDs for the top files.
|
|
154
|
+
const countRows = this.db
|
|
155
|
+
.prepare(`SELECT f.value AS file, COUNT(*) AS cnt
|
|
156
|
+
FROM observations o, json_each(o.files) AS f
|
|
157
|
+
WHERE o.project_id = ? AND o.deleted_at IS NULL
|
|
158
|
+
GROUP BY f.value
|
|
159
|
+
ORDER BY cnt DESC
|
|
160
|
+
LIMIT ?`)
|
|
161
|
+
.all(projectId, Math.floor(limit));
|
|
162
|
+
return countRows.map(({ file, cnt }) => {
|
|
163
|
+
const idRows = this.db
|
|
164
|
+
.prepare(`SELECT o.id FROM observations o, json_each(o.files) AS f
|
|
165
|
+
WHERE o.project_id = ? AND o.deleted_at IS NULL AND f.value = ?
|
|
166
|
+
ORDER BY o.created_at DESC`)
|
|
167
|
+
.all(projectId, file);
|
|
168
|
+
return { file, count: cnt, ids: idRows.map((r) => r.id) };
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/core/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EACL,sBAAsB,EACtB,iBAAiB,GAIlB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAoBzC,SAAS,gBAAgB,CAAC,GAAmB;IAC3C,OAAO,iBAAiB,CAAC,KAAK,CAAC;QAC7B,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAa;QACxC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAa;QACtC,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,aAAa,EAAE,GAAG,CAAC,aAAa;QAChC,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,CAAC,UAAU;KAC3B,CAAC,CAAC;AACL,CAAC;AAMD,MAAM,OAAO,WAAW;IACO;IAA7B,YAA6B,EAAM;QAAN,OAAE,GAAF,EAAE,CAAI;IAAG,CAAC;IAEvC,MAAM,CAAC,KAAuB,EAAE,UAAyB,EAAE;QACzD,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAE9C,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnD,+DAA+D;QAC/D,oDAAoD;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YACnC,IAAI,CAAC,EAAE;iBACJ,OAAO,CACN;;;sEAG4D,CAC7D;iBACA,GAAG,CACF,EAAE,EACF,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,IAAI,EACX,KAAK,EACL,IAAI,EACJ,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAC5B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAC3B,UAAU,EACV,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,CACpB,CAAC;QACN,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,SAAS,EAAE,CAAC;QAEhB,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,EAAE,CAEhE,CAAC;QACd,OAAO,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5C,CAAC;IAED,aAAa,CACX,SAAiB,EACjB,UAAwD,EAAE;QAE1D,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc;YAClC,CAAC,CAAC,sBAAsB;YACxB,CAAC,CAAC,6CAA6C,CAAC;QAClD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,8BAA8B,KAAK,6BAA6B,KAAK,EAAE,CAAC;aAChF,GAAG,CAAC,SAAS,CAAqB,CAAC;QACtC,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,UAAU,CAAC,EAAU;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,4EAA4E,CAAC;aACrF,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QACvB,OAAO,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,cAAc,CAAC,SAAiB;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,oFAAoF,CAAC;aAC7F,GAAG,CAAC,SAAS,CAAkB,CAAC;QACnC,OAAO,GAAG,CAAC,CAAC,CAAC;IACf,CAAC;IAED,YAAY,CAAC,EAAU,EAAE,SAAiB;QACxC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED,qBAAqB,CAAC,SAAiB,EAAE,QAAgB,IAAI;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;yCAEiC,CAClC;aACA,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAqB,CAAC;QACzD,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,kBAAkB,CAAC,SAAiB;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;+EACuE,CACxE;aACA,GAAG,CAAC,SAAS,CAAwC,CAAC;QACzD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,SAAiB;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;uBAEe,CAChB;aACA,GAAG,CAAC,SAAS,CAA2C,CAAC;QAC5D,MAAM,GAAG,GAAG;YACV,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,CAAC;YACT,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,CAAC;SAC0B,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,gBAAgB,CAAC,SAAiB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN;qDAC6C,CAC9C;aACA,GAAG,CAAC,SAAS,CAAsB,CAAC;QACvC,OAAO,GAAG,CAAC,KAAK,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,SAAiB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN;qDAC6C,CAC9C;aACA,GAAG,CAAC,SAAS,CAAsB,CAAC;QACvC,OAAO,GAAG,CAAC,KAAK,CAAC;IACnB,CAAC;IAED,gEAAgE;IAChE,sCAAsC;IACtC,KAAK,CAAC,SAAiB,EAAE,WAAmB;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;4EACoE,CACrE;aACA,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,wDAAwD;IACxD,UAAU,CAAC,SAAiB,EAAE,OAAe,EAAE,KAAK,GAAG,GAAG;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;0CAEkC,CACnC;aACA,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAqB,CAAC;QAClE,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,yCAAyC;IACzC,UAAU,CAAC,SAAiB,EAAE,IAAqB,EAAE,KAAK,GAAG,EAAE;QAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;0CAEkC,CACnC;aACA,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAqB,CAAC;QAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,yDAAyD;IACzD,wDAAwD;IACxD,iBAAiB,CACf,SAAiB,EACjB,KAAK,GAAG,EAAE;QAEV,uEAAuE;QACvE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE;aACtB,OAAO,CACN;;;;;iBAKS,CACV;aACA,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAoC,CAAC;QAExE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE;YACrC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;iBACnB,OAAO,CACN;;sCAE4B,CAC7B;iBACA,GAAG,CAAC,SAAS,EAAE,IAAI,CAAqB,CAAC;YAC5C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AAIA,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGhD;AAED,wBAAgB,SAAS,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAEpD"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { encode } from 'gpt-tokenizer';
|
|
2
|
+
// gpt-tokenizer is a BPE approximation, not the exact tokenizer Claude uses.
|
|
3
|
+
// When estimates drift, prefer undercounting over overclaiming.
|
|
4
|
+
export function countTokens(text) {
|
|
5
|
+
if (!text)
|
|
6
|
+
return 0;
|
|
7
|
+
return encode(text).length;
|
|
8
|
+
}
|
|
9
|
+
export function sumTokens(...parts) {
|
|
10
|
+
return parts.reduce((n, p) => n + countTokens(p), 0);
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=tokens.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,6EAA6E;AAC7E,gEAAgE;AAChE,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAG,KAAe;IAC1C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface Turn {
|
|
2
|
+
role: 'user' | 'assistant' | 'system' | 'tool';
|
|
3
|
+
text: string;
|
|
4
|
+
timestamp?: string;
|
|
5
|
+
files_touched?: string[];
|
|
6
|
+
}
|
|
7
|
+
export declare function parseTranscript(raw: string): Turn[];
|
|
8
|
+
export declare function parseTranscriptFile(path: string): Turn[];
|
|
9
|
+
export declare function renderTurns(turns: Turn[]): string;
|
|
10
|
+
export interface PromptResponsePair {
|
|
11
|
+
prompt: string;
|
|
12
|
+
response: string;
|
|
13
|
+
files_touched: string[];
|
|
14
|
+
}
|
|
15
|
+
export declare function extractPromptResponsePairs(turns: Turn[]): PromptResponsePair[];
|
|
16
|
+
//# sourceMappingURL=transcript.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcript.d.ts","sourceRoot":"","sources":["../../src/core/transcript.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAkED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,EAAE,CA8BnD;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,CAExD;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,CAEjD;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAMD,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,kBAAkB,EAAE,CA2B9E"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
const FILE_TOOL_NAMES = new Set(['Read', 'Edit', 'Write', 'NotebookEdit']);
|
|
3
|
+
function extractFilesFromBlock(block) {
|
|
4
|
+
if (block == null || typeof block !== 'object')
|
|
5
|
+
return [];
|
|
6
|
+
const b = block;
|
|
7
|
+
if (b.type !== 'tool_use' || b.name === undefined)
|
|
8
|
+
return [];
|
|
9
|
+
if (!FILE_TOOL_NAMES.has(b.name))
|
|
10
|
+
return [];
|
|
11
|
+
const input = b.input;
|
|
12
|
+
const p = input?.file_path ?? input?.notebook_path;
|
|
13
|
+
return typeof p === 'string' && p.length > 0 ? [p] : [];
|
|
14
|
+
}
|
|
15
|
+
function coerceContent(content) {
|
|
16
|
+
if (typeof content === 'string')
|
|
17
|
+
return { text: content, files: [] };
|
|
18
|
+
if (Array.isArray(content)) {
|
|
19
|
+
const files = [];
|
|
20
|
+
const text = content
|
|
21
|
+
.map((block) => {
|
|
22
|
+
if (block == null)
|
|
23
|
+
return '';
|
|
24
|
+
if (typeof block === 'string')
|
|
25
|
+
return block;
|
|
26
|
+
if (typeof block === 'object' &&
|
|
27
|
+
'text' in block &&
|
|
28
|
+
typeof block.text === 'string') {
|
|
29
|
+
return block.text;
|
|
30
|
+
}
|
|
31
|
+
if (typeof block === 'object' &&
|
|
32
|
+
'type' in block &&
|
|
33
|
+
block.type === 'tool_use') {
|
|
34
|
+
const b = block;
|
|
35
|
+
files.push(...extractFilesFromBlock(block));
|
|
36
|
+
return `[tool_use ${b.name ?? ''} ${JSON.stringify(b.input ?? {})}]`;
|
|
37
|
+
}
|
|
38
|
+
if (typeof block === 'object' &&
|
|
39
|
+
'type' in block &&
|
|
40
|
+
block.type === 'tool_result') {
|
|
41
|
+
const b = block;
|
|
42
|
+
return `[tool_result ${coerceContent(b.content).text}]`;
|
|
43
|
+
}
|
|
44
|
+
return '';
|
|
45
|
+
})
|
|
46
|
+
.filter((s) => s.length > 0)
|
|
47
|
+
.join('\n');
|
|
48
|
+
return { text, files };
|
|
49
|
+
}
|
|
50
|
+
return { text: '', files: [] };
|
|
51
|
+
}
|
|
52
|
+
export function parseTranscript(raw) {
|
|
53
|
+
const trimmed = raw.trim();
|
|
54
|
+
if (trimmed.length === 0)
|
|
55
|
+
return [];
|
|
56
|
+
const looksLikeJsonl = trimmed.includes('\n{') || (trimmed.startsWith('{') && trimmed.endsWith('}'));
|
|
57
|
+
if (!looksLikeJsonl) {
|
|
58
|
+
return [{ role: 'user', text: trimmed }];
|
|
59
|
+
}
|
|
60
|
+
const lines = trimmed.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
|
61
|
+
const turns = [];
|
|
62
|
+
for (const line of lines) {
|
|
63
|
+
let obj;
|
|
64
|
+
try {
|
|
65
|
+
obj = JSON.parse(line);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const role = (obj.message?.role ?? obj.role ?? obj.type ?? 'user');
|
|
71
|
+
if (role !== 'user' && role !== 'assistant' && role !== 'system' && role !== 'tool')
|
|
72
|
+
continue;
|
|
73
|
+
const content = obj.message?.content ?? obj.content;
|
|
74
|
+
const { text, files } = coerceContent(content);
|
|
75
|
+
if (text.length === 0)
|
|
76
|
+
continue;
|
|
77
|
+
const turn = { role, text };
|
|
78
|
+
if (obj.timestamp)
|
|
79
|
+
turn.timestamp = obj.timestamp;
|
|
80
|
+
if (files.length > 0)
|
|
81
|
+
turn.files_touched = files;
|
|
82
|
+
turns.push(turn);
|
|
83
|
+
}
|
|
84
|
+
return turns;
|
|
85
|
+
}
|
|
86
|
+
export function parseTranscriptFile(path) {
|
|
87
|
+
return parseTranscript(readFileSync(path, 'utf8'));
|
|
88
|
+
}
|
|
89
|
+
export function renderTurns(turns) {
|
|
90
|
+
return turns.map((t) => `${t.role}: ${t.text}`).join('\n\n');
|
|
91
|
+
}
|
|
92
|
+
// Extract user→assistant turn pairs for cache population. Only pairs where
|
|
93
|
+
// the user turn is immediately followed by one or more assistant turns are
|
|
94
|
+
// included. Assistant turns get concatenated so tool-heavy replies still
|
|
95
|
+
// attribute to the single user prompt that kicked them off.
|
|
96
|
+
export function extractPromptResponsePairs(turns) {
|
|
97
|
+
const pairs = [];
|
|
98
|
+
for (let i = 0; i < turns.length; i += 1) {
|
|
99
|
+
const t = turns[i];
|
|
100
|
+
if (!t || t.role !== 'user')
|
|
101
|
+
continue;
|
|
102
|
+
const responses = [];
|
|
103
|
+
const files = new Set();
|
|
104
|
+
let j = i + 1;
|
|
105
|
+
while (j < turns.length) {
|
|
106
|
+
const next = turns[j];
|
|
107
|
+
if (!next)
|
|
108
|
+
break;
|
|
109
|
+
if (next.role === 'user')
|
|
110
|
+
break;
|
|
111
|
+
if (next.role === 'assistant') {
|
|
112
|
+
responses.push(next.text);
|
|
113
|
+
for (const f of next.files_touched ?? [])
|
|
114
|
+
files.add(f);
|
|
115
|
+
}
|
|
116
|
+
j += 1;
|
|
117
|
+
}
|
|
118
|
+
if (responses.length > 0) {
|
|
119
|
+
pairs.push({
|
|
120
|
+
prompt: t.text,
|
|
121
|
+
response: responses.join('\n\n'),
|
|
122
|
+
files_touched: [...files],
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return pairs;
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=transcript.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcript.js","sourceRoot":"","sources":["../../src/core/transcript.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAsBvC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;AAE3E,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC1D,MAAM,CAAC,GAAG,KAA0D,CAAC;IACrE,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAC7D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,CAAC,CAAC,KAAqE,CAAC;IACtF,MAAM,CAAC,GAAG,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,aAAa,CAAC;IACnD,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB;IACrC,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACrE,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,OAAO;aACjB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO,EAAE,CAAC;YAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC5C,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,MAAM,IAAI,KAAK;gBACf,OAAQ,KAA2B,CAAC,IAAI,KAAK,QAAQ,EACrD,CAAC;gBACD,OAAQ,KAA0B,CAAC,IAAI,CAAC;YAC1C,CAAC;YACD,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,MAAM,IAAI,KAAK;gBACd,KAA0B,CAAC,IAAI,KAAK,UAAU,EAC/C,CAAC;gBACD,MAAM,CAAC,GAAG,KAA2C,CAAC;gBACtD,KAAK,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC5C,OAAO,aAAa,CAAC,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC;YACvE,CAAC;YACD,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,MAAM,IAAI,KAAK;gBACd,KAA0B,CAAC,IAAI,KAAK,aAAa,EAClD,CAAC;gBACD,MAAM,CAAC,GAAG,KAA8B,CAAC;gBACzC,OAAO,gBAAgB,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC;YAC1D,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAC3B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,cAAc,GAClB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAChF,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,GAAmB,CAAC;QACxB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmB,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,MAAM,CAAiB,CAAC;QACnF,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,MAAM;YAAE,SAAS;QAC9F,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC;QACpD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,MAAM,IAAI,GAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;QAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,OAAO,eAAe,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/D,CAAC;AAQD,2EAA2E;AAC3E,2EAA2E;AAC3E,yEAAyE;AACzE,4DAA4D;AAC5D,MAAM,UAAU,0BAA0B,CAAC,KAAa;IACtD,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QACtC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC,IAAI;gBAAE,MAAM;YACjB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBAAE,MAAM;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC9B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,IAAI,EAAE;oBAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;YACD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,CAAC,CAAC,IAAI;gBACd,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;gBAChC,aAAa,EAAE,CAAC,GAAG,KAAK,CAAC;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
-- 001_init.sql
|
|
2
|
+
-- Initial schema: observations (+ FTS5), cache_entries, file_fingerprints.
|
|
3
|
+
-- SQLite is the source of truth; the markdown index is a derived mirror.
|
|
4
|
+
|
|
5
|
+
CREATE TABLE IF NOT EXISTS observations (
|
|
6
|
+
id TEXT PRIMARY KEY,
|
|
7
|
+
project_id TEXT NOT NULL,
|
|
8
|
+
session_id TEXT NOT NULL,
|
|
9
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','learning','bugfix','file_summary','command','other')),
|
|
10
|
+
title TEXT NOT NULL,
|
|
11
|
+
body TEXT NOT NULL,
|
|
12
|
+
files TEXT NOT NULL DEFAULT '[]', -- JSON array
|
|
13
|
+
tags TEXT NOT NULL DEFAULT '[]', -- JSON array
|
|
14
|
+
created_at INTEGER NOT NULL,
|
|
15
|
+
tokens_saved INTEGER NOT NULL DEFAULT 0,
|
|
16
|
+
tokens_spent INTEGER NOT NULL DEFAULT 0, -- extraction cost; used by stats
|
|
17
|
+
superseded_by TEXT,
|
|
18
|
+
embedding BLOB, -- 384-dim f32, populated once embeddings land
|
|
19
|
+
deleted_at INTEGER
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id);
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
|
|
24
|
+
CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at);
|
|
25
|
+
CREATE INDEX IF NOT EXISTS idx_observations_kind ON observations(kind);
|
|
26
|
+
CREATE INDEX IF NOT EXISTS idx_observations_deleted ON observations(deleted_at);
|
|
27
|
+
|
|
28
|
+
-- FTS5 index over title + body + tags. BM25 retriever reads this directly.
|
|
29
|
+
-- `content='observations'` makes the FTS table an external-content table; triggers keep it in sync.
|
|
30
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
|
|
31
|
+
title,
|
|
32
|
+
body,
|
|
33
|
+
tags,
|
|
34
|
+
content='observations',
|
|
35
|
+
content_rowid='rowid',
|
|
36
|
+
tokenize='porter unicode61'
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
CREATE TRIGGER IF NOT EXISTS observations_ai
|
|
40
|
+
AFTER INSERT ON observations
|
|
41
|
+
BEGIN
|
|
42
|
+
INSERT INTO observations_fts(rowid, title, body, tags)
|
|
43
|
+
VALUES (new.rowid, new.title, new.body, new.tags);
|
|
44
|
+
END;
|
|
45
|
+
|
|
46
|
+
CREATE TRIGGER IF NOT EXISTS observations_ad
|
|
47
|
+
AFTER DELETE ON observations
|
|
48
|
+
BEGIN
|
|
49
|
+
INSERT INTO observations_fts(observations_fts, rowid, title, body, tags)
|
|
50
|
+
VALUES ('delete', old.rowid, old.title, old.body, old.tags);
|
|
51
|
+
END;
|
|
52
|
+
|
|
53
|
+
CREATE TRIGGER IF NOT EXISTS observations_au
|
|
54
|
+
AFTER UPDATE ON observations
|
|
55
|
+
BEGIN
|
|
56
|
+
INSERT INTO observations_fts(observations_fts, rowid, title, body, tags)
|
|
57
|
+
VALUES ('delete', old.rowid, old.title, old.body, old.tags);
|
|
58
|
+
INSERT INTO observations_fts(rowid, title, body, tags)
|
|
59
|
+
VALUES (new.rowid, new.title, new.body, new.tags);
|
|
60
|
+
END;
|
|
61
|
+
|
|
62
|
+
CREATE TABLE IF NOT EXISTS cache_entries (
|
|
63
|
+
id TEXT PRIMARY KEY,
|
|
64
|
+
prompt_hash TEXT NOT NULL UNIQUE,
|
|
65
|
+
prompt_text TEXT NOT NULL,
|
|
66
|
+
prompt_embedding BLOB,
|
|
67
|
+
response TEXT NOT NULL,
|
|
68
|
+
model TEXT NOT NULL,
|
|
69
|
+
context_fingerprint TEXT NOT NULL,
|
|
70
|
+
fingerprint_version INTEGER NOT NULL DEFAULT 1,
|
|
71
|
+
created_at INTEGER NOT NULL,
|
|
72
|
+
last_hit_at INTEGER NOT NULL,
|
|
73
|
+
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
74
|
+
false_hit_count INTEGER NOT NULL DEFAULT 0,
|
|
75
|
+
invalidated INTEGER NOT NULL DEFAULT 0
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
CREATE INDEX IF NOT EXISTS idx_cache_prompt_hash ON cache_entries(prompt_hash);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS idx_cache_last_hit ON cache_entries(last_hit_at);
|
|
80
|
+
|
|
81
|
+
CREATE TABLE IF NOT EXISTS file_fingerprints (
|
|
82
|
+
project_id TEXT NOT NULL,
|
|
83
|
+
path TEXT NOT NULL,
|
|
84
|
+
content_hash TEXT NOT NULL,
|
|
85
|
+
mtime INTEGER NOT NULL,
|
|
86
|
+
tokens INTEGER NOT NULL,
|
|
87
|
+
summary TEXT,
|
|
88
|
+
summary_hash TEXT,
|
|
89
|
+
PRIMARY KEY (project_id, path)
|
|
90
|
+
);
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
-- 002_cache_files.sql
|
|
2
|
+
-- Track the files referenced by a cached prompt/response pair so we can
|
|
3
|
+
-- re-hash them on lookup and invalidate stale entries. Without this column
|
|
4
|
+
-- we have no way to know which files to fingerprint.
|
|
5
|
+
|
|
6
|
+
ALTER TABLE cache_entries ADD COLUMN files_touched TEXT NOT NULL DEFAULT '[]';
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
-- 003_retrieval_stats.sql
|
|
2
|
+
-- Per-project, per-strategy retrieval counters and total cache hit tracking.
|
|
3
|
+
|
|
4
|
+
CREATE TABLE IF NOT EXISTS retrieval_stats (
|
|
5
|
+
project_id TEXT NOT NULL,
|
|
6
|
+
strategy TEXT NOT NULL,
|
|
7
|
+
call_count INTEGER NOT NULL DEFAULT 0,
|
|
8
|
+
last_called_at INTEGER,
|
|
9
|
+
PRIMARY KEY (project_id, strategy)
|
|
10
|
+
);
|
|
11
|
+
|
|
12
|
+
CREATE TABLE IF NOT EXISTS cache_hit_stats (
|
|
13
|
+
project_id TEXT NOT NULL PRIMARY KEY,
|
|
14
|
+
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
15
|
+
miss_count INTEGER NOT NULL DEFAULT 0,
|
|
16
|
+
last_hit_at INTEGER
|
|
17
|
+
);
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { type DB } from '../core/db.js';
|
|
3
|
+
import { type LlmCaller } from '../core/extractor.js';
|
|
4
|
+
import type { Config } from '../core/schema.js';
|
|
5
|
+
export declare const HookPayloadSchema: z.ZodEffects<z.ZodObject<{
|
|
6
|
+
session_id: z.ZodOptional<z.ZodString>;
|
|
7
|
+
cwd: z.ZodOptional<z.ZodString>;
|
|
8
|
+
project_id: z.ZodOptional<z.ZodString>;
|
|
9
|
+
transcript: z.ZodOptional<z.ZodString>;
|
|
10
|
+
transcript_path: z.ZodOptional<z.ZodString>;
|
|
11
|
+
hook_event_name: z.ZodOptional<z.ZodString>;
|
|
12
|
+
}, "strip", z.ZodTypeAny, {
|
|
13
|
+
project_id?: string | undefined;
|
|
14
|
+
session_id?: string | undefined;
|
|
15
|
+
cwd?: string | undefined;
|
|
16
|
+
transcript?: string | undefined;
|
|
17
|
+
transcript_path?: string | undefined;
|
|
18
|
+
hook_event_name?: string | undefined;
|
|
19
|
+
}, {
|
|
20
|
+
project_id?: string | undefined;
|
|
21
|
+
session_id?: string | undefined;
|
|
22
|
+
cwd?: string | undefined;
|
|
23
|
+
transcript?: string | undefined;
|
|
24
|
+
transcript_path?: string | undefined;
|
|
25
|
+
hook_event_name?: string | undefined;
|
|
26
|
+
}>, {
|
|
27
|
+
project_id?: string | undefined;
|
|
28
|
+
session_id?: string | undefined;
|
|
29
|
+
cwd?: string | undefined;
|
|
30
|
+
transcript?: string | undefined;
|
|
31
|
+
transcript_path?: string | undefined;
|
|
32
|
+
hook_event_name?: string | undefined;
|
|
33
|
+
}, {
|
|
34
|
+
project_id?: string | undefined;
|
|
35
|
+
session_id?: string | undefined;
|
|
36
|
+
cwd?: string | undefined;
|
|
37
|
+
transcript?: string | undefined;
|
|
38
|
+
transcript_path?: string | undefined;
|
|
39
|
+
hook_event_name?: string | undefined;
|
|
40
|
+
}>;
|
|
41
|
+
export type HookPayload = z.infer<typeof HookPayloadSchema>;
|
|
42
|
+
export declare function readToEnd(stream: NodeJS.ReadableStream): Promise<string>;
|
|
43
|
+
export interface RunOptions {
|
|
44
|
+
cwd?: string;
|
|
45
|
+
config?: Config;
|
|
46
|
+
caller?: LlmCaller;
|
|
47
|
+
db?: DB;
|
|
48
|
+
dbPath?: string;
|
|
49
|
+
indexPath?: string;
|
|
50
|
+
projectName?: string;
|
|
51
|
+
now?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface RunResult {
|
|
54
|
+
projectId: string;
|
|
55
|
+
inserted: number;
|
|
56
|
+
tokensSpent: number;
|
|
57
|
+
tokensSavedTotal: number;
|
|
58
|
+
indexPath: string;
|
|
59
|
+
cacheEntriesAdded: number;
|
|
60
|
+
embeddingsAdded: number;
|
|
61
|
+
summariesGenerated: number;
|
|
62
|
+
}
|
|
63
|
+
export declare function runPostSession(payload: HookPayload, opts?: RunOptions): Promise<RunResult>;
|
|
64
|
+
export declare function main(): Promise<void>;
|
|
65
|
+
//# sourceMappingURL=post_session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"post_session.d.ts","sourceRoot":"","sources":["../../src/hooks/post_session.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAU,KAAK,EAAE,EAAE,MAAM,eAAe,CAAC;AAGhD,OAAO,EAIL,KAAK,SAAS,EACf,MAAM,sBAAsB,CAAC;AAiB9B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAY1B,CAAC;AAEL,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAQ9E;AAwGD,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,EAAE,CAAC,EAAE,EAAE,CAAC;IACR,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAsB,cAAc,CAClC,OAAO,EAAE,WAAW,EACpB,IAAI,GAAE,UAAe,GACpB,OAAO,CAAC,SAAS,CAAC,CA4GpB;AAED,wBAAsB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAqB1C"}
|