foldkeep 0.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foldkeep/__init__.py +16 -0
- foldkeep/__main__.py +3 -0
- foldkeep/cli.py +246 -0
- foldkeep/core/__init__.py +0 -0
- foldkeep/core/compressor.py +346 -0
- foldkeep/diff.py +105 -0
- foldkeep/estimate.py +28 -0
- foldkeep/mcp_server.py +189 -0
- foldkeep/memory/__init__.py +0 -0
- foldkeep/memory/store.py +92 -0
- foldkeep/packing.py +63 -0
- foldkeep/selftest.py +154 -0
- foldkeep/session.py +423 -0
- foldkeep-0.9.2.dist-info/METADATA +117 -0
- foldkeep-0.9.2.dist-info/RECORD +18 -0
- foldkeep-0.9.2.dist-info/WHEEL +5 -0
- foldkeep-0.9.2.dist-info/licenses/LICENSE +21 -0
- foldkeep-0.9.2.dist-info/top_level.txt +1 -0
foldkeep/session.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
"""Agent session hooks -- rolling compression for long conversations (v0.8).
|
|
2
|
+
|
|
3
|
+
A session is an ordered list of turns (user/assistant). As the session grows,
|
|
4
|
+
oldest raw turns are automatically rolled up into compressed form ("head"),
|
|
5
|
+
while recent turns stay verbatim ("tail"). render() assembles the context
|
|
6
|
+
within an optional token budget, RECENCY-FIRST: newest turns stay verbatim,
|
|
7
|
+
older ones are dropped -- but their facts survive via two pinned layers:
|
|
8
|
+
- entity registry (SCREAMING_CASE tokens, one line)
|
|
9
|
+
- fact lines (salient sentence per turn; shown only for turns that
|
|
10
|
+
fell out of the body -- render stays read-only)
|
|
11
|
+
"""
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import sqlite3
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
from .core.compressor import ENT_RE, FastLingua
|
|
19
|
+
from .estimate import estimate_tokens
|
|
20
|
+
|
|
21
|
+
_DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "memories.db")
|
|
22
|
+
NL = chr(10)
|
|
23
|
+
NUM_RE = re.compile(r"\d+(?:\.\d+)?")
|
|
24
|
+
|
|
25
|
+
def _sig(text):
|
|
26
|
+
"""Diversity signature of a fact line: entities PLUS numbers --
|
|
27
|
+
'PASS 12' vs 'PASS 20' differ only in digits, and digits are facts."""
|
|
28
|
+
return frozenset(ENT_RE.findall(text)) | frozenset(NUM_RE.findall(text))
|
|
29
|
+
|
|
30
|
+
_TOK_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_.]*|\d+(?:\.\d+)?")
|
|
31
|
+
|
|
32
|
+
def _lean(text, win=28, cap=200):
|
|
33
|
+
"""Context windows around every entity/number occurrence -- the
|
|
34
|
+
evidence an agent needs, minus connective filler. ~3x cheaper than
|
|
35
|
+
the verbatim sentence while keeping exact substrings intact."""
|
|
36
|
+
spans = []
|
|
37
|
+
for m in _TOK_RE.finditer(text):
|
|
38
|
+
if ENT_RE.fullmatch(m.group()) or NUM_RE.fullmatch(m.group()):
|
|
39
|
+
spans.append((max(0, m.start() - win),
|
|
40
|
+
min(len(text), m.end() + win)))
|
|
41
|
+
if not spans:
|
|
42
|
+
return text[:cap]
|
|
43
|
+
spans.sort()
|
|
44
|
+
merged = [list(spans[0])]
|
|
45
|
+
for s, e in spans[1:]:
|
|
46
|
+
if s <= merged[-1][1] + 8:
|
|
47
|
+
merged[-1][1] = max(merged[-1][1], e)
|
|
48
|
+
else:
|
|
49
|
+
merged.append([s, e])
|
|
50
|
+
parts = []
|
|
51
|
+
pos = 0
|
|
52
|
+
for s, e in merged:
|
|
53
|
+
if s > pos:
|
|
54
|
+
parts.append("...")
|
|
55
|
+
parts.append(text[s:e].strip())
|
|
56
|
+
pos = e
|
|
57
|
+
if sum(len(p) for p in parts) > cap:
|
|
58
|
+
break
|
|
59
|
+
out = " ... ".join(parts)
|
|
60
|
+
return out[:cap]
|
|
61
|
+
_FL_PIN = None
|
|
62
|
+
|
|
63
|
+
def _pin_fl():
|
|
64
|
+
"""Lazy FastLingua for pin-time fact compression."""
|
|
65
|
+
global _FL_PIN
|
|
66
|
+
if _FL_PIN is None:
|
|
67
|
+
_FL_PIN = FastLingua("balanced", 0.25)
|
|
68
|
+
return _FL_PIN
|
|
69
|
+
|
|
70
|
+
def _conn():
|
|
71
|
+
c = sqlite3.connect(_DB)
|
|
72
|
+
c.execute("""CREATE TABLE IF NOT EXISTS sessions (
|
|
73
|
+
name TEXT PRIMARY KEY, project TEXT NOT NULL DEFAULT 'default',
|
|
74
|
+
created REAL)""")
|
|
75
|
+
c.execute("""CREATE TABLE IF NOT EXISTS turns (
|
|
76
|
+
sid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
77
|
+
name TEXT NOT NULL, seq INTEGER NOT NULL,
|
|
78
|
+
role TEXT NOT NULL, kind TEXT NOT NULL DEFAULT 'raw',
|
|
79
|
+
text TEXT NOT NULL, orig_chars INTEGER,
|
|
80
|
+
created REAL)""")
|
|
81
|
+
c.execute("""CREATE TABLE IF NOT EXISTS entities (
|
|
82
|
+
name TEXT NOT NULL, ent TEXT NOT NULL)""")
|
|
83
|
+
c.execute("""CREATE TABLE IF NOT EXISTS facts (
|
|
84
|
+
name TEXT NOT NULL, seq INTEGER NOT NULL,
|
|
85
|
+
text TEXT NOT NULL)""")
|
|
86
|
+
# v0.9 lossless base (learned from pi-fold): originals survive
|
|
87
|
+
# rollup/consolidate and stay expandable byte-for-byte
|
|
88
|
+
cols = [r[1] for r in c.execute("PRAGMA table_info(turns)").fetchall()]
|
|
89
|
+
if "orig_text" not in cols:
|
|
90
|
+
c.execute("ALTER TABLE turns ADD COLUMN orig_text TEXT")
|
|
91
|
+
c.execute("""CREATE TABLE IF NOT EXISTS originals (
|
|
92
|
+
name TEXT NOT NULL, seq INTEGER NOT NULL,
|
|
93
|
+
text TEXT NOT NULL, PRIMARY KEY (name, seq))""")
|
|
94
|
+
return c
|
|
95
|
+
|
|
96
|
+
_FENCE_RE = re.compile(r"```|~~~")
|
|
97
|
+
|
|
98
|
+
def _intolerant(text):
|
|
99
|
+
"""Loss-tolerance routing (learned from leanctx/LLMLingua): code
|
|
100
|
+
fences, JSON payloads and tool-call shapes must NOT be compressed --
|
|
101
|
+
distortion there destroys semantics, not filler. Such turns are kept
|
|
102
|
+
verbatim or dropped-to-facts, never hard-compressed."""
|
|
103
|
+
if _FENCE_RE.search(text):
|
|
104
|
+
return True
|
|
105
|
+
if text.lstrip()[:1] in "{[":
|
|
106
|
+
try:
|
|
107
|
+
json.loads(text)
|
|
108
|
+
return True
|
|
109
|
+
except Exception:
|
|
110
|
+
pass
|
|
111
|
+
if re.search(r'"(?:tool_use_id|tool_call|function_call)"\s*:', text):
|
|
112
|
+
return True
|
|
113
|
+
return False
|
|
114
|
+
|
|
115
|
+
def new(name, project="default"):
|
|
116
|
+
"""Create (or reset) a named session: clears its previous turns."""
|
|
117
|
+
with _conn() as c:
|
|
118
|
+
c.execute("INSERT OR REPLACE INTO sessions VALUES (?,?,?)",
|
|
119
|
+
(name, project, time.time()))
|
|
120
|
+
c.execute("DELETE FROM turns WHERE name=?", (name,))
|
|
121
|
+
c.execute("DELETE FROM entities WHERE name=?", (name,))
|
|
122
|
+
c.execute("DELETE FROM facts WHERE name=?", (name,))
|
|
123
|
+
return name
|
|
124
|
+
|
|
125
|
+
def _rows(name):
|
|
126
|
+
with _conn() as c:
|
|
127
|
+
return c.execute("SELECT sid,seq,role,kind,text,orig_chars FROM turns "
|
|
128
|
+
"WHERE name=? ORDER BY seq", (name,)).fetchall()
|
|
129
|
+
|
|
130
|
+
def push(name, role, text, roll_threshold=2600):
|
|
131
|
+
with _conn() as c:
|
|
132
|
+
mx = c.execute("SELECT COALESCE(MAX(seq),0) FROM turns WHERE name=?",
|
|
133
|
+
(name,)).fetchone()[0]
|
|
134
|
+
c.execute("INSERT INTO turns (name,seq,role,kind,text,orig_chars,"
|
|
135
|
+
"created,orig_text) VALUES (?,?,?,?,?,?,?,?)",
|
|
136
|
+
(name, mx + 1, role, "raw", text, len(text), time.time(),
|
|
137
|
+
text))
|
|
138
|
+
# pin a LEAN fact line: context windows around entities/numbers
|
|
139
|
+
# of the salient sentence -- exact substrings survive, filler
|
|
140
|
+
# does not; render shows it only for out-of-body turns
|
|
141
|
+
sal = _salient(text)
|
|
142
|
+
lean = _lean(sal)
|
|
143
|
+
c.execute("INSERT OR REPLACE INTO facts VALUES (?,?,?)",
|
|
144
|
+
(name, mx + 1, lean or sal[:96]))
|
|
145
|
+
rolled = _rollup(name, roll_threshold)
|
|
146
|
+
return {"seq": mx + 1, "rolled": rolled or 0}
|
|
147
|
+
|
|
148
|
+
def _salient(text, cap=160):
|
|
149
|
+
"""The turn's POINT: in agent transcripts the decision sentence
|
|
150
|
+
precedes tool-output noise, so take the FIRST entity-bearing
|
|
151
|
+
sentence (falling back to the first sentence), not the longest --
|
|
152
|
+
longest routinely selects a noise line full of URLs/hashes."""
|
|
153
|
+
sents = [s.strip() for s in re.split(r"(?<=[.!?\u3002])\s+|\n+", text)
|
|
154
|
+
if s.strip()]
|
|
155
|
+
if not sents:
|
|
156
|
+
return text[:cap]
|
|
157
|
+
for s in sents:
|
|
158
|
+
if ENT_RE.search(s):
|
|
159
|
+
return s[:cap]
|
|
160
|
+
return sents[0][:cap]
|
|
161
|
+
|
|
162
|
+
def _rollup(name, threshold):
|
|
163
|
+
fl = FastLingua("balanced", 0.45)
|
|
164
|
+
rows = [r for r in _rows(name) if r[3] == "raw"]
|
|
165
|
+
toks = [estimate_tokens(r[4]) for r in rows]
|
|
166
|
+
total = sum(toks)
|
|
167
|
+
if total <= threshold:
|
|
168
|
+
return 0
|
|
169
|
+
target = threshold * 0.55
|
|
170
|
+
run, cut = 0, len(rows)
|
|
171
|
+
for i in range(len(rows) - 1, -1, -1):
|
|
172
|
+
run += toks[i]
|
|
173
|
+
if run <= target:
|
|
174
|
+
cut = i
|
|
175
|
+
else:
|
|
176
|
+
break
|
|
177
|
+
n = 0
|
|
178
|
+
ents = set()
|
|
179
|
+
with _conn() as c:
|
|
180
|
+
for r in rows[:cut]:
|
|
181
|
+
if _intolerant(r[4]):
|
|
182
|
+
continue # zero-tolerance content stays raw
|
|
183
|
+
comp, _st = fl.compress(r[4])
|
|
184
|
+
c.execute("UPDATE turns SET kind='comp', text=?, orig_chars=?, "
|
|
185
|
+
"orig_text=COALESCE(orig_text,?) WHERE sid=?",
|
|
186
|
+
(comp, r[5], r[4], r[0]))
|
|
187
|
+
ents.update(ENT_RE.findall(r[4]))
|
|
188
|
+
n += 1
|
|
189
|
+
# entity registry: rolled-away turns keep entity facts reachable
|
|
190
|
+
for e in ents:
|
|
191
|
+
c.execute("INSERT OR IGNORE INTO entities VALUES (?,?)", (name, e))
|
|
192
|
+
_consolidate(name)
|
|
193
|
+
return n
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _consolidate(name, max_head=120, keep=60):
|
|
197
|
+
"""Hierarchical head merge: too many compressed rows -> re-compress the
|
|
198
|
+
oldest block into ONE denser row so the head keeps shrinking."""
|
|
199
|
+
with _conn() as c:
|
|
200
|
+
comp = c.execute("SELECT sid,text FROM turns WHERE name=? AND "
|
|
201
|
+
"kind='comp' ORDER BY seq", (name,)).fetchall()
|
|
202
|
+
if len(comp) <= max_head:
|
|
203
|
+
return 0
|
|
204
|
+
old = comp[:-keep]
|
|
205
|
+
joined = NL.join(t for _sid, t in old)
|
|
206
|
+
fl = FastLingua("aggressive", 0.45)
|
|
207
|
+
merged, _st = fl.compress(joined)
|
|
208
|
+
sids = [sid for sid, _t in old]
|
|
209
|
+
# stash verbatim originals before the merge deletes the rows
|
|
210
|
+
keep_rows = c.execute(
|
|
211
|
+
"SELECT seq, COALESCE(orig_text, text) FROM turns "
|
|
212
|
+
"WHERE sid IN (%s)" % ",".join("?" * len(sids)), sids).fetchall()
|
|
213
|
+
c.executemany("INSERT OR REPLACE INTO originals VALUES (?,?,?)",
|
|
214
|
+
[(name, sq, tx) for sq, tx in keep_rows])
|
|
215
|
+
c.execute("DELETE FROM turns WHERE sid IN (%s)"
|
|
216
|
+
% ",".join("?" * len(sids)), sids)
|
|
217
|
+
c.execute("INSERT INTO turns (name,seq,role,kind,text,orig_chars,"
|
|
218
|
+
"created) VALUES (?,?,?,?,?,?,?)",
|
|
219
|
+
(name, int(time.time()), "system", "comp", merged,
|
|
220
|
+
len(joined), time.time()))
|
|
221
|
+
return len(sids)
|
|
222
|
+
|
|
223
|
+
def _env_int(key, dflt):
|
|
224
|
+
"""Layer-budget knob (learned from ECC context controls): operators
|
|
225
|
+
cap injected context via env without touching code."""
|
|
226
|
+
try:
|
|
227
|
+
return int(os.environ.get(key, dflt))
|
|
228
|
+
except (TypeError, ValueError):
|
|
229
|
+
return dflt
|
|
230
|
+
|
|
231
|
+
def _registry(name, cap=None):
|
|
232
|
+
"""Session-level entity registry, priority-ordered:
|
|
233
|
+
SCREAMING_CASE fact tokens first, then the rest. Deterministic.
|
|
234
|
+
Cap defaults to OC_REGISTRY_MAX env (ECC-style context control)."""
|
|
235
|
+
if cap is None:
|
|
236
|
+
cap = _env_int("OC_REGISTRY_MAX", 48)
|
|
237
|
+
with _conn() as c:
|
|
238
|
+
rows = c.execute("SELECT DISTINCT ent FROM entities WHERE name=?",
|
|
239
|
+
(name,)).fetchall()
|
|
240
|
+
ents = [r[0] for r in rows]
|
|
241
|
+
ents.sort(key=lambda e: (0 if re.match(r"^[A-Z][A-Z0-9_]{2,}$", e)
|
|
242
|
+
else 1, e))
|
|
243
|
+
return ents[:cap]
|
|
244
|
+
|
|
245
|
+
def _fact_block(name, max_tok=320, exclude=()):
|
|
246
|
+
"""Pinned fact lines for turns NOT present in the body (oldest kept
|
|
247
|
+
first when trimming). Keeps render read-only and deterministic."""
|
|
248
|
+
with _conn() as c:
|
|
249
|
+
frows = c.execute(
|
|
250
|
+
"SELECT seq,text FROM facts WHERE name=? ORDER BY seq DESC "
|
|
251
|
+
"LIMIT 64", (name,)).fetchall()
|
|
252
|
+
frows = [(sq, tx) for sq, tx in frows if sq not in exclude]
|
|
253
|
+
cands = list(reversed(frows[-24:]))
|
|
254
|
+
if not cands:
|
|
255
|
+
return ""
|
|
256
|
+
header = "=== pinned facts (out-of-body turns) ==="
|
|
257
|
+
|
|
258
|
+
def _blk(items):
|
|
259
|
+
ls = ["[t%d] %s" % (sq, tx) for sq, tx in sorted(items)]
|
|
260
|
+
return header + NL + NL.join(ls)
|
|
261
|
+
|
|
262
|
+
# greedy SET-COVER knapsack: repeatedly take the line adding the
|
|
263
|
+
# most NEW entities (shorter lines win ties), subject to the cap --
|
|
264
|
+
# spreads coverage across turns instead of clustering on a few
|
|
265
|
+
# entity-dense neighbours
|
|
266
|
+
sigs = [(sq, tx, _sig(tx)) for sq, tx in cands]
|
|
267
|
+
chosen, seen = [], set()
|
|
268
|
+
pool = list(sigs)
|
|
269
|
+
while pool:
|
|
270
|
+
best_i, best_key = 0, None
|
|
271
|
+
for i, (sq, tx, sg) in enumerate(pool):
|
|
272
|
+
key = (-len(sg - seen), len(tx))
|
|
273
|
+
if best_key is None or key < best_key:
|
|
274
|
+
best_i, best_key = i, key
|
|
275
|
+
sq, tx, sg = pool.pop(best_i)
|
|
276
|
+
if estimate_tokens(_blk(chosen + [(sq, tx)])) <= max_tok:
|
|
277
|
+
chosen.append((sq, tx))
|
|
278
|
+
seen |= sg
|
|
279
|
+
if not chosen:
|
|
280
|
+
return ""
|
|
281
|
+
return _blk(chosen)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def render(name, budget=None):
|
|
285
|
+
"""Recency-first assembly: walk NEWEST -> OLDEST so under budget
|
|
286
|
+
pressure the newest turns stay verbatim and the OLDEST are dropped
|
|
287
|
+
(their facts survive via registry + pinned-fact layers)."""
|
|
288
|
+
used, dropped = 0, 0
|
|
289
|
+
fl_hard = FastLingua("balanced", 0.25)
|
|
290
|
+
rows = _rows(name)
|
|
291
|
+
ents = _registry(name)
|
|
292
|
+
reg = ""
|
|
293
|
+
if ents:
|
|
294
|
+
reg = "=== key facts: %s ===" % " ".join(ents)
|
|
295
|
+
used += estimate_tokens(reg)
|
|
296
|
+
# reserve a slice of the budget for pinned facts up-front so the
|
|
297
|
+
# fact layer can NEVER trigger an eviction spiral against the body;
|
|
298
|
+
# floor of 170 tok lets the fact layer carry ALL facts at tiny
|
|
299
|
+
# budgets (body turns go hard-compressed, facts remain the guarantee)
|
|
300
|
+
reserve = int(min(320, max(170, budget * 0.35))) if budget else 0
|
|
301
|
+
eff = (budget - reserve) if budget else None
|
|
302
|
+
pairs, verbatim = [], set() # verbatim seqs: facts NOT needed for them
|
|
303
|
+
for sid, seq, role, kind, text, ochars in reversed(rows):
|
|
304
|
+
tok = estimate_tokens(text)
|
|
305
|
+
tag = "comp" if kind == "comp" else "raw"
|
|
306
|
+
if eff is None or used + tok <= eff:
|
|
307
|
+
pairs.append((seq, "[%02d][%s/%s] %s" % (seq, role, tag, text)))
|
|
308
|
+
verbatim.add(seq)
|
|
309
|
+
used += tok
|
|
310
|
+
continue
|
|
311
|
+
comp, _st = fl_hard.compress(text)
|
|
312
|
+
tok2 = estimate_tokens(comp)
|
|
313
|
+
if (not _intolerant(text) and comp.strip() and tok2 < tok
|
|
314
|
+
and used + tok2 <= eff):
|
|
315
|
+
# hard-compress is LOSSY (numbers/format can die) -- the turn
|
|
316
|
+
# stays in the body but its pinned fact line still shows
|
|
317
|
+
pairs.append((seq, "[%02d][%s/hard] %s" % (seq, role, comp)))
|
|
318
|
+
used += tok2
|
|
319
|
+
else:
|
|
320
|
+
dropped += 1
|
|
321
|
+
pairs.reverse() # now oldest-first
|
|
322
|
+
parts = [tx for _sq, tx in pairs]
|
|
323
|
+
fact_block = ""
|
|
324
|
+
if reserve:
|
|
325
|
+
room = budget - used
|
|
326
|
+
if room > 20:
|
|
327
|
+
fact_block = _fact_block(name, max_tok=min(320, room),
|
|
328
|
+
exclude=verbatim)
|
|
329
|
+
if fact_block:
|
|
330
|
+
used += estimate_tokens(fact_block)
|
|
331
|
+
# min-viable floor: never return a completely empty render when the
|
|
332
|
+
# session has turns and a positive budget -- fall back to the last
|
|
333
|
+
# turn, hard-compressed and hard-truncated to fit.
|
|
334
|
+
if not parts and rows and budget and int(budget) > 0:
|
|
335
|
+
sid, seq, role, kind, text, ochars = rows[-1]
|
|
336
|
+
comp, _st = fl_hard.compress(text)
|
|
337
|
+
maxchars = max(int(budget) * 4, 200)
|
|
338
|
+
if len(comp) > maxchars:
|
|
339
|
+
comp = comp[:maxchars] + " ...[truncated]"
|
|
340
|
+
parts.append("[%02d][%s/floor] %s" % (seq, role, comp))
|
|
341
|
+
used = estimate_tokens(comp)
|
|
342
|
+
body = NL.join(parts)
|
|
343
|
+
pre = [b for b in (reg, fact_block) if b]
|
|
344
|
+
if pre:
|
|
345
|
+
body = NL.join(pre) + NL + body
|
|
346
|
+
head = "=== session %s | ~%d tok%s ===" % (
|
|
347
|
+
name, used, "" if budget is None else " / budget %d" % budget)
|
|
348
|
+
if dropped:
|
|
349
|
+
head += " | dropped %d old turns" % dropped
|
|
350
|
+
return head + NL + body, used, dropped
|
|
351
|
+
|
|
352
|
+
def expand(name, seq):
|
|
353
|
+
"""Lossless recovery (learned from pi-fold): fetch the verbatim
|
|
354
|
+
original of any turn -- survives rollup, consolidate and render
|
|
355
|
+
drops. Returns None if the turn never existed."""
|
|
356
|
+
with _conn() as c:
|
|
357
|
+
row = c.execute("SELECT kind, text, orig_text FROM turns "
|
|
358
|
+
"WHERE name=? AND seq=?", (name, seq)).fetchone()
|
|
359
|
+
if row:
|
|
360
|
+
kind, text, orig = row
|
|
361
|
+
if orig:
|
|
362
|
+
return orig
|
|
363
|
+
if kind == "raw":
|
|
364
|
+
return text
|
|
365
|
+
row = c.execute("SELECT text FROM originals WHERE name=? AND seq=?",
|
|
366
|
+
(name, seq)).fetchone()
|
|
367
|
+
return row[0] if row else None
|
|
368
|
+
|
|
369
|
+
def search(name, query, limit=10):
|
|
370
|
+
"""Cross-session recall (learned from hermes-agent's FTS5 session
|
|
371
|
+
search): substring match over verbatim originals + pinned facts,
|
|
372
|
+
newest first. Works even for turns that were rolled away or merged,
|
|
373
|
+
because originals survive (v0.9 lossless base)."""
|
|
374
|
+
pat = "%" + query.replace("%", "").replace("_", " ") + "%"
|
|
375
|
+
like = "%" + query.replace("%", "") + "%"
|
|
376
|
+
with _conn() as c:
|
|
377
|
+
rows = c.execute(
|
|
378
|
+
"SELECT seq, role, COALESCE(orig_text, text) FROM turns "
|
|
379
|
+
"WHERE name=? AND (COALESCE(orig_text, text) LIKE ? "
|
|
380
|
+
"OR COALESCE(orig_text, text) LIKE ?) "
|
|
381
|
+
"ORDER BY seq DESC LIMIT ?",
|
|
382
|
+
(name, like, pat, limit)).fetchall()
|
|
383
|
+
facts = c.execute(
|
|
384
|
+
"SELECT seq, text FROM facts WHERE name=? AND text LIKE ? "
|
|
385
|
+
"ORDER BY seq DESC LIMIT ?", (name, like, limit)).fetchall()
|
|
386
|
+
return {"turns": [{"seq": s, "role": r, "text": t}
|
|
387
|
+
for s, r, t in rows],
|
|
388
|
+
"facts": [{"seq": s, "text": t} for s, t in facts]}
|
|
389
|
+
|
|
390
|
+
def export_md(name, path=None):
|
|
391
|
+
"""Inspectable Markdown vault (learned from ECC's Memory Vault):
|
|
392
|
+
dump the FULL session -- every turn verbatim via originals, plus
|
|
393
|
+
registry and fact layers -- to a human-readable string/file."""
|
|
394
|
+
lines = ["# session %s" % name, ""]
|
|
395
|
+
ents = _registry(name)
|
|
396
|
+
if ents:
|
|
397
|
+
lines += ["## key facts", "", " ".join(ents), ""]
|
|
398
|
+
lines += ["## turns", ""]
|
|
399
|
+
for _sid, seq, role, kind, text, _oc in _rows(name):
|
|
400
|
+
body = expand(name, seq)
|
|
401
|
+
if body is None:
|
|
402
|
+
body = text
|
|
403
|
+
lines.append("### [%02d] %s (%s)" % (seq, role, kind))
|
|
404
|
+
lines += ["", body, ""]
|
|
405
|
+
lines += ["## pinned facts", ""]
|
|
406
|
+
with _conn() as c:
|
|
407
|
+
frows = c.execute("SELECT seq,text FROM facts WHERE name=? "
|
|
408
|
+
"ORDER BY seq", (name,)).fetchall()
|
|
409
|
+
for sq, tx in frows:
|
|
410
|
+
lines.append("- [t%d] %s" % (sq, tx))
|
|
411
|
+
md = NL.join(lines)
|
|
412
|
+
if path:
|
|
413
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
414
|
+
f.write(md)
|
|
415
|
+
return md
|
|
416
|
+
|
|
417
|
+
def stat(name):
|
|
418
|
+
rows = _rows(name)
|
|
419
|
+
raw = [r for r in rows if r[3] == "raw"]
|
|
420
|
+
comp = [r for r in rows if r[3] == "comp"]
|
|
421
|
+
return {"turns": len(rows),
|
|
422
|
+
"raw": len(raw), "comp": len(comp),
|
|
423
|
+
"est_tokens": sum(estimate_tokens(r[4]) for r in rows)}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: foldkeep
|
|
3
|
+
Version: 0.9.2
|
|
4
|
+
Summary: Zero-dependency lossless context folding for agent sessions: pinned facts, tolerance routing, byte-for-byte expand.
|
|
5
|
+
Author: foldkeep contributors
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 foldkeep contributors
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
Keywords: llm,agent,context-window,memory,compression
|
|
28
|
+
Classifier: Development Status :: 4 - Beta
|
|
29
|
+
Classifier: Intended Audience :: Developers
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Programming Language :: Python :: 3
|
|
32
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
33
|
+
Requires-Python: >=3.10
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
License-File: LICENSE
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# foldkeep
|
|
39
|
+
|
|
40
|
+
**Fold the context. Keep the facts.**
|
|
41
|
+
|
|
42
|
+
foldkeep is a zero-dependency, deterministic context-compression engine for
|
|
43
|
+
long agent conversations. It rolls old turns into compressed form while a
|
|
44
|
+
pinned fact layer keeps every decision reachable — and a lossless base lets
|
|
45
|
+
you expand ANY turn back byte-for-byte, even after rolling and merging.
|
|
46
|
+
|
|
47
|
+
Pure Python stdlib. No LLM calls. Millisecond latency. SQLite-backed.
|
|
48
|
+
|
|
49
|
+
## Why
|
|
50
|
+
|
|
51
|
+
Agent transcripts grow until the model drowns. Naive fixes lose the past:
|
|
52
|
+
tail-truncation drops early decisions; naive compression mangles numbers,
|
|
53
|
+
code and tool payloads. foldkeep keeps three guarantees:
|
|
54
|
+
|
|
55
|
+
1. **Pinned facts survive** — every turn pins a salient evidence line;
|
|
56
|
+
a set-cover knapsack picks which lines fill the leftover budget so
|
|
57
|
+
entity coverage is maximized, not clustered.
|
|
58
|
+
2. **Zero-tolerance content is never mangled** — code fences, JSON payloads
|
|
59
|
+
and tool-call shapes are routed away from lossy compression.
|
|
60
|
+
3. **Nothing is ever lost** — originals persist on disk; `expand(name, seq)`
|
|
61
|
+
recovers any turn byte-for-byte after rollup, consolidation or render
|
|
62
|
+
drops.
|
|
63
|
+
|
|
64
|
+
## Quickstart
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from foldkeep import session as sess
|
|
68
|
+
|
|
69
|
+
sess.new("my-session")
|
|
70
|
+
for role, text in conversation: # push turns as they happen
|
|
71
|
+
sess.push("my-session", role, text) # auto-rolls when long
|
|
72
|
+
|
|
73
|
+
out, used, dropped = sess.render("my-session", budget=800)
|
|
74
|
+
# -> compressed body + key-facts registry + pinned-fact block, <= 800 tok
|
|
75
|
+
|
|
76
|
+
orig = sess.expand("my-session", 3) # byte-for-byte original of turn 3
|
|
77
|
+
hits = sess.search("my-session", "TKT-9001") # recall across rolled turns
|
|
78
|
+
md = sess.export_md("my-session", "vault.md") # inspectable Markdown vault
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
CLI:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
python -m foldkeep selftest # 16-case correctness battery
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Proven, not promised
|
|
88
|
+
|
|
89
|
+
| Suite | Result |
|
|
90
|
+
|---|---|
|
|
91
|
+
| selftest (correctness) | 16/16 |
|
|
92
|
+
| stress_audit (adversarial probes) | 22/22 |
|
|
93
|
+
| marathon (long-session checkpoints) | 9/9, push ≤ 12 ms, render ≤ 80 ms |
|
|
94
|
+
| agent_selftest (real build-session replay, equal budgets) | wins 5/6 cells vs tail-only & head-trunc |
|
|
95
|
+
| chat_selftest (this project's own chat replay) | 12/12, recall 9/9 under forced rolling |
|
|
96
|
+
|
|
97
|
+
Benchmarks live in this repo (`agent_selftest.py`, `chat_selftest.py`,
|
|
98
|
+
`stress_audit.py`, `long_session_test.py`) — every number reproducible.
|
|
99
|
+
|
|
100
|
+
## Design
|
|
101
|
+
|
|
102
|
+
- **Rolling compression**: newest stays verbatim, oldest compresses into a
|
|
103
|
+
head; hierarchical re-merge keeps the head shrinking.
|
|
104
|
+
- **Three-layer render**: compressed body + one-line entity registry +
|
|
105
|
+
set-cover pinned-fact block, all inside your token budget.
|
|
106
|
+
- **Loss-tolerance routing**: fences / JSON / tool-call shapes bypass
|
|
107
|
+
lossy paths entirely.
|
|
108
|
+
- **Lossless base**: `orig_text` column + `originals` table + `expand()`.
|
|
109
|
+
- **Operator knobs**: `OC_REGISTRY_MAX` env caps registry injection.
|
|
110
|
+
|
|
111
|
+
## Status
|
|
112
|
+
|
|
113
|
+
v0.9.2. Research notes (`RESEARCH.md`) document the competitive landscape
|
|
114
|
+
(LLMLingua, leanctx, mnesis, FoldAgent, pi-fold, ECC, hermes-agent) and what
|
|
115
|
+
we learned from each.
|
|
116
|
+
|
|
117
|
+
MIT licensed. Windows/macOS/Linux, Python 3.10+.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
foldkeep/__init__.py,sha256=YgI1i0HkgOC7Tyy_6FvUHd8QfF67BvTEoL78kOn3d64,654
|
|
2
|
+
foldkeep/__main__.py,sha256=wT62aXCuN7iwRPQoswnmeGiBS5Cou0CIBvMxbOTfvFs,33
|
|
3
|
+
foldkeep/cli.py,sha256=klhNcHWfvy3fTnLW37acwYdfQ82iL_Xdc9yk6t-eQVE,10193
|
|
4
|
+
foldkeep/diff.py,sha256=XbQXeYAI0J30SI8toFvl69mEh5nuDyLjdXqYwrehHFk,3812
|
|
5
|
+
foldkeep/estimate.py,sha256=9b-ycl1kV6GtLxfd1jaiM17Frw1Ae7RxiiouYTvwOpM,1084
|
|
6
|
+
foldkeep/mcp_server.py,sha256=MEcfMIglstnEHfinuhgf7DC9koQd4MpTqaHDFKdjSek,8245
|
|
7
|
+
foldkeep/packing.py,sha256=c4bIaB37_wvUDjB27nJoiXvB2q_HuxlnqkucTxnf0t4,2469
|
|
8
|
+
foldkeep/selftest.py,sha256=Ty3mgppgnyo26N3BB7VMdkHORE1Pxke8bCv5jA73M9o,5993
|
|
9
|
+
foldkeep/session.py,sha256=z5TwHaCZjvhsFQuTAHsY8ZFW9RH4CMwDmTEc3PbyLMY,17797
|
|
10
|
+
foldkeep/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
foldkeep/core/compressor.py,sha256=Eand_M0KPNHbgZwP-RU2KAdEiFKIBaS-b0_fUIWdo9U,15642
|
|
12
|
+
foldkeep/memory/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
foldkeep/memory/store.py,sha256=6o7a7AMiboS1VZL3AgErmLWpm9OfQQJxMPGxzx4JVNw,3322
|
|
14
|
+
foldkeep-0.9.2.dist-info/licenses/LICENSE,sha256=VFosHB0I5sHdQeeGhyQSyCZUEzDh6hXe5w8sUEMFhQY,1097
|
|
15
|
+
foldkeep-0.9.2.dist-info/METADATA,sha256=G40-TvlErwV-h3HBpLkcfsV-JTENCtwRRek5rNCkSC8,5102
|
|
16
|
+
foldkeep-0.9.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
17
|
+
foldkeep-0.9.2.dist-info/top_level.txt,sha256=rPCGfvrjkGK93LGcI4VTNyG3NqFejwB6pSBzmXOJXjs,9
|
|
18
|
+
foldkeep-0.9.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 foldkeep contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
foldkeep
|