foldkeep 0.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foldkeep/__init__.py +16 -0
- foldkeep/__main__.py +3 -0
- foldkeep/cli.py +246 -0
- foldkeep/core/__init__.py +0 -0
- foldkeep/core/compressor.py +346 -0
- foldkeep/diff.py +105 -0
- foldkeep/estimate.py +28 -0
- foldkeep/mcp_server.py +189 -0
- foldkeep/memory/__init__.py +0 -0
- foldkeep/memory/store.py +92 -0
- foldkeep/packing.py +63 -0
- foldkeep/selftest.py +154 -0
- foldkeep/session.py +423 -0
- foldkeep-0.9.2.dist-info/METADATA +117 -0
- foldkeep-0.9.2.dist-info/RECORD +18 -0
- foldkeep-0.9.2.dist-info/WHEEL +5 -0
- foldkeep-0.9.2.dist-info/licenses/LICENSE +21 -0
- foldkeep-0.9.2.dist-info/top_level.txt +1 -0
foldkeep/diff.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Context Diff -- git-like version chain for agent contexts (v0.4).
|
|
2
|
+
|
|
3
|
+
Store: old_state + delta = new_state (full snapshots kept sparse)
|
|
4
|
+
Delta format: [["=",i1,i2] | ["replace"/"delete"/"insert",i1,i2,[new lines]]]
|
|
5
|
+
Reconstruction walks back to the nearest full snapshot and replays deltas.
|
|
6
|
+
"""
|
|
7
|
+
import difflib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import sqlite3
|
|
11
|
+
import time
|
|
12
|
+
|
|
13
|
+
from .core.compressor import FastLingua
|
|
14
|
+
|
|
15
|
+
_DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "memories.db")
|
|
16
|
+
|
|
17
|
+
def _conn():
|
|
18
|
+
c = sqlite3.connect(_DB)
|
|
19
|
+
c.execute("""CREATE TABLE IF NOT EXISTS versions (
|
|
20
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
21
|
+
project TEXT NOT NULL DEFAULT 'default',
|
|
22
|
+
label TEXT NOT NULL DEFAULT '',
|
|
23
|
+
parent_id INTEGER,
|
|
24
|
+
full_comp TEXT,
|
|
25
|
+
delta TEXT,
|
|
26
|
+
orig_chars INTEGER, comp_chars INTEGER,
|
|
27
|
+
created REAL)""")
|
|
28
|
+
return c
|
|
29
|
+
|
|
30
|
+
def _latest(project):
|
|
31
|
+
with _conn() as c:
|
|
32
|
+
row = c.execute("SELECT id FROM versions WHERE project=? "
|
|
33
|
+
"ORDER BY id DESC LIMIT 1", (project,)).fetchone()
|
|
34
|
+
return row[0] if row else None
|
|
35
|
+
|
|
36
|
+
def commit(project, text, label=""):
|
|
37
|
+
fl = FastLingua("balanced", 0.5)
|
|
38
|
+
comp, st = fl.compress(text)
|
|
39
|
+
parent = _latest(project)
|
|
40
|
+
use_full, delta_json = True, None
|
|
41
|
+
if parent is not None:
|
|
42
|
+
ptext = show(parent)
|
|
43
|
+
if ptext:
|
|
44
|
+
sm = difflib.SequenceMatcher(None, ptext.split("\n"), comp.split("\n"))
|
|
45
|
+
ops, churn = [], 0
|
|
46
|
+
for tag, i1, i2, j1, j2 in sm.get_opcodes():
|
|
47
|
+
if tag == "equal":
|
|
48
|
+
ops.append(["=", i1, i2])
|
|
49
|
+
else:
|
|
50
|
+
ops.append([tag, i1, i2, comp.split("\n")[j1:j2]])
|
|
51
|
+
churn += (i2 - i1) + (j2 - j1)
|
|
52
|
+
if churn / max(len(ptext) + len(comp), 1) < 0.6:
|
|
53
|
+
use_full, delta_json = False, json.dumps(ops, ensure_ascii=False)
|
|
54
|
+
with _conn() as c:
|
|
55
|
+
cur = c.execute(
|
|
56
|
+
"INSERT INTO versions (project,label,parent_id,full_comp,delta,"
|
|
57
|
+
"orig_chars,comp_chars,created) VALUES (?,?,?,?,?,?,?,?)",
|
|
58
|
+
(project, label, None if use_full else parent,
|
|
59
|
+
comp if use_full else None, delta_json,
|
|
60
|
+
st["orig"], st["comp"], time.time()))
|
|
61
|
+
return {"id": cur.lastrowid, "orig": st["orig"], "comp": st["comp"],
|
|
62
|
+
"ratio": st["ratio"], "stored": "full" if use_full else "delta"}
|
|
63
|
+
|
|
64
|
+
def _row(vid):
|
|
65
|
+
with _conn() as c:
|
|
66
|
+
return c.execute("SELECT id,parent_id,full_comp,delta FROM versions "
|
|
67
|
+
"WHERE id=?", (vid,)).fetchone()
|
|
68
|
+
|
|
69
|
+
def show(vid):
|
|
70
|
+
chain = []
|
|
71
|
+
cur = vid
|
|
72
|
+
while cur is not None:
|
|
73
|
+
row = _row(cur)
|
|
74
|
+
if not row:
|
|
75
|
+
return None
|
|
76
|
+
_id, pid, full, delta = row
|
|
77
|
+
if full is not None:
|
|
78
|
+
base = full
|
|
79
|
+
break
|
|
80
|
+
chain.append(delta)
|
|
81
|
+
cur = pid
|
|
82
|
+
else:
|
|
83
|
+
return None
|
|
84
|
+
for dj in reversed(chain):
|
|
85
|
+
lines = base.split("\n")
|
|
86
|
+
out = []
|
|
87
|
+
for op in json.loads(dj):
|
|
88
|
+
if op[0] == "=":
|
|
89
|
+
out.extend(lines[op[1]:op[2]])
|
|
90
|
+
else:
|
|
91
|
+
out.extend(op[3])
|
|
92
|
+
base = "\n".join(out)
|
|
93
|
+
return base
|
|
94
|
+
|
|
95
|
+
def restore(vid):
|
|
96
|
+
return show(vid)
|
|
97
|
+
|
|
98
|
+
def log(project="default", limit=20):
|
|
99
|
+
with _conn() as c:
|
|
100
|
+
rows = c.execute("SELECT id,label,parent_id,orig_chars,comp_chars,"
|
|
101
|
+
"(full_comp IS NOT NULL),created FROM versions "
|
|
102
|
+
"WHERE project=? ORDER BY id DESC LIMIT ?",
|
|
103
|
+
(project, limit)).fetchall()
|
|
104
|
+
return [{"id": r[0], "label": r[1], "parent": r[2], "orig": r[3],
|
|
105
|
+
"comp": r[4], "full": bool(r[5]), "ts": r[6]} for r in rows]
|
foldkeep/estimate.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Token & cost estimation heuristics (no tokenizer dependency)."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
_PRICING = {
|
|
5
|
+
"gpt-4o": 2.50, "gpt-4o-mini": 0.15,
|
|
6
|
+
"claude-sonnet": 3.00, "claude-haiku": 0.80,
|
|
7
|
+
"deepseek-v3": 0.27, "qwen-max": 1.60,
|
|
8
|
+
"local": 0.0,
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
def estimate_tokens(text, model=None):
|
|
12
|
+
# model accepted for API symmetry; token count is a model-agnostic
|
|
13
|
+
# heuristic (CJK ~1.5 chars/token, other ~4 chars/token).
|
|
14
|
+
zh = len(re.findall(r"[\u4e00-\u9fff]", text))
|
|
15
|
+
other = max(len(text) - zh, 0)
|
|
16
|
+
return int(zh / 1.5 + other / 4.0)
|
|
17
|
+
|
|
18
|
+
def estimate_cost(text, model="claude-sonnet"):
|
|
19
|
+
t = estimate_tokens(text)
|
|
20
|
+
rate = _PRICING.get(model, 0.0)
|
|
21
|
+
return {"tokens": t, "usd": round(t * rate / 1000000.0, 6), "model": model}
|
|
22
|
+
|
|
23
|
+
def savings(orig, comp, model="claude-sonnet"):
|
|
24
|
+
a, b = estimate_cost(orig, model), estimate_cost(comp, model)
|
|
25
|
+
saved = max(a["tokens"] - b["tokens"], 0)
|
|
26
|
+
return {"tokens_saved": saved,
|
|
27
|
+
"usd_saved": round(a["usd"] - b["usd"], 6),
|
|
28
|
+
"ratio": round(b["tokens"] / max(a["tokens"], 1), 3)}
|
foldkeep/mcp_server.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""FoldKeep MCP Server (zero-dependency, stdio JSON-RPC).
|
|
2
|
+
|
|
3
|
+
Run: python -m foldkeep.mcp_server
|
|
4
|
+
Expose three tools to any MCP client (Claude Desktop, Cursor, ...):
|
|
5
|
+
compress_context(text, ratio?) -> compressed text + stats
|
|
6
|
+
save_context(name, text, kind?, project?) -> compress + store in memory bank
|
|
7
|
+
search_context(query, project?, limit?) -> ranked memories
|
|
8
|
+
Protocol: newline-delimited JSON-RPC 2.0 (MCP stdio transport).
|
|
9
|
+
"""
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
from .core.compressor import FastLingua
|
|
14
|
+
from .memory import store
|
|
15
|
+
from .diff import commit as diff_commit, show as diff_show
|
|
16
|
+
from . import session as sess
|
|
17
|
+
|
|
18
|
+
SERVER_INFO = {"name": "foldkeep", "version": "0.3.0"}
|
|
19
|
+
PROTOCOL = "2024-11-05"
|
|
20
|
+
|
|
21
|
+
TOOLS = [
|
|
22
|
+
{"name": "compress_context",
|
|
23
|
+
"description": "Compress text with entity-preserving FastLingua. Returns compressed text and stats.",
|
|
24
|
+
"inputSchema": {"type": "object",
|
|
25
|
+
"properties": {"text": {"type": "string"},
|
|
26
|
+
"ratio": {"type": "number", "default": 0.5}},
|
|
27
|
+
"required": ["text"]}},
|
|
28
|
+
{"name": "save_context",
|
|
29
|
+
"description": "Compress text and store it in the memory bank under a name (kind: snapshot|history|decision|summary).",
|
|
30
|
+
"inputSchema": {"type": "object",
|
|
31
|
+
"properties": {"name": {"type": "string"},
|
|
32
|
+
"text": {"type": "string"},
|
|
33
|
+
"kind": {"type": "string",
|
|
34
|
+
"enum": ["snapshot", "history", "decision", "summary"],
|
|
35
|
+
"default": "snapshot"},
|
|
36
|
+
"project": {"type": "string", "default": "default"}},
|
|
37
|
+
"required": ["name", "text"]}},
|
|
38
|
+
{"name": "context_commit",
|
|
39
|
+
"description": "Commit a new context version for a project (git-like chain, stores delta vs previous).",
|
|
40
|
+
"inputSchema": {"type": "object",
|
|
41
|
+
"properties": {"text": {"type": "string"},
|
|
42
|
+
"project": {"type": "string", "default": "default"},
|
|
43
|
+
"label": {"type": "string", "default": ""}},
|
|
44
|
+
"required": ["text"]}},
|
|
45
|
+
{"name": "context_restore",
|
|
46
|
+
"description": "Restore (reconstruct) a context version by id.",
|
|
47
|
+
"inputSchema": {"type": "object",
|
|
48
|
+
"properties": {"id": {"type": "number"}},
|
|
49
|
+
"required": ["id"]}},
|
|
50
|
+
{"name": "session_push",
|
|
51
|
+
"description": "Append a turn to a named long-running session; old turns auto-compress.",
|
|
52
|
+
"inputSchema": {"type": "object",
|
|
53
|
+
"properties": {"name": {"type": "string"},
|
|
54
|
+
"role": {"type": "string",
|
|
55
|
+
"enum": ["user", "assistant", "system"]},
|
|
56
|
+
"text": {"type": "string"}},
|
|
57
|
+
"required": ["name", "role", "text"]}},
|
|
58
|
+
{"name": "session_render",
|
|
59
|
+
"description": "Render session context within optional token budget (compressed head + raw tail).",
|
|
60
|
+
"inputSchema": {"type": "object",
|
|
61
|
+
"properties": {"name": {"type": "string"},
|
|
62
|
+
"budget": {"type": "number"}},
|
|
63
|
+
"required": ["name"]}},
|
|
64
|
+
{"name": "search_context",
|
|
65
|
+
"description": "Search stored memories by keywords (matches entity tags and content).",
|
|
66
|
+
"inputSchema": {"type": "object",
|
|
67
|
+
"properties": {"query": {"type": "string"},
|
|
68
|
+
"project": {"type": "string"},
|
|
69
|
+
"limit": {"type": "number", "default": 5}},
|
|
70
|
+
"required": ["query"]}},
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
def _tool_compress(args):
|
|
74
|
+
text = args.get("text", "")
|
|
75
|
+
ratio = float(args.get("ratio", 0.5))
|
|
76
|
+
fl = FastLingua("balanced", ratio)
|
|
77
|
+
comp, st = fl.compress(text)
|
|
78
|
+
return ("compressed %d -> %d chars (%.1f%%), entities %d\n\n%s"
|
|
79
|
+
% (st["orig"], st["comp"], st["ratio"] * 100, st["entities"], comp))
|
|
80
|
+
|
|
81
|
+
def _tool_save(args):
|
|
82
|
+
name = args.get("name", "")
|
|
83
|
+
text = args.get("text", "")
|
|
84
|
+
kind = args.get("kind", "snapshot")
|
|
85
|
+
project = args.get("project", "default")
|
|
86
|
+
fl = FastLingua("balanced", 0.5)
|
|
87
|
+
comp, st = fl.compress(text)
|
|
88
|
+
store.save(name, text, comp, project=project, kind=kind)
|
|
89
|
+
return ("saved [%s/%s] %s | %d -> %d chars (%.1f%%)"
|
|
90
|
+
% (project, kind, name, st["orig"], st["comp"], st["ratio"] * 100))
|
|
91
|
+
|
|
92
|
+
def _tool_search(args):
|
|
93
|
+
query = args.get("query", "")
|
|
94
|
+
project = args.get("project")
|
|
95
|
+
limit = int(args.get("limit", 5))
|
|
96
|
+
hits = store.search(query, project=project, limit=limit)
|
|
97
|
+
if not hits:
|
|
98
|
+
return "(no matches)"
|
|
99
|
+
return "\n".join("%s [%s/%s] score=%d %dchars"
|
|
100
|
+
% (h["name"], h["project"], h["kind"], h["score"], h["chars"])
|
|
101
|
+
for h in hits)
|
|
102
|
+
|
|
103
|
+
def _tool_ctx_commit(args):
|
|
104
|
+
r = diff_commit(args.get("project", "default"), args.get("text", ""),
|
|
105
|
+
args.get("label", ""))
|
|
106
|
+
return ("committed v%d | %d -> %d chars (%.1f%%) | stored %s"
|
|
107
|
+
% (r["id"], r["orig"], r["comp"], r["ratio"] * 100, r["stored"]))
|
|
108
|
+
|
|
109
|
+
def _tool_ctx_restore(args):
|
|
110
|
+
text = diff_show(int(args.get("id", 0)))
|
|
111
|
+
return text if text is not None else "(version not found)"
|
|
112
|
+
|
|
113
|
+
def _tool_sess_push(args):
|
|
114
|
+
r = sess.push(args.get("name", ""), args.get("role", "user"),
|
|
115
|
+
args.get("text", ""))
|
|
116
|
+
st = sess.stat(args.get("name", ""))
|
|
117
|
+
return ("pushed #%d | rolled %d | session %d turns ~%d tok"
|
|
118
|
+
% (r["seq"], r["rolled"], st["turns"], st["est_tokens"]))
|
|
119
|
+
|
|
120
|
+
def _tool_sess_render(args):
|
|
121
|
+
b = args.get("budget")
|
|
122
|
+
out, used, dropped = sess.render(args.get("name", ""),
|
|
123
|
+
int(b) if b else None)
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
_HANDLERS = {"session_push": _tool_sess_push,
|
|
127
|
+
"session_render": _tool_sess_render,
|
|
128
|
+
"context_commit": _tool_ctx_commit,
|
|
129
|
+
"context_restore": _tool_ctx_restore,
|
|
130
|
+
"compress_context": _tool_compress,
|
|
131
|
+
"save_context": _tool_save,
|
|
132
|
+
"search_context": _tool_search}
|
|
133
|
+
|
|
134
|
+
def _reply(msg_id, result):
|
|
135
|
+
print(json.dumps({"jsonrpc": "2.0", "id": msg_id, "result": result}),
|
|
136
|
+
flush=True)
|
|
137
|
+
|
|
138
|
+
def _error(msg_id, code, message):
|
|
139
|
+
print(json.dumps({"jsonrpc": "2.0", "id": msg_id,
|
|
140
|
+
"error": {"code": code, "message": message}}),
|
|
141
|
+
flush=True)
|
|
142
|
+
|
|
143
|
+
def handle(msg):
|
|
144
|
+
method = msg.get("method", "")
|
|
145
|
+
msg_id = msg.get("id")
|
|
146
|
+
if method == "initialize":
|
|
147
|
+
_reply(msg_id, {"protocolVersion": PROTOCOL,
|
|
148
|
+
"capabilities": {"tools": {}},
|
|
149
|
+
"serverInfo": SERVER_INFO})
|
|
150
|
+
elif method == "tools/list":
|
|
151
|
+
_reply(msg_id, {"tools": TOOLS})
|
|
152
|
+
elif method == "tools/call":
|
|
153
|
+
params = msg.get("params", {})
|
|
154
|
+
name = params.get("name", "")
|
|
155
|
+
args = params.get("arguments", {})
|
|
156
|
+
fn = _HANDLERS.get(name)
|
|
157
|
+
if not fn:
|
|
158
|
+
_error(msg_id, -32602, "unknown tool: %s" % name)
|
|
159
|
+
return
|
|
160
|
+
spec = next((t for t in TOOLS if t["name"] == name), None)
|
|
161
|
+
missing = [k for k in (spec["inputSchema"].get("required", []) if spec else [])
|
|
162
|
+
if k not in args]
|
|
163
|
+
if missing:
|
|
164
|
+
_reply(msg_id, {"content": [{"type": "text",
|
|
165
|
+
"text": "missing required arguments: %s" % ", ".join(missing)}],
|
|
166
|
+
"isError": True})
|
|
167
|
+
return
|
|
168
|
+
try:
|
|
169
|
+
_reply(msg_id, {"content": [{"type": "text", "text": fn(args)}]})
|
|
170
|
+
except Exception as ex:
|
|
171
|
+
_reply(msg_id, {"content": [{"type": "text", "text": "error: %s" % ex}],
|
|
172
|
+
"isError": True})
|
|
173
|
+
elif method.startswith("notifications/"):
|
|
174
|
+
pass
|
|
175
|
+
elif msg_id is not None:
|
|
176
|
+
_error(msg_id, -32601, "method not found: %s" % method)
|
|
177
|
+
|
|
178
|
+
def main():
|
|
179
|
+
for line in sys.stdin:
|
|
180
|
+
line = line.strip()
|
|
181
|
+
if not line:
|
|
182
|
+
continue
|
|
183
|
+
try:
|
|
184
|
+
handle(json.loads(line))
|
|
185
|
+
except Exception as ex:
|
|
186
|
+
_error(None, -32700, "parse error: %s" % ex)
|
|
187
|
+
|
|
188
|
+
if __name__ == "__main__":
|
|
189
|
+
main()
|
|
File without changes
|
foldkeep/memory/store.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""FoldKeep memory bank (SQLite, stdlib only).
|
|
2
|
+
|
|
3
|
+
Kinds: snapshot | history | decision | summary
|
|
4
|
+
API kept compatible with v0.1 JSON store, plus search() and projects.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import sqlite3
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from ..core.compressor import ENT_RE, KEEP_KEYWORDS
|
|
12
|
+
from ..estimate import estimate_tokens
|
|
13
|
+
|
|
14
|
+
_DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "memories.db")
|
|
15
|
+
|
|
16
|
+
def _conn():
|
|
17
|
+
c = sqlite3.connect(_DB)
|
|
18
|
+
c.execute("""CREATE TABLE IF NOT EXISTS memories (
|
|
19
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
20
|
+
project TEXT NOT NULL DEFAULT 'default',
|
|
21
|
+
kind TEXT NOT NULL DEFAULT 'snapshot',
|
|
22
|
+
name TEXT NOT NULL UNIQUE,
|
|
23
|
+
tags TEXT NOT NULL DEFAULT '',
|
|
24
|
+
orig_chars INTEGER, comp_chars INTEGER, tokens_est INTEGER,
|
|
25
|
+
text_comp TEXT NOT NULL,
|
|
26
|
+
created REAL)""")
|
|
27
|
+
return c
|
|
28
|
+
|
|
29
|
+
def _tags_of(text):
|
|
30
|
+
ents = sorted(set(ENT_RE.findall(text)))
|
|
31
|
+
kws = [k for k in KEEP_KEYWORDS if k in text]
|
|
32
|
+
return " ".join(ents + kws)[:500]
|
|
33
|
+
|
|
34
|
+
def save(name, orig_text, comp_text, meta=None, project="default", kind="snapshot"):
|
|
35
|
+
with _conn() as c:
|
|
36
|
+
c.execute("INSERT OR REPLACE INTO memories "
|
|
37
|
+
"(project,kind,name,tags,orig_chars,comp_chars,tokens_est,text_comp,created) "
|
|
38
|
+
"VALUES (?,?,?,?,?,?,?,?,?)",
|
|
39
|
+
(project, kind, name, _tags_of(orig_text),
|
|
40
|
+
len(orig_text), len(comp_text),
|
|
41
|
+
estimate_tokens(orig_text), comp_text, time.time()))
|
|
42
|
+
return _DB
|
|
43
|
+
|
|
44
|
+
def get(name):
|
|
45
|
+
with _conn() as c:
|
|
46
|
+
row = c.execute("SELECT * FROM memories WHERE name=?", (name,)).fetchone()
|
|
47
|
+
if not row:
|
|
48
|
+
return None
|
|
49
|
+
cols = ["id", "project", "kind", "name", "tags", "orig_chars",
|
|
50
|
+
"comp_chars", "tokens_est", "text_comp", "created"]
|
|
51
|
+
return dict(zip(cols, row))
|
|
52
|
+
|
|
53
|
+
def delete(name):
|
|
54
|
+
with _conn() as c:
|
|
55
|
+
cur = c.execute("DELETE FROM memories WHERE name=?", (name,))
|
|
56
|
+
return cur.rowcount > 0
|
|
57
|
+
|
|
58
|
+
def list_all(project=None):
|
|
59
|
+
q = "SELECT name, project, kind, orig_chars, comp_chars, created FROM memories"
|
|
60
|
+
args = ()
|
|
61
|
+
if project:
|
|
62
|
+
q += " WHERE project=?"
|
|
63
|
+
args = (project,)
|
|
64
|
+
q += " ORDER BY created DESC"
|
|
65
|
+
with _conn() as c:
|
|
66
|
+
rows = c.execute(q, args).fetchall()
|
|
67
|
+
return [{"name": r[0], "project": r[1], "kind": r[2],
|
|
68
|
+
"orig": r[3], "comp": r[4], "ts": r[5]} for r in rows]
|
|
69
|
+
|
|
70
|
+
def search(query, project=None, limit=5):
|
|
71
|
+
terms = [t for t in re.split(r"\s+", query.strip()) if t]
|
|
72
|
+
if not terms:
|
|
73
|
+
return []
|
|
74
|
+
q = "SELECT name, project, kind, tags, text_comp, comp_chars FROM memories"
|
|
75
|
+
args = []
|
|
76
|
+
conds = []
|
|
77
|
+
if project:
|
|
78
|
+
conds.append("project=?")
|
|
79
|
+
args.append(project)
|
|
80
|
+
with _conn() as c:
|
|
81
|
+
rows = c.execute(q + (" WHERE " + " AND ".join(conds) if conds else ""), args).fetchall()
|
|
82
|
+
scored = []
|
|
83
|
+
for name, proj, kind, tags, text, nchars in rows:
|
|
84
|
+
hay_t = tags.lower()
|
|
85
|
+
hay_c = (text or "").lower()
|
|
86
|
+
score = sum(2 for t in terms if t.lower() in hay_t)
|
|
87
|
+
score += sum(1 for t in terms if t.lower() in hay_c)
|
|
88
|
+
if score:
|
|
89
|
+
scored.append({"name": name, "project": proj, "kind": kind,
|
|
90
|
+
"score": score, "chars": nchars})
|
|
91
|
+
scored.sort(key=lambda x: -x["score"])
|
|
92
|
+
return scored[:limit]
|
foldkeep/packing.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Budget packing: fit multiple documents into a token budget (v0.2).
|
|
2
|
+
|
|
3
|
+
Strategy:
|
|
4
|
+
1. compress every document (FastLingua, balanced)
|
|
5
|
+
2. score importance = entities*3 + keep-keywords*2 + 1
|
|
6
|
+
3. greedy knapsack by importance-per-token
|
|
7
|
+
4. overflow items get a second, harder compression pass
|
|
8
|
+
5. still overflowing -> dropped and reported (never exceed budget)
|
|
9
|
+
"""
|
|
10
|
+
from .core.compressor import FastLingua, ENT_RE, KEEP_KEYWORDS
|
|
11
|
+
from .estimate import estimate_tokens
|
|
12
|
+
|
|
13
|
+
def _importance(text):
|
|
14
|
+
ents = len(set(ENT_RE.findall(text)))
|
|
15
|
+
kw = sum(1 for k in KEEP_KEYWORDS if k in text)
|
|
16
|
+
return 1.0 + ents * 3.0 + kw * 2.0
|
|
17
|
+
|
|
18
|
+
def pack(items, budget, ratio=0.5):
|
|
19
|
+
"""items: list of (name, text). Returns a result dict (never exceeds budget)."""
|
|
20
|
+
fl = FastLingua("balanced", ratio)
|
|
21
|
+
cands = []
|
|
22
|
+
for name, text in items:
|
|
23
|
+
comp, st = fl.compress(text)
|
|
24
|
+
tok = estimate_tokens(comp)
|
|
25
|
+
imp = _importance(text)
|
|
26
|
+
cands.append({"name": name, "orig": text, "text": comp,
|
|
27
|
+
"tokens": tok, "imp": imp,
|
|
28
|
+
"density": imp / max(tok, 1)})
|
|
29
|
+
cands.sort(key=lambda c: -c["density"])
|
|
30
|
+
used, packed, dropped = 0, [], []
|
|
31
|
+
for c in cands:
|
|
32
|
+
if c["tokens"] <= 0 and c["orig"].strip():
|
|
33
|
+
dropped.append(c)
|
|
34
|
+
continue
|
|
35
|
+
if used + c["tokens"] <= budget:
|
|
36
|
+
packed.append(c)
|
|
37
|
+
used += c["tokens"]
|
|
38
|
+
continue
|
|
39
|
+
fl2 = FastLingua("balanced", max(ratio * 0.5, 0.15))
|
|
40
|
+
comp2, _st2 = fl2.compress(c["orig"])
|
|
41
|
+
tok2 = estimate_tokens(comp2)
|
|
42
|
+
if tok2 > 0 and used + tok2 <= budget:
|
|
43
|
+
c["text"], c["tokens"] = comp2, tok2
|
|
44
|
+
c["hard"] = True
|
|
45
|
+
packed.append(c)
|
|
46
|
+
used += tok2
|
|
47
|
+
else:
|
|
48
|
+
dropped.append(c)
|
|
49
|
+
return {"budget": budget, "used": used,
|
|
50
|
+
"packed": packed, "dropped": dropped}
|
|
51
|
+
|
|
52
|
+
def render(result):
|
|
53
|
+
lines = ["=== FoldKeep Pack | budget %d tok | used %d tok ==="
|
|
54
|
+
% (result["budget"], result["used"])]
|
|
55
|
+
for c in result["packed"]:
|
|
56
|
+
tag = " (hard-compressed)" if c.get("hard") else ""
|
|
57
|
+
lines.append("--- [%s] %d tok%s ---" % (c["name"], c["tokens"], tag))
|
|
58
|
+
lines.append(c["text"])
|
|
59
|
+
if result["dropped"]:
|
|
60
|
+
names = ", ".join("%s(~%dtok)" % (c["name"], c["tokens"])
|
|
61
|
+
for c in result["dropped"])
|
|
62
|
+
lines.append("=== DROPPED (over budget): %s ===" % names)
|
|
63
|
+
return "\n".join(lines)
|
foldkeep/selftest.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""FoldKeep self-test battery (16 edge cases). Run: oc selftest"""
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
def run(verbose=True):
|
|
8
|
+
from .core.compressor import FastLingua
|
|
9
|
+
from . import store, diff, session as sess, packing
|
|
10
|
+
from .estimate import estimate_cost
|
|
11
|
+
|
|
12
|
+
R = []
|
|
13
|
+
def case(name, fn):
|
|
14
|
+
try:
|
|
15
|
+
fn(); R.append((name, "PASS", ""))
|
|
16
|
+
except Exception as ex:
|
|
17
|
+
R.append((name, "FAIL", "%s: %s" % (type(ex).__name__, ex)))
|
|
18
|
+
|
|
19
|
+
def c1():
|
|
20
|
+
fl = FastLingua("balanced", 0.5)
|
|
21
|
+
comp, st = fl.compress("")
|
|
22
|
+
assert st["orig"] == 0 and comp == ""
|
|
23
|
+
case("compress empty", c1)
|
|
24
|
+
|
|
25
|
+
def c2():
|
|
26
|
+
fl = FastLingua("balanced", 0.5)
|
|
27
|
+
t = "\u4f1a\u8bae\u8bb0\u5f55\uff1a\u4eca\u5929\u4e0ealice\u8ba8\u8bba\u4e86FastDeploy\u7684\u4e0a\u7ebf\u5ef6\u671f\u95ee\u9898\uff0c\u539f\u56e0\u662f\u6570\u636e\u5e93\u8fde\u63a5\u8d85\u65f6\uff0c\u51b3\u5b9a\u4e0b\u5468\u4fee\u590d\u3002" * 6
|
|
28
|
+
comp, st = fl.compress(t)
|
|
29
|
+
assert 0 < st["ratio"] <= 1.0 and len(comp) > 0
|
|
30
|
+
case("compress zh long", c2)
|
|
31
|
+
|
|
32
|
+
def c3():
|
|
33
|
+
fl = FastLingua("balanced", 0.5)
|
|
34
|
+
comp, st = fl.compress("https://api.example.com/ticket/42 https://db.internal.net/x")
|
|
35
|
+
assert "https://api.example.com/ticket/42" in comp
|
|
36
|
+
case("url preserved", c3)
|
|
37
|
+
|
|
38
|
+
def c4():
|
|
39
|
+
info = estimate_cost("hello world this is a test", "no-such-model")
|
|
40
|
+
assert info["usd"] >= 0
|
|
41
|
+
case("unknown model cost", c4)
|
|
42
|
+
|
|
43
|
+
def c5():
|
|
44
|
+
store.save("qa-uni", "\u5185\u5bb9\uff1aConnectionTimeout \u51b3\u7b56 \u56de\u6eda",
|
|
45
|
+
"\u5185\u5bb9 ConnectionTimeout")
|
|
46
|
+
g = store.get("qa-uni")
|
|
47
|
+
assert g and "ConnectionTimeout" in g["text_comp"]
|
|
48
|
+
store.delete("qa-uni")
|
|
49
|
+
case("store unicode", c5)
|
|
50
|
+
|
|
51
|
+
def c6():
|
|
52
|
+
store.save("qa-zh", "\u51b3\u7b56\uff1amax_connections 200 ConnectionTimeout",
|
|
53
|
+
"\u51b3\u7b56 max_connections")
|
|
54
|
+
hits = store.search("\u6570\u636e\u5e93 \u51b3\u7b56")
|
|
55
|
+
assert isinstance(hits, list)
|
|
56
|
+
store.delete("qa-zh")
|
|
57
|
+
case("store search", c6)
|
|
58
|
+
|
|
59
|
+
def c7():
|
|
60
|
+
fl = FastLingua("balanced", 0.5)
|
|
61
|
+
txt = ("Incident A: ConnectionTimeout on db.internal.net ticket 77. "
|
|
62
|
+
"Root cause pool exhaustion. Fix config.yaml.")
|
|
63
|
+
for i in range(5):
|
|
64
|
+
r = diff.commit("qa-chain", txt, "v%d" % i)
|
|
65
|
+
exp, _st = fl.compress(txt)
|
|
66
|
+
assert diff.show(r["id"]) == exp, "mismatch at %d" % i
|
|
67
|
+
txt = txt.replace("ticket 77", "ticket 7%d" % (i + 8))
|
|
68
|
+
txt = txt.replace("pool exhaustion", "pool exhaustion level %d" % i)
|
|
69
|
+
case("diff chain x5 exact", c7)
|
|
70
|
+
|
|
71
|
+
def c8():
|
|
72
|
+
t = "Same content decision alice bob ticket 42"
|
|
73
|
+
a = diff.commit("qa-same", t, "a")
|
|
74
|
+
b = diff.commit("qa-same", t, "b")
|
|
75
|
+
assert diff.show(b["id"]) == diff.show(a["id"])
|
|
76
|
+
case("diff identical", c8)
|
|
77
|
+
|
|
78
|
+
def c9():
|
|
79
|
+
sess.push("qa-empty", "user", "")
|
|
80
|
+
out, used, dropped = sess.render("qa-empty")
|
|
81
|
+
assert "[01]" in out
|
|
82
|
+
case("session empty turn", c9)
|
|
83
|
+
|
|
84
|
+
def c10():
|
|
85
|
+
sess.push("qa-tiny", "user", "x " * 500)
|
|
86
|
+
out, used, dropped = sess.render("qa-tiny", budget=5)
|
|
87
|
+
# v0.7.2 contract: min-viable floor -> never empty, bounded output
|
|
88
|
+
assert "/floor]" in out and 0 < used <= 100, (used, out[:80])
|
|
89
|
+
case("session micro budget", c10)
|
|
90
|
+
|
|
91
|
+
def c11():
|
|
92
|
+
r = packing.pack([], 100)
|
|
93
|
+
assert r["used"] == 0 and not r["packed"]
|
|
94
|
+
case("pack empty", c11)
|
|
95
|
+
|
|
96
|
+
def c12():
|
|
97
|
+
r = packing.pack([("big", "word " * 3000)], 10)
|
|
98
|
+
assert r["used"] <= 10 and len(r["dropped"]) == 1, (r["used"], len(r["dropped"]))
|
|
99
|
+
case("pack over-budget drop", c12)
|
|
100
|
+
|
|
101
|
+
def c13():
|
|
102
|
+
fl = FastLingua("balanced", 0.5, anchors=True)
|
|
103
|
+
comp, st = fl.compress(
|
|
104
|
+
"Incident A: ConnectionTimeout on db.internal.net ticket 77 "
|
|
105
|
+
"because pool exhaustion. "
|
|
106
|
+
"Meeting B: alice delayed FastDeploy v2.3 since worker.js OOM. "
|
|
107
|
+
"Decision C: raise max_connections to 200 so p99 recovers.")
|
|
108
|
+
tags = re.findall(r"\[F\d+\]", comp)
|
|
109
|
+
assert len(tags) >= 2 and len(tags) == len(set(tags)), repr(comp)
|
|
110
|
+
case("anchors multi unique", c13)
|
|
111
|
+
|
|
112
|
+
def c14():
|
|
113
|
+
t = "repeat block alpha beta gamma. " * 40
|
|
114
|
+
a, _s1 = FastLingua("balanced", 0.5, collapse=False).compress(t)
|
|
115
|
+
b, _s2 = FastLingua("balanced", 0.5, collapse=True).compress(t)
|
|
116
|
+
assert len(b) <= len(a)
|
|
117
|
+
case("collapse repetition", c14)
|
|
118
|
+
|
|
119
|
+
def _mcp(msgs):
|
|
120
|
+
p = subprocess.run([sys.executable, "-m", "foldkeep.mcp_server"],
|
|
121
|
+
input=msgs.encode("utf-8"), cwd=_pkg_root(),
|
|
122
|
+
capture_output=True, timeout=60)
|
|
123
|
+
return [l for l in p.stdout.decode("utf-8").strip().split("\n") if l.strip()]
|
|
124
|
+
|
|
125
|
+
def _pkg_root():
|
|
126
|
+
import os
|
|
127
|
+
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
128
|
+
|
|
129
|
+
def c15():
|
|
130
|
+
msgs = "\n".join(["not-json",
|
|
131
|
+
'{"jsonrpc":"2.0","id":1,"method":"tools/list"}']) + "\n"
|
|
132
|
+
lines = _mcp(msgs)
|
|
133
|
+
assert any('"tools"' in l for l in lines) and any("-32700" in l for l in lines), lines
|
|
134
|
+
case("mcp bad-line resilient", c15)
|
|
135
|
+
|
|
136
|
+
def c16():
|
|
137
|
+
msg = json.dumps({"jsonrpc": "2.0", "id": 9, "method": "tools/call",
|
|
138
|
+
"params": {"name": "save_context",
|
|
139
|
+
"arguments": {"name": "x"}}}) + "\n"
|
|
140
|
+
out = "\n".join(_mcp(msg))
|
|
141
|
+
assert '"isError": true' in out, out[:200]
|
|
142
|
+
case("mcp missing arg -> isError", c16)
|
|
143
|
+
|
|
144
|
+
if verbose:
|
|
145
|
+
for name, status, detail in R:
|
|
146
|
+
print("%-28s %s %s" % (name, status, detail))
|
|
147
|
+
fails = [r for r in R if r[1] == "FAIL"]
|
|
148
|
+
return R, fails
|
|
149
|
+
|
|
150
|
+
if __name__ == "__main__":
|
|
151
|
+
R, fails = run()
|
|
152
|
+
print("---")
|
|
153
|
+
print("TOTAL %d | PASS %d | FAIL %d" % (len(R), len(R) - len(fails), len(fails)))
|
|
154
|
+
sys.exit(1 if fails else 0)
|