boring-agent-memory 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- boring_agent_memory/__init__.py +12 -0
- boring_agent_memory/api.py +23 -0
- boring_agent_memory/benchmark.py +524 -0
- boring_agent_memory/canonical.py +73 -0
- boring_agent_memory/chunking.py +425 -0
- boring_agent_memory/cli.py +298 -0
- boring_agent_memory/config.py +162 -0
- boring_agent_memory/embeddings.py +253 -0
- boring_agent_memory/eval.py +271 -0
- boring_agent_memory/index.py +449 -0
- boring_agent_memory/ingest.py +175 -0
- boring_agent_memory/privacy.py +84 -0
- boring_agent_memory/query.py +233 -0
- boring_agent_memory/schema.py +212 -0
- boring_agent_memory/server.py +41 -0
- boring_agent_memory/snippets.py +26 -0
- boring_agent_memory-0.2.2.dist-info/METADATA +162 -0
- boring_agent_memory-0.2.2.dist-info/RECORD +22 -0
- boring_agent_memory-0.2.2.dist-info/WHEEL +5 -0
- boring_agent_memory-0.2.2.dist-info/entry_points.txt +2 -0
- boring_agent_memory-0.2.2.dist-info/licenses/LICENSE +22 -0
- boring_agent_memory-0.2.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import sqlite3
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from dataclasses import asdict, dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .schema import connect, init_db, require_index
|
|
10
|
+
from .snippets import plain_snippet
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class QueryResult:
|
|
15
|
+
id: str
|
|
16
|
+
chunk_id: str
|
|
17
|
+
source_type: str
|
|
18
|
+
source_path: str
|
|
19
|
+
workspace: str
|
|
20
|
+
title: str
|
|
21
|
+
heading: str
|
|
22
|
+
start_line: int
|
|
23
|
+
end_line: int
|
|
24
|
+
citation: str
|
|
25
|
+
score: float
|
|
26
|
+
snippet: str
|
|
27
|
+
strategy: str
|
|
28
|
+
|
|
29
|
+
def to_dict(self) -> dict[str, object]:
|
|
30
|
+
return asdict(self)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def query_memory(
|
|
34
|
+
db_path: Path | str,
|
|
35
|
+
query: str,
|
|
36
|
+
limit: int = 5,
|
|
37
|
+
source_type: str | None = None,
|
|
38
|
+
workspace: Path | str | None = None,
|
|
39
|
+
) -> list[QueryResult]:
|
|
40
|
+
conn = connect(require_index(db_path))
|
|
41
|
+
init_db(conn)
|
|
42
|
+
try:
|
|
43
|
+
workspace_path = Path(workspace).expanduser().resolve() if workspace else None
|
|
44
|
+
results = _query_memory(conn, query, limit, source_type, workspace_path)
|
|
45
|
+
finally:
|
|
46
|
+
conn.close()
|
|
47
|
+
return results
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _query_memory(
|
|
51
|
+
conn: sqlite3.Connection,
|
|
52
|
+
query: str,
|
|
53
|
+
limit: int,
|
|
54
|
+
source_type: str | None,
|
|
55
|
+
workspace: Path | None,
|
|
56
|
+
) -> list[QueryResult]:
|
|
57
|
+
if limit <= 0:
|
|
58
|
+
return []
|
|
59
|
+
|
|
60
|
+
seen_documents: set[str] = set()
|
|
61
|
+
results: list[QueryResult] = []
|
|
62
|
+
|
|
63
|
+
def collect(rows: Iterable[sqlite3.Row], strategy: str) -> bool:
|
|
64
|
+
# Rows are chunks but results are documents, so read only until enough distinct documents appear.
|
|
65
|
+
for row in rows:
|
|
66
|
+
if row["id"] in seen_documents:
|
|
67
|
+
continue
|
|
68
|
+
seen_documents.add(row["id"])
|
|
69
|
+
results.append(row_to_result(row, strategy, query))
|
|
70
|
+
if len(results) >= limit:
|
|
71
|
+
return True
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
for strategy, fts_query in build_fts_queries(query):
|
|
75
|
+
try:
|
|
76
|
+
if collect(run_fts_query(conn, fts_query, source_type, workspace), strategy):
|
|
77
|
+
return results
|
|
78
|
+
except sqlite3.DatabaseError:
|
|
79
|
+
continue
|
|
80
|
+
|
|
81
|
+
collect(run_like_query(conn, query, source_type, workspace), "like")
|
|
82
|
+
return results
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def build_fts_queries(query: str) -> list[tuple[str, str]]:
|
|
86
|
+
cleaned = query.strip()
|
|
87
|
+
tokens = tokenize_query(cleaned)
|
|
88
|
+
queries: list[tuple[str, str]] = []
|
|
89
|
+
|
|
90
|
+
if cleaned:
|
|
91
|
+
queries.append(("phrase", quote_fts(cleaned)))
|
|
92
|
+
if tokens:
|
|
93
|
+
queries.append(("and", " AND ".join(quote_fts(token) for token in tokens)))
|
|
94
|
+
queries.append(("or", " OR ".join(quote_fts(token) for token in tokens)))
|
|
95
|
+
return queries
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def tokenize_query(query: str) -> list[str]:
|
|
99
|
+
tokens = re.findall(r"[A-Za-z0-9_./:-]+|[\u4e00-\u9fff]+", query)
|
|
100
|
+
return [token.strip("./:-") for token in tokens if token.strip("./:-")]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def quote_fts(value: str) -> str:
|
|
104
|
+
return '"' + value.replace('"', '""') + '"'
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def run_fts_query(
|
|
108
|
+
conn: sqlite3.Connection,
|
|
109
|
+
fts_query: str,
|
|
110
|
+
source_type: str | None,
|
|
111
|
+
workspace: Path | None,
|
|
112
|
+
) -> Iterable[sqlite3.Row]:
|
|
113
|
+
params: list[object] = [fts_query]
|
|
114
|
+
source_filter = ""
|
|
115
|
+
if source_type:
|
|
116
|
+
source_filter = "AND d.source_type = ?"
|
|
117
|
+
params.append(source_type)
|
|
118
|
+
workspace_filter = ""
|
|
119
|
+
if workspace:
|
|
120
|
+
workspace_filter = "AND d.workspace = ?"
|
|
121
|
+
params.append(workspace.as_posix())
|
|
122
|
+
|
|
123
|
+
return conn.execute(
|
|
124
|
+
f"""
|
|
125
|
+
SELECT
|
|
126
|
+
d.id,
|
|
127
|
+
c.id AS chunk_id,
|
|
128
|
+
d.source_type,
|
|
129
|
+
d.source_path,
|
|
130
|
+
d.workspace,
|
|
131
|
+
d.title,
|
|
132
|
+
c.heading,
|
|
133
|
+
c.start_line,
|
|
134
|
+
c.end_line,
|
|
135
|
+
rank AS score,
|
|
136
|
+
snippet(chunks_fts, 3, '[', ']', '...', 28) AS snippet
|
|
137
|
+
FROM chunks_fts
|
|
138
|
+
JOIN chunks c ON c.id = chunks_fts.chunk_id
|
|
139
|
+
JOIN documents d ON d.id = c.document_id
|
|
140
|
+
WHERE chunks_fts MATCH ?
|
|
141
|
+
AND rank MATCH 'bm25(0.0, 2.5, 1.8, 1.0, 0.25)'
|
|
142
|
+
{source_filter}
|
|
143
|
+
{workspace_filter}
|
|
144
|
+
ORDER BY rank ASC, c.id ASC
|
|
145
|
+
""",
|
|
146
|
+
params,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def escape_like(value: str) -> str:
|
|
151
|
+
"""Escape LIKE wildcards so the query is matched as literal text."""
|
|
152
|
+
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def run_like_query(
|
|
156
|
+
conn: sqlite3.Connection,
|
|
157
|
+
query: str,
|
|
158
|
+
source_type: str | None,
|
|
159
|
+
workspace: Path | None,
|
|
160
|
+
) -> Iterable[sqlite3.Row]:
|
|
161
|
+
like = f"%{escape_like(query.lower())}%"
|
|
162
|
+
params: list[object] = [like, like, like, like]
|
|
163
|
+
source_filter = ""
|
|
164
|
+
if source_type:
|
|
165
|
+
source_filter = "AND d.source_type = ?"
|
|
166
|
+
params.append(source_type)
|
|
167
|
+
workspace_filter = ""
|
|
168
|
+
if workspace:
|
|
169
|
+
workspace_filter = "AND d.workspace = ?"
|
|
170
|
+
params.append(workspace.as_posix())
|
|
171
|
+
params.extend([like, like])
|
|
172
|
+
|
|
173
|
+
return conn.execute(
|
|
174
|
+
f"""
|
|
175
|
+
SELECT
|
|
176
|
+
d.id,
|
|
177
|
+
c.id AS chunk_id,
|
|
178
|
+
d.source_type,
|
|
179
|
+
d.source_path,
|
|
180
|
+
d.workspace,
|
|
181
|
+
d.title,
|
|
182
|
+
c.heading,
|
|
183
|
+
c.start_line,
|
|
184
|
+
c.end_line,
|
|
185
|
+
999.0 AS score,
|
|
186
|
+
c.content AS snippet
|
|
187
|
+
FROM chunks c
|
|
188
|
+
JOIN documents d ON d.id = c.document_id
|
|
189
|
+
WHERE (
|
|
190
|
+
lower(d.title) LIKE ? ESCAPE '\\'
|
|
191
|
+
OR lower(c.heading) LIKE ? ESCAPE '\\'
|
|
192
|
+
OR lower(c.content) LIKE ? ESCAPE '\\'
|
|
193
|
+
OR lower(d.source_path) LIKE ? ESCAPE '\\'
|
|
194
|
+
)
|
|
195
|
+
{source_filter}
|
|
196
|
+
{workspace_filter}
|
|
197
|
+
ORDER BY
|
|
198
|
+
CASE
|
|
199
|
+
WHEN lower(d.title) LIKE ? ESCAPE '\\' THEN 0
|
|
200
|
+
WHEN lower(d.source_path) LIKE ? ESCAPE '\\' THEN 1
|
|
201
|
+
ELSE 2
|
|
202
|
+
END,
|
|
203
|
+
d.title ASC,
|
|
204
|
+
c.ordinal ASC
|
|
205
|
+
""",
|
|
206
|
+
params,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def row_to_result(row: sqlite3.Row, strategy: str, query: str) -> QueryResult:
|
|
211
|
+
snippet = row["snippet"]
|
|
212
|
+
if strategy == "like":
|
|
213
|
+
snippet = plain_snippet(snippet, query)
|
|
214
|
+
heading = row["heading"]
|
|
215
|
+
location = f"L{row['start_line']}-L{row['end_line']}"
|
|
216
|
+
citation = f"{row['source_path']}:{location}"
|
|
217
|
+
if heading:
|
|
218
|
+
citation = f"{row['source_path']}#{heading}:{location}"
|
|
219
|
+
return QueryResult(
|
|
220
|
+
id=row["id"],
|
|
221
|
+
chunk_id=row["chunk_id"],
|
|
222
|
+
source_type=row["source_type"],
|
|
223
|
+
source_path=row["source_path"],
|
|
224
|
+
workspace=row["workspace"],
|
|
225
|
+
title=row["title"],
|
|
226
|
+
heading=heading,
|
|
227
|
+
start_line=int(row["start_line"]),
|
|
228
|
+
end_line=int(row["end_line"]),
|
|
229
|
+
citation=citation,
|
|
230
|
+
score=float(row["score"]),
|
|
231
|
+
snippet=snippet,
|
|
232
|
+
strategy=strategy,
|
|
233
|
+
)
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .chunking import chunk_text
|
|
7
|
+
|
|
8
|
+
DEFAULT_DB_PATH = Path(".bam/memory.db")
|
|
9
|
+
SCHEMA_VERSION = 2
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def require_index(db_path: Path | str) -> Path:
|
|
13
|
+
"""Reject a missing index, because connecting would silently create an empty one."""
|
|
14
|
+
path = Path(db_path).expanduser()
|
|
15
|
+
if not path.exists():
|
|
16
|
+
raise FileNotFoundError(f"index does not exist: {path}; run bam build first")
|
|
17
|
+
return path
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def connect(db_path: Path | str = DEFAULT_DB_PATH) -> sqlite3.Connection:
|
|
21
|
+
path = Path(db_path).expanduser()
|
|
22
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
23
|
+
conn = sqlite3.connect(path)
|
|
24
|
+
conn.row_factory = sqlite3.Row
|
|
25
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
26
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
27
|
+
conn.execute("PRAGMA secure_delete=ON")
|
|
28
|
+
return conn
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def init_db(conn: sqlite3.Connection) -> None:
|
|
32
|
+
"""Create or transactionally migrate the index schema."""
|
|
33
|
+
version = schema_version(conn)
|
|
34
|
+
if version > SCHEMA_VERSION:
|
|
35
|
+
raise sqlite3.DatabaseError(
|
|
36
|
+
f"index schema {version} is newer than supported schema {SCHEMA_VERSION}"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
has_legacy_records = _table_exists(conn, "records")
|
|
40
|
+
if version == SCHEMA_VERSION and not has_legacy_records:
|
|
41
|
+
_enable_fts_secure_delete(conn)
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
started_transaction = not conn.in_transaction
|
|
45
|
+
if started_transaction:
|
|
46
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
47
|
+
try:
|
|
48
|
+
if has_legacy_records:
|
|
49
|
+
_migrate_v1_to_v2(conn)
|
|
50
|
+
else:
|
|
51
|
+
_create_v2_schema(conn)
|
|
52
|
+
conn.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
|
53
|
+
_enable_fts_secure_delete(conn)
|
|
54
|
+
if started_transaction:
|
|
55
|
+
conn.commit()
|
|
56
|
+
except Exception:
|
|
57
|
+
if started_transaction:
|
|
58
|
+
conn.rollback()
|
|
59
|
+
raise
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def schema_version(conn: sqlite3.Connection) -> int:
|
|
63
|
+
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def fts5_available(conn: sqlite3.Connection) -> bool:
|
|
67
|
+
try:
|
|
68
|
+
conn.execute(
|
|
69
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS temp._bam_fts5_probe USING fts5(x)"
|
|
70
|
+
)
|
|
71
|
+
conn.execute("DROP TABLE IF EXISTS temp._bam_fts5_probe")
|
|
72
|
+
return True
|
|
73
|
+
except sqlite3.DatabaseError:
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def clear_index(conn: sqlite3.Connection) -> None:
|
|
78
|
+
"""Clear index rows inside the caller's transaction."""
|
|
79
|
+
conn.execute("DELETE FROM chunks_fts")
|
|
80
|
+
conn.execute("DELETE FROM chunks")
|
|
81
|
+
conn.execute("DELETE FROM documents")
|
|
82
|
+
conn.execute("DELETE FROM index_metadata")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _create_v2_schema(conn: sqlite3.Connection) -> None:
|
|
86
|
+
statements = (
|
|
87
|
+
"""CREATE TABLE IF NOT EXISTS documents (
|
|
88
|
+
id TEXT PRIMARY KEY,
|
|
89
|
+
source_type TEXT NOT NULL,
|
|
90
|
+
source_path TEXT NOT NULL,
|
|
91
|
+
workspace TEXT NOT NULL,
|
|
92
|
+
title TEXT NOT NULL,
|
|
93
|
+
content TEXT NOT NULL,
|
|
94
|
+
source_hash TEXT NOT NULL,
|
|
95
|
+
content_hash TEXT NOT NULL,
|
|
96
|
+
metadata_json TEXT NOT NULL,
|
|
97
|
+
updated_at TEXT NOT NULL,
|
|
98
|
+
UNIQUE(workspace, source_type, source_path)
|
|
99
|
+
)""",
|
|
100
|
+
"""CREATE INDEX IF NOT EXISTS idx_documents_source_type
|
|
101
|
+
ON documents(source_type)""",
|
|
102
|
+
"""CREATE INDEX IF NOT EXISTS idx_documents_workspace
|
|
103
|
+
ON documents(workspace)""",
|
|
104
|
+
"""CREATE INDEX IF NOT EXISTS idx_documents_source_hash
|
|
105
|
+
ON documents(source_hash)""",
|
|
106
|
+
"""CREATE TABLE IF NOT EXISTS chunks (
|
|
107
|
+
id TEXT PRIMARY KEY,
|
|
108
|
+
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
109
|
+
heading TEXT NOT NULL,
|
|
110
|
+
heading_key TEXT NOT NULL,
|
|
111
|
+
ordinal INTEGER NOT NULL,
|
|
112
|
+
start_line INTEGER NOT NULL,
|
|
113
|
+
end_line INTEGER NOT NULL,
|
|
114
|
+
content TEXT NOT NULL,
|
|
115
|
+
content_hash TEXT NOT NULL,
|
|
116
|
+
UNIQUE(document_id, heading_key, ordinal)
|
|
117
|
+
)""",
|
|
118
|
+
"""CREATE INDEX IF NOT EXISTS idx_chunks_document_id
|
|
119
|
+
ON chunks(document_id)""",
|
|
120
|
+
"""CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
121
|
+
chunk_id UNINDEXED,
|
|
122
|
+
title,
|
|
123
|
+
heading,
|
|
124
|
+
content,
|
|
125
|
+
source_path,
|
|
126
|
+
tokenize = 'unicode61'
|
|
127
|
+
)""",
|
|
128
|
+
"""CREATE TABLE IF NOT EXISTS index_metadata (
|
|
129
|
+
singleton INTEGER PRIMARY KEY CHECK(singleton = 1),
|
|
130
|
+
config_fingerprint TEXT NOT NULL,
|
|
131
|
+
config_json TEXT NOT NULL,
|
|
132
|
+
built_at TEXT NOT NULL,
|
|
133
|
+
chunker_version INTEGER NOT NULL,
|
|
134
|
+
chunk_size INTEGER NOT NULL
|
|
135
|
+
)""",
|
|
136
|
+
"""CREATE VIEW IF NOT EXISTS records AS
|
|
137
|
+
SELECT
|
|
138
|
+
id, source_type, source_path, workspace, title, content,
|
|
139
|
+
content_hash, metadata_json, updated_at
|
|
140
|
+
FROM documents""",
|
|
141
|
+
)
|
|
142
|
+
for statement in statements:
|
|
143
|
+
conn.execute(statement)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _migrate_v1_to_v2(conn: sqlite3.Connection) -> None:
|
|
147
|
+
legacy_rows = conn.execute("SELECT * FROM records ORDER BY source_path").fetchall()
|
|
148
|
+
conn.execute("DROP TABLE IF EXISTS records_fts")
|
|
149
|
+
conn.execute("ALTER TABLE records RENAME TO legacy_records")
|
|
150
|
+
_create_v2_schema(conn)
|
|
151
|
+
|
|
152
|
+
for row in legacy_rows:
|
|
153
|
+
chunk = chunk_text(row["id"], row["content"], "", max_chars=0)[0]
|
|
154
|
+
conn.execute(
|
|
155
|
+
"""
|
|
156
|
+
INSERT INTO documents (
|
|
157
|
+
id, source_type, source_path, workspace, title, content,
|
|
158
|
+
source_hash, content_hash, metadata_json, updated_at
|
|
159
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
160
|
+
""",
|
|
161
|
+
(
|
|
162
|
+
row["id"],
|
|
163
|
+
row["source_type"],
|
|
164
|
+
row["source_path"],
|
|
165
|
+
row["workspace"],
|
|
166
|
+
row["title"],
|
|
167
|
+
row["content"],
|
|
168
|
+
"",
|
|
169
|
+
chunk.content_hash,
|
|
170
|
+
row["metadata_json"],
|
|
171
|
+
row["updated_at"],
|
|
172
|
+
),
|
|
173
|
+
)
|
|
174
|
+
conn.execute(
|
|
175
|
+
"""
|
|
176
|
+
INSERT INTO chunks (
|
|
177
|
+
id, document_id, heading, heading_key, ordinal,
|
|
178
|
+
start_line, end_line, content, content_hash
|
|
179
|
+
) VALUES (?, ?, '', 'document', 0, 1, ?, ?, ?)
|
|
180
|
+
""",
|
|
181
|
+
(
|
|
182
|
+
chunk.id,
|
|
183
|
+
row["id"],
|
|
184
|
+
chunk.end_line,
|
|
185
|
+
chunk.content,
|
|
186
|
+
chunk.content_hash,
|
|
187
|
+
),
|
|
188
|
+
)
|
|
189
|
+
conn.execute(
|
|
190
|
+
"""
|
|
191
|
+
INSERT INTO chunks_fts (chunk_id, title, heading, content, source_path)
|
|
192
|
+
VALUES (?, ?, '', ?, ?)
|
|
193
|
+
""",
|
|
194
|
+
(chunk.id, row["title"], chunk.content, row["source_path"]),
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
conn.execute("DROP TABLE legacy_records")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
201
|
+
row = conn.execute(
|
|
202
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
|
|
203
|
+
(table,),
|
|
204
|
+
).fetchone()
|
|
205
|
+
return row is not None
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _enable_fts_secure_delete(conn: sqlite3.Connection) -> None:
|
|
209
|
+
try:
|
|
210
|
+
conn.execute("INSERT INTO chunks_fts(chunks_fts, rank) VALUES('secure-delete', 1)")
|
|
211
|
+
except sqlite3.DatabaseError:
|
|
212
|
+
pass
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TextIO
|
|
7
|
+
|
|
8
|
+
from .query import query_memory
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def serve_stdio(
|
|
12
|
+
db_path: Path | str,
|
|
13
|
+
stdin: TextIO = sys.stdin,
|
|
14
|
+
stdout: TextIO = sys.stdout,
|
|
15
|
+
) -> int:
|
|
16
|
+
for line in stdin:
|
|
17
|
+
line = line.strip()
|
|
18
|
+
if not line:
|
|
19
|
+
continue
|
|
20
|
+
try:
|
|
21
|
+
request = json.loads(line)
|
|
22
|
+
query = str(request["query"])
|
|
23
|
+
limit = int(request.get("limit", 5))
|
|
24
|
+
source_type = request.get("source_type")
|
|
25
|
+
workspace = request.get("workspace")
|
|
26
|
+
results = [
|
|
27
|
+
result.to_dict()
|
|
28
|
+
for result in query_memory(
|
|
29
|
+
db_path,
|
|
30
|
+
query,
|
|
31
|
+
limit=limit,
|
|
32
|
+
source_type=source_type,
|
|
33
|
+
workspace=workspace,
|
|
34
|
+
)
|
|
35
|
+
]
|
|
36
|
+
response = {"ok": True, "query": query, "results": results}
|
|
37
|
+
except Exception as exc: # noqa: BLE001 - stdio server must report errors as data.
|
|
38
|
+
response = {"ok": False, "error": str(exc)}
|
|
39
|
+
stdout.write(json.dumps(response, ensure_ascii=False) + "\n")
|
|
40
|
+
stdout.flush()
|
|
41
|
+
return 0
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def plain_snippet(content: str, query: str, max_chars: int = 220) -> str:
|
|
7
|
+
if not content:
|
|
8
|
+
return ""
|
|
9
|
+
|
|
10
|
+
terms = [t for t in re.findall(r"[\w\u4e00-\u9fff]+", query, flags=re.UNICODE) if t]
|
|
11
|
+
lower = content.lower()
|
|
12
|
+
start = 0
|
|
13
|
+
for term in terms:
|
|
14
|
+
idx = lower.find(term.lower())
|
|
15
|
+
if idx >= 0:
|
|
16
|
+
start = max(0, idx - max_chars // 3)
|
|
17
|
+
break
|
|
18
|
+
|
|
19
|
+
snippet = content[start : start + max_chars].replace("\n", " ")
|
|
20
|
+
snippet = re.sub(r"\s+", " ", snippet).strip()
|
|
21
|
+
if start > 0:
|
|
22
|
+
snippet = "..." + snippet
|
|
23
|
+
if start + max_chars < len(content):
|
|
24
|
+
snippet += "..."
|
|
25
|
+
return snippet
|
|
26
|
+
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: boring-agent-memory
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: A boring, local-first BM25 memory layer for AI agents.
|
|
5
|
+
Author: Boring Agent Memory contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/hskl18/boring-agent-memory
|
|
8
|
+
Project-URL: Repository, https://github.com/hskl18/boring-agent-memory
|
|
9
|
+
Project-URL: Issues, https://github.com/hskl18/boring-agent-memory/issues
|
|
10
|
+
Keywords: agent-memory,bm25,sqlite,fts5,rag,local-first,retrieval,source-grounded,privacy-preserving
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
|
+
Classifier: Topic :: Database
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: embeddings
|
|
25
|
+
Requires-Dist: fastembed<0.9,>=0.8.0; extra == "embeddings"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# Boring Agent Memory
|
|
29
|
+
|
|
30
|
+
[](https://github.com/hskl18/boring-agent-memory/actions/workflows/tests.yml)
|
|
31
|
+
|
|
32
|
+
Local BM25 recall over the files your agent already trusts, with line citations and a check that each source still matches what was indexed.
|
|
33
|
+
|
|
34
|
+
**Status: alpha.** The CLI, Python API, and index schema may still change before 1.0.
|
|
35
|
+
|
|
36
|
+
Most agent memory failures are not failures of semantic search.
|
|
37
|
+
They are failures of trust: stale state, missing provenance, private data captured by default, and memory overriding the source of truth.
|
|
38
|
+
|
|
39
|
+
Boring Agent Memory indexes the docs, skills, runbooks, bug logs, and ADRs you choose with SQLite FTS5.
|
|
40
|
+
It returns source paths and snippets, and your agent reads the cited file before it acts.
|
|
41
|
+
It needs no hosted service, embedding API, vector database, model download, or account.
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
python -m pip install boring-agent-memory
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
It needs Python 3.11 or newer.
|
|
50
|
+
|
|
51
|
+
## Try It
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
mkdir notes
|
|
55
|
+
printf '# Deploys\n\nRoll back database migrations within 15 minutes.\n' > notes/deploys.md
|
|
56
|
+
|
|
57
|
+
bam build --include notes
|
|
58
|
+
bam query "roll back migrations"
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
```text
|
|
62
|
+
deploys.md score=-3e-06
|
|
63
|
+
/path/to/notes/deploys.md
|
|
64
|
+
# Deploys
|
|
65
|
+
|
|
66
|
+
[Roll] [back] database [migrations] within 15 minutes.
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Now change the file and check whether the index still matches it:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
printf '# Deploys\n\nRoll back within 30 minutes.\n' > notes/deploys.md
|
|
73
|
+
|
|
74
|
+
bam inspect notes/deploys.md
|
|
75
|
+
bam update --include notes
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`bam inspect` reports `content_hash_match: False` until `bam update` picks up the edit.
|
|
79
|
+
|
|
80
|
+
## Use Your Own Files
|
|
81
|
+
|
|
82
|
+
Point a config at the files you trust and save it as `memory.yaml`:
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
index_path: ~/.bam/memory.db
|
|
86
|
+
source_type: canonical_file
|
|
87
|
+
workspace: ~/project/app
|
|
88
|
+
|
|
89
|
+
include:
|
|
90
|
+
- ~/agent/skills
|
|
91
|
+
- ~/project/*/docs
|
|
92
|
+
|
|
93
|
+
exclude:
|
|
94
|
+
- "**/secrets/**"
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
bam build --config memory.yaml --json
|
|
99
|
+
bam update --config memory.yaml --dry-run --json
|
|
100
|
+
bam update --config memory.yaml --json
|
|
101
|
+
|
|
102
|
+
bam --db ~/.bam/memory.db query "rollback policy" --limit 5 --json
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
A database that does not exist is an error, so a mistyped path cannot look like an empty memory.
|
|
106
|
+
|
|
107
|
+
## Use It From An Agent
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from boring_agent_memory import memory_query
|
|
111
|
+
|
|
112
|
+
results = memory_query(
|
|
113
|
+
"rollback policy for database migrations",
|
|
114
|
+
limit=5,
|
|
115
|
+
source_type="canonical_file",
|
|
116
|
+
db_path="~/.bam/memory.db",
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
for result in results:
|
|
120
|
+
print(result.citation)
|
|
121
|
+
print(result.snippet)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Other processes can use `bam serve --stdio`, which speaks JSON lines. See the [CLI reference](https://github.com/hskl18/boring-agent-memory/blob/main/docs/cli.md).
|
|
125
|
+
|
|
126
|
+
Agent instructions to copy are in [docs/agent-prompt.md](https://github.com/hskl18/boring-agent-memory/blob/main/docs/agent-prompt.md), with setups for [Claude Code](https://github.com/hskl18/boring-agent-memory/blob/main/examples/claude_code.md), [Codex](https://github.com/hskl18/boring-agent-memory/blob/main/examples/codex.md), and [Hermes](https://github.com/hskl18/boring-agent-memory/blob/main/docs/hermes-integration.md).
|
|
127
|
+
|
|
128
|
+
## Evidence
|
|
129
|
+
|
|
130
|
+
On Benchmark v2's 12 synthetic queries, chunked BM25 finds the right lines first every time, while whole-document BM25 finds the right file 91% of the time and never the right lines.
|
|
131
|
+
These are deterministic fixture results, not production traffic or a broad semantic-memory claim.
|
|
132
|
+
See [Benchmark v2](https://github.com/hskl18/boring-agent-memory/blob/main/docs/benchmark-v2.md) and [why BM25 comes first](https://github.com/hskl18/boring-agent-memory/blob/main/docs/canonical-first-memory.md#why-bm25-first).
|
|
133
|
+
|
|
134
|
+
## What It Is Not
|
|
135
|
+
|
|
136
|
+
- Not a chatbot brain or an auto-capture memory system. It indexes only paths you name.
|
|
137
|
+
- Not a replacement for semantic search over vague queries.
|
|
138
|
+
- Not an authority over current files. Canonical files come first, BM25 recall second, model memory last.
|
|
139
|
+
- Not a hosted service or a vector database.
|
|
140
|
+
|
|
141
|
+
## Privacy
|
|
142
|
+
|
|
143
|
+
Secret-bearing paths are skipped and common secret patterns are redacted before indexing, but these are guardrails, not a formal secret scanner.
|
|
144
|
+
Read the [privacy model](https://github.com/hskl18/boring-agent-memory/blob/main/docs/privacy-model.md) before choosing what to index.
|
|
145
|
+
|
|
146
|
+
## Documentation
|
|
147
|
+
|
|
148
|
+
- [CLI reference](https://github.com/hskl18/boring-agent-memory/blob/main/docs/cli.md)
|
|
149
|
+
- [Configuration](https://github.com/hskl18/boring-agent-memory/blob/main/docs/configuration.md)
|
|
150
|
+
- [Python API](https://github.com/hskl18/boring-agent-memory/blob/main/docs/python-api.md)
|
|
151
|
+
- [Architecture](https://github.com/hskl18/boring-agent-memory/blob/main/docs/architecture.md)
|
|
152
|
+
- [Comparison with other memory tools](https://github.com/hskl18/boring-agent-memory/blob/main/docs/comparison.md)
|
|
153
|
+
- [Status and known limits](https://github.com/hskl18/boring-agent-memory/blob/main/docs/status-and-gaps.md)
|
|
154
|
+
- [Roadmap](https://github.com/hskl18/boring-agent-memory/blob/main/docs/roadmap.md)
|
|
155
|
+
|
|
156
|
+
## Contributing
|
|
157
|
+
|
|
158
|
+
See [CONTRIBUTING.md](https://github.com/hskl18/boring-agent-memory/blob/main/CONTRIBUTING.md), and read [SECURITY.md](https://github.com/hskl18/boring-agent-memory/blob/main/SECURITY.md) before sharing logs, configs, or memory fixtures.
|
|
159
|
+
|
|
160
|
+
## License
|
|
161
|
+
|
|
162
|
+
MIT
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
boring_agent_memory/__init__.py,sha256=BpzIw7PhQE0f_3mDCSvqfwW4cY8aX9gGvOyRGzhRdiY,277
|
|
2
|
+
boring_agent_memory/api.py,sha256=fnNirQFK1L_bXRPvzgJfvogVTymXC_FFVm3BeyFpZAk,543
|
|
3
|
+
boring_agent_memory/benchmark.py,sha256=EnyvZqIAzTQX7Ik09Y1001_B2A4meA6WjbcbFswqbQc,18377
|
|
4
|
+
boring_agent_memory/canonical.py,sha256=dibyz1-vQFt41hqPRtf2ovsEEmNe6K20hSDhiQG54dU,2260
|
|
5
|
+
boring_agent_memory/chunking.py,sha256=qIDlFbl-eDtD0GauzT9AeIiCNuYL6Vj_bRX-iDjJcmM,13514
|
|
6
|
+
boring_agent_memory/cli.py,sha256=pV4ylAVBKGU9W6wdQcm3pyYw5L7_AOp3QIauydKSvlc,12690
|
|
7
|
+
boring_agent_memory/config.py,sha256=m158Ib3l0u9gsE3w7iiuhrqdcYafRokeDPQkJP6f698,5404
|
|
8
|
+
boring_agent_memory/embeddings.py,sha256=bUicUeA6JZv5EVYNB92HJJiu5bj84AGxFjOsKYHR-F4,9234
|
|
9
|
+
boring_agent_memory/eval.py,sha256=WlVLLkiAHUBcuqOW4phXykCb8Y_QnCysmbeALsJOJVE,9865
|
|
10
|
+
boring_agent_memory/index.py,sha256=6cR7gDpxCvqdn9owFv30TMXLMOcJj95_B_wQ2kQLaEs,15076
|
|
11
|
+
boring_agent_memory/ingest.py,sha256=zl5B-rC81Mf1uk2C979tGJEQfKU8Jbd7_JTh7L4Shcs,4880
|
|
12
|
+
boring_agent_memory/privacy.py,sha256=9tz-xvK5FhOes-DHHbw0cu1tq5Px_0NygT8AKDwV7gg,2203
|
|
13
|
+
boring_agent_memory/query.py,sha256=uSkdj_wMJB2q__NrUQPbIhMCY9XiRyTdtINsCpcxVQU,6530
|
|
14
|
+
boring_agent_memory/schema.py,sha256=DtTDCwt0eSrW7qUPIYMQqY5LRLQN972dy6RpdfXxOm8,7084
|
|
15
|
+
boring_agent_memory/server.py,sha256=wEFzrKpyzMxmjcQ4RKzI-JABsl99WAYhHnadUVUi8co,1233
|
|
16
|
+
boring_agent_memory/snippets.py,sha256=CKt5K3Kd-xXw78A0kl9WeyB9EAkT097AN0QguUzTNls,700
|
|
17
|
+
boring_agent_memory-0.2.2.dist-info/licenses/LICENSE,sha256=fsBCl4I4dL3u79HotmimZWbp-MzULu1zMc6QDUVTFZE,1090
|
|
18
|
+
boring_agent_memory-0.2.2.dist-info/METADATA,sha256=fnIFiWW6brD_0qdxiQ0vc6nfkMUBXERAma3qZ8b8NmU,6396
|
|
19
|
+
boring_agent_memory-0.2.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
20
|
+
boring_agent_memory-0.2.2.dist-info/entry_points.txt,sha256=5YIq6c9FM_Jrtq5MmnM5YvOPd2nRC-ZNELtQSURUnfc,53
|
|
21
|
+
boring_agent_memory-0.2.2.dist-info/top_level.txt,sha256=X-HqElxuI6jmSCU7bPww8yLIWHROF9aEwN7kjHZlkGo,20
|
|
22
|
+
boring_agent_memory-0.2.2.dist-info/RECORD,,
|