syncmill 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syncmill/__init__.py +11 -0
- syncmill/agents.py +2064 -0
- syncmill/artifact_io.py +201 -0
- syncmill/artifacts.py +1946 -0
- syncmill/board.py +2060 -0
- syncmill/breaker.py +327 -0
- syncmill/cli.py +4986 -0
- syncmill/command_errors.py +32 -0
- syncmill/config.py +1118 -0
- syncmill/dashboard.py +935 -0
- syncmill/gates.py +179 -0
- syncmill/gateway.py +234 -0
- syncmill/git_env.py +99 -0
- syncmill/isolated_session.py +431 -0
- syncmill/matcher.py +127 -0
- syncmill/memory.py +1067 -0
- syncmill/memory_backend.py +1907 -0
- syncmill/memory_migration.py +962 -0
- syncmill/model_routing.py +482 -0
- syncmill/models.py +244 -0
- syncmill/monitor.py +775 -0
- syncmill/phoenix_exporter.py +339 -0
- syncmill/planning.py +512 -0
- syncmill/platform_support.py +14 -0
- syncmill/preflight.py +1129 -0
- syncmill/project.py +1067 -0
- syncmill/report.py +502 -0
- syncmill/review_candidates.py +107 -0
- syncmill/room_compare.py +2380 -0
- syncmill/room_raw.py +484 -0
- syncmill/room_viewer.py +604 -0
- syncmill/rooms.py +4044 -0
- syncmill/session.py +444 -0
- syncmill/strategies/__init__.py +82 -0
- syncmill/strategies/base.py +910 -0
- syncmill/strategies/compete.py +233 -0
- syncmill/strategies/council.py +429 -0
- syncmill/strategies/decompose.py +570 -0
- syncmill/strategies/pipeline.py +221 -0
- syncmill/strategies/route.py +130 -0
- syncmill/supervisor.py +1423 -0
- syncmill/teamfile.py +571 -0
- syncmill/telemetry_hook.py +231 -0
- syncmill/terminal.py +245 -0
- syncmill/tracer.py +548 -0
- syncmill/tui.py +3537 -0
- syncmill/tui_collaboration.py +499 -0
- syncmill/tui_jobs.py +1407 -0
- syncmill/tui_render.py +156 -0
- syncmill/workflow.py +3746 -0
- syncmill/worktree.py +1045 -0
- syncmill-0.2.0.dist-info/METADATA +853 -0
- syncmill-0.2.0.dist-info/RECORD +56 -0
- syncmill-0.2.0.dist-info/WHEEL +4 -0
- syncmill-0.2.0.dist-info/entry_points.txt +2 -0
- syncmill-0.2.0.dist-info/licenses/LICENSE +191 -0
|
@@ -0,0 +1,1907 @@
|
|
|
1
|
+
"""Backend seam for syncmill's long-term memory.
|
|
2
|
+
|
|
3
|
+
``MemoryBackend`` names the contract ``SyncmillMemory`` imposes on its
|
|
4
|
+
storage provider. Backends raise freely: ``SyncmillMemory`` owns graceful
|
|
5
|
+
degradation, and adapters must not add retries or fallback behavior.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import copy
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Iterable, Sequence
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from time import monotonic
|
|
22
|
+
from typing import Any, Protocol, TypedDict
|
|
23
|
+
from uuid import UUID, uuid4
|
|
24
|
+
|
|
25
|
+
from syncmill.config import MemoryConfig
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class WriteBlockedError(RuntimeError):
|
|
31
|
+
"""The backend's redaction guard blocked this content.
|
|
32
|
+
|
|
33
|
+
Distinct from ordinary backend failures so ``SyncmillMemory`` can skip the
|
|
34
|
+
direct-add fallback — the same guard runs on the same content there and can
|
|
35
|
+
only re-block (duplicate audit noise, never a different outcome)."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class MemoryRow(TypedDict, total=False):
|
|
39
|
+
"""One search/get result row; ``namespace`` is required for get isolation."""
|
|
40
|
+
|
|
41
|
+
id: str
|
|
42
|
+
content: str
|
|
43
|
+
score: float
|
|
44
|
+
source: str
|
|
45
|
+
namespace: str
|
|
46
|
+
tags: list[str]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class BackendCapabilities:
|
|
51
|
+
"""Capabilities used by later backend/namespace-lifecycle phases."""
|
|
52
|
+
|
|
53
|
+
supports_sessions: bool = True
|
|
54
|
+
supports_tag_filter: bool = True
|
|
55
|
+
namespace_delete_is_atomic: bool = True
|
|
56
|
+
verbatim_storage: bool = True
|
|
57
|
+
# Whether search_multi is one native merged-ranking call (memtomem's
|
|
58
|
+
# comma-namespace union filter) vs an adapter-side per-namespace loop.
|
|
59
|
+
native_multi_namespace_search: bool = True
|
|
60
|
+
# Whether share_chunk records structural provenance (a chunk_links row)
|
|
61
|
+
# beside the shared-from tag, or the tag alone.
|
|
62
|
+
share_provenance_links: bool = True
|
|
63
|
+
namespace_pattern: re.Pattern[str] = field(
|
|
64
|
+
default_factory=lambda: re.compile(r"^[A-Za-z0-9._-]+(?::[A-Za-z0-9._-]+)*$")
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class MemoryBackend(Protocol):
|
|
69
|
+
"""Async, raise-freely storage contract for ``SyncmillMemory``.
|
|
70
|
+
|
|
71
|
+
File-backed adapters may additionally expose ``purge_namespace(ns) -> int``:
|
|
72
|
+
return only after deleting owned sources, rows and local cached results;
|
|
73
|
+
raise on an unprovable/partial purge. SyncmillMemory discovers this optional
|
|
74
|
+
async capability without requiring it from fileless or third-party adapters.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def capabilities(self) -> BackendCapabilities: ...
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def agent_namespace_prefix(self) -> str: ...
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def shared_namespace(self) -> str: ...
|
|
85
|
+
|
|
86
|
+
def validate_namespace(self, ns: str) -> bool: ...
|
|
87
|
+
|
|
88
|
+
async def add(self, content: str, *, tags: list[str], namespace: str) -> None: ...
|
|
89
|
+
|
|
90
|
+
async def search(
|
|
91
|
+
self,
|
|
92
|
+
query: str,
|
|
93
|
+
*,
|
|
94
|
+
namespace: str,
|
|
95
|
+
top_k: int,
|
|
96
|
+
tag_filter: str | None = None,
|
|
97
|
+
) -> list[MemoryRow]: ...
|
|
98
|
+
|
|
99
|
+
async def search_multi(
|
|
100
|
+
self,
|
|
101
|
+
query: str,
|
|
102
|
+
*,
|
|
103
|
+
namespaces: Sequence[str],
|
|
104
|
+
top_k: int,
|
|
105
|
+
tag_filter: str | None = None,
|
|
106
|
+
) -> list[MemoryRow]: ...
|
|
107
|
+
|
|
108
|
+
async def add_returning_ids(
|
|
109
|
+
self, content: str, *, tags: list[str], namespace: str
|
|
110
|
+
) -> list[str]:
|
|
111
|
+
"""Like ``add`` but returns every freshly indexed chunk id (a long
|
|
112
|
+
content can split into several chunks). Empty when the store minted no
|
|
113
|
+
fresh id (e.g. a content-hash dedup). May raise ``WriteBlockedError``."""
|
|
114
|
+
...
|
|
115
|
+
|
|
116
|
+
async def share_chunk(
|
|
117
|
+
self, chunk_id: str, *, target_namespace: str
|
|
118
|
+
) -> str | None: ...
|
|
119
|
+
|
|
120
|
+
async def register_namespace(
|
|
121
|
+
self, ns: str, *, description: str | None = None, color: str | None = None
|
|
122
|
+
) -> None:
|
|
123
|
+
"""Unconditional description/color metadata upsert (mem_agent_register
|
|
124
|
+
semantics; ``None`` leaves a field untouched). Metadata is a
|
|
125
|
+
memtomem-native concept — mem0/LangGraph adapters no-op, exactly as
|
|
126
|
+
their ``ensure_namespace`` already does."""
|
|
127
|
+
...
|
|
128
|
+
|
|
129
|
+
async def get(self, chunk_id: str) -> MemoryRow | None: ...
|
|
130
|
+
|
|
131
|
+
async def ensure_namespace(self, ns: str, *, description: str = "") -> None: ...
|
|
132
|
+
|
|
133
|
+
async def ensure_ready(self) -> None: ...
|
|
134
|
+
|
|
135
|
+
async def status(self) -> dict[str, Any]: ...
|
|
136
|
+
|
|
137
|
+
async def close(self) -> None: ...
|
|
138
|
+
|
|
139
|
+
async def count_namespace(self, ns: str) -> int: ...
|
|
140
|
+
|
|
141
|
+
async def delete_namespace(self, ns: str) -> int: ...
|
|
142
|
+
|
|
143
|
+
async def start_session(self, agent_id: str, namespace: str) -> str | None: ...
|
|
144
|
+
|
|
145
|
+
async def log_event(self, kind: str, text: str) -> None: ...
|
|
146
|
+
|
|
147
|
+
async def end_session(self, summary: str | None) -> None: ...
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# The mem_agent_share provenance tag. On a re-share the chain is collapsed to
|
|
151
|
+
# the immediate parent only (prior markers dropped) — mirroring memtomem's
|
|
152
|
+
# _build_shared_tags so syncmill copies read identically to MCP-tool shares.
|
|
153
|
+
_SHARED_FROM_PREFIX = "shared-from="
|
|
154
|
+
|
|
155
|
+
# Ceiling on how many source locks one purge will hold at once. Each is an open
|
|
156
|
+
# file descriptor held for the whole operation, so an unbounded namespace could
|
|
157
|
+
# exhaust the process limit mid-purge — with the board GC waiting on it, since
|
|
158
|
+
# collection runs after an outcome or an empty claim. Refusing loudly leaves the
|
|
159
|
+
# namespace collectable, which is the same fail-closed posture as every other
|
|
160
|
+
# unprovable outcome here. A board-run scratch namespace writes a handful of
|
|
161
|
+
# day files, so this is a guard against pathology, not a working limit.
|
|
162
|
+
_MAX_PURGE_SOURCE_LOCKS = 256
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _get_memtomem_atomic() -> Any:
|
|
166
|
+
"""Import memtomem's atomic file locking helpers with cross-version compatibility.
|
|
167
|
+
|
|
168
|
+
memtomem 0.5+ moved _atomic from memtomem.context._atomic to memtomem._atomic.
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
from memtomem import _atomic
|
|
172
|
+
|
|
173
|
+
return _atomic
|
|
174
|
+
except ImportError:
|
|
175
|
+
from memtomem.context import _atomic
|
|
176
|
+
|
|
177
|
+
return _atomic
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _utc_date_str() -> str:
|
|
181
|
+
"""Daily memory-file name, matching MemtomemStore.add's target choice."""
|
|
182
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _user_base(config: Any) -> Path:
|
|
186
|
+
"""The user-tier base every syncmill write and purge is anchored to.
|
|
187
|
+
|
|
188
|
+
No ``allow_project_tier``: that flag is for a caller with its own consent
|
|
189
|
+
gate for git-tracked (``project_shared``) writes, and syncmill writes
|
|
190
|
+
unattended with none — so a base that is a registered project tier is
|
|
191
|
+
refused, the posture upstream applies to its derived-target writers.
|
|
192
|
+
"""
|
|
193
|
+
from memtomem.memory_scope import require_user_base
|
|
194
|
+
|
|
195
|
+
return require_user_base(
|
|
196
|
+
config.indexing.memory_dirs, config.indexing.project_memory_dirs
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _target_tier(config: Any, path: Path) -> str:
|
|
201
|
+
"""The ADR-0011 tier memtomem gates and labels ``path`` with.
|
|
202
|
+
|
|
203
|
+
Upstream's own resolver, so this boundary and memtomem's write gate can
|
|
204
|
+
never disagree about a path's tier. It raises for a path whose owning tier
|
|
205
|
+
and indexed tier differ, or that sits in an unregistered project tree.
|
|
206
|
+
"""
|
|
207
|
+
from memtomem.integrations.langgraph import _resolve_target_scope
|
|
208
|
+
|
|
209
|
+
return _resolve_target_scope(
|
|
210
|
+
path.expanduser().resolve(),
|
|
211
|
+
config.indexing.memory_dirs,
|
|
212
|
+
config.indexing.project_memory_dirs,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _read_only_roots(config: Any) -> set[Path]:
|
|
217
|
+
"""Index roots memtomem reads but may never rewrite (``read_only_memory_dirs``)."""
|
|
218
|
+
return {
|
|
219
|
+
Path(d).expanduser().resolve()
|
|
220
|
+
for d in getattr(config.indexing, "read_only_memory_dirs", ()) or ()
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _refuse_unwritable_target(comp: Any, target: Path) -> None:
|
|
225
|
+
"""Refuse a day file this unattended writer must not append to.
|
|
226
|
+
|
|
227
|
+
Mirrors the checks memtomem 0.6's own add makes on its derived target,
|
|
228
|
+
narrowed to what syncmill can do without asking anyone: a symlinked day
|
|
229
|
+
file (syncmill never makes one, and following it can land the append in
|
|
230
|
+
a tier the base check never saw), a target outside the user tier, a
|
|
231
|
+
read-only source, or one the indexer excludes (it would take the append
|
|
232
|
+
and index nothing). Raised errors degrade the write like any backend
|
|
233
|
+
failure.
|
|
234
|
+
"""
|
|
235
|
+
if target.is_symlink():
|
|
236
|
+
raise RuntimeError(f"refusing to append to a symlinked day file: {target}")
|
|
237
|
+
tier = _target_tier(comp.config, target)
|
|
238
|
+
if tier != "user":
|
|
239
|
+
raise RuntimeError(
|
|
240
|
+
f"refusing an unattended write into the {tier} tier: {target}"
|
|
241
|
+
)
|
|
242
|
+
if comp.index_engine.is_read_only_source(target):
|
|
243
|
+
raise RuntimeError(f"refusing to append to a read-only source: {target}")
|
|
244
|
+
if comp.index_engine.is_excluded(target):
|
|
245
|
+
raise RuntimeError(f"refusing to append to an excluded source: {target}")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _namespace_lock_path(base: Path, ns: str) -> Path:
|
|
249
|
+
"""syncmill's own per-namespace lock, held ABOVE memtomem's per-file one.
|
|
250
|
+
|
|
251
|
+
A purge has to exclude writers to a NAMESPACE, and memtomem's locks are
|
|
252
|
+
per-FILE (its L2 sidecar), so the natural move — lock the file a writer is
|
|
253
|
+
about to touch — cannot work: the writer derives its target BEFORE taking
|
|
254
|
+
any lock, so a process suspended between those two steps (a stopped
|
|
255
|
+
process, a descheduled thread, a slow guard) resumes holding a target for
|
|
256
|
+
an arbitrarily old date. Enumerating candidate dates cannot fix that, since
|
|
257
|
+
there is no bound on how long a writer may hold a stale target. This is the
|
|
258
|
+
same shape as the bundle-path defect in the workflow store: a paused
|
|
259
|
+
process defeats any check-then-act, so the exclusion has to be on the thing
|
|
260
|
+
both sides agree about — here the namespace — and it has to be taken before
|
|
261
|
+
the racing side computes anything.
|
|
262
|
+
|
|
263
|
+
So this lock is keyed by (memory base, namespace) and nothing else, and the
|
|
264
|
+
write path takes it before it reads the clock. Being date-free is the whole
|
|
265
|
+
point: no argument about rollovers is required, and none is made.
|
|
266
|
+
|
|
267
|
+
**Lock ordering.** This sits ABOVE memtomem's documented memory-file domain
|
|
268
|
+
(``context/_atomic.py``, L0-L4): a holder may go on to take an L2 sidecar,
|
|
269
|
+
never the reverse. Only syncmill takes it, so that order is enforceable by
|
|
270
|
+
inspection of this module alone.
|
|
271
|
+
|
|
272
|
+
It lives beside the day files rather than in syncmill's state root because
|
|
273
|
+
the writer must already be able to write there, so this adds no new failure
|
|
274
|
+
mode to an advisory write path — an unwritable state root would otherwise
|
|
275
|
+
turn every memory write into an error. The leading dot matches memtomem's
|
|
276
|
+
own sidecar convention, and the name can never satisfy ``_is_day_file``, so
|
|
277
|
+
the sweep does not treat it as a source file.
|
|
278
|
+
"""
|
|
279
|
+
digest = hashlib.sha256(f"{os.path.realpath(base)}\0{ns}".encode()).hexdigest()[:32]
|
|
280
|
+
return base / f".syncmill-ns-{digest}.lock"
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# ASCII digits only: `\d` also matches other Unicode decimal digits, so a
|
|
284
|
+
# full-width `2026-08-01` would pass a date the writer cannot produce.
|
|
285
|
+
_DAY_FILE_DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _is_day_file(name: str, suffix: str) -> bool:
|
|
289
|
+
"""Whether ``name`` is a day file for ``suffix`` — a DATE plus that suffix.
|
|
290
|
+
|
|
291
|
+
Suffix-matching alone is not enough at a destructive boundary: it accepts
|
|
292
|
+
any prefix, so an unrelated ``backup--{slug}-{digest}.md`` or
|
|
293
|
+
``notes.{ns}.md`` would be unlinked. The namespace digest proves which
|
|
294
|
+
namespace a day file belongs to, never that the file IS a day file.
|
|
295
|
+
|
|
296
|
+
The date must be one ``strftime("%Y-%m-%d")`` could have produced, shape
|
|
297
|
+
AND calendar: every name this accepts is one syncmill or memtomem wrote,
|
|
298
|
+
so anything a writer cannot emit — ``2026-99-99``, non-ASCII digits — is
|
|
299
|
+
somebody else's file that happens to end the same way.
|
|
300
|
+
"""
|
|
301
|
+
if not name.endswith(suffix):
|
|
302
|
+
return False
|
|
303
|
+
date_part = name[: -len(suffix)]
|
|
304
|
+
if not _DAY_FILE_DATE.fullmatch(date_part):
|
|
305
|
+
return False
|
|
306
|
+
try:
|
|
307
|
+
datetime.strptime(date_part, "%Y-%m-%d")
|
|
308
|
+
except ValueError:
|
|
309
|
+
return False
|
|
310
|
+
return True
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _index_roots(config: Any, base: Path) -> list[Path]:
|
|
314
|
+
"""Every directory the indexer treats as a root, user tier first.
|
|
315
|
+
|
|
316
|
+
Sweeping only the user tier leaves a native day file under a registered
|
|
317
|
+
project root on disk, where the next pass re-indexes it exactly like a
|
|
318
|
+
user-tier one — so the namespace the caller was told had been purged comes
|
|
319
|
+
back. ``all_index_roots`` is upstream's own single source of truth for
|
|
320
|
+
consumers that must act on every indexable file.
|
|
321
|
+
|
|
322
|
+
Every root is verified to be a readable directory and anything else RAISES,
|
|
323
|
+
including a missing ``all_index_roots`` (every pinned version has it, so its
|
|
324
|
+
absence is version skew, not an old-config shape to guess around). This
|
|
325
|
+
boundary fails CLOSED for the same reason the caller does: skipping a root
|
|
326
|
+
that is merely unreachable right now — an unmounted project tier — would
|
|
327
|
+
report a clean zero-file purge, let the caller tombstone the namespace, and
|
|
328
|
+
resurrect it when the root came back. An unverifiable root means the
|
|
329
|
+
namespace stays collectable and a later sweep retries it.
|
|
330
|
+
|
|
331
|
+
``base`` (the caller's already-resolved ``require_user_base``) leads so the
|
|
332
|
+
two never fork on how the user tier is resolved, but it is checked like any
|
|
333
|
+
other root.
|
|
334
|
+
"""
|
|
335
|
+
all_roots = getattr(config.indexing, "all_index_roots", None)
|
|
336
|
+
if not callable(all_roots):
|
|
337
|
+
raise RuntimeError(
|
|
338
|
+
"memtomem config exposes no all_index_roots(); cannot prove every "
|
|
339
|
+
"index root was swept"
|
|
340
|
+
)
|
|
341
|
+
roots: dict[Path, None] = dict.fromkeys(
|
|
342
|
+
[base, *(Path(e).expanduser().resolve() for e in all_roots())]
|
|
343
|
+
)
|
|
344
|
+
for root in roots:
|
|
345
|
+
if not root.is_dir() or not os.access(root, os.R_OK | os.X_OK):
|
|
346
|
+
raise RuntimeError(
|
|
347
|
+
f"index root is not a readable directory: {root}; cannot prove "
|
|
348
|
+
"its sources were swept"
|
|
349
|
+
)
|
|
350
|
+
return list(roots)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _namespace_day_file_suffixes(config: Any, ns: str) -> list[str]:
|
|
354
|
+
"""The day-file suffixes that identify ``ns``, both write shapes.
|
|
355
|
+
|
|
356
|
+
``{date}.{ns}.md`` is what this backend's replicated add flow writes;
|
|
357
|
+
``{date}--{slug}-{digest}.md`` is what memtomem (0.4+) natively writes for the
|
|
358
|
+
same namespace. The native suffix comes from upstream's own
|
|
359
|
+
``day_file_name``, so an upstream rename breaks a test rather than silently
|
|
360
|
+
stranding files.
|
|
361
|
+
"""
|
|
362
|
+
from memtomem.memory_scope import day_file_name
|
|
363
|
+
|
|
364
|
+
suffixes = [f".{ns.replace(':', '.')}.md"]
|
|
365
|
+
# day_file_name gives the DEFAULT namespace the bare "{date}.md" shape,
|
|
366
|
+
# leaving just ".md" for an empty date_str — a suffix every file in the
|
|
367
|
+
# store ends with. Only the digest-suffixed non-default shape is swept.
|
|
368
|
+
native_suffix = day_file_name(ns, config.namespace.default_namespace, date_str="")
|
|
369
|
+
if native_suffix != ".md":
|
|
370
|
+
suffixes.append(native_suffix)
|
|
371
|
+
return suffixes
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _namespace_source_paths(config: Any, ns: str) -> list[Path]:
|
|
375
|
+
"""Every day file belonging to ``ns``, across every index root.
|
|
376
|
+
|
|
377
|
+
The single matcher. ``_sweep_namespace_sources`` unlinks what this returns
|
|
378
|
+
and ``purge_namespace`` locks it first, so the two can never disagree about
|
|
379
|
+
which files a namespace owns — the matching is the destructive boundary.
|
|
380
|
+
"""
|
|
381
|
+
# Raises on an empty memory_dirs, which the contract wants surfaced
|
|
382
|
+
# rather than reported as a zero-file purge.
|
|
383
|
+
base = _user_base(config)
|
|
384
|
+
suffixes = _namespace_day_file_suffixes(config, ns)
|
|
385
|
+
found: list[Path] = []
|
|
386
|
+
for root in _index_roots(config, base):
|
|
387
|
+
# scandir, not glob: pathlib SWALLOWS a scan-time OSError and yields
|
|
388
|
+
# nothing, so an unreadable root would look like an empty one and
|
|
389
|
+
# report a clean purge over files it never managed to look at. The
|
|
390
|
+
# os.access preflight cannot settle this on its own — it can
|
|
391
|
+
# disagree with the real operation under ACLs, and the permissions
|
|
392
|
+
# can change between the check and the scan.
|
|
393
|
+
with os.scandir(root) as entries:
|
|
394
|
+
for entry in entries:
|
|
395
|
+
if any(_is_day_file(entry.name, suffix) for suffix in suffixes):
|
|
396
|
+
found.append(Path(entry.path))
|
|
397
|
+
return found
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class UnprovablePurgeError(RuntimeError):
|
|
401
|
+
"""Purge is unacknowledged; partial deletion may have occurred and is retryable."""
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _check_purge_descriptor_budget(needed: int) -> None:
|
|
405
|
+
"""Leave room for SQLite, tracing and concurrent threads while holding L2.
|
|
406
|
+
|
|
407
|
+
The 256-source ceiling alone exceeds a stock macOS 256-fd soft limit.
|
|
408
|
+
Measure current headroom instead of raising the process-wide limit. This
|
|
409
|
+
is admission control, not a reservation: a later OS refusal still unwinds
|
|
410
|
+
every acquired lock before the exception reaches the GC caller.
|
|
411
|
+
"""
|
|
412
|
+
import resource
|
|
413
|
+
|
|
414
|
+
soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
415
|
+
if soft == resource.RLIM_INFINITY:
|
|
416
|
+
return
|
|
417
|
+
fd_root = (
|
|
418
|
+
Path("/proc/self/fd") if Path("/proc/self/fd").is_dir() else Path("/dev/fd")
|
|
419
|
+
)
|
|
420
|
+
available = max(0, soft - len(os.listdir(fd_root)) - 32)
|
|
421
|
+
if needed > available:
|
|
422
|
+
raise UnprovablePurgeError(
|
|
423
|
+
f"purge needs {needed} source files locked but only {available} "
|
|
424
|
+
"file descriptors are available with safety headroom; "
|
|
425
|
+
"raise the process file-descriptor limit or review the store; "
|
|
426
|
+
"no source files were deleted"
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _purgeable_sources(
|
|
431
|
+
config: Any, ns: str, roots: list[Path], paths: Iterable[Path]
|
|
432
|
+
) -> set[Path]:
|
|
433
|
+
"""The subset of ``paths`` this purge may lock and unlink for ``ns``.
|
|
434
|
+
|
|
435
|
+
Held to one rule whatever the origin: an exact ``{date}`` + namespace-suffix
|
|
436
|
+
name, directly inside one of the roots the indexer reads. The disk scan
|
|
437
|
+
satisfies it by construction; DATABASE paths have had no such check, and a
|
|
438
|
+
row's ``source_file`` is data, not a promise — the pre-upgrade residue is a
|
|
439
|
+
namespace-SHARED ``{date}.md`` this store deliberately never deletes, and a
|
|
440
|
+
corrupt row could name anything at all.
|
|
441
|
+
|
|
442
|
+
**Symlinks are refused outright, not skipped.** memtomem RESOLVES a path
|
|
443
|
+
before deriving its sidecar, so a link and its target have different lock
|
|
444
|
+
identities: the purge would take ``.link.lock`` while an indexer holds
|
|
445
|
+
``.target.lock``, exclude nothing, and still report success. Rather than
|
|
446
|
+
mirror upstream's resolution — coupling this boundary to an implementation
|
|
447
|
+
detail that can drift — a symlinked candidate makes the whole purge
|
|
448
|
+
unprovable. Board-run scratch files are never symlinks, so this is a loud
|
|
449
|
+
refusal for a store that is not what this code assumes, not a working path.
|
|
450
|
+
|
|
451
|
+
A DB path that fails the name/root rule is likewise not silently dropped:
|
|
452
|
+
it may be exactly the in-flight reader whose lock we needed. Ignoring it is
|
|
453
|
+
how rows come back after an acknowledged purge, so it too is unprovable.
|
|
454
|
+
"""
|
|
455
|
+
keep: set[Path] = set()
|
|
456
|
+
suffixes = _namespace_day_file_suffixes(config, ns)
|
|
457
|
+
read_only = _read_only_roots(config)
|
|
458
|
+
for path in paths:
|
|
459
|
+
if path.is_symlink():
|
|
460
|
+
raise UnprovablePurgeError(
|
|
461
|
+
f"source {path} for {ns} is a symlink, whose lock identity "
|
|
462
|
+
"differs from the one memtomem derives; refusing to purge"
|
|
463
|
+
)
|
|
464
|
+
# Refused, not skipped, and before anything is deleted: a skipped
|
|
465
|
+
# source would survive the purge with its rows gone, and the caller
|
|
466
|
+
# would tombstone a namespace that comes back on the next reindex.
|
|
467
|
+
if path.parent.resolve() in read_only:
|
|
468
|
+
raise UnprovablePurgeError(
|
|
469
|
+
f"source {path} for {ns} is in a read-only memory root, which "
|
|
470
|
+
"memtomem may index but never rewrite; refusing to purge"
|
|
471
|
+
)
|
|
472
|
+
try:
|
|
473
|
+
tier = _target_tier(config, path)
|
|
474
|
+
except Exception as exc:
|
|
475
|
+
raise UnprovablePurgeError(
|
|
476
|
+
f"cannot establish the tier of source {path} for {ns}: {exc}"
|
|
477
|
+
) from exc
|
|
478
|
+
if tier == "project_shared":
|
|
479
|
+
raise UnprovablePurgeError(
|
|
480
|
+
f"source {path} for {ns} is in a git-tracked project_shared tier; "
|
|
481
|
+
"deleting it needs a consent syncmill cannot ask for unattended"
|
|
482
|
+
)
|
|
483
|
+
if path.parent in roots and any(
|
|
484
|
+
_is_day_file(path.name, suffix) for suffix in suffixes
|
|
485
|
+
):
|
|
486
|
+
keep.add(path)
|
|
487
|
+
continue
|
|
488
|
+
raise UnprovablePurgeError(
|
|
489
|
+
f"the database attributes {path} to {ns}, but it is not a day file "
|
|
490
|
+
"inside an index root; it cannot be locked, so this purge cannot "
|
|
491
|
+
"prove it removed everything"
|
|
492
|
+
)
|
|
493
|
+
return keep
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _sweep_namespace_sources(config: Any, ns: str) -> int:
|
|
497
|
+
"""Unlink every day file belonging to ``ns``, across every index root."""
|
|
498
|
+
removed = 0
|
|
499
|
+
roots = _index_roots(config, _user_base(config))
|
|
500
|
+
for path in _purgeable_sources(config, ns, roots, _namespace_source_paths(config, ns)):
|
|
501
|
+
os.unlink(path)
|
|
502
|
+
removed += 1
|
|
503
|
+
return removed
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _entry_title() -> str:
|
|
507
|
+
"""A heading that is unique per write.
|
|
508
|
+
|
|
509
|
+
Historical rationale: memtomem 0.3's ``append_entry`` defaulted to
|
|
510
|
+
``## Entry {now}`` at SECOND resolution, and two entries under the same
|
|
511
|
+
heading in one file are re-chunked into a SINGLE chunk — so a second write
|
|
512
|
+
inside the same second silently swallowed the first entry's text and
|
|
513
|
+
RE-MINTED its chunk id. That id had already been returned to the caller,
|
|
514
|
+
so the provenance share that follows looked the chunk up, found nothing,
|
|
515
|
+
and fell back to a direct write with the link quietly gone. Two rapid
|
|
516
|
+
writes to one namespace is not a corner case here: it is what a
|
|
517
|
+
pipeline/council/decompose handoff does.
|
|
518
|
+
|
|
519
|
+
memtomem 0.4's own default is now unique too (ms timestamp + full uuid);
|
|
520
|
+
syncmill keeps supplying an explicit title of the same strength because
|
|
521
|
+
the deterministic shape is pinned by hermetic tests and must not drift
|
|
522
|
+
with the upstream default. It stays a plain ``## …`` heading, which is all
|
|
523
|
+
the chunker requires, and only the heading changes — the ``> created:``
|
|
524
|
+
line memtomem's own parsers read keeps its second resolution.
|
|
525
|
+
"""
|
|
526
|
+
stamp = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
|
527
|
+
return f"Entry {stamp} {uuid4().hex}"
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _shared_tags(tags: Sequence[str], source_id: str) -> list[str]:
|
|
531
|
+
kept = [t for t in tags if not t.startswith(_SHARED_FROM_PREFIX)]
|
|
532
|
+
kept.append(f"{_SHARED_FROM_PREFIX}{source_id}")
|
|
533
|
+
return kept
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _merge_rows(row_lists: Iterable[list[MemoryRow]], top_k: int) -> list[MemoryRow]:
|
|
537
|
+
"""Adapter-side multi-namespace merge: dedupe by id keeping the highest
|
|
538
|
+
score, sort by score descending, truncate. The fallback for backends
|
|
539
|
+
without a native union-namespace search."""
|
|
540
|
+
merged: dict[str, MemoryRow] = {}
|
|
541
|
+
for rows in row_lists:
|
|
542
|
+
for row in rows:
|
|
543
|
+
prior = merged.get(row["id"])
|
|
544
|
+
if prior is None or row.get("score", 0.0) > prior.get("score", 0.0):
|
|
545
|
+
merged[row["id"]] = row
|
|
546
|
+
return sorted(merged.values(), key=lambda row: -row.get("score", 0.0))[:top_k]
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
_FENCE_RE = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})(.*)$")
|
|
550
|
+
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$")
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _propagate_entry_tags_to_headings(content: str, tags: list[str]) -> str:
|
|
554
|
+
"""Propagate entry tags to interior markdown headings.
|
|
555
|
+
|
|
556
|
+
Memtomem daily files chunk by ATX headings matching `^(#{1,6})\\s+(.+)$`.
|
|
557
|
+
append_entry emits the `> tags:` blockquote only under the entry's
|
|
558
|
+
top heading. When an entry contains interior markdown headings,
|
|
559
|
+
memtomem splits the entry into multiple chunks, but subsequent chunks
|
|
560
|
+
lack a `> tags:` blockquote and receive empty tags (tags=()), making them
|
|
561
|
+
unreachable via `tag_filter` (e.g. `board-result:item-42`).
|
|
562
|
+
|
|
563
|
+
Attaching `> tags: [...]` directly beneath each interior heading
|
|
564
|
+
(outside fenced code blocks) ensures every resulting chunk inherits
|
|
565
|
+
the entry's tags while preserving the verbatim content and heading
|
|
566
|
+
structure without mutating text into bold labels.
|
|
567
|
+
"""
|
|
568
|
+
if not tags:
|
|
569
|
+
return content
|
|
570
|
+
tag_line = f"> tags: {json.dumps(list(tags), ensure_ascii=False)}"
|
|
571
|
+
lines = content.splitlines(keepends=True)
|
|
572
|
+
out: list[str] = []
|
|
573
|
+
fence_char: str | None = None
|
|
574
|
+
fence_len = 0
|
|
575
|
+
|
|
576
|
+
for line in lines:
|
|
577
|
+
line_no_ending = line.rstrip("\r\n")
|
|
578
|
+
fence_match = _FENCE_RE.match(line_no_ending)
|
|
579
|
+
|
|
580
|
+
if fence_char is None:
|
|
581
|
+
if fence_match:
|
|
582
|
+
run = fence_match.group(1)
|
|
583
|
+
fence_char = run[0]
|
|
584
|
+
fence_len = len(run)
|
|
585
|
+
out.append(line)
|
|
586
|
+
continue
|
|
587
|
+
else:
|
|
588
|
+
if (
|
|
589
|
+
fence_match
|
|
590
|
+
and fence_match.group(1)[0] == fence_char
|
|
591
|
+
and len(fence_match.group(1)) >= fence_len
|
|
592
|
+
and not fence_match.group(2).strip()
|
|
593
|
+
):
|
|
594
|
+
fence_char = None
|
|
595
|
+
fence_len = 0
|
|
596
|
+
out.append(line)
|
|
597
|
+
continue
|
|
598
|
+
|
|
599
|
+
out.append(line)
|
|
600
|
+
if _HEADING_RE.match(line_no_ending):
|
|
601
|
+
line_ending = "\r\n" if line.endswith("\r\n") else "\n"
|
|
602
|
+
if not line.endswith(("\r", "\n")):
|
|
603
|
+
out.append(line_ending)
|
|
604
|
+
out.append(f"{tag_line}{line_ending}\n")
|
|
605
|
+
|
|
606
|
+
return "".join(out)
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
class MemtomemBackend:
|
|
610
|
+
"""Default backend and the sole owner of memtomem-private coupling."""
|
|
611
|
+
|
|
612
|
+
def __init__(self, config: MemoryConfig):
|
|
613
|
+
# Keep imports lazy: import or construction failures are advisory and are
|
|
614
|
+
# caught by SyncmillMemory's existing construction boundary.
|
|
615
|
+
from memtomem.constants import AGENT_NAMESPACE_PREFIX, SHARED_NAMESPACE
|
|
616
|
+
|
|
617
|
+
try:
|
|
618
|
+
from memtomem.integrations.langgraph import MemtomemStore
|
|
619
|
+
except (ImportError, AttributeError):
|
|
620
|
+
from memtomem.integrations import MemtomemStore
|
|
621
|
+
|
|
622
|
+
self._agent_prefix = AGENT_NAMESPACE_PREFIX
|
|
623
|
+
self._shared_ns = SHARED_NAMESPACE
|
|
624
|
+
overrides: dict[str, dict[str, Any]] = {
|
|
625
|
+
"storage": {"sqlite_path": config.db_path}
|
|
626
|
+
}
|
|
627
|
+
if config.memory_dir is not None:
|
|
628
|
+
overrides["indexing"] = {"memory_dirs": [config.memory_dir]}
|
|
629
|
+
# Kept for status(): reading it back off the store would touch the
|
|
630
|
+
# private `_config_overrides`, and an upstream rename would then report
|
|
631
|
+
# version skew as a connection failure.
|
|
632
|
+
self._overrides = overrides
|
|
633
|
+
self._store = MemtomemStore(config_overrides=overrides)
|
|
634
|
+
self._caps = BackendCapabilities()
|
|
635
|
+
|
|
636
|
+
@property
|
|
637
|
+
def capabilities(self) -> BackendCapabilities:
|
|
638
|
+
return self._caps
|
|
639
|
+
|
|
640
|
+
@property
|
|
641
|
+
def agent_namespace_prefix(self) -> str:
|
|
642
|
+
return self._agent_prefix
|
|
643
|
+
|
|
644
|
+
@property
|
|
645
|
+
def shared_namespace(self) -> str:
|
|
646
|
+
return self._shared_ns
|
|
647
|
+
|
|
648
|
+
def validate_namespace(self, ns: str) -> bool:
|
|
649
|
+
return bool(self._caps.namespace_pattern.fullmatch(ns))
|
|
650
|
+
|
|
651
|
+
async def add(self, content: str, *, tags: list[str], namespace: str) -> None:
|
|
652
|
+
# The SAME flow as add_returning_ids (ids discarded), so both writes land
|
|
653
|
+
# in the namespace-suffixed daily file. Routing this through
|
|
654
|
+
# MemtomemStore.add instead would append to the namespace-SHARED
|
|
655
|
+
# `{date}.md`, which breaks namespace isolation in two ways. The markdown
|
|
656
|
+
# is the index's source of truth while delete_by_namespace only removes
|
|
657
|
+
# DB rows, so a GC'd `board-run:*` chunk survives in a file that a later
|
|
658
|
+
# `shared` write re-indexes — and, its rows now gone, its text is treated
|
|
659
|
+
# as new and stamped `shared`, leaking board-internal content into global
|
|
660
|
+
# knowledge. And because one file's chunks are content-hash keyed, an
|
|
661
|
+
# identical write to a second namespace is classified unchanged and
|
|
662
|
+
# silently dropped.
|
|
663
|
+
await self.add_returning_ids(content, tags=tags, namespace=namespace)
|
|
664
|
+
|
|
665
|
+
async def search(
|
|
666
|
+
self,
|
|
667
|
+
query: str,
|
|
668
|
+
*,
|
|
669
|
+
namespace: str,
|
|
670
|
+
top_k: int,
|
|
671
|
+
tag_filter: str | None = None,
|
|
672
|
+
) -> list[MemoryRow]:
|
|
673
|
+
if tag_filter is not None:
|
|
674
|
+
return await self._store.search(
|
|
675
|
+
query,
|
|
676
|
+
namespace=namespace,
|
|
677
|
+
top_k=top_k,
|
|
678
|
+
tag_filter=tag_filter,
|
|
679
|
+
)
|
|
680
|
+
return await self._store.search(query, namespace=namespace, top_k=top_k)
|
|
681
|
+
|
|
682
|
+
async def search_multi(
|
|
683
|
+
self,
|
|
684
|
+
query: str,
|
|
685
|
+
*,
|
|
686
|
+
namespaces: Sequence[str],
|
|
687
|
+
top_k: int,
|
|
688
|
+
tag_filter: str | None = None,
|
|
689
|
+
) -> list[MemoryRow]:
|
|
690
|
+
# One native merged-ranking call: NamespaceFilter.parse treats a comma
|
|
691
|
+
# list as a union filter — the same shape mem_agent_search builds. The
|
|
692
|
+
# namespace charset pattern forbids commas, but callers (search_context
|
|
693
|
+
# builds names from free-form agent ids) don't all validate — so enforce
|
|
694
|
+
# it here, or a comma-bearing name would silently widen the union
|
|
695
|
+
# filter to namespaces the caller never asked for. Raising is fine on
|
|
696
|
+
# this contract; SyncmillMemory degrades per call.
|
|
697
|
+
if any("," in ns for ns in namespaces):
|
|
698
|
+
raise ValueError(f"namespace contains a comma: {list(namespaces)!r}")
|
|
699
|
+
namespace = ",".join(namespaces)
|
|
700
|
+
if tag_filter is not None:
|
|
701
|
+
return await self._store.search(
|
|
702
|
+
query,
|
|
703
|
+
namespace=namespace,
|
|
704
|
+
top_k=top_k,
|
|
705
|
+
tag_filter=tag_filter,
|
|
706
|
+
)
|
|
707
|
+
return await self._store.search(query, namespace=namespace, top_k=top_k)
|
|
708
|
+
|
|
709
|
+
async def add_returning_ids(
|
|
710
|
+
self, content: str, *, tags: list[str], namespace: str
|
|
711
|
+
) -> list[str]:
|
|
712
|
+
# MemtomemStore.add discards IndexingStats.new_chunk_ids, so replicate
|
|
713
|
+
# its short write flow (guard → lock → mix-refusal → append_entry →
|
|
714
|
+
# index_file(lock_held=True)) to keep the ids. Lazy imports:
|
|
715
|
+
# construction failures stay advisory.
|
|
716
|
+
atomic = _get_memtomem_atomic()
|
|
717
|
+
_CRUD_SIDECAR_LOCK_BUDGET_S = atomic._CRUD_SIDECAR_LOCK_BUDGET_S
|
|
718
|
+
async_file_lock = atomic.async_file_lock
|
|
719
|
+
from memtomem.privacy import enforce_write_guard
|
|
720
|
+
|
|
721
|
+
comp = await self._store._ensure_init()
|
|
722
|
+
guard = enforce_write_guard(
|
|
723
|
+
content,
|
|
724
|
+
surface="syncmill_add",
|
|
725
|
+
audit_context={"namespace": namespace},
|
|
726
|
+
)
|
|
727
|
+
# `blocked_project_shared` (memtomem 0.6) is the hard refusal for a
|
|
728
|
+
# git-tracked tier; unreachable while every syncmill target is `user`,
|
|
729
|
+
# but a decision this code does not name must never fall through as
|
|
730
|
+
# an allow.
|
|
731
|
+
if guard.decision in ("blocked", "blocked_project_shared"):
|
|
732
|
+
raise WriteBlockedError(
|
|
733
|
+
f"memtomem redaction guard refused the write ({guard.decision})"
|
|
734
|
+
)
|
|
735
|
+
base = _user_base(comp.config)
|
|
736
|
+
# BEFORE the clock is read. `purge_namespace` holds this same lock for
|
|
737
|
+
# the whole of its sweep → row delete → verify, so a write either
|
|
738
|
+
# completes entirely before a purge begins or waits for it — never
|
|
739
|
+
# lands a file whose rows the purge then takes. Deriving the target
|
|
740
|
+
# first and locking only that file cannot give this: a writer suspended
|
|
741
|
+
# between the two resumes with a stale target the purge never knew to
|
|
742
|
+
# lock (see `_namespace_lock_path`).
|
|
743
|
+
async with async_file_lock(
|
|
744
|
+
_namespace_lock_path(base, namespace),
|
|
745
|
+
timeout=_CRUD_SIDECAR_LOCK_BUDGET_S,
|
|
746
|
+
):
|
|
747
|
+
return await self._locked_append(
|
|
748
|
+
comp, base, content, tags=tags, namespace=namespace
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
async def _locked_append(
|
|
752
|
+
self,
|
|
753
|
+
comp: Any,
|
|
754
|
+
base: Path,
|
|
755
|
+
content: str,
|
|
756
|
+
*,
|
|
757
|
+
tags: list[str],
|
|
758
|
+
namespace: str,
|
|
759
|
+
) -> list[str]:
|
|
760
|
+
"""The append itself, with this namespace's writers already excluded."""
|
|
761
|
+
atomic = _get_memtomem_atomic()
|
|
762
|
+
_CRUD_SIDECAR_LOCK_BUDGET_S = atomic._CRUD_SIDECAR_LOCK_BUDGET_S
|
|
763
|
+
memory_lock_path = atomic.memory_lock_path
|
|
764
|
+
async_file_lock = atomic.async_file_lock
|
|
765
|
+
from memtomem.memory_scope import namespace_mix_refusal
|
|
766
|
+
from memtomem.tools.memory_writer import append_entry
|
|
767
|
+
|
|
768
|
+
# A NAMESPACE-SUFFIXED daily file, deliberately diverging from
|
|
769
|
+
# MemtomemStore.add's shared `{date}.md`: under memtomem's default
|
|
770
|
+
# config, re-indexing one file that holds the same content-hash twice
|
|
771
|
+
# collapses the chunks to the LAST write — so the two-step share
|
|
772
|
+
# (identical content in agent-ns then `shared`, the mem_agent_share
|
|
773
|
+
# copy) would DELETE the private original and orphan the provenance
|
|
774
|
+
# link (a real FOREIGN KEY failure the opt-in real smoke caught).
|
|
775
|
+
# Separate files per namespace keep both chunks alive. The ':'→'.'
|
|
776
|
+
# mapping could collide only for a namespace literally containing the
|
|
777
|
+
# dotted form (e.g. "agent-runtime.claude") — unreachable with
|
|
778
|
+
# syncmill's fixed namespace vocabulary.
|
|
779
|
+
target = base / f"{_utc_date_str()}.{namespace.replace(':', '.')}.md"
|
|
780
|
+
# An EXPLICIT per-write heading, never append_entry's second-resolution
|
|
781
|
+
# default: two entries sharing a heading in one file re-chunk into one,
|
|
782
|
+
# swallowing the earlier text and re-minting the id already handed to
|
|
783
|
+
# the caller (see `_entry_title`).
|
|
784
|
+
#
|
|
785
|
+
# `format_entry_block` IGNORES the title when the content supplies its
|
|
786
|
+
# own `## ` heading, and in that branch it emits the tags blockquote
|
|
787
|
+
# BEFORE that heading — so such a payload gets a shared heading (the
|
|
788
|
+
# merge this guards against) AND loses its tags to the previous entry's
|
|
789
|
+
# section. syncmill's writers deliberately label themselves in bold
|
|
790
|
+
# instead (see memory.py's heading note); warn rather than mutate the
|
|
791
|
+
# content, since the backend contract is verbatim storage.
|
|
792
|
+
# `## ` ONLY, because that is the branch the pinned writer actually
|
|
793
|
+
# takes: memtomem's `format_entry_block` (0.4 through 0.6.8) tests
|
|
794
|
+
# `stripped.startswith("## ")`. Widening this to any ATX heading was
|
|
795
|
+
# wrong — calling the real formatter with `# H1` or `### H3` keeps the
|
|
796
|
+
# unique title AND the tags, so the warning would have described a
|
|
797
|
+
# failure that does not happen. If the upstream condition ever widens,
|
|
798
|
+
# widen this with it, against the real formatter.
|
|
799
|
+
if content.strip().startswith("## "):
|
|
800
|
+
logger.warning(
|
|
801
|
+
"memory payload starts with a '## ' heading, so its unique "
|
|
802
|
+
"per-entry heading and tags will be dropped by the writer; "
|
|
803
|
+
"label the payload in bold instead"
|
|
804
|
+
)
|
|
805
|
+
# memtomem's write-locking contract (since 0.4, its issue #2005): the append
|
|
806
|
+
# and its re-index must share one cross-process sidecar-lock span, or
|
|
807
|
+
# the watcher can index the freshly appended file FIRST — without the
|
|
808
|
+
# explicit namespace — and the agent-private chunk lands permanently in
|
|
809
|
+
# `default`, exposed to non-private searches with the provenance share
|
|
810
|
+
# broken (reproduced against the real 0.4 store). The mix-refusal read
|
|
811
|
+
# sits inside the same span for the same reason: a guard that inspects
|
|
812
|
+
# a file another writer may change before the append is decoration.
|
|
813
|
+
# `lock_held=True` skips index_file's nested sidecar acquire, which
|
|
814
|
+
# would otherwise self-deadlock (memtomem #1587). A lock timeout
|
|
815
|
+
# (another process holds it past the budget) raises TimeoutError, which
|
|
816
|
+
# SyncmillMemory degrades per call like any backend failure.
|
|
817
|
+
# Checked before the acquire as well as inside it: `memory_lock_path`
|
|
818
|
+
# resolves the target, so a symlinked day file would otherwise get its
|
|
819
|
+
# sidecar created wherever the link points. Inside the span the checks
|
|
820
|
+
# are authoritative, since the file can change until the lock is held.
|
|
821
|
+
_refuse_unwritable_target(comp, target)
|
|
822
|
+
# `memory_lock_path`, never `_lock_path_for`: memtomem's indexer keys
|
|
823
|
+
# the L2 sidecar on the RESOLVED path, and a lock keyed any other way
|
|
824
|
+
# would not exclude it (upstream #2130).
|
|
825
|
+
async with async_file_lock(
|
|
826
|
+
memory_lock_path(target), timeout=_CRUD_SIDECAR_LOCK_BUDGET_S
|
|
827
|
+
):
|
|
828
|
+
_refuse_unwritable_target(comp, target)
|
|
829
|
+
mix_err = await namespace_mix_refusal(
|
|
830
|
+
index_engine=comp.index_engine,
|
|
831
|
+
storage=comp.storage,
|
|
832
|
+
default_namespace=comp.config.namespace.default_namespace,
|
|
833
|
+
target=target,
|
|
834
|
+
effective_ns=namespace,
|
|
835
|
+
override_hint="a different target file",
|
|
836
|
+
)
|
|
837
|
+
if mix_err is not None:
|
|
838
|
+
# Unreachable with syncmill's per-namespace files short of
|
|
839
|
+
# legacy residue; refusing beats restamping stored rows.
|
|
840
|
+
raise RuntimeError(
|
|
841
|
+
f"memtomem namespace mix refused for {target}: {mix_err}"
|
|
842
|
+
)
|
|
843
|
+
prepared = _propagate_entry_tags_to_headings(content, list(tags))
|
|
844
|
+
append_entry(target, prepared, title=_entry_title(), tags=list(tags))
|
|
845
|
+
stats = await comp.index_engine.index_file(
|
|
846
|
+
target, namespace=namespace, already_scanned=True, lock_held=True
|
|
847
|
+
)
|
|
848
|
+
# MCP-write parity — see `add`.
|
|
849
|
+
comp.search_pipeline.invalidate_cache()
|
|
850
|
+
return [str(cid) for cid in getattr(stats, "new_chunk_ids", ()) or ()]
|
|
851
|
+
|
|
852
|
+
async def share_chunk(self, chunk_id: str, *, target_namespace: str) -> str | None:
|
|
853
|
+
# mem_agent_share semantics: COPY the chunk into the target namespace
|
|
854
|
+
# with a collapsed shared-from tag, then record a chunk_links row per
|
|
855
|
+
# copied chunk. NOTE: like MemtomemStore.get, the source lookup is NOT
|
|
856
|
+
# namespace-scoped — today's only caller feeds ids it just minted; a
|
|
857
|
+
# future promote-style surface must add a source-namespace check (the
|
|
858
|
+
# gateway `get_shared` lesson) before exposing this to untrusted ids.
|
|
859
|
+
source = await self._store.get(chunk_id)
|
|
860
|
+
if source is None:
|
|
861
|
+
return None
|
|
862
|
+
new_ids = await self.add_returning_ids(
|
|
863
|
+
source["content"],
|
|
864
|
+
tags=_shared_tags(source.get("tags", []), chunk_id),
|
|
865
|
+
namespace=target_namespace,
|
|
866
|
+
)
|
|
867
|
+
for new_id in new_ids:
|
|
868
|
+
# Link recording is best-effort (the mem_agent_share posture): the
|
|
869
|
+
# copy is durable, so a failed link must not report a failed share.
|
|
870
|
+
try:
|
|
871
|
+
storage = await self._storage()
|
|
872
|
+
await storage.add_chunk_link(
|
|
873
|
+
UUID(chunk_id),
|
|
874
|
+
UUID(new_id),
|
|
875
|
+
"shared",
|
|
876
|
+
target_namespace,
|
|
877
|
+
)
|
|
878
|
+
except Exception:
|
|
879
|
+
logger.warning(
|
|
880
|
+
"share provenance link failed; copy is durable", exc_info=True
|
|
881
|
+
)
|
|
882
|
+
return new_ids[0] if new_ids else None
|
|
883
|
+
|
|
884
|
+
async def register_namespace(
|
|
885
|
+
self, ns: str, *, description: str | None = None, color: str | None = None
|
|
886
|
+
) -> None:
|
|
887
|
+
# mem_agent_register semantics: unconditional upsert (None leaves the
|
|
888
|
+
# existing value untouched) — unlike ensure_namespace's create-if-missing.
|
|
889
|
+
await (await self._storage()).set_namespace_meta(
|
|
890
|
+
ns, description=description, color=color
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
async def get(self, chunk_id: str) -> MemoryRow | None:
|
|
894
|
+
return await self._store.get(chunk_id)
|
|
895
|
+
|
|
896
|
+
async def ensure_namespace(self, ns: str, *, description: str = "") -> None:
|
|
897
|
+
storage = await self._storage()
|
|
898
|
+
if await storage.get_namespace_meta(ns) is None:
|
|
899
|
+
await storage.set_namespace_meta(ns, description=description)
|
|
900
|
+
|
|
901
|
+
async def ensure_ready(self) -> None:
|
|
902
|
+
await self._store._ensure_init()
|
|
903
|
+
|
|
904
|
+
async def status(self) -> dict[str, Any]:
|
|
905
|
+
await self.ensure_ready()
|
|
906
|
+
# Our own overrides dict, not the store's private `_config_overrides` —
|
|
907
|
+
# an upstream rename of that attribute would surface as AttributeError,
|
|
908
|
+
# which SyncmillMemory.status reports as a CONNECTION failure (version
|
|
909
|
+
# skew misdiagnosed as an outage).
|
|
910
|
+
db_path = self._overrides.get("storage", {}).get("sqlite_path", "default")
|
|
911
|
+
info: dict[str, Any] = {"db_path": db_path}
|
|
912
|
+
try:
|
|
913
|
+
comp = getattr(self._store, "_components", None)
|
|
914
|
+
if comp is not None:
|
|
915
|
+
storage = getattr(comp, "storage", None)
|
|
916
|
+
if storage is not None and hasattr(storage, "get_stats"):
|
|
917
|
+
stats = await storage.get_stats()
|
|
918
|
+
if isinstance(stats, dict):
|
|
919
|
+
for k in (
|
|
920
|
+
"total_chunks",
|
|
921
|
+
"total_sources",
|
|
922
|
+
"source_files",
|
|
923
|
+
"dense_vectors",
|
|
924
|
+
):
|
|
925
|
+
if k in stats:
|
|
926
|
+
info[k] = stats[k]
|
|
927
|
+
cfg = getattr(comp, "config", None)
|
|
928
|
+
if cfg is not None:
|
|
929
|
+
emb = getattr(cfg, "embedding", None)
|
|
930
|
+
if emb is not None:
|
|
931
|
+
if hasattr(emb, "model"):
|
|
932
|
+
info["embedding_model"] = emb.model
|
|
933
|
+
if hasattr(emb, "provider"):
|
|
934
|
+
info["embedding_provider"] = emb.provider
|
|
935
|
+
except Exception:
|
|
936
|
+
logger.debug("memtomem status diagnostic probe omitted", exc_info=True)
|
|
937
|
+
return info
|
|
938
|
+
|
|
939
|
+
async def close(self) -> None:
|
|
940
|
+
await self._store.close()
|
|
941
|
+
|
|
942
|
+
async def _storage(self) -> Any:
|
|
943
|
+
await self._store._ensure_init()
|
|
944
|
+
return self._store._components.storage
|
|
945
|
+
|
|
946
|
+
async def count_namespace(self, ns: str) -> int:
|
|
947
|
+
metas = await (await self._storage()).list_namespace_meta()
|
|
948
|
+
return next(
|
|
949
|
+
(m.get("chunk_count", 0) for m in metas if m.get("namespace") == ns),
|
|
950
|
+
0,
|
|
951
|
+
)
|
|
952
|
+
|
|
953
|
+
async def delete_namespace(self, ns: str) -> int:
|
|
954
|
+
removed = await (await self._storage()).delete_by_namespace(ns)
|
|
955
|
+
await self._invalidate_search_cache()
|
|
956
|
+
return removed
|
|
957
|
+
|
|
958
|
+
async def _invalidate_search_cache(self, *, fail_closed: bool = False) -> None:
|
|
959
|
+
"""Drop THIS PROCESS's search-pipeline TTL result cache after a delete.
|
|
960
|
+
|
|
961
|
+
The same MCP-parity rule the write path follows, for the same reason in
|
|
962
|
+
the other direction: the pipeline caches result sets, so a reader that
|
|
963
|
+
searched before a delete keeps being served the deleted rows until the
|
|
964
|
+
TTL expires. For a ``board-run:*`` purge that means handing a later
|
|
965
|
+
reader exactly the board-internal text the purge existed to remove —
|
|
966
|
+
the leak this whole path guards, arriving from memory instead of from a
|
|
967
|
+
reindex.
|
|
968
|
+
|
|
969
|
+
**Process-local, and that is a real limit.** The cache lives in the
|
|
970
|
+
pipeline object, so another process holding its own ``MemtomemBackend``
|
|
971
|
+
over the same store keeps serving its own cached rows until its TTL
|
|
972
|
+
expires. Nothing here can reach it; closing that needs a shared
|
|
973
|
+
generation or epoch upstream. So the purge's postcondition is stated in
|
|
974
|
+
terms of files, rows, and this process's cache — never "no cached copy
|
|
975
|
+
anywhere".
|
|
976
|
+
|
|
977
|
+
``fail_closed`` for the purge path: the caller TOMBSTONES on success, so
|
|
978
|
+
a namespace whose cached rows this process may still serve must not be
|
|
979
|
+
reported as purged. The plain delete path stays best-effort, where a
|
|
980
|
+
stale cache expires on its own and there is no tombstone to mislead.
|
|
981
|
+
"""
|
|
982
|
+
try:
|
|
983
|
+
comp = await self._store._ensure_init()
|
|
984
|
+
comp.search_pipeline.invalidate_cache()
|
|
985
|
+
except Exception:
|
|
986
|
+
if fail_closed:
|
|
987
|
+
raise
|
|
988
|
+
logger.warning("search cache invalidation failed", exc_info=True)
|
|
989
|
+
|
|
990
|
+
async def delete_namespace_sources(self, ns: str) -> int:
|
|
991
|
+
"""Remove the markdown source files this backend wrote for ``ns``.
|
|
992
|
+
|
|
993
|
+
``delete_by_namespace`` only removes database rows, but the markdown is
|
|
994
|
+
what the index is rebuilt FROM: a watcher, ``mm index``, or any later
|
|
995
|
+
write to the same file re-indexes the deleted text and — its rows now
|
|
996
|
+
gone — treats it as new content. Without this a deleted namespace comes
|
|
997
|
+
back on the next reindex.
|
|
998
|
+
|
|
999
|
+
Two file shapes hold a namespace's content, and both are matched: the
|
|
1000
|
+
``{date}.{ns}.md`` files this backend's replicated add flow writes, and
|
|
1001
|
+
the ``{date}--{slug}-{digest}.md`` files memtomem (0.4+) natively writes
|
|
1002
|
+
for the same namespace (CLI ``mm add``, the ``mem_add`` MCP tool — the
|
|
1003
|
+
surfaces that call ``day_file_name``; the watcher only re-indexes what
|
|
1004
|
+
they wrote, which is what makes a surviving file resurrect).
|
|
1005
|
+
|
|
1006
|
+
A name is matched only when it is a whole ``{date}`` + one of those two
|
|
1007
|
+
suffixes: a bare ``*{suffix}`` glob would also take ``backup--{slug}-
|
|
1008
|
+
{digest}.md`` or ``notes.{ns}.md``, and the digest proves the namespace,
|
|
1009
|
+
never that syncmill wrote the file or that the prefix is a date. For the
|
|
1010
|
+
same reason ``ns`` is never interpolated INTO a glob pattern — it is
|
|
1011
|
+
caller-supplied, and a ``*`` or ``[`` in it would silently widen the
|
|
1012
|
+
sweep — so the glob is the literal ``*.md`` and every candidate is
|
|
1013
|
+
compared as an exact string.
|
|
1014
|
+
|
|
1015
|
+
RAISES on any failure rather than reporting a misleading zero: the
|
|
1016
|
+
caller tombstones a purged namespace, and a namespace whose source is
|
|
1017
|
+
still on disk has not been purged — it must stay collectable so a
|
|
1018
|
+
later sweep retries it. Every root the indexer reads is swept, not just
|
|
1019
|
+
the user tier: a file under a registered project root is re-indexed
|
|
1020
|
+
exactly like a user-tier one, so leaving it behind resurrects the
|
|
1021
|
+
namespace the caller was told had been purged. A root that cannot be
|
|
1022
|
+
verified (missing, not a directory, unreadable — an unmounted project
|
|
1023
|
+
tier) RAISES rather than being skipped, for the same reason: skipping
|
|
1024
|
+
it would report a clean purge over files still on disk.
|
|
1025
|
+
|
|
1026
|
+
**Known limitations.** (1) A store written by an earlier version put
|
|
1027
|
+
every namespace in one shared ``{date}.md``. Those files are not
|
|
1028
|
+
matched here and cannot be — the entries carry no namespace, so there
|
|
1029
|
+
is nothing to select on, and deleting the file would take shared
|
|
1030
|
+
knowledge with it. Scratch written before the upgrade can therefore
|
|
1031
|
+
still be re-indexed. (2) The legacy suffix maps ``:`` to ``.``, which is
|
|
1032
|
+
not injective: ``a:b`` and ``a.b`` produce the same file name. That
|
|
1033
|
+
comes from the WRITE side (see ``add``) and predates this method, and
|
|
1034
|
+
upstream's mix-guard means such a file only ever holds one of the two —
|
|
1035
|
+
so the exposure is deleting the other namespace's source, not mixing
|
|
1036
|
+
them. Unreachable for the only caller, which builds ``board-run:{uuid}``
|
|
1037
|
+
from a uuid4; closing it would mean changing the write-side name and
|
|
1038
|
+
orphaning every file already on disk.
|
|
1039
|
+
"""
|
|
1040
|
+
config = (await self._store._ensure_init()).config
|
|
1041
|
+
return _sweep_namespace_sources(config, ns)
|
|
1042
|
+
|
|
1043
|
+
async def purge_namespace(self, ns: str) -> int:
|
|
1044
|
+
"""Remove a namespace's sources AND rows with writers locked out.
|
|
1045
|
+
|
|
1046
|
+
The two-call sequence this replaces — sweep sources, then delete rows —
|
|
1047
|
+
had no exclusion between its halves. A write landing in that window
|
|
1048
|
+
creates a day file AND its row, the row delete then takes the row, and
|
|
1049
|
+
the markdown is left with nothing claiming it: the next reindex reads
|
|
1050
|
+
board-internal text as new content and stamps it with whatever
|
|
1051
|
+
namespace that pass resolves, which is how scratch reaches global
|
|
1052
|
+
``shared``. Ordering and a second verify sweep NARROW that window; only
|
|
1053
|
+
holding the writers out closes it.
|
|
1054
|
+
|
|
1055
|
+
**What is excluded, exactly.** Two levels, because two different things
|
|
1056
|
+
touch these files.
|
|
1057
|
+
|
|
1058
|
+
WRITERS take the namespace lock: the add flow is the only producer of
|
|
1059
|
+
``board-run:*`` content, it has a single write path (``add`` routes
|
|
1060
|
+
through ``add_returning_ids``), and that path takes this namespace's
|
|
1061
|
+
lock BEFORE it derives any file name. So a write either finishes before
|
|
1062
|
+
this purge starts or waits for it.
|
|
1063
|
+
|
|
1064
|
+
READERS — memtomem's watcher and manual indexers — take each file's L2
|
|
1065
|
+
sidecar instead, and an indexer that has read a file but not yet
|
|
1066
|
+
committed its rows would otherwise resurrect the namespace right after
|
|
1067
|
+
this method reported success. So the purge also takes the L2 sidecar of
|
|
1068
|
+
every source that EXISTS, enumerated from disk under the namespace lock.
|
|
1069
|
+
That is not the date enumeration that made the first version of this
|
|
1070
|
+
unsound: those were paths a writer might LATER name, unbounded because a
|
|
1071
|
+
suspended writer's target can be arbitrarily old; these are paths found
|
|
1072
|
+
on disk now.
|
|
1073
|
+
|
|
1074
|
+
An earlier version instead locked the writer's per-file L2 sidecar for
|
|
1075
|
+
today's and tomorrow's date. That was unsound, and the reason is worth
|
|
1076
|
+
keeping: the writer derives its target BEFORE taking any lock, so a
|
|
1077
|
+
process suspended in between resumes holding a target for an arbitrarily
|
|
1078
|
+
old date, which no finite set of dates covers. See
|
|
1079
|
+
``_namespace_lock_path``.
|
|
1080
|
+
|
|
1081
|
+
**What is NOT excluded.** A memtomem-native writer (``mm add``, the
|
|
1082
|
+
``mem_add`` MCP tool) does not take syncmill's namespace lock, so it can
|
|
1083
|
+
still interleave. Note this is NOT because its file name is unknowable —
|
|
1084
|
+
``day_file_name`` derives it deterministically, and the digest is over
|
|
1085
|
+
the namespace, not the date — but because upstream has no namespace-level
|
|
1086
|
+
lock to share (its documented lock domain is per-file, L0-L4 in
|
|
1087
|
+
``context/_atomic.py``) and locking its file paths would need the same
|
|
1088
|
+
unsound date enumeration. Nothing writes a ``board-run:{uuid4}`` scratch
|
|
1089
|
+
namespace that way, since syncmill mints those ids and no other surface
|
|
1090
|
+
knows them, but the guarantee is scoped to syncmill's own writers rather
|
|
1091
|
+
than absolute.
|
|
1092
|
+
|
|
1093
|
+
The verify sweep therefore stays, and now means something narrow and
|
|
1094
|
+
sharp: it can only fire for a native writer, since syncmill's are locked
|
|
1095
|
+
out and in-flight indexers are held. It RAISES rather than reporting a
|
|
1096
|
+
count, because a purge that cannot be proven complete must leave the
|
|
1097
|
+
namespace collectable — the caller tombstones on success, and a
|
|
1098
|
+
namespace with a live producer must not be retired from collection.
|
|
1099
|
+
Note what it is NOT: a proof of completeness. It samples once, so a
|
|
1100
|
+
native write landing after it is still missed; it is best-effort
|
|
1101
|
+
detection of the one interleaving the locks do not cover.
|
|
1102
|
+
|
|
1103
|
+
A lock the budget cannot obtain raises TimeoutError, which the caller
|
|
1104
|
+
degrades like any other backend failure — again leaving the namespace
|
|
1105
|
+
collectable, with no retry here (a scratch GC has a later chance, and
|
|
1106
|
+
blocking board dispatch on it would be worse).
|
|
1107
|
+
|
|
1108
|
+
Returns the number of source files removed.
|
|
1109
|
+
"""
|
|
1110
|
+
from contextlib import AsyncExitStack
|
|
1111
|
+
|
|
1112
|
+
atomic = _get_memtomem_atomic()
|
|
1113
|
+
_CRUD_SIDECAR_LOCK_BUDGET_S = atomic._CRUD_SIDECAR_LOCK_BUDGET_S
|
|
1114
|
+
memory_lock_path = atomic.memory_lock_path
|
|
1115
|
+
async_file_lock = atomic.async_file_lock
|
|
1116
|
+
|
|
1117
|
+
config = (await self._store._ensure_init()).config
|
|
1118
|
+
base = _user_base(config)
|
|
1119
|
+
# ONE deadline for the whole operation, started BEFORE the namespace
|
|
1120
|
+
# acquisition. Starting it after would let that lock spend a full budget
|
|
1121
|
+
# and every source lock spend another, so a single purge could take two
|
|
1122
|
+
# budgets plus scans — and `delete_board_runs` walks up to 100
|
|
1123
|
+
# namespaces in one maintenance pass, multiplying it.
|
|
1124
|
+
deadline = monotonic() + _CRUD_SIDECAR_LOCK_BUDGET_S
|
|
1125
|
+
|
|
1126
|
+
def _remaining() -> float:
|
|
1127
|
+
left = deadline - monotonic()
|
|
1128
|
+
if left <= 0:
|
|
1129
|
+
raise TimeoutError(
|
|
1130
|
+
f"purge of {ns} exceeded {_CRUD_SIDECAR_LOCK_BUDGET_S}s "
|
|
1131
|
+
"before the purge completed"
|
|
1132
|
+
)
|
|
1133
|
+
return left
|
|
1134
|
+
|
|
1135
|
+
async with AsyncExitStack() as stack:
|
|
1136
|
+
await stack.enter_async_context(
|
|
1137
|
+
async_file_lock(_namespace_lock_path(base, ns), timeout=_remaining())
|
|
1138
|
+
)
|
|
1139
|
+
# The namespace lock stops a syncmill WRITE, but memtomem's watcher
|
|
1140
|
+
# and manual indexers are READERS of these files: they take the
|
|
1141
|
+
# file's L2 sidecar, not this lock. One can hold L2, read a
|
|
1142
|
+
# board-run file, and pause before its DB transaction — and if the
|
|
1143
|
+
# purge ran through, it would unlink the file, delete the rows,
|
|
1144
|
+
# verify clean, and report success, after which the indexer resumes
|
|
1145
|
+
# and upserts its buffered chunks into a namespace the caller has
|
|
1146
|
+
# already been told is gone.
|
|
1147
|
+
#
|
|
1148
|
+
# So take the L2 sidecar of every source before touching any of
|
|
1149
|
+
# them. The candidate set is the UNION of what is on disk and what
|
|
1150
|
+
# the DB still attributes to this namespace: a source an indexer is
|
|
1151
|
+
# holding can already have been unlinked externally, in which case
|
|
1152
|
+
# disk alone sees nothing and that indexer would commit its rows
|
|
1153
|
+
# after the purge returned. Neither source of truth covers the other.
|
|
1154
|
+
#
|
|
1155
|
+
# Re-snapshot after acquiring, because taking a lock takes time and
|
|
1156
|
+
# the set can grow underneath. The loop converges when a pass adds
|
|
1157
|
+
# nothing new.
|
|
1158
|
+
storage = await self._storage()
|
|
1159
|
+
locked: set[Path] = set()
|
|
1160
|
+
while True:
|
|
1161
|
+
# Verified ONCE per snapshot, and a failure propagates. Turning
|
|
1162
|
+
# it into "skip this path" would be fail-OPEN at the operation
|
|
1163
|
+
# level: a transient error while validating could drop exactly
|
|
1164
|
+
# the externally-unlinked source an indexer is holding, and the
|
|
1165
|
+
# purge would then delete its rows and report success.
|
|
1166
|
+
roots = _index_roots(config, base)
|
|
1167
|
+
candidates = set(_namespace_source_paths(config, ns))
|
|
1168
|
+
candidates |= _purgeable_sources(
|
|
1169
|
+
config,
|
|
1170
|
+
ns,
|
|
1171
|
+
roots,
|
|
1172
|
+
(Path(raw) for raw in await storage.list_sources_by_namespace(ns)),
|
|
1173
|
+
)
|
|
1174
|
+
# The disk half goes through the same rule, so a symlinked day
|
|
1175
|
+
# file is refused there too rather than locked under the wrong
|
|
1176
|
+
# identity.
|
|
1177
|
+
candidates = _purgeable_sources(config, ns, roots, candidates)
|
|
1178
|
+
fresh = candidates - locked
|
|
1179
|
+
if not fresh:
|
|
1180
|
+
break
|
|
1181
|
+
if len(locked) + len(fresh) > _MAX_PURGE_SOURCE_LOCKS:
|
|
1182
|
+
raise UnprovablePurgeError(
|
|
1183
|
+
f"namespace {ns} has more than {_MAX_PURGE_SOURCE_LOCKS} "
|
|
1184
|
+
"source files; refusing to hold that many locks at once "
|
|
1185
|
+
"rather than exhaust file descriptors mid-purge"
|
|
1186
|
+
)
|
|
1187
|
+
_check_purge_descriptor_budget(len(fresh))
|
|
1188
|
+
for lock_path in sorted({memory_lock_path(p) for p in fresh}, key=str):
|
|
1189
|
+
await stack.enter_async_context(
|
|
1190
|
+
async_file_lock(lock_path, timeout=_remaining())
|
|
1191
|
+
)
|
|
1192
|
+
locked |= fresh
|
|
1193
|
+
# SOURCES FIRST, still: the rows are what keep a surviving file
|
|
1194
|
+
# harmless, since a reindex that recognizes the text leaves its
|
|
1195
|
+
# namespace alone. Under the lock no syncmill writer can interleave,
|
|
1196
|
+
# so the pair is atomic against them; the order continues to decide
|
|
1197
|
+
# what a FAILURE between the two leaves behind.
|
|
1198
|
+
#
|
|
1199
|
+
# Only paths whose L2 this purge HOLDS are unlinked. Sweeping by
|
|
1200
|
+
# match instead would mutate a file that appeared after the snapshot
|
|
1201
|
+
# without holding its lock — the very thing the locking is for.
|
|
1202
|
+
removed = 0
|
|
1203
|
+
for path in sorted(locked, key=str):
|
|
1204
|
+
_remaining()
|
|
1205
|
+
try:
|
|
1206
|
+
os.unlink(path)
|
|
1207
|
+
except FileNotFoundError:
|
|
1208
|
+
# Either the DB knew a path disk no longer has, or something
|
|
1209
|
+
# external removed it. Its rows still go below; the count
|
|
1210
|
+
# reports what THIS purge removed.
|
|
1211
|
+
continue
|
|
1212
|
+
removed += 1
|
|
1213
|
+
_remaining()
|
|
1214
|
+
await storage.delete_by_namespace(ns)
|
|
1215
|
+
# Inside the span: a reader that searched before this purge would
|
|
1216
|
+
# otherwise keep being served the purged rows from this process's
|
|
1217
|
+
# TTL cache. Fail CLOSED here — the caller tombstones on success, so
|
|
1218
|
+
# a namespace whose cached rows may still be served locally must not
|
|
1219
|
+
# be reported as purged.
|
|
1220
|
+
_remaining()
|
|
1221
|
+
await self._invalidate_search_cache(fail_closed=True)
|
|
1222
|
+
# Anything still matching is a path this purge never held: a writer
|
|
1223
|
+
# it cannot exclude produced it.
|
|
1224
|
+
raced = len(_namespace_source_paths(config, ns))
|
|
1225
|
+
_remaining()
|
|
1226
|
+
if raced:
|
|
1227
|
+
raise UnprovablePurgeError(
|
|
1228
|
+
f"namespace {ns} gained {raced} source file(s) during its purge; "
|
|
1229
|
+
"a writer this lock cannot exclude is active, so the purge is "
|
|
1230
|
+
"not provably complete"
|
|
1231
|
+
)
|
|
1232
|
+
return removed
|
|
1233
|
+
|
|
1234
|
+
async def start_session(self, agent_id: str, namespace: str) -> str | None:
|
|
1235
|
+
return await self._store.start_session(agent_id=agent_id, namespace=namespace)
|
|
1236
|
+
|
|
1237
|
+
async def log_event(self, kind: str, text: str) -> None:
|
|
1238
|
+
await self._store.log_event(kind, text)
|
|
1239
|
+
|
|
1240
|
+
async def end_session(self, summary: str | None) -> None:
|
|
1241
|
+
await self._store.end_session(summary=summary)
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
_LOCAL_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
|
|
1245
|
+
_LOCAL_DIMS = 384
|
|
1246
|
+
|
|
1247
|
+
|
|
1248
|
+
class _DisabledLLM:
|
|
1249
|
+
"""Fail closed if a future mem0 path accidentally tries inference."""
|
|
1250
|
+
|
|
1251
|
+
def generate_response(self, *args: Any, **kwargs: Any) -> Any:
|
|
1252
|
+
raise RuntimeError("syncmill's mem0 backend disables LLM inference")
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
|
1256
|
+
merged = copy.deepcopy(base)
|
|
1257
|
+
for key, value in override.items():
|
|
1258
|
+
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
|
1259
|
+
merged[key] = _deep_merge(merged[key], value)
|
|
1260
|
+
else:
|
|
1261
|
+
merged[key] = copy.deepcopy(value)
|
|
1262
|
+
return merged
|
|
1263
|
+
|
|
1264
|
+
|
|
1265
|
+
def _resolve_mem0_config(config: MemoryConfig) -> tuple[Path, dict[str, Any]]:
|
|
1266
|
+
"""Resolve Mem0 defaults without importing Mem0 or touching the filesystem.
|
|
1267
|
+
|
|
1268
|
+
Migration storage binding consumes the same resolved vector/history settings
|
|
1269
|
+
as the runtime adapter. Keeping this step pure lets migration commands reject
|
|
1270
|
+
a mismatched destination before they construct an SDK client or create a DB.
|
|
1271
|
+
"""
|
|
1272
|
+
mem0_cfg = config.mem0
|
|
1273
|
+
data_dir = Path(mem0_cfg.data_dir).expanduser()
|
|
1274
|
+
defaults: dict[str, Any] = {
|
|
1275
|
+
"vector_store": {
|
|
1276
|
+
"provider": "qdrant",
|
|
1277
|
+
"config": {
|
|
1278
|
+
"collection_name": "syncmill",
|
|
1279
|
+
"path": str(data_dir / "qdrant"),
|
|
1280
|
+
"embedding_model_dims": _LOCAL_DIMS,
|
|
1281
|
+
},
|
|
1282
|
+
},
|
|
1283
|
+
"embedder": {
|
|
1284
|
+
"provider": "fastembed",
|
|
1285
|
+
"config": {
|
|
1286
|
+
"model": _LOCAL_MODEL,
|
|
1287
|
+
"embedding_dims": _LOCAL_DIMS,
|
|
1288
|
+
},
|
|
1289
|
+
},
|
|
1290
|
+
# AsyncMemory constructs an LLM even though infer=False never uses it.
|
|
1291
|
+
"llm": {
|
|
1292
|
+
"provider": "openai",
|
|
1293
|
+
"config": {
|
|
1294
|
+
"model": "gpt-5-mini",
|
|
1295
|
+
"api_key": "syncmill-inference-disabled",
|
|
1296
|
+
"openai_base_url": "http://127.0.0.1:9/v1",
|
|
1297
|
+
},
|
|
1298
|
+
},
|
|
1299
|
+
"history_db_path": str(data_dir / "history.db"),
|
|
1300
|
+
"version": "v1.1",
|
|
1301
|
+
}
|
|
1302
|
+
raw = copy.deepcopy(mem0_cfg.config)
|
|
1303
|
+
resolved = _deep_merge(defaults, raw)
|
|
1304
|
+
vector = resolved.get("vector_store", {})
|
|
1305
|
+
if vector.get("provider") != "qdrant":
|
|
1306
|
+
raise ValueError("mem0 backend requires vector_store.provider='qdrant'")
|
|
1307
|
+
|
|
1308
|
+
embedder = resolved.get("embedder", {})
|
|
1309
|
+
embed_cfg = embedder.get("config", {})
|
|
1310
|
+
user_embedder = raw.get("embedder", {})
|
|
1311
|
+
user_embed_cfg = user_embedder.get("config", {})
|
|
1312
|
+
customized_embedder = (
|
|
1313
|
+
embedder.get("provider") != "fastembed"
|
|
1314
|
+
or embed_cfg.get("model") != _LOCAL_MODEL
|
|
1315
|
+
)
|
|
1316
|
+
if customized_embedder and "embedding_dims" not in user_embed_cfg:
|
|
1317
|
+
raise ValueError(
|
|
1318
|
+
"custom mem0 embedders must set config.embedding_dims explicitly"
|
|
1319
|
+
)
|
|
1320
|
+
if embedder.get("provider") != "fastembed" and "model" not in user_embed_cfg:
|
|
1321
|
+
raise ValueError("custom mem0 embedders must set config.model explicitly")
|
|
1322
|
+
dims = embed_cfg.get("embedding_dims")
|
|
1323
|
+
if not isinstance(dims, int) or isinstance(dims, bool) or dims < 1:
|
|
1324
|
+
raise ValueError("mem0 embedder.config.embedding_dims must be positive")
|
|
1325
|
+
vector_cfg = vector.setdefault("config", {})
|
|
1326
|
+
vector_dims = vector_cfg.get("embedding_model_dims")
|
|
1327
|
+
user_vector_cfg = raw.get("vector_store", {}).get("config", {})
|
|
1328
|
+
if (
|
|
1329
|
+
"embedding_model_dims" in user_vector_cfg
|
|
1330
|
+
and vector_dims is not None
|
|
1331
|
+
and vector_dims != dims
|
|
1332
|
+
):
|
|
1333
|
+
raise ValueError(
|
|
1334
|
+
"mem0 vector_store embedding_model_dims must match embedder dimensions"
|
|
1335
|
+
)
|
|
1336
|
+
vector_cfg["embedding_model_dims"] = dims
|
|
1337
|
+
|
|
1338
|
+
# A remote/custom Qdrant client and a local path are mutually exclusive.
|
|
1339
|
+
if any(k in user_vector_cfg for k in ("host", "url", "client")):
|
|
1340
|
+
vector_cfg.pop("path", None)
|
|
1341
|
+
elif "path" in vector_cfg:
|
|
1342
|
+
vector_cfg["path"] = str(Path(vector_cfg["path"]).expanduser())
|
|
1343
|
+
resolved["history_db_path"] = str(Path(resolved["history_db_path"]).expanduser())
|
|
1344
|
+
return data_dir, resolved
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
class Mem0Backend:
|
|
1348
|
+
"""Optional mem0 OSS reference backend, constrained to Qdrant.
|
|
1349
|
+
|
|
1350
|
+
The adapter maps each syncmill namespace to one mem0 ``user_id`` and forces
|
|
1351
|
+
``infer=False`` on writes. Qdrant is deliberately required because P2 needs
|
|
1352
|
+
exact namespace counts and proof that bulk deletion reached zero.
|
|
1353
|
+
"""
|
|
1354
|
+
|
|
1355
|
+
def __init__(self, config: MemoryConfig):
|
|
1356
|
+
mem0_cfg = config.mem0
|
|
1357
|
+
data_dir, resolved = _resolve_mem0_config(config)
|
|
1358
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
1359
|
+
|
|
1360
|
+
# mem0 snapshots this setting on the process's first mem0 import. Syncmill
|
|
1361
|
+
# defaults that first import to no anonymous telemetry; changing the
|
|
1362
|
+
# setting after another component imports mem0 requires a process restart.
|
|
1363
|
+
os.environ["MEM0_TELEMETRY"] = "true" if mem0_cfg.telemetry else "false"
|
|
1364
|
+
try:
|
|
1365
|
+
from mem0 import AsyncMemory
|
|
1366
|
+
from mem0.configs.base import MemoryConfig as Mem0MemoryConfig
|
|
1367
|
+
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
|
1368
|
+
except ModuleNotFoundError as exc:
|
|
1369
|
+
raise RuntimeError(
|
|
1370
|
+
"mem0 backend is not installed; install syncmill[mem0] "
|
|
1371
|
+
"or run `uv sync --extra mem0`"
|
|
1372
|
+
) from exc
|
|
1373
|
+
|
|
1374
|
+
self._store = AsyncMemory(config=Mem0MemoryConfig(**resolved))
|
|
1375
|
+
self._store.llm = _DisabledLLM()
|
|
1376
|
+
self._filter_type = Filter
|
|
1377
|
+
self._field_condition_type = FieldCondition
|
|
1378
|
+
self._match_value_type = MatchValue
|
|
1379
|
+
self._data_dir = str(data_dir)
|
|
1380
|
+
self._resolved = resolved
|
|
1381
|
+
self._caps = BackendCapabilities(
|
|
1382
|
+
supports_sessions=False,
|
|
1383
|
+
supports_tag_filter=True,
|
|
1384
|
+
namespace_delete_is_atomic=False,
|
|
1385
|
+
native_multi_namespace_search=False,
|
|
1386
|
+
share_provenance_links=False,
|
|
1387
|
+
)
|
|
1388
|
+
self._closed = False
|
|
1389
|
+
|
|
1390
|
+
@property
|
|
1391
|
+
def capabilities(self) -> BackendCapabilities:
|
|
1392
|
+
return self._caps
|
|
1393
|
+
|
|
1394
|
+
@property
|
|
1395
|
+
def agent_namespace_prefix(self) -> str:
|
|
1396
|
+
return "agent-runtime:"
|
|
1397
|
+
|
|
1398
|
+
@property
|
|
1399
|
+
def shared_namespace(self) -> str:
|
|
1400
|
+
return "shared"
|
|
1401
|
+
|
|
1402
|
+
def validate_namespace(self, ns: str) -> bool:
|
|
1403
|
+
return bool(self._caps.namespace_pattern.fullmatch(ns))
|
|
1404
|
+
|
|
1405
|
+
async def add(self, content: str, *, tags: list[str], namespace: str) -> None:
|
|
1406
|
+
await self.add_returning_ids(content, tags=tags, namespace=namespace)
|
|
1407
|
+
|
|
1408
|
+
async def add_returning_ids(
|
|
1409
|
+
self, content: str, *, tags: list[str], namespace: str
|
|
1410
|
+
) -> list[str]:
|
|
1411
|
+
response = await self._store.add(
|
|
1412
|
+
[{"role": "user", "content": content}],
|
|
1413
|
+
user_id=namespace,
|
|
1414
|
+
metadata={"syncmill_tags": list(tags)},
|
|
1415
|
+
infer=False,
|
|
1416
|
+
)
|
|
1417
|
+
rows = response.get("results", []) if isinstance(response, dict) else []
|
|
1418
|
+
if len(rows) != 1 or rows[0].get("memory") != content:
|
|
1419
|
+
raise RuntimeError("mem0 did not confirm one verbatim memory write")
|
|
1420
|
+
row_id = rows[0].get("id")
|
|
1421
|
+
return [str(row_id)] if row_id else []
|
|
1422
|
+
|
|
1423
|
+
def _filters(self, namespace: str, tag_filter: str | None) -> dict[str, Any]:
|
|
1424
|
+
filters: dict[str, Any] = {"user_id": namespace}
|
|
1425
|
+
if tag_filter is not None:
|
|
1426
|
+
tags = [tag.strip() for tag in tag_filter.split(",") if tag.strip()]
|
|
1427
|
+
if tags:
|
|
1428
|
+
filters["OR"] = [{"syncmill_tags": tag} for tag in tags]
|
|
1429
|
+
return filters
|
|
1430
|
+
|
|
1431
|
+
@staticmethod
|
|
1432
|
+
def _row(row: dict[str, Any]) -> MemoryRow | None:
|
|
1433
|
+
namespace = row.get("user_id")
|
|
1434
|
+
if not namespace:
|
|
1435
|
+
return None
|
|
1436
|
+
metadata = row.get("metadata") or {}
|
|
1437
|
+
raw_tags = metadata.get("syncmill_tags")
|
|
1438
|
+
tags = [str(tag) for tag in raw_tags] if isinstance(raw_tags, list) else []
|
|
1439
|
+
return MemoryRow(
|
|
1440
|
+
id=str(row.get("id", "")),
|
|
1441
|
+
content=str(row.get("memory", "")),
|
|
1442
|
+
score=float(row.get("score") or 0.0),
|
|
1443
|
+
source=str(metadata.get("source") or "mem0"),
|
|
1444
|
+
namespace=str(namespace),
|
|
1445
|
+
tags=tags,
|
|
1446
|
+
)
|
|
1447
|
+
|
|
1448
|
+
async def search(
|
|
1449
|
+
self,
|
|
1450
|
+
query: str,
|
|
1451
|
+
*,
|
|
1452
|
+
namespace: str,
|
|
1453
|
+
top_k: int,
|
|
1454
|
+
tag_filter: str | None = None,
|
|
1455
|
+
) -> list[MemoryRow]:
|
|
1456
|
+
response = await self._store.search(
|
|
1457
|
+
query,
|
|
1458
|
+
top_k=top_k,
|
|
1459
|
+
filters=self._filters(namespace, tag_filter),
|
|
1460
|
+
threshold=0.0,
|
|
1461
|
+
rerank=False,
|
|
1462
|
+
)
|
|
1463
|
+
raw_rows = response.get("results", []) if isinstance(response, dict) else []
|
|
1464
|
+
rows = [mapped for raw in raw_rows if (mapped := self._row(raw))]
|
|
1465
|
+
return [row for row in rows if row.get("namespace") == namespace]
|
|
1466
|
+
|
|
1467
|
+
async def search_multi(
|
|
1468
|
+
self,
|
|
1469
|
+
query: str,
|
|
1470
|
+
*,
|
|
1471
|
+
namespaces: Sequence[str],
|
|
1472
|
+
top_k: int,
|
|
1473
|
+
tag_filter: str | None = None,
|
|
1474
|
+
) -> list[MemoryRow]:
|
|
1475
|
+
return _merge_rows(
|
|
1476
|
+
[
|
|
1477
|
+
await self.search(
|
|
1478
|
+
query, namespace=ns, top_k=top_k, tag_filter=tag_filter
|
|
1479
|
+
)
|
|
1480
|
+
for ns in namespaces
|
|
1481
|
+
],
|
|
1482
|
+
top_k,
|
|
1483
|
+
)
|
|
1484
|
+
|
|
1485
|
+
async def share_chunk(self, chunk_id: str, *, target_namespace: str) -> str | None:
|
|
1486
|
+
source = await self.get(chunk_id)
|
|
1487
|
+
if source is None:
|
|
1488
|
+
return None
|
|
1489
|
+
new_ids = await self.add_returning_ids(
|
|
1490
|
+
source["content"],
|
|
1491
|
+
tags=_shared_tags(source.get("tags", []), chunk_id),
|
|
1492
|
+
namespace=target_namespace,
|
|
1493
|
+
)
|
|
1494
|
+
return new_ids[0] if new_ids else None
|
|
1495
|
+
|
|
1496
|
+
async def register_namespace(
|
|
1497
|
+
self, ns: str, *, description: str | None = None, color: str | None = None
|
|
1498
|
+
) -> None:
|
|
1499
|
+
return None
|
|
1500
|
+
|
|
1501
|
+
async def get(self, chunk_id: str) -> MemoryRow | None:
|
|
1502
|
+
raw = await self._store.get(chunk_id)
|
|
1503
|
+
return self._row(raw) if isinstance(raw, dict) else None
|
|
1504
|
+
|
|
1505
|
+
async def ensure_namespace(self, ns: str, *, description: str = "") -> None:
|
|
1506
|
+
return None
|
|
1507
|
+
|
|
1508
|
+
async def ensure_ready(self) -> None:
|
|
1509
|
+
await self.count_namespace("syncmill-readiness-probe")
|
|
1510
|
+
|
|
1511
|
+
async def status(self) -> dict[str, Any]:
|
|
1512
|
+
await self.ensure_ready()
|
|
1513
|
+
vector = self._resolved["vector_store"]
|
|
1514
|
+
embedder = self._resolved["embedder"]
|
|
1515
|
+
return {
|
|
1516
|
+
"data_dir": self._data_dir,
|
|
1517
|
+
"vector_store": "qdrant",
|
|
1518
|
+
"collection": vector["config"].get("collection_name", "mem0"),
|
|
1519
|
+
"embedder": f"{embedder.get('provider')}/{embedder['config'].get('model')}",
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
def _qdrant_store(self) -> Any:
|
|
1523
|
+
store = self._store.vector_store
|
|
1524
|
+
if not hasattr(store, "client") or not hasattr(store, "collection_name"):
|
|
1525
|
+
raise RuntimeError("mem0 Qdrant internals are unavailable")
|
|
1526
|
+
return store
|
|
1527
|
+
|
|
1528
|
+
async def count_namespace(self, ns: str) -> int:
|
|
1529
|
+
store = self._qdrant_store()
|
|
1530
|
+
count_filter = self._filter_type(
|
|
1531
|
+
must=[
|
|
1532
|
+
self._field_condition_type(
|
|
1533
|
+
key="user_id", match=self._match_value_type(value=ns)
|
|
1534
|
+
)
|
|
1535
|
+
]
|
|
1536
|
+
)
|
|
1537
|
+
response = await asyncio.to_thread(
|
|
1538
|
+
store.client.count,
|
|
1539
|
+
collection_name=store.collection_name,
|
|
1540
|
+
count_filter=count_filter,
|
|
1541
|
+
exact=True,
|
|
1542
|
+
)
|
|
1543
|
+
return int(response.count)
|
|
1544
|
+
|
|
1545
|
+
async def delete_namespace(self, ns: str) -> int:
|
|
1546
|
+
original = await self.count_namespace(ns)
|
|
1547
|
+
remaining = original
|
|
1548
|
+
while remaining:
|
|
1549
|
+
await self._store.delete_all(user_id=ns)
|
|
1550
|
+
after = await self.count_namespace(ns)
|
|
1551
|
+
if after >= remaining:
|
|
1552
|
+
raise RuntimeError(
|
|
1553
|
+
f"mem0 namespace deletion made no progress for {ns!r}"
|
|
1554
|
+
)
|
|
1555
|
+
remaining = after
|
|
1556
|
+
return original
|
|
1557
|
+
|
|
1558
|
+
async def start_session(self, agent_id: str, namespace: str) -> str | None:
|
|
1559
|
+
return None
|
|
1560
|
+
|
|
1561
|
+
async def log_event(self, kind: str, text: str) -> None:
|
|
1562
|
+
return None
|
|
1563
|
+
|
|
1564
|
+
async def end_session(self, summary: str | None) -> None:
|
|
1565
|
+
return None
|
|
1566
|
+
|
|
1567
|
+
async def close(self) -> None:
|
|
1568
|
+
if self._closed:
|
|
1569
|
+
return
|
|
1570
|
+
self._closed = True
|
|
1571
|
+
# Teardown must not depend on the private Qdrant shape used by count/delete.
|
|
1572
|
+
# mem0 2.0.x closes its history DB, while the client close releases the
|
|
1573
|
+
# embedded Qdrant path lock when that client is still discoverable.
|
|
1574
|
+
vector_store = getattr(self._store, "vector_store", None)
|
|
1575
|
+
client = getattr(vector_store, "client", None)
|
|
1576
|
+
try:
|
|
1577
|
+
self._store.close()
|
|
1578
|
+
finally:
|
|
1579
|
+
close = getattr(client, "close", None)
|
|
1580
|
+
if close is not None:
|
|
1581
|
+
await asyncio.to_thread(close)
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
_LANGGRAPH_NS = ("syncmill", "memory", "items")
|
|
1585
|
+
# One page of asearch results; count_namespace's stop condition compares
|
|
1586
|
+
# against this same constant so the two can't drift.
|
|
1587
|
+
_LANGGRAPH_PAGE_SIZE = 100
|
|
1588
|
+
# Iteration backstop for count_namespace: far above any real namespace (a
|
|
1589
|
+
# syncmill namespace holds task results, not a corpus), hit only by a store
|
|
1590
|
+
# whose asearch ignores `offset` and would otherwise loop forever.
|
|
1591
|
+
_LANGGRAPH_COUNT_CAP = 1_000_000
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def _tag_key(tag: str) -> str:
|
|
1595
|
+
return "tag_" + hashlib.sha256(tag.encode("utf-8")).hexdigest()
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
class LangGraphBackend:
|
|
1599
|
+
"""Optional persistent LangGraph AsyncSqliteStore reference backend."""
|
|
1600
|
+
|
|
1601
|
+
def __init__(self, config: MemoryConfig):
|
|
1602
|
+
self._db_path = str(Path(config.langgraph.db_path).expanduser())
|
|
1603
|
+
self._store: Any = None
|
|
1604
|
+
self._store_cm: Any = None
|
|
1605
|
+
self._init_lock = asyncio.Lock()
|
|
1606
|
+
self._closed = False
|
|
1607
|
+
self._caps = BackendCapabilities(
|
|
1608
|
+
supports_sessions=False,
|
|
1609
|
+
supports_tag_filter=True,
|
|
1610
|
+
namespace_delete_is_atomic=False,
|
|
1611
|
+
native_multi_namespace_search=False,
|
|
1612
|
+
share_provenance_links=False,
|
|
1613
|
+
)
|
|
1614
|
+
|
|
1615
|
+
@property
|
|
1616
|
+
def capabilities(self) -> BackendCapabilities:
|
|
1617
|
+
return self._caps
|
|
1618
|
+
|
|
1619
|
+
@property
|
|
1620
|
+
def agent_namespace_prefix(self) -> str:
|
|
1621
|
+
return "agent-runtime:"
|
|
1622
|
+
|
|
1623
|
+
@property
|
|
1624
|
+
def shared_namespace(self) -> str:
|
|
1625
|
+
return "shared"
|
|
1626
|
+
|
|
1627
|
+
def validate_namespace(self, ns: str) -> bool:
|
|
1628
|
+
return bool(self._caps.namespace_pattern.fullmatch(ns))
|
|
1629
|
+
|
|
1630
|
+
async def _ensure_store(self) -> Any:
|
|
1631
|
+
if self._closed:
|
|
1632
|
+
raise RuntimeError("LangGraph backend is closed")
|
|
1633
|
+
if self._store is not None:
|
|
1634
|
+
return self._store
|
|
1635
|
+
async with self._init_lock:
|
|
1636
|
+
if self._store is not None:
|
|
1637
|
+
return self._store
|
|
1638
|
+
try:
|
|
1639
|
+
from fastembed import TextEmbedding
|
|
1640
|
+
from langchain_core.embeddings import Embeddings
|
|
1641
|
+
from langgraph.store.sqlite.aio import AsyncSqliteStore
|
|
1642
|
+
except ModuleNotFoundError as exc:
|
|
1643
|
+
raise RuntimeError(
|
|
1644
|
+
"LangGraph backend is not installed; install syncmill[langgraph] "
|
|
1645
|
+
"or run `uv sync --extra langgraph`"
|
|
1646
|
+
) from exc
|
|
1647
|
+
|
|
1648
|
+
class LocalEmbeddings(Embeddings):
|
|
1649
|
+
def __init__(self) -> None:
|
|
1650
|
+
self.model = TextEmbedding(model_name=_LOCAL_MODEL)
|
|
1651
|
+
|
|
1652
|
+
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
1653
|
+
return [list(vector) for vector in self.model.embed(texts)]
|
|
1654
|
+
|
|
1655
|
+
def embed_query(self, text: str) -> list[float]:
|
|
1656
|
+
return self.embed_documents([text])[0]
|
|
1657
|
+
|
|
1658
|
+
async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
1659
|
+
return await asyncio.to_thread(self.embed_documents, texts)
|
|
1660
|
+
|
|
1661
|
+
async def aembed_query(self, text: str) -> list[float]:
|
|
1662
|
+
return (await self.aembed_documents([text]))[0]
|
|
1663
|
+
|
|
1664
|
+
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
|
1665
|
+
self._store_cm = AsyncSqliteStore.from_conn_string(
|
|
1666
|
+
self._db_path,
|
|
1667
|
+
index={
|
|
1668
|
+
"dims": _LOCAL_DIMS,
|
|
1669
|
+
"embed": LocalEmbeddings(),
|
|
1670
|
+
"text_fields": ["content"],
|
|
1671
|
+
},
|
|
1672
|
+
)
|
|
1673
|
+
self._store = await self._store_cm.__aenter__()
|
|
1674
|
+
try:
|
|
1675
|
+
await self._store.setup()
|
|
1676
|
+
except Exception:
|
|
1677
|
+
await self._store_cm.__aexit__(None, None, None)
|
|
1678
|
+
self._store = self._store_cm = None
|
|
1679
|
+
raise
|
|
1680
|
+
return self._store
|
|
1681
|
+
|
|
1682
|
+
@staticmethod
|
|
1683
|
+
def _value(content: str, tags: list[str], namespace: str) -> dict[str, Any]:
|
|
1684
|
+
value: dict[str, Any] = {
|
|
1685
|
+
"schema_version": 1,
|
|
1686
|
+
"namespace": namespace,
|
|
1687
|
+
"content": content,
|
|
1688
|
+
"tags": list(tags),
|
|
1689
|
+
"source": "langgraph",
|
|
1690
|
+
}
|
|
1691
|
+
value.update({_tag_key(tag): True for tag in tags})
|
|
1692
|
+
return value
|
|
1693
|
+
|
|
1694
|
+
@staticmethod
|
|
1695
|
+
def _row(item: Any) -> MemoryRow | None:
|
|
1696
|
+
value = getattr(item, "value", None)
|
|
1697
|
+
if not isinstance(value, dict):
|
|
1698
|
+
return None
|
|
1699
|
+
namespace = value.get("namespace")
|
|
1700
|
+
content = value.get("content")
|
|
1701
|
+
tags = value.get("tags", [])
|
|
1702
|
+
if not isinstance(namespace, str) or not isinstance(content, str):
|
|
1703
|
+
return None
|
|
1704
|
+
if not isinstance(tags, list) or not all(isinstance(t, str) for t in tags):
|
|
1705
|
+
return None
|
|
1706
|
+
return MemoryRow(
|
|
1707
|
+
id=str(item.key),
|
|
1708
|
+
content=content,
|
|
1709
|
+
score=float(getattr(item, "score", None) or 0.0),
|
|
1710
|
+
source=str(value.get("source") or "langgraph"),
|
|
1711
|
+
namespace=namespace,
|
|
1712
|
+
tags=list(tags),
|
|
1713
|
+
)
|
|
1714
|
+
|
|
1715
|
+
async def add(self, content: str, *, tags: list[str], namespace: str) -> None:
|
|
1716
|
+
await self.add_returning_ids(content, tags=tags, namespace=namespace)
|
|
1717
|
+
|
|
1718
|
+
async def add_returning_ids(
|
|
1719
|
+
self, content: str, *, tags: list[str], namespace: str
|
|
1720
|
+
) -> list[str]:
|
|
1721
|
+
store = await self._ensure_store()
|
|
1722
|
+
key = str(uuid4())
|
|
1723
|
+
await store.aput(
|
|
1724
|
+
_LANGGRAPH_NS,
|
|
1725
|
+
key,
|
|
1726
|
+
self._value(content, tags, namespace),
|
|
1727
|
+
index=["content"],
|
|
1728
|
+
)
|
|
1729
|
+
return [key]
|
|
1730
|
+
|
|
1731
|
+
async def search(
|
|
1732
|
+
self,
|
|
1733
|
+
query: str,
|
|
1734
|
+
*,
|
|
1735
|
+
namespace: str,
|
|
1736
|
+
top_k: int,
|
|
1737
|
+
tag_filter: str | None = None,
|
|
1738
|
+
) -> list[MemoryRow]:
|
|
1739
|
+
store = await self._ensure_store()
|
|
1740
|
+
tags = (
|
|
1741
|
+
[tag.strip() for tag in tag_filter.split(",") if tag.strip()]
|
|
1742
|
+
if tag_filter
|
|
1743
|
+
else []
|
|
1744
|
+
)
|
|
1745
|
+
filters = [{"namespace": namespace, _tag_key(tag): True} for tag in tags] or [
|
|
1746
|
+
{"namespace": namespace}
|
|
1747
|
+
]
|
|
1748
|
+
merged: dict[str, MemoryRow] = {}
|
|
1749
|
+
for exact_filter in filters:
|
|
1750
|
+
items = await store.asearch(
|
|
1751
|
+
_LANGGRAPH_NS,
|
|
1752
|
+
query=query,
|
|
1753
|
+
filter=exact_filter,
|
|
1754
|
+
limit=top_k,
|
|
1755
|
+
)
|
|
1756
|
+
for item in items:
|
|
1757
|
+
row = self._row(item)
|
|
1758
|
+
if row is None or row.get("namespace") != namespace:
|
|
1759
|
+
continue
|
|
1760
|
+
prior = merged.get(row["id"])
|
|
1761
|
+
if prior is None or row.get("score", 0.0) > prior.get("score", 0.0):
|
|
1762
|
+
merged[row["id"]] = row
|
|
1763
|
+
return sorted(merged.values(), key=lambda row: -row.get("score", 0.0))[:top_k]
|
|
1764
|
+
|
|
1765
|
+
async def search_multi(
|
|
1766
|
+
self,
|
|
1767
|
+
query: str,
|
|
1768
|
+
*,
|
|
1769
|
+
namespaces: Sequence[str],
|
|
1770
|
+
top_k: int,
|
|
1771
|
+
tag_filter: str | None = None,
|
|
1772
|
+
) -> list[MemoryRow]:
|
|
1773
|
+
return _merge_rows(
|
|
1774
|
+
[
|
|
1775
|
+
await self.search(
|
|
1776
|
+
query, namespace=ns, top_k=top_k, tag_filter=tag_filter
|
|
1777
|
+
)
|
|
1778
|
+
for ns in namespaces
|
|
1779
|
+
],
|
|
1780
|
+
top_k,
|
|
1781
|
+
)
|
|
1782
|
+
|
|
1783
|
+
async def share_chunk(self, chunk_id: str, *, target_namespace: str) -> str | None:
|
|
1784
|
+
source = await self.get(chunk_id)
|
|
1785
|
+
if source is None:
|
|
1786
|
+
return None
|
|
1787
|
+
new_ids = await self.add_returning_ids(
|
|
1788
|
+
source["content"],
|
|
1789
|
+
tags=_shared_tags(source.get("tags", []), chunk_id),
|
|
1790
|
+
namespace=target_namespace,
|
|
1791
|
+
)
|
|
1792
|
+
return new_ids[0] if new_ids else None
|
|
1793
|
+
|
|
1794
|
+
async def register_namespace(
|
|
1795
|
+
self, ns: str, *, description: str | None = None, color: str | None = None
|
|
1796
|
+
) -> None:
|
|
1797
|
+
return None
|
|
1798
|
+
|
|
1799
|
+
async def get(self, chunk_id: str) -> MemoryRow | None:
|
|
1800
|
+
return self._row(
|
|
1801
|
+
await (await self._ensure_store()).aget(_LANGGRAPH_NS, chunk_id)
|
|
1802
|
+
)
|
|
1803
|
+
|
|
1804
|
+
async def ensure_namespace(self, ns: str, *, description: str = "") -> None:
|
|
1805
|
+
return None
|
|
1806
|
+
|
|
1807
|
+
async def ensure_ready(self) -> None:
|
|
1808
|
+
await self._ensure_store()
|
|
1809
|
+
|
|
1810
|
+
async def status(self) -> dict[str, Any]:
|
|
1811
|
+
await self.ensure_ready()
|
|
1812
|
+
return {
|
|
1813
|
+
"db_path": self._db_path,
|
|
1814
|
+
"store": "langgraph/AsyncSqliteStore",
|
|
1815
|
+
"embedder": f"fastembed/{_LOCAL_MODEL}",
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
async def _page(
|
|
1819
|
+
self, ns: str, *, limit: int = _LANGGRAPH_PAGE_SIZE, offset: int = 0
|
|
1820
|
+
) -> list[Any]:
|
|
1821
|
+
return await (await self._ensure_store()).asearch(
|
|
1822
|
+
_LANGGRAPH_NS,
|
|
1823
|
+
query=None,
|
|
1824
|
+
filter={"namespace": ns},
|
|
1825
|
+
limit=limit,
|
|
1826
|
+
offset=offset,
|
|
1827
|
+
)
|
|
1828
|
+
|
|
1829
|
+
async def count_namespace(self, ns: str) -> int:
|
|
1830
|
+
total = 0
|
|
1831
|
+
while True:
|
|
1832
|
+
page = await self._page(ns, offset=total)
|
|
1833
|
+
if not page:
|
|
1834
|
+
return total
|
|
1835
|
+
total += len(page)
|
|
1836
|
+
if len(page) < _LANGGRAPH_PAGE_SIZE:
|
|
1837
|
+
return total
|
|
1838
|
+
# A store that ignores `offset` returns the same full page forever;
|
|
1839
|
+
# without a cap that spins unbounded (the mem0 adapter's no-progress
|
|
1840
|
+
# guard, adapted to offset paging).
|
|
1841
|
+
if total > _LANGGRAPH_COUNT_CAP:
|
|
1842
|
+
raise RuntimeError(
|
|
1843
|
+
f"LangGraph namespace count made no progress for {ns!r}"
|
|
1844
|
+
)
|
|
1845
|
+
|
|
1846
|
+
async def delete_namespace(self, ns: str) -> int:
|
|
1847
|
+
original = await self.count_namespace(ns)
|
|
1848
|
+
store = await self._ensure_store()
|
|
1849
|
+
remaining = original
|
|
1850
|
+
while True:
|
|
1851
|
+
page = await self._page(ns)
|
|
1852
|
+
if not page:
|
|
1853
|
+
break
|
|
1854
|
+
for item in page:
|
|
1855
|
+
await store.adelete(_LANGGRAPH_NS, item.key)
|
|
1856
|
+
# No-progress guard (the mem0 adapter's recipe): if a pass deleted
|
|
1857
|
+
# nothing — adelete no-ops on a mismatched key, a permission issue —
|
|
1858
|
+
# the same page would come back forever, an unbounded hang worse
|
|
1859
|
+
# than the failure it hides.
|
|
1860
|
+
after = await self.count_namespace(ns)
|
|
1861
|
+
if after >= remaining:
|
|
1862
|
+
raise RuntimeError(
|
|
1863
|
+
f"LangGraph namespace deletion made no progress for {ns!r}"
|
|
1864
|
+
)
|
|
1865
|
+
remaining = after
|
|
1866
|
+
if await self.count_namespace(ns):
|
|
1867
|
+
raise RuntimeError(
|
|
1868
|
+
f"LangGraph namespace deletion did not reach zero for {ns!r}"
|
|
1869
|
+
)
|
|
1870
|
+
return original
|
|
1871
|
+
|
|
1872
|
+
async def start_session(self, agent_id: str, namespace: str) -> str | None:
|
|
1873
|
+
return None
|
|
1874
|
+
|
|
1875
|
+
async def log_event(self, kind: str, text: str) -> None:
|
|
1876
|
+
return None
|
|
1877
|
+
|
|
1878
|
+
async def end_session(self, summary: str | None) -> None:
|
|
1879
|
+
return None
|
|
1880
|
+
|
|
1881
|
+
async def close(self) -> None:
|
|
1882
|
+
if self._closed:
|
|
1883
|
+
return
|
|
1884
|
+
self._closed = True
|
|
1885
|
+
if self._store_cm is not None:
|
|
1886
|
+
await self._store_cm.__aexit__(None, None, None)
|
|
1887
|
+
self._store = self._store_cm = None
|
|
1888
|
+
|
|
1889
|
+
|
|
1890
|
+
def build_backend(config: MemoryConfig) -> MemoryBackend:
|
|
1891
|
+
"""Build the configured backend, refusing non-verbatim storage."""
|
|
1892
|
+
|
|
1893
|
+
if config.backend == "memtomem":
|
|
1894
|
+
backend: MemoryBackend = MemtomemBackend(config)
|
|
1895
|
+
elif config.backend == "mem0":
|
|
1896
|
+
backend = Mem0Backend(config)
|
|
1897
|
+
elif config.backend == "langgraph":
|
|
1898
|
+
backend = LangGraphBackend(config)
|
|
1899
|
+
else: # guarded by MemoryConfig's Literal, defensive for programmatic callers
|
|
1900
|
+
raise ValueError(f"unknown memory backend: {config.backend!r}")
|
|
1901
|
+
# Defensive, like the unknown-backend branch above: no shipped backend sets
|
|
1902
|
+
# verbatim_storage=False (all three leave the default True), so this is
|
|
1903
|
+
# reachable only from tests/monkeypatching — it exists to make the contract
|
|
1904
|
+
# explicit for a future adapter, not to guard a live path.
|
|
1905
|
+
if not backend.capabilities.verbatim_storage:
|
|
1906
|
+
raise ValueError(f"backend {config.backend!r} cannot store verbatim content")
|
|
1907
|
+
return backend
|