syncmill 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. syncmill/__init__.py +11 -0
  2. syncmill/agents.py +2064 -0
  3. syncmill/artifact_io.py +201 -0
  4. syncmill/artifacts.py +1946 -0
  5. syncmill/board.py +2060 -0
  6. syncmill/breaker.py +327 -0
  7. syncmill/cli.py +4986 -0
  8. syncmill/command_errors.py +32 -0
  9. syncmill/config.py +1118 -0
  10. syncmill/dashboard.py +935 -0
  11. syncmill/gates.py +179 -0
  12. syncmill/gateway.py +234 -0
  13. syncmill/git_env.py +99 -0
  14. syncmill/isolated_session.py +431 -0
  15. syncmill/matcher.py +127 -0
  16. syncmill/memory.py +1067 -0
  17. syncmill/memory_backend.py +1907 -0
  18. syncmill/memory_migration.py +962 -0
  19. syncmill/model_routing.py +482 -0
  20. syncmill/models.py +244 -0
  21. syncmill/monitor.py +775 -0
  22. syncmill/phoenix_exporter.py +339 -0
  23. syncmill/planning.py +512 -0
  24. syncmill/platform_support.py +14 -0
  25. syncmill/preflight.py +1129 -0
  26. syncmill/project.py +1067 -0
  27. syncmill/report.py +502 -0
  28. syncmill/review_candidates.py +107 -0
  29. syncmill/room_compare.py +2380 -0
  30. syncmill/room_raw.py +484 -0
  31. syncmill/room_viewer.py +604 -0
  32. syncmill/rooms.py +4044 -0
  33. syncmill/session.py +444 -0
  34. syncmill/strategies/__init__.py +82 -0
  35. syncmill/strategies/base.py +910 -0
  36. syncmill/strategies/compete.py +233 -0
  37. syncmill/strategies/council.py +429 -0
  38. syncmill/strategies/decompose.py +570 -0
  39. syncmill/strategies/pipeline.py +221 -0
  40. syncmill/strategies/route.py +130 -0
  41. syncmill/supervisor.py +1423 -0
  42. syncmill/teamfile.py +571 -0
  43. syncmill/telemetry_hook.py +231 -0
  44. syncmill/terminal.py +245 -0
  45. syncmill/tracer.py +548 -0
  46. syncmill/tui.py +3537 -0
  47. syncmill/tui_collaboration.py +499 -0
  48. syncmill/tui_jobs.py +1407 -0
  49. syncmill/tui_render.py +156 -0
  50. syncmill/workflow.py +3746 -0
  51. syncmill/worktree.py +1045 -0
  52. syncmill-0.2.0.dist-info/METADATA +853 -0
  53. syncmill-0.2.0.dist-info/RECORD +56 -0
  54. syncmill-0.2.0.dist-info/WHEEL +4 -0
  55. syncmill-0.2.0.dist-info/entry_points.txt +2 -0
  56. syncmill-0.2.0.dist-info/licenses/LICENSE +191 -0
@@ -0,0 +1,1907 @@
1
+ """Backend seam for syncmill's long-term memory.
2
+
3
+ ``MemoryBackend`` names the contract ``SyncmillMemory`` imposes on its
4
+ storage provider. Backends raise freely: ``SyncmillMemory`` owns graceful
5
+ degradation, and adapters must not add retries or fallback behavior.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ import copy
12
+ import hashlib
13
+ import json
14
+ import logging
15
+ import os
16
+ import re
17
+ from collections.abc import Iterable, Sequence
18
+ from dataclasses import dataclass, field
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+ from time import monotonic
22
+ from typing import Any, Protocol, TypedDict
23
+ from uuid import UUID, uuid4
24
+
25
+ from syncmill.config import MemoryConfig
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+
30
+ class WriteBlockedError(RuntimeError):
31
+ """The backend's redaction guard blocked this content.
32
+
33
+ Distinct from ordinary backend failures so ``SyncmillMemory`` can skip the
34
+ direct-add fallback — the same guard runs on the same content there and can
35
+ only re-block (duplicate audit noise, never a different outcome)."""
36
+
37
+
38
+ class MemoryRow(TypedDict, total=False):
39
+ """One search/get result row; ``namespace`` is required for get isolation."""
40
+
41
+ id: str
42
+ content: str
43
+ score: float
44
+ source: str
45
+ namespace: str
46
+ tags: list[str]
47
+
48
+
49
+ @dataclass(frozen=True)
50
+ class BackendCapabilities:
51
+ """Capabilities used by later backend/namespace-lifecycle phases."""
52
+
53
+ supports_sessions: bool = True
54
+ supports_tag_filter: bool = True
55
+ namespace_delete_is_atomic: bool = True
56
+ verbatim_storage: bool = True
57
+ # Whether search_multi is one native merged-ranking call (memtomem's
58
+ # comma-namespace union filter) vs an adapter-side per-namespace loop.
59
+ native_multi_namespace_search: bool = True
60
+ # Whether share_chunk records structural provenance (a chunk_links row)
61
+ # beside the shared-from tag, or the tag alone.
62
+ share_provenance_links: bool = True
63
+ namespace_pattern: re.Pattern[str] = field(
64
+ default_factory=lambda: re.compile(r"^[A-Za-z0-9._-]+(?::[A-Za-z0-9._-]+)*$")
65
+ )
66
+
67
+
68
+ class MemoryBackend(Protocol):
69
+ """Async, raise-freely storage contract for ``SyncmillMemory``.
70
+
71
+ File-backed adapters may additionally expose ``purge_namespace(ns) -> int``:
72
+ return only after deleting owned sources, rows and local cached results;
73
+ raise on an unprovable/partial purge. SyncmillMemory discovers this optional
74
+ async capability without requiring it from fileless or third-party adapters.
75
+ """
76
+
77
+ @property
78
+ def capabilities(self) -> BackendCapabilities: ...
79
+
80
+ @property
81
+ def agent_namespace_prefix(self) -> str: ...
82
+
83
+ @property
84
+ def shared_namespace(self) -> str: ...
85
+
86
+ def validate_namespace(self, ns: str) -> bool: ...
87
+
88
+ async def add(self, content: str, *, tags: list[str], namespace: str) -> None: ...
89
+
90
+ async def search(
91
+ self,
92
+ query: str,
93
+ *,
94
+ namespace: str,
95
+ top_k: int,
96
+ tag_filter: str | None = None,
97
+ ) -> list[MemoryRow]: ...
98
+
99
+ async def search_multi(
100
+ self,
101
+ query: str,
102
+ *,
103
+ namespaces: Sequence[str],
104
+ top_k: int,
105
+ tag_filter: str | None = None,
106
+ ) -> list[MemoryRow]: ...
107
+
108
+ async def add_returning_ids(
109
+ self, content: str, *, tags: list[str], namespace: str
110
+ ) -> list[str]:
111
+ """Like ``add`` but returns every freshly indexed chunk id (a long
112
+ content can split into several chunks). Empty when the store minted no
113
+ fresh id (e.g. a content-hash dedup). May raise ``WriteBlockedError``."""
114
+ ...
115
+
116
+ async def share_chunk(
117
+ self, chunk_id: str, *, target_namespace: str
118
+ ) -> str | None: ...
119
+
120
+ async def register_namespace(
121
+ self, ns: str, *, description: str | None = None, color: str | None = None
122
+ ) -> None:
123
+ """Unconditional description/color metadata upsert (mem_agent_register
124
+ semantics; ``None`` leaves a field untouched). Metadata is a
125
+ memtomem-native concept — mem0/LangGraph adapters no-op, exactly as
126
+ their ``ensure_namespace`` already does."""
127
+ ...
128
+
129
+ async def get(self, chunk_id: str) -> MemoryRow | None: ...
130
+
131
+ async def ensure_namespace(self, ns: str, *, description: str = "") -> None: ...
132
+
133
+ async def ensure_ready(self) -> None: ...
134
+
135
+ async def status(self) -> dict[str, Any]: ...
136
+
137
+ async def close(self) -> None: ...
138
+
139
+ async def count_namespace(self, ns: str) -> int: ...
140
+
141
+ async def delete_namespace(self, ns: str) -> int: ...
142
+
143
+ async def start_session(self, agent_id: str, namespace: str) -> str | None: ...
144
+
145
+ async def log_event(self, kind: str, text: str) -> None: ...
146
+
147
+ async def end_session(self, summary: str | None) -> None: ...
148
+
149
+
150
+ # The mem_agent_share provenance tag. On a re-share the chain is collapsed to
151
+ # the immediate parent only (prior markers dropped) — mirroring memtomem's
152
+ # _build_shared_tags so syncmill copies read identically to MCP-tool shares.
153
+ _SHARED_FROM_PREFIX = "shared-from="
154
+
155
+ # Ceiling on how many source locks one purge will hold at once. Each is an open
156
+ # file descriptor held for the whole operation, so an unbounded namespace could
157
+ # exhaust the process limit mid-purge — with the board GC waiting on it, since
158
+ # collection runs after an outcome or an empty claim. Refusing loudly leaves the
159
+ # namespace collectable, which is the same fail-closed posture as every other
160
+ # unprovable outcome here. A board-run scratch namespace writes a handful of
161
+ # day files, so this is a guard against pathology, not a working limit.
162
+ _MAX_PURGE_SOURCE_LOCKS = 256
163
+
164
+
165
+ def _get_memtomem_atomic() -> Any:
166
+ """Import memtomem's atomic file locking helpers with cross-version compatibility.
167
+
168
+ memtomem 0.5+ moved _atomic from memtomem.context._atomic to memtomem._atomic.
169
+ """
170
+ try:
171
+ from memtomem import _atomic
172
+
173
+ return _atomic
174
+ except ImportError:
175
+ from memtomem.context import _atomic
176
+
177
+ return _atomic
178
+
179
+
180
+ def _utc_date_str() -> str:
181
+ """Daily memory-file name, matching MemtomemStore.add's target choice."""
182
+ return datetime.now(timezone.utc).strftime("%Y-%m-%d")
183
+
184
+
185
+ def _user_base(config: Any) -> Path:
186
+ """The user-tier base every syncmill write and purge is anchored to.
187
+
188
+ No ``allow_project_tier``: that flag is for a caller with its own consent
189
+ gate for git-tracked (``project_shared``) writes, and syncmill writes
190
+ unattended with none — so a base that is a registered project tier is
191
+ refused, the posture upstream applies to its derived-target writers.
192
+ """
193
+ from memtomem.memory_scope import require_user_base
194
+
195
+ return require_user_base(
196
+ config.indexing.memory_dirs, config.indexing.project_memory_dirs
197
+ )
198
+
199
+
200
+ def _target_tier(config: Any, path: Path) -> str:
201
+ """The ADR-0011 tier memtomem gates and labels ``path`` with.
202
+
203
+ Upstream's own resolver, so this boundary and memtomem's write gate can
204
+ never disagree about a path's tier. It raises for a path whose owning tier
205
+ and indexed tier differ, or that sits in an unregistered project tree.
206
+ """
207
+ from memtomem.integrations.langgraph import _resolve_target_scope
208
+
209
+ return _resolve_target_scope(
210
+ path.expanduser().resolve(),
211
+ config.indexing.memory_dirs,
212
+ config.indexing.project_memory_dirs,
213
+ )
214
+
215
+
216
+ def _read_only_roots(config: Any) -> set[Path]:
217
+ """Index roots memtomem reads but may never rewrite (``read_only_memory_dirs``)."""
218
+ return {
219
+ Path(d).expanduser().resolve()
220
+ for d in getattr(config.indexing, "read_only_memory_dirs", ()) or ()
221
+ }
222
+
223
+
224
+ def _refuse_unwritable_target(comp: Any, target: Path) -> None:
225
+ """Refuse a day file this unattended writer must not append to.
226
+
227
+ Mirrors the checks memtomem 0.6's own add makes on its derived target,
228
+ narrowed to what syncmill can do without asking anyone: a symlinked day
229
+ file (syncmill never makes one, and following it can land the append in
230
+ a tier the base check never saw), a target outside the user tier, a
231
+ read-only source, or one the indexer excludes (it would take the append
232
+ and index nothing). Raised errors degrade the write like any backend
233
+ failure.
234
+ """
235
+ if target.is_symlink():
236
+ raise RuntimeError(f"refusing to append to a symlinked day file: {target}")
237
+ tier = _target_tier(comp.config, target)
238
+ if tier != "user":
239
+ raise RuntimeError(
240
+ f"refusing an unattended write into the {tier} tier: {target}"
241
+ )
242
+ if comp.index_engine.is_read_only_source(target):
243
+ raise RuntimeError(f"refusing to append to a read-only source: {target}")
244
+ if comp.index_engine.is_excluded(target):
245
+ raise RuntimeError(f"refusing to append to an excluded source: {target}")
246
+
247
+
248
+ def _namespace_lock_path(base: Path, ns: str) -> Path:
249
+ """syncmill's own per-namespace lock, held ABOVE memtomem's per-file one.
250
+
251
+ A purge has to exclude writers to a NAMESPACE, and memtomem's locks are
252
+ per-FILE (its L2 sidecar), so the natural move — lock the file a writer is
253
+ about to touch — cannot work: the writer derives its target BEFORE taking
254
+ any lock, so a process suspended between those two steps (a stopped
255
+ process, a descheduled thread, a slow guard) resumes holding a target for
256
+ an arbitrarily old date. Enumerating candidate dates cannot fix that, since
257
+ there is no bound on how long a writer may hold a stale target. This is the
258
+ same shape as the bundle-path defect in the workflow store: a paused
259
+ process defeats any check-then-act, so the exclusion has to be on the thing
260
+ both sides agree about — here the namespace — and it has to be taken before
261
+ the racing side computes anything.
262
+
263
+ So this lock is keyed by (memory base, namespace) and nothing else, and the
264
+ write path takes it before it reads the clock. Being date-free is the whole
265
+ point: no argument about rollovers is required, and none is made.
266
+
267
+ **Lock ordering.** This sits ABOVE memtomem's documented memory-file domain
268
+ (``context/_atomic.py``, L0-L4): a holder may go on to take an L2 sidecar,
269
+ never the reverse. Only syncmill takes it, so that order is enforceable by
270
+ inspection of this module alone.
271
+
272
+ It lives beside the day files rather than in syncmill's state root because
273
+ the writer must already be able to write there, so this adds no new failure
274
+ mode to an advisory write path — an unwritable state root would otherwise
275
+ turn every memory write into an error. The leading dot matches memtomem's
276
+ own sidecar convention, and the name can never satisfy ``_is_day_file``, so
277
+ the sweep does not treat it as a source file.
278
+ """
279
+ digest = hashlib.sha256(f"{os.path.realpath(base)}\0{ns}".encode()).hexdigest()[:32]
280
+ return base / f".syncmill-ns-{digest}.lock"
281
+
282
+
283
+ # ASCII digits only: `\d` also matches other Unicode decimal digits, so a
284
+ # full-width `2026-08-01` would pass a date the writer cannot produce.
285
+ _DAY_FILE_DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")
286
+
287
+
288
+ def _is_day_file(name: str, suffix: str) -> bool:
289
+ """Whether ``name`` is a day file for ``suffix`` — a DATE plus that suffix.
290
+
291
+ Suffix-matching alone is not enough at a destructive boundary: it accepts
292
+ any prefix, so an unrelated ``backup--{slug}-{digest}.md`` or
293
+ ``notes.{ns}.md`` would be unlinked. The namespace digest proves which
294
+ namespace a day file belongs to, never that the file IS a day file.
295
+
296
+ The date must be one ``strftime("%Y-%m-%d")`` could have produced, shape
297
+ AND calendar: every name this accepts is one syncmill or memtomem wrote,
298
+ so anything a writer cannot emit — ``2026-99-99``, non-ASCII digits — is
299
+ somebody else's file that happens to end the same way.
300
+ """
301
+ if not name.endswith(suffix):
302
+ return False
303
+ date_part = name[: -len(suffix)]
304
+ if not _DAY_FILE_DATE.fullmatch(date_part):
305
+ return False
306
+ try:
307
+ datetime.strptime(date_part, "%Y-%m-%d")
308
+ except ValueError:
309
+ return False
310
+ return True
311
+
312
+
313
+ def _index_roots(config: Any, base: Path) -> list[Path]:
314
+ """Every directory the indexer treats as a root, user tier first.
315
+
316
+ Sweeping only the user tier leaves a native day file under a registered
317
+ project root on disk, where the next pass re-indexes it exactly like a
318
+ user-tier one — so the namespace the caller was told had been purged comes
319
+ back. ``all_index_roots`` is upstream's own single source of truth for
320
+ consumers that must act on every indexable file.
321
+
322
+ Every root is verified to be a readable directory and anything else RAISES,
323
+ including a missing ``all_index_roots`` (every pinned version has it, so its
324
+ absence is version skew, not an old-config shape to guess around). This
325
+ boundary fails CLOSED for the same reason the caller does: skipping a root
326
+ that is merely unreachable right now — an unmounted project tier — would
327
+ report a clean zero-file purge, let the caller tombstone the namespace, and
328
+ resurrect it when the root came back. An unverifiable root means the
329
+ namespace stays collectable and a later sweep retries it.
330
+
331
+ ``base`` (the caller's already-resolved ``require_user_base``) leads so the
332
+ two never fork on how the user tier is resolved, but it is checked like any
333
+ other root.
334
+ """
335
+ all_roots = getattr(config.indexing, "all_index_roots", None)
336
+ if not callable(all_roots):
337
+ raise RuntimeError(
338
+ "memtomem config exposes no all_index_roots(); cannot prove every "
339
+ "index root was swept"
340
+ )
341
+ roots: dict[Path, None] = dict.fromkeys(
342
+ [base, *(Path(e).expanduser().resolve() for e in all_roots())]
343
+ )
344
+ for root in roots:
345
+ if not root.is_dir() or not os.access(root, os.R_OK | os.X_OK):
346
+ raise RuntimeError(
347
+ f"index root is not a readable directory: {root}; cannot prove "
348
+ "its sources were swept"
349
+ )
350
+ return list(roots)
351
+
352
+
353
+ def _namespace_day_file_suffixes(config: Any, ns: str) -> list[str]:
354
+ """The day-file suffixes that identify ``ns``, both write shapes.
355
+
356
+ ``{date}.{ns}.md`` is what this backend's replicated add flow writes;
357
+ ``{date}--{slug}-{digest}.md`` is what memtomem (0.4+) natively writes for the
358
+ same namespace. The native suffix comes from upstream's own
359
+ ``day_file_name``, so an upstream rename breaks a test rather than silently
360
+ stranding files.
361
+ """
362
+ from memtomem.memory_scope import day_file_name
363
+
364
+ suffixes = [f".{ns.replace(':', '.')}.md"]
365
+ # day_file_name gives the DEFAULT namespace the bare "{date}.md" shape,
366
+ # leaving just ".md" for an empty date_str — a suffix every file in the
367
+ # store ends with. Only the digest-suffixed non-default shape is swept.
368
+ native_suffix = day_file_name(ns, config.namespace.default_namespace, date_str="")
369
+ if native_suffix != ".md":
370
+ suffixes.append(native_suffix)
371
+ return suffixes
372
+
373
+
374
+ def _namespace_source_paths(config: Any, ns: str) -> list[Path]:
375
+ """Every day file belonging to ``ns``, across every index root.
376
+
377
+ The single matcher. ``_sweep_namespace_sources`` unlinks what this returns
378
+ and ``purge_namespace`` locks it first, so the two can never disagree about
379
+ which files a namespace owns — the matching is the destructive boundary.
380
+ """
381
+ # Raises on an empty memory_dirs, which the contract wants surfaced
382
+ # rather than reported as a zero-file purge.
383
+ base = _user_base(config)
384
+ suffixes = _namespace_day_file_suffixes(config, ns)
385
+ found: list[Path] = []
386
+ for root in _index_roots(config, base):
387
+ # scandir, not glob: pathlib SWALLOWS a scan-time OSError and yields
388
+ # nothing, so an unreadable root would look like an empty one and
389
+ # report a clean purge over files it never managed to look at. The
390
+ # os.access preflight cannot settle this on its own — it can
391
+ # disagree with the real operation under ACLs, and the permissions
392
+ # can change between the check and the scan.
393
+ with os.scandir(root) as entries:
394
+ for entry in entries:
395
+ if any(_is_day_file(entry.name, suffix) for suffix in suffixes):
396
+ found.append(Path(entry.path))
397
+ return found
398
+
399
+
400
+ class UnprovablePurgeError(RuntimeError):
401
+ """Purge is unacknowledged; partial deletion may have occurred and is retryable."""
402
+
403
+
404
+ def _check_purge_descriptor_budget(needed: int) -> None:
405
+ """Leave room for SQLite, tracing and concurrent threads while holding L2.
406
+
407
+ The 256-source ceiling alone exceeds a stock macOS 256-fd soft limit.
408
+ Measure current headroom instead of raising the process-wide limit. This
409
+ is admission control, not a reservation: a later OS refusal still unwinds
410
+ every acquired lock before the exception reaches the GC caller.
411
+ """
412
+ import resource
413
+
414
+ soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
415
+ if soft == resource.RLIM_INFINITY:
416
+ return
417
+ fd_root = (
418
+ Path("/proc/self/fd") if Path("/proc/self/fd").is_dir() else Path("/dev/fd")
419
+ )
420
+ available = max(0, soft - len(os.listdir(fd_root)) - 32)
421
+ if needed > available:
422
+ raise UnprovablePurgeError(
423
+ f"purge needs {needed} source files locked but only {available} "
424
+ "file descriptors are available with safety headroom; "
425
+ "raise the process file-descriptor limit or review the store; "
426
+ "no source files were deleted"
427
+ )
428
+
429
+
430
+ def _purgeable_sources(
431
+ config: Any, ns: str, roots: list[Path], paths: Iterable[Path]
432
+ ) -> set[Path]:
433
+ """The subset of ``paths`` this purge may lock and unlink for ``ns``.
434
+
435
+ Held to one rule whatever the origin: an exact ``{date}`` + namespace-suffix
436
+ name, directly inside one of the roots the indexer reads. The disk scan
437
+ satisfies it by construction; DATABASE paths have had no such check, and a
438
+ row's ``source_file`` is data, not a promise — the pre-upgrade residue is a
439
+ namespace-SHARED ``{date}.md`` this store deliberately never deletes, and a
440
+ corrupt row could name anything at all.
441
+
442
+ **Symlinks are refused outright, not skipped.** memtomem RESOLVES a path
443
+ before deriving its sidecar, so a link and its target have different lock
444
+ identities: the purge would take ``.link.lock`` while an indexer holds
445
+ ``.target.lock``, exclude nothing, and still report success. Rather than
446
+ mirror upstream's resolution — coupling this boundary to an implementation
447
+ detail that can drift — a symlinked candidate makes the whole purge
448
+ unprovable. Board-run scratch files are never symlinks, so this is a loud
449
+ refusal for a store that is not what this code assumes, not a working path.
450
+
451
+ A DB path that fails the name/root rule is likewise not silently dropped:
452
+ it may be exactly the in-flight reader whose lock we needed. Ignoring it is
453
+ how rows come back after an acknowledged purge, so it too is unprovable.
454
+ """
455
+ keep: set[Path] = set()
456
+ suffixes = _namespace_day_file_suffixes(config, ns)
457
+ read_only = _read_only_roots(config)
458
+ for path in paths:
459
+ if path.is_symlink():
460
+ raise UnprovablePurgeError(
461
+ f"source {path} for {ns} is a symlink, whose lock identity "
462
+ "differs from the one memtomem derives; refusing to purge"
463
+ )
464
+ # Refused, not skipped, and before anything is deleted: a skipped
465
+ # source would survive the purge with its rows gone, and the caller
466
+ # would tombstone a namespace that comes back on the next reindex.
467
+ if path.parent.resolve() in read_only:
468
+ raise UnprovablePurgeError(
469
+ f"source {path} for {ns} is in a read-only memory root, which "
470
+ "memtomem may index but never rewrite; refusing to purge"
471
+ )
472
+ try:
473
+ tier = _target_tier(config, path)
474
+ except Exception as exc:
475
+ raise UnprovablePurgeError(
476
+ f"cannot establish the tier of source {path} for {ns}: {exc}"
477
+ ) from exc
478
+ if tier == "project_shared":
479
+ raise UnprovablePurgeError(
480
+ f"source {path} for {ns} is in a git-tracked project_shared tier; "
481
+ "deleting it needs a consent syncmill cannot ask for unattended"
482
+ )
483
+ if path.parent in roots and any(
484
+ _is_day_file(path.name, suffix) for suffix in suffixes
485
+ ):
486
+ keep.add(path)
487
+ continue
488
+ raise UnprovablePurgeError(
489
+ f"the database attributes {path} to {ns}, but it is not a day file "
490
+ "inside an index root; it cannot be locked, so this purge cannot "
491
+ "prove it removed everything"
492
+ )
493
+ return keep
494
+
495
+
496
+ def _sweep_namespace_sources(config: Any, ns: str) -> int:
497
+ """Unlink every day file belonging to ``ns``, across every index root."""
498
+ removed = 0
499
+ roots = _index_roots(config, _user_base(config))
500
+ for path in _purgeable_sources(config, ns, roots, _namespace_source_paths(config, ns)):
501
+ os.unlink(path)
502
+ removed += 1
503
+ return removed
504
+
505
+
506
+ def _entry_title() -> str:
507
+ """A heading that is unique per write.
508
+
509
+ Historical rationale: memtomem 0.3's ``append_entry`` defaulted to
510
+ ``## Entry {now}`` at SECOND resolution, and two entries under the same
511
+ heading in one file are re-chunked into a SINGLE chunk — so a second write
512
+ inside the same second silently swallowed the first entry's text and
513
+ RE-MINTED its chunk id. That id had already been returned to the caller,
514
+ so the provenance share that follows looked the chunk up, found nothing,
515
+ and fell back to a direct write with the link quietly gone. Two rapid
516
+ writes to one namespace is not a corner case here: it is what a
517
+ pipeline/council/decompose handoff does.
518
+
519
+ memtomem 0.4's own default is now unique too (ms timestamp + full uuid);
520
+ syncmill keeps supplying an explicit title of the same strength because
521
+ the deterministic shape is pinned by hermetic tests and must not drift
522
+ with the upstream default. It stays a plain ``## …`` heading, which is all
523
+ the chunker requires, and only the heading changes — the ``> created:``
524
+ line memtomem's own parsers read keeps its second resolution.
525
+ """
526
+ stamp = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
527
+ return f"Entry {stamp} {uuid4().hex}"
528
+
529
+
530
+ def _shared_tags(tags: Sequence[str], source_id: str) -> list[str]:
531
+ kept = [t for t in tags if not t.startswith(_SHARED_FROM_PREFIX)]
532
+ kept.append(f"{_SHARED_FROM_PREFIX}{source_id}")
533
+ return kept
534
+
535
+
536
+ def _merge_rows(row_lists: Iterable[list[MemoryRow]], top_k: int) -> list[MemoryRow]:
537
+ """Adapter-side multi-namespace merge: dedupe by id keeping the highest
538
+ score, sort by score descending, truncate. The fallback for backends
539
+ without a native union-namespace search."""
540
+ merged: dict[str, MemoryRow] = {}
541
+ for rows in row_lists:
542
+ for row in rows:
543
+ prior = merged.get(row["id"])
544
+ if prior is None or row.get("score", 0.0) > prior.get("score", 0.0):
545
+ merged[row["id"]] = row
546
+ return sorted(merged.values(), key=lambda row: -row.get("score", 0.0))[:top_k]
547
+
548
+
549
+ _FENCE_RE = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})(.*)$")
550
+ _HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$")
551
+
552
+
553
+ def _propagate_entry_tags_to_headings(content: str, tags: list[str]) -> str:
554
+ """Propagate entry tags to interior markdown headings.
555
+
556
+ Memtomem daily files chunk by ATX headings matching `^(#{1,6})\\s+(.+)$`.
557
+ append_entry emits the `> tags:` blockquote only under the entry's
558
+ top heading. When an entry contains interior markdown headings,
559
+ memtomem splits the entry into multiple chunks, but subsequent chunks
560
+ lack a `> tags:` blockquote and receive empty tags (tags=()), making them
561
+ unreachable via `tag_filter` (e.g. `board-result:item-42`).
562
+
563
+ Attaching `> tags: [...]` directly beneath each interior heading
564
+ (outside fenced code blocks) ensures every resulting chunk inherits
565
+ the entry's tags while preserving the verbatim content and heading
566
+ structure without mutating text into bold labels.
567
+ """
568
+ if not tags:
569
+ return content
570
+ tag_line = f"> tags: {json.dumps(list(tags), ensure_ascii=False)}"
571
+ lines = content.splitlines(keepends=True)
572
+ out: list[str] = []
573
+ fence_char: str | None = None
574
+ fence_len = 0
575
+
576
+ for line in lines:
577
+ line_no_ending = line.rstrip("\r\n")
578
+ fence_match = _FENCE_RE.match(line_no_ending)
579
+
580
+ if fence_char is None:
581
+ if fence_match:
582
+ run = fence_match.group(1)
583
+ fence_char = run[0]
584
+ fence_len = len(run)
585
+ out.append(line)
586
+ continue
587
+ else:
588
+ if (
589
+ fence_match
590
+ and fence_match.group(1)[0] == fence_char
591
+ and len(fence_match.group(1)) >= fence_len
592
+ and not fence_match.group(2).strip()
593
+ ):
594
+ fence_char = None
595
+ fence_len = 0
596
+ out.append(line)
597
+ continue
598
+
599
+ out.append(line)
600
+ if _HEADING_RE.match(line_no_ending):
601
+ line_ending = "\r\n" if line.endswith("\r\n") else "\n"
602
+ if not line.endswith(("\r", "\n")):
603
+ out.append(line_ending)
604
+ out.append(f"{tag_line}{line_ending}\n")
605
+
606
+ return "".join(out)
607
+
608
+
609
+ class MemtomemBackend:
610
+ """Default backend and the sole owner of memtomem-private coupling."""
611
+
612
+ def __init__(self, config: MemoryConfig):
613
+ # Keep imports lazy: import or construction failures are advisory and are
614
+ # caught by SyncmillMemory's existing construction boundary.
615
+ from memtomem.constants import AGENT_NAMESPACE_PREFIX, SHARED_NAMESPACE
616
+
617
+ try:
618
+ from memtomem.integrations.langgraph import MemtomemStore
619
+ except (ImportError, AttributeError):
620
+ from memtomem.integrations import MemtomemStore
621
+
622
+ self._agent_prefix = AGENT_NAMESPACE_PREFIX
623
+ self._shared_ns = SHARED_NAMESPACE
624
+ overrides: dict[str, dict[str, Any]] = {
625
+ "storage": {"sqlite_path": config.db_path}
626
+ }
627
+ if config.memory_dir is not None:
628
+ overrides["indexing"] = {"memory_dirs": [config.memory_dir]}
629
+ # Kept for status(): reading it back off the store would touch the
630
+ # private `_config_overrides`, and an upstream rename would then report
631
+ # version skew as a connection failure.
632
+ self._overrides = overrides
633
+ self._store = MemtomemStore(config_overrides=overrides)
634
+ self._caps = BackendCapabilities()
635
+
636
+ @property
637
+ def capabilities(self) -> BackendCapabilities:
638
+ return self._caps
639
+
640
+ @property
641
+ def agent_namespace_prefix(self) -> str:
642
+ return self._agent_prefix
643
+
644
+ @property
645
+ def shared_namespace(self) -> str:
646
+ return self._shared_ns
647
+
648
+ def validate_namespace(self, ns: str) -> bool:
649
+ return bool(self._caps.namespace_pattern.fullmatch(ns))
650
+
651
+ async def add(self, content: str, *, tags: list[str], namespace: str) -> None:
652
+ # The SAME flow as add_returning_ids (ids discarded), so both writes land
653
+ # in the namespace-suffixed daily file. Routing this through
654
+ # MemtomemStore.add instead would append to the namespace-SHARED
655
+ # `{date}.md`, which breaks namespace isolation in two ways. The markdown
656
+ # is the index's source of truth while delete_by_namespace only removes
657
+ # DB rows, so a GC'd `board-run:*` chunk survives in a file that a later
658
+ # `shared` write re-indexes — and, its rows now gone, its text is treated
659
+ # as new and stamped `shared`, leaking board-internal content into global
660
+ # knowledge. And because one file's chunks are content-hash keyed, an
661
+ # identical write to a second namespace is classified unchanged and
662
+ # silently dropped.
663
+ await self.add_returning_ids(content, tags=tags, namespace=namespace)
664
+
665
+ async def search(
666
+ self,
667
+ query: str,
668
+ *,
669
+ namespace: str,
670
+ top_k: int,
671
+ tag_filter: str | None = None,
672
+ ) -> list[MemoryRow]:
673
+ if tag_filter is not None:
674
+ return await self._store.search(
675
+ query,
676
+ namespace=namespace,
677
+ top_k=top_k,
678
+ tag_filter=tag_filter,
679
+ )
680
+ return await self._store.search(query, namespace=namespace, top_k=top_k)
681
+
682
+ async def search_multi(
683
+ self,
684
+ query: str,
685
+ *,
686
+ namespaces: Sequence[str],
687
+ top_k: int,
688
+ tag_filter: str | None = None,
689
+ ) -> list[MemoryRow]:
690
+ # One native merged-ranking call: NamespaceFilter.parse treats a comma
691
+ # list as a union filter — the same shape mem_agent_search builds. The
692
+ # namespace charset pattern forbids commas, but callers (search_context
693
+ # builds names from free-form agent ids) don't all validate — so enforce
694
+ # it here, or a comma-bearing name would silently widen the union
695
+ # filter to namespaces the caller never asked for. Raising is fine on
696
+ # this contract; SyncmillMemory degrades per call.
697
+ if any("," in ns for ns in namespaces):
698
+ raise ValueError(f"namespace contains a comma: {list(namespaces)!r}")
699
+ namespace = ",".join(namespaces)
700
+ if tag_filter is not None:
701
+ return await self._store.search(
702
+ query,
703
+ namespace=namespace,
704
+ top_k=top_k,
705
+ tag_filter=tag_filter,
706
+ )
707
+ return await self._store.search(query, namespace=namespace, top_k=top_k)
708
+
709
+ async def add_returning_ids(
710
+ self, content: str, *, tags: list[str], namespace: str
711
+ ) -> list[str]:
712
+ # MemtomemStore.add discards IndexingStats.new_chunk_ids, so replicate
713
+ # its short write flow (guard → lock → mix-refusal → append_entry →
714
+ # index_file(lock_held=True)) to keep the ids. Lazy imports:
715
+ # construction failures stay advisory.
716
+ atomic = _get_memtomem_atomic()
717
+ _CRUD_SIDECAR_LOCK_BUDGET_S = atomic._CRUD_SIDECAR_LOCK_BUDGET_S
718
+ async_file_lock = atomic.async_file_lock
719
+ from memtomem.privacy import enforce_write_guard
720
+
721
+ comp = await self._store._ensure_init()
722
+ guard = enforce_write_guard(
723
+ content,
724
+ surface="syncmill_add",
725
+ audit_context={"namespace": namespace},
726
+ )
727
+ # `blocked_project_shared` (memtomem 0.6) is the hard refusal for a
728
+ # git-tracked tier; unreachable while every syncmill target is `user`,
729
+ # but a decision this code does not name must never fall through as
730
+ # an allow.
731
+ if guard.decision in ("blocked", "blocked_project_shared"):
732
+ raise WriteBlockedError(
733
+ f"memtomem redaction guard refused the write ({guard.decision})"
734
+ )
735
+ base = _user_base(comp.config)
736
+ # BEFORE the clock is read. `purge_namespace` holds this same lock for
737
+ # the whole of its sweep → row delete → verify, so a write either
738
+ # completes entirely before a purge begins or waits for it — never
739
+ # lands a file whose rows the purge then takes. Deriving the target
740
+ # first and locking only that file cannot give this: a writer suspended
741
+ # between the two resumes with a stale target the purge never knew to
742
+ # lock (see `_namespace_lock_path`).
743
+ async with async_file_lock(
744
+ _namespace_lock_path(base, namespace),
745
+ timeout=_CRUD_SIDECAR_LOCK_BUDGET_S,
746
+ ):
747
+ return await self._locked_append(
748
+ comp, base, content, tags=tags, namespace=namespace
749
+ )
750
+
751
+ async def _locked_append(
752
+ self,
753
+ comp: Any,
754
+ base: Path,
755
+ content: str,
756
+ *,
757
+ tags: list[str],
758
+ namespace: str,
759
+ ) -> list[str]:
760
+ """The append itself, with this namespace's writers already excluded."""
761
+ atomic = _get_memtomem_atomic()
762
+ _CRUD_SIDECAR_LOCK_BUDGET_S = atomic._CRUD_SIDECAR_LOCK_BUDGET_S
763
+ memory_lock_path = atomic.memory_lock_path
764
+ async_file_lock = atomic.async_file_lock
765
+ from memtomem.memory_scope import namespace_mix_refusal
766
+ from memtomem.tools.memory_writer import append_entry
767
+
768
+ # A NAMESPACE-SUFFIXED daily file, deliberately diverging from
769
+ # MemtomemStore.add's shared `{date}.md`: under memtomem's default
770
+ # config, re-indexing one file that holds the same content-hash twice
771
+ # collapses the chunks to the LAST write — so the two-step share
772
+ # (identical content in agent-ns then `shared`, the mem_agent_share
773
+ # copy) would DELETE the private original and orphan the provenance
774
+ # link (a real FOREIGN KEY failure the opt-in real smoke caught).
775
+ # Separate files per namespace keep both chunks alive. The ':'→'.'
776
+ # mapping could collide only for a namespace literally containing the
777
+ # dotted form (e.g. "agent-runtime.claude") — unreachable with
778
+ # syncmill's fixed namespace vocabulary.
779
+ target = base / f"{_utc_date_str()}.{namespace.replace(':', '.')}.md"
780
+ # An EXPLICIT per-write heading, never append_entry's second-resolution
781
+ # default: two entries sharing a heading in one file re-chunk into one,
782
+ # swallowing the earlier text and re-minting the id already handed to
783
+ # the caller (see `_entry_title`).
784
+ #
785
+ # `format_entry_block` IGNORES the title when the content supplies its
786
+ # own `## ` heading, and in that branch it emits the tags blockquote
787
+ # BEFORE that heading — so such a payload gets a shared heading (the
788
+ # merge this guards against) AND loses its tags to the previous entry's
789
+ # section. syncmill's writers deliberately label themselves in bold
790
+ # instead (see memory.py's heading note); warn rather than mutate the
791
+ # content, since the backend contract is verbatim storage.
792
+ # `## ` ONLY, because that is the branch the pinned writer actually
793
+ # takes: memtomem's `format_entry_block` (0.4 through 0.6.8) tests
794
+ # `stripped.startswith("## ")`. Widening this to any ATX heading was
795
+ # wrong — calling the real formatter with `# H1` or `### H3` keeps the
796
+ # unique title AND the tags, so the warning would have described a
797
+ # failure that does not happen. If the upstream condition ever widens,
798
+ # widen this with it, against the real formatter.
799
+ if content.strip().startswith("## "):
800
+ logger.warning(
801
+ "memory payload starts with a '## ' heading, so its unique "
802
+ "per-entry heading and tags will be dropped by the writer; "
803
+ "label the payload in bold instead"
804
+ )
805
+ # memtomem's write-locking contract (since 0.4, its issue #2005): the append
806
+ # and its re-index must share one cross-process sidecar-lock span, or
807
+ # the watcher can index the freshly appended file FIRST — without the
808
+ # explicit namespace — and the agent-private chunk lands permanently in
809
+ # `default`, exposed to non-private searches with the provenance share
810
+ # broken (reproduced against the real 0.4 store). The mix-refusal read
811
+ # sits inside the same span for the same reason: a guard that inspects
812
+ # a file another writer may change before the append is decoration.
813
+ # `lock_held=True` skips index_file's nested sidecar acquire, which
814
+ # would otherwise self-deadlock (memtomem #1587). A lock timeout
815
+ # (another process holds it past the budget) raises TimeoutError, which
816
+ # SyncmillMemory degrades per call like any backend failure.
817
+ # Checked before the acquire as well as inside it: `memory_lock_path`
818
+ # resolves the target, so a symlinked day file would otherwise get its
819
+ # sidecar created wherever the link points. Inside the span the checks
820
+ # are authoritative, since the file can change until the lock is held.
821
+ _refuse_unwritable_target(comp, target)
822
+ # `memory_lock_path`, never `_lock_path_for`: memtomem's indexer keys
823
+ # the L2 sidecar on the RESOLVED path, and a lock keyed any other way
824
+ # would not exclude it (upstream #2130).
825
+ async with async_file_lock(
826
+ memory_lock_path(target), timeout=_CRUD_SIDECAR_LOCK_BUDGET_S
827
+ ):
828
+ _refuse_unwritable_target(comp, target)
829
+ mix_err = await namespace_mix_refusal(
830
+ index_engine=comp.index_engine,
831
+ storage=comp.storage,
832
+ default_namespace=comp.config.namespace.default_namespace,
833
+ target=target,
834
+ effective_ns=namespace,
835
+ override_hint="a different target file",
836
+ )
837
+ if mix_err is not None:
838
+ # Unreachable with syncmill's per-namespace files short of
839
+ # legacy residue; refusing beats restamping stored rows.
840
+ raise RuntimeError(
841
+ f"memtomem namespace mix refused for {target}: {mix_err}"
842
+ )
843
+ prepared = _propagate_entry_tags_to_headings(content, list(tags))
844
+ append_entry(target, prepared, title=_entry_title(), tags=list(tags))
845
+ stats = await comp.index_engine.index_file(
846
+ target, namespace=namespace, already_scanned=True, lock_held=True
847
+ )
848
+ # MCP-write parity — see `add`.
849
+ comp.search_pipeline.invalidate_cache()
850
+ return [str(cid) for cid in getattr(stats, "new_chunk_ids", ()) or ()]
851
+
852
+ async def share_chunk(self, chunk_id: str, *, target_namespace: str) -> str | None:
853
+ # mem_agent_share semantics: COPY the chunk into the target namespace
854
+ # with a collapsed shared-from tag, then record a chunk_links row per
855
+ # copied chunk. NOTE: like MemtomemStore.get, the source lookup is NOT
856
+ # namespace-scoped — today's only caller feeds ids it just minted; a
857
+ # future promote-style surface must add a source-namespace check (the
858
+ # gateway `get_shared` lesson) before exposing this to untrusted ids.
859
+ source = await self._store.get(chunk_id)
860
+ if source is None:
861
+ return None
862
+ new_ids = await self.add_returning_ids(
863
+ source["content"],
864
+ tags=_shared_tags(source.get("tags", []), chunk_id),
865
+ namespace=target_namespace,
866
+ )
867
+ for new_id in new_ids:
868
+ # Link recording is best-effort (the mem_agent_share posture): the
869
+ # copy is durable, so a failed link must not report a failed share.
870
+ try:
871
+ storage = await self._storage()
872
+ await storage.add_chunk_link(
873
+ UUID(chunk_id),
874
+ UUID(new_id),
875
+ "shared",
876
+ target_namespace,
877
+ )
878
+ except Exception:
879
+ logger.warning(
880
+ "share provenance link failed; copy is durable", exc_info=True
881
+ )
882
+ return new_ids[0] if new_ids else None
883
+
884
+ async def register_namespace(
885
+ self, ns: str, *, description: str | None = None, color: str | None = None
886
+ ) -> None:
887
+ # mem_agent_register semantics: unconditional upsert (None leaves the
888
+ # existing value untouched) — unlike ensure_namespace's create-if-missing.
889
+ await (await self._storage()).set_namespace_meta(
890
+ ns, description=description, color=color
891
+ )
892
+
893
+ async def get(self, chunk_id: str) -> MemoryRow | None:
894
+ return await self._store.get(chunk_id)
895
+
896
+ async def ensure_namespace(self, ns: str, *, description: str = "") -> None:
897
+ storage = await self._storage()
898
+ if await storage.get_namespace_meta(ns) is None:
899
+ await storage.set_namespace_meta(ns, description=description)
900
+
901
+ async def ensure_ready(self) -> None:
902
+ await self._store._ensure_init()
903
+
904
+ async def status(self) -> dict[str, Any]:
905
+ await self.ensure_ready()
906
+ # Our own overrides dict, not the store's private `_config_overrides` —
907
+ # an upstream rename of that attribute would surface as AttributeError,
908
+ # which SyncmillMemory.status reports as a CONNECTION failure (version
909
+ # skew misdiagnosed as an outage).
910
+ db_path = self._overrides.get("storage", {}).get("sqlite_path", "default")
911
+ info: dict[str, Any] = {"db_path": db_path}
912
+ try:
913
+ comp = getattr(self._store, "_components", None)
914
+ if comp is not None:
915
+ storage = getattr(comp, "storage", None)
916
+ if storage is not None and hasattr(storage, "get_stats"):
917
+ stats = await storage.get_stats()
918
+ if isinstance(stats, dict):
919
+ for k in (
920
+ "total_chunks",
921
+ "total_sources",
922
+ "source_files",
923
+ "dense_vectors",
924
+ ):
925
+ if k in stats:
926
+ info[k] = stats[k]
927
+ cfg = getattr(comp, "config", None)
928
+ if cfg is not None:
929
+ emb = getattr(cfg, "embedding", None)
930
+ if emb is not None:
931
+ if hasattr(emb, "model"):
932
+ info["embedding_model"] = emb.model
933
+ if hasattr(emb, "provider"):
934
+ info["embedding_provider"] = emb.provider
935
+ except Exception:
936
+ logger.debug("memtomem status diagnostic probe omitted", exc_info=True)
937
+ return info
938
+
939
+ async def close(self) -> None:
940
+ await self._store.close()
941
+
942
+ async def _storage(self) -> Any:
943
+ await self._store._ensure_init()
944
+ return self._store._components.storage
945
+
946
+ async def count_namespace(self, ns: str) -> int:
947
+ metas = await (await self._storage()).list_namespace_meta()
948
+ return next(
949
+ (m.get("chunk_count", 0) for m in metas if m.get("namespace") == ns),
950
+ 0,
951
+ )
952
+
953
+ async def delete_namespace(self, ns: str) -> int:
954
+ removed = await (await self._storage()).delete_by_namespace(ns)
955
+ await self._invalidate_search_cache()
956
+ return removed
957
+
958
+ async def _invalidate_search_cache(self, *, fail_closed: bool = False) -> None:
959
+ """Drop THIS PROCESS's search-pipeline TTL result cache after a delete.
960
+
961
+ The same MCP-parity rule the write path follows, for the same reason in
962
+ the other direction: the pipeline caches result sets, so a reader that
963
+ searched before a delete keeps being served the deleted rows until the
964
+ TTL expires. For a ``board-run:*`` purge that means handing a later
965
+ reader exactly the board-internal text the purge existed to remove —
966
+ the leak this whole path guards, arriving from memory instead of from a
967
+ reindex.
968
+
969
+ **Process-local, and that is a real limit.** The cache lives in the
970
+ pipeline object, so another process holding its own ``MemtomemBackend``
971
+ over the same store keeps serving its own cached rows until its TTL
972
+ expires. Nothing here can reach it; closing that needs a shared
973
+ generation or epoch upstream. So the purge's postcondition is stated in
974
+ terms of files, rows, and this process's cache — never "no cached copy
975
+ anywhere".
976
+
977
+ ``fail_closed`` for the purge path: the caller TOMBSTONES on success, so
978
+ a namespace whose cached rows this process may still serve must not be
979
+ reported as purged. The plain delete path stays best-effort, where a
980
+ stale cache expires on its own and there is no tombstone to mislead.
981
+ """
982
+ try:
983
+ comp = await self._store._ensure_init()
984
+ comp.search_pipeline.invalidate_cache()
985
+ except Exception:
986
+ if fail_closed:
987
+ raise
988
+ logger.warning("search cache invalidation failed", exc_info=True)
989
+
990
+ async def delete_namespace_sources(self, ns: str) -> int:
991
+ """Remove the markdown source files this backend wrote for ``ns``.
992
+
993
+ ``delete_by_namespace`` only removes database rows, but the markdown is
994
+ what the index is rebuilt FROM: a watcher, ``mm index``, or any later
995
+ write to the same file re-indexes the deleted text and — its rows now
996
+ gone — treats it as new content. Without this a deleted namespace comes
997
+ back on the next reindex.
998
+
999
+ Two file shapes hold a namespace's content, and both are matched: the
1000
+ ``{date}.{ns}.md`` files this backend's replicated add flow writes, and
1001
+ the ``{date}--{slug}-{digest}.md`` files memtomem (0.4+) natively writes
1002
+ for the same namespace (CLI ``mm add``, the ``mem_add`` MCP tool — the
1003
+ surfaces that call ``day_file_name``; the watcher only re-indexes what
1004
+ they wrote, which is what makes a surviving file resurrect).
1005
+
1006
+ A name is matched only when it is a whole ``{date}`` + one of those two
1007
+ suffixes: a bare ``*{suffix}`` glob would also take ``backup--{slug}-
1008
+ {digest}.md`` or ``notes.{ns}.md``, and the digest proves the namespace,
1009
+ never that syncmill wrote the file or that the prefix is a date. For the
1010
+ same reason ``ns`` is never interpolated INTO a glob pattern — it is
1011
+ caller-supplied, and a ``*`` or ``[`` in it would silently widen the
1012
+ sweep — so the glob is the literal ``*.md`` and every candidate is
1013
+ compared as an exact string.
1014
+
1015
+ RAISES on any failure rather than reporting a misleading zero: the
1016
+ caller tombstones a purged namespace, and a namespace whose source is
1017
+ still on disk has not been purged — it must stay collectable so a
1018
+ later sweep retries it. Every root the indexer reads is swept, not just
1019
+ the user tier: a file under a registered project root is re-indexed
1020
+ exactly like a user-tier one, so leaving it behind resurrects the
1021
+ namespace the caller was told had been purged. A root that cannot be
1022
+ verified (missing, not a directory, unreadable — an unmounted project
1023
+ tier) RAISES rather than being skipped, for the same reason: skipping
1024
+ it would report a clean purge over files still on disk.
1025
+
1026
+ **Known limitations.** (1) A store written by an earlier version put
1027
+ every namespace in one shared ``{date}.md``. Those files are not
1028
+ matched here and cannot be — the entries carry no namespace, so there
1029
+ is nothing to select on, and deleting the file would take shared
1030
+ knowledge with it. Scratch written before the upgrade can therefore
1031
+ still be re-indexed. (2) The legacy suffix maps ``:`` to ``.``, which is
1032
+ not injective: ``a:b`` and ``a.b`` produce the same file name. That
1033
+ comes from the WRITE side (see ``add``) and predates this method, and
1034
+ upstream's mix-guard means such a file only ever holds one of the two —
1035
+ so the exposure is deleting the other namespace's source, not mixing
1036
+ them. Unreachable for the only caller, which builds ``board-run:{uuid}``
1037
+ from a uuid4; closing it would mean changing the write-side name and
1038
+ orphaning every file already on disk.
1039
+ """
1040
+ config = (await self._store._ensure_init()).config
1041
+ return _sweep_namespace_sources(config, ns)
1042
+
1043
+ async def purge_namespace(self, ns: str) -> int:
1044
+ """Remove a namespace's sources AND rows with writers locked out.
1045
+
1046
+ The two-call sequence this replaces — sweep sources, then delete rows —
1047
+ had no exclusion between its halves. A write landing in that window
1048
+ creates a day file AND its row, the row delete then takes the row, and
1049
+ the markdown is left with nothing claiming it: the next reindex reads
1050
+ board-internal text as new content and stamps it with whatever
1051
+ namespace that pass resolves, which is how scratch reaches global
1052
+ ``shared``. Ordering and a second verify sweep NARROW that window; only
1053
+ holding the writers out closes it.
1054
+
1055
+ **What is excluded, exactly.** Two levels, because two different things
1056
+ touch these files.
1057
+
1058
+ WRITERS take the namespace lock: the add flow is the only producer of
1059
+ ``board-run:*`` content, it has a single write path (``add`` routes
1060
+ through ``add_returning_ids``), and that path takes this namespace's
1061
+ lock BEFORE it derives any file name. So a write either finishes before
1062
+ this purge starts or waits for it.
1063
+
1064
+ READERS — memtomem's watcher and manual indexers — take each file's L2
1065
+ sidecar instead, and an indexer that has read a file but not yet
1066
+ committed its rows would otherwise resurrect the namespace right after
1067
+ this method reported success. So the purge also takes the L2 sidecar of
1068
+ every source that EXISTS, enumerated from disk under the namespace lock.
1069
+ That is not the date enumeration that made the first version of this
1070
+ unsound: those were paths a writer might LATER name, unbounded because a
1071
+ suspended writer's target can be arbitrarily old; these are paths found
1072
+ on disk now.
1073
+
1074
+ An earlier version instead locked the writer's per-file L2 sidecar for
1075
+ today's and tomorrow's date. That was unsound, and the reason is worth
1076
+ keeping: the writer derives its target BEFORE taking any lock, so a
1077
+ process suspended in between resumes holding a target for an arbitrarily
1078
+ old date, which no finite set of dates covers. See
1079
+ ``_namespace_lock_path``.
1080
+
1081
+ **What is NOT excluded.** A memtomem-native writer (``mm add``, the
1082
+ ``mem_add`` MCP tool) does not take syncmill's namespace lock, so it can
1083
+ still interleave. Note this is NOT because its file name is unknowable —
1084
+ ``day_file_name`` derives it deterministically, and the digest is over
1085
+ the namespace, not the date — but because upstream has no namespace-level
1086
+ lock to share (its documented lock domain is per-file, L0-L4 in
1087
+ ``context/_atomic.py``) and locking its file paths would need the same
1088
+ unsound date enumeration. Nothing writes a ``board-run:{uuid4}`` scratch
1089
+ namespace that way, since syncmill mints those ids and no other surface
1090
+ knows them, but the guarantee is scoped to syncmill's own writers rather
1091
+ than absolute.
1092
+
1093
+ The verify sweep therefore stays, and now means something narrow and
1094
+ sharp: it can only fire for a native writer, since syncmill's are locked
1095
+ out and in-flight indexers are held. It RAISES rather than reporting a
1096
+ count, because a purge that cannot be proven complete must leave the
1097
+ namespace collectable — the caller tombstones on success, and a
1098
+ namespace with a live producer must not be retired from collection.
1099
+ Note what it is NOT: a proof of completeness. It samples once, so a
1100
+ native write landing after it is still missed; it is best-effort
1101
+ detection of the one interleaving the locks do not cover.
1102
+
1103
+ A lock the budget cannot obtain raises TimeoutError, which the caller
1104
+ degrades like any other backend failure — again leaving the namespace
1105
+ collectable, with no retry here (a scratch GC has a later chance, and
1106
+ blocking board dispatch on it would be worse).
1107
+
1108
+ Returns the number of source files removed.
1109
+ """
1110
+ from contextlib import AsyncExitStack
1111
+
1112
+ atomic = _get_memtomem_atomic()
1113
+ _CRUD_SIDECAR_LOCK_BUDGET_S = atomic._CRUD_SIDECAR_LOCK_BUDGET_S
1114
+ memory_lock_path = atomic.memory_lock_path
1115
+ async_file_lock = atomic.async_file_lock
1116
+
1117
+ config = (await self._store._ensure_init()).config
1118
+ base = _user_base(config)
1119
+ # ONE deadline for the whole operation, started BEFORE the namespace
1120
+ # acquisition. Starting it after would let that lock spend a full budget
1121
+ # and every source lock spend another, so a single purge could take two
1122
+ # budgets plus scans — and `delete_board_runs` walks up to 100
1123
+ # namespaces in one maintenance pass, multiplying it.
1124
+ deadline = monotonic() + _CRUD_SIDECAR_LOCK_BUDGET_S
1125
+
1126
+ def _remaining() -> float:
1127
+ left = deadline - monotonic()
1128
+ if left <= 0:
1129
+ raise TimeoutError(
1130
+ f"purge of {ns} exceeded {_CRUD_SIDECAR_LOCK_BUDGET_S}s "
1131
+ "before the purge completed"
1132
+ )
1133
+ return left
1134
+
1135
+ async with AsyncExitStack() as stack:
1136
+ await stack.enter_async_context(
1137
+ async_file_lock(_namespace_lock_path(base, ns), timeout=_remaining())
1138
+ )
1139
+ # The namespace lock stops a syncmill WRITE, but memtomem's watcher
1140
+ # and manual indexers are READERS of these files: they take the
1141
+ # file's L2 sidecar, not this lock. One can hold L2, read a
1142
+ # board-run file, and pause before its DB transaction — and if the
1143
+ # purge ran through, it would unlink the file, delete the rows,
1144
+ # verify clean, and report success, after which the indexer resumes
1145
+ # and upserts its buffered chunks into a namespace the caller has
1146
+ # already been told is gone.
1147
+ #
1148
+ # So take the L2 sidecar of every source before touching any of
1149
+ # them. The candidate set is the UNION of what is on disk and what
1150
+ # the DB still attributes to this namespace: a source an indexer is
1151
+ # holding can already have been unlinked externally, in which case
1152
+ # disk alone sees nothing and that indexer would commit its rows
1153
+ # after the purge returned. Neither source of truth covers the other.
1154
+ #
1155
+ # Re-snapshot after acquiring, because taking a lock takes time and
1156
+ # the set can grow underneath. The loop converges when a pass adds
1157
+ # nothing new.
1158
+ storage = await self._storage()
1159
+ locked: set[Path] = set()
1160
+ while True:
1161
+ # Verified ONCE per snapshot, and a failure propagates. Turning
1162
+ # it into "skip this path" would be fail-OPEN at the operation
1163
+ # level: a transient error while validating could drop exactly
1164
+ # the externally-unlinked source an indexer is holding, and the
1165
+ # purge would then delete its rows and report success.
1166
+ roots = _index_roots(config, base)
1167
+ candidates = set(_namespace_source_paths(config, ns))
1168
+ candidates |= _purgeable_sources(
1169
+ config,
1170
+ ns,
1171
+ roots,
1172
+ (Path(raw) for raw in await storage.list_sources_by_namespace(ns)),
1173
+ )
1174
+ # The disk half goes through the same rule, so a symlinked day
1175
+ # file is refused there too rather than locked under the wrong
1176
+ # identity.
1177
+ candidates = _purgeable_sources(config, ns, roots, candidates)
1178
+ fresh = candidates - locked
1179
+ if not fresh:
1180
+ break
1181
+ if len(locked) + len(fresh) > _MAX_PURGE_SOURCE_LOCKS:
1182
+ raise UnprovablePurgeError(
1183
+ f"namespace {ns} has more than {_MAX_PURGE_SOURCE_LOCKS} "
1184
+ "source files; refusing to hold that many locks at once "
1185
+ "rather than exhaust file descriptors mid-purge"
1186
+ )
1187
+ _check_purge_descriptor_budget(len(fresh))
1188
+ for lock_path in sorted({memory_lock_path(p) for p in fresh}, key=str):
1189
+ await stack.enter_async_context(
1190
+ async_file_lock(lock_path, timeout=_remaining())
1191
+ )
1192
+ locked |= fresh
1193
+ # SOURCES FIRST, still: the rows are what keep a surviving file
1194
+ # harmless, since a reindex that recognizes the text leaves its
1195
+ # namespace alone. Under the lock no syncmill writer can interleave,
1196
+ # so the pair is atomic against them; the order continues to decide
1197
+ # what a FAILURE between the two leaves behind.
1198
+ #
1199
+ # Only paths whose L2 this purge HOLDS are unlinked. Sweeping by
1200
+ # match instead would mutate a file that appeared after the snapshot
1201
+ # without holding its lock — the very thing the locking is for.
1202
+ removed = 0
1203
+ for path in sorted(locked, key=str):
1204
+ _remaining()
1205
+ try:
1206
+ os.unlink(path)
1207
+ except FileNotFoundError:
1208
+ # Either the DB knew a path disk no longer has, or something
1209
+ # external removed it. Its rows still go below; the count
1210
+ # reports what THIS purge removed.
1211
+ continue
1212
+ removed += 1
1213
+ _remaining()
1214
+ await storage.delete_by_namespace(ns)
1215
+ # Inside the span: a reader that searched before this purge would
1216
+ # otherwise keep being served the purged rows from this process's
1217
+ # TTL cache. Fail CLOSED here — the caller tombstones on success, so
1218
+ # a namespace whose cached rows may still be served locally must not
1219
+ # be reported as purged.
1220
+ _remaining()
1221
+ await self._invalidate_search_cache(fail_closed=True)
1222
+ # Anything still matching is a path this purge never held: a writer
1223
+ # it cannot exclude produced it.
1224
+ raced = len(_namespace_source_paths(config, ns))
1225
+ _remaining()
1226
+ if raced:
1227
+ raise UnprovablePurgeError(
1228
+ f"namespace {ns} gained {raced} source file(s) during its purge; "
1229
+ "a writer this lock cannot exclude is active, so the purge is "
1230
+ "not provably complete"
1231
+ )
1232
+ return removed
1233
+
1234
+ async def start_session(self, agent_id: str, namespace: str) -> str | None:
1235
+ return await self._store.start_session(agent_id=agent_id, namespace=namespace)
1236
+
1237
+ async def log_event(self, kind: str, text: str) -> None:
1238
+ await self._store.log_event(kind, text)
1239
+
1240
+ async def end_session(self, summary: str | None) -> None:
1241
+ await self._store.end_session(summary=summary)
1242
+
1243
+
1244
+ _LOCAL_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
1245
+ _LOCAL_DIMS = 384
1246
+
1247
+
1248
+ class _DisabledLLM:
1249
+ """Fail closed if a future mem0 path accidentally tries inference."""
1250
+
1251
+ def generate_response(self, *args: Any, **kwargs: Any) -> Any:
1252
+ raise RuntimeError("syncmill's mem0 backend disables LLM inference")
1253
+
1254
+
1255
+ def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
1256
+ merged = copy.deepcopy(base)
1257
+ for key, value in override.items():
1258
+ if isinstance(value, dict) and isinstance(merged.get(key), dict):
1259
+ merged[key] = _deep_merge(merged[key], value)
1260
+ else:
1261
+ merged[key] = copy.deepcopy(value)
1262
+ return merged
1263
+
1264
+
1265
+ def _resolve_mem0_config(config: MemoryConfig) -> tuple[Path, dict[str, Any]]:
1266
+ """Resolve Mem0 defaults without importing Mem0 or touching the filesystem.
1267
+
1268
+ Migration storage binding consumes the same resolved vector/history settings
1269
+ as the runtime adapter. Keeping this step pure lets migration commands reject
1270
+ a mismatched destination before they construct an SDK client or create a DB.
1271
+ """
1272
+ mem0_cfg = config.mem0
1273
+ data_dir = Path(mem0_cfg.data_dir).expanduser()
1274
+ defaults: dict[str, Any] = {
1275
+ "vector_store": {
1276
+ "provider": "qdrant",
1277
+ "config": {
1278
+ "collection_name": "syncmill",
1279
+ "path": str(data_dir / "qdrant"),
1280
+ "embedding_model_dims": _LOCAL_DIMS,
1281
+ },
1282
+ },
1283
+ "embedder": {
1284
+ "provider": "fastembed",
1285
+ "config": {
1286
+ "model": _LOCAL_MODEL,
1287
+ "embedding_dims": _LOCAL_DIMS,
1288
+ },
1289
+ },
1290
+ # AsyncMemory constructs an LLM even though infer=False never uses it.
1291
+ "llm": {
1292
+ "provider": "openai",
1293
+ "config": {
1294
+ "model": "gpt-5-mini",
1295
+ "api_key": "syncmill-inference-disabled",
1296
+ "openai_base_url": "http://127.0.0.1:9/v1",
1297
+ },
1298
+ },
1299
+ "history_db_path": str(data_dir / "history.db"),
1300
+ "version": "v1.1",
1301
+ }
1302
+ raw = copy.deepcopy(mem0_cfg.config)
1303
+ resolved = _deep_merge(defaults, raw)
1304
+ vector = resolved.get("vector_store", {})
1305
+ if vector.get("provider") != "qdrant":
1306
+ raise ValueError("mem0 backend requires vector_store.provider='qdrant'")
1307
+
1308
+ embedder = resolved.get("embedder", {})
1309
+ embed_cfg = embedder.get("config", {})
1310
+ user_embedder = raw.get("embedder", {})
1311
+ user_embed_cfg = user_embedder.get("config", {})
1312
+ customized_embedder = (
1313
+ embedder.get("provider") != "fastembed"
1314
+ or embed_cfg.get("model") != _LOCAL_MODEL
1315
+ )
1316
+ if customized_embedder and "embedding_dims" not in user_embed_cfg:
1317
+ raise ValueError(
1318
+ "custom mem0 embedders must set config.embedding_dims explicitly"
1319
+ )
1320
+ if embedder.get("provider") != "fastembed" and "model" not in user_embed_cfg:
1321
+ raise ValueError("custom mem0 embedders must set config.model explicitly")
1322
+ dims = embed_cfg.get("embedding_dims")
1323
+ if not isinstance(dims, int) or isinstance(dims, bool) or dims < 1:
1324
+ raise ValueError("mem0 embedder.config.embedding_dims must be positive")
1325
+ vector_cfg = vector.setdefault("config", {})
1326
+ vector_dims = vector_cfg.get("embedding_model_dims")
1327
+ user_vector_cfg = raw.get("vector_store", {}).get("config", {})
1328
+ if (
1329
+ "embedding_model_dims" in user_vector_cfg
1330
+ and vector_dims is not None
1331
+ and vector_dims != dims
1332
+ ):
1333
+ raise ValueError(
1334
+ "mem0 vector_store embedding_model_dims must match embedder dimensions"
1335
+ )
1336
+ vector_cfg["embedding_model_dims"] = dims
1337
+
1338
+ # A remote/custom Qdrant client and a local path are mutually exclusive.
1339
+ if any(k in user_vector_cfg for k in ("host", "url", "client")):
1340
+ vector_cfg.pop("path", None)
1341
+ elif "path" in vector_cfg:
1342
+ vector_cfg["path"] = str(Path(vector_cfg["path"]).expanduser())
1343
+ resolved["history_db_path"] = str(Path(resolved["history_db_path"]).expanduser())
1344
+ return data_dir, resolved
1345
+
1346
+
1347
+ class Mem0Backend:
1348
+ """Optional mem0 OSS reference backend, constrained to Qdrant.
1349
+
1350
+ The adapter maps each syncmill namespace to one mem0 ``user_id`` and forces
1351
+ ``infer=False`` on writes. Qdrant is deliberately required because P2 needs
1352
+ exact namespace counts and proof that bulk deletion reached zero.
1353
+ """
1354
+
1355
+ def __init__(self, config: MemoryConfig):
1356
+ mem0_cfg = config.mem0
1357
+ data_dir, resolved = _resolve_mem0_config(config)
1358
+ data_dir.mkdir(parents=True, exist_ok=True)
1359
+
1360
+ # mem0 snapshots this setting on the process's first mem0 import. Syncmill
1361
+ # defaults that first import to no anonymous telemetry; changing the
1362
+ # setting after another component imports mem0 requires a process restart.
1363
+ os.environ["MEM0_TELEMETRY"] = "true" if mem0_cfg.telemetry else "false"
1364
+ try:
1365
+ from mem0 import AsyncMemory
1366
+ from mem0.configs.base import MemoryConfig as Mem0MemoryConfig
1367
+ from qdrant_client.models import FieldCondition, Filter, MatchValue
1368
+ except ModuleNotFoundError as exc:
1369
+ raise RuntimeError(
1370
+ "mem0 backend is not installed; install syncmill[mem0] "
1371
+ "or run `uv sync --extra mem0`"
1372
+ ) from exc
1373
+
1374
+ self._store = AsyncMemory(config=Mem0MemoryConfig(**resolved))
1375
+ self._store.llm = _DisabledLLM()
1376
+ self._filter_type = Filter
1377
+ self._field_condition_type = FieldCondition
1378
+ self._match_value_type = MatchValue
1379
+ self._data_dir = str(data_dir)
1380
+ self._resolved = resolved
1381
+ self._caps = BackendCapabilities(
1382
+ supports_sessions=False,
1383
+ supports_tag_filter=True,
1384
+ namespace_delete_is_atomic=False,
1385
+ native_multi_namespace_search=False,
1386
+ share_provenance_links=False,
1387
+ )
1388
+ self._closed = False
1389
+
1390
+ @property
1391
+ def capabilities(self) -> BackendCapabilities:
1392
+ return self._caps
1393
+
1394
+ @property
1395
+ def agent_namespace_prefix(self) -> str:
1396
+ return "agent-runtime:"
1397
+
1398
+ @property
1399
+ def shared_namespace(self) -> str:
1400
+ return "shared"
1401
+
1402
+ def validate_namespace(self, ns: str) -> bool:
1403
+ return bool(self._caps.namespace_pattern.fullmatch(ns))
1404
+
1405
+ async def add(self, content: str, *, tags: list[str], namespace: str) -> None:
1406
+ await self.add_returning_ids(content, tags=tags, namespace=namespace)
1407
+
1408
+ async def add_returning_ids(
1409
+ self, content: str, *, tags: list[str], namespace: str
1410
+ ) -> list[str]:
1411
+ response = await self._store.add(
1412
+ [{"role": "user", "content": content}],
1413
+ user_id=namespace,
1414
+ metadata={"syncmill_tags": list(tags)},
1415
+ infer=False,
1416
+ )
1417
+ rows = response.get("results", []) if isinstance(response, dict) else []
1418
+ if len(rows) != 1 or rows[0].get("memory") != content:
1419
+ raise RuntimeError("mem0 did not confirm one verbatim memory write")
1420
+ row_id = rows[0].get("id")
1421
+ return [str(row_id)] if row_id else []
1422
+
1423
+ def _filters(self, namespace: str, tag_filter: str | None) -> dict[str, Any]:
1424
+ filters: dict[str, Any] = {"user_id": namespace}
1425
+ if tag_filter is not None:
1426
+ tags = [tag.strip() for tag in tag_filter.split(",") if tag.strip()]
1427
+ if tags:
1428
+ filters["OR"] = [{"syncmill_tags": tag} for tag in tags]
1429
+ return filters
1430
+
1431
+ @staticmethod
1432
+ def _row(row: dict[str, Any]) -> MemoryRow | None:
1433
+ namespace = row.get("user_id")
1434
+ if not namespace:
1435
+ return None
1436
+ metadata = row.get("metadata") or {}
1437
+ raw_tags = metadata.get("syncmill_tags")
1438
+ tags = [str(tag) for tag in raw_tags] if isinstance(raw_tags, list) else []
1439
+ return MemoryRow(
1440
+ id=str(row.get("id", "")),
1441
+ content=str(row.get("memory", "")),
1442
+ score=float(row.get("score") or 0.0),
1443
+ source=str(metadata.get("source") or "mem0"),
1444
+ namespace=str(namespace),
1445
+ tags=tags,
1446
+ )
1447
+
1448
+ async def search(
1449
+ self,
1450
+ query: str,
1451
+ *,
1452
+ namespace: str,
1453
+ top_k: int,
1454
+ tag_filter: str | None = None,
1455
+ ) -> list[MemoryRow]:
1456
+ response = await self._store.search(
1457
+ query,
1458
+ top_k=top_k,
1459
+ filters=self._filters(namespace, tag_filter),
1460
+ threshold=0.0,
1461
+ rerank=False,
1462
+ )
1463
+ raw_rows = response.get("results", []) if isinstance(response, dict) else []
1464
+ rows = [mapped for raw in raw_rows if (mapped := self._row(raw))]
1465
+ return [row for row in rows if row.get("namespace") == namespace]
1466
+
1467
+ async def search_multi(
1468
+ self,
1469
+ query: str,
1470
+ *,
1471
+ namespaces: Sequence[str],
1472
+ top_k: int,
1473
+ tag_filter: str | None = None,
1474
+ ) -> list[MemoryRow]:
1475
+ return _merge_rows(
1476
+ [
1477
+ await self.search(
1478
+ query, namespace=ns, top_k=top_k, tag_filter=tag_filter
1479
+ )
1480
+ for ns in namespaces
1481
+ ],
1482
+ top_k,
1483
+ )
1484
+
1485
+ async def share_chunk(self, chunk_id: str, *, target_namespace: str) -> str | None:
1486
+ source = await self.get(chunk_id)
1487
+ if source is None:
1488
+ return None
1489
+ new_ids = await self.add_returning_ids(
1490
+ source["content"],
1491
+ tags=_shared_tags(source.get("tags", []), chunk_id),
1492
+ namespace=target_namespace,
1493
+ )
1494
+ return new_ids[0] if new_ids else None
1495
+
1496
+ async def register_namespace(
1497
+ self, ns: str, *, description: str | None = None, color: str | None = None
1498
+ ) -> None:
1499
+ return None
1500
+
1501
+ async def get(self, chunk_id: str) -> MemoryRow | None:
1502
+ raw = await self._store.get(chunk_id)
1503
+ return self._row(raw) if isinstance(raw, dict) else None
1504
+
1505
+ async def ensure_namespace(self, ns: str, *, description: str = "") -> None:
1506
+ return None
1507
+
1508
+ async def ensure_ready(self) -> None:
1509
+ await self.count_namespace("syncmill-readiness-probe")
1510
+
1511
+ async def status(self) -> dict[str, Any]:
1512
+ await self.ensure_ready()
1513
+ vector = self._resolved["vector_store"]
1514
+ embedder = self._resolved["embedder"]
1515
+ return {
1516
+ "data_dir": self._data_dir,
1517
+ "vector_store": "qdrant",
1518
+ "collection": vector["config"].get("collection_name", "mem0"),
1519
+ "embedder": f"{embedder.get('provider')}/{embedder['config'].get('model')}",
1520
+ }
1521
+
1522
+ def _qdrant_store(self) -> Any:
1523
+ store = self._store.vector_store
1524
+ if not hasattr(store, "client") or not hasattr(store, "collection_name"):
1525
+ raise RuntimeError("mem0 Qdrant internals are unavailable")
1526
+ return store
1527
+
1528
+ async def count_namespace(self, ns: str) -> int:
1529
+ store = self._qdrant_store()
1530
+ count_filter = self._filter_type(
1531
+ must=[
1532
+ self._field_condition_type(
1533
+ key="user_id", match=self._match_value_type(value=ns)
1534
+ )
1535
+ ]
1536
+ )
1537
+ response = await asyncio.to_thread(
1538
+ store.client.count,
1539
+ collection_name=store.collection_name,
1540
+ count_filter=count_filter,
1541
+ exact=True,
1542
+ )
1543
+ return int(response.count)
1544
+
1545
+ async def delete_namespace(self, ns: str) -> int:
1546
+ original = await self.count_namespace(ns)
1547
+ remaining = original
1548
+ while remaining:
1549
+ await self._store.delete_all(user_id=ns)
1550
+ after = await self.count_namespace(ns)
1551
+ if after >= remaining:
1552
+ raise RuntimeError(
1553
+ f"mem0 namespace deletion made no progress for {ns!r}"
1554
+ )
1555
+ remaining = after
1556
+ return original
1557
+
1558
+ async def start_session(self, agent_id: str, namespace: str) -> str | None:
1559
+ return None
1560
+
1561
+ async def log_event(self, kind: str, text: str) -> None:
1562
+ return None
1563
+
1564
+ async def end_session(self, summary: str | None) -> None:
1565
+ return None
1566
+
1567
+ async def close(self) -> None:
1568
+ if self._closed:
1569
+ return
1570
+ self._closed = True
1571
+ # Teardown must not depend on the private Qdrant shape used by count/delete.
1572
+ # mem0 2.0.x closes its history DB, while the client close releases the
1573
+ # embedded Qdrant path lock when that client is still discoverable.
1574
+ vector_store = getattr(self._store, "vector_store", None)
1575
+ client = getattr(vector_store, "client", None)
1576
+ try:
1577
+ self._store.close()
1578
+ finally:
1579
+ close = getattr(client, "close", None)
1580
+ if close is not None:
1581
+ await asyncio.to_thread(close)
1582
+
1583
+
1584
+ _LANGGRAPH_NS = ("syncmill", "memory", "items")
1585
+ # One page of asearch results; count_namespace's stop condition compares
1586
+ # against this same constant so the two can't drift.
1587
+ _LANGGRAPH_PAGE_SIZE = 100
1588
+ # Iteration backstop for count_namespace: far above any real namespace (a
1589
+ # syncmill namespace holds task results, not a corpus), hit only by a store
1590
+ # whose asearch ignores `offset` and would otherwise loop forever.
1591
+ _LANGGRAPH_COUNT_CAP = 1_000_000
1592
+
1593
+
1594
+ def _tag_key(tag: str) -> str:
1595
+ return "tag_" + hashlib.sha256(tag.encode("utf-8")).hexdigest()
1596
+
1597
+
1598
+ class LangGraphBackend:
1599
+ """Optional persistent LangGraph AsyncSqliteStore reference backend."""
1600
+
1601
+ def __init__(self, config: MemoryConfig):
1602
+ self._db_path = str(Path(config.langgraph.db_path).expanduser())
1603
+ self._store: Any = None
1604
+ self._store_cm: Any = None
1605
+ self._init_lock = asyncio.Lock()
1606
+ self._closed = False
1607
+ self._caps = BackendCapabilities(
1608
+ supports_sessions=False,
1609
+ supports_tag_filter=True,
1610
+ namespace_delete_is_atomic=False,
1611
+ native_multi_namespace_search=False,
1612
+ share_provenance_links=False,
1613
+ )
1614
+
1615
+ @property
1616
+ def capabilities(self) -> BackendCapabilities:
1617
+ return self._caps
1618
+
1619
+ @property
1620
+ def agent_namespace_prefix(self) -> str:
1621
+ return "agent-runtime:"
1622
+
1623
+ @property
1624
+ def shared_namespace(self) -> str:
1625
+ return "shared"
1626
+
1627
+ def validate_namespace(self, ns: str) -> bool:
1628
+ return bool(self._caps.namespace_pattern.fullmatch(ns))
1629
+
1630
+ async def _ensure_store(self) -> Any:
1631
+ if self._closed:
1632
+ raise RuntimeError("LangGraph backend is closed")
1633
+ if self._store is not None:
1634
+ return self._store
1635
+ async with self._init_lock:
1636
+ if self._store is not None:
1637
+ return self._store
1638
+ try:
1639
+ from fastembed import TextEmbedding
1640
+ from langchain_core.embeddings import Embeddings
1641
+ from langgraph.store.sqlite.aio import AsyncSqliteStore
1642
+ except ModuleNotFoundError as exc:
1643
+ raise RuntimeError(
1644
+ "LangGraph backend is not installed; install syncmill[langgraph] "
1645
+ "or run `uv sync --extra langgraph`"
1646
+ ) from exc
1647
+
1648
+ class LocalEmbeddings(Embeddings):
1649
+ def __init__(self) -> None:
1650
+ self.model = TextEmbedding(model_name=_LOCAL_MODEL)
1651
+
1652
+ def embed_documents(self, texts: list[str]) -> list[list[float]]:
1653
+ return [list(vector) for vector in self.model.embed(texts)]
1654
+
1655
+ def embed_query(self, text: str) -> list[float]:
1656
+ return self.embed_documents([text])[0]
1657
+
1658
+ async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
1659
+ return await asyncio.to_thread(self.embed_documents, texts)
1660
+
1661
+ async def aembed_query(self, text: str) -> list[float]:
1662
+ return (await self.aembed_documents([text]))[0]
1663
+
1664
+ Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
1665
+ self._store_cm = AsyncSqliteStore.from_conn_string(
1666
+ self._db_path,
1667
+ index={
1668
+ "dims": _LOCAL_DIMS,
1669
+ "embed": LocalEmbeddings(),
1670
+ "text_fields": ["content"],
1671
+ },
1672
+ )
1673
+ self._store = await self._store_cm.__aenter__()
1674
+ try:
1675
+ await self._store.setup()
1676
+ except Exception:
1677
+ await self._store_cm.__aexit__(None, None, None)
1678
+ self._store = self._store_cm = None
1679
+ raise
1680
+ return self._store
1681
+
1682
+ @staticmethod
1683
+ def _value(content: str, tags: list[str], namespace: str) -> dict[str, Any]:
1684
+ value: dict[str, Any] = {
1685
+ "schema_version": 1,
1686
+ "namespace": namespace,
1687
+ "content": content,
1688
+ "tags": list(tags),
1689
+ "source": "langgraph",
1690
+ }
1691
+ value.update({_tag_key(tag): True for tag in tags})
1692
+ return value
1693
+
1694
+ @staticmethod
1695
+ def _row(item: Any) -> MemoryRow | None:
1696
+ value = getattr(item, "value", None)
1697
+ if not isinstance(value, dict):
1698
+ return None
1699
+ namespace = value.get("namespace")
1700
+ content = value.get("content")
1701
+ tags = value.get("tags", [])
1702
+ if not isinstance(namespace, str) or not isinstance(content, str):
1703
+ return None
1704
+ if not isinstance(tags, list) or not all(isinstance(t, str) for t in tags):
1705
+ return None
1706
+ return MemoryRow(
1707
+ id=str(item.key),
1708
+ content=content,
1709
+ score=float(getattr(item, "score", None) or 0.0),
1710
+ source=str(value.get("source") or "langgraph"),
1711
+ namespace=namespace,
1712
+ tags=list(tags),
1713
+ )
1714
+
1715
+ async def add(self, content: str, *, tags: list[str], namespace: str) -> None:
1716
+ await self.add_returning_ids(content, tags=tags, namespace=namespace)
1717
+
1718
+ async def add_returning_ids(
1719
+ self, content: str, *, tags: list[str], namespace: str
1720
+ ) -> list[str]:
1721
+ store = await self._ensure_store()
1722
+ key = str(uuid4())
1723
+ await store.aput(
1724
+ _LANGGRAPH_NS,
1725
+ key,
1726
+ self._value(content, tags, namespace),
1727
+ index=["content"],
1728
+ )
1729
+ return [key]
1730
+
1731
+ async def search(
1732
+ self,
1733
+ query: str,
1734
+ *,
1735
+ namespace: str,
1736
+ top_k: int,
1737
+ tag_filter: str | None = None,
1738
+ ) -> list[MemoryRow]:
1739
+ store = await self._ensure_store()
1740
+ tags = (
1741
+ [tag.strip() for tag in tag_filter.split(",") if tag.strip()]
1742
+ if tag_filter
1743
+ else []
1744
+ )
1745
+ filters = [{"namespace": namespace, _tag_key(tag): True} for tag in tags] or [
1746
+ {"namespace": namespace}
1747
+ ]
1748
+ merged: dict[str, MemoryRow] = {}
1749
+ for exact_filter in filters:
1750
+ items = await store.asearch(
1751
+ _LANGGRAPH_NS,
1752
+ query=query,
1753
+ filter=exact_filter,
1754
+ limit=top_k,
1755
+ )
1756
+ for item in items:
1757
+ row = self._row(item)
1758
+ if row is None or row.get("namespace") != namespace:
1759
+ continue
1760
+ prior = merged.get(row["id"])
1761
+ if prior is None or row.get("score", 0.0) > prior.get("score", 0.0):
1762
+ merged[row["id"]] = row
1763
+ return sorted(merged.values(), key=lambda row: -row.get("score", 0.0))[:top_k]
1764
+
1765
+ async def search_multi(
1766
+ self,
1767
+ query: str,
1768
+ *,
1769
+ namespaces: Sequence[str],
1770
+ top_k: int,
1771
+ tag_filter: str | None = None,
1772
+ ) -> list[MemoryRow]:
1773
+ return _merge_rows(
1774
+ [
1775
+ await self.search(
1776
+ query, namespace=ns, top_k=top_k, tag_filter=tag_filter
1777
+ )
1778
+ for ns in namespaces
1779
+ ],
1780
+ top_k,
1781
+ )
1782
+
1783
+ async def share_chunk(self, chunk_id: str, *, target_namespace: str) -> str | None:
1784
+ source = await self.get(chunk_id)
1785
+ if source is None:
1786
+ return None
1787
+ new_ids = await self.add_returning_ids(
1788
+ source["content"],
1789
+ tags=_shared_tags(source.get("tags", []), chunk_id),
1790
+ namespace=target_namespace,
1791
+ )
1792
+ return new_ids[0] if new_ids else None
1793
+
1794
+ async def register_namespace(
1795
+ self, ns: str, *, description: str | None = None, color: str | None = None
1796
+ ) -> None:
1797
+ return None
1798
+
1799
+ async def get(self, chunk_id: str) -> MemoryRow | None:
1800
+ return self._row(
1801
+ await (await self._ensure_store()).aget(_LANGGRAPH_NS, chunk_id)
1802
+ )
1803
+
1804
+ async def ensure_namespace(self, ns: str, *, description: str = "") -> None:
1805
+ return None
1806
+
1807
+ async def ensure_ready(self) -> None:
1808
+ await self._ensure_store()
1809
+
1810
+ async def status(self) -> dict[str, Any]:
1811
+ await self.ensure_ready()
1812
+ return {
1813
+ "db_path": self._db_path,
1814
+ "store": "langgraph/AsyncSqliteStore",
1815
+ "embedder": f"fastembed/{_LOCAL_MODEL}",
1816
+ }
1817
+
1818
+ async def _page(
1819
+ self, ns: str, *, limit: int = _LANGGRAPH_PAGE_SIZE, offset: int = 0
1820
+ ) -> list[Any]:
1821
+ return await (await self._ensure_store()).asearch(
1822
+ _LANGGRAPH_NS,
1823
+ query=None,
1824
+ filter={"namespace": ns},
1825
+ limit=limit,
1826
+ offset=offset,
1827
+ )
1828
+
1829
+ async def count_namespace(self, ns: str) -> int:
1830
+ total = 0
1831
+ while True:
1832
+ page = await self._page(ns, offset=total)
1833
+ if not page:
1834
+ return total
1835
+ total += len(page)
1836
+ if len(page) < _LANGGRAPH_PAGE_SIZE:
1837
+ return total
1838
+ # A store that ignores `offset` returns the same full page forever;
1839
+ # without a cap that spins unbounded (the mem0 adapter's no-progress
1840
+ # guard, adapted to offset paging).
1841
+ if total > _LANGGRAPH_COUNT_CAP:
1842
+ raise RuntimeError(
1843
+ f"LangGraph namespace count made no progress for {ns!r}"
1844
+ )
1845
+
1846
+ async def delete_namespace(self, ns: str) -> int:
1847
+ original = await self.count_namespace(ns)
1848
+ store = await self._ensure_store()
1849
+ remaining = original
1850
+ while True:
1851
+ page = await self._page(ns)
1852
+ if not page:
1853
+ break
1854
+ for item in page:
1855
+ await store.adelete(_LANGGRAPH_NS, item.key)
1856
+ # No-progress guard (the mem0 adapter's recipe): if a pass deleted
1857
+ # nothing — adelete no-ops on a mismatched key, a permission issue —
1858
+ # the same page would come back forever, an unbounded hang worse
1859
+ # than the failure it hides.
1860
+ after = await self.count_namespace(ns)
1861
+ if after >= remaining:
1862
+ raise RuntimeError(
1863
+ f"LangGraph namespace deletion made no progress for {ns!r}"
1864
+ )
1865
+ remaining = after
1866
+ if await self.count_namespace(ns):
1867
+ raise RuntimeError(
1868
+ f"LangGraph namespace deletion did not reach zero for {ns!r}"
1869
+ )
1870
+ return original
1871
+
1872
+ async def start_session(self, agent_id: str, namespace: str) -> str | None:
1873
+ return None
1874
+
1875
+ async def log_event(self, kind: str, text: str) -> None:
1876
+ return None
1877
+
1878
+ async def end_session(self, summary: str | None) -> None:
1879
+ return None
1880
+
1881
+ async def close(self) -> None:
1882
+ if self._closed:
1883
+ return
1884
+ self._closed = True
1885
+ if self._store_cm is not None:
1886
+ await self._store_cm.__aexit__(None, None, None)
1887
+ self._store = self._store_cm = None
1888
+
1889
+
1890
+ def build_backend(config: MemoryConfig) -> MemoryBackend:
1891
+ """Build the configured backend, refusing non-verbatim storage."""
1892
+
1893
+ if config.backend == "memtomem":
1894
+ backend: MemoryBackend = MemtomemBackend(config)
1895
+ elif config.backend == "mem0":
1896
+ backend = Mem0Backend(config)
1897
+ elif config.backend == "langgraph":
1898
+ backend = LangGraphBackend(config)
1899
+ else: # guarded by MemoryConfig's Literal, defensive for programmatic callers
1900
+ raise ValueError(f"unknown memory backend: {config.backend!r}")
1901
+ # Defensive, like the unknown-backend branch above: no shipped backend sets
1902
+ # verbatim_storage=False (all three leave the default True), so this is
1903
+ # reachable only from tests/monkeypatching — it exists to make the contract
1904
+ # explicit for a future adapter, not to guard a live path.
1905
+ if not backend.capabilities.verbatim_storage:
1906
+ raise ValueError(f"backend {config.backend!r} cannot store verbatim content")
1907
+ return backend