albert-memory 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. albert/__init__.py +3 -0
  2. albert/admin.py +532 -0
  3. albert/api.py +698 -0
  4. albert/automatic_memory_release.json +14 -0
  5. albert/benchmark.py +272 -0
  6. albert/benchmark_cli.py +37 -0
  7. albert/capture.py +1051 -0
  8. albert/capture_installer.py +1000 -0
  9. albert/capture_promotion.py +298 -0
  10. albert/capture_retention.py +72 -0
  11. albert/capture_sdk.py +558 -0
  12. albert/capture_security.py +79 -0
  13. albert/chunking.py +79 -0
  14. albert/classifier.py +263 -0
  15. albert/claude_adapter.py +357 -0
  16. albert/claude_lifecycle.py +418 -0
  17. albert/claude_package.json +19 -0
  18. albert/claude_plugin/.claude-plugin/plugin.json +11 -0
  19. albert/claude_plugin/.mcp.json +13 -0
  20. albert/claude_plugin/hooks/hooks.json +70 -0
  21. albert/clusters.py +140 -0
  22. albert/codex_adapter.py +308 -0
  23. albert/codex_hooks/hooks.json +49 -0
  24. albert/codex_lifecycle.py +485 -0
  25. albert/codex_package.json +19 -0
  26. albert/compatibility.py +121 -0
  27. albert/config.py +144 -0
  28. albert/console.py +508 -0
  29. albert/console_backends.py +236 -0
  30. albert/console_map.py +518 -0
  31. albert/console_ops.py +919 -0
  32. albert/credentials.py +616 -0
  33. albert/cursor_adapter.py +320 -0
  34. albert/cursor_hooks/hooks.json +11 -0
  35. albert/cursor_rules/albert-memory.mdc +11 -0
  36. albert/db.py +68 -0
  37. albert/embeddings.py +91 -0
  38. albert/graph.py +623 -0
  39. albert/graph_builders.py +879 -0
  40. albert/graph_cypher.py +669 -0
  41. albert/graph_store.py +376 -0
  42. albert/host_targets/claude-code.json +45 -0
  43. albert/host_targets/codex.json +45 -0
  44. albert/host_targets/openclaw.json +45 -0
  45. albert/host_targets.py +271 -0
  46. albert/links.py +428 -0
  47. albert/mcp_server.py +457 -0
  48. albert/memory_safety.py +111 -0
  49. albert/models.py +611 -0
  50. albert/openclaw_adapter.py +326 -0
  51. albert/openclaw_adapter_trust.json +5 -0
  52. albert/openclaw_package.json +22 -0
  53. albert/openclaw_plugin/index.js +135 -0
  54. albert/openclaw_plugin/openclaw.plugin.json +25 -0
  55. albert/openclaw_plugin/package.json +12 -0
  56. albert/portability.py +291 -0
  57. albert/provenance.py +105 -0
  58. albert/recorder.py +166 -0
  59. albert/release_gate.py +83 -0
  60. albert/schemas.py +617 -0
  61. albert/scope.py +26 -0
  62. albert/search.py +447 -0
  63. albert/security.py +150 -0
  64. albert/services.py +1316 -0
  65. albert/session_context.py +162 -0
  66. albert/trace_feed.py +165 -0
  67. albert/trace_writer.py +131 -0
  68. albert/tracing.py +75 -0
  69. albert/worker.py +202 -0
  70. albert/workspaces.py +256 -0
  71. albert_memory-0.4.0.dist-info/METADATA +554 -0
  72. albert_memory-0.4.0.dist-info/RECORD +74 -0
  73. albert_memory-0.4.0.dist-info/WHEEL +4 -0
  74. albert_memory-0.4.0.dist-info/entry_points.txt +14 -0
albert/host_targets.py ADDED
@@ -0,0 +1,271 @@
1
+ """Versioned, fail-closed host lifecycle target contracts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ import stat
9
+ from dataclasses import dataclass
10
+ from pathlib import Path, PurePosixPath
11
+
12
+
13
+ class HostTargetError(RuntimeError):
14
+ """A host target cannot be trusted or negotiated."""
15
+
16
+
17
+ HOST_TARGET_SCHEMA = "com.jiggai.albert.host-target"
18
+ HOST_TARGET_SCHEMA_VERSION = 1
19
+ CAPABILITY_PATTERN = re.compile(r"^[a-z][a-z0-9._-]{0,99}$")
20
+ VERSION_PATTERN = re.compile(r"^(\d+)\.(\d+)\.(\d+)$")
21
+ RANGE_PATTERN = re.compile(r"^(>=|>|<=|<|==)(\d+)\.(\d+)\.(\d+)$")
22
+ TARGET_PATH = Path(__file__).with_name("host_targets")
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class HostTarget:
27
+ target_id: str
28
+ schema_version: int
29
+ descriptor_version: str
30
+ package: dict[str, str]
31
+ host: dict[str, object]
32
+ hooks: dict[str, object]
33
+ install: dict[str, object]
34
+ credentials: dict[str, object]
35
+ ownership: dict[str, str]
36
+ lifecycle: dict[str, object]
37
+ probes: tuple[str, ...]
38
+ capabilities: tuple[str, ...]
39
+
40
+
41
+ def _version(value: object, label: str) -> tuple[int, int, int]:
42
+ if not isinstance(value, str) or not (match := VERSION_PATTERN.fullmatch(value)):
43
+ raise HostTargetError(f"{label} must be an exact semantic version")
44
+ return tuple(int(part) for part in match.groups())
45
+
46
+
47
+ def _version_in_range(version: tuple[int, int, int], declared: object) -> bool:
48
+ if not isinstance(declared, str):
49
+ return False
50
+ clauses = [part.strip() for part in declared.split(",") if part.strip()]
51
+ if not clauses:
52
+ return False
53
+ for clause in clauses:
54
+ match = RANGE_PATTERN.fullmatch(clause)
55
+ if not match:
56
+ return False
57
+ target = tuple(int(value) for value in match.groups()[1:])
58
+ if not {
59
+ ">=": version >= target,
60
+ ">": version > target,
61
+ "<=": version <= target,
62
+ "<": version < target,
63
+ "==": version == target,
64
+ }[match.group(1)]:
65
+ return False
66
+ return True
67
+
68
+
69
+ def _string_map(value: object, keys: set[str], label: str) -> dict[str, str]:
70
+ if (
71
+ not isinstance(value, dict)
72
+ or set(value) != keys
73
+ or not all(isinstance(item, str) and item for item in value.values())
74
+ ):
75
+ raise HostTargetError(f"host target {label} is malformed")
76
+ return dict(value)
77
+
78
+
79
+ def validate_host_target(payload: object) -> HostTarget:
80
+ """Validate every field before a target can cause host or filesystem changes."""
81
+ if not isinstance(payload, dict) or set(payload) != {
82
+ "schema",
83
+ "schema_version",
84
+ "descriptor_version",
85
+ "target_id",
86
+ "enabled",
87
+ "package",
88
+ "host",
89
+ "hooks",
90
+ "install",
91
+ "credentials",
92
+ "ownership",
93
+ "lifecycle",
94
+ "probes",
95
+ "capabilities",
96
+ }:
97
+ raise HostTargetError("host target contains unknown or missing fields")
98
+ if (
99
+ payload["schema"] != HOST_TARGET_SCHEMA
100
+ or type(payload["schema_version"]) is not int
101
+ or payload["schema_version"] != 1
102
+ ):
103
+ raise HostTargetError("host target schema or version is unsupported")
104
+ if payload["enabled"] is not True:
105
+ raise HostTargetError("host target is disabled")
106
+ target_id = payload["target_id"]
107
+ if not isinstance(target_id, str) or not re.fullmatch(r"[a-z][a-z0-9-]{0,49}", target_id):
108
+ raise HostTargetError("host target identifier is invalid")
109
+ _version(payload["descriptor_version"], "descriptor_version")
110
+
111
+ package = _string_map(
112
+ payload["package"], {"distribution", "version", "artifact_provenance"}, "package"
113
+ )
114
+ _version(package["version"], "package.version")
115
+ if package["artifact_provenance"] != "github-oidc-dsse+sha256":
116
+ raise HostTargetError("host target package provenance is unsupported")
117
+
118
+ host = payload["host"]
119
+ if not isinstance(host, dict) or set(host) != {"family", "tested", "supported"}:
120
+ raise HostTargetError("host target host compatibility is malformed")
121
+ tested = _version(host["tested"], "host.tested")
122
+ if host["family"] != target_id or not _version_in_range(tested, host["supported"]):
123
+ raise HostTargetError("host target tested release is outside its supported range")
124
+
125
+ hooks = payload["hooks"]
126
+ if (
127
+ not isinstance(hooks, dict)
128
+ or set(hooks) != {"fidelity", "present", "absent"}
129
+ or hooks["fidelity"] not in {"full", "reduced"}
130
+ or not all(isinstance(hooks[key], list) for key in ("present", "absent"))
131
+ ):
132
+ raise HostTargetError("host target hook fidelity is malformed")
133
+ present = hooks["present"]
134
+ absent = hooks["absent"]
135
+ if (
136
+ not all(
137
+ isinstance(item, str) and CAPABILITY_PATTERN.fullmatch(item)
138
+ for item in present + absent
139
+ )
140
+ or len(present) != len(set(present))
141
+ or len(absent) != len(set(absent))
142
+ or set(present) & set(absent)
143
+ or (hooks["fidelity"] == "full" and absent)
144
+ ):
145
+ raise HostTargetError("host target hooks are contradictory or fabricated")
146
+
147
+ install = payload["install"]
148
+ if not isinstance(install, dict) or set(install) != {"root", "reload", "uninstall"}:
149
+ raise HostTargetError("host target install contract is malformed")
150
+ root = install["root"]
151
+ if (
152
+ not isinstance(root, str)
153
+ or not root.startswith("~/")
154
+ or ".." in PurePosixPath(root[2:]).parts
155
+ or not all(
156
+ isinstance(install[key], list)
157
+ and install[key]
158
+ and all(isinstance(arg, str) and arg for arg in install[key])
159
+ for key in ("reload", "uninstall")
160
+ )
161
+ ):
162
+ raise HostTargetError("host target install path or commands are unsafe")
163
+
164
+ credentials = payload["credentials"]
165
+ if credentials != {
166
+ "acquisition": ["hidden_prompt", "private_stdin", "private_fd"],
167
+ "storage": "owned_file_0600",
168
+ "forbidden_surfaces": [
169
+ "args",
170
+ "ordinary_env",
171
+ "host_config",
172
+ "logs",
173
+ "diagnostics",
174
+ "artifacts",
175
+ ],
176
+ }:
177
+ raise HostTargetError("host target credential handling is unsafe")
178
+ ownership = _string_map(payload["ownership"], {"marker", "scope"}, "ownership")
179
+ if ownership["scope"] != "target-owned-tree-and-entry":
180
+ raise HostTargetError("host target ownership scope is unsafe")
181
+
182
+ lifecycle = payload["lifecycle"]
183
+ if (
184
+ not isinstance(lifecycle, dict)
185
+ or set(lifecycle) != {"upgrade", "rotate", "revoke", "rollback", "direct_brain"}
186
+ or not all(value is True for value in lifecycle.values())
187
+ ):
188
+ raise HostTargetError("host target lifecycle contract is incomplete")
189
+ probes = payload["probes"]
190
+ capabilities = payload["capabilities"]
191
+ if (
192
+ not isinstance(probes, list)
193
+ or probes != ["host_config", "runtime", "brain_ready", "capture_recall"]
194
+ or not isinstance(capabilities, list)
195
+ or capabilities != sorted(set(capabilities))
196
+ or not capabilities
197
+ or not all(
198
+ isinstance(item, str) and CAPABILITY_PATTERN.fullmatch(item) for item in capabilities
199
+ )
200
+ ):
201
+ raise HostTargetError("host target probes or capabilities are invalid")
202
+ return HostTarget(
203
+ target_id=target_id,
204
+ schema_version=payload["schema_version"],
205
+ descriptor_version=payload["descriptor_version"],
206
+ package=package,
207
+ host=dict(host),
208
+ hooks=dict(hooks),
209
+ install=dict(install),
210
+ credentials=dict(credentials),
211
+ ownership=ownership,
212
+ lifecycle=dict(lifecycle),
213
+ probes=tuple(probes),
214
+ capabilities=tuple(capabilities),
215
+ )
216
+
217
+
218
+ def load_host_target(target_id: str) -> HostTarget:
219
+ """Load only explicitly shipped targets; unknown targets fail closed."""
220
+ if target_id not in {"openclaw", "claude-code", "codex"}:
221
+ raise HostTargetError(f"unknown or disabled host target: {target_id}")
222
+ try:
223
+ payload = json.loads((TARGET_PATH / f"{target_id}.json").read_text(encoding="utf-8"))
224
+ except (OSError, json.JSONDecodeError) as exc:
225
+ raise HostTargetError("host target descriptor is unavailable or invalid") from exc
226
+ target = validate_host_target(payload)
227
+ if target.target_id != target_id:
228
+ raise HostTargetError("host target descriptor identity mismatch")
229
+ return target
230
+
231
+
232
+ def negotiate_host_target(
233
+ target_id: str,
234
+ host_version: tuple[int, int, int],
235
+ required_capabilities: set[str],
236
+ *,
237
+ target: HostTarget | None = None,
238
+ ) -> HostTarget:
239
+ negotiated = target or load_host_target(target_id)
240
+ if negotiated.target_id != target_id or not _version_in_range(
241
+ host_version, negotiated.host["supported"]
242
+ ):
243
+ raise HostTargetError("host target version is unsupported")
244
+ if not required_capabilities <= set(negotiated.capabilities):
245
+ raise HostTargetError("host target lacks required capabilities")
246
+ return negotiated
247
+
248
+
249
+ def assert_owned_install_root(root: Path, *, expected_marker: str) -> None:
250
+ """Reject foreign ownership, symlink traversal, and widened modes."""
251
+ if root.is_symlink():
252
+ raise HostTargetError("host target install root must not be a symlink")
253
+ current = root
254
+ while not current.exists() and current != current.parent:
255
+ current = current.parent
256
+ while current != current.parent:
257
+ if current.is_symlink():
258
+ raise HostTargetError("host target install path contains a symlink")
259
+ current = current.parent
260
+ if not root.exists():
261
+ return
262
+ marker_path = root / "install.json"
263
+ try:
264
+ marker = json.loads(marker_path.read_text(encoding="utf-8"))
265
+ except (OSError, json.JSONDecodeError) as exc:
266
+ raise HostTargetError("host target ownership marker is unavailable") from exc
267
+ if not isinstance(marker, dict) or marker.get("owner") != expected_marker:
268
+ raise HostTargetError("host target installation has foreign ownership")
269
+ for private_path in (marker_path, root / "credentials.env", root / "credentials.json"):
270
+ if private_path.exists() and stat.S_IMODE(os.stat(private_path).st_mode) != 0o600:
271
+ raise HostTargetError("host target private file mode is unsafe")
albert/links.py ADDED
@@ -0,0 +1,428 @@
1
+ """Derived links between memories.
2
+
3
+ `similar` and `sequence` links are written when a memory is indexed; `recalled`
4
+ links and recall statistics come from the flight recorder's traces. Everything
5
+ here is rebuildable and must never fail the indexing that triggers it.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ import re
12
+ from datetime import UTC, datetime, timedelta
13
+ from typing import Any
14
+ from uuid import UUID
15
+
16
+ from sqlalchemy import Float, and_, cast, or_, select
17
+ from sqlalchemy.exc import IntegrityError
18
+ from sqlalchemy.orm import Session
19
+
20
+ from albert.config import get_settings
21
+ from albert.models import (
22
+ ConsoleState,
23
+ Episode,
24
+ Memory,
25
+ MemoryChunk,
26
+ MemoryLink,
27
+ MemoryRecall,
28
+ MemoryStat,
29
+ Trace,
30
+ )
31
+ from albert.recorder import current_recorder, span
32
+ from albert.search import _cosine, _enable_iterative_scan
33
+
34
+ logger = logging.getLogger("albert.links")
35
+ PROBE_CHUNKS = 8
36
+ DATED_NAME = re.compile(r"\d{4}-\d{2}-\d{2}")
37
+
38
+
39
+ def _is_postgres(session: Session) -> bool:
40
+ return session.bind is not None and session.bind.dialect.name == "postgresql"
41
+
42
+
43
+ def _pair(first: UUID, second: UUID) -> tuple[UUID, UUID]:
44
+ """Canonical order for an undirected link: smaller id first."""
45
+ return (first, second) if str(first) < str(second) else (second, first)
46
+
47
+
48
+ def nearest_memories(
49
+ session: Session, memory: Memory, *, k: int, minimum: float
50
+ ) -> list[tuple[UUID, float]]:
51
+ """Other active memories in the same workspace, by best chunk-pair cosine similarity."""
52
+ chunks = list(
53
+ session.scalars(
54
+ select(MemoryChunk)
55
+ .where(MemoryChunk.memory_id == memory.id, MemoryChunk.embedding.is_not(None))
56
+ .order_by(MemoryChunk.chunk_index)
57
+ )
58
+ )
59
+ if not chunks:
60
+ return []
61
+ scope = [
62
+ MemoryChunk.organization_id == memory.organization_id,
63
+ MemoryChunk.memory_id != memory.id,
64
+ MemoryChunk.embedding_model == chunks[0].embedding_model,
65
+ MemoryChunk.embedding.is_not(None),
66
+ Memory.status == "active",
67
+ (
68
+ MemoryChunk.workspace_id == memory.workspace_id
69
+ if memory.workspace_id is not None
70
+ else MemoryChunk.workspace_id.is_(None)
71
+ ),
72
+ ]
73
+ best: dict[UUID, float] = {}
74
+ if _is_postgres(session):
75
+ _enable_iterative_scan(session)
76
+ for chunk in chunks[:PROBE_CHUNKS]:
77
+ distance = cast(MemoryChunk.embedding.op("<=>")(chunk.embedding), Float)
78
+ rows = session.execute(
79
+ select(MemoryChunk.memory_id, distance)
80
+ .join(Memory, Memory.id == MemoryChunk.memory_id)
81
+ .where(*scope)
82
+ .order_by(distance)
83
+ .limit(k * 8)
84
+ )
85
+ for other_id, value in rows:
86
+ similarity = 1.0 - float(value)
87
+ if similarity > best.get(other_id, -1.0):
88
+ best[other_id] = similarity
89
+ else:
90
+ others = session.execute(
91
+ select(MemoryChunk.memory_id, MemoryChunk.embedding)
92
+ .join(Memory, Memory.id == MemoryChunk.memory_id)
93
+ .where(*scope)
94
+ ).all()
95
+ mine = [list(chunk.embedding) for chunk in chunks[:PROBE_CHUNKS]]
96
+ for other_id, embedding in others:
97
+ similarity = max(_cosine(list(embedding or []), vector) for vector in mine)
98
+ if similarity > best.get(other_id, -1.0):
99
+ best[other_id] = similarity
100
+ ranked = sorted(best.items(), key=lambda item: (-item[1], str(item[0])))
101
+ return [(other_id, score) for other_id, score in ranked if score >= minimum][:k]
102
+
103
+
104
+ def _touching(memory_id: UUID, kind: str) -> Any:
105
+ return (
106
+ MemoryLink.kind == kind,
107
+ or_(MemoryLink.source_memory_id == memory_id, MemoryLink.target_memory_id == memory_id),
108
+ )
109
+
110
+
111
+ def drop_links_for_memory(session: Session, memory_id: UUID) -> None:
112
+ """Remove every link touching a memory (used when it is forgotten)."""
113
+ session.query(MemoryLink).filter(
114
+ or_(MemoryLink.source_memory_id == memory_id, MemoryLink.target_memory_id == memory_id)
115
+ ).delete(synchronize_session=False)
116
+
117
+
118
+ def drop_similar_links(session: Session, memory_id: UUID) -> None:
119
+ """Remove a memory's `similar` links; they describe text that has just changed.
120
+
121
+ `sequence` and `recalled` links stay: they come from where the memory sits
122
+ and how it has been used, not from what it currently says.
123
+ """
124
+ session.query(MemoryLink).filter(*_touching(memory_id, "similar")).delete(
125
+ synchronize_session=False
126
+ )
127
+ session.flush()
128
+
129
+
130
+ def refresh_similar_links(session: Session, memory: Memory) -> int:
131
+ """Replace the memory's `similar` links with its current nearest neighbours."""
132
+ settings = get_settings()
133
+ drop_similar_links(session, memory.id)
134
+ neighbours = nearest_memories(
135
+ session, memory, k=settings.link_neighbors, minimum=settings.link_similarity_min
136
+ )
137
+ for other_id, similarity in neighbours:
138
+ source, target = _pair(memory.id, other_id)
139
+ session.add(
140
+ MemoryLink(
141
+ organization_id=memory.organization_id,
142
+ workspace_id=memory.workspace_id,
143
+ source_memory_id=source,
144
+ target_memory_id=target,
145
+ kind="similar",
146
+ weight=round(similarity, 4),
147
+ )
148
+ )
149
+ session.flush()
150
+ return len(neighbours)
151
+
152
+
153
+ def _dated_parent(source_uri: str) -> str | None:
154
+ """The directory of a dated source file, or None when the name carries no date."""
155
+ parent, _, name = source_uri.rpartition("/")
156
+ return parent + "/" if parent and DATED_NAME.search(name) else None
157
+
158
+
159
+ def link_sequence(session: Session, memory: Memory) -> int:
160
+ """Link the previous dated entry from the same source directory to this memory."""
161
+ if memory.episode_id is None:
162
+ return 0
163
+ episode = session.get(Episode, memory.episode_id)
164
+ if episode is None:
165
+ return 0
166
+ parent = _dated_parent(episode.source_uri)
167
+ if parent is None:
168
+ return 0
169
+ candidates = session.scalars(
170
+ select(Episode)
171
+ .where(
172
+ Episode.organization_id == episode.organization_id,
173
+ (
174
+ Episode.workspace_id == episode.workspace_id
175
+ if episode.workspace_id is not None
176
+ else Episode.workspace_id.is_(None)
177
+ ),
178
+ Episode.id != episode.id,
179
+ Episode.deleted_at.is_(None),
180
+ Episode.source_uri.like(f"{parent}%"),
181
+ Episode.occurred_at < episode.occurred_at,
182
+ )
183
+ .order_by(Episode.occurred_at.desc())
184
+ .limit(20)
185
+ )
186
+ for previous in candidates:
187
+ if _dated_parent(previous.source_uri) != parent:
188
+ continue # a deeper directory or an undated sibling
189
+ earlier = session.scalar(
190
+ select(Memory).where(Memory.episode_id == previous.id, Memory.status == "active")
191
+ )
192
+ if earlier is None:
193
+ continue
194
+ exists = session.scalar(
195
+ select(MemoryLink.id).where(
196
+ MemoryLink.source_memory_id == earlier.id,
197
+ MemoryLink.target_memory_id == memory.id,
198
+ MemoryLink.kind == "sequence",
199
+ )
200
+ )
201
+ if exists is None:
202
+ session.add(
203
+ MemoryLink(
204
+ organization_id=memory.organization_id,
205
+ workspace_id=memory.workspace_id,
206
+ source_memory_id=earlier.id,
207
+ target_memory_id=memory.id,
208
+ kind="sequence",
209
+ weight=1.0,
210
+ )
211
+ )
212
+ session.flush()
213
+ return 1
214
+ return 0
215
+
216
+
217
+ def refresh_links_for_memory(session: Session, memory: Memory) -> dict[str, int]:
218
+ """Compute a memory's links inside a `link` span; never raise."""
219
+ result = {"similar": 0, "sequence": 0}
220
+ recorder = current_recorder()
221
+ if recorder is not None:
222
+ # The live map pulses and refetches on this once the memory is indexed.
223
+ recorder.note(stored_ids=[str(memory.id)])
224
+ with span("link") as handle:
225
+ try:
226
+ with session.begin_nested():
227
+ result["similar"] = refresh_similar_links(session, memory)
228
+ result["sequence"] = link_sequence(session, memory)
229
+ except Exception as exc:
230
+ logger.exception("link computation failed for memory %s", memory.id)
231
+ handle.set(error=type(exc).__name__)
232
+ handle.set(**result)
233
+ return result
234
+
235
+
236
+ def rebuild_links(session: Session, organization_id: UUID | None = None) -> dict[str, int]:
237
+ """Recompute `similar` and `sequence` links for every active memory in scope.
238
+
239
+ Unlike incremental indexing this yields the exact union of every memory's
240
+ nearest neighbours, including links a newer memory would add to an older one.
241
+ """
242
+ settings = get_settings()
243
+ scope = [MemoryLink.kind.in_(("similar", "sequence"))]
244
+ memory_scope = [Memory.status == "active"]
245
+ if organization_id is not None:
246
+ scope.append(MemoryLink.organization_id == organization_id)
247
+ memory_scope.append(Memory.organization_id == organization_id)
248
+ session.query(MemoryLink).filter(*scope).delete(synchronize_session=False)
249
+ session.flush()
250
+ memories = list(session.scalars(select(Memory).where(*memory_scope).order_by(Memory.id)))
251
+ pairs: dict[tuple[UUID, UUID], tuple[Memory, float]] = {}
252
+ for memory in memories:
253
+ for other_id, similarity in nearest_memories(
254
+ session, memory, k=settings.link_neighbors, minimum=settings.link_similarity_min
255
+ ):
256
+ key = _pair(memory.id, other_id)
257
+ if key not in pairs or similarity > pairs[key][1]:
258
+ pairs[key] = (memory, similarity)
259
+ for (source, target), (memory, similarity) in pairs.items():
260
+ session.add(
261
+ MemoryLink(
262
+ organization_id=memory.organization_id,
263
+ workspace_id=memory.workspace_id,
264
+ source_memory_id=source,
265
+ target_memory_id=target,
266
+ kind="similar",
267
+ weight=round(similarity, 4),
268
+ )
269
+ )
270
+ session.flush()
271
+ sequence = sum(link_sequence(session, memory) for memory in memories)
272
+ return {"memories": len(memories), "similar": len(pairs), "sequence": sequence}
273
+
274
+
275
+ COUNTED_TRACE_NAMES = ("POST /v1/search", "POST /v1/context/assemble")
276
+ COUNTED_STATUSES = ("ok", "degraded")
277
+ RECALL_CURSOR_KEY = "recall_cursor"
278
+ RECALL_PAIR_DEPTH = 5
279
+
280
+
281
+ def _lock_cursor(session: Session) -> ConsoleState | None:
282
+ """The cursor row, locked for this transaction; None when another worker holds it.
283
+
284
+ Two workers running housekeeping at once would otherwise read the same
285
+ cursor and count the same traces twice.
286
+ """
287
+ statement = select(ConsoleState).where(ConsoleState.key == RECALL_CURSOR_KEY)
288
+ if _is_postgres(session):
289
+ statement = statement.with_for_update(skip_locked=True)
290
+ state = session.scalar(statement)
291
+ if state is not None:
292
+ return state
293
+ exists = session.scalar(select(ConsoleState.key).where(ConsoleState.key == RECALL_CURSOR_KEY))
294
+ if exists is not None:
295
+ return None # present but locked by another worker
296
+ try:
297
+ with session.begin_nested():
298
+ state = ConsoleState(key=RECALL_CURSOR_KEY, value={})
299
+ session.add(state)
300
+ session.flush()
301
+ except IntegrityError:
302
+ return None # another worker created it first and is processing
303
+ return state
304
+
305
+
306
+ def _cursor_of(state: ConsoleState) -> tuple[datetime, UUID] | None:
307
+ raw = (state.value or {}).get("cursor")
308
+ if not raw:
309
+ return None
310
+ written_at, _, trace_id = str(raw).partition("|")
311
+ return datetime.fromisoformat(written_at), UUID(trace_id)
312
+
313
+
314
+ def _record_recall(session: Session, trace: Trace) -> None:
315
+ raw_ids = (trace.summary or {}).get("memory_ids") or []
316
+ wanted: list[UUID] = []
317
+ for raw in raw_ids:
318
+ try:
319
+ memory_id = UUID(str(raw))
320
+ except ValueError:
321
+ continue
322
+ if memory_id not in wanted:
323
+ wanted.append(memory_id)
324
+ if not wanted or trace.organization_id is None:
325
+ return
326
+ workspaces = dict(
327
+ session.execute(
328
+ select(Memory.id, Memory.workspace_id).where(
329
+ Memory.id.in_(wanted),
330
+ Memory.organization_id == trace.organization_id,
331
+ Memory.status == "active",
332
+ )
333
+ ).all()
334
+ )
335
+ query = str((trace.summary or {}).get("query") or "")[:500]
336
+ for rank, memory_id in enumerate(wanted, start=1):
337
+ if memory_id not in workspaces:
338
+ continue # forgotten since the search ran
339
+ session.add(
340
+ MemoryRecall(
341
+ memory_id=memory_id,
342
+ organization_id=trace.organization_id,
343
+ trace_id=trace.id,
344
+ recalled_at=trace.started_at,
345
+ rank=rank,
346
+ query=query,
347
+ principal_id=trace.principal_id,
348
+ )
349
+ )
350
+ stat = session.get(MemoryStat, memory_id)
351
+ if stat is None:
352
+ stat = MemoryStat(
353
+ memory_id=memory_id, organization_id=trace.organization_id, recall_count=0
354
+ )
355
+ session.add(stat)
356
+ stat.recall_count += 1
357
+ if stat.last_recalled_at is None or _aware(stat.last_recalled_at) < _aware(
358
+ trace.started_at
359
+ ):
360
+ stat.last_recalled_at = trace.started_at
361
+ top = [memory_id for memory_id in wanted if memory_id in workspaces][:RECALL_PAIR_DEPTH]
362
+ for index, first in enumerate(top):
363
+ for second in top[index + 1 :]:
364
+ source, target = _pair(first, second)
365
+ link = session.scalar(
366
+ select(MemoryLink).where(
367
+ MemoryLink.source_memory_id == source,
368
+ MemoryLink.target_memory_id == target,
369
+ MemoryLink.kind == "recalled",
370
+ )
371
+ )
372
+ if link is None:
373
+ shared = workspaces[source] if workspaces[source] == workspaces[target] else None
374
+ session.add(
375
+ MemoryLink(
376
+ organization_id=trace.organization_id,
377
+ workspace_id=shared,
378
+ source_memory_id=source,
379
+ target_memory_id=target,
380
+ kind="recalled",
381
+ weight=1.0,
382
+ count=1,
383
+ )
384
+ )
385
+ else:
386
+ link.count += 1
387
+ link.weight = float(link.count)
388
+ session.flush()
389
+
390
+
391
+ def _aware(value: datetime) -> datetime:
392
+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
393
+
394
+
395
+ def process_recall_traces(
396
+ session: Session, *, batch: int = 500, settle_seconds: float = 2.0
397
+ ) -> int:
398
+ """Turn counted search traces newer than the cursor into recalls, stats and links.
399
+
400
+ The cursor moves in the same transaction as the rows it accounts for, so a
401
+ restart never double counts. Traces younger than `settle_seconds` wait for
402
+ the next pass: the trace writer stamps `written_at` just before it commits.
403
+ """
404
+ state = _lock_cursor(session)
405
+ if state is None:
406
+ session.rollback()
407
+ return 0
408
+ cursor = _cursor_of(state)
409
+ statement = select(Trace).where(
410
+ Trace.name.in_(COUNTED_TRACE_NAMES),
411
+ Trace.status.in_(COUNTED_STATUSES),
412
+ Trace.written_at <= datetime.now(UTC) - timedelta(seconds=settle_seconds),
413
+ )
414
+ if cursor is not None:
415
+ written_at, trace_id = cursor
416
+ statement = statement.where(
417
+ or_(
418
+ Trace.written_at > written_at,
419
+ and_(Trace.written_at == written_at, Trace.id > trace_id),
420
+ )
421
+ )
422
+ traces = list(session.scalars(statement.order_by(Trace.written_at, Trace.id).limit(batch)))
423
+ for trace in traces:
424
+ _record_recall(session, trace)
425
+ if traces:
426
+ state.value = {"cursor": f"{traces[-1].written_at.isoformat()}|{traces[-1].id}"}
427
+ session.commit()
428
+ return len(traces)