albert-memory 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- albert/__init__.py +3 -0
- albert/admin.py +532 -0
- albert/api.py +698 -0
- albert/automatic_memory_release.json +14 -0
- albert/benchmark.py +272 -0
- albert/benchmark_cli.py +37 -0
- albert/capture.py +1051 -0
- albert/capture_installer.py +1000 -0
- albert/capture_promotion.py +298 -0
- albert/capture_retention.py +72 -0
- albert/capture_sdk.py +558 -0
- albert/capture_security.py +79 -0
- albert/chunking.py +79 -0
- albert/classifier.py +263 -0
- albert/claude_adapter.py +357 -0
- albert/claude_lifecycle.py +418 -0
- albert/claude_package.json +19 -0
- albert/claude_plugin/.claude-plugin/plugin.json +11 -0
- albert/claude_plugin/.mcp.json +13 -0
- albert/claude_plugin/hooks/hooks.json +70 -0
- albert/clusters.py +140 -0
- albert/codex_adapter.py +308 -0
- albert/codex_hooks/hooks.json +49 -0
- albert/codex_lifecycle.py +485 -0
- albert/codex_package.json +19 -0
- albert/compatibility.py +121 -0
- albert/config.py +144 -0
- albert/console.py +508 -0
- albert/console_backends.py +236 -0
- albert/console_map.py +518 -0
- albert/console_ops.py +919 -0
- albert/credentials.py +616 -0
- albert/cursor_adapter.py +320 -0
- albert/cursor_hooks/hooks.json +11 -0
- albert/cursor_rules/albert-memory.mdc +11 -0
- albert/db.py +68 -0
- albert/embeddings.py +91 -0
- albert/graph.py +623 -0
- albert/graph_builders.py +879 -0
- albert/graph_cypher.py +669 -0
- albert/graph_store.py +376 -0
- albert/host_targets/claude-code.json +45 -0
- albert/host_targets/codex.json +45 -0
- albert/host_targets/openclaw.json +45 -0
- albert/host_targets.py +271 -0
- albert/links.py +428 -0
- albert/mcp_server.py +457 -0
- albert/memory_safety.py +111 -0
- albert/models.py +611 -0
- albert/openclaw_adapter.py +326 -0
- albert/openclaw_adapter_trust.json +5 -0
- albert/openclaw_package.json +22 -0
- albert/openclaw_plugin/index.js +135 -0
- albert/openclaw_plugin/openclaw.plugin.json +25 -0
- albert/openclaw_plugin/package.json +12 -0
- albert/portability.py +291 -0
- albert/provenance.py +105 -0
- albert/recorder.py +166 -0
- albert/release_gate.py +83 -0
- albert/schemas.py +617 -0
- albert/scope.py +26 -0
- albert/search.py +447 -0
- albert/security.py +150 -0
- albert/services.py +1316 -0
- albert/session_context.py +162 -0
- albert/trace_feed.py +165 -0
- albert/trace_writer.py +131 -0
- albert/tracing.py +75 -0
- albert/worker.py +202 -0
- albert/workspaces.py +256 -0
- albert_memory-0.4.0.dist-info/METADATA +554 -0
- albert_memory-0.4.0.dist-info/RECORD +74 -0
- albert_memory-0.4.0.dist-info/WHEEL +4 -0
- albert_memory-0.4.0.dist-info/entry_points.txt +14 -0
albert/host_targets.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Versioned, fail-closed host lifecycle target contracts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import stat
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path, PurePosixPath
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class HostTargetError(RuntimeError):
|
|
14
|
+
"""A host target cannot be trusted or negotiated."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
HOST_TARGET_SCHEMA = "com.jiggai.albert.host-target"
|
|
18
|
+
HOST_TARGET_SCHEMA_VERSION = 1
|
|
19
|
+
CAPABILITY_PATTERN = re.compile(r"^[a-z][a-z0-9._-]{0,99}$")
|
|
20
|
+
VERSION_PATTERN = re.compile(r"^(\d+)\.(\d+)\.(\d+)$")
|
|
21
|
+
RANGE_PATTERN = re.compile(r"^(>=|>|<=|<|==)(\d+)\.(\d+)\.(\d+)$")
|
|
22
|
+
TARGET_PATH = Path(__file__).with_name("host_targets")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class HostTarget:
|
|
27
|
+
target_id: str
|
|
28
|
+
schema_version: int
|
|
29
|
+
descriptor_version: str
|
|
30
|
+
package: dict[str, str]
|
|
31
|
+
host: dict[str, object]
|
|
32
|
+
hooks: dict[str, object]
|
|
33
|
+
install: dict[str, object]
|
|
34
|
+
credentials: dict[str, object]
|
|
35
|
+
ownership: dict[str, str]
|
|
36
|
+
lifecycle: dict[str, object]
|
|
37
|
+
probes: tuple[str, ...]
|
|
38
|
+
capabilities: tuple[str, ...]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _version(value: object, label: str) -> tuple[int, int, int]:
|
|
42
|
+
if not isinstance(value, str) or not (match := VERSION_PATTERN.fullmatch(value)):
|
|
43
|
+
raise HostTargetError(f"{label} must be an exact semantic version")
|
|
44
|
+
return tuple(int(part) for part in match.groups())
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _version_in_range(version: tuple[int, int, int], declared: object) -> bool:
|
|
48
|
+
if not isinstance(declared, str):
|
|
49
|
+
return False
|
|
50
|
+
clauses = [part.strip() for part in declared.split(",") if part.strip()]
|
|
51
|
+
if not clauses:
|
|
52
|
+
return False
|
|
53
|
+
for clause in clauses:
|
|
54
|
+
match = RANGE_PATTERN.fullmatch(clause)
|
|
55
|
+
if not match:
|
|
56
|
+
return False
|
|
57
|
+
target = tuple(int(value) for value in match.groups()[1:])
|
|
58
|
+
if not {
|
|
59
|
+
">=": version >= target,
|
|
60
|
+
">": version > target,
|
|
61
|
+
"<=": version <= target,
|
|
62
|
+
"<": version < target,
|
|
63
|
+
"==": version == target,
|
|
64
|
+
}[match.group(1)]:
|
|
65
|
+
return False
|
|
66
|
+
return True
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _string_map(value: object, keys: set[str], label: str) -> dict[str, str]:
|
|
70
|
+
if (
|
|
71
|
+
not isinstance(value, dict)
|
|
72
|
+
or set(value) != keys
|
|
73
|
+
or not all(isinstance(item, str) and item for item in value.values())
|
|
74
|
+
):
|
|
75
|
+
raise HostTargetError(f"host target {label} is malformed")
|
|
76
|
+
return dict(value)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def validate_host_target(payload: object) -> HostTarget:
|
|
80
|
+
"""Validate every field before a target can cause host or filesystem changes."""
|
|
81
|
+
if not isinstance(payload, dict) or set(payload) != {
|
|
82
|
+
"schema",
|
|
83
|
+
"schema_version",
|
|
84
|
+
"descriptor_version",
|
|
85
|
+
"target_id",
|
|
86
|
+
"enabled",
|
|
87
|
+
"package",
|
|
88
|
+
"host",
|
|
89
|
+
"hooks",
|
|
90
|
+
"install",
|
|
91
|
+
"credentials",
|
|
92
|
+
"ownership",
|
|
93
|
+
"lifecycle",
|
|
94
|
+
"probes",
|
|
95
|
+
"capabilities",
|
|
96
|
+
}:
|
|
97
|
+
raise HostTargetError("host target contains unknown or missing fields")
|
|
98
|
+
if (
|
|
99
|
+
payload["schema"] != HOST_TARGET_SCHEMA
|
|
100
|
+
or type(payload["schema_version"]) is not int
|
|
101
|
+
or payload["schema_version"] != 1
|
|
102
|
+
):
|
|
103
|
+
raise HostTargetError("host target schema or version is unsupported")
|
|
104
|
+
if payload["enabled"] is not True:
|
|
105
|
+
raise HostTargetError("host target is disabled")
|
|
106
|
+
target_id = payload["target_id"]
|
|
107
|
+
if not isinstance(target_id, str) or not re.fullmatch(r"[a-z][a-z0-9-]{0,49}", target_id):
|
|
108
|
+
raise HostTargetError("host target identifier is invalid")
|
|
109
|
+
_version(payload["descriptor_version"], "descriptor_version")
|
|
110
|
+
|
|
111
|
+
package = _string_map(
|
|
112
|
+
payload["package"], {"distribution", "version", "artifact_provenance"}, "package"
|
|
113
|
+
)
|
|
114
|
+
_version(package["version"], "package.version")
|
|
115
|
+
if package["artifact_provenance"] != "github-oidc-dsse+sha256":
|
|
116
|
+
raise HostTargetError("host target package provenance is unsupported")
|
|
117
|
+
|
|
118
|
+
host = payload["host"]
|
|
119
|
+
if not isinstance(host, dict) or set(host) != {"family", "tested", "supported"}:
|
|
120
|
+
raise HostTargetError("host target host compatibility is malformed")
|
|
121
|
+
tested = _version(host["tested"], "host.tested")
|
|
122
|
+
if host["family"] != target_id or not _version_in_range(tested, host["supported"]):
|
|
123
|
+
raise HostTargetError("host target tested release is outside its supported range")
|
|
124
|
+
|
|
125
|
+
hooks = payload["hooks"]
|
|
126
|
+
if (
|
|
127
|
+
not isinstance(hooks, dict)
|
|
128
|
+
or set(hooks) != {"fidelity", "present", "absent"}
|
|
129
|
+
or hooks["fidelity"] not in {"full", "reduced"}
|
|
130
|
+
or not all(isinstance(hooks[key], list) for key in ("present", "absent"))
|
|
131
|
+
):
|
|
132
|
+
raise HostTargetError("host target hook fidelity is malformed")
|
|
133
|
+
present = hooks["present"]
|
|
134
|
+
absent = hooks["absent"]
|
|
135
|
+
if (
|
|
136
|
+
not all(
|
|
137
|
+
isinstance(item, str) and CAPABILITY_PATTERN.fullmatch(item)
|
|
138
|
+
for item in present + absent
|
|
139
|
+
)
|
|
140
|
+
or len(present) != len(set(present))
|
|
141
|
+
or len(absent) != len(set(absent))
|
|
142
|
+
or set(present) & set(absent)
|
|
143
|
+
or (hooks["fidelity"] == "full" and absent)
|
|
144
|
+
):
|
|
145
|
+
raise HostTargetError("host target hooks are contradictory or fabricated")
|
|
146
|
+
|
|
147
|
+
install = payload["install"]
|
|
148
|
+
if not isinstance(install, dict) or set(install) != {"root", "reload", "uninstall"}:
|
|
149
|
+
raise HostTargetError("host target install contract is malformed")
|
|
150
|
+
root = install["root"]
|
|
151
|
+
if (
|
|
152
|
+
not isinstance(root, str)
|
|
153
|
+
or not root.startswith("~/")
|
|
154
|
+
or ".." in PurePosixPath(root[2:]).parts
|
|
155
|
+
or not all(
|
|
156
|
+
isinstance(install[key], list)
|
|
157
|
+
and install[key]
|
|
158
|
+
and all(isinstance(arg, str) and arg for arg in install[key])
|
|
159
|
+
for key in ("reload", "uninstall")
|
|
160
|
+
)
|
|
161
|
+
):
|
|
162
|
+
raise HostTargetError("host target install path or commands are unsafe")
|
|
163
|
+
|
|
164
|
+
credentials = payload["credentials"]
|
|
165
|
+
if credentials != {
|
|
166
|
+
"acquisition": ["hidden_prompt", "private_stdin", "private_fd"],
|
|
167
|
+
"storage": "owned_file_0600",
|
|
168
|
+
"forbidden_surfaces": [
|
|
169
|
+
"args",
|
|
170
|
+
"ordinary_env",
|
|
171
|
+
"host_config",
|
|
172
|
+
"logs",
|
|
173
|
+
"diagnostics",
|
|
174
|
+
"artifacts",
|
|
175
|
+
],
|
|
176
|
+
}:
|
|
177
|
+
raise HostTargetError("host target credential handling is unsafe")
|
|
178
|
+
ownership = _string_map(payload["ownership"], {"marker", "scope"}, "ownership")
|
|
179
|
+
if ownership["scope"] != "target-owned-tree-and-entry":
|
|
180
|
+
raise HostTargetError("host target ownership scope is unsafe")
|
|
181
|
+
|
|
182
|
+
lifecycle = payload["lifecycle"]
|
|
183
|
+
if (
|
|
184
|
+
not isinstance(lifecycle, dict)
|
|
185
|
+
or set(lifecycle) != {"upgrade", "rotate", "revoke", "rollback", "direct_brain"}
|
|
186
|
+
or not all(value is True for value in lifecycle.values())
|
|
187
|
+
):
|
|
188
|
+
raise HostTargetError("host target lifecycle contract is incomplete")
|
|
189
|
+
probes = payload["probes"]
|
|
190
|
+
capabilities = payload["capabilities"]
|
|
191
|
+
if (
|
|
192
|
+
not isinstance(probes, list)
|
|
193
|
+
or probes != ["host_config", "runtime", "brain_ready", "capture_recall"]
|
|
194
|
+
or not isinstance(capabilities, list)
|
|
195
|
+
or capabilities != sorted(set(capabilities))
|
|
196
|
+
or not capabilities
|
|
197
|
+
or not all(
|
|
198
|
+
isinstance(item, str) and CAPABILITY_PATTERN.fullmatch(item) for item in capabilities
|
|
199
|
+
)
|
|
200
|
+
):
|
|
201
|
+
raise HostTargetError("host target probes or capabilities are invalid")
|
|
202
|
+
return HostTarget(
|
|
203
|
+
target_id=target_id,
|
|
204
|
+
schema_version=payload["schema_version"],
|
|
205
|
+
descriptor_version=payload["descriptor_version"],
|
|
206
|
+
package=package,
|
|
207
|
+
host=dict(host),
|
|
208
|
+
hooks=dict(hooks),
|
|
209
|
+
install=dict(install),
|
|
210
|
+
credentials=dict(credentials),
|
|
211
|
+
ownership=ownership,
|
|
212
|
+
lifecycle=dict(lifecycle),
|
|
213
|
+
probes=tuple(probes),
|
|
214
|
+
capabilities=tuple(capabilities),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def load_host_target(target_id: str) -> HostTarget:
|
|
219
|
+
"""Load only explicitly shipped targets; unknown targets fail closed."""
|
|
220
|
+
if target_id not in {"openclaw", "claude-code", "codex"}:
|
|
221
|
+
raise HostTargetError(f"unknown or disabled host target: {target_id}")
|
|
222
|
+
try:
|
|
223
|
+
payload = json.loads((TARGET_PATH / f"{target_id}.json").read_text(encoding="utf-8"))
|
|
224
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
225
|
+
raise HostTargetError("host target descriptor is unavailable or invalid") from exc
|
|
226
|
+
target = validate_host_target(payload)
|
|
227
|
+
if target.target_id != target_id:
|
|
228
|
+
raise HostTargetError("host target descriptor identity mismatch")
|
|
229
|
+
return target
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def negotiate_host_target(
|
|
233
|
+
target_id: str,
|
|
234
|
+
host_version: tuple[int, int, int],
|
|
235
|
+
required_capabilities: set[str],
|
|
236
|
+
*,
|
|
237
|
+
target: HostTarget | None = None,
|
|
238
|
+
) -> HostTarget:
|
|
239
|
+
negotiated = target or load_host_target(target_id)
|
|
240
|
+
if negotiated.target_id != target_id or not _version_in_range(
|
|
241
|
+
host_version, negotiated.host["supported"]
|
|
242
|
+
):
|
|
243
|
+
raise HostTargetError("host target version is unsupported")
|
|
244
|
+
if not required_capabilities <= set(negotiated.capabilities):
|
|
245
|
+
raise HostTargetError("host target lacks required capabilities")
|
|
246
|
+
return negotiated
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def assert_owned_install_root(root: Path, *, expected_marker: str) -> None:
|
|
250
|
+
"""Reject foreign ownership, symlink traversal, and widened modes."""
|
|
251
|
+
if root.is_symlink():
|
|
252
|
+
raise HostTargetError("host target install root must not be a symlink")
|
|
253
|
+
current = root
|
|
254
|
+
while not current.exists() and current != current.parent:
|
|
255
|
+
current = current.parent
|
|
256
|
+
while current != current.parent:
|
|
257
|
+
if current.is_symlink():
|
|
258
|
+
raise HostTargetError("host target install path contains a symlink")
|
|
259
|
+
current = current.parent
|
|
260
|
+
if not root.exists():
|
|
261
|
+
return
|
|
262
|
+
marker_path = root / "install.json"
|
|
263
|
+
try:
|
|
264
|
+
marker = json.loads(marker_path.read_text(encoding="utf-8"))
|
|
265
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
266
|
+
raise HostTargetError("host target ownership marker is unavailable") from exc
|
|
267
|
+
if not isinstance(marker, dict) or marker.get("owner") != expected_marker:
|
|
268
|
+
raise HostTargetError("host target installation has foreign ownership")
|
|
269
|
+
for private_path in (marker_path, root / "credentials.env", root / "credentials.json"):
|
|
270
|
+
if private_path.exists() and stat.S_IMODE(os.stat(private_path).st_mode) != 0o600:
|
|
271
|
+
raise HostTargetError("host target private file mode is unsafe")
|
albert/links.py
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
"""Derived links between memories.
|
|
2
|
+
|
|
3
|
+
`similar` and `sequence` links are written when a memory is indexed; `recalled`
|
|
4
|
+
links and recall statistics come from the flight recorder's traces. Everything
|
|
5
|
+
here is rebuildable and must never fail the indexing that triggers it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import re
|
|
12
|
+
from datetime import UTC, datetime, timedelta
|
|
13
|
+
from typing import Any
|
|
14
|
+
from uuid import UUID
|
|
15
|
+
|
|
16
|
+
from sqlalchemy import Float, and_, cast, or_, select
|
|
17
|
+
from sqlalchemy.exc import IntegrityError
|
|
18
|
+
from sqlalchemy.orm import Session
|
|
19
|
+
|
|
20
|
+
from albert.config import get_settings
|
|
21
|
+
from albert.models import (
|
|
22
|
+
ConsoleState,
|
|
23
|
+
Episode,
|
|
24
|
+
Memory,
|
|
25
|
+
MemoryChunk,
|
|
26
|
+
MemoryLink,
|
|
27
|
+
MemoryRecall,
|
|
28
|
+
MemoryStat,
|
|
29
|
+
Trace,
|
|
30
|
+
)
|
|
31
|
+
from albert.recorder import current_recorder, span
|
|
32
|
+
from albert.search import _cosine, _enable_iterative_scan
|
|
33
|
+
|
|
34
|
+
logger = logging.getLogger("albert.links")
|
|
35
|
+
PROBE_CHUNKS = 8
|
|
36
|
+
DATED_NAME = re.compile(r"\d{4}-\d{2}-\d{2}")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _is_postgres(session: Session) -> bool:
|
|
40
|
+
return session.bind is not None and session.bind.dialect.name == "postgresql"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _pair(first: UUID, second: UUID) -> tuple[UUID, UUID]:
|
|
44
|
+
"""Canonical order for an undirected link: smaller id first."""
|
|
45
|
+
return (first, second) if str(first) < str(second) else (second, first)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def nearest_memories(
|
|
49
|
+
session: Session, memory: Memory, *, k: int, minimum: float
|
|
50
|
+
) -> list[tuple[UUID, float]]:
|
|
51
|
+
"""Other active memories in the same workspace, by best chunk-pair cosine similarity."""
|
|
52
|
+
chunks = list(
|
|
53
|
+
session.scalars(
|
|
54
|
+
select(MemoryChunk)
|
|
55
|
+
.where(MemoryChunk.memory_id == memory.id, MemoryChunk.embedding.is_not(None))
|
|
56
|
+
.order_by(MemoryChunk.chunk_index)
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
if not chunks:
|
|
60
|
+
return []
|
|
61
|
+
scope = [
|
|
62
|
+
MemoryChunk.organization_id == memory.organization_id,
|
|
63
|
+
MemoryChunk.memory_id != memory.id,
|
|
64
|
+
MemoryChunk.embedding_model == chunks[0].embedding_model,
|
|
65
|
+
MemoryChunk.embedding.is_not(None),
|
|
66
|
+
Memory.status == "active",
|
|
67
|
+
(
|
|
68
|
+
MemoryChunk.workspace_id == memory.workspace_id
|
|
69
|
+
if memory.workspace_id is not None
|
|
70
|
+
else MemoryChunk.workspace_id.is_(None)
|
|
71
|
+
),
|
|
72
|
+
]
|
|
73
|
+
best: dict[UUID, float] = {}
|
|
74
|
+
if _is_postgres(session):
|
|
75
|
+
_enable_iterative_scan(session)
|
|
76
|
+
for chunk in chunks[:PROBE_CHUNKS]:
|
|
77
|
+
distance = cast(MemoryChunk.embedding.op("<=>")(chunk.embedding), Float)
|
|
78
|
+
rows = session.execute(
|
|
79
|
+
select(MemoryChunk.memory_id, distance)
|
|
80
|
+
.join(Memory, Memory.id == MemoryChunk.memory_id)
|
|
81
|
+
.where(*scope)
|
|
82
|
+
.order_by(distance)
|
|
83
|
+
.limit(k * 8)
|
|
84
|
+
)
|
|
85
|
+
for other_id, value in rows:
|
|
86
|
+
similarity = 1.0 - float(value)
|
|
87
|
+
if similarity > best.get(other_id, -1.0):
|
|
88
|
+
best[other_id] = similarity
|
|
89
|
+
else:
|
|
90
|
+
others = session.execute(
|
|
91
|
+
select(MemoryChunk.memory_id, MemoryChunk.embedding)
|
|
92
|
+
.join(Memory, Memory.id == MemoryChunk.memory_id)
|
|
93
|
+
.where(*scope)
|
|
94
|
+
).all()
|
|
95
|
+
mine = [list(chunk.embedding) for chunk in chunks[:PROBE_CHUNKS]]
|
|
96
|
+
for other_id, embedding in others:
|
|
97
|
+
similarity = max(_cosine(list(embedding or []), vector) for vector in mine)
|
|
98
|
+
if similarity > best.get(other_id, -1.0):
|
|
99
|
+
best[other_id] = similarity
|
|
100
|
+
ranked = sorted(best.items(), key=lambda item: (-item[1], str(item[0])))
|
|
101
|
+
return [(other_id, score) for other_id, score in ranked if score >= minimum][:k]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _touching(memory_id: UUID, kind: str) -> Any:
|
|
105
|
+
return (
|
|
106
|
+
MemoryLink.kind == kind,
|
|
107
|
+
or_(MemoryLink.source_memory_id == memory_id, MemoryLink.target_memory_id == memory_id),
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def drop_links_for_memory(session: Session, memory_id: UUID) -> None:
|
|
112
|
+
"""Remove every link touching a memory (used when it is forgotten)."""
|
|
113
|
+
session.query(MemoryLink).filter(
|
|
114
|
+
or_(MemoryLink.source_memory_id == memory_id, MemoryLink.target_memory_id == memory_id)
|
|
115
|
+
).delete(synchronize_session=False)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def drop_similar_links(session: Session, memory_id: UUID) -> None:
|
|
119
|
+
"""Remove a memory's `similar` links; they describe text that has just changed.
|
|
120
|
+
|
|
121
|
+
`sequence` and `recalled` links stay: they come from where the memory sits
|
|
122
|
+
and how it has been used, not from what it currently says.
|
|
123
|
+
"""
|
|
124
|
+
session.query(MemoryLink).filter(*_touching(memory_id, "similar")).delete(
|
|
125
|
+
synchronize_session=False
|
|
126
|
+
)
|
|
127
|
+
session.flush()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def refresh_similar_links(session: Session, memory: Memory) -> int:
|
|
131
|
+
"""Replace the memory's `similar` links with its current nearest neighbours."""
|
|
132
|
+
settings = get_settings()
|
|
133
|
+
drop_similar_links(session, memory.id)
|
|
134
|
+
neighbours = nearest_memories(
|
|
135
|
+
session, memory, k=settings.link_neighbors, minimum=settings.link_similarity_min
|
|
136
|
+
)
|
|
137
|
+
for other_id, similarity in neighbours:
|
|
138
|
+
source, target = _pair(memory.id, other_id)
|
|
139
|
+
session.add(
|
|
140
|
+
MemoryLink(
|
|
141
|
+
organization_id=memory.organization_id,
|
|
142
|
+
workspace_id=memory.workspace_id,
|
|
143
|
+
source_memory_id=source,
|
|
144
|
+
target_memory_id=target,
|
|
145
|
+
kind="similar",
|
|
146
|
+
weight=round(similarity, 4),
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
session.flush()
|
|
150
|
+
return len(neighbours)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _dated_parent(source_uri: str) -> str | None:
|
|
154
|
+
"""The directory of a dated source file, or None when the name carries no date."""
|
|
155
|
+
parent, _, name = source_uri.rpartition("/")
|
|
156
|
+
return parent + "/" if parent and DATED_NAME.search(name) else None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def link_sequence(session: Session, memory: Memory) -> int:
|
|
160
|
+
"""Link the previous dated entry from the same source directory to this memory."""
|
|
161
|
+
if memory.episode_id is None:
|
|
162
|
+
return 0
|
|
163
|
+
episode = session.get(Episode, memory.episode_id)
|
|
164
|
+
if episode is None:
|
|
165
|
+
return 0
|
|
166
|
+
parent = _dated_parent(episode.source_uri)
|
|
167
|
+
if parent is None:
|
|
168
|
+
return 0
|
|
169
|
+
candidates = session.scalars(
|
|
170
|
+
select(Episode)
|
|
171
|
+
.where(
|
|
172
|
+
Episode.organization_id == episode.organization_id,
|
|
173
|
+
(
|
|
174
|
+
Episode.workspace_id == episode.workspace_id
|
|
175
|
+
if episode.workspace_id is not None
|
|
176
|
+
else Episode.workspace_id.is_(None)
|
|
177
|
+
),
|
|
178
|
+
Episode.id != episode.id,
|
|
179
|
+
Episode.deleted_at.is_(None),
|
|
180
|
+
Episode.source_uri.like(f"{parent}%"),
|
|
181
|
+
Episode.occurred_at < episode.occurred_at,
|
|
182
|
+
)
|
|
183
|
+
.order_by(Episode.occurred_at.desc())
|
|
184
|
+
.limit(20)
|
|
185
|
+
)
|
|
186
|
+
for previous in candidates:
|
|
187
|
+
if _dated_parent(previous.source_uri) != parent:
|
|
188
|
+
continue # a deeper directory or an undated sibling
|
|
189
|
+
earlier = session.scalar(
|
|
190
|
+
select(Memory).where(Memory.episode_id == previous.id, Memory.status == "active")
|
|
191
|
+
)
|
|
192
|
+
if earlier is None:
|
|
193
|
+
continue
|
|
194
|
+
exists = session.scalar(
|
|
195
|
+
select(MemoryLink.id).where(
|
|
196
|
+
MemoryLink.source_memory_id == earlier.id,
|
|
197
|
+
MemoryLink.target_memory_id == memory.id,
|
|
198
|
+
MemoryLink.kind == "sequence",
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
if exists is None:
|
|
202
|
+
session.add(
|
|
203
|
+
MemoryLink(
|
|
204
|
+
organization_id=memory.organization_id,
|
|
205
|
+
workspace_id=memory.workspace_id,
|
|
206
|
+
source_memory_id=earlier.id,
|
|
207
|
+
target_memory_id=memory.id,
|
|
208
|
+
kind="sequence",
|
|
209
|
+
weight=1.0,
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
session.flush()
|
|
213
|
+
return 1
|
|
214
|
+
return 0
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def refresh_links_for_memory(session: Session, memory: Memory) -> dict[str, int]:
|
|
218
|
+
"""Compute a memory's links inside a `link` span; never raise."""
|
|
219
|
+
result = {"similar": 0, "sequence": 0}
|
|
220
|
+
recorder = current_recorder()
|
|
221
|
+
if recorder is not None:
|
|
222
|
+
# The live map pulses and refetches on this once the memory is indexed.
|
|
223
|
+
recorder.note(stored_ids=[str(memory.id)])
|
|
224
|
+
with span("link") as handle:
|
|
225
|
+
try:
|
|
226
|
+
with session.begin_nested():
|
|
227
|
+
result["similar"] = refresh_similar_links(session, memory)
|
|
228
|
+
result["sequence"] = link_sequence(session, memory)
|
|
229
|
+
except Exception as exc:
|
|
230
|
+
logger.exception("link computation failed for memory %s", memory.id)
|
|
231
|
+
handle.set(error=type(exc).__name__)
|
|
232
|
+
handle.set(**result)
|
|
233
|
+
return result
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def rebuild_links(session: Session, organization_id: UUID | None = None) -> dict[str, int]:
|
|
237
|
+
"""Recompute `similar` and `sequence` links for every active memory in scope.
|
|
238
|
+
|
|
239
|
+
Unlike incremental indexing this yields the exact union of every memory's
|
|
240
|
+
nearest neighbours, including links a newer memory would add to an older one.
|
|
241
|
+
"""
|
|
242
|
+
settings = get_settings()
|
|
243
|
+
scope = [MemoryLink.kind.in_(("similar", "sequence"))]
|
|
244
|
+
memory_scope = [Memory.status == "active"]
|
|
245
|
+
if organization_id is not None:
|
|
246
|
+
scope.append(MemoryLink.organization_id == organization_id)
|
|
247
|
+
memory_scope.append(Memory.organization_id == organization_id)
|
|
248
|
+
session.query(MemoryLink).filter(*scope).delete(synchronize_session=False)
|
|
249
|
+
session.flush()
|
|
250
|
+
memories = list(session.scalars(select(Memory).where(*memory_scope).order_by(Memory.id)))
|
|
251
|
+
pairs: dict[tuple[UUID, UUID], tuple[Memory, float]] = {}
|
|
252
|
+
for memory in memories:
|
|
253
|
+
for other_id, similarity in nearest_memories(
|
|
254
|
+
session, memory, k=settings.link_neighbors, minimum=settings.link_similarity_min
|
|
255
|
+
):
|
|
256
|
+
key = _pair(memory.id, other_id)
|
|
257
|
+
if key not in pairs or similarity > pairs[key][1]:
|
|
258
|
+
pairs[key] = (memory, similarity)
|
|
259
|
+
for (source, target), (memory, similarity) in pairs.items():
|
|
260
|
+
session.add(
|
|
261
|
+
MemoryLink(
|
|
262
|
+
organization_id=memory.organization_id,
|
|
263
|
+
workspace_id=memory.workspace_id,
|
|
264
|
+
source_memory_id=source,
|
|
265
|
+
target_memory_id=target,
|
|
266
|
+
kind="similar",
|
|
267
|
+
weight=round(similarity, 4),
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
session.flush()
|
|
271
|
+
sequence = sum(link_sequence(session, memory) for memory in memories)
|
|
272
|
+
return {"memories": len(memories), "similar": len(pairs), "sequence": sequence}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
COUNTED_TRACE_NAMES = ("POST /v1/search", "POST /v1/context/assemble")
|
|
276
|
+
COUNTED_STATUSES = ("ok", "degraded")
|
|
277
|
+
RECALL_CURSOR_KEY = "recall_cursor"
|
|
278
|
+
RECALL_PAIR_DEPTH = 5
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _lock_cursor(session: Session) -> ConsoleState | None:
|
|
282
|
+
"""The cursor row, locked for this transaction; None when another worker holds it.
|
|
283
|
+
|
|
284
|
+
Two workers running housekeeping at once would otherwise read the same
|
|
285
|
+
cursor and count the same traces twice.
|
|
286
|
+
"""
|
|
287
|
+
statement = select(ConsoleState).where(ConsoleState.key == RECALL_CURSOR_KEY)
|
|
288
|
+
if _is_postgres(session):
|
|
289
|
+
statement = statement.with_for_update(skip_locked=True)
|
|
290
|
+
state = session.scalar(statement)
|
|
291
|
+
if state is not None:
|
|
292
|
+
return state
|
|
293
|
+
exists = session.scalar(select(ConsoleState.key).where(ConsoleState.key == RECALL_CURSOR_KEY))
|
|
294
|
+
if exists is not None:
|
|
295
|
+
return None # present but locked by another worker
|
|
296
|
+
try:
|
|
297
|
+
with session.begin_nested():
|
|
298
|
+
state = ConsoleState(key=RECALL_CURSOR_KEY, value={})
|
|
299
|
+
session.add(state)
|
|
300
|
+
session.flush()
|
|
301
|
+
except IntegrityError:
|
|
302
|
+
return None # another worker created it first and is processing
|
|
303
|
+
return state
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _cursor_of(state: ConsoleState) -> tuple[datetime, UUID] | None:
|
|
307
|
+
raw = (state.value or {}).get("cursor")
|
|
308
|
+
if not raw:
|
|
309
|
+
return None
|
|
310
|
+
written_at, _, trace_id = str(raw).partition("|")
|
|
311
|
+
return datetime.fromisoformat(written_at), UUID(trace_id)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _record_recall(session: Session, trace: Trace) -> None:
|
|
315
|
+
raw_ids = (trace.summary or {}).get("memory_ids") or []
|
|
316
|
+
wanted: list[UUID] = []
|
|
317
|
+
for raw in raw_ids:
|
|
318
|
+
try:
|
|
319
|
+
memory_id = UUID(str(raw))
|
|
320
|
+
except ValueError:
|
|
321
|
+
continue
|
|
322
|
+
if memory_id not in wanted:
|
|
323
|
+
wanted.append(memory_id)
|
|
324
|
+
if not wanted or trace.organization_id is None:
|
|
325
|
+
return
|
|
326
|
+
workspaces = dict(
|
|
327
|
+
session.execute(
|
|
328
|
+
select(Memory.id, Memory.workspace_id).where(
|
|
329
|
+
Memory.id.in_(wanted),
|
|
330
|
+
Memory.organization_id == trace.organization_id,
|
|
331
|
+
Memory.status == "active",
|
|
332
|
+
)
|
|
333
|
+
).all()
|
|
334
|
+
)
|
|
335
|
+
query = str((trace.summary or {}).get("query") or "")[:500]
|
|
336
|
+
for rank, memory_id in enumerate(wanted, start=1):
|
|
337
|
+
if memory_id not in workspaces:
|
|
338
|
+
continue # forgotten since the search ran
|
|
339
|
+
session.add(
|
|
340
|
+
MemoryRecall(
|
|
341
|
+
memory_id=memory_id,
|
|
342
|
+
organization_id=trace.organization_id,
|
|
343
|
+
trace_id=trace.id,
|
|
344
|
+
recalled_at=trace.started_at,
|
|
345
|
+
rank=rank,
|
|
346
|
+
query=query,
|
|
347
|
+
principal_id=trace.principal_id,
|
|
348
|
+
)
|
|
349
|
+
)
|
|
350
|
+
stat = session.get(MemoryStat, memory_id)
|
|
351
|
+
if stat is None:
|
|
352
|
+
stat = MemoryStat(
|
|
353
|
+
memory_id=memory_id, organization_id=trace.organization_id, recall_count=0
|
|
354
|
+
)
|
|
355
|
+
session.add(stat)
|
|
356
|
+
stat.recall_count += 1
|
|
357
|
+
if stat.last_recalled_at is None or _aware(stat.last_recalled_at) < _aware(
|
|
358
|
+
trace.started_at
|
|
359
|
+
):
|
|
360
|
+
stat.last_recalled_at = trace.started_at
|
|
361
|
+
top = [memory_id for memory_id in wanted if memory_id in workspaces][:RECALL_PAIR_DEPTH]
|
|
362
|
+
for index, first in enumerate(top):
|
|
363
|
+
for second in top[index + 1 :]:
|
|
364
|
+
source, target = _pair(first, second)
|
|
365
|
+
link = session.scalar(
|
|
366
|
+
select(MemoryLink).where(
|
|
367
|
+
MemoryLink.source_memory_id == source,
|
|
368
|
+
MemoryLink.target_memory_id == target,
|
|
369
|
+
MemoryLink.kind == "recalled",
|
|
370
|
+
)
|
|
371
|
+
)
|
|
372
|
+
if link is None:
|
|
373
|
+
shared = workspaces[source] if workspaces[source] == workspaces[target] else None
|
|
374
|
+
session.add(
|
|
375
|
+
MemoryLink(
|
|
376
|
+
organization_id=trace.organization_id,
|
|
377
|
+
workspace_id=shared,
|
|
378
|
+
source_memory_id=source,
|
|
379
|
+
target_memory_id=target,
|
|
380
|
+
kind="recalled",
|
|
381
|
+
weight=1.0,
|
|
382
|
+
count=1,
|
|
383
|
+
)
|
|
384
|
+
)
|
|
385
|
+
else:
|
|
386
|
+
link.count += 1
|
|
387
|
+
link.weight = float(link.count)
|
|
388
|
+
session.flush()
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _aware(value: datetime) -> datetime:
|
|
392
|
+
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def process_recall_traces(
|
|
396
|
+
session: Session, *, batch: int = 500, settle_seconds: float = 2.0
|
|
397
|
+
) -> int:
|
|
398
|
+
"""Turn counted search traces newer than the cursor into recalls, stats and links.
|
|
399
|
+
|
|
400
|
+
The cursor moves in the same transaction as the rows it accounts for, so a
|
|
401
|
+
restart never double counts. Traces younger than `settle_seconds` wait for
|
|
402
|
+
the next pass: the trace writer stamps `written_at` just before it commits.
|
|
403
|
+
"""
|
|
404
|
+
state = _lock_cursor(session)
|
|
405
|
+
if state is None:
|
|
406
|
+
session.rollback()
|
|
407
|
+
return 0
|
|
408
|
+
cursor = _cursor_of(state)
|
|
409
|
+
statement = select(Trace).where(
|
|
410
|
+
Trace.name.in_(COUNTED_TRACE_NAMES),
|
|
411
|
+
Trace.status.in_(COUNTED_STATUSES),
|
|
412
|
+
Trace.written_at <= datetime.now(UTC) - timedelta(seconds=settle_seconds),
|
|
413
|
+
)
|
|
414
|
+
if cursor is not None:
|
|
415
|
+
written_at, trace_id = cursor
|
|
416
|
+
statement = statement.where(
|
|
417
|
+
or_(
|
|
418
|
+
Trace.written_at > written_at,
|
|
419
|
+
and_(Trace.written_at == written_at, Trace.id > trace_id),
|
|
420
|
+
)
|
|
421
|
+
)
|
|
422
|
+
traces = list(session.scalars(statement.order_by(Trace.written_at, Trace.id).limit(batch)))
|
|
423
|
+
for trace in traces:
|
|
424
|
+
_record_recall(session, trace)
|
|
425
|
+
if traces:
|
|
426
|
+
state.value = {"cursor": f"{traces[-1].written_at.isoformat()}|{traces[-1].id}"}
|
|
427
|
+
session.commit()
|
|
428
|
+
return len(traces)
|