superlocalmemory 3.6.2 → 3.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +180 -0
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/core/consolidation_engine.py +8 -0
- package/src/superlocalmemory/core/engine.py +6 -1
- package/src/superlocalmemory/core/fact_consolidator.py +54 -13
- package/src/superlocalmemory/core/graph_pruner.py +55 -19
- package/src/superlocalmemory/core/maintenance.py +9 -0
- package/src/superlocalmemory/core/store_pipeline.py +120 -51
- package/src/superlocalmemory/mcp/_stdin_guard.py +60 -0
- package/src/superlocalmemory/mcp/server.py +34 -5
- package/src/superlocalmemory/optimize/adapters/wrap.py +12 -3
- package/src/superlocalmemory/optimize/cache/manager.py +63 -9
- package/src/superlocalmemory/optimize/compress/router.py +7 -9
- package/src/superlocalmemory/optimize/proxy/_helpers.py +236 -1
- package/src/superlocalmemory/optimize/proxy/anthropic_surface.py +187 -5
- package/src/superlocalmemory/optimize/proxy/gemini_surface.py +373 -14
- package/src/superlocalmemory/optimize/proxy/openai_surface.py +329 -7
- package/src/superlocalmemory/optimize/proxy/server.py +10 -3
- package/src/superlocalmemory/retrieval/engine.py +17 -6
- package/src/superlocalmemory/retrieval/entity_channel.py +10 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +3 -1
- package/src/superlocalmemory/storage/database.py +113 -9
- package/src/superlocalmemory.egg-info/PKG-INFO +1 -1
- package/src/superlocalmemory.egg-info/SOURCES.txt +3 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,186 @@ All notable changes to SuperLocalMemory V3 will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [3.6.4] - 2026-06-09 — Memory-integrity & reliability hardening
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- **`remember` write-path integrity:** fact storage is now idempotent across all memory
|
|
13
|
+
lifecycle states and every write path — storing the same fact twice is one fact. A transient
|
|
14
|
+
backend error during extraction or consolidation can no longer leave a memory without a
|
|
15
|
+
retrievable fact (graceful fallback).
|
|
16
|
+
- **Graph & vector consistency:** edges and embeddings stay correct as memories age,
|
|
17
|
+
consolidate, and are archived (no stale or orphaned graph/vector entries influencing recall).
|
|
18
|
+
- **MCP stdio stability:** resolved a connection-lifecycle edge case that could prematurely
|
|
19
|
+
end a session with strict MCP hosts.
|
|
20
|
+
|
|
21
|
+
### Performance
|
|
22
|
+
|
|
23
|
+
- Faster recall on large knowledge bases; tighter memory bounds in hot paths.
|
|
24
|
+
|
|
25
|
+
### Internal
|
|
26
|
+
|
|
27
|
+
- Expanded automated test coverage for write-path and graph integrity.
|
|
28
|
+
|
|
29
|
+
## [3.6.3] - 2026-06-08 — Cache + compression now work for Claude Code, Claude Desktop, Codex CLI
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
|
|
33
|
+
- **CRITICAL — Cache permanently bypassed for ALL tool-bearing clients (Claude Code, Claude Desktop, Codex CLI):**
|
|
34
|
+
`anthropic_surface.py` and `openai_surface.py` both wrapped every cache operation with
|
|
35
|
+
`if not has_tools and proxy.hooks.cache:`. Because Claude Code ALWAYS sends a `tools` array,
|
|
36
|
+
caching was structurally impossible — zero savings regardless of how many identical prompts
|
|
37
|
+
were sent. Fix: removed all four `has_tools` guards across both surfaces. Cache now fires
|
|
38
|
+
unconditionally for every request on both streaming and non-streaming paths.
|
|
39
|
+
|
|
40
|
+
- **CRITICAL — Anthropic streaming SSE parser returned `None` for tool_use responses:**
|
|
41
|
+
`_parse_sse_to_json()` only accumulated `text_delta` events. Any response containing a
|
|
42
|
+
`tool_use` content block (every Claude Code response) caused the parser to emit an empty
|
|
43
|
+
content array and return `None`, meaning the completed SSE stream was never stored even if the
|
|
44
|
+
`has_tools` guard had been removed. Full rewrite: the parser now tracks content blocks by
|
|
45
|
+
index, handles both `text_delta` and `input_json_delta` events, and assembles complete
|
|
46
|
+
`tool_use` entries (with `id`, `name`, `input` JSON object) in the stored JSON. Returns
|
|
47
|
+
`None` only on genuinely incomplete streams (missing `message_start` or `message_stop`).
|
|
48
|
+
|
|
49
|
+
- **CRITICAL — Anthropic SSE cache replay did not emit tool_use blocks:**
|
|
50
|
+
`_sse_from_cached_json()` only replayed `text` content blocks. Tool-use blocks were silently
|
|
51
|
+
dropped, producing a truncated response on cache hits. Fixed: the replay function now handles
|
|
52
|
+
`tool_use` blocks — emits `content_block_start` with `{type:"tool_use", id, name, input:{}}`,
|
|
53
|
+
then `input_json_delta` chunks (50-char pieces), then `content_block_stop`. Clients receive
|
|
54
|
+
a byte-for-byte equivalent of the original SSE stream.
|
|
55
|
+
|
|
56
|
+
- **CRITICAL — OpenAI SSE parser returned `None` for tool_calls responses:**
|
|
57
|
+
`_parse_openai_sse_to_json()` detected `tool_calls` in the delta and returned `None`
|
|
58
|
+
immediately. OpenAI-compatible clients (Codex CLI, Antigravity) were therefore never cached.
|
|
59
|
+
Fixed: parser now accumulates `tool_calls` by `choice_index → tc_index → {id, type, function}`
|
|
60
|
+
and stores them in the assembled `chat.completion` JSON. Tool call arguments are joined from
|
|
61
|
+
streaming `arguments` deltas.
|
|
62
|
+
|
|
63
|
+
- **CRITICAL — OpenAI SSE cache replay dropped tool_calls entirely:**
|
|
64
|
+
`_openai_sse_from_cached_json()` only replayed text content. Tool calls were silently dropped.
|
|
65
|
+
Fixed: replays `tool_calls` as proper OpenAI SSE delta events — first chunk with role +
|
|
66
|
+
tool_call headers (id, type, name, empty arguments), then argument chunks (50-char pieces),
|
|
67
|
+
then finish chunk. Preserves the streaming contract with clients.
|
|
68
|
+
|
|
69
|
+
- **CompressRouter never instantiated — `compress_hook=None` always:**
|
|
70
|
+
`_load_hooks()` in `server.py` had dead code: `if config.compress_enabled: pass`. The
|
|
71
|
+
`CompressRouter` singleton was never created, so compression was silently a no-op for every
|
|
72
|
+
session since v3.6.0. Fixed: `_load_hooks()` now calls `CompressRouter.get_instance()` and
|
|
73
|
+
wires it into the `HookChain`. Daemon log now correctly reports `compress_hook=CompressRouter`.
|
|
74
|
+
|
|
75
|
+
- **MetricsCollector never wired to CompressRouter — `compress_runs=0` always:**
|
|
76
|
+
`CompressRouter.set_metrics()` was never called during proxy startup, so
|
|
77
|
+
`_metrics_counters=None` permanently. Result: `compress_runs` counter was always 0 in the
|
|
78
|
+
dashboard even when compression was running. Fixed: `_load_hooks()` calls
|
|
79
|
+
`compress_hook.set_metrics(MetricsCollector.get_instance())` immediately after instantiation.
|
|
80
|
+
|
|
81
|
+
- **`on_compress` signature mismatch — metrics counter never incremented:**
|
|
82
|
+
`CompressRouter._compress_messages()` called `self._metrics_counters.on_compress(saved, lossy)`
|
|
83
|
+
where `saved` was bytes-saved and `lossy` was a bool. `MetricsCollector.on_compress()` expects
|
|
84
|
+
`(bytes_original, bytes_after)`. The mismatch meant `compress_runs` and `bytes_saved` were
|
|
85
|
+
always wrong even after wiring. Fixed: caller now passes `(before_tokens, after_tokens)`.
|
|
86
|
+
|
|
87
|
+
- **`is_tool_msg` in `CompressRouter` skipped ALL user messages from compression:**
|
|
88
|
+
The original guard was `is_tool_msg = (role == "tool" or role == "user")`. This silently
|
|
89
|
+
skipped every `user` turn, including long tool-result messages (the main source of savings
|
|
90
|
+
in Claude Code sessions). Fixed: only OpenAI `role=="tool"` messages are skipped. Anthropic
|
|
91
|
+
`tool_result` blocks (embedded in user message content arrays) are now compressed by
|
|
92
|
+
`_compress_content_block()`, which handles the nested structure correctly.
|
|
93
|
+
|
|
94
|
+
### Tests
|
|
95
|
+
|
|
96
|
+
- `tests/optimize/proxy/test_openai_surface.py`: updated `test_parse_openai_sse_to_json_tool_calls_returns_none`
|
|
97
|
+
→ renamed to `test_parse_openai_sse_to_json_tool_calls_cached`, asserts valid JSON returned
|
|
98
|
+
with `tool_calls` array preserved instead of `None`.
|
|
99
|
+
- `tests/optimize/proxy/test_server.py`: updated `test_load_hooks_compress_enabled_placeholder`
|
|
100
|
+
→ renamed to `test_load_hooks_compress_enabled_loads_router`, asserts `hooks.compress is not None`
|
|
101
|
+
and `isinstance(hooks.compress, CompressRouter)`.
|
|
102
|
+
- All 83 proxy tests pass (0 failures).
|
|
103
|
+
|
|
104
|
+
### Documentation
|
|
105
|
+
|
|
106
|
+
- `docs/proxy-setup.md`: removed "Cache fires only for requests WITHOUT tools" caveat. Updated
|
|
107
|
+
"What Gets Cached" table — Claude Code, Claude Desktop, Codex CLI now show `✓ Yes`. Added
|
|
108
|
+
explanation of how tool-use caching works (SSE accumulate → parse → store → replay as SSE).
|
|
109
|
+
Updated troubleshooting section — removed stale "tool-bearing requests are bypassed" note,
|
|
110
|
+
added actionable checklist for diagnosing zero-savings scenarios.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## [3.6.3] - 2026-06-08 — Proxy streaming cache fix for Anthropic, OpenAI surfaces
|
|
115
|
+
|
|
116
|
+
### Fixed
|
|
117
|
+
|
|
118
|
+
- **CRITICAL — Anthropic streaming cache never populated (miss permanently 0 savings):**
|
|
119
|
+
`anthropic_surface.py`'s streaming path called `_stream_forward()` (no-op passthrough)
|
|
120
|
+
instead of the new `_stream_and_cache_forward()`. Claude Code, AGY, and every other
|
|
121
|
+
streaming Anthropic client could never populate the cache because the response body was
|
|
122
|
+
never accumulated. Cache was always empty, `tokens_saved` was always 0, regardless of how
|
|
123
|
+
many identical prompts were sent. Fix: streaming path now checks cache on the way in
|
|
124
|
+
(`_safe_cache_check`), runs compression on the request body, and passes a `store_callback`
|
|
125
|
+
to `_stream_and_cache_forward()` that accumulates the SSE stream, parses it via
|
|
126
|
+
`_parse_sse_to_json`, and stores the assembled JSON message after `message_stop` is seen.
|
|
127
|
+
Second identical streaming call returns a properly re-emitted SSE stream from cache
|
|
128
|
+
(verified: `msg_id` identical, no upstream call on hit).
|
|
129
|
+
|
|
130
|
+
- **CRITICAL — OpenAI streaming surface: cache bypassed + `_safe_compress` NameError:**
|
|
131
|
+
`openai_surface.py` had the same streaming bypass bug AND a missing import — `_safe_compress`
|
|
132
|
+
was called on the non-streaming compression path but never imported, causing a silent
|
|
133
|
+
`NameError` on any non-streaming request with compression enabled. Both issues fixed:
|
|
134
|
+
(1) streaming path now wires `_stream_and_cache_forward` with `_parse_openai_sse_to_json`
|
|
135
|
+
and `_openai_sse_from_cached_json` helpers (OpenAI SSE format differs from Anthropic's —
|
|
136
|
+
uses `[DONE]` sentinel and `chat.completion.chunk` objects). (2) `_safe_compress` added
|
|
137
|
+
to imports. Cache hit on OpenAI streaming calls now returns a re-emitted SSE stream with
|
|
138
|
+
proper `chat.completion.chunk` events and `[DONE]` terminator.
|
|
139
|
+
|
|
140
|
+
- **`_stream_and_cache_forward` completion marker was Anthropic-only:**
|
|
141
|
+
The `finally` block checked for `b"message_stop"` to detect a complete stream before
|
|
142
|
+
calling `on_complete`. OpenAI SSE streams end with `data: [DONE]\n\n` — not `message_stop`.
|
|
143
|
+
Result: OpenAI streaming responses were never stored in cache even after the fix above
|
|
144
|
+
because the `on_complete` callback was never fired. Fixed by checking both markers:
|
|
145
|
+
`b"message_stop"` (Anthropic) OR `b"[DONE]"` (OpenAI / any compatible provider).
|
|
146
|
+
|
|
147
|
+
- **`_stream_and_cache_forward` redundant join:** `full = b"".join(acc)` recomputed the
|
|
148
|
+
join that `_joined` had already computed. Fixed to reuse `_joined` directly.
|
|
149
|
+
|
|
150
|
+
- **`server.py` version string stuck at `"3.6.0"`:** `_PROXY_VERSION` was not updated
|
|
151
|
+
during the 3.6.1 and 3.6.2 releases. Fixed to `"3.6.3"`. The `/health` endpoint now
|
|
152
|
+
correctly reports `"version":"3.6.3"`.
|
|
153
|
+
|
|
154
|
+
- **`CacheManager.get()` returned `None` on miss, discarding the cache key:** Store
|
|
155
|
+
condition `cache_result.cache_key` was always falsy on miss because `get()` returned
|
|
156
|
+
`None` (no `CachedResponse` object). Non-streaming responses after a miss were never
|
|
157
|
+
stored. Fixed: `get()` now returns `CachedResponse(hit=False, data=None, cache_key=key)`
|
|
158
|
+
so the key propagates to the store condition.
|
|
159
|
+
|
|
160
|
+
- **`CacheManager.check()` never called `MetricsCollector.on_miss()`:** Miss events were
|
|
161
|
+
not counted, so `hits/(hits+misses)` was always 0 in the dashboard. Fixed: `check()`
|
|
162
|
+
calls `MetricsCollector.get_instance().on_miss()` when `result.hit is False`.
|
|
163
|
+
|
|
164
|
+
### Added
|
|
165
|
+
|
|
166
|
+
- `_parse_openai_sse_to_json(sse_bytes)`: assembles OpenAI streaming chunks into a
|
|
167
|
+
single `chat.completion` JSON for cache storage. Handles multi-index choices, usage
|
|
168
|
+
capture, `tool_calls` detection (never caches tool responses), and `[DONE]` sentinel.
|
|
169
|
+
- `_openai_sse_from_cached_json(cached_bytes)`: replays a stored `chat.completion` as
|
|
170
|
+
a proper OpenAI SSE stream for cache-hit responses. Emits role chunk, content chunks
|
|
171
|
+
(100-char batches), finish chunk, and `[DONE]`. Preserves the streaming contract with
|
|
172
|
+
clients (Codex CLI, Antigravity, openai-python).
|
|
173
|
+
- `docs/proxy-setup.md`: comprehensive per-CLI proxy activation guide covering Claude Code
|
|
174
|
+
CLI, Claude Desktop, Cursor, Windsurf, AGY/Antigravity, Gemini CLI, Codex CLI, Python
|
|
175
|
+
anthropic/openai SDK, Node.js SDK, LangChain, LlamaIndex, SDK adapter, and raw curl.
|
|
176
|
+
Includes an honest "What Gets Cached" table showing which clients benefit from caching.
|
|
177
|
+
|
|
178
|
+
### Tests
|
|
179
|
+
|
|
180
|
+
- `tests/optimize/proxy/test_openai_surface.py`: 9 new tests covering
|
|
181
|
+
`_parse_openai_sse_to_json` (complete stream, missing `[DONE]`, tool calls, empty bytes,
|
|
182
|
+
usage capture) and `_openai_sse_from_cached_json` (roundtrip, bad JSON, wrong object
|
|
183
|
+
type) plus an end-to-end streaming cache miss→store→hit cycle.
|
|
184
|
+
- All 583 optimize tests pass (4 skipped — platform-specific).
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
8
188
|
## [3.6.2] - 2026-06-08 — wrap dry_run fix for config-file mechanism (macOS CI)
|
|
9
189
|
|
|
10
190
|
### Fixed
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "superlocalmemory",
|
|
3
|
-
"version": "3.6.
|
|
3
|
+
"version": "3.6.4",
|
|
4
4
|
"description": "Information-geometric agent memory with mathematical guarantees. 4-channel retrieval, Fisher-Rao similarity, zero-LLM mode, EU AI Act compliant. Works with Claude, Cursor, Windsurf, and 17+ AI tools.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-memory",
|
package/pyproject.toml
CHANGED
|
@@ -28,7 +28,7 @@ if "OMP_NUM_THREADS" not in os.environ:
|
|
|
28
28
|
os.environ["OMP_NUM_THREADS"] = "2"
|
|
29
29
|
# ---------------------------------------------------------------------------
|
|
30
30
|
|
|
31
|
-
__version__ = "3.6.
|
|
31
|
+
__version__ = "3.6.3"
|
|
32
32
|
|
|
33
33
|
_REQUIRED_VERSIONS = {
|
|
34
34
|
"sentence_transformers": "5.3.0",
|
|
@@ -745,6 +745,14 @@ class ConsolidationEngine:
|
|
|
745
745
|
"""
|
|
746
746
|
from superlocalmemory.storage.models import _new_id
|
|
747
747
|
|
|
748
|
+
# NOTE (core-promotion-02, intentionally NOT changed): the audit
|
|
749
|
+
# flagged placeholder core blocks ("No data available.") + version
|
|
750
|
+
# churn. Investigation showed BOTH the "always 5 blocks" layout and the
|
|
751
|
+
# monotonic version-on-recompile counter are intentional, tested design
|
|
752
|
+
# (see test_consolidation_engine), and the read-side injection filter
|
|
753
|
+
# already hides placeholders from agents. Suppressing or de-churning
|
|
754
|
+
# here breaks that design for a cosmetic gain — so it is left as-is.
|
|
755
|
+
|
|
748
756
|
# Get existing version for increment
|
|
749
757
|
existing = self._db.get_core_block(profile_id, block_type)
|
|
750
758
|
version = (existing["version"] + 1) if existing else 1
|
|
@@ -281,13 +281,18 @@ class MemoryEngine:
|
|
|
281
281
|
# V3.2: ConsolidationEngine (Phase 5) -- sleep-time consolidation
|
|
282
282
|
from superlocalmemory.core.summarizer import Summarizer
|
|
283
283
|
summarizer = Summarizer(self._config)
|
|
284
|
+
# P1-5 (core-promotion-01): wire a real behavioral store. Previously
|
|
285
|
+
# hardcoded None → _compile_behavioral_block always returned the
|
|
286
|
+
# "No behavioral patterns detected yet." placeholder, so behavioral
|
|
287
|
+
# patterns never reached the always-injected core block (dead feature).
|
|
288
|
+
from superlocalmemory.core.recall_pipeline import _get_behavioral_tracker
|
|
284
289
|
self._consolidation_engine = _init_consolidation(
|
|
285
290
|
self._config, self._db,
|
|
286
291
|
auto_linker=self._auto_linker,
|
|
287
292
|
graph_analyzer=self._graph_analyzer,
|
|
288
293
|
temporal_validator=self._temporal_validator,
|
|
289
294
|
summarizer=summarizer,
|
|
290
|
-
behavioral_store=
|
|
295
|
+
behavioral_store=_get_behavioral_tracker(self._db),
|
|
291
296
|
embedder=self._embedder, # v3.4.7: for CCQ worker
|
|
292
297
|
llm=getattr(self, "_llm", None), # v3.4.7: for CCQ worker
|
|
293
298
|
)
|
|
@@ -226,19 +226,41 @@ def _consolidate_cluster(
|
|
|
226
226
|
except (json.JSONDecodeError, TypeError):
|
|
227
227
|
pass
|
|
228
228
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
))
|
|
229
|
+
# P0-3 (dedup-complete-01): apply the SAME content-idempotency invariant
|
|
230
|
+
# as storage.database.store_fact — but on THIS cursor so it stays inside
|
|
231
|
+
# the cluster SAVEPOINT. Previously this raw INSERT bypassed dedup, so a
|
|
232
|
+
# consolidated summary identical to an existing live fact created a
|
|
233
|
+
# duplicate row and never reinforced evidence. Now: reinforce-or-insert.
|
|
234
|
+
# (Excludes 'archived' = soft-deleted, mirroring store_fact.)
|
|
235
|
+
_existing = c.execute(
|
|
236
|
+
"SELECT fact_id FROM atomic_facts "
|
|
237
|
+
"WHERE profile_id = ? AND content = ? "
|
|
238
|
+
"AND lifecycle IN ('active', 'warm', 'cold') "
|
|
239
|
+
"ORDER BY created_at LIMIT 1",
|
|
240
|
+
(profile_id, summary),
|
|
241
|
+
).fetchone()
|
|
242
|
+
if _existing:
|
|
243
|
+
new_fact_id = _existing["fact_id"]
|
|
244
|
+
c.execute(
|
|
245
|
+
"UPDATE atomic_facts "
|
|
246
|
+
"SET evidence_count = evidence_count + ?, access_count = access_count + 1 "
|
|
247
|
+
"WHERE fact_id = ?",
|
|
248
|
+
(len(facts), new_fact_id),
|
|
249
|
+
)
|
|
250
|
+
else:
|
|
251
|
+
c.execute("""
|
|
252
|
+
INSERT INTO atomic_facts
|
|
253
|
+
(fact_id, memory_id, profile_id, content, fact_type,
|
|
254
|
+
entities_json, canonical_entities_json,
|
|
255
|
+
confidence, importance, evidence_count, access_count,
|
|
256
|
+
created_at, lifecycle)
|
|
257
|
+
VALUES (?, '', ?, ?, 'semantic', ?, ?, ?, 0.8, ?, 0, ?, 'active')
|
|
258
|
+
""", (
|
|
259
|
+
new_fact_id, profile_id, summary,
|
|
260
|
+
json.dumps(list(all_entities)),
|
|
261
|
+
json.dumps(list(all_entities)),
|
|
262
|
+
round(avg_confidence, 3), len(facts), now,
|
|
263
|
+
))
|
|
242
264
|
|
|
243
265
|
# Record the consolidation
|
|
244
266
|
consolidation_id = uuid.uuid4().hex[:16]
|
|
@@ -257,6 +279,25 @@ def _consolidate_cluster(
|
|
|
257
279
|
(*fact_ids, profile_id),
|
|
258
280
|
)
|
|
259
281
|
|
|
282
|
+
# P1-4 (graph-integrity-01): archived facts must stop influencing
|
|
283
|
+
# graph-based ranking. The association_edges FK is ON DELETE CASCADE
|
|
284
|
+
# only (no ON UPDATE), so archiving via UPDATE leaves orphaned edges
|
|
285
|
+
# that spreading_activation still reads. Remove edges touching the
|
|
286
|
+
# archived facts, and set their retention zone so ForgettingFilter
|
|
287
|
+
# excludes them. Inside the SAVEPOINT for atomicity.
|
|
288
|
+
c.execute(
|
|
289
|
+
f"DELETE FROM association_edges "
|
|
290
|
+
f"WHERE profile_id = ? "
|
|
291
|
+
f"AND (source_fact_id IN ({placeholders}) "
|
|
292
|
+
f" OR target_fact_id IN ({placeholders}))",
|
|
293
|
+
(profile_id, *fact_ids, *fact_ids),
|
|
294
|
+
)
|
|
295
|
+
c.execute(
|
|
296
|
+
f"UPDATE fact_retention SET lifecycle_zone = 'archive' "
|
|
297
|
+
f"WHERE profile_id = ? AND fact_id IN ({placeholders})",
|
|
298
|
+
(profile_id, *fact_ids),
|
|
299
|
+
)
|
|
300
|
+
|
|
260
301
|
c.execute(f"RELEASE SAVEPOINT {savepoint_name}")
|
|
261
302
|
|
|
262
303
|
except Exception:
|
|
@@ -56,6 +56,7 @@ def prune_graph(
|
|
|
56
56
|
"self_loops_removed": 0,
|
|
57
57
|
"duplicates_removed": 0,
|
|
58
58
|
"hub_edges_removed": 0,
|
|
59
|
+
"association_orphans_removed": 0, # gi-04
|
|
59
60
|
"total_before": 0,
|
|
60
61
|
"total_after": 0,
|
|
61
62
|
}
|
|
@@ -83,6 +84,9 @@ def prune_graph(
|
|
|
83
84
|
stats["hub_edges_removed"] = _cap_node_degree(
|
|
84
85
|
c, profile_id, _MAX_DEGREE_PER_NODE, dry_run,
|
|
85
86
|
)
|
|
87
|
+
stats["association_orphans_removed"] = _remove_orphan_association_edges(
|
|
88
|
+
c, profile_id, dry_run,
|
|
89
|
+
)
|
|
86
90
|
|
|
87
91
|
if dry_run:
|
|
88
92
|
c.execute("ROLLBACK")
|
|
@@ -122,6 +126,34 @@ def prune_graph(
|
|
|
122
126
|
return stats
|
|
123
127
|
|
|
124
128
|
|
|
129
|
+
def _remove_orphan_association_edges(
|
|
130
|
+
c: sqlite3.Cursor,
|
|
131
|
+
profile_id: str,
|
|
132
|
+
dry_run: bool,
|
|
133
|
+
) -> int:
|
|
134
|
+
"""gi-04: remove association_edges whose source/target fact no longer
|
|
135
|
+
exists in atomic_facts.
|
|
136
|
+
|
|
137
|
+
prune_graph historically only touched graph_edges despite its docstring
|
|
138
|
+
claiming "all graph pruning". Hard-deleted facts leave orphaned
|
|
139
|
+
association_edges (the FK cascade only fires under FK-on connections),
|
|
140
|
+
which spreading_activation then has to scan. Returns rows removed.
|
|
141
|
+
"""
|
|
142
|
+
where = (
|
|
143
|
+
"profile_id = ? AND ("
|
|
144
|
+
"source_fact_id NOT IN (SELECT fact_id FROM atomic_facts WHERE profile_id = ?) "
|
|
145
|
+
"OR target_fact_id NOT IN (SELECT fact_id FROM atomic_facts WHERE profile_id = ?))"
|
|
146
|
+
)
|
|
147
|
+
c.execute(f"SELECT COUNT(*) AS cnt FROM association_edges WHERE {where}",
|
|
148
|
+
(profile_id, profile_id, profile_id))
|
|
149
|
+
n = c.fetchone()["cnt"]
|
|
150
|
+
if dry_run or not n:
|
|
151
|
+
return n
|
|
152
|
+
c.execute(f"DELETE FROM association_edges WHERE {where}",
|
|
153
|
+
(profile_id, profile_id, profile_id))
|
|
154
|
+
return c.rowcount
|
|
155
|
+
|
|
156
|
+
|
|
125
157
|
def _remove_orphan_edges(
|
|
126
158
|
c: sqlite3.Cursor,
|
|
127
159
|
profile_id: str,
|
|
@@ -321,60 +353,64 @@ def _cap_node_degree(
|
|
|
321
353
|
2. Edges with rn > max_degree are deleted in a single DELETE statement.
|
|
322
354
|
Requires SQLite 3.25+ (window functions). System is on 3.53.1.
|
|
323
355
|
"""
|
|
356
|
+
# gi-03: cap BOTH out-degree (PARTITION BY source_id) AND in-degree
|
|
357
|
+
# (PARTITION BY target_id). Previously only out-degree was capped, so hub
|
|
358
|
+
# nodes accumulated unbounded in-degree (observed up to 1457), inflating
|
|
359
|
+
# entity-channel fan-in cost. An edge is removed if it exceeds max_degree
|
|
360
|
+
# in EITHER direction (low-weight to both its endpoints). Computed in one
|
|
361
|
+
# window-function pass, no Python loops.
|
|
324
362
|
if dry_run:
|
|
325
363
|
c.execute(
|
|
326
364
|
"""
|
|
327
365
|
SELECT COUNT(*) as cnt FROM (
|
|
328
366
|
SELECT edge_id,
|
|
329
|
-
ROW_NUMBER() OVER (
|
|
330
|
-
|
|
331
|
-
) as rn
|
|
367
|
+
ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY weight DESC) as out_rn,
|
|
368
|
+
ROW_NUMBER() OVER (PARTITION BY target_id ORDER BY weight DESC) as in_rn
|
|
332
369
|
FROM graph_edges
|
|
333
370
|
WHERE profile_id = ?
|
|
334
|
-
) WHERE
|
|
371
|
+
) WHERE out_rn > ? OR in_rn > ?
|
|
335
372
|
""",
|
|
336
|
-
(profile_id, max_degree),
|
|
373
|
+
(profile_id, max_degree, max_degree),
|
|
337
374
|
)
|
|
338
375
|
excess = c.fetchone()["cnt"]
|
|
339
376
|
logger.info(
|
|
340
|
-
"(dry-run) _cap_node_degree: ~%d edges would be removed (max_degree=%d)",
|
|
377
|
+
"(dry-run) _cap_node_degree: ~%d edges would be removed (max_degree=%d, in+out)",
|
|
341
378
|
excess, max_degree,
|
|
342
379
|
)
|
|
343
380
|
return excess
|
|
344
381
|
|
|
345
|
-
# Step 1:
|
|
346
|
-
c.execute("
|
|
347
|
-
c.execute("
|
|
382
|
+
# Step 1: collect edges exceeding the cap in either direction (one pass).
|
|
383
|
+
c.execute("DROP TABLE IF EXISTS _slm_cap_del")
|
|
384
|
+
c.execute("CREATE TEMP TABLE _slm_cap_del (edge_id TEXT PRIMARY KEY)")
|
|
348
385
|
c.execute(
|
|
349
386
|
"""
|
|
350
|
-
INSERT INTO
|
|
387
|
+
INSERT OR IGNORE INTO _slm_cap_del (edge_id)
|
|
351
388
|
SELECT edge_id FROM (
|
|
352
389
|
SELECT edge_id,
|
|
353
|
-
ROW_NUMBER() OVER (
|
|
354
|
-
|
|
355
|
-
) as rn
|
|
390
|
+
ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY weight DESC) as out_rn,
|
|
391
|
+
ROW_NUMBER() OVER (PARTITION BY target_id ORDER BY weight DESC) as in_rn
|
|
356
392
|
FROM graph_edges
|
|
357
393
|
WHERE profile_id = ?
|
|
358
|
-
) WHERE
|
|
394
|
+
) WHERE out_rn > ? OR in_rn > ?
|
|
359
395
|
""",
|
|
360
|
-
(profile_id, max_degree),
|
|
396
|
+
(profile_id, max_degree, max_degree),
|
|
361
397
|
)
|
|
362
398
|
|
|
363
|
-
# Step 2: delete
|
|
399
|
+
# Step 2: delete the over-cap edges (single DELETE).
|
|
364
400
|
c.execute(
|
|
365
401
|
"""
|
|
366
402
|
DELETE FROM graph_edges
|
|
367
403
|
WHERE profile_id = ?
|
|
368
|
-
AND edge_id
|
|
404
|
+
AND edge_id IN (SELECT edge_id FROM _slm_cap_del)
|
|
369
405
|
""",
|
|
370
406
|
(profile_id,),
|
|
371
407
|
)
|
|
372
408
|
deleted = c.rowcount
|
|
373
409
|
|
|
374
|
-
c.execute("DROP TABLE IF EXISTS
|
|
410
|
+
c.execute("DROP TABLE IF EXISTS _slm_cap_del")
|
|
375
411
|
|
|
376
412
|
logger.info(
|
|
377
|
-
"_cap_node_degree: deleted %d low-weight edges (max_degree=%d)",
|
|
413
|
+
"_cap_node_degree: deleted %d low-weight edges (max_degree=%d, in+out capped)",
|
|
378
414
|
deleted, max_degree,
|
|
379
415
|
)
|
|
380
416
|
return deleted
|
|
@@ -107,8 +107,17 @@ def run_maintenance(
|
|
|
107
107
|
"fisher_coupled": 0,
|
|
108
108
|
"sheaf_checked": 0,
|
|
109
109
|
"entity_summaries_consolidated": 0, # V3.4.40
|
|
110
|
+
"orphan_metadata_gc": 0, # v3.6.4 (P1-3)
|
|
110
111
|
}
|
|
111
112
|
|
|
113
|
+
# P1-3 (embeddings-vector-02): sweep orphaned embedding_metadata left by
|
|
114
|
+
# any FK-off delete path, so the semantic channel never maps to dead facts.
|
|
115
|
+
# Runs before the early-return so it sweeps even for empty profiles.
|
|
116
|
+
try:
|
|
117
|
+
counts["orphan_metadata_gc"] = db.gc_orphaned_embedding_metadata()
|
|
118
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
119
|
+
logger.debug("orphan metadata GC skipped: %s", exc)
|
|
120
|
+
|
|
112
121
|
facts = db.get_all_facts(profile_id)
|
|
113
122
|
if not facts:
|
|
114
123
|
return counts
|