superlocalmemory 3.4.10 → 3.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -11
- package/docs/skill-evolution.md +77 -10
- package/ide/hooks/tool-event-hook.sh +4 -4
- package/package.json +1 -1
- package/pyproject.toml +3 -2
- package/src/superlocalmemory/cli/commands.py +170 -0
- package/src/superlocalmemory/cli/main.py +21 -0
- package/src/superlocalmemory/cli/setup_wizard.py +54 -11
- package/src/superlocalmemory/core/config.py +35 -0
- package/src/superlocalmemory/core/consolidation_engine.py +128 -0
- package/src/superlocalmemory/core/embedding_worker.py +1 -1
- package/src/superlocalmemory/core/engine.py +12 -0
- package/src/superlocalmemory/core/fact_consolidator.py +425 -0
- package/src/superlocalmemory/core/graph_pruner.py +290 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +20 -0
- package/src/superlocalmemory/core/recall_pipeline.py +9 -0
- package/src/superlocalmemory/core/tier_manager.py +325 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +6 -5
- package/src/superlocalmemory/evolution/__init__.py +29 -0
- package/src/superlocalmemory/evolution/blind_verifier.py +115 -0
- package/src/superlocalmemory/evolution/evolution_store.py +302 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +181 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +555 -0
- package/src/superlocalmemory/evolution/triggers.py +367 -0
- package/src/superlocalmemory/evolution/types.py +92 -0
- package/src/superlocalmemory/hooks/hook_handlers.py +13 -0
- package/src/superlocalmemory/learning/skill_performance_miner.py +44 -11
- package/src/superlocalmemory/mcp/server.py +4 -0
- package/src/superlocalmemory/mcp/tools_evolution.py +338 -0
- package/src/superlocalmemory/retrieval/engine.py +64 -4
- package/src/superlocalmemory/retrieval/forgetting_filter.py +22 -7
- package/src/superlocalmemory/retrieval/strategy.py +2 -2
- package/src/superlocalmemory/server/routes/behavioral.py +19 -15
- package/src/superlocalmemory/server/routes/evolution.py +213 -0
- package/src/superlocalmemory/server/routes/tiers.py +195 -0
- package/src/superlocalmemory/server/unified_daemon.py +36 -5
- package/src/superlocalmemory/storage/schema_v3411.py +149 -0
- package/src/superlocalmemory/ui/index.html +5 -2
- package/src/superlocalmemory/ui/js/lifecycle.js +83 -0
- package/src/superlocalmemory/ui/js/ng-skills.js +394 -10
- package/src/superlocalmemory.egg-info/PKG-INFO +609 -0
- package/src/superlocalmemory.egg-info/SOURCES.txt +335 -0
- package/src/superlocalmemory.egg-info/dependency_links.txt +1 -0
- package/src/superlocalmemory.egg-info/entry_points.txt +2 -0
- package/src/superlocalmemory.egg-info/requires.txt +55 -0
- package/src/superlocalmemory.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Evolution API routes — dashboard endpoints for skill evolution engine.
|
|
6
|
+
|
|
7
|
+
Routes: /api/evolution/status, /api/evolution/enable, /api/evolution/run
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from fastapi import APIRouter
|
|
14
|
+
|
|
15
|
+
from .helpers import get_active_profile, MEMORY_DIR
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger("superlocalmemory.routes.evolution")
|
|
18
|
+
router = APIRouter()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@router.get("/api/evolution/status")
|
|
22
|
+
async def evolution_status():
|
|
23
|
+
"""Get evolution engine status, backend, and recent history."""
|
|
24
|
+
try:
|
|
25
|
+
import json as _json
|
|
26
|
+
from superlocalmemory.evolution.skill_evolver import detect_backend
|
|
27
|
+
from superlocalmemory.evolution.evolution_store import EvolutionStore
|
|
28
|
+
|
|
29
|
+
# Read config directly from config.json (SLMConfig.load doesn't serialize evolution)
|
|
30
|
+
config_path = MEMORY_DIR / "config.json"
|
|
31
|
+
evo_cfg = {}
|
|
32
|
+
if config_path.exists():
|
|
33
|
+
with open(config_path) as f:
|
|
34
|
+
cfg = _json.load(f)
|
|
35
|
+
evo_cfg = cfg.get("evolution", {})
|
|
36
|
+
|
|
37
|
+
enabled = evo_cfg.get("enabled", False)
|
|
38
|
+
backend = detect_backend() if enabled else "none"
|
|
39
|
+
db_path = str(MEMORY_DIR / "memory.db")
|
|
40
|
+
|
|
41
|
+
store = EvolutionStore(db_path)
|
|
42
|
+
stats = store.get_stats()
|
|
43
|
+
recent = store.get_recent(limit=10)
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
"enabled": enabled,
|
|
47
|
+
"backend": backend,
|
|
48
|
+
"config": {
|
|
49
|
+
"backend_setting": evo_cfg.get("backend", "auto"),
|
|
50
|
+
"max_per_cycle": evo_cfg.get("max_evolutions_per_cycle", 3),
|
|
51
|
+
},
|
|
52
|
+
"stats": {
|
|
53
|
+
"total": stats.get("total", 0),
|
|
54
|
+
"promoted": stats.get("by_status", {}).get("promoted", 0),
|
|
55
|
+
"rejected": stats.get("by_status", {}).get("rejected", 0),
|
|
56
|
+
"failed": stats.get("by_status", {}).get("failed", 0),
|
|
57
|
+
"cycle_budget_remaining": stats.get("cycle_budget_remaining", 3),
|
|
58
|
+
},
|
|
59
|
+
"recent": [
|
|
60
|
+
{
|
|
61
|
+
"id": r.id,
|
|
62
|
+
"skill_name": r.skill_name,
|
|
63
|
+
"evolution_type": r.evolution_type.value,
|
|
64
|
+
"trigger": r.trigger.value,
|
|
65
|
+
"status": r.status.value,
|
|
66
|
+
"mutation_summary": r.mutation_summary,
|
|
67
|
+
"blind_verified": r.blind_verified,
|
|
68
|
+
"created_at": r.created_at,
|
|
69
|
+
}
|
|
70
|
+
for r in recent
|
|
71
|
+
],
|
|
72
|
+
}
|
|
73
|
+
except Exception as e:
|
|
74
|
+
logger.debug("evolution_status error: %s", e)
|
|
75
|
+
return {"enabled": False, "backend": "none", "error": str(e)}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@router.post("/api/evolution/enable")
|
|
79
|
+
async def evolution_enable():
|
|
80
|
+
"""Enable skill evolution engine. Writes directly to config.json."""
|
|
81
|
+
try:
|
|
82
|
+
import json as _json
|
|
83
|
+
|
|
84
|
+
config_path = MEMORY_DIR / "config.json"
|
|
85
|
+
cfg = {}
|
|
86
|
+
if config_path.exists():
|
|
87
|
+
with open(config_path) as f:
|
|
88
|
+
cfg = _json.load(f)
|
|
89
|
+
|
|
90
|
+
if "evolution" not in cfg:
|
|
91
|
+
cfg["evolution"] = {}
|
|
92
|
+
cfg["evolution"]["enabled"] = True
|
|
93
|
+
cfg["evolution"]["backend"] = "auto"
|
|
94
|
+
|
|
95
|
+
with open(config_path, "w") as f:
|
|
96
|
+
_json.dump(cfg, f, indent=2)
|
|
97
|
+
|
|
98
|
+
return {"ok": True, "message": "Evolution enabled. Will use auto-detected backend."}
|
|
99
|
+
except Exception as e:
|
|
100
|
+
logger.error("evolution_enable error: %s", e)
|
|
101
|
+
return {"ok": False, "error": str(e)}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@router.post("/api/evolution/run")
|
|
105
|
+
async def evolution_run():
|
|
106
|
+
"""Manually trigger an evolution cycle."""
|
|
107
|
+
try:
|
|
108
|
+
import json as _json
|
|
109
|
+
from superlocalmemory.evolution.skill_evolver import SkillEvolver
|
|
110
|
+
|
|
111
|
+
config_path = MEMORY_DIR / "config.json"
|
|
112
|
+
evo_cfg = {}
|
|
113
|
+
if config_path.exists():
|
|
114
|
+
with open(config_path) as f:
|
|
115
|
+
evo_cfg = _json.load(f).get("evolution", {})
|
|
116
|
+
|
|
117
|
+
if not evo_cfg.get("enabled", False):
|
|
118
|
+
return {"ok": False, "error": "Evolution is disabled. Enable first."}
|
|
119
|
+
|
|
120
|
+
profile = get_active_profile()
|
|
121
|
+
db_path = str(MEMORY_DIR / "memory.db")
|
|
122
|
+
|
|
123
|
+
# Build a minimal config object for the evolver
|
|
124
|
+
class _EvoCfg:
|
|
125
|
+
enabled = True
|
|
126
|
+
backend = evo_cfg.get("backend", "auto")
|
|
127
|
+
max_evolutions_per_cycle = evo_cfg.get("max_evolutions_per_cycle", 3)
|
|
128
|
+
class _Cfg:
|
|
129
|
+
evolution = _EvoCfg()
|
|
130
|
+
|
|
131
|
+
evolver = SkillEvolver(db_path, _Cfg())
|
|
132
|
+
result = evolver.run_consolidation_cycle(profile)
|
|
133
|
+
|
|
134
|
+
return {"ok": True, **result}
|
|
135
|
+
except Exception as e:
|
|
136
|
+
logger.error("evolution_run error: %s", e)
|
|
137
|
+
return {"ok": False, "error": str(e)}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@router.get("/api/evolution/lineage")
|
|
141
|
+
async def evolution_lineage(skill_name: str = ""):
|
|
142
|
+
"""Get evolution lineage for a skill or all skills.
|
|
143
|
+
|
|
144
|
+
Returns lineage records and a tree structure grouped by root skill.
|
|
145
|
+
"""
|
|
146
|
+
try:
|
|
147
|
+
import sqlite3 as _sqlite3
|
|
148
|
+
|
|
149
|
+
db_path = str(MEMORY_DIR / "memory.db")
|
|
150
|
+
conn = _sqlite3.connect(db_path, timeout=10)
|
|
151
|
+
conn.row_factory = _sqlite3.Row
|
|
152
|
+
|
|
153
|
+
if skill_name:
|
|
154
|
+
rows = conn.execute(
|
|
155
|
+
"SELECT id, skill_name, parent_skill_id, evolution_type, "
|
|
156
|
+
"trigger_type, generation, status, mutation_summary, "
|
|
157
|
+
"blind_verified, created_at, completed_at "
|
|
158
|
+
"FROM skill_evolution_log "
|
|
159
|
+
"WHERE skill_name = ? OR parent_skill_id = ? "
|
|
160
|
+
"ORDER BY created_at ASC",
|
|
161
|
+
(skill_name, skill_name),
|
|
162
|
+
).fetchall()
|
|
163
|
+
else:
|
|
164
|
+
rows = conn.execute(
|
|
165
|
+
"SELECT id, skill_name, parent_skill_id, evolution_type, "
|
|
166
|
+
"trigger_type, generation, status, mutation_summary, "
|
|
167
|
+
"blind_verified, created_at, completed_at "
|
|
168
|
+
"FROM skill_evolution_log "
|
|
169
|
+
"ORDER BY created_at DESC LIMIT 100",
|
|
170
|
+
).fetchall()
|
|
171
|
+
|
|
172
|
+
conn.close()
|
|
173
|
+
|
|
174
|
+
lineage = [
|
|
175
|
+
{
|
|
176
|
+
"id": dict(r)["id"],
|
|
177
|
+
"skill_name": dict(r)["skill_name"],
|
|
178
|
+
"parent_skill_id": dict(r).get("parent_skill_id", ""),
|
|
179
|
+
"evolution_type": dict(r)["evolution_type"],
|
|
180
|
+
"trigger": dict(r)["trigger_type"],
|
|
181
|
+
"generation": dict(r).get("generation", 0),
|
|
182
|
+
"status": dict(r)["status"],
|
|
183
|
+
"mutation_summary": dict(r).get("mutation_summary", ""),
|
|
184
|
+
"blind_verified": bool(dict(r).get("blind_verified", 0)),
|
|
185
|
+
"created_at": dict(r).get("created_at", ""),
|
|
186
|
+
"completed_at": dict(r).get("completed_at", ""),
|
|
187
|
+
}
|
|
188
|
+
for r in rows
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
# Build tree structure: group by root skill
|
|
192
|
+
tree: dict = {}
|
|
193
|
+
for entry in lineage:
|
|
194
|
+
root = entry.get("parent_skill_id") or entry["skill_name"]
|
|
195
|
+
if root not in tree:
|
|
196
|
+
tree[root] = {"root": root, "evolutions": []}
|
|
197
|
+
tree[root]["evolutions"].append({
|
|
198
|
+
"id": entry["id"],
|
|
199
|
+
"skill_name": entry["skill_name"],
|
|
200
|
+
"evolution_type": entry["evolution_type"],
|
|
201
|
+
"status": entry["status"],
|
|
202
|
+
"generation": entry["generation"],
|
|
203
|
+
"created_at": entry["created_at"],
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
"lineage": lineage,
|
|
208
|
+
"lineage_count": len(lineage),
|
|
209
|
+
"tree": tree,
|
|
210
|
+
}
|
|
211
|
+
except Exception as e:
|
|
212
|
+
logger.debug("evolution_lineage error: %s", e)
|
|
213
|
+
return {"lineage": [], "lineage_count": 0, "tree": {}, "error": str(e)}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
"""SuperLocalMemory V3.4.11 "Scale-Ready" - Tier Management Routes
|
|
5
|
+
|
|
6
|
+
Routes: /api/tiers/stats, /api/tiers/evaluate, /api/tiers/pin, /api/tiers/unpin
|
|
7
|
+
|
|
8
|
+
Uses lightweight sqlite3 directly (not MemoryEngine) for fast dashboard queries.
|
|
9
|
+
All connections use WAL mode + busy_timeout for concurrency safety.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
import re
|
|
14
|
+
import sqlite3
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from datetime import datetime, UTC
|
|
17
|
+
|
|
18
|
+
from fastapi import APIRouter, HTTPException, Request
|
|
19
|
+
from pydantic import BaseModel, Field
|
|
20
|
+
|
|
21
|
+
from .helpers import DB_PATH
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger("superlocalmemory.routes.tiers")
|
|
24
|
+
router = APIRouter()
|
|
25
|
+
|
|
26
|
+
_PROFILE_PATTERN = re.compile(r'^[a-zA-Z0-9_-]+$')
|
|
27
|
+
_MAX_REASON_LENGTH = 500
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class PinRequest(BaseModel):
|
|
31
|
+
fact_id: str = Field(..., min_length=1)
|
|
32
|
+
reason: str = Field(default="", max_length=_MAX_REASON_LENGTH)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@contextmanager
|
|
36
|
+
def _db():
|
|
37
|
+
"""Context-managed DB connection with WAL + busy_timeout."""
|
|
38
|
+
conn = sqlite3.connect(str(DB_PATH))
|
|
39
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
40
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
41
|
+
conn.row_factory = sqlite3.Row
|
|
42
|
+
try:
|
|
43
|
+
yield conn
|
|
44
|
+
finally:
|
|
45
|
+
conn.close()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _validate_profile(profile_id: str) -> str:
|
|
49
|
+
"""Validate profile_id against allowed pattern."""
|
|
50
|
+
if not profile_id or not _PROFILE_PATTERN.match(profile_id):
|
|
51
|
+
raise HTTPException(status_code=400, detail="Invalid profile_id")
|
|
52
|
+
return profile_id
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@router.get("/api/tiers/stats")
|
|
56
|
+
async def tier_stats(profile_id: str = "default"):
|
|
57
|
+
"""Get tier distribution stats."""
|
|
58
|
+
profile_id = _validate_profile(profile_id)
|
|
59
|
+
|
|
60
|
+
with _db() as conn:
|
|
61
|
+
try:
|
|
62
|
+
c = conn.cursor()
|
|
63
|
+
c.execute(
|
|
64
|
+
"SELECT lifecycle, COUNT(*) as cnt FROM atomic_facts "
|
|
65
|
+
"WHERE profile_id = ? GROUP BY lifecycle", (profile_id,),
|
|
66
|
+
)
|
|
67
|
+
dist = {row["lifecycle"]: row["cnt"] for row in c.fetchall()}
|
|
68
|
+
|
|
69
|
+
pinned = 0
|
|
70
|
+
try:
|
|
71
|
+
c.execute(
|
|
72
|
+
"SELECT COUNT(*) as c FROM pinned_facts "
|
|
73
|
+
"WHERE profile_id = ?", (profile_id,),
|
|
74
|
+
)
|
|
75
|
+
pinned = c.fetchone()["c"]
|
|
76
|
+
except sqlite3.OperationalError:
|
|
77
|
+
pass # pinned_facts table may not exist yet
|
|
78
|
+
|
|
79
|
+
total = sum(dist.values())
|
|
80
|
+
return {
|
|
81
|
+
"active": dist.get("active", 0),
|
|
82
|
+
"warm": dist.get("warm", 0),
|
|
83
|
+
"cold": dist.get("cold", 0),
|
|
84
|
+
"archived": dist.get("archived", 0),
|
|
85
|
+
"total": total,
|
|
86
|
+
"pinned": pinned,
|
|
87
|
+
"active_pct": round(
|
|
88
|
+
dist.get("active", 0) / max(total, 1) * 100, 1,
|
|
89
|
+
),
|
|
90
|
+
}
|
|
91
|
+
except Exception as exc:
|
|
92
|
+
logger.error("tier_stats failed: %s", exc, exc_info=True)
|
|
93
|
+
raise HTTPException(
|
|
94
|
+
status_code=500, detail="Internal storage error",
|
|
95
|
+
) from None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@router.post("/api/tiers/evaluate")
|
|
99
|
+
async def evaluate_tiers_route(request: Request, profile_id: str = "default"):
|
|
100
|
+
"""Manually trigger tier evaluation.
|
|
101
|
+
|
|
102
|
+
Uses the shared engine (via lazy import) instead of re-initializing
|
|
103
|
+
DatabaseManager on every request.
|
|
104
|
+
"""
|
|
105
|
+
profile_id = _validate_profile(profile_id)
|
|
106
|
+
|
|
107
|
+
try:
|
|
108
|
+
from superlocalmemory.core.tier_manager import evaluate_tiers
|
|
109
|
+
from .helpers import get_engine_lazy
|
|
110
|
+
|
|
111
|
+
engine = get_engine_lazy(request.app.state)
|
|
112
|
+
if engine is None or not hasattr(engine, '_db') or engine._db is None:
|
|
113
|
+
raise HTTPException(
|
|
114
|
+
status_code=503, detail="Engine not initialized",
|
|
115
|
+
)
|
|
116
|
+
stats = evaluate_tiers(engine._db, profile_id)
|
|
117
|
+
return {"success": True, "stats": stats}
|
|
118
|
+
except HTTPException:
|
|
119
|
+
raise
|
|
120
|
+
except Exception as exc:
|
|
121
|
+
logger.error("evaluate_tiers failed: %s", exc, exc_info=True)
|
|
122
|
+
raise HTTPException(
|
|
123
|
+
status_code=500, detail="Tier evaluation failed",
|
|
124
|
+
) from None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@router.post("/api/tiers/pin")
|
|
128
|
+
async def pin_fact_route(request: PinRequest, profile_id: str = "default"):
|
|
129
|
+
"""Pin a fact to stay in active tier forever.
|
|
130
|
+
|
|
131
|
+
Validates fact exists in the specified profile before pinning.
|
|
132
|
+
"""
|
|
133
|
+
profile_id = _validate_profile(profile_id)
|
|
134
|
+
|
|
135
|
+
with _db() as conn:
|
|
136
|
+
try:
|
|
137
|
+
# Verify fact exists in this profile
|
|
138
|
+
c = conn.cursor()
|
|
139
|
+
c.execute(
|
|
140
|
+
"SELECT fact_id FROM atomic_facts "
|
|
141
|
+
"WHERE fact_id = ? AND profile_id = ?",
|
|
142
|
+
(request.fact_id, profile_id),
|
|
143
|
+
)
|
|
144
|
+
if c.fetchone() is None:
|
|
145
|
+
raise HTTPException(
|
|
146
|
+
status_code=404,
|
|
147
|
+
detail=f"Fact {request.fact_id[:8]}... not found",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
now = datetime.now(UTC).isoformat()
|
|
151
|
+
conn.execute(
|
|
152
|
+
"INSERT OR REPLACE INTO pinned_facts "
|
|
153
|
+
"(fact_id, profile_id, pinned_at, reason) "
|
|
154
|
+
"VALUES (?, ?, ?, ?)",
|
|
155
|
+
(request.fact_id, profile_id, now, request.reason),
|
|
156
|
+
)
|
|
157
|
+
conn.execute(
|
|
158
|
+
"UPDATE atomic_facts SET lifecycle = 'active' "
|
|
159
|
+
"WHERE fact_id = ? AND profile_id = ?",
|
|
160
|
+
(request.fact_id, profile_id),
|
|
161
|
+
)
|
|
162
|
+
conn.commit()
|
|
163
|
+
return {"success": True, "message": f"Fact {request.fact_id[:8]}... pinned"}
|
|
164
|
+
except HTTPException:
|
|
165
|
+
raise
|
|
166
|
+
except Exception as exc:
|
|
167
|
+
logger.error("pin_fact failed: %s", exc, exc_info=True)
|
|
168
|
+
raise HTTPException(
|
|
169
|
+
status_code=500, detail="Failed to pin fact",
|
|
170
|
+
) from None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@router.post("/api/tiers/unpin")
|
|
174
|
+
async def unpin_fact_route(request: PinRequest, profile_id: str = "default"):
|
|
175
|
+
"""Unpin a fact, allowing normal tier demotion.
|
|
176
|
+
|
|
177
|
+
Lifecycle stays 'active' until the next tier evaluation cycle demotes it
|
|
178
|
+
based on access patterns. This is intentional — immediate demotion would
|
|
179
|
+
surprise the user.
|
|
180
|
+
"""
|
|
181
|
+
profile_id = _validate_profile(profile_id)
|
|
182
|
+
|
|
183
|
+
with _db() as conn:
|
|
184
|
+
try:
|
|
185
|
+
conn.execute(
|
|
186
|
+
"DELETE FROM pinned_facts WHERE fact_id = ? AND profile_id = ?",
|
|
187
|
+
(request.fact_id, profile_id),
|
|
188
|
+
)
|
|
189
|
+
conn.commit()
|
|
190
|
+
return {"success": True, "unpinned": True}
|
|
191
|
+
except Exception as exc:
|
|
192
|
+
logger.error("unpin_fact failed: %s", exc, exc_info=True)
|
|
193
|
+
raise HTTPException(
|
|
194
|
+
status_code=500, detail="Failed to unpin fact",
|
|
195
|
+
) from None
|
|
@@ -268,6 +268,20 @@ async def lifespan(application: FastAPI):
|
|
|
268
268
|
if reranker and hasattr(reranker, 'warmup_sync'):
|
|
269
269
|
reranker.warmup_sync(timeout=120)
|
|
270
270
|
|
|
271
|
+
# V3.4.11: Pre-warm embedding worker (load ONNX model on startup)
|
|
272
|
+
# Without this, first recall takes 60-90s for model load.
|
|
273
|
+
# Same pattern as reranker warmup above.
|
|
274
|
+
import threading
|
|
275
|
+
def _warmup_embedder():
|
|
276
|
+
try:
|
|
277
|
+
embedder = getattr(retrieval_eng, '_embedder', None) if retrieval_eng else None
|
|
278
|
+
if embedder and hasattr(embedder, 'embed'):
|
|
279
|
+
embedder.embed("warmup")
|
|
280
|
+
logger.info("Embedding worker pre-warmed (ONNX model loaded)")
|
|
281
|
+
except Exception as exc:
|
|
282
|
+
logger.warning("Embedding warmup failed: %s", exc)
|
|
283
|
+
threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
|
|
284
|
+
|
|
271
285
|
except Exception as exc:
|
|
272
286
|
logger.warning("Engine init failed: %s", exc)
|
|
273
287
|
application.state.engine = None
|
|
@@ -318,6 +332,8 @@ async def lifespan(application: FastAPI):
|
|
|
318
332
|
if enable_legacy:
|
|
319
333
|
asyncio.create_task(_start_legacy_redirect(_DEFAULT_PORT, _LEGACY_PORT))
|
|
320
334
|
|
|
335
|
+
global _start_time
|
|
336
|
+
_start_time = time.monotonic()
|
|
321
337
|
_last_activity = time.monotonic()
|
|
322
338
|
logger.info("Unified daemon ready on port %d (24/7 mode)" if idle_timeout <= 0
|
|
323
339
|
else "Unified daemon ready on port %d (idle timeout: %ds)",
|
|
@@ -422,7 +438,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
422
438
|
return JSONResponse(
|
|
423
439
|
status_code=429,
|
|
424
440
|
content={"error": "Too many requests."},
|
|
425
|
-
headers={"Retry-After": str(limiter
|
|
441
|
+
headers={"Retry-After": str(getattr(limiter, 'window', 60))},
|
|
426
442
|
)
|
|
427
443
|
response = await call_next(request)
|
|
428
444
|
response.headers["X-RateLimit-Remaining"] = str(remaining)
|
|
@@ -472,6 +488,19 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
472
488
|
application.include_router(profiles_router)
|
|
473
489
|
application.include_router(backup_router)
|
|
474
490
|
application.include_router(data_io_router)
|
|
491
|
+
|
|
492
|
+
# Optional routers — ImportError-safe so missing modules don't crash startup
|
|
493
|
+
try:
|
|
494
|
+
from superlocalmemory.server.routes.tiers import router as tiers_router
|
|
495
|
+
application.include_router(tiers_router)
|
|
496
|
+
except ImportError:
|
|
497
|
+
logger.debug("tiers_router not available")
|
|
498
|
+
|
|
499
|
+
try:
|
|
500
|
+
from superlocalmemory.server.routes.evolution import router as evolution_router
|
|
501
|
+
application.include_router(evolution_router)
|
|
502
|
+
except ImportError:
|
|
503
|
+
logger.debug("evolution_router not available")
|
|
475
504
|
application.include_router(events_router)
|
|
476
505
|
application.include_router(agents_router)
|
|
477
506
|
application.include_router(ws_router)
|
|
@@ -542,13 +571,16 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
542
571
|
}
|
|
543
572
|
|
|
544
573
|
@application.get("/recall")
|
|
545
|
-
async def recall(q: str = "", limit: int = 20):
|
|
574
|
+
async def recall(q: str = "", query: str = "", limit: int = 20):
|
|
546
575
|
_update_activity()
|
|
576
|
+
search_query = q or query # Accept both ?q= and ?query= for compatibility
|
|
547
577
|
engine = application.state.engine
|
|
548
578
|
if engine is None:
|
|
549
579
|
raise HTTPException(503, detail="Engine not initialized")
|
|
580
|
+
if not search_query:
|
|
581
|
+
return {"results": [], "count": 0, "query_type": "none", "retrieval_time_ms": 0}
|
|
550
582
|
try:
|
|
551
|
-
response = engine.recall(
|
|
583
|
+
response = engine.recall(search_query, limit=limit)
|
|
552
584
|
results = [
|
|
553
585
|
{
|
|
554
586
|
"content": r.fact.content,
|
|
@@ -590,7 +622,6 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
590
622
|
async def status():
|
|
591
623
|
_update_activity()
|
|
592
624
|
engine = application.state.engine
|
|
593
|
-
uptime = time.monotonic() - _last_activity
|
|
594
625
|
fact_count = engine.fact_count if engine else 0
|
|
595
626
|
mode = engine._config.mode.value if engine and hasattr(engine, '_config') else "unknown"
|
|
596
627
|
return {
|
|
@@ -656,7 +687,7 @@ def _start_memory_watchdog() -> None:
|
|
|
656
687
|
"""
|
|
657
688
|
import threading
|
|
658
689
|
|
|
659
|
-
MAX_WORKER_MB =
|
|
690
|
+
MAX_WORKER_MB = 4096 # 4GB per worker — ONNX full model is 1.6GB + overhead
|
|
660
691
|
|
|
661
692
|
def watchdog_loop():
|
|
662
693
|
while True:
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""SuperLocalMemory V3.4.11 "Scale-Ready" — Schema Extensions.
|
|
6
|
+
|
|
7
|
+
New tables for tiered storage + future backend integrations:
|
|
8
|
+
- pinned_facts: User-pinned facts that stay in hot tier forever
|
|
9
|
+
- backend_status: Tracks initialization state of LanceDB/KùzuDB backends
|
|
10
|
+
- fact_consolidations: History of merged/consolidated facts
|
|
11
|
+
|
|
12
|
+
Design rules (inherited):
|
|
13
|
+
- CREATE IF NOT EXISTS for idempotency
|
|
14
|
+
- profile_id where applicable
|
|
15
|
+
- Never ALTER existing column types
|
|
16
|
+
|
|
17
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
import sqlite3
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# DDL — Pinned Facts (user override on lifecycle demotion)
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
_PINNED_FACTS_DDL = """
|
|
32
|
+
CREATE TABLE IF NOT EXISTS pinned_facts (
|
|
33
|
+
fact_id TEXT PRIMARY KEY,
|
|
34
|
+
profile_id TEXT DEFAULT 'default',
|
|
35
|
+
pinned_at TEXT NOT NULL,
|
|
36
|
+
reason TEXT DEFAULT ''
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
CREATE INDEX IF NOT EXISTS idx_pinned_facts_profile
|
|
40
|
+
ON pinned_facts(profile_id);
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
# DDL — Backend Status (LanceDB, KùzuDB, sqlite-vec tracking)
|
|
45
|
+
# ---------------------------------------------------------------------------
|
|
46
|
+
|
|
47
|
+
_BACKEND_STATUS_DDL = """
|
|
48
|
+
CREATE TABLE IF NOT EXISTS backend_status (
|
|
49
|
+
backend_name TEXT PRIMARY KEY,
|
|
50
|
+
status TEXT DEFAULT 'not_initialized',
|
|
51
|
+
record_count INTEGER DEFAULT 0,
|
|
52
|
+
last_sync_at TEXT,
|
|
53
|
+
error_message TEXT DEFAULT '',
|
|
54
|
+
config TEXT DEFAULT '{}'
|
|
55
|
+
);
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
# ---------------------------------------------------------------------------
|
|
59
|
+
# DDL — Fact Consolidations (merge history)
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
_FACT_CONSOLIDATIONS_DDL = """
|
|
63
|
+
CREATE TABLE IF NOT EXISTS fact_consolidations (
|
|
64
|
+
consolidation_id TEXT PRIMARY KEY,
|
|
65
|
+
profile_id TEXT DEFAULT 'default',
|
|
66
|
+
consolidated_fact_id TEXT NOT NULL,
|
|
67
|
+
source_fact_ids TEXT NOT NULL,
|
|
68
|
+
strategy TEXT DEFAULT 'entity_cluster',
|
|
69
|
+
created_at TEXT NOT NULL
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
CREATE INDEX IF NOT EXISTS idx_fact_consolidations_profile
|
|
73
|
+
ON fact_consolidations(profile_id);
|
|
74
|
+
CREATE INDEX IF NOT EXISTS idx_fact_consolidations_target
|
|
75
|
+
ON fact_consolidations(consolidated_fact_id);
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
# ---------------------------------------------------------------------------
|
|
79
|
+
# DDL — Index on lifecycle for fast tier queries
|
|
80
|
+
# ---------------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
_LIFECYCLE_INDEX_DDL = """
|
|
83
|
+
CREATE INDEX IF NOT EXISTS idx_atomic_facts_lifecycle
|
|
84
|
+
ON atomic_facts(lifecycle, profile_id);
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# ---------------------------------------------------------------------------
|
|
89
|
+
# Migration runner
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
def apply_v3411_schema(db_path: str | sqlite3.Connection) -> dict:
|
|
93
|
+
"""Apply all v3.4.11 schema changes. Idempotent."""
|
|
94
|
+
result = {"applied": [], "errors": []}
|
|
95
|
+
|
|
96
|
+
if isinstance(db_path, sqlite3.Connection):
|
|
97
|
+
conn = db_path
|
|
98
|
+
own_connection = False
|
|
99
|
+
else:
|
|
100
|
+
conn = sqlite3.connect(str(db_path))
|
|
101
|
+
own_connection = True
|
|
102
|
+
|
|
103
|
+
try:
|
|
104
|
+
for name, ddl in [
|
|
105
|
+
("pinned_facts", _PINNED_FACTS_DDL),
|
|
106
|
+
("backend_status", _BACKEND_STATUS_DDL),
|
|
107
|
+
("fact_consolidations", _FACT_CONSOLIDATIONS_DDL),
|
|
108
|
+
("lifecycle_index", _LIFECYCLE_INDEX_DDL),
|
|
109
|
+
]:
|
|
110
|
+
try:
|
|
111
|
+
conn.executescript(ddl)
|
|
112
|
+
result["applied"].append(name)
|
|
113
|
+
except sqlite3.OperationalError as e:
|
|
114
|
+
result["errors"].append(f"{name}: {e}")
|
|
115
|
+
|
|
116
|
+
# Seed default backend entries
|
|
117
|
+
now = __import__("datetime").datetime.now().isoformat()
|
|
118
|
+
for backend in ("sqlite_vec", "sqlite_graph", "lancedb", "kuzu"):
|
|
119
|
+
try:
|
|
120
|
+
conn.execute(
|
|
121
|
+
"INSERT OR IGNORE INTO backend_status "
|
|
122
|
+
"(backend_name, status, config) VALUES (?, ?, ?)",
|
|
123
|
+
(backend, "not_initialized" if backend in ("lancedb", "kuzu") else "active", "{}"),
|
|
124
|
+
)
|
|
125
|
+
except sqlite3.OperationalError:
|
|
126
|
+
pass
|
|
127
|
+
|
|
128
|
+
# Mark version
|
|
129
|
+
try:
|
|
130
|
+
conn.execute(
|
|
131
|
+
"INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
|
|
132
|
+
("3.4.11", now),
|
|
133
|
+
)
|
|
134
|
+
except sqlite3.OperationalError:
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
conn.commit()
|
|
138
|
+
|
|
139
|
+
if result["applied"]:
|
|
140
|
+
logger.info("Schema v3.4.11 applied: %s", ", ".join(result["applied"]))
|
|
141
|
+
|
|
142
|
+
except Exception as e:
|
|
143
|
+
result["errors"].append(f"fatal: {e}")
|
|
144
|
+
logger.error("Schema v3.4.11 migration failed: %s", e)
|
|
145
|
+
finally:
|
|
146
|
+
if own_connection:
|
|
147
|
+
conn.close()
|
|
148
|
+
|
|
149
|
+
return result
|
|
@@ -1519,10 +1519,12 @@
|
|
|
1519
1519
|
<h5 class="mb-0"><i class="bi bi-hourglass-split text-warning"></i> Memory Lifecycle</h5>
|
|
1520
1520
|
<div>
|
|
1521
1521
|
<span class="badge bg-secondary me-2" id="lifecycle-profile-badge">default</span>
|
|
1522
|
-
<button class="btn btn-sm btn-outline-
|
|
1522
|
+
<button class="btn btn-sm btn-outline-success" onclick="evaluateTiersNow()"><i class="bi bi-arrow-repeat"></i> Evaluate Tiers</button>
|
|
1523
|
+
<button class="btn btn-sm btn-outline-info ms-1" onclick="compactDryRun()"><i class="bi bi-funnel"></i> Preview Compaction</button>
|
|
1523
1524
|
<button class="btn btn-sm btn-outline-warning ms-1" onclick="compactExecute()"><i class="bi bi-lightning"></i> Compact Now</button>
|
|
1524
1525
|
</div>
|
|
1525
1526
|
</div>
|
|
1527
|
+
|
|
1526
1528
|
<!-- State Distribution -->
|
|
1527
1529
|
<div class="row g-3 mb-4" id="lifecycle-states-row">
|
|
1528
1530
|
<div class="col-md-2"><div class="card p-3 text-center"><div class="text-success fw-bold fs-3" id="lc-active-count">-</div><small class="text-muted">Active</small></div></div>
|
|
@@ -1741,6 +1743,7 @@
|
|
|
1741
1743
|
<div style="margin-top:8px">Loading skill performance data...</div>
|
|
1742
1744
|
</div>
|
|
1743
1745
|
</div>
|
|
1746
|
+
<div id="skill-lineage-container" class="mt-3"></div>
|
|
1744
1747
|
<div id="skill-detail-panel" style="display:none;margin-top:24px"></div>
|
|
1745
1748
|
</div>
|
|
1746
1749
|
|
|
@@ -2150,7 +2153,7 @@
|
|
|
2150
2153
|
<script src="static/js/ng-health.js?v=345"></script>
|
|
2151
2154
|
<script src="static/js/ng-ingestion.js?v=345"></script>
|
|
2152
2155
|
<script src="static/js/ng-entities.js?v=3410"></script>
|
|
2153
|
-
<script src="static/js/ng-skills.js?v=
|
|
2156
|
+
<script src="static/js/ng-skills.js?v=3411"></script>
|
|
2154
2157
|
<script src="static/js/ng-mesh.js?v=345"></script>
|
|
2155
2158
|
<script src="static/js/ng-shell.js?v=3410"></script>
|
|
2156
2159
|
|