superlocalmemory 3.4.10 → 3.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +17 -11
  2. package/docs/skill-evolution.md +77 -10
  3. package/ide/hooks/tool-event-hook.sh +4 -4
  4. package/package.json +1 -1
  5. package/pyproject.toml +3 -2
  6. package/src/superlocalmemory/cli/commands.py +170 -0
  7. package/src/superlocalmemory/cli/main.py +21 -0
  8. package/src/superlocalmemory/cli/setup_wizard.py +54 -11
  9. package/src/superlocalmemory/core/config.py +35 -0
  10. package/src/superlocalmemory/core/consolidation_engine.py +128 -0
  11. package/src/superlocalmemory/core/embedding_worker.py +1 -1
  12. package/src/superlocalmemory/core/engine.py +12 -0
  13. package/src/superlocalmemory/core/fact_consolidator.py +425 -0
  14. package/src/superlocalmemory/core/graph_pruner.py +290 -0
  15. package/src/superlocalmemory/core/maintenance_scheduler.py +20 -0
  16. package/src/superlocalmemory/core/recall_pipeline.py +9 -0
  17. package/src/superlocalmemory/core/tier_manager.py +325 -0
  18. package/src/superlocalmemory/encoding/entity_resolver.py +6 -5
  19. package/src/superlocalmemory/evolution/__init__.py +29 -0
  20. package/src/superlocalmemory/evolution/blind_verifier.py +115 -0
  21. package/src/superlocalmemory/evolution/evolution_store.py +302 -0
  22. package/src/superlocalmemory/evolution/mutation_generator.py +181 -0
  23. package/src/superlocalmemory/evolution/skill_evolver.py +555 -0
  24. package/src/superlocalmemory/evolution/triggers.py +367 -0
  25. package/src/superlocalmemory/evolution/types.py +92 -0
  26. package/src/superlocalmemory/hooks/hook_handlers.py +13 -0
  27. package/src/superlocalmemory/learning/skill_performance_miner.py +44 -11
  28. package/src/superlocalmemory/mcp/server.py +4 -0
  29. package/src/superlocalmemory/mcp/tools_evolution.py +338 -0
  30. package/src/superlocalmemory/retrieval/engine.py +64 -4
  31. package/src/superlocalmemory/retrieval/forgetting_filter.py +22 -7
  32. package/src/superlocalmemory/retrieval/strategy.py +2 -2
  33. package/src/superlocalmemory/server/routes/behavioral.py +19 -15
  34. package/src/superlocalmemory/server/routes/evolution.py +213 -0
  35. package/src/superlocalmemory/server/routes/tiers.py +195 -0
  36. package/src/superlocalmemory/server/unified_daemon.py +36 -5
  37. package/src/superlocalmemory/storage/schema_v3411.py +149 -0
  38. package/src/superlocalmemory/ui/index.html +5 -2
  39. package/src/superlocalmemory/ui/js/lifecycle.js +83 -0
  40. package/src/superlocalmemory/ui/js/ng-skills.js +394 -10
  41. package/src/superlocalmemory.egg-info/PKG-INFO +609 -0
  42. package/src/superlocalmemory.egg-info/SOURCES.txt +335 -0
  43. package/src/superlocalmemory.egg-info/dependency_links.txt +1 -0
  44. package/src/superlocalmemory.egg-info/entry_points.txt +2 -0
  45. package/src/superlocalmemory.egg-info/requires.txt +55 -0
  46. package/src/superlocalmemory.egg-info/top_level.txt +1 -0
@@ -0,0 +1,213 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """Evolution API routes — dashboard endpoints for skill evolution engine.
6
+
7
+ Routes: /api/evolution/status, /api/evolution/enable, /api/evolution/run
8
+ """
9
+
10
+ import logging
11
+ from pathlib import Path
12
+
13
+ from fastapi import APIRouter
14
+
15
+ from .helpers import get_active_profile, MEMORY_DIR
16
+
17
+ logger = logging.getLogger("superlocalmemory.routes.evolution")
18
+ router = APIRouter()
19
+
20
+
21
+ @router.get("/api/evolution/status")
22
+ async def evolution_status():
23
+ """Get evolution engine status, backend, and recent history."""
24
+ try:
25
+ import json as _json
26
+ from superlocalmemory.evolution.skill_evolver import detect_backend
27
+ from superlocalmemory.evolution.evolution_store import EvolutionStore
28
+
29
+ # Read config directly from config.json (SLMConfig.load doesn't serialize evolution)
30
+ config_path = MEMORY_DIR / "config.json"
31
+ evo_cfg = {}
32
+ if config_path.exists():
33
+ with open(config_path) as f:
34
+ cfg = _json.load(f)
35
+ evo_cfg = cfg.get("evolution", {})
36
+
37
+ enabled = evo_cfg.get("enabled", False)
38
+ backend = detect_backend() if enabled else "none"
39
+ db_path = str(MEMORY_DIR / "memory.db")
40
+
41
+ store = EvolutionStore(db_path)
42
+ stats = store.get_stats()
43
+ recent = store.get_recent(limit=10)
44
+
45
+ return {
46
+ "enabled": enabled,
47
+ "backend": backend,
48
+ "config": {
49
+ "backend_setting": evo_cfg.get("backend", "auto"),
50
+ "max_per_cycle": evo_cfg.get("max_evolutions_per_cycle", 3),
51
+ },
52
+ "stats": {
53
+ "total": stats.get("total", 0),
54
+ "promoted": stats.get("by_status", {}).get("promoted", 0),
55
+ "rejected": stats.get("by_status", {}).get("rejected", 0),
56
+ "failed": stats.get("by_status", {}).get("failed", 0),
57
+ "cycle_budget_remaining": stats.get("cycle_budget_remaining", 3),
58
+ },
59
+ "recent": [
60
+ {
61
+ "id": r.id,
62
+ "skill_name": r.skill_name,
63
+ "evolution_type": r.evolution_type.value,
64
+ "trigger": r.trigger.value,
65
+ "status": r.status.value,
66
+ "mutation_summary": r.mutation_summary,
67
+ "blind_verified": r.blind_verified,
68
+ "created_at": r.created_at,
69
+ }
70
+ for r in recent
71
+ ],
72
+ }
73
+ except Exception as e:
74
+ logger.debug("evolution_status error: %s", e)
75
+ return {"enabled": False, "backend": "none", "error": str(e)}
76
+
77
+
78
+ @router.post("/api/evolution/enable")
79
+ async def evolution_enable():
80
+ """Enable skill evolution engine. Writes directly to config.json."""
81
+ try:
82
+ import json as _json
83
+
84
+ config_path = MEMORY_DIR / "config.json"
85
+ cfg = {}
86
+ if config_path.exists():
87
+ with open(config_path) as f:
88
+ cfg = _json.load(f)
89
+
90
+ if "evolution" not in cfg:
91
+ cfg["evolution"] = {}
92
+ cfg["evolution"]["enabled"] = True
93
+ cfg["evolution"]["backend"] = "auto"
94
+
95
+ with open(config_path, "w") as f:
96
+ _json.dump(cfg, f, indent=2)
97
+
98
+ return {"ok": True, "message": "Evolution enabled. Will use auto-detected backend."}
99
+ except Exception as e:
100
+ logger.error("evolution_enable error: %s", e)
101
+ return {"ok": False, "error": str(e)}
102
+
103
+
104
+ @router.post("/api/evolution/run")
105
+ async def evolution_run():
106
+ """Manually trigger an evolution cycle."""
107
+ try:
108
+ import json as _json
109
+ from superlocalmemory.evolution.skill_evolver import SkillEvolver
110
+
111
+ config_path = MEMORY_DIR / "config.json"
112
+ evo_cfg = {}
113
+ if config_path.exists():
114
+ with open(config_path) as f:
115
+ evo_cfg = _json.load(f).get("evolution", {})
116
+
117
+ if not evo_cfg.get("enabled", False):
118
+ return {"ok": False, "error": "Evolution is disabled. Enable first."}
119
+
120
+ profile = get_active_profile()
121
+ db_path = str(MEMORY_DIR / "memory.db")
122
+
123
+ # Build a minimal config object for the evolver
124
+ class _EvoCfg:
125
+ enabled = True
126
+ backend = evo_cfg.get("backend", "auto")
127
+ max_evolutions_per_cycle = evo_cfg.get("max_evolutions_per_cycle", 3)
128
+ class _Cfg:
129
+ evolution = _EvoCfg()
130
+
131
+ evolver = SkillEvolver(db_path, _Cfg())
132
+ result = evolver.run_consolidation_cycle(profile)
133
+
134
+ return {"ok": True, **result}
135
+ except Exception as e:
136
+ logger.error("evolution_run error: %s", e)
137
+ return {"ok": False, "error": str(e)}
138
+
139
+
140
+ @router.get("/api/evolution/lineage")
141
+ async def evolution_lineage(skill_name: str = ""):
142
+ """Get evolution lineage for a skill or all skills.
143
+
144
+ Returns lineage records and a tree structure grouped by root skill.
145
+ """
146
+ try:
147
+ import sqlite3 as _sqlite3
148
+
149
+ db_path = str(MEMORY_DIR / "memory.db")
150
+ conn = _sqlite3.connect(db_path, timeout=10)
151
+ conn.row_factory = _sqlite3.Row
152
+
153
+ if skill_name:
154
+ rows = conn.execute(
155
+ "SELECT id, skill_name, parent_skill_id, evolution_type, "
156
+ "trigger_type, generation, status, mutation_summary, "
157
+ "blind_verified, created_at, completed_at "
158
+ "FROM skill_evolution_log "
159
+ "WHERE skill_name = ? OR parent_skill_id = ? "
160
+ "ORDER BY created_at ASC",
161
+ (skill_name, skill_name),
162
+ ).fetchall()
163
+ else:
164
+ rows = conn.execute(
165
+ "SELECT id, skill_name, parent_skill_id, evolution_type, "
166
+ "trigger_type, generation, status, mutation_summary, "
167
+ "blind_verified, created_at, completed_at "
168
+ "FROM skill_evolution_log "
169
+ "ORDER BY created_at DESC LIMIT 100",
170
+ ).fetchall()
171
+
172
+ conn.close()
173
+
174
+ lineage = [
175
+ {
176
+ "id": dict(r)["id"],
177
+ "skill_name": dict(r)["skill_name"],
178
+ "parent_skill_id": dict(r).get("parent_skill_id", ""),
179
+ "evolution_type": dict(r)["evolution_type"],
180
+ "trigger": dict(r)["trigger_type"],
181
+ "generation": dict(r).get("generation", 0),
182
+ "status": dict(r)["status"],
183
+ "mutation_summary": dict(r).get("mutation_summary", ""),
184
+ "blind_verified": bool(dict(r).get("blind_verified", 0)),
185
+ "created_at": dict(r).get("created_at", ""),
186
+ "completed_at": dict(r).get("completed_at", ""),
187
+ }
188
+ for r in rows
189
+ ]
190
+
191
+ # Build tree structure: group by root skill
192
+ tree: dict = {}
193
+ for entry in lineage:
194
+ root = entry.get("parent_skill_id") or entry["skill_name"]
195
+ if root not in tree:
196
+ tree[root] = {"root": root, "evolutions": []}
197
+ tree[root]["evolutions"].append({
198
+ "id": entry["id"],
199
+ "skill_name": entry["skill_name"],
200
+ "evolution_type": entry["evolution_type"],
201
+ "status": entry["status"],
202
+ "generation": entry["generation"],
203
+ "created_at": entry["created_at"],
204
+ })
205
+
206
+ return {
207
+ "lineage": lineage,
208
+ "lineage_count": len(lineage),
209
+ "tree": tree,
210
+ }
211
+ except Exception as e:
212
+ logger.debug("evolution_lineage error: %s", e)
213
+ return {"lineage": [], "lineage_count": 0, "tree": {}, "error": str(e)}
@@ -0,0 +1,195 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+ """SuperLocalMemory V3.4.11 "Scale-Ready" - Tier Management Routes
5
+
6
+ Routes: /api/tiers/stats, /api/tiers/evaluate, /api/tiers/pin, /api/tiers/unpin
7
+
8
+ Uses lightweight sqlite3 directly (not MemoryEngine) for fast dashboard queries.
9
+ All connections use WAL mode + busy_timeout for concurrency safety.
10
+ """
11
+
12
+ import logging
13
+ import re
14
+ import sqlite3
15
+ from contextlib import contextmanager
16
+ from datetime import datetime, UTC
17
+
18
+ from fastapi import APIRouter, HTTPException, Request
19
+ from pydantic import BaseModel, Field
20
+
21
+ from .helpers import DB_PATH
22
+
23
+ logger = logging.getLogger("superlocalmemory.routes.tiers")
24
+ router = APIRouter()
25
+
26
+ _PROFILE_PATTERN = re.compile(r'^[a-zA-Z0-9_-]+$')
27
+ _MAX_REASON_LENGTH = 500
28
+
29
+
30
+ class PinRequest(BaseModel):
31
+ fact_id: str = Field(..., min_length=1)
32
+ reason: str = Field(default="", max_length=_MAX_REASON_LENGTH)
33
+
34
+
35
+ @contextmanager
36
+ def _db():
37
+ """Context-managed DB connection with WAL + busy_timeout."""
38
+ conn = sqlite3.connect(str(DB_PATH))
39
+ conn.execute("PRAGMA journal_mode=WAL")
40
+ conn.execute("PRAGMA busy_timeout=5000")
41
+ conn.row_factory = sqlite3.Row
42
+ try:
43
+ yield conn
44
+ finally:
45
+ conn.close()
46
+
47
+
48
+ def _validate_profile(profile_id: str) -> str:
49
+ """Validate profile_id against allowed pattern."""
50
+ if not profile_id or not _PROFILE_PATTERN.match(profile_id):
51
+ raise HTTPException(status_code=400, detail="Invalid profile_id")
52
+ return profile_id
53
+
54
+
55
+ @router.get("/api/tiers/stats")
56
+ async def tier_stats(profile_id: str = "default"):
57
+ """Get tier distribution stats."""
58
+ profile_id = _validate_profile(profile_id)
59
+
60
+ with _db() as conn:
61
+ try:
62
+ c = conn.cursor()
63
+ c.execute(
64
+ "SELECT lifecycle, COUNT(*) as cnt FROM atomic_facts "
65
+ "WHERE profile_id = ? GROUP BY lifecycle", (profile_id,),
66
+ )
67
+ dist = {row["lifecycle"]: row["cnt"] for row in c.fetchall()}
68
+
69
+ pinned = 0
70
+ try:
71
+ c.execute(
72
+ "SELECT COUNT(*) as c FROM pinned_facts "
73
+ "WHERE profile_id = ?", (profile_id,),
74
+ )
75
+ pinned = c.fetchone()["c"]
76
+ except sqlite3.OperationalError:
77
+ pass # pinned_facts table may not exist yet
78
+
79
+ total = sum(dist.values())
80
+ return {
81
+ "active": dist.get("active", 0),
82
+ "warm": dist.get("warm", 0),
83
+ "cold": dist.get("cold", 0),
84
+ "archived": dist.get("archived", 0),
85
+ "total": total,
86
+ "pinned": pinned,
87
+ "active_pct": round(
88
+ dist.get("active", 0) / max(total, 1) * 100, 1,
89
+ ),
90
+ }
91
+ except Exception as exc:
92
+ logger.error("tier_stats failed: %s", exc, exc_info=True)
93
+ raise HTTPException(
94
+ status_code=500, detail="Internal storage error",
95
+ ) from None
96
+
97
+
98
+ @router.post("/api/tiers/evaluate")
99
+ async def evaluate_tiers_route(request: Request, profile_id: str = "default"):
100
+ """Manually trigger tier evaluation.
101
+
102
+ Uses the shared engine (via lazy import) instead of re-initializing
103
+ DatabaseManager on every request.
104
+ """
105
+ profile_id = _validate_profile(profile_id)
106
+
107
+ try:
108
+ from superlocalmemory.core.tier_manager import evaluate_tiers
109
+ from .helpers import get_engine_lazy
110
+
111
+ engine = get_engine_lazy(request.app.state)
112
+ if engine is None or not hasattr(engine, '_db') or engine._db is None:
113
+ raise HTTPException(
114
+ status_code=503, detail="Engine not initialized",
115
+ )
116
+ stats = evaluate_tiers(engine._db, profile_id)
117
+ return {"success": True, "stats": stats}
118
+ except HTTPException:
119
+ raise
120
+ except Exception as exc:
121
+ logger.error("evaluate_tiers failed: %s", exc, exc_info=True)
122
+ raise HTTPException(
123
+ status_code=500, detail="Tier evaluation failed",
124
+ ) from None
125
+
126
+
127
+ @router.post("/api/tiers/pin")
128
+ async def pin_fact_route(request: PinRequest, profile_id: str = "default"):
129
+ """Pin a fact to stay in active tier forever.
130
+
131
+ Validates fact exists in the specified profile before pinning.
132
+ """
133
+ profile_id = _validate_profile(profile_id)
134
+
135
+ with _db() as conn:
136
+ try:
137
+ # Verify fact exists in this profile
138
+ c = conn.cursor()
139
+ c.execute(
140
+ "SELECT fact_id FROM atomic_facts "
141
+ "WHERE fact_id = ? AND profile_id = ?",
142
+ (request.fact_id, profile_id),
143
+ )
144
+ if c.fetchone() is None:
145
+ raise HTTPException(
146
+ status_code=404,
147
+ detail=f"Fact {request.fact_id[:8]}... not found",
148
+ )
149
+
150
+ now = datetime.now(UTC).isoformat()
151
+ conn.execute(
152
+ "INSERT OR REPLACE INTO pinned_facts "
153
+ "(fact_id, profile_id, pinned_at, reason) "
154
+ "VALUES (?, ?, ?, ?)",
155
+ (request.fact_id, profile_id, now, request.reason),
156
+ )
157
+ conn.execute(
158
+ "UPDATE atomic_facts SET lifecycle = 'active' "
159
+ "WHERE fact_id = ? AND profile_id = ?",
160
+ (request.fact_id, profile_id),
161
+ )
162
+ conn.commit()
163
+ return {"success": True, "message": f"Fact {request.fact_id[:8]}... pinned"}
164
+ except HTTPException:
165
+ raise
166
+ except Exception as exc:
167
+ logger.error("pin_fact failed: %s", exc, exc_info=True)
168
+ raise HTTPException(
169
+ status_code=500, detail="Failed to pin fact",
170
+ ) from None
171
+
172
+
173
+ @router.post("/api/tiers/unpin")
174
+ async def unpin_fact_route(request: PinRequest, profile_id: str = "default"):
175
+ """Unpin a fact, allowing normal tier demotion.
176
+
177
+ Lifecycle stays 'active' until the next tier evaluation cycle demotes it
178
+ based on access patterns. This is intentional — immediate demotion would
179
+ surprise the user.
180
+ """
181
+ profile_id = _validate_profile(profile_id)
182
+
183
+ with _db() as conn:
184
+ try:
185
+ conn.execute(
186
+ "DELETE FROM pinned_facts WHERE fact_id = ? AND profile_id = ?",
187
+ (request.fact_id, profile_id),
188
+ )
189
+ conn.commit()
190
+ return {"success": True, "unpinned": True}
191
+ except Exception as exc:
192
+ logger.error("unpin_fact failed: %s", exc, exc_info=True)
193
+ raise HTTPException(
194
+ status_code=500, detail="Failed to unpin fact",
195
+ ) from None
@@ -268,6 +268,20 @@ async def lifespan(application: FastAPI):
268
268
  if reranker and hasattr(reranker, 'warmup_sync'):
269
269
  reranker.warmup_sync(timeout=120)
270
270
 
271
+ # V3.4.11: Pre-warm embedding worker (load ONNX model on startup)
272
+ # Without this, first recall takes 60-90s for model load.
273
+ # Same pattern as reranker warmup above.
274
+ import threading
275
+ def _warmup_embedder():
276
+ try:
277
+ embedder = getattr(retrieval_eng, '_embedder', None) if retrieval_eng else None
278
+ if embedder and hasattr(embedder, 'embed'):
279
+ embedder.embed("warmup")
280
+ logger.info("Embedding worker pre-warmed (ONNX model loaded)")
281
+ except Exception as exc:
282
+ logger.warning("Embedding warmup failed: %s", exc)
283
+ threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
284
+
271
285
  except Exception as exc:
272
286
  logger.warning("Engine init failed: %s", exc)
273
287
  application.state.engine = None
@@ -318,6 +332,8 @@ async def lifespan(application: FastAPI):
318
332
  if enable_legacy:
319
333
  asyncio.create_task(_start_legacy_redirect(_DEFAULT_PORT, _LEGACY_PORT))
320
334
 
335
+ global _start_time
336
+ _start_time = time.monotonic()
321
337
  _last_activity = time.monotonic()
322
338
  logger.info("Unified daemon ready on port %d (24/7 mode)" if idle_timeout <= 0
323
339
  else "Unified daemon ready on port %d (idle timeout: %ds)",
@@ -422,7 +438,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
422
438
  return JSONResponse(
423
439
  status_code=429,
424
440
  content={"error": "Too many requests."},
425
- headers={"Retry-After": str(limiter.window_seconds)},
441
+ headers={"Retry-After": str(getattr(limiter, 'window', 60))},
426
442
  )
427
443
  response = await call_next(request)
428
444
  response.headers["X-RateLimit-Remaining"] = str(remaining)
@@ -472,6 +488,19 @@ def _register_dashboard_routes(application: FastAPI) -> None:
472
488
  application.include_router(profiles_router)
473
489
  application.include_router(backup_router)
474
490
  application.include_router(data_io_router)
491
+
492
+ # Optional routers — ImportError-safe so missing modules don't crash startup
493
+ try:
494
+ from superlocalmemory.server.routes.tiers import router as tiers_router
495
+ application.include_router(tiers_router)
496
+ except ImportError:
497
+ logger.debug("tiers_router not available")
498
+
499
+ try:
500
+ from superlocalmemory.server.routes.evolution import router as evolution_router
501
+ application.include_router(evolution_router)
502
+ except ImportError:
503
+ logger.debug("evolution_router not available")
475
504
  application.include_router(events_router)
476
505
  application.include_router(agents_router)
477
506
  application.include_router(ws_router)
@@ -542,13 +571,16 @@ def _register_daemon_routes(application: FastAPI) -> None:
542
571
  }
543
572
 
544
573
  @application.get("/recall")
545
- async def recall(q: str = "", limit: int = 20):
574
+ async def recall(q: str = "", query: str = "", limit: int = 20):
546
575
  _update_activity()
576
+ search_query = q or query # Accept both ?q= and ?query= for compatibility
547
577
  engine = application.state.engine
548
578
  if engine is None:
549
579
  raise HTTPException(503, detail="Engine not initialized")
580
+ if not search_query:
581
+ return {"results": [], "count": 0, "query_type": "none", "retrieval_time_ms": 0}
550
582
  try:
551
- response = engine.recall(q, limit=limit)
583
+ response = engine.recall(search_query, limit=limit)
552
584
  results = [
553
585
  {
554
586
  "content": r.fact.content,
@@ -590,7 +622,6 @@ def _register_daemon_routes(application: FastAPI) -> None:
590
622
  async def status():
591
623
  _update_activity()
592
624
  engine = application.state.engine
593
- uptime = time.monotonic() - _last_activity
594
625
  fact_count = engine.fact_count if engine else 0
595
626
  mode = engine._config.mode.value if engine and hasattr(engine, '_config') else "unknown"
596
627
  return {
@@ -656,7 +687,7 @@ def _start_memory_watchdog() -> None:
656
687
  """
657
688
  import threading
658
689
 
659
- MAX_WORKER_MB = 2048 # 2GB per worker — kill if exceeded
690
+ MAX_WORKER_MB = 4096 # 4GB per worker — ONNX full model is 1.6GB + overhead
660
691
 
661
692
  def watchdog_loop():
662
693
  while True:
@@ -0,0 +1,149 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """SuperLocalMemory V3.4.11 "Scale-Ready" — Schema Extensions.
6
+
7
+ New tables for tiered storage + future backend integrations:
8
+ - pinned_facts: User-pinned facts that stay in hot tier forever
9
+ - backend_status: Tracks initialization state of LanceDB/KùzuDB backends
10
+ - fact_consolidations: History of merged/consolidated facts
11
+
12
+ Design rules (inherited):
13
+ - CREATE IF NOT EXISTS for idempotency
14
+ - profile_id where applicable
15
+ - Never ALTER existing column types
16
+
17
+ Part of Qualixar | Author: Varun Pratap Bhardwaj
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import logging
23
+ import sqlite3
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+ # ---------------------------------------------------------------------------
28
+ # DDL — Pinned Facts (user override on lifecycle demotion)
29
+ # ---------------------------------------------------------------------------
30
+
31
+ _PINNED_FACTS_DDL = """
32
+ CREATE TABLE IF NOT EXISTS pinned_facts (
33
+ fact_id TEXT PRIMARY KEY,
34
+ profile_id TEXT DEFAULT 'default',
35
+ pinned_at TEXT NOT NULL,
36
+ reason TEXT DEFAULT ''
37
+ );
38
+
39
+ CREATE INDEX IF NOT EXISTS idx_pinned_facts_profile
40
+ ON pinned_facts(profile_id);
41
+ """
42
+
43
+ # ---------------------------------------------------------------------------
44
+ # DDL — Backend Status (LanceDB, KùzuDB, sqlite-vec tracking)
45
+ # ---------------------------------------------------------------------------
46
+
47
+ _BACKEND_STATUS_DDL = """
48
+ CREATE TABLE IF NOT EXISTS backend_status (
49
+ backend_name TEXT PRIMARY KEY,
50
+ status TEXT DEFAULT 'not_initialized',
51
+ record_count INTEGER DEFAULT 0,
52
+ last_sync_at TEXT,
53
+ error_message TEXT DEFAULT '',
54
+ config TEXT DEFAULT '{}'
55
+ );
56
+ """
57
+
58
+ # ---------------------------------------------------------------------------
59
+ # DDL — Fact Consolidations (merge history)
60
+ # ---------------------------------------------------------------------------
61
+
62
+ _FACT_CONSOLIDATIONS_DDL = """
63
+ CREATE TABLE IF NOT EXISTS fact_consolidations (
64
+ consolidation_id TEXT PRIMARY KEY,
65
+ profile_id TEXT DEFAULT 'default',
66
+ consolidated_fact_id TEXT NOT NULL,
67
+ source_fact_ids TEXT NOT NULL,
68
+ strategy TEXT DEFAULT 'entity_cluster',
69
+ created_at TEXT NOT NULL
70
+ );
71
+
72
+ CREATE INDEX IF NOT EXISTS idx_fact_consolidations_profile
73
+ ON fact_consolidations(profile_id);
74
+ CREATE INDEX IF NOT EXISTS idx_fact_consolidations_target
75
+ ON fact_consolidations(consolidated_fact_id);
76
+ """
77
+
78
+ # ---------------------------------------------------------------------------
79
+ # DDL — Index on lifecycle for fast tier queries
80
+ # ---------------------------------------------------------------------------
81
+
82
+ _LIFECYCLE_INDEX_DDL = """
83
+ CREATE INDEX IF NOT EXISTS idx_atomic_facts_lifecycle
84
+ ON atomic_facts(lifecycle, profile_id);
85
+ """
86
+
87
+
88
+ # ---------------------------------------------------------------------------
89
+ # Migration runner
90
+ # ---------------------------------------------------------------------------
91
+
92
+ def apply_v3411_schema(db_path: str | sqlite3.Connection) -> dict:
93
+ """Apply all v3.4.11 schema changes. Idempotent."""
94
+ result = {"applied": [], "errors": []}
95
+
96
+ if isinstance(db_path, sqlite3.Connection):
97
+ conn = db_path
98
+ own_connection = False
99
+ else:
100
+ conn = sqlite3.connect(str(db_path))
101
+ own_connection = True
102
+
103
+ try:
104
+ for name, ddl in [
105
+ ("pinned_facts", _PINNED_FACTS_DDL),
106
+ ("backend_status", _BACKEND_STATUS_DDL),
107
+ ("fact_consolidations", _FACT_CONSOLIDATIONS_DDL),
108
+ ("lifecycle_index", _LIFECYCLE_INDEX_DDL),
109
+ ]:
110
+ try:
111
+ conn.executescript(ddl)
112
+ result["applied"].append(name)
113
+ except sqlite3.OperationalError as e:
114
+ result["errors"].append(f"{name}: {e}")
115
+
116
+ # Seed default backend entries
117
+ now = __import__("datetime").datetime.now().isoformat()
118
+ for backend in ("sqlite_vec", "sqlite_graph", "lancedb", "kuzu"):
119
+ try:
120
+ conn.execute(
121
+ "INSERT OR IGNORE INTO backend_status "
122
+ "(backend_name, status, config) VALUES (?, ?, ?)",
123
+ (backend, "not_initialized" if backend in ("lancedb", "kuzu") else "active", "{}"),
124
+ )
125
+ except sqlite3.OperationalError:
126
+ pass
127
+
128
+ # Mark version
129
+ try:
130
+ conn.execute(
131
+ "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
132
+ ("3.4.11", now),
133
+ )
134
+ except sqlite3.OperationalError:
135
+ pass
136
+
137
+ conn.commit()
138
+
139
+ if result["applied"]:
140
+ logger.info("Schema v3.4.11 applied: %s", ", ".join(result["applied"]))
141
+
142
+ except Exception as e:
143
+ result["errors"].append(f"fatal: {e}")
144
+ logger.error("Schema v3.4.11 migration failed: %s", e)
145
+ finally:
146
+ if own_connection:
147
+ conn.close()
148
+
149
+ return result
@@ -1519,10 +1519,12 @@
1519
1519
  <h5 class="mb-0"><i class="bi bi-hourglass-split text-warning"></i> Memory Lifecycle</h5>
1520
1520
  <div>
1521
1521
  <span class="badge bg-secondary me-2" id="lifecycle-profile-badge">default</span>
1522
- <button class="btn btn-sm btn-outline-info" onclick="compactDryRun()"><i class="bi bi-funnel"></i> Preview Compaction</button>
1522
+ <button class="btn btn-sm btn-outline-success" onclick="evaluateTiersNow()"><i class="bi bi-arrow-repeat"></i> Evaluate Tiers</button>
1523
+ <button class="btn btn-sm btn-outline-info ms-1" onclick="compactDryRun()"><i class="bi bi-funnel"></i> Preview Compaction</button>
1523
1524
  <button class="btn btn-sm btn-outline-warning ms-1" onclick="compactExecute()"><i class="bi bi-lightning"></i> Compact Now</button>
1524
1525
  </div>
1525
1526
  </div>
1527
+
1526
1528
  <!-- State Distribution -->
1527
1529
  <div class="row g-3 mb-4" id="lifecycle-states-row">
1528
1530
  <div class="col-md-2"><div class="card p-3 text-center"><div class="text-success fw-bold fs-3" id="lc-active-count">-</div><small class="text-muted">Active</small></div></div>
@@ -1741,6 +1743,7 @@
1741
1743
  <div style="margin-top:8px">Loading skill performance data...</div>
1742
1744
  </div>
1743
1745
  </div>
1746
+ <div id="skill-lineage-container" class="mt-3"></div>
1744
1747
  <div id="skill-detail-panel" style="display:none;margin-top:24px"></div>
1745
1748
  </div>
1746
1749
 
@@ -2150,7 +2153,7 @@
2150
2153
  <script src="static/js/ng-health.js?v=345"></script>
2151
2154
  <script src="static/js/ng-ingestion.js?v=345"></script>
2152
2155
  <script src="static/js/ng-entities.js?v=3410"></script>
2153
- <script src="static/js/ng-skills.js?v=3410"></script>
2156
+ <script src="static/js/ng-skills.js?v=3411"></script>
2154
2157
  <script src="static/js/ng-mesh.js?v=345"></script>
2155
2158
  <script src="static/js/ng-shell.js?v=3410"></script>
2156
2159