superlocalmemory 3.4.8 → 3.4.10

package/src/superlocalmemory/cli/commands.py

@@ -58,6 +58,8 @@ def dispatch(args: Namespace) -> None:
         "reap": cmd_reap,
         # V3.3.21 daemon
         "serve": cmd_serve,
+        # V3.4.9 nuclear restart
+        "restart": cmd_restart,
         # V3.4.3 ingestion adapters
         "adapters": cmd_adapters,
         # V3.4.8 external observation ingestion
@@ -146,6 +148,193 @@ def cmd_serve(args: Namespace) -> None:
 # -- Ingestion Adapters (V3.4.3) ------------------------------------------
 
 
+def cmd_restart(args: Namespace) -> None:
+    """Nuclear restart: kill ALL orphans, clean state, start fresh, verify health.
+
+    5-step pipeline:
+      1. Kill ALL SLM processes (daemon + workers + orphans)
+      2. Clean stale PID/port/lock files
+      3. Start fresh daemon
+      4. Wait for engine warmup + verify health
+      5. Optionally open dashboard
+    """
+    import os
+    import time
+    from pathlib import Path
+
+    use_json = getattr(args, "json", False)
+    open_dashboard = getattr(args, "dashboard", False)
+    slm_dir = Path.home() / ".superlocalmemory"
+    steps: list[dict] = []
+
+    def _log(step: int, name: str, status: str, detail: str = ""):
+        entry = {"step": step, "name": name, "status": status, "detail": detail}
+        steps.append(entry)
+        if not use_json:
+            icon = {"ok": "+", "warn": "!", "fail": "x"}.get(status, " ")
+            print(f"  [{icon}] Step {step}: {name}" + (f" — {detail}" if detail else ""))
+
+    if not use_json:
+        print()
+        print("  SLM Full System Restart")
+        print("  " + "=" * 40)
+        print()
+
+    # Step 1: Kill ALL SLM processes
+    killed = 0
+    try:
+        import psutil
+        my_pid = os.getpid()
+        targets = [
+            "superlocalmemory.server.unified_daemon",
+            "superlocalmemory.core.embedding_worker",
+            "superlocalmemory.core.recall_worker",
+            "superlocalmemory.core.reranker_worker",
+            "superlocalmemory.cli.daemon",
+        ]
+        for proc in psutil.process_iter(["pid", "cmdline"]):
+            try:
+                if proc.pid == my_pid:
+                    continue
+                cmdline = " ".join(proc.info.get("cmdline") or [])
+                if any(t in cmdline for t in targets):
+                    for child in proc.children(recursive=True):
+                        try:
+                            child.kill()
+                            killed += 1
+                        except (psutil.NoSuchProcess, psutil.AccessDenied):
+                            pass
+                    proc.kill()
+                    killed += 1
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+    except ImportError:
+        # Fallback: pkill
+        import subprocess as _sp
+        for pattern in [
+            "superlocalmemory.server.unified_daemon",
+            "superlocalmemory.core.embedding_worker",
+            "superlocalmemory.core.recall_worker",
+            "superlocalmemory.core.reranker_worker",
+        ]:
+            try:
+                r = _sp.run(["pkill", "-9", "-f", pattern], capture_output=True, timeout=5)
+                if r.returncode == 0:
+                    killed += 1
+            except Exception:
+                pass
+
+    _log(1, "Kill all SLM processes", "ok", f"{killed} processes killed")
+    time.sleep(3)
+
+    # Step 2: Clean stale files
+    cleaned = []
+    for fname in ("daemon.pid", "daemon.port", "daemon.lock"):
+        fpath = slm_dir / fname
+        if fpath.exists():
+            fpath.unlink(missing_ok=True)
+            cleaned.append(fname)
+    _log(2, "Clean stale state files", "ok",
+         f"removed: {', '.join(cleaned)}" if cleaned else "already clean")
+
+    # Step 3: Start fresh daemon
+    time.sleep(1)
+    from superlocalmemory.cli.daemon import ensure_daemon
+    started = ensure_daemon()
+    _log(3, "Start fresh daemon", "ok" if started else "fail",
+         "daemon started" if started else "failed to start — check slm doctor")
+
+    if not started:
+        if use_json:
+            from superlocalmemory.cli.json_output import json_print
+            json_print("restart", data={"steps": steps, "success": False},
+                       next_actions=[{"command": "slm doctor", "description": "Diagnose issues"}])
+        else:
+            print("\n  Restart FAILED at step 3. Run: slm doctor")
+        return
+
+    # Step 4: Wait for warmup + verify health
+    if not use_json:
+        print("  [ ] Step 4: Waiting for engine warmup (up to 30s)...", end="", flush=True)
+
+    health = None
+    engine_ok = False
+    for attempt in range(15):
+        time.sleep(2)
+        try:
+            from superlocalmemory.cli.daemon import daemon_request
+            health = daemon_request("GET", "/health")
+            if health and health.get("engine") == "initialized":
+                engine_ok = True
+                break
+        except Exception:
+            pass
+
+    if not use_json:
+        print("\r", end="")  # Clear the waiting line
+
+    if engine_ok:
+        version = health.get("version", "?")
+        pid = health.get("pid", "?")
+        _log(4, "Engine health verified", "ok", f"v{version}, PID {pid}, engine=initialized")
+    else:
+        engine_state = health.get("engine", "unknown") if health else "unreachable"
+        _log(4, "Engine health check", "warn",
+             f"engine={engine_state} — may still be warming up. Try again in 30s.")
+
+    # Step 5: Database integrity check
+    try:
+        import sqlite3
+        db_path = slm_dir / "memory.db"
+        if db_path.exists():
+            conn = sqlite3.connect(str(db_path))
+            integrity = conn.execute("PRAGMA integrity_check").fetchone()[0]
+            fact_count = conn.execute("SELECT COUNT(*) FROM atomic_facts").fetchone()[0]
+            entity_count = conn.execute("SELECT COUNT(*) FROM canonical_entities").fetchone()[0]
+            conn.close()
+            _log(5, "Database integrity", "ok" if integrity == "ok" else "fail",
+                 f"integrity={integrity}, {fact_count} facts, {entity_count} entities")
+        else:
+            _log(5, "Database check", "warn", "no database yet — will create on first use")
+    except Exception as exc:
+        _log(5, "Database check", "warn", str(exc))
+
+    # Step 6 (optional): Open dashboard
+    if open_dashboard:
+        try:
+            import webbrowser
+            from superlocalmemory.cli.daemon import _get_port
+            port = _get_port()
+            url = f"http://localhost:{port}"
+            webbrowser.open(url)
+            _log(6, "Dashboard opened", "ok", url)
+        except Exception as exc:
+            _log(6, "Dashboard open", "fail", str(exc))
+
+    # Summary
+    all_ok = all(s["status"] == "ok" for s in steps)
+
+    if use_json:
+        from superlocalmemory.cli.json_output import json_print
+        json_print("restart", data={
+            "steps": steps, "success": all_ok,
+            "processes_killed": killed,
+            "version": health.get("version") if health else None,
+        }, next_actions=[
+            {"command": "slm serve status", "description": "Check daemon status"},
+            {"command": "slm dashboard", "description": "Open dashboard"},
+        ])
+        return
+
+    print()
+    if all_ok:
+        print("  All systems operational.")
+    else:
+        warnings = [s for s in steps if s["status"] != "ok"]
+        print(f"  {len(warnings)} issue(s) — check details above.")
+    print()
+
+
 def cmd_ingest(args: Namespace) -> None:
     """Import external observations into SLM learning pipeline."""
     from superlocalmemory.cli.ingest_cmd import cmd_ingest as _ingest

package/src/superlocalmemory/cli/ingest_cmd.py

@@ -76,28 +76,43 @@ def _error(msg: str, use_json: bool) -> None:
 
 
 def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
-    """Ingest ECC session summaries from transcript JSONL files.
+    """Ingest ECC observations into SLM tool_events.
 
-    Scans the Claude projects directory for session JSONL files and
-    extracts tool usage patterns from them.
+    Scans TWO sources:
+    1. ECC observation files (~/.claude/homunculus/projects/*/observations.jsonl)
+       — rich data: tool input/output, session_id, project context
+    2. Claude session transcript files (~/.claude/projects/*.jsonl)
+       — fallback: tool_use blocks from conversation history
+
+    v3.4.10: Now preserves input_summary and output_summary from ECC observations.
     """
     result = {"source": "ecc", "ingested": 0, "skipped": 0, "dry_run": dry_run}
 
-    # Find ECC session files
+    files: list[Path] = []
+
     if file_path:
         files = [Path(file_path)]
     else:
-        # Auto-discover: scan Claude project session files
+        # Source 1: ECC observation files (RICH data — preferred)
+        ecc_dir = Path.home() / ".claude" / "homunculus" / "projects"
+        if ecc_dir.exists():
+            ecc_files = sorted(
+                ecc_dir.rglob("observations.jsonl"),
+                key=lambda p: p.stat().st_mtime, reverse=True,
+            )
+            files.extend(ecc_files[:20])
+
+        # Source 2: Claude session transcripts (fallback)
         claude_dir = Path.home() / ".claude" / "projects"
-        if not claude_dir.exists():
-            result["error"] = f"Claude projects dir not found: {claude_dir}"
-            return result
-        files = sorted(claude_dir.rglob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True)
-        # Limit to recent files (last 20)
-        files = files[:20]
+        if claude_dir.exists():
+            claude_files = sorted(
+                claude_dir.rglob("*.jsonl"),
+                key=lambda p: p.stat().st_mtime, reverse=True,
+            )
+            files.extend(claude_files[:20])
 
     if not files:
-        result["error"] = "No session files found"
+        result["error"] = "No ECC observation or session files found"
         return result
 
     result["files_scanned"] = len(files)
@@ -116,7 +131,6 @@ def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
                         result["skipped"] += 1
                         continue
 
-                    # Extract tool usage from ECC session records
                     extracted = _extract_tool_events_from_record(record)
                     events.extend(extracted)
         except (OSError, PermissionError):
@@ -128,7 +142,6 @@ def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
         result["sample"] = events[:5]
         return result
 
-    # Write to tool_events table
     if events:
         ingested = _write_tool_events(events)
         result["ingested"] = ingested
@@ -139,47 +152,79 @@ def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
 
 
 def _extract_tool_events_from_record(record: dict) -> list[dict]:
-    """Extract tool events from a single ECC/Claude session JSONL record."""
+    """Extract tool events from a single ECC/Claude session JSONL record.
+
+    Handles three formats:
+    1. ECC observation format: {"event": "tool_complete", "tool": "X", "input": "...", "output": "..."}
+    2. Claude transcript format: {"type": "assistant", "content": [{"type": "tool_use", ...}]}
+    3. Direct tool event format: {"tool_name": "X", "event_type": "complete"}
+
+    v3.4.10: Preserves input_summary and output_summary from all formats.
+    """
     events = []
+    now_iso = datetime.now(timezone.utc).isoformat()
+
+    # Format 1: ECC observation format (from ~/.claude/homunculus/projects/*/observations.jsonl)
+    if "event" in record and "tool" in record:
+        event_type_raw = record.get("event", "")
+        if event_type_raw in ("tool_complete", "tool_start"):
+            event_type = "complete" if event_type_raw == "tool_complete" else "invoke"
+            events.append({
+                "tool_name": record["tool"],
+                "event_type": event_type,
+                "input_summary": str(record.get("input", ""))[:500],
+                "output_summary": str(record.get("output", ""))[:500],
+                "session_id": record.get("session", "ecc_import"),
+                "project_path": record.get("project_name", ""),
+                "created_at": record.get("timestamp", now_iso),
+            })
+        return events
 
-    # Handle ECC summary format
+    # Format 2: Claude transcript format
     if "type" in record:
         rtype = record.get("type", "")
-
-        # Tool use records
         if rtype == "assistant" and "content" in record:
             content = record.get("content", [])
             if isinstance(content, list):
                 for block in content:
                     if isinstance(block, dict) and block.get("type") == "tool_use":
                         tool_name = block.get("name", "unknown")
+                        raw_input = block.get("input", {})
+                        input_str = json.dumps(raw_input, default=str)[:500] if isinstance(raw_input, dict) else str(raw_input)[:500]
                         events.append({
                             "tool_name": tool_name,
                             "event_type": "complete",
+                            "input_summary": input_str,
+                            "output_summary": "",
                             "session_id": record.get("session_id", "ecc_import"),
-                            "created_at": record.get("timestamp", datetime.now(timezone.utc).isoformat()),
+                            "project_path": "",
+                            "created_at": record.get("timestamp", now_iso),
                         })
-
-            # Also extract from tool_use type directly
-            if isinstance(content, list):
-                for block in content:
-                    if isinstance(block, dict) and block.get("type") == "tool_result":
+                    elif isinstance(block, dict) and block.get("type") == "tool_result":
                         tool_name = block.get("tool_use_id", "unknown")
                         is_error = block.get("is_error", False)
+                        raw_content = block.get("content", "")
+                        output_str = str(raw_content)[:500] if raw_content else ""
                         events.append({
                             "tool_name": tool_name,
                             "event_type": "error" if is_error else "complete",
+                            "input_summary": "",
+                            "output_summary": output_str,
                             "session_id": record.get("session_id", "ecc_import"),
-                            "created_at": record.get("timestamp", datetime.now(timezone.utc).isoformat()),
+                            "project_path": "",
+                            "created_at": record.get("timestamp", now_iso),
                         })
 
-    # Handle direct tool event format (from hook output)
+    # Format 3: Direct tool event format (from hook output)
     if "tool_name" in record and "event_type" in record:
         events.append({
             "tool_name": record["tool_name"],
             "event_type": record.get("event_type", "complete"),
+            "input_summary": str(record.get("input_summary", ""))[:500],
+            "output_summary": str(record.get("output_summary", ""))[:500],
             "session_id": record.get("session_id", "ecc_import"),
-            "created_at": record.get("created_at", datetime.now(timezone.utc).isoformat()),
+            "project_path": record.get("project_path", ""),
+            "created_at": record.get("created_at", now_iso),
         })
 
     return events
@@ -228,7 +273,11 @@ def _ingest_jsonl(file_path: str, *, dry_run: bool = False) -> dict:
 
 
 def _write_tool_events(events: list[dict]) -> int:
-    """Write tool events to SLM's memory.db tool_events table."""
+    """Write tool events to SLM's memory.db tool_events table.
+
+    v3.4.10: Preserves input_summary, output_summary, and project_path
+    from enriched sources (ECC observations, enriched hook).
+    """
     db_path = MEMORY_DB
     if not db_path.exists():
         return 0
@@ -243,11 +292,14 @@ def _write_tool_events(events: list[dict]) -> int:
                     "INSERT INTO tool_events "
                     "(session_id, profile_id, project_path, tool_name, event_type, "
                     " input_summary, output_summary, duration_ms, metadata, created_at) "
-                    "VALUES (?, 'default', '', ?, ?, '', '', 0, '{}', ?)",
+                    "VALUES (?, 'default', ?, ?, ?, ?, ?, 0, '{}', ?)",
                     (
                         ev.get("session_id", "import"),
+                        ev.get("project_path", ""),
                         ev["tool_name"],
                         ev.get("event_type", "complete"),
+                        ev.get("input_summary", ""),
+                        ev.get("output_summary", ""),
                         ev.get("created_at", datetime.now(timezone.utc).isoformat()),
                     ),
                 )

package/src/superlocalmemory/cli/main.py

@@ -199,6 +199,17 @@ def main() -> None:
         help="start (default), stop, status, install (OS service), uninstall",
     )
 
+    # V3.4.9: Full system restart with health verification
+    restart_p = sub.add_parser(
+        "restart",
+        help="Nuclear restart: kill orphans, clean state, start fresh, verify health",
+    )
+    restart_p.add_argument(
+        "--dashboard", action="store_true",
+        help="Open dashboard after restart",
+    )
+    restart_p.add_argument("--json", action="store_true", help="Output structured JSON")
+
     # -- Profiles ------------------------------------------------------
     profile_p = sub.add_parser("profile", help="Profile management (list/switch/create)")
     profile_p.add_argument(

package/src/superlocalmemory/core/consolidation_engine.py

@@ -178,6 +178,16 @@ class ConsolidationEngine:
                     logger.debug("Soft prompt generation (non-fatal): %s", exc)
                     results["soft_prompts"] = {"error": str(exc)}
 
+                # Step 10 (v3.4.10): Mine skill performance from tool events.
+                # Creates per-skill assertions and skill correlation patterns.
+                try:
+                    from superlocalmemory.learning.skill_performance_miner import SkillPerformanceMiner
+                    spm = SkillPerformanceMiner(self._db.db_path)
+                    results["skill_performance"] = spm.mine(profile_id)
+                except Exception as exc:
+                    logger.debug("Skill performance mining (non-fatal): %s", exc)
+                    results["skill_performance"] = {"error": str(exc)}
+
             results["success"] = True
         except Exception as exc:
             logger.warning(

package/src/superlocalmemory/core/engine.py

@@ -139,6 +139,13 @@ class MemoryEngine:
         except Exception as exc:
             logger.debug("V3.4.7 schema migration: %s", exc)
 
+        # V3.4.10: Apply "Fortress" schema (backup_destinations, entity_blacklist)
+        try:
+            from superlocalmemory.storage.schema_v3410 import apply_v3410_schema
+            apply_v3410_schema(str(self._db.db_path))
+        except Exception as exc:
+            logger.debug("V3.4.10 schema migration: %s", exc)
+
         self._embedder = init_embedder(self._config)
 
         if self._caps.llm_fact_extraction:

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -75,7 +75,7 @@ class MaintenanceScheduler:
         self._timer.start()
 
     def _run(self) -> None:
-        """Execute maintenance and schedule next run."""
+        """Execute maintenance + auto-backup check, then schedule next run."""
         if not self._running:
             return
         try:
@@ -84,8 +84,29 @@ class MaintenanceScheduler:
             logger.info("Scheduled maintenance complete: %s", counts)
         except Exception as exc:
             logger.warning("Scheduled maintenance failed: %s", exc)
-        finally:
-            self._schedule_next()
+
+        # V3.4.10: Check if auto-backup is due
+        try:
+            from superlocalmemory.infra.backup import BackupManager
+            manager = BackupManager(db_path=self._db.db_path)
+            filename = manager.check_and_backup()
+            if filename:
+                logger.info("Auto-backup created: %s", filename)
+                self._sync_cloud_destinations(manager)
+        except Exception as exc:
+            logger.debug("Auto-backup check skipped: %s", exc)
+
+        self._schedule_next()
+
+    def _sync_cloud_destinations(self, manager: object) -> None:
+        """Push latest backup to configured cloud destinations."""
+        try:
+            from superlocalmemory.infra.cloud_backup import sync_all_destinations
+            sync_all_destinations(self._db.db_path)
+        except ImportError:
+            pass  # cloud_backup module not available yet
+        except Exception as exc:
+            logger.warning("Cloud sync failed (non-critical): %s", exc)
 
     def __del__(self) -> None:
         try: