superlocalmemory 3.4.9 → 3.4.11

package/src/superlocalmemory/cli/commands.py

@@ -58,10 +58,15 @@ def dispatch(args: Namespace) -> None:
         "reap": cmd_reap,
         # V3.3.21 daemon
         "serve": cmd_serve,
+        # V3.4.9 nuclear restart
+        "restart": cmd_restart,
         # V3.4.3 ingestion adapters
         "adapters": cmd_adapters,
         # V3.4.8 external observation ingestion
         "ingest": cmd_ingest,
+        # V3.4.11 skill evolution
+        "config": cmd_config,
+        "evolve": cmd_evolve,
     }
     handler = handlers.get(args.command)
     if handler:
@@ -146,12 +151,366 @@ def cmd_serve(args: Namespace) -> None:
 # -- Ingestion Adapters (V3.4.3) ------------------------------------------
 
 
+def cmd_restart(args: Namespace) -> None:
+    """Nuclear restart: kill ALL orphans, clean state, start fresh, verify health.
+
+    5-step pipeline:
+      1. Kill ALL SLM processes (daemon + workers + orphans)
+      2. Clean stale PID/port/lock files
+      3. Start fresh daemon
+      4. Wait for engine warmup + verify health
+      5. Optionally open dashboard
+    """
+    import os
+    import time
+    from pathlib import Path
+
+    use_json = getattr(args, "json", False)
+    open_dashboard = getattr(args, "dashboard", False)
+    slm_dir = Path.home() / ".superlocalmemory"
+    steps: list[dict] = []
+
+    def _log(step: int, name: str, status: str, detail: str = ""):
+        entry = {"step": step, "name": name, "status": status, "detail": detail}
+        steps.append(entry)
+        if not use_json:
+            icon = {"ok": "+", "warn": "!", "fail": "x"}.get(status, " ")
+            print(f"  [{icon}] Step {step}: {name}" + (f" — {detail}" if detail else ""))
+
+    if not use_json:
+        print()
+        print("  SLM Full System Restart")
+        print("  " + "=" * 40)
+        print()
+
+    # Step 1: Kill ALL SLM processes
+    killed = 0
+    try:
+        import psutil
+        my_pid = os.getpid()
+        targets = [
+            "superlocalmemory.server.unified_daemon",
+            "superlocalmemory.core.embedding_worker",
+            "superlocalmemory.core.recall_worker",
+            "superlocalmemory.core.reranker_worker",
+            "superlocalmemory.cli.daemon",
+        ]
+        for proc in psutil.process_iter(["pid", "cmdline"]):
+            try:
+                if proc.pid == my_pid:
+                    continue
+                cmdline = " ".join(proc.info.get("cmdline") or [])
+                if any(t in cmdline for t in targets):
+                    for child in proc.children(recursive=True):
+                        try:
+                            child.kill()
+                            killed += 1
+                        except (psutil.NoSuchProcess, psutil.AccessDenied):
+                            pass
+                    proc.kill()
+                    killed += 1
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+    except ImportError:
+        # Fallback: pkill
+        import subprocess as _sp
+        for pattern in [
+            "superlocalmemory.server.unified_daemon",
+            "superlocalmemory.core.embedding_worker",
+            "superlocalmemory.core.recall_worker",
+            "superlocalmemory.core.reranker_worker",
+        ]:
+            try:
+                r = _sp.run(["pkill", "-9", "-f", pattern], capture_output=True, timeout=5)
+                if r.returncode == 0:
+                    killed += 1
+            except Exception:
+                pass
+
+    _log(1, "Kill all SLM processes", "ok", f"{killed} processes killed")
+    time.sleep(3)
+
+    # Step 2: Clean stale files
+    cleaned = []
+    for fname in ("daemon.pid", "daemon.port", "daemon.lock"):
+        fpath = slm_dir / fname
+        if fpath.exists():
+            fpath.unlink(missing_ok=True)
+            cleaned.append(fname)
+    _log(2, "Clean stale state files", "ok",
+         f"removed: {', '.join(cleaned)}" if cleaned else "already clean")
+
+    # Step 3: Start fresh daemon
+    time.sleep(1)
+    from superlocalmemory.cli.daemon import ensure_daemon
+    started = ensure_daemon()
+    _log(3, "Start fresh daemon", "ok" if started else "fail",
+         "daemon started" if started else "failed to start — check slm doctor")
+
+    if not started:
+        if use_json:
+            from superlocalmemory.cli.json_output import json_print
+            json_print("restart", data={"steps": steps, "success": False},
+                       next_actions=[{"command": "slm doctor", "description": "Diagnose issues"}])
+        else:
+            print("\n  Restart FAILED at step 3. Run: slm doctor")
+        return
+
+    # Step 4: Wait for warmup + verify health
+    if not use_json:
+        print("  [ ] Step 4: Waiting for engine warmup (up to 30s)...", end="", flush=True)
+
+    health = None
+    engine_ok = False
+    for attempt in range(15):
+        time.sleep(2)
+        try:
+            from superlocalmemory.cli.daemon import daemon_request
+            health = daemon_request("GET", "/health")
+            if health and health.get("engine") == "initialized":
+                engine_ok = True
+                break
+        except Exception:
+            pass
+
+    if not use_json:
+        print("\r", end="")  # Clear the waiting line
+
+    if engine_ok:
+        version = health.get("version", "?")
+        pid = health.get("pid", "?")
+        _log(4, "Engine health verified", "ok", f"v{version}, PID {pid}, engine=initialized")
+    else:
+        engine_state = health.get("engine", "unknown") if health else "unreachable"
+        _log(4, "Engine health check", "warn",
+             f"engine={engine_state} — may still be warming up. Try again in 30s.")
+
+    # Step 5: Database integrity check
+    try:
+        import sqlite3
+        db_path = slm_dir / "memory.db"
+        if db_path.exists():
+            conn = sqlite3.connect(str(db_path))
+            integrity = conn.execute("PRAGMA integrity_check").fetchone()[0]
+            fact_count = conn.execute("SELECT COUNT(*) FROM atomic_facts").fetchone()[0]
+            entity_count = conn.execute("SELECT COUNT(*) FROM canonical_entities").fetchone()[0]
+            conn.close()
+            _log(5, "Database integrity", "ok" if integrity == "ok" else "fail",
+                 f"integrity={integrity}, {fact_count} facts, {entity_count} entities")
+        else:
+            _log(5, "Database check", "warn", "no database yet — will create on first use")
+    except Exception as exc:
+        _log(5, "Database check", "warn", str(exc))
+
+    # Step 6 (optional): Open dashboard
+    if open_dashboard:
+        try:
+            import webbrowser
+            from superlocalmemory.cli.daemon import _get_port
+            port = _get_port()
+            url = f"http://localhost:{port}"
+            webbrowser.open(url)
+            _log(6, "Dashboard opened", "ok", url)
+        except Exception as exc:
+            _log(6, "Dashboard open", "fail", str(exc))
+
+    # Summary
+    all_ok = all(s["status"] == "ok" for s in steps)
+
+    if use_json:
+        from superlocalmemory.cli.json_output import json_print
+        json_print("restart", data={
+            "steps": steps, "success": all_ok,
+            "processes_killed": killed,
+            "version": health.get("version") if health else None,
+        }, next_actions=[
+            {"command": "slm serve status", "description": "Check daemon status"},
+            {"command": "slm dashboard", "description": "Open dashboard"},
+        ])
+        return
+
+    print()
+    if all_ok:
+        print("  All systems operational.")
+    else:
+        warnings = [s for s in steps if s["status"] != "ok"]
+        print(f"  {len(warnings)} issue(s) — check details above.")
+    print()
+
+
 def cmd_ingest(args: Namespace) -> None:
     """Import external observations into SLM learning pipeline."""
     from superlocalmemory.cli.ingest_cmd import cmd_ingest as _ingest
     _ingest(args)
 
 
+# -- Config & Evolution (V3.4.11) -----------------------------------------------
+
+
+def cmd_config(args: Namespace) -> None:
+    """Get or set config values (dot-notation).
+
+    Usage:
+      slm config set evolution.enabled true
+      slm config set evolution.backend auto
+      slm config get evolution.enabled
+      slm config get evolution.backend
+    """
+    import json
+    from pathlib import Path
+
+    use_json = getattr(args, "json", False)
+    action = getattr(args, "action", "get")
+    key = getattr(args, "key", "")
+    value = getattr(args, "value", None)
+
+    config_path = Path.home() / ".superlocalmemory" / "config.json"
+
+    # Read existing config
+    cfg: dict = {}
+    if config_path.exists():
+        try:
+            cfg = json.loads(config_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    if action == "get":
+        # Parse dot-notation key (e.g. "evolution.enabled")
+        parts = key.split(".") if key else []
+        node = cfg
+        for part in parts:
+            if isinstance(node, dict) and part in node:
+                node = node[part]
+            else:
+                node = None
+                break
+
+        if use_json:
+            from superlocalmemory.cli.json_output import json_print
+            json_print("config", data={"key": key, "value": node})
+        else:
+            if node is None:
+                print(f"{key}: (not set)")
+            else:
+                print(f"{key}: {node}")
+
+    elif action == "set":
+        _ALLOWED_CONFIG_KEYS = {
+            "evolution.enabled", "evolution.backend", "evolution.max_evolutions_per_cycle",
+            "mesh_enabled", "daemon_idle_timeout", "entity_compilation_enabled",
+        }
+        if key not in _ALLOWED_CONFIG_KEYS:
+            if use_json:
+                from superlocalmemory.cli.json_output import json_print
+                json_print("config", error={
+                    "code": "DISALLOWED_KEY",
+                    "message": f"'{key}' is not a configurable key. Allowed: {', '.join(sorted(_ALLOWED_CONFIG_KEYS))}",
+                })
+            else:
+                print(f"Error: '{key}' is not a configurable key. Allowed: {', '.join(sorted(_ALLOWED_CONFIG_KEYS))}")
+            sys.exit(1)
+
+        if not key or value is None:
+            if use_json:
+                from superlocalmemory.cli.json_output import json_print
+                json_print("config", error={
+                    "code": "INVALID_INPUT",
+                    "message": "Usage: slm config set <key> <value>",
+                })
+            else:
+                print("Usage: slm config set <key> <value>")
+            sys.exit(1)
+
+        # Parse value: booleans, numbers, strings
+        parsed_value: object
+        if value.lower() in ("true", "yes", "on"):
+            parsed_value = True
+        elif value.lower() in ("false", "no", "off"):
+            parsed_value = False
+        else:
+            try:
+                parsed_value = int(value)
+            except ValueError:
+                try:
+                    parsed_value = float(value)
+                except ValueError:
+                    parsed_value = value
+
+        # Set via dot-notation (e.g. "evolution.enabled" -> cfg["evolution"]["enabled"])
+        parts = key.split(".")
+        node = cfg
+        for part in parts[:-1]:
+            if part not in node or not isinstance(node.get(part), dict):
+                node[part] = {}
+            node = node[part]
+        old_value = node.get(parts[-1])
+        node[parts[-1]] = parsed_value
+
+        # Write back
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        config_path.write_text(json.dumps(cfg, indent=2) + "\n")
+
+        if use_json:
+            from superlocalmemory.cli.json_output import json_print
+            json_print("config", data={
+                "key": key, "old_value": old_value, "new_value": parsed_value,
+            })
+        else:
+            print(f"{key}: {old_value} -> {parsed_value}")
+
+    else:
+        if use_json:
+            from superlocalmemory.cli.json_output import json_print
+            json_print("config", error={
+                "code": "UNKNOWN_ACTION",
+                "message": f"Unknown action: {action}. Use 'get' or 'set'.",
+            })
+        else:
+            print(f"Unknown config action: {action}. Use 'get' or 'set'.")
+        sys.exit(1)
+
+
+def cmd_evolve(args: Namespace) -> None:
+    """Run skill evolution for a session (called from Stop hook).
+
+    Reads config.json to check if evolution is enabled.
+    If enabled, imports SkillEvolver and runs run_post_session().
+    If disabled, exits silently (zero output for fire-and-forget).
+    """
+    import json
+    from pathlib import Path
+
+    session_id = getattr(args, "session", "") or ""
+    profile = getattr(args, "profile", "default") or "default"
+
+    if not session_id:
+        return  # Silent exit — nothing to do without a session
+
+    # Check if evolution is enabled via config.json
+    config_path = Path.home() / ".superlocalmemory" / "config.json"
+    try:
+        cfg = json.loads(config_path.read_text()) if config_path.exists() else {}
+    except (json.JSONDecodeError, OSError):
+        return  # Config unreadable — silent exit
+
+    evolution_cfg = cfg.get("evolution", {})
+    if not evolution_cfg.get("enabled", False):
+        return  # Disabled — silent exit
+
+    # Heavy imports only if enabled (this runs as a Popen child)
+    try:
+        from superlocalmemory.evolution.skill_evolver import SkillEvolver
+
+        db_path = Path.home() / ".superlocalmemory" / "memory.db"
+        if not db_path.exists():
+            return
+
+        evolver = SkillEvolver(db_path)
+        evolver.run_post_session(session_id, profile)
+    except Exception:
+        pass  # Best-effort — don't crash the Stop hook
+
+
 def cmd_adapters(args: Namespace) -> None:
     """Manage ingestion adapters (Gmail, Calendar, Transcript).
 

package/src/superlocalmemory/cli/ingest_cmd.py

@@ -76,28 +76,43 @@ def _error(msg: str, use_json: bool) -> None:
 
 
 def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
-    """Ingest ECC session summaries from transcript JSONL files.
+    """Ingest ECC observations into SLM tool_events.
 
-    Scans the Claude projects directory for session JSONL files and
-    extracts tool usage patterns from them.
+    Scans TWO sources:
+    1. ECC observation files (~/.claude/homunculus/projects/*/observations.jsonl)
+       — rich data: tool input/output, session_id, project context
+    2. Claude session transcript files (~/.claude/projects/*.jsonl)
+       — fallback: tool_use blocks from conversation history
+
+    v3.4.10: Now preserves input_summary and output_summary from ECC observations.
     """
     result = {"source": "ecc", "ingested": 0, "skipped": 0, "dry_run": dry_run}
 
-    # Find ECC session files
+    files: list[Path] = []
+
     if file_path:
         files = [Path(file_path)]
     else:
-        # Auto-discover: scan Claude project session files
+        # Source 1: ECC observation files (RICH data — preferred)
+        ecc_dir = Path.home() / ".claude" / "homunculus" / "projects"
+        if ecc_dir.exists():
+            ecc_files = sorted(
+                ecc_dir.rglob("observations.jsonl"),
+                key=lambda p: p.stat().st_mtime, reverse=True,
+            )
+            files.extend(ecc_files[:20])
+
+        # Source 2: Claude session transcripts (fallback)
         claude_dir = Path.home() / ".claude" / "projects"
-        if not claude_dir.exists():
-            result["error"] = f"Claude projects dir not found: {claude_dir}"
-            return result
-        files = sorted(claude_dir.rglob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True)
-        # Limit to recent files (last 20)
-        files = files[:20]
+        if claude_dir.exists():
+            claude_files = sorted(
+                claude_dir.rglob("*.jsonl"),
+                key=lambda p: p.stat().st_mtime, reverse=True,
+            )
+            files.extend(claude_files[:20])
 
     if not files:
-        result["error"] = "No session files found"
+        result["error"] = "No ECC observation or session files found"
         return result
 
     result["files_scanned"] = len(files)
@@ -116,7 +131,6 @@ def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
                         result["skipped"] += 1
                         continue
 
-                    # Extract tool usage from ECC session records
                     extracted = _extract_tool_events_from_record(record)
                     events.extend(extracted)
         except (OSError, PermissionError):
@@ -128,7 +142,6 @@ def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
         result["sample"] = events[:5]
         return result
 
-    # Write to tool_events table
     if events:
         ingested = _write_tool_events(events)
         result["ingested"] = ingested
@@ -139,47 +152,79 @@ def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
 
 
 def _extract_tool_events_from_record(record: dict) -> list[dict]:
-    """Extract tool events from a single ECC/Claude session JSONL record."""
+    """Extract tool events from a single ECC/Claude session JSONL record.
+
+    Handles three formats:
+    1. ECC observation format: {"event": "tool_complete", "tool": "X", "input": "...", "output": "..."}
+    2. Claude transcript format: {"type": "assistant", "content": [{"type": "tool_use", ...}]}
+    3. Direct tool event format: {"tool_name": "X", "event_type": "complete"}
+
+    v3.4.10: Preserves input_summary and output_summary from all formats.
+    """
     events = []
+    now_iso = datetime.now(timezone.utc).isoformat()
+
+    # Format 1: ECC observation format (from ~/.claude/homunculus/projects/*/observations.jsonl)
+    if "event" in record and "tool" in record:
+        event_type_raw = record.get("event", "")
+        if event_type_raw in ("tool_complete", "tool_start"):
+            event_type = "complete" if event_type_raw == "tool_complete" else "invoke"
+            events.append({
+                "tool_name": record["tool"],
+                "event_type": event_type,
+                "input_summary": str(record.get("input", ""))[:500],
+                "output_summary": str(record.get("output", ""))[:500],
+                "session_id": record.get("session", "ecc_import"),
+                "project_path": record.get("project_name", ""),
+                "created_at": record.get("timestamp", now_iso),
+            })
+        return events
 
-    # Handle ECC summary format
+    # Format 2: Claude transcript format
     if "type" in record:
         rtype = record.get("type", "")
-
-        # Tool use records
         if rtype == "assistant" and "content" in record:
             content = record.get("content", [])
             if isinstance(content, list):
                 for block in content:
                     if isinstance(block, dict) and block.get("type") == "tool_use":
                         tool_name = block.get("name", "unknown")
+                        raw_input = block.get("input", {})
+                        input_str = json.dumps(raw_input, default=str)[:500] if isinstance(raw_input, dict) else str(raw_input)[:500]
                         events.append({
                             "tool_name": tool_name,
                             "event_type": "complete",
+                            "input_summary": input_str,
+                            "output_summary": "",
                             "session_id": record.get("session_id", "ecc_import"),
-                            "created_at": record.get("timestamp", datetime.now(timezone.utc).isoformat()),
+                            "project_path": "",
+                            "created_at": record.get("timestamp", now_iso),
                         })
-
-            # Also extract from tool_use type directly
-            if isinstance(content, list):
-                for block in content:
-                    if isinstance(block, dict) and block.get("type") == "tool_result":
+                    elif isinstance(block, dict) and block.get("type") == "tool_result":
                         tool_name = block.get("tool_use_id", "unknown")
                         is_error = block.get("is_error", False)
+                        raw_content = block.get("content", "")
+                        output_str = str(raw_content)[:500] if raw_content else ""
                         events.append({
                             "tool_name": tool_name,
                             "event_type": "error" if is_error else "complete",
+                            "input_summary": "",
+                            "output_summary": output_str,
                             "session_id": record.get("session_id", "ecc_import"),
-                            "created_at": record.get("timestamp", datetime.now(timezone.utc).isoformat()),
+                            "project_path": "",
+                            "created_at": record.get("timestamp", now_iso),
                         })
 
-    # Handle direct tool event format (from hook output)
+    # Format 3: Direct tool event format (from hook output)
     if "tool_name" in record and "event_type" in record:
         events.append({
             "tool_name": record["tool_name"],
             "event_type": record.get("event_type", "complete"),
+            "input_summary": str(record.get("input_summary", ""))[:500],
+            "output_summary": str(record.get("output_summary", ""))[:500],
             "session_id": record.get("session_id", "ecc_import"),
-            "created_at": record.get("created_at", datetime.now(timezone.utc).isoformat()),
+            "project_path": record.get("project_path", ""),
+            "created_at": record.get("created_at", now_iso),
         })
 
     return events
@@ -228,7 +273,11 @@ def _ingest_jsonl(file_path: str, *, dry_run: bool = False) -> dict:
 
 
 def _write_tool_events(events: list[dict]) -> int:
-    """Write tool events to SLM's memory.db tool_events table."""
+    """Write tool events to SLM's memory.db tool_events table.
+
+    v3.4.10: Preserves input_summary, output_summary, and project_path
+    from enriched sources (ECC observations, enriched hook).
+    """
     db_path = MEMORY_DB
     if not db_path.exists():
         return 0
@@ -243,11 +292,14 @@ def _write_tool_events(events: list[dict]) -> int:
                     "INSERT INTO tool_events "
                     "(session_id, profile_id, project_path, tool_name, event_type, "
                     " input_summary, output_summary, duration_ms, metadata, created_at) "
-                    "VALUES (?, 'default', '', ?, ?, '', '', 0, '{}', ?)",
+                    "VALUES (?, 'default', ?, ?, ?, ?, ?, 0, '{}', ?)",
                     (
                         ev.get("session_id", "import"),
+                        ev.get("project_path", ""),
                         ev["tool_name"],
                         ev.get("event_type", "complete"),
+                        ev.get("input_summary", ""),
+                        ev.get("output_summary", ""),
                         ev.get("created_at", datetime.now(timezone.utc).isoformat()),
                     ),
                 )

package/src/superlocalmemory/cli/main.py

@@ -199,6 +199,17 @@ def main() -> None:
         help="start (default), stop, status, install (OS service), uninstall",
     )
 
+    # V3.4.9: Full system restart with health verification
+    restart_p = sub.add_parser(
+        "restart",
+        help="Nuclear restart: kill orphans, clean state, start fresh, verify health",
+    )
+    restart_p.add_argument(
+        "--dashboard", action="store_true",
+        help="Open dashboard after restart",
+    )
+    restart_p.add_argument("--json", action="store_true", help="Output structured JSON")
+
     # -- Profiles ------------------------------------------------------
     profile_p = sub.add_parser("profile", help="Profile management (list/switch/create)")
     profile_p.add_argument(
@@ -302,6 +313,26 @@ def main() -> None:
     )
     ingest_p.add_argument("--json", action="store_true", help="Output structured JSON")
 
+    # V3.4.11: Config get/set (dot-notation)
+    config_p = sub.add_parser(
+        "config",
+        help="Get or set config values (e.g. slm config set evolution.enabled true)",
+    )
+    config_p.add_argument(
+        "action", choices=["get", "set"], help="Action: get or set",
+    )
+    config_p.add_argument("key", help="Config key in dot notation (e.g. evolution.enabled)")
+    config_p.add_argument("value", nargs="?", default=None, help="Value to set (for 'set' action)")
+    config_p.add_argument("--json", action="store_true", help="Output structured JSON")
+
+    # V3.4.11: Skill evolution (called from Stop hook, fire-and-forget)
+    evolve_p = sub.add_parser(
+        "evolve",
+        help="Run post-session skill evolution (internal, called by Stop hook)",
+    )
+    evolve_p.add_argument("--session", default="", help="Session ID to process")
+    evolve_p.add_argument("--profile", default="default", help="Profile ID")
+
     args = parser.parse_args()
 
     if not args.command:
@@ -317,6 +348,7 @@ def main() -> None:
     # Cross-platform: macOS + Windows + Linux.
     _NO_DAEMON_COMMANDS = {
         "setup", "mode", "provider", "connect", "migrate", "mcp", "warmup",
+        "config", "evolve",
     }
     if args.command not in _NO_DAEMON_COMMANDS:
         try: