superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/cli/ingest_cmd.py

@@ -1,327 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""CLI handler for `slm ingest` — import external observations into SLM.
-
-Supported sources:
-  --source ecc    Import ECC (Everything Claude Code) session summaries
-  --source jsonl  Import generic JSONL observations
-
-Each imported record becomes a tool_event in the behavioral learning pipeline,
-so the AssertionMiner can learn from them.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import sqlite3
-import sys
-from argparse import Namespace
-from datetime import datetime, timezone
-from pathlib import Path
-
-from superlocalmemory.infra.data_root import DynamicStatePath
-
-logger = logging.getLogger(__name__)
-
-# Backward-compatible public paths that resolve at call time instead of
-# freezing Path.home() during module import.
-MEMORY_DIR = DynamicStatePath()
-MEMORY_DB = DynamicStatePath("memory.db")
-
-
-def cmd_ingest(args: Namespace) -> None:
-    """Ingest external observations into SLM tool_events table."""
-    source = getattr(args, "source", "ecc")
-    file_path = getattr(args, "file", "")
-    use_json = getattr(args, "json", False)
-    dry_run = getattr(args, "dry_run", False)
-
-    if source == "ecc":
-        result = _ingest_ecc(file_path, dry_run=dry_run)
-    elif source == "jsonl":
-        if not file_path:
-            _error("--file required for jsonl source", use_json)
-            return
-        result = _ingest_jsonl(file_path, dry_run=dry_run)
-    else:
-        _error(f"Unknown source: {source}", use_json)
-        return
-
-    if use_json:
-        from superlocalmemory.cli.json_output import json_print
-        json_print("ingest", data=result, next_actions=[
-            {"command": "slm consolidate --cognitive", "description": "Run consolidation to mine assertions"},
-        ])
-        return
-
-    if result.get("error"):
-        print(f"Error: {result['error']}")
-        sys.exit(1)
-
-    print(f"Ingested: {result['ingested']} events from {source}")
-    if result.get("skipped"):
-        print(f"Skipped:  {result['skipped']} (duplicates/invalid)")
-    if dry_run:
-        print("(dry run — no data written)")
-
-
-def _error(msg: str, use_json: bool) -> None:
-    if use_json:
-        from superlocalmemory.cli.json_output import json_print
-        json_print("ingest", error={"code": "INGEST_ERROR", "message": msg})
-    else:
-        print(f"Error: {msg}")
-    sys.exit(1)
-
-
-def _ingest_ecc(file_path: str, *, dry_run: bool = False) -> dict:
-    """Ingest ECC observations into SLM tool_events.
-
-    Scans TWO sources:
-    1. ECC observation files (~/.claude/homunculus/projects/*/observations.jsonl)
-       — rich data: tool input/output, session_id, project context
-    2. Claude session transcript files (~/.claude/projects/*.jsonl)
-       — fallback: tool_use blocks from conversation history
-
-    v3.4.10: Now preserves input_summary and output_summary from ECC observations.
-    """
-    result = {"source": "ecc", "ingested": 0, "skipped": 0, "dry_run": dry_run}
-
-    files: list[Path] = []
-
-    if file_path:
-        files = [Path(file_path)]
-    else:
-        # Source 1: ECC observation files (RICH data — preferred)
-        ecc_dir = Path.home() / ".claude" / "homunculus" / "projects"
-        if ecc_dir.exists():
-            ecc_files = sorted(
-                ecc_dir.rglob("observations.jsonl"),
-                key=lambda p: p.stat().st_mtime, reverse=True,
-            )
-            files.extend(ecc_files[:20])
-
-        # Source 2: Claude session transcripts (fallback)
-        claude_dir = Path.home() / ".claude" / "projects"
-        if claude_dir.exists():
-            claude_files = sorted(
-                claude_dir.rglob("*.jsonl"),
-                key=lambda p: p.stat().st_mtime, reverse=True,
-            )
-            files.extend(claude_files[:20])
-
-    if not files:
-        result["error"] = "No ECC observation or session files found"
-        return result
-
-    result["files_scanned"] = len(files)
-    events = []
-
-    for fpath in files:
-        try:
-            with open(fpath) as f:
-                for line in f:
-                    line = line.strip()
-                    if not line:
-                        continue
-                    try:
-                        record = json.loads(line)
-                    except json.JSONDecodeError:
-                        result["skipped"] += 1
-                        continue
-
-                    extracted = _extract_tool_events_from_record(record)
-                    events.extend(extracted)
-        except (OSError, PermissionError):
-            result["skipped"] += 1
-            continue
-
-    if dry_run:
-        result["ingested"] = len(events)
-        result["sample"] = events[:5]
-        return result
-
-    if events:
-        ingested = _write_tool_events(events)
-        result["ingested"] = ingested
-    else:
-        result["ingested"] = 0
-
-    return result
-
-
-def _extract_tool_events_from_record(record: dict) -> list[dict]:
-    """Extract tool events from a single ECC/Claude session JSONL record.
-
-    Handles three formats:
-    1. ECC observation format: {"event": "tool_complete", "tool": "X", "input": "...", "output": "..."}
-    2. Claude transcript format: {"type": "assistant", "content": [{"type": "tool_use", ...}]}
-    3. Direct tool event format: {"tool_name": "X", "event_type": "complete"}
-
-    v3.4.10: Preserves input_summary and output_summary from all formats.
-    """
-    events = []
-    now_iso = datetime.now(timezone.utc).isoformat()
-
-    # Format 1: ECC observation format (from ~/.claude/homunculus/projects/*/observations.jsonl)
-    if "event" in record and "tool" in record:
-        event_type_raw = record.get("event", "")
-        if event_type_raw in ("tool_complete", "tool_start"):
-            event_type = "complete" if event_type_raw == "tool_complete" else "invoke"
-            events.append({
-                "tool_name": record["tool"],
-                "event_type": event_type,
-                "input_summary": str(record.get("input", ""))[:500],
-                "output_summary": str(record.get("output", ""))[:500],
-                "session_id": record.get("session", "ecc_import"),
-                "project_path": record.get("project_name", ""),
-                "created_at": record.get("timestamp", now_iso),
-            })
-        return events
-
-    # Format 2: Claude transcript format
-    if "type" in record:
-        rtype = record.get("type", "")
-        if rtype == "assistant" and "content" in record:
-            content = record.get("content", [])
-            if isinstance(content, list):
-                for block in content:
-                    if isinstance(block, dict) and block.get("type") == "tool_use":
-                        tool_name = block.get("name", "unknown")
-                        raw_input = block.get("input", {})
-                        input_str = json.dumps(raw_input, default=str)[:500] if isinstance(raw_input, dict) else str(raw_input)[:500]
-                        events.append({
-                            "tool_name": tool_name,
-                            "event_type": "complete",
-                            "input_summary": input_str,
-                            "output_summary": "",
-                            "session_id": record.get("session_id", "ecc_import"),
-                            "project_path": "",
-                            "created_at": record.get("timestamp", now_iso),
-                        })
-                    elif isinstance(block, dict) and block.get("type") == "tool_result":
-                        tool_name = block.get("tool_use_id", "unknown")
-                        is_error = block.get("is_error", False)
-                        raw_content = block.get("content", "")
-                        output_str = str(raw_content)[:500] if raw_content else ""
-                        events.append({
-                            "tool_name": tool_name,
-                            "event_type": "error" if is_error else "complete",
-                            "input_summary": "",
-                            "output_summary": output_str,
-                            "session_id": record.get("session_id", "ecc_import"),
-                            "project_path": "",
-                            "created_at": record.get("timestamp", now_iso),
-                        })
-
-    # Format 3: Direct tool event format (from hook output)
-    if "tool_name" in record and "event_type" in record:
-        events.append({
-            "tool_name": record["tool_name"],
-            "event_type": record.get("event_type", "complete"),
-            "input_summary": str(record.get("input_summary", ""))[:500],
-            "output_summary": str(record.get("output_summary", ""))[:500],
-            "session_id": record.get("session_id", "ecc_import"),
-            "project_path": record.get("project_path", ""),
-            "created_at": record.get("created_at", now_iso),
-        })
-
-    return events
-
-
-def _ingest_jsonl(file_path: str, *, dry_run: bool = False) -> dict:
-    """Ingest generic JSONL file with tool event records."""
-    result = {"source": "jsonl", "ingested": 0, "skipped": 0, "dry_run": dry_run}
-
-    fpath = Path(file_path)
-    if not fpath.exists():
-        result["error"] = f"File not found: {file_path}"
-        return result
-
-    events = []
-    with open(fpath) as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                record = json.loads(line)
-            except json.JSONDecodeError:
-                result["skipped"] += 1
-                continue
-
-            if "tool_name" not in record:
-                result["skipped"] += 1
-                continue
-
-            events.append({
-                "tool_name": record["tool_name"],
-                "event_type": record.get("event_type", "complete"),
-                "session_id": record.get("session_id", "jsonl_import"),
-                "created_at": record.get("created_at", datetime.now(timezone.utc).isoformat()),
-            })
-
-    if dry_run:
-        result["ingested"] = len(events)
-        return result
-
-    if events:
-        result["ingested"] = _write_tool_events(events)
-
-    return result
-
-
-def _write_tool_events(events: list[dict]) -> int:
-    """Write tool events to SLM's memory.db tool_events table.
-
-    v3.4.10: Preserves input_summary, output_summary, and project_path
-    from enriched sources (ECC observations, enriched hook).
-
-    Concurrency note
-    ----------------
-    ``slm ingest`` is a SEPARATE OS process — the daemon's threading.RLock
-    write-lock cannot help here.  PRAGMA busy_timeout is the only cross-process
-    lever: it makes SQLite WAIT for the single writer (daemon / hook) to finish
-    instead of immediately raising SQLITE_BUSY.  The ``timeout`` kwarg to
-    sqlite3.connect is the Python-level retry budget (same duration).
-    """
-    db_path = Path(MEMORY_DB)
-    if not db_path.exists():
-        return 0
-
-    _busy_ms = 10_000
-    conn = sqlite3.connect(str(db_path), timeout=_busy_ms / 1000.0)
-    count = 0
-
-    try:
-        conn.execute(f"PRAGMA busy_timeout={_busy_ms}")
-        for ev in events:
-            try:
-                conn.execute(
-                    "INSERT INTO tool_events "
-                    "(session_id, profile_id, project_path, tool_name, event_type, "
-                    " input_summary, output_summary, duration_ms, metadata, created_at) "
-                    "VALUES (?, 'default', ?, ?, ?, ?, ?, 0, '{}', ?)",
-                    (
-                        ev.get("session_id", "import"),
-                        ev.get("project_path", ""),
-                        ev["tool_name"],
-                        ev.get("event_type", "complete"),
-                        ev.get("input_summary", ""),
-                        ev.get("output_summary", ""),
-                        ev.get("created_at", datetime.now(timezone.utc).isoformat()),
-                    ),
-                )
-                count += 1
-            except sqlite3.Error:
-                continue
-        conn.commit()
-    finally:
-        conn.close()
-
-    return count

package/src/superlocalmemory/cli/json_output.py

@@ -1,81 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Shared JSON envelope for agent-native CLI output.
-
-Follows the 2026 agent-native CLI standard:
-- Consistent envelope: success, command, version, data/error
-- HATEOAS next_actions for agent guidance
-- Metadata for execution context
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import json
-
-
-def _get_version() -> str:
-    """Read version from package.json (npm), pyproject.toml, or metadata."""
-    from pathlib import Path
-    pkg_root = Path(__file__).resolve().parent.parent.parent.parent
-    # 1. package.json (npm installs)
-    try:
-        pkg_json = pkg_root / "package.json"
-        if pkg_json.exists():
-            with open(pkg_json) as f:
-                v = json.load(f).get("version", "")
-                if v:
-                    return v
-    except Exception:
-        pass
-    # 2. pyproject.toml (pip installs)
-    try:
-        import tomllib
-        toml_path = pkg_root / "pyproject.toml"
-        if toml_path.exists():
-            with open(toml_path, "rb") as f:
-                return tomllib.load(f)["project"]["version"]
-    except Exception:
-        pass
-    # 3. importlib.metadata fallback
-    try:
-        from importlib.metadata import version
-        return version("superlocalmemory")
-    except Exception:
-        pass
-    return "unknown"
-
-
-def json_print(
-    command: str,
-    *,
-    data: dict | None = None,
-    error: dict | None = None,
-    next_actions: list[dict] | None = None,
-    metadata: dict | None = None,
-) -> None:
-    """Print a standard JSON envelope to stdout.
-
-    Success envelope:
-        {"success": true, "command": "...", "version": "...", "data": {...}}
-
-    Error envelope:
-        {"success": false, "command": "...", "version": "...", "error": {...}}
-    """
-    envelope: dict = {
-        "success": error is None,
-        "command": command,
-        "version": _get_version(),
-    }
-    if error is not None:
-        envelope["error"] = error
-    else:
-        envelope["data"] = data if data is not None else {}
-    if metadata:
-        envelope["metadata"] = metadata
-    if next_actions:
-        envelope["next_actions"] = next_actions
-    print(json.dumps(envelope, indent=2, default=str))

package/src/superlocalmemory/cli/loop_cmd.py

@@ -1,187 +0,0 @@
-"""``slm loop`` — inspect and demonstrate bounded loops backed by SLM memory.
-
-Subcommands:
-
-* ``slm loop demo``            run a built-in, keyless convergence demo (stub
-                              proposer + deterministic gate) and persist every
-                              lap to SLM memory. Proves the engine + ledger
-                              end to end without a credentialed agent.
-* ``slm loop history [--name]`` list recorded loop runs from SLM memory.
-* ``slm loop show <run_id>``   show every lap of one run.
-
-The durable value here is that a loop's history lives in the same SLM data
-root as everything else the agent remembers — queryable via ``slm recall`` and
-visible in the dashboard. Reads and the demo run through an in-process engine
-store rooted at the active data root (``SLM_DATA_DIR`` or
-``~/.superlocalmemory``); point ``SLM_DATA_DIR`` elsewhere to sandbox a run.
-"""
-
-from __future__ import annotations
-
-import json
-from argparse import Namespace
-from pathlib import Path
-from typing import Any
-
-from superlocalmemory.infra.data_root import canonical_data_root
-from superlocalmemory.loops import (
-    Bounds,
-    LapResult,
-    SLMMemoryLedger,
-    Verdict,
-    open_engine_store,
-    run_bounded_loop,
-)
-
-
-def _data_root() -> Path:
-    return canonical_data_root()
-
-
-class _FailOpenLedger:
-    """Wrap a ledger so a memory write hiccup never aborts a running loop.
-
-    A ledger is observability; the loop's correctness comes from the gate.
-    Write errors are counted and surfaced after the run rather than raised
-    mid-flight, so we never silently pretend they did not happen.
-    """
-
-    def __init__(self, inner: Any) -> None:
-        self._inner = inner
-        self.write_errors: list[str] = []
-
-    def record(self, entry: Any) -> None:
-        try:
-            self._inner.record(entry)
-        except Exception as exc:  # pragma: no cover - defensive path
-            self.write_errors.append(f"lap {getattr(entry, 'lap', '?')}: {exc}")
-
-    def laps(self, run_id: str) -> list:
-        return self._inner.laps(run_id)
-
-    def runs(self, name: str) -> list:
-        return self._inner.runs(name)
-
-
-def _open_ledger() -> tuple[SLMMemoryLedger, Any]:
-    store = open_engine_store(_data_root() / "memory.db")
-    return SLMMemoryLedger(store), store
-
-
-def cmd_loop(args: Namespace) -> None:
-    action = getattr(args, "loop_command", None)
-    if action == "demo":
-        _cmd_demo(args)
-    elif action == "history":
-        _cmd_history(args)
-    elif action == "show":
-        _cmd_show(args)
-    else:
-        print("Usage: slm loop {demo|history|show} [options]")
-
-
-def _cmd_demo(args: Namespace) -> None:
-    """Run the convergence demo: the gate fails twice, then passes on lap 3."""
-    iterations = int(getattr(args, "iterations", 10) or 10)
-    as_json = bool(getattr(args, "json", False))
-    pass_on = 3
-
-    ledger, store = _open_ledger()
-    guarded = _FailOpenLedger(ledger)
-    try:
-        outcome = run_bounded_loop(
-            "convergence-demo",
-            bounds=Bounds(max_iterations=iterations),
-            runner=lambda lap: LapResult(changed=True, tokens=8),
-            gate=lambda lap: Verdict(lap >= pass_on, f"demo gate: lap {lap}"),
-            ledger=guarded,
-        )
-        laps = guarded.laps(outcome.run_id)
-    finally:
-        store.close()
-
-    if as_json:
-        print(json.dumps({
-            "status": outcome.status.value,
-            "reason": outcome.reason,
-            "laps": outcome.laps,
-            "run_id": outcome.run_id,
-            "ledger": [
-                {"lap": e.lap, "decision": e.decision, "passed": e.passed}
-                for e in laps
-            ],
-            "write_errors": guarded.write_errors,
-        }, indent=2))
-        return
-
-    mark = "✓" if outcome.ok else "✗"
-    print(f"{mark} [{outcome.status.value}] {outcome.reason} (laps: {outcome.laps})")
-    for e in laps:
-        gate = "gate-pass" if e.passed else "gate-fail"
-        print(f"   lap {e.lap}: {e.decision:<8} {gate}  {e.detail}")
-    print(f"run_id: {outcome.run_id}  (recall with tag loop:convergence-demo)")
-    if guarded.write_errors:
-        print(f"WARNING: {len(guarded.write_errors)} ledger write error(s): "
-              f"{guarded.write_errors[0]}")
-
-
-def _cmd_history(args: Namespace) -> None:
-    name = getattr(args, "name", None) or "convergence-demo"
-    as_json = bool(getattr(args, "json", False))
-    ledger, store = _open_ledger()
-    try:
-        run_ids = ledger.runs(name)
-        rows = []
-        for rid in run_ids:
-            laps = ledger.laps(rid)
-            last = laps[-1] if laps else None
-            rows.append({
-                "run_id": rid,
-                "laps": len(laps),
-                "final": last.decision if last else "unknown",
-                "ts": last.ts if last else "",
-            })
-    finally:
-        store.close()
-
-    if as_json:
-        print(json.dumps({"name": name, "runs": rows}, indent=2))
-        return
-    if not rows:
-        print(f"No recorded runs for loop '{name}'.")
-        return
-    print(f"Runs for loop '{name}':")
-    for r in rows:
-        print(f"  {r['run_id']:<28} laps={r['laps']:<3} final={r['final']:<8} {r['ts']}")
-
-
-def _cmd_show(args: Namespace) -> None:
-    run_id = getattr(args, "run_id", None)
-    as_json = bool(getattr(args, "json", False))
-    if not run_id:
-        print("Usage: slm loop show <run_id>")
-        return
-    ledger, store = _open_ledger()
-    try:
-        laps = ledger.laps(run_id)
-    finally:
-        store.close()
-
-    if as_json:
-        print(json.dumps({
-            "run_id": run_id,
-            "laps": [
-                {"lap": e.lap, "ts": e.ts, "decision": e.decision,
-                 "passed": e.passed, "detail": e.detail, "budget": e.budget}
-                for e in laps
-            ],
-        }, indent=2))
-        return
-    if not laps:
-        print(f"No ledger entries for run '{run_id}'.")
-        return
-    print(f"Loop run {run_id} ({laps[0].name}):")
-    for e in laps:
-        gate = "gate-pass" if e.passed else "gate-fail"
-        print(f"  lap {e.lap}: {e.decision:<8} {gate}  {e.detail}  "
-              f"[tokens={e.budget.get('tokens', 0)}]")