superlocalmemory 4.1.6 → 4.1.8

package/src/superlocalmemory/hooks/codex_assets.py

@@ -2,6 +2,9 @@
 
 from __future__ import annotations
 
+import hashlib
+import json
+import re
 import shutil
 import sysconfig
 from pathlib import Path
@@ -24,11 +27,31 @@ SKILLS = (
 # Codex subagent files written to ~/.codex/agents (content built by _agent_files()).
 AGENTS = ("slm-memory-advisor.toml", "slm-optimize-advisor.toml")
 
-_MEMORY_ADVISOR_TOML = (
-    'name = "slm-memory-advisor"\n'
-    'description = "Use SuperLocalMemory safely: initialize once, recall before remember, and store only durable atomic facts."\n'
-    'instructions = "Use SLM for memory discipline only. Check results before claiming success; preserve private scope unless the user explicitly asks to share."\n'
-)
+#: Digests of the agent files this installer last wrote, kept beside them.  An
+#: agent file whose current content still matches its recorded digest is ours to
+#: refresh; anything else belongs to whoever changed it and is left alone.  A
+#: file with no recorded digest predates this manifest and is treated as theirs,
+#: which is the safe reading: this installer once replaced two hand-maintained
+#: 4.9 KB advisors with its own one-line stubs and there was nothing to restore
+#: from.
+MANIFEST_NAME = ".slm-managed.json"
+
+#: Used only when the advisor source document is unavailable — an installation
+#: that ships no agent sources still gets a usable, if terse, subagent.
+_FALLBACKS = {
+    "slm-memory-advisor.toml": (
+        "Use SuperLocalMemory safely: initialize once, recall before remember, "
+        "and store only durable atomic facts.",
+        "Use SLM for memory discipline only. Check results before claiming success; "
+        "preserve private scope unless the user explicitly asks to share.",
+    ),
+    "slm-optimize-advisor.toml": (
+        "Apply SuperLocalMemory's no-proxy context-optimization rules — reversible "
+        "compression of large tool output and KV-caching of repeated reads/searches.",
+        "Reduce context-window pressure with the Surface-B tools (reversible CCR "
+        "compression + a per-agent KV cache); fail-open — never block the task.",
+    ),
+}
 
 
 def _source_root() -> Path:
@@ -49,67 +72,164 @@ def _agents_source_root() -> Path | None:
     return installed if installed.exists() else None
 
 
-def _optimize_advisor_toml() -> str:
-    """Build the optimize-advisor TOML from the canonical advisor doc so Codex
-    ships the FULL decision rules (the 8-rule tree), not a one-line stub. Falls
-    back to a short instruction only if the source doc is unavailable.
+def _split_frontmatter(text: str) -> tuple[str, str]:
+    """Return (frontmatter, body); frontmatter is "" when the doc has none."""
+    if not text.startswith("---"):
+        return "", text.strip()
+    end = text.find("\n---", 3)
+    if end == -1:
+        return "", text.strip()
+    return text[3:end], text[end + 4:].strip()
+
+
+def _description(frontmatter: str, default: str) -> str:
+    """Read `description:`, joining YAML folded (`>`) blocks onto one line."""
+    folded = re.search(r"^description:\s*>[-+]?\s*\n((?:[ \t]+\S.*\n?)+)", frontmatter, re.M)
+    if folded:
+        return " ".join(line.strip() for line in folded.group(1).splitlines() if line.strip())
+    inline = re.search(r"^description:\s*(\S.*)$", frontmatter, re.M)
+    return inline.group(1).strip() if inline else default
+
+
+def _agent_toml(filename: str) -> str:
+    """Build one subagent's TOML, preferring the canonical advisor document so
+    Codex ships the advisor's full decision rules rather than a summary of them.
     """
-    description = (
-        "Apply SuperLocalMemory's no-proxy context-optimization rules — reversible "
-        "compression of large tool output and KV-caching of repeated reads/searches."
-    )
-    body = ""
+    name = filename.removesuffix(".toml")
+    default_description, default_body = _FALLBACKS[filename]
+    description, body = default_description, default_body
+
     root = _agents_source_root()
     if root is not None:
-        src = root / "slm-optimize-advisor.md"
-        if src.exists():
-            text = src.read_text(encoding="utf-8")
-            if text.startswith("---"):  # strip YAML frontmatter, keep the guidance body
-                end = text.find("\n---", 3)
-                if end != -1:
-                    text = text[end + 4:]
-            body = text.strip()
-    if not body:
-        body = (
-            "Reduce context-window pressure with the Surface-B tools (reversible CCR "
-            "compression + a per-agent KV cache); fail-open — never block the task."
-        )
-    # TOML literal multi-line string ('''...'''): no escape processing, and the
-    # advisor body contains no ''' sequence.
-    return (
-        'name = "slm-optimize-advisor"\n'
-        f'description = "{description}"\n'
-        f"instructions = '''\n{body}\n'''\n"
-    )
+        source = root / f"{name}.md"
+        if source.exists():
+            frontmatter, source_body = _split_frontmatter(source.read_text(encoding="utf-8"))
+            if source_body:
+                description = _description(frontmatter, default_description)
+                body = source_body
+
+    # description is a TOML basic string, so quotes and backslashes must be
+    # escaped — the advisor descriptions really do contain quoted questions.
+    escaped = description.replace("\\", "\\\\").replace('"', '\\"')
+    # A literal multi-line string ('''...''') performs no escape processing,
+    # which keeps the advisor markdown byte-exact.  It cannot contain the
+    # sequence that closes it, so fall back to an escaped basic string in that
+    # case rather than editing the advisor's own text.
+    if "'''" in body:
+        basic = body.replace("\\", "\\\\").replace('"', '\\"')
+        instructions = '"""\n' + basic + '\n"""'
+    else:
+        instructions = "'''\n" + body + "\n'''"
+    return f'name = "{name}"\ndescription = "{escaped}"\ninstructions = {instructions}\n'
 
 
 def _agent_files() -> dict:
     """Return {filename: TOML content} for the Codex subagents."""
-    return {
-        "slm-memory-advisor.toml": _MEMORY_ADVISOR_TOML,
-        "slm-optimize-advisor.toml": _optimize_advisor_toml(),
-    }
+    return {filename: _agent_toml(filename) for filename in AGENTS}
+
+
+def _digest(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
 
+def _read_manifest(agents_root: Path) -> dict:
+    try:
+        loaded = json.loads((agents_root / MANIFEST_NAME).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+    return loaded if isinstance(loaded, dict) else {}
 
-def install_assets(*, home: Path | None = None, dry_run: bool = False) -> dict:
-    """Copy only named SLM assets; never rewrite user-owned assets."""
+
+def _is_ours(target: Path, manifest: dict) -> bool:
+    """True when this installer may overwrite `target`.
+
+    Either it does not exist yet, or its content is byte-identical to what this
+    installer last recorded writing there.
+    """
+    if not target.exists():
+        return True
+    recorded = manifest.get(target.name)
+    if not recorded:
+        return False
+    try:
+        return _digest(target.read_text(encoding="utf-8")) == recorded
+    except OSError:
+        return False
+
+
+def _skills_read_elsewhere(home: Path, skills_root: Path) -> list[str]:
+    """Skill paths Codex may read that this installer does not write.
+
+    Some setups point ~/.codex/skills/<name> at a checkout instead of using the
+    copies under ~/.agents/skills, in which case writing the copies refreshes
+    nothing Codex will load.  Report those paths so the caller can say so rather
+    than claiming a refresh it did not perform.
+    """
+    elsewhere = []
+    for skill in SKILLS:
+        candidate = home / ".codex" / "skills" / skill
+        if not candidate.exists() and not candidate.is_symlink():
+            continue
+        resolved = candidate.resolve() if candidate.is_symlink() else candidate
+        if resolved != (skills_root / skill).resolve():
+            elsewhere.append(str(candidate))
+    return elsewhere
+
+
+def install_assets(*, home: Path | None = None, dry_run: bool = False, force: bool = False) -> dict:
+    """Copy only named SLM assets; never rewrite user-owned assets.
+
+    An agent file that this installer did not write, or that has been edited
+    since it did, is preserved and reported under ``agents_preserved`` rather
+    than overwritten.  Pass ``force=True`` to overwrite anyway, which first
+    copies the existing file aside with a ``.bak`` suffix.
+    """
     home = home or Path.home()
     source = _source_root()
     missing = [skill for skill in SKILLS if not (source / skill / "SKILL.md").exists()]
     if missing:
         return {"success": False, "errors": [f"missing bundled skills: {', '.join(missing)}"]}
-    if dry_run:
-        return {"success": True, "skills": list(SKILLS), "agents": list(AGENTS), "dry_run": True}
+
     skills_root, agents_root = home / ".agents" / "skills", home / ".codex" / "agents"
+    manifest = _read_manifest(agents_root)
+    planned = _agent_files()
+
+    writable, preserved = [], []
+    for filename, content in planned.items():
+        target = agents_root / filename
+        if force or _is_ours(target, manifest):
+            writable.append((filename, content, target))
+        else:
+            preserved.append(str(target))
+
+    skill_targets = [skills_root / skill / "SKILL.md" for skill in SKILLS]
+    result = {
+        "success": True,
+        "dry_run": dry_run,
+        "skills": list(SKILLS),
+        "agents": [filename for filename, _, _ in writable],
+        "skills_written": [str(path) for path in skill_targets],
+        "agents_written": [str(target) for _, _, target in writable],
+        "agents_preserved": preserved,
+        "skills_read_elsewhere": _skills_read_elsewhere(home, skills_root),
+    }
+    if dry_run:
+        return result
+
     skills_root.mkdir(parents=True, exist_ok=True)
     agents_root.mkdir(parents=True, exist_ok=True)
     for skill in SKILLS:
         target = skills_root / skill
         target.mkdir(parents=True, exist_ok=True)
         shutil.copy2(source / skill / "SKILL.md", target / "SKILL.md")
-    for filename, content in _agent_files().items():
-        (agents_root / filename).write_text(content, encoding="utf-8")
-    return {"success": True, "skills": list(SKILLS), "agents": list(AGENTS), "dry_run": False}
+    for filename, content, target in writable:
+        if force and target.exists():
+            shutil.copy2(target, target.with_suffix(target.suffix + ".bak"))
+        target.write_text(content, encoding="utf-8")
+        manifest[filename] = _digest(content)
+    if writable:
+        (agents_root / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+    return result
 
 
 def remove_assets(*, home: Path | None = None, dry_run: bool = False) -> dict:

package/src/superlocalmemory/hooks/post_tool_outcome_hook.py

@@ -83,6 +83,122 @@ def _validate(marker: str) -> str | None:
         return None
 
 
+# --- Behavioural telemetry -------------------------------------------------
+# ``tool_events`` is the record of what the agent did, and several readers
+# depend on it: assertion mining, skill-performance mining, and the engagement
+# features that settle a recall against the actions which followed it. Each of
+# those reads the table; none of them writes it. The only writers were an
+# explicit ``log_tool_event`` call and a bulk importer, so on an ordinary
+# install the table simply stopped receiving invocations and every reader
+# quietly aged out with it.
+#
+# This hook already runs on exactly the right edge -- after each tool completes,
+# with the session that ran it -- so it records the invocation here. It happens
+# before the marker scan and independently of it: whether a recalled memory was
+# named in the output decides what a *reward* is worth, not whether the action
+# occurred at all.
+_MAX_SUMMARY_LEN = 500
+
+# Compiled once at import. This path now runs on every tool call rather than
+# only on the ~1-in-5 that carry a marker, so per-call ``re`` compilation or a
+# module import would be paid on the hot path for no benefit.
+_SECRET_PATTERNS = (
+    (re.compile(r"\b(?:sk-|pk-|api[_-]?key[_-]?)[A-Za-z0-9_-]{10,}\b"),
+     "[REDACTED]"),
+    (re.compile(r"\b[A-Za-z0-9+/]{40,}={0,2}\b"), "[REDACTED]"),
+    (re.compile(r"password\s*[=:]\s*\S+", re.IGNORECASE),
+     "password=[REDACTED]"),
+)
+
+
+def _scrub(text: str) -> str:
+    """Strip credential-shaped substrings from telemetry text."""
+    for pattern, replacement in _SECRET_PATTERNS:
+        text = pattern.sub(replacement, text)
+    return text
+
+
+def _summary(raw: object) -> str:
+    """Truncate-then-scrub ``raw`` for a summary column.
+
+    Truncation comes first so the regex cost is bounded by
+    ``_MAX_SUMMARY_LEN`` and not by the size of a tool response.
+    """
+    if raw is None:
+        return ""
+    if not isinstance(raw, str):
+        try:
+            import json as _json
+            raw = _json.dumps(raw, default=str)
+        except Exception:
+            try:
+                raw = str(raw)
+            except Exception:
+                return ""
+    return _scrub(raw[:_MAX_SUMMARY_LEN])
+
+
+def _record_tool_event(
+    session_id: str,
+    tool_name: str,
+    payload: dict,
+    response_text: str,
+) -> bool:
+    """Append one ``tool_events`` row. Never raises; returns whether it wrote.
+
+    Failure is silent by design: telemetry must never be the reason a tool call
+    reports a problem to the user, and the hook contract is to exit 0 whatever
+    happens.
+    """
+    if not tool_name:
+        return False
+    try:
+        import sqlite3
+        from datetime import datetime, timezone
+
+        try:
+            from superlocalmemory.hooks.session_registry import (
+                resolve_active_profile,
+            )
+            profile_id = resolve_active_profile() or "default"
+        except Exception:
+            profile_id = "default"
+
+        project_path = ""
+        raw_cwd = payload.get("cwd")
+        if isinstance(raw_cwd, str):
+            project_path = raw_cwd[:_MAX_SUMMARY_LEN]
+
+        conn = sqlite3.connect(str(_memory_db_path()), timeout=0.05)
+        try:
+            conn.execute("PRAGMA busy_timeout=50")
+            conn.execute(
+                "INSERT INTO tool_events "
+                "(session_id, profile_id, project_path, tool_name, event_type,"
+                " input_summary, output_summary, duration_ms, metadata,"
+                " created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    session_id,
+                    profile_id,
+                    project_path,
+                    tool_name,
+                    "complete",
+                    _summary(payload.get("tool_input")),
+                    _summary(response_text),
+                    0,
+                    "{}",
+                    datetime.now(timezone.utc).isoformat(),
+                ),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+        return True
+    except Exception:
+        return False
+
+
 def _inner_main() -> str:
     """Return an ``outcome`` string (for perf log); never raises."""
     payload = read_stdin_json()
@@ -119,6 +235,12 @@ def _inner_main() -> str:
 
     # Response scan — capped BEFORE regex (bound O(cap)).
     response_text = summarize_response(payload.get("tool_response"))
+
+    # The invocation is recorded before anything is decided about markers. An
+    # action the agent took is a fact about the session on its own terms; the
+    # marker only decides whether it also settles a pending recall.
+    _record_tool_event(session_id, tool_name, payload, response_text)
+
     if not response_text:
         return "no_response"
 

package/src/superlocalmemory/learning/bandit.py

@@ -326,9 +326,20 @@ class ContextualBandit:
         play_id: int,
         reward: float,
         kind: str = "proxy_position",
+        weight: float = 1.0,
     ) -> bool:
         """Apply the reward to the (profile, stratum, arm) posterior.
 
+        ``weight`` scales how much this one observation counts, and exists for
+        inverse-propensity correction: the policy chose what was shown, so an
+        arm it almost always shows produces weak evidence whatever happens to
+        it, while one it rarely shows produces strong evidence. Weighting by
+        the inverse of that probability is what keeps the posterior from
+        recording popularity the policy manufactured. See ``propensity.py``.
+
+        A weight of 1.0 is the uncorrected update and the default, so a caller
+        with no competitor posteriors to estimate against changes nothing.
+
         Returns True on success. Never raises — DB failures logged at WARN.
         Cache invalidated on success (B5).
         """
@@ -343,6 +354,15 @@ class ContextualBandit:
         elif reward_f > 1.0:
             reward_f = 1.0
 
+        try:
+            weight_f = float(weight)
+        except (TypeError, ValueError):
+            weight_f = 1.0
+        # A non-positive weight would either freeze the arm or subtract
+        # evidence; neither is a meaningful observation.
+        if weight_f <= 0.0:
+            weight_f = 1.0
+
         try:
             conn = _conn_for(self._db_path)
         except sqlite3.Error as exc:  # pragma: no cover — defensive
@@ -389,8 +409,8 @@ class ContextualBandit:
                 "    plays = plays + 1, "
                 "    last_played_at = ? "
                 "WHERE profile_id = ? AND stratum = ? AND arm_id = ?",
-                (cap, reward_f, cap, 1.0 - reward_f, now,
-                 profile_id, stratum, arm_id),
+                (cap, weight_f * reward_f, cap, weight_f * (1.0 - reward_f),
+                 now, profile_id, stratum, arm_id),
             )
             conn.execute(
                 "UPDATE bandit_plays "