switchroom 0.12.26 → 0.12.28

package/vendor/hindsight-memory/scripts/lib/llm.py

@@ -0,0 +1,146 @@
+"""LLM provider detection for Hindsight's fact extraction.
+
+Port of: detectLLMConfig() in index.js
+
+When running hindsight-embed locally (daemon mode), it needs an LLM to
+extract facts from retained conversations. This module detects the LLM
+config using the same priority chain as Openclaw:
+
+  1. HINDSIGHT_API_LLM_* environment variables (highest priority)
+  2. Plugin config (llmProvider, llmModel, llmApiKeyEnv)
+  3. Auto-detect from standard provider env vars
+  4. External API mode (server-side LLM, no local config needed)
+"""
+
+import os
+
+# Provider detection table — same order as Openclaw
+PROVIDER_DETECTION = [
+    {"name": "openai", "key_env": "OPENAI_API_KEY"},
+    {"name": "anthropic", "key_env": "ANTHROPIC_API_KEY"},
+    {"name": "gemini", "key_env": "GEMINI_API_KEY"},
+    {"name": "groq", "key_env": "GROQ_API_KEY"},
+    {"name": "ollama", "key_env": ""},
+    {"name": "openai-codex", "key_env": ""},
+    {"name": "claude-code", "key_env": ""},
+]
+
+# Providers that don't require an API key
+NO_KEY_REQUIRED = {"ollama", "openai-codex", "claude-code"}
+
+
+def _find_provider(name):
+    """Find a provider entry by name."""
+    for p in PROVIDER_DETECTION:
+        if p["name"] == name:
+            return p
+    return None
+
+
+def detect_llm_config(config: dict) -> dict:
+    """Detect LLM configuration.
+
+    Returns dict with: provider, api_key, model, base_url, source.
+    Returns None values for external API mode (server handles LLM).
+    Raises RuntimeError if no configuration found and not in external API mode.
+    """
+    override_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER")
+    override_model = os.environ.get("HINDSIGHT_API_LLM_MODEL")
+    override_key = os.environ.get("HINDSIGHT_API_LLM_API_KEY")
+    override_base_url = os.environ.get("HINDSIGHT_API_LLM_BASE_URL")
+
+    # Priority 1: HINDSIGHT_API_LLM_PROVIDER env var
+    if override_provider:
+        if not override_key and override_provider not in NO_KEY_REQUIRED:
+            raise RuntimeError(
+                f'HINDSIGHT_API_LLM_PROVIDER is set to "{override_provider}" but HINDSIGHT_API_LLM_API_KEY is not set.'
+            )
+        pinfo = _find_provider(override_provider)
+        return {
+            "provider": override_provider,
+            "api_key": override_key or "",
+            "model": override_model,
+            "base_url": override_base_url,
+            "source": "HINDSIGHT_API_LLM_PROVIDER override",
+        }
+
+    # Priority 2: Plugin config llmProvider/llmModel
+    cfg_provider = config.get("llmProvider")
+    if cfg_provider:
+        pinfo = _find_provider(cfg_provider)
+        api_key = ""
+        key_env_name = config.get("llmApiKeyEnv")
+        if key_env_name:
+            api_key = os.environ.get(key_env_name, "")
+        elif pinfo and pinfo["key_env"]:
+            api_key = os.environ.get(pinfo["key_env"], "")
+
+        if not api_key and cfg_provider not in NO_KEY_REQUIRED:
+            key_source = key_env_name or (pinfo["key_env"] if pinfo else "unknown")
+            raise RuntimeError(
+                f'Plugin config llmProvider is "{cfg_provider}" but no API key found. Expected env var: {key_source}'
+            )
+        return {
+            "provider": cfg_provider,
+            "api_key": api_key,
+            "model": config.get("llmModel") or override_model,
+            "base_url": override_base_url,
+            "source": "plugin config",
+        }
+
+    # Priority 3: Auto-detect from standard provider env vars
+    for pinfo in PROVIDER_DETECTION:
+        if pinfo["name"] in NO_KEY_REQUIRED:
+            continue  # Must be explicitly requested
+        if not pinfo["key_env"]:
+            continue
+        api_key = os.environ.get(pinfo["key_env"], "")
+        if api_key:
+            return {
+                "provider": pinfo["name"],
+                "api_key": api_key,
+                "model": override_model,
+                "base_url": override_base_url,
+                "source": f"auto-detected from {pinfo['key_env']}",
+            }
+
+    # Priority 4: External API mode — server handles LLM
+    if config.get("hindsightApiUrl"):
+        return {
+            "provider": None,
+            "api_key": None,
+            "model": None,
+            "base_url": None,
+            "source": "external-api-mode-no-llm",
+        }
+
+    raise RuntimeError(
+        "No LLM configuration found for Hindsight.\n\n"
+        "Option 1: Set a standard provider API key (auto-detect):\n"
+        "  export OPENAI_API_KEY=sk-your-key\n"
+        "  export ANTHROPIC_API_KEY=your-key\n\n"
+        "Option 2: Override with Hindsight-specific env vars:\n"
+        "  export HINDSIGHT_API_LLM_PROVIDER=openai\n"
+        "  export HINDSIGHT_API_LLM_API_KEY=sk-your-key\n\n"
+        "Option 3: Use an external Hindsight API (server-side LLM):\n"
+        "  Set hindsightApiUrl in settings.json or HINDSIGHT_API_URL env var\n\n"
+        "The model will be selected automatically by Hindsight. To override: export HINDSIGHT_API_LLM_MODEL=your-model"
+    )
+
+
+def get_llm_env_vars(llm_config: dict) -> dict:
+    """Build environment variables for hindsight-embed daemon from LLM config.
+
+    These are passed to the daemon subprocess so it knows which LLM to use
+    for fact extraction.
+    """
+    env = {}
+    if llm_config.get("provider"):
+        env["HINDSIGHT_API_LLM_PROVIDER"] = llm_config["provider"]
+    if llm_config.get("api_key"):
+        env["HINDSIGHT_API_LLM_API_KEY"] = llm_config["api_key"]
+    if llm_config.get("model"):
+        env["HINDSIGHT_API_LLM_MODEL"] = llm_config["model"]
+    if llm_config.get("base_url"):
+        env["HINDSIGHT_API_LLM_BASE_URL"] = llm_config["base_url"]
+    return env

package/vendor/hindsight-memory/scripts/lib/pending.py

@@ -0,0 +1,218 @@
+"""Persistent queue for failed retain payloads.
+
+When a SessionEnd retain fails, the only on-disk record of the turn's
+memory is the just-closed transcript — and the agent thinks it was
+persisted. To prevent silent data loss (#1071), session_end.py
+serializes the *exact retain payload* it would have POSTed into
+``~/.hindsight/pending-retains/<unix-ms>-<short-uuid>.json``. The next
+SessionStart drains the directory: oldest first, success deletes,
+failure bumps an attempt counter (up to MAX_ATTEMPTS) and leaves the
+entry for the run after that.
+
+Layout
+------
+``~/.hindsight/pending-retains/`` (mode 0700, may contain sensitive
+memory payloads).
+
+Inside a Switchroom docker agent, ``$HOME`` is the agent UID's home
+inside the container, which is NOT a bind-mounted volume. The queue
+therefore survives session-to-session within a container's lifetime
+(the common case: claude session ends → container keeps running → next
+session drains) but NOT container recreate. That's deliberate: this is
+a rescue queue for transient retain failures, not a long-term DLQ.
+If the upstream is broken long enough that the agent container gets
+recreated, the operator has bigger problems and ``switchroom doctor``
+already surfaced the backlog.
+Each entry is a JSON file ``<unix-ms>-<short-uuid>.json`` containing::
+
+    {
+      "schema": 1,
+      "api_url":     "<resolved Hindsight URL at time of failure>",
+      "api_token":   "<bearer token or null>",
+      "bank_id":     "<derived bank id>",
+      "document_id": "<retain document id>",
+      "content":     "<formatted transcript>",
+      "context":     "<retainContext>",
+      "metadata":    {...},
+      "tags":        [...] or null,
+      "failed_at":   "<ISO-8601 UTC>",
+      "error_class": "<exception class name>",
+      "error_message": "<str(e)>",
+      "attempt_count": 1
+    }
+
+The file is written via ``write tmp + rename`` so concurrent agents
+sharing ``$HOME`` (legacy installs) never observe a half-written entry.
+
+Bounded directory
+-----------------
+``MAX_ENTRIES`` (1000) caps the queue. When full, ``enqueue()`` refuses
+the entry and returns ``None`` — the caller logs loudly and the operator
+is expected to drain manually. A chronically full queue means upstream
+is broken for a long time; piling on more entries doesn't help.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import uuid
+from typing import Optional
+
+
+SCHEMA = 1
+MAX_ENTRIES = 1000
+MAX_ATTEMPTS = 5
+
+
+def pending_dir() -> str:
+    """Return the pending-retains directory path.
+
+    Override with ``HINDSIGHT_PENDING_DIR`` for tests. Default:
+    ``$HOME/.hindsight/pending-retains/``.
+    """
+    override = os.environ.get("HINDSIGHT_PENDING_DIR")
+    if override:
+        return override
+    return os.path.join(os.path.expanduser("~"), ".hindsight", "pending-retains")
+
+
+def _ensure_dir() -> str:
+    """Create the queue dir with mode 0700 if missing. Return its path."""
+    d = pending_dir()
+    if not os.path.isdir(d):
+        os.makedirs(d, mode=0o700, exist_ok=True)
+    else:
+        # Tighten perms if a previous run created it with looser bits.
+        try:
+            mode = os.stat(d).st_mode & 0o777
+            if mode != 0o700:
+                os.chmod(d, 0o700)
+        except OSError:
+            pass
+    return d
+
+
+def _list_entries(d: str) -> list[str]:
+    """Return sorted filenames (oldest first by lexicographic order on
+    the ``<unix-ms>-<uuid>.json`` filename pattern).
+    """
+    try:
+        names = [n for n in os.listdir(d) if n.endswith(".json")]
+    except FileNotFoundError:
+        return []
+    names.sort()
+    return names
+
+
+def count() -> int:
+    """Number of pending entries. Safe to call when dir doesn't exist."""
+    d = pending_dir()
+    return len(_list_entries(d))
+
+
+def enqueue(payload: dict, error: BaseException) -> Optional[str]:
+    """Persist a failed retain payload.
+
+    ``payload`` carries the exact arguments that would have gone to
+    ``client.retain()`` plus connection info (``api_url``, ``api_token``)
+    so the drainer can rebuild the client without re-resolving config.
+
+    Returns the absolute path of the written entry, or ``None`` if the
+    queue is full (``MAX_ENTRIES`` reached). Atomic: writes ``<name>.tmp``
+    then renames to ``<name>``.
+    """
+    d = _ensure_dir()
+    if len(_list_entries(d)) >= MAX_ENTRIES:
+        return None
+
+    entry = dict(payload)
+    entry["schema"] = SCHEMA
+    entry["failed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    entry["error_class"] = type(error).__name__
+    entry["error_message"] = str(error)
+    entry.setdefault("attempt_count", 1)
+
+    ts_ms = int(time.time() * 1000)
+    short_uuid = uuid.uuid4().hex[:12]
+    name = f"{ts_ms}-{short_uuid}.json"
+    final = os.path.join(d, name)
+    tmp = final + ".tmp"
+
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(entry, f, ensure_ascii=False)
+    os.chmod(tmp, 0o600)
+    os.rename(tmp, final)
+    return final
+
+
+def iter_entries() -> list[tuple[str, dict]]:
+    """Return ``[(path, entry_dict), ...]`` oldest first.
+
+    Unreadable / malformed files are skipped silently — the drainer
+    handles its own logging. We never crash the SessionStart hook on
+    a corrupt entry.
+    """
+    d = pending_dir()
+    out: list[tuple[str, dict]] = []
+    for name in _list_entries(d):
+        p = os.path.join(d, name)
+        try:
+            with open(p, encoding="utf-8") as f:
+                out.append((p, json.load(f)))
+        except (OSError, json.JSONDecodeError):
+            continue
+    return out
+
+
+def delete_entry(path: str) -> bool:
+    """Remove a queue entry. Returns True on success, False otherwise."""
+    try:
+        os.remove(path)
+        return True
+    except OSError:
+        return False
+
+
+def update_attempt(path: str, entry: dict, error: BaseException) -> bool:
+    """Persist an updated attempt count + error info back to ``path``.
+
+    Atomic: writes ``<path>.tmp`` then renames. Returns True on success.
+    """
+    entry["attempt_count"] = int(entry.get("attempt_count", 1)) + 1
+    entry["last_attempt_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    entry["error_class"] = type(error).__name__
+    entry["error_message"] = str(error)
+    try:
+        tmp = path + ".tmp"
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(entry, f, ensure_ascii=False)
+        os.chmod(tmp, 0o600)
+        os.rename(tmp, path)
+        return True
+    except OSError:
+        return False
+
+
+def mark_dead(path: str, entry: dict) -> Optional[str]:
+    """Convert an entry that exceeded ``MAX_ATTEMPTS`` into a permanent
+    failure marker. Renames ``<path>`` to ``<path>.dead`` so the queue
+    no longer drains it but operators can still inspect.
+
+    Returns the marker path, or ``None`` if the rename failed.
+    """
+    entry["dead_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    dead_path = path + ".dead"
+    try:
+        # Best-effort: write the final state first so the marker shows
+        # the death timestamp + last error.
+        tmp = path + ".tmp"
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(entry, f, ensure_ascii=False)
+        os.chmod(tmp, 0o600)
+        os.rename(tmp, path)
+        os.rename(path, dead_path)
+        return dead_path
+    except OSError:
+        return None

package/vendor/hindsight-memory/scripts/lib/state.py

@@ -0,0 +1,196 @@
+"""File-based state persistence.
+
+Claude Code hooks are ephemeral processes — state must be persisted to files.
+Uses $CLAUDE_PLUGIN_DATA/state/ as the storage directory.
+"""
+
+import json
+import os
+import re
+import sys
+
+# fcntl is Unix-only; import conditionally so the module loads on Windows
+if sys.platform != "win32":
+    import fcntl
+else:
+    fcntl = None
+
+
+def _state_dir() -> str:
+    """Get the state directory, creating it if needed."""
+    plugin_data = os.environ.get("CLAUDE_PLUGIN_DATA", "")
+    if not plugin_data:
+        # Fallback to a temp location for testing
+        plugin_data = os.path.join(os.path.expanduser("~"), ".claude", "plugins", "data", "hindsight-memory")
+    state_dir = os.path.join(plugin_data, "state")
+    os.makedirs(state_dir, exist_ok=True)
+    return state_dir
+
+
+def _safe_filename(name: str) -> str:
+    """Sanitize a filename to prevent path traversal.
+
+    Strips path separators, .., and control characters. Mirrors Openclaw's
+    sanitizeFilename().
+    """
+    # Replace path separators and dangerous patterns
+    name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "_", name)
+    # Collapse .. to prevent traversal
+    name = name.replace("..", "_")
+    # Limit length
+    name = name[:200]
+    return name or "state"
+
+
+def _state_file(name: str) -> str:
+    """Get path for a state file. Name is sanitized to prevent traversal."""
+    safe = _safe_filename(name)
+    path = os.path.join(_state_dir(), safe)
+    # Final guard: resolved path must be inside state_dir
+    resolved = os.path.realpath(path)
+    expected_dir = os.path.realpath(_state_dir())
+    if not resolved.startswith(expected_dir + os.sep) and resolved != expected_dir:
+        raise ValueError(f"State file path escapes state directory: {name!r}")
+    return path
+
+
+def read_state(name: str, default=None):
+    """Read a JSON state file. Returns default if not found."""
+    path = _state_file(name)
+    if not os.path.exists(path):
+        return default
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return default
+
+
+def write_state(name: str, data):
+    """Write data to a JSON state file atomically."""
+    path = _state_file(name)
+    tmp_path = path + ".tmp"
+    try:
+        with open(tmp_path, "w") as f:
+            json.dump(data, f)
+        os.replace(tmp_path, path)
+    except OSError:
+        # Best-effort cleanup
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+
+
+def get_turn_count(session_id: str) -> int:
+    """Get the current turn count for a session."""
+    turns = read_state("turns.json", {})
+    return turns.get(session_id, 0)
+
+
+def increment_turn_count(session_id: str) -> int:
+    """Increment and return the turn count for a session.
+
+    Uses flock on Unix to prevent race conditions between concurrent hook
+    processes (e.g. async Stop + new UserPromptSubmit). On Windows, flock is
+    unavailable so we proceed without a lock — minor races here are harmless.
+    """
+    lock_path = _state_file("turns.lock")
+    if fcntl is not None:
+        try:
+            lock_fd = open(lock_path, "w")
+            fcntl.flock(lock_fd, fcntl.LOCK_EX)
+            try:
+                turns = read_state("turns.json", {})
+                turns[session_id] = turns.get(session_id, 0) + 1
+                # Cap tracked sessions to prevent unbounded growth
+                if len(turns) > 10000:
+                    sorted_keys = sorted(turns.keys())
+                    for k in sorted_keys[: len(sorted_keys) // 2]:
+                        del turns[k]
+                write_state("turns.json", turns)
+                return turns[session_id]
+            finally:
+                fcntl.flock(lock_fd, fcntl.LOCK_UN)
+                lock_fd.close()
+        except OSError:
+            pass
+
+    # Fallback: proceed without lock (Windows or lock acquisition failed)
+    turns = read_state("turns.json", {})
+    turns[session_id] = turns.get(session_id, 0) + 1
+    # Cap tracked sessions to prevent unbounded growth
+    if len(turns) > 10000:
+        sorted_keys = sorted(turns.keys())
+        for k in sorted_keys[: len(sorted_keys) // 2]:
+            del turns[k]
+    write_state("turns.json", turns)
+    return turns[session_id]
+
+
+def _locked_read_modify_write(state_name: str, lock_name: str, modify_fn):
+    """Read-modify-write a state file under flock.
+
+    modify_fn receives the current state dict and returns (updated_dict, result).
+    Returns the result from modify_fn.
+    """
+    lock_path = _state_file(lock_name)
+    if fcntl is not None:
+        try:
+            lock_fd = open(lock_path, "w")
+            fcntl.flock(lock_fd, fcntl.LOCK_EX)
+            try:
+                data = read_state(state_name, {})
+                data, result = modify_fn(data)
+                write_state(state_name, data)
+                return result
+            finally:
+                fcntl.flock(lock_fd, fcntl.LOCK_UN)
+                lock_fd.close()
+        except OSError:
+            pass
+
+    # Fallback without lock
+    data = read_state(state_name, {})
+    data, result = modify_fn(data)
+    write_state(state_name, data)
+    return result
+
+
+def track_retention(session_id: str, message_count: int) -> tuple:
+    """Track retention state and detect compaction.
+
+    Compares the current message count against the last retained count for this
+    session.  When the transcript shrinks (compaction), increments a chunk counter
+    so the caller can use a distinct document_id, preserving the pre-compaction
+    document.
+
+    Returns:
+        (chunk_index, compacted) — chunk_index for building document_id,
+        compacted is True if compaction was detected this call.
+    """
+
+    def _update(data):
+        entry = data.get(session_id, {"message_count": 0, "chunk": 0})
+        last_count = entry["message_count"]
+        chunk = entry["chunk"]
+        compacted = False
+
+        if message_count < last_count:
+            # Transcript shrank — compaction happened
+            chunk += 1
+            compacted = True
+
+        entry["message_count"] = message_count
+        entry["chunk"] = chunk
+        data[session_id] = entry
+
+        # Cap tracked sessions
+        if len(data) > 10000:
+            sorted_keys = sorted(data.keys())
+            for k in sorted_keys[: len(sorted_keys) // 2]:
+                del data[k]
+
+        return data, (chunk, compacted)
+
+    return _locked_read_modify_write("retention_tracking.json", "retention_tracking.lock", _update)