switchroom 0.18.11 → 0.18.13

package/telegram-plugin/worktree-watch-cwds.ts

@@ -90,13 +90,30 @@ function defaultDeriveName(agentDir: string): string {
   return leaf;
 }
 
+/**
+ * Two-tier owner-identity resolution shared by `ownedWorktreeCwds` and
+ * `refreshOwnedWorktreeHeartbeats`:
+ *   Tier 1 — env fast path (`SWITCHROOM_AGENT_NAME`).
+ *   Tier 2 — durable fallback derived from the agent's OWN directory basename.
+ * Returns "" when neither source yields a usable identity — callers MUST treat
+ * "" as fail-closed (never guess ownership).
+ */
+export function resolveOwnerIdentity(
+  self: string | undefined,
+  agentDir: string | null | undefined,
+  deriveName?: (agentDir: string) => string,
+): string {
+  let resolved: string = self != null ? self : "";
+  if (resolved === "" && agentDir != null && agentDir !== "") {
+    const derive = deriveName ?? defaultDeriveName;
+    resolved = derive(agentDir) || "";
+  }
+  return resolved;
+}
+
 export function ownedWorktreeCwds(opts: OwnedWorktreeCwdsOptions): string[] {
   // Tier 1: env fast path. Tier 2: durable agentDir-derived fallback.
-  let resolved: string = opts.self != null ? opts.self : "";
-  if (resolved === "" && opts.agentDir != null && opts.agentDir !== "") {
-    const derive = opts.deriveName ?? defaultDeriveName;
-    resolved = derive(opts.agentDir) || "";
-  }
+  const resolved = resolveOwnerIdentity(opts.self, opts.agentDir, opts.deriveName);
 
   if (resolved === "") {
     // Both env and durable config unavailable. Keep the historical
@@ -133,3 +150,175 @@ export function ownedWorktreeCwds(opts: OwnedWorktreeCwdsOptions): string[] {
     return [];
   }
 }
+
+/**
+ * Default minimum interval (ms) between heartbeat writes for the same
+ * worktree record. The gateway invokes the refresh on every ~1s rescan tick;
+ * writing every tick would be needless churn. Refreshing at most every 2 min
+ * keeps every live claim's heartbeat FAR fresher than the reaper's 10-min
+ * `STALE_THRESHOLD_MS`, so a live claim never reads as stale, while a dead
+ * gateway (no ticks) lets the heartbeat age out and become reap-eligible.
+ */
+export const DEFAULT_HEARTBEAT_REFRESH_INTERVAL_MS = 2 * 60_000;
+
+/** Minimal record shape needed to refresh a worktree heartbeat. */
+export interface WorktreeHeartbeatRecord {
+  id: string;
+  ownerAgent?: string;
+  /** ISO 8601 timestamp of the record's current heartbeat (for throttling). */
+  heartbeatAt?: string;
+}
+
+export interface RefreshOwnedHeartbeatsOptions {
+  /** The agent's identity — `process.env.SWITCHROOM_AGENT_NAME` (fast path). */
+  self: string | undefined;
+  /** Durable, non-env identity fallback: the agent's OWN directory. */
+  agentDir?: string | null;
+  /** Host-global registry read (`listRecords` from src/worktree/registry). */
+  listRecords: () => WorktreeHeartbeatRecord[];
+  /** Advance one record's heartbeat (`touchHeartbeat` from the registry). */
+  touchHeartbeat: (id: string) => void;
+  /**
+   * Skip a touch while the record's heartbeat is younger than this (ms).
+   * Defaults to `DEFAULT_HEARTBEAT_REFRESH_INTERVAL_MS`. Set to 0 to always
+   * touch (used by tests).
+   */
+  minRefreshIntervalMs?: number;
+  /** `Date.now` override for tests. */
+  now?: () => number;
+  /** Injectable name derivation (defaults to `path.basename`). */
+  deriveName?: (agentDir: string) => string;
+  /** Best-effort error sink. */
+  log?: (msg: string) => void;
+}
+
+/**
+ * Refresh the heartbeat of every worktree THIS agent owns.
+ *
+ * This is the production driver that keeps `touchHeartbeat` (previously dead —
+ * F1/H3) alive: the gateway calls it on the same rescan tick that re-derives
+ * the watched worktree cwds, so a claim held by a LIVE agent has its heartbeat
+ * advanced continuously and never trips the reaper's staleness gate. When the
+ * owning gateway dies, the ticks stop, the heartbeat ages past
+ * `STALE_THRESHOLD_MS`, and the (now fail-safe) reaper can reclaim the truly
+ * abandoned claim.
+ *
+ * Fail-CLOSED, mirroring `ownedWorktreeCwds`:
+ *   - Unresolved identity ⇒ touch nothing (never guess ownership).
+ *   - Ownerless records (`ownerAgent` undefined) are NEVER matched — we must
+ *     not advance another agent's / an unattributable claim's heartbeat.
+ *   - A registry read failure ⇒ touch nothing.
+ *   - A per-record touch failure is swallowed (logged) — one bad record must
+ *     not abort the rest, and this runs on the hot watch loop.
+ *
+ * @returns the number of heartbeats actually advanced this call.
+ */
+export function refreshOwnedWorktreeHeartbeats(
+  opts: RefreshOwnedHeartbeatsOptions,
+): number {
+  const identity = resolveOwnerIdentity(opts.self, opts.agentDir, opts.deriveName);
+  if (identity === "") return 0; // fail-closed: never guess ownership
+
+  const nowMs = (opts.now ?? Date.now)();
+  const minInterval =
+    opts.minRefreshIntervalMs ?? DEFAULT_HEARTBEAT_REFRESH_INTERVAL_MS;
+
+  let records: WorktreeHeartbeatRecord[];
+  try {
+    records = opts.listRecords();
+  } catch {
+    return 0;
+  }
+
+  let touched = 0;
+  for (const r of records) {
+    if (r.ownerAgent !== identity) continue; // ownerless never matched
+    // Throttle: skip if the heartbeat is still comfortably fresh.
+    if (minInterval > 0 && r.heartbeatAt != null) {
+      const age = nowMs - new Date(r.heartbeatAt).getTime();
+      if (Number.isFinite(age) && age >= 0 && age < minInterval) continue;
+    }
+    try {
+      opts.touchHeartbeat(r.id);
+      touched++;
+    } catch (err) {
+      opts.log?.(
+        `worktree heartbeat refresh failed for ${r.id}: ${(err as Error).message}`,
+      );
+    }
+  }
+  return touched;
+}
+
+/** A registry record carrying everything both watch operations need. */
+export type WorktreeWatchRecord = WorktreeOwnershipRecord & WorktreeHeartbeatRecord;
+
+export interface WorktreeWatchProviderOptions {
+  /** The agent's identity — `process.env.SWITCHROOM_AGENT_NAME` (fast path). */
+  self: string | undefined;
+  /** Durable, non-env identity fallback: the agent's OWN directory. */
+  agentDir?: string | null;
+  /** Host-global registry read (`listRecords` from src/worktree/registry). */
+  listRecords: () => WorktreeWatchRecord[];
+  /** Advance one record's heartbeat (`touchHeartbeat` from the registry). */
+  touchHeartbeat: (id: string) => void;
+  /** Injectable realpath for the cwd derivation (defaults to fs.realpathSync). */
+  realpath?: (p: string) => string;
+  /** Injectable name derivation from `agentDir` (defaults to path.basename). */
+  deriveName?: (agentDir: string) => string;
+  /** Heartbeat throttle window (ms); see refreshOwnedWorktreeHeartbeats. */
+  minRefreshIntervalMs?: number;
+  /** `Date.now` override for tests. */
+  now?: () => number;
+  /** Best-effort log sink shared by both operations. */
+  log?: (msg: string) => void;
+}
+
+/**
+ * Build the `extraWatchCwdsProvider` closure the gateway installs on the
+ * subagent watcher.
+ *
+ * The provider is invoked on every ~1s rescan tick and does TWO things on that
+ * single tick:
+ *   1. advances the heartbeat of every worktree THIS agent owns
+ *      (`refreshOwnedWorktreeHeartbeats` — the production driver that keeps
+ *      `touchHeartbeat` alive; without it every claim reads "stale" 10 min
+ *      after creation and the reaper's staleness guarantee collapses), AND
+ *   2. returns the set of owned worktree cwds for the watcher to also watch
+ *      (`ownedWorktreeCwds`).
+ *
+ * It is extracted from the gateway (rather than inlined) SO THAT the wiring —
+ * specifically that the provider actually DRIVES heartbeats, not merely returns
+ * cwds — is under direct unit test. Deleting the heartbeat refresh here is
+ * caught by the provider's behaviour test (it would return cwds but stop
+ * advancing heartbeats), which the pre-extraction inline closure could not
+ * assert against.
+ *
+ * Both operations use the SAME two-tier identity (env fast path + agentDir
+ * fallback), so a single `agentDir` kill-switch on the caller governs both.
+ * Fully best-effort: neither operation throws out of the returned closure.
+ */
+export function makeWorktreeWatchProvider(
+  opts: WorktreeWatchProviderOptions,
+): () => string[] {
+  return () => {
+    refreshOwnedWorktreeHeartbeats({
+      self: opts.self,
+      agentDir: opts.agentDir,
+      listRecords: opts.listRecords,
+      touchHeartbeat: opts.touchHeartbeat,
+      minRefreshIntervalMs: opts.minRefreshIntervalMs,
+      now: opts.now,
+      deriveName: opts.deriveName,
+      log: opts.log,
+    });
+    return ownedWorktreeCwds({
+      self: opts.self,
+      agentDir: opts.agentDir,
+      listRecords: opts.listRecords,
+      realpath: opts.realpath,
+      deriveName: opts.deriveName,
+      log: opts.log,
+    });
+  };
+}

package/vendor/hindsight-memory/scripts/drain_pending.py

@@ -9,15 +9,22 @@ the queue no longer drains it but the operator can still inspect via
 
 Boundaries
 ----------
-* Per-entry HTTP timeout: ``HINDSIGHT_DRAIN_TIMEOUT`` (default 5s).
+* Per-entry HTTP timeout: ``HINDSIGHT_DRAIN_TIMEOUT`` (default 5s), but
+  clamped per entry to the budget still remaining (see below) so a
+  single slow entry can never overshoot the wall-clock cap. The default
+  timeout (5s) intentionally exceeds the default budget (4s): the clamp,
+  not the raw timeout, is what bounds a slow entry.
+* Total wall-clock cap: ``HINDSIGHT_DRAIN_BUDGET_S`` (default 4s) so
+  drain never blocks SessionStart longer than the upstream hook timeout
+  permits. This is the authoritative bound; the per-entry timeout is
+  clamped down to ``max(1, remaining budget)`` before each request, so
+  even one slow upstream entry overshoots the budget by at most the
+  clamp floor (~1s), not by ``HINDSIGHT_DRAIN_TIMEOUT - budget``.
 * Stall guard: if ``STALL_THRESHOLD`` (3) consecutive entries fail with
   the same error class, we stop draining for this session — that's a
   systemic outage, not a transient flake, and continuing would only
   burn the SessionStart timeout budget. The remaining entries stay
   queued for the next session.
-* Total wall-clock cap: ``HINDSIGHT_DRAIN_BUDGET_S`` (default 4s) so
-  drain never blocks SessionStart longer than the upstream
-  hook timeout permits.
 
 Standalone usage::
 
@@ -113,13 +120,22 @@ def drain(config: dict | None = None) -> dict:
     last_error_class: str | None = None
 
     for path, entry in entries:
-        if time.monotonic() - started > budget:
+        elapsed = time.monotonic() - started
+        if elapsed > budget:
             summary["budget_exceeded"] = True
             debug_log(config, "drain_pending: total budget exceeded, stopping")
             break
 
+        # Clamp the per-entry HTTP timeout to the budget still remaining
+        # (#1094 item 2). Without this, a single slow entry using the full
+        # HINDSIGHT_DRAIN_TIMEOUT (default 5s) overshoots the total budget
+        # (default 4s). Floor at 1s so we still give a near-exhausted
+        # budget one bounded shot rather than a 0s (instant-fail) request.
+        remaining = budget - elapsed
+        effective_timeout = max(1, min(timeout, int(remaining) if remaining >= 1 else 1))
+
         try:
-            _retry_one(entry, timeout=timeout)
+            _retry_one(entry, timeout=effective_timeout)
         except Exception as e:
             err_class = type(e).__name__
             if err_class == last_error_class:

package/vendor/hindsight-memory/scripts/lib/client.py

@@ -93,15 +93,22 @@ class HindsightClient:
                 pass
             raise RuntimeError(f"HTTP {e.code} from {url}: {body_text}") from e
 
-    def health_check(self, timeout: int = 5) -> bool:
+    def health_check(self, timeout: int = 5, retries: int = HEALTH_CHECK_RETRIES) -> bool:
         """Check if the Hindsight server is reachable.
 
-        Mirrors Openclaw's checkExternalApiHealth: retries up to 3 times
-        with 2s delay between attempts.
+        Mirrors Openclaw's checkExternalApiHealth: retries up to
+        ``retries`` times (default ``HEALTH_CHECK_RETRIES`` = 3) with
+        ``HEALTH_CHECK_DELAY`` (2s) between attempts.
+
+        Time-budgeted callers (e.g. the SessionStart drain gate,
+        #1094) should pass ``retries=1``: against a HUNG server the
+        default loop costs ~retries*timeout + (retries-1)*delay of wall
+        clock, which blows a 5s hook budget.
         """
         import time
 
-        for attempt in range(1, HEALTH_CHECK_RETRIES + 1):
+        retries = max(1, retries)
+        for attempt in range(1, retries + 1):
             try:
                 url = f"{self.api_url}/health"
                 req = urllib.request.Request(url, headers=self._headers(), method="GET")
@@ -110,7 +117,7 @@ class HindsightClient:
                         return True
             except Exception:
                 pass
-            if attempt < HEALTH_CHECK_RETRIES:
+            if attempt < retries:
                 time.sleep(HEALTH_CHECK_DELAY)
         return False
 

package/vendor/hindsight-memory/scripts/lib/directives.py

@@ -183,6 +183,27 @@ def parse_active_directives_block(text: str) -> list:
     return contents
 
 
+# Length-aware guard for terse rules (#2912). Forward coverage alone
+# (|rule ∩ directive| / |rule|) is easy to satisfy when the rule has only a
+# couple of significant tokens: any long directive that happens to contain
+# those few words scores ~1.0 and silently swallows a genuinely-new short
+# rule, skipping a legitimate re-prompt. For such short rules we additionally
+# require REVERSE coverage — the matched directive's significant-token set must
+# also be substantially covered by the rule's — which is a Jaccard-style
+# bidirectional check. That fails precisely the "few tokens diluted inside a
+# long unrelated directive" case while still passing a terse rule that restates
+# a comparably terse directive.
+#
+# Constants chosen against the real tokenizer output:
+#   - < 4 significant tokens is "short" (1-3 tokens: the regime where a single
+#     incidental word swings forward coverage past 0.6).
+#   - reverse coverage >= 0.5 keeps near-equal-size restatements deduped
+#     (e.g. a 2-token rule vs a 4-token directive → 2/4 = 0.5) but rejects a
+#     2-token rule diluted inside a 6+-token directive (2/6 ≈ 0.33 < 0.5).
+_SHORT_RULE_TOKEN_LIMIT = 4
+_SHORT_RULE_REVERSE_COVERAGE = 0.5
+
+
 def rule_already_captured(
     rule_text: str, directive_contents: list, threshold: float = 0.6
 ) -> bool:
@@ -193,15 +214,29 @@ def rule_already_captured(
     best-matching directive. A high coverage ratio means the restated rule adds
     (almost) no new significant words over one already stored — i.e. a
     duplicate. Deterministic; no model/API call.
+
+    For terse rules (fewer than ``_SHORT_RULE_TOKEN_LIMIT`` significant tokens)
+    forward coverage is not sufficient — see the module comment above — so a
+    reverse-coverage guard is also required before declaring a match.
     """
     rule_tokens = _dedup_tokens(rule_text)
     if not rule_tokens:
         return False
+    is_short_rule = len(rule_tokens) < _SHORT_RULE_TOKEN_LIMIT
     for content in directive_contents:
         d_tokens = _dedup_tokens(content)
         if not d_tokens:
             continue
-        covered = len(rule_tokens & d_tokens) / len(rule_tokens)
-        if covered >= threshold:
-            return True
+        intersection = len(rule_tokens & d_tokens)
+        covered = intersection / len(rule_tokens)
+        if covered < threshold:
+            continue
+        if is_short_rule:
+            # Bidirectional guard: the directive must not be much larger than
+            # the rule, or those few shared tokens are incidental overlap
+            # rather than a true restatement.
+            reverse_covered = intersection / len(d_tokens)
+            if reverse_covered < _SHORT_RULE_REVERSE_COVERAGE:
+                continue
+        return True
     return False

package/vendor/hindsight-memory/scripts/lib/pending.py

@@ -197,22 +197,49 @@ def update_attempt(path: str, entry: dict, error: BaseException) -> bool:
 
 def mark_dead(path: str, entry: dict) -> Optional[str]:
     """Convert an entry that exceeded ``MAX_ATTEMPTS`` into a permanent
-    failure marker. Renames ``<path>`` to ``<path>.dead`` so the queue
-    no longer drains it but operators can still inspect.
-
-    Returns the marker path, or ``None`` if the rename failed.
+    failure marker at ``<path>.dead`` so the queue no longer drains it
+    but operators can still inspect.
+
+    Returns the marker path, or ``None`` if it failed.
+
+    Crash-window invariant (#1094 item 3): **a live ``<path>.json`` entry
+    must never carry a ``dead_at`` stamp.** The old two-step form violated
+    this — it wrote the dead_at-stamped payload back to the *live* path
+    (rename tmp -> path) and only then renamed path -> path.dead, so a
+    crash between the two renames left a live entry with ``dead_at`` set
+    that the drainer would re-enter and re-bump. Here we instead:
+
+      1. write the dead_at-stamped payload to ``<path>.tmp``
+      2. ``os.replace(tmp, dead_path)``  — the .dead marker appears in one
+         atomic step (never drained: the drainer only lists ``*.json``)
+      3. ``os.unlink(path)``            — drop the original live entry
+
+    At every crash point the invariant holds: the ``dead_at`` stamp only
+    ever lands on ``<path>.dead``. A crash after step 2 leaves both the
+    (stale, no-dead_at) live entry and the .dead marker; the next drain
+    re-marks it dead (os.replace overwrites the marker idempotently),
+    never observing a live entry with dead_at.
     """
     entry["dead_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
     dead_path = path + ".dead"
+    tmp = path + ".tmp"
     try:
-        # Best-effort: write the final state first so the marker shows
-        # the death timestamp + last error.
-        tmp = path + ".tmp"
         with open(tmp, "w", encoding="utf-8") as f:
             json.dump(entry, f, ensure_ascii=False)
         os.chmod(tmp, 0o600)
-        os.rename(tmp, path)
-        os.rename(path, dead_path)
+        os.replace(tmp, dead_path)
+        # Marker is durable now; removing the original never resurrects a
+        # dead_at-stamped live entry. Best-effort — a leftover live entry
+        # is self-healing (re-marked dead on the next pass).
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
         return dead_path
     except OSError:
+        # Clean up a possibly-orphaned tmp so it doesn't linger.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
         return None

package/vendor/hindsight-memory/scripts/session_end.py

@@ -40,7 +40,16 @@ from lib.pending import MAX_ENTRIES, count as pending_count, enqueue as pending_
 # Exit codes:
 #   0 — success (or retain skipped for benign reasons)
 #   1 — retain failed AND was queued to pending-retains (recoverable)
-#   2 — retain failed AND the queue rejected it (chronic backlog)
+#   2 — retain failed but nothing was queued — either the queue rejected
+#       it (chronic backlog) or there was no payload to queue (the
+#       failure happened before one was built). Not recoverable via the
+#       pending-retains drain, so it's semantically "dropped", not
+#       "queued".
+# Only the sign of the exit code is load-bearing downstream: Claude
+# Code's hook runner routes any non-zero SessionEnd exit to the issue
+# sink (bin/run-hook.sh / #424); nothing distinguishes 1 from 2
+# programmatically, so this is a correctness/clarity fix, not a
+# contract change.
 EXIT_OK = 0
 EXIT_QUEUED = 1
 EXIT_DROPPED = 2
@@ -98,8 +107,10 @@ def main() -> int:
                     exit_code = EXIT_QUEUED
             else:
                 # No payload to queue — the failure happened before we
-                # finished building one (e.g. URL resolution).
-                exit_code = EXIT_QUEUED
+                # finished building one (e.g. URL resolution). Nothing
+                # landed in pending-retains, so this is a drop, not a
+                # queue (#1094 item 5): EXIT_DROPPED, not EXIT_QUEUED.
+                exit_code = EXIT_DROPPED
 
     # Stop daemon if we started it. Always runs, even on retain failure,
     # so we don't leak a daemon process.

package/vendor/hindsight-memory/scripts/session_start.py

@@ -53,6 +53,27 @@ def main():
         prestart_daemon_background(config, debug_fn=_dbg)
         return
 
+    # Mode 1 (external hindsightApiUrl) reachability gate (#1094 item 1).
+    # get_api_url() returns the external URL WITHOUT probing it, so an
+    # external server that's down would otherwise slip past the guard
+    # above and let the drain run against nothing — bumping attempt
+    # counters on healthy entries until the stall guard trips. Modes 2/3
+    # already gate via _check_health inside get_api_url, so only Mode 1
+    # needs an explicit probe here. On failure, skip the drain and return
+    # (no local daemon to pre-start in Mode 1 — the external server is the
+    # operator's responsibility); queued entries stay for the next session.
+    # Single bounded attempt (retries=1, timeout=2): the default 3-retry
+    # health_check against a HUNG (timing-out) server would cost ~10s
+    # (3*2s timeouts + 2*2s sleeps) — worse than the attempt-counter
+    # bumps this gate prevents, and over the 5s SessionStart hook budget.
+    # Worst-case probe cost here is ~2s.
+    if config.get("hindsightApiUrl") and not client.health_check(timeout=2, retries=1):
+        debug_log(
+            config,
+            f"External Hindsight at {api_url} unreachable, skipping drain",
+        )
+        return
+
     # Drain any retains that session_end.py queued on failure (#1071).
     # Bounded by HINDSIGHT_DRAIN_BUDGET_S so a slow upstream can't pin
     # the SessionStart hook. The drain is best-effort — failures stay

package/vendor/hindsight-memory/scripts/tests/test_directives.py

@@ -255,6 +255,44 @@ class TestDirectiveDedup(unittest.TestCase):
     def test_empty_rule_is_not_captured(self):
         self.assertFalse(rule_already_captured("", ["Always use British spelling."]))
 
+    # --- #2912: length-aware guard for terse rules -----------------------
+
+    def test_terse_new_rule_not_swallowed_by_long_directive(self):
+        # A genuinely-new terse rule whose few significant tokens all appear
+        # inside a much longer, unrelated directive must NOT be deduped — the
+        # verifier should still re-prompt. Rule tokens {tabs, indentation} are
+        # both present in the long directive, so forward coverage is 1.0, but
+        # they are incidental overlap inside a broader rule.
+        contents = parse_active_directives_block(
+            self._block(
+                "Always use spaces not tabs for indentation and trim "
+                "trailing whitespace on every saved source file."
+            )
+        )
+        self.assertFalse(rule_already_captured("Use tabs for indentation.", contents))
+
+    def test_terse_rule_restating_equally_terse_directive_is_captured(self):
+        # Two comparably terse rules about the same thing → real duplicate.
+        contents = parse_active_directives_block(self._block("Use British spelling."))
+        self.assertTrue(rule_already_captured("Always use British spelling.", contents))
+
+    def test_long_rule_boundary_unchanged_at_0_6(self):
+        # Long rules (>= _SHORT_RULE_TOKEN_LIMIT significant tokens) keep the
+        # plain forward-coverage 0.6 behavior — the guard does not engage.
+        directive = "capital python numeric boolean spelling timezone"
+        # Rule shares 3 of its 5 significant tokens (0.6 exactly) → captured.
+        self.assertTrue(
+            rule_already_captured(
+                "capital python numeric quarterly monthly", [directive]
+            )
+        )
+        # Rule shares 2 of 5 (0.4 < 0.6) → not captured.
+        self.assertFalse(
+            rule_already_captured(
+                "capital python quarterly monthly weekly", [directive]
+            )
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

package/vendor/hindsight-memory/tests/test_drain_pending.py

@@ -162,6 +162,74 @@ class DrainPendingTest(unittest.TestCase):
         remaining = [n for n in os.listdir(self._pending) if n.endswith(".json")]
         self.assertEqual(len(remaining), 10)
 
+    def test_per_entry_timeout_clamped_to_remaining_budget(self):
+        # #1094 item 2: default HINDSIGHT_DRAIN_TIMEOUT (5) exceeds the
+        # budget (4 here), so the effective per-entry timeout must be
+        # clamped DOWN to the remaining budget — never the raw 5s.
+        self._env.stop()
+        self._env = patch.dict(
+            os.environ,
+            {
+                "HINDSIGHT_PENDING_DIR": self._pending,
+                "HINDSIGHT_DRAIN_TIMEOUT": "5",
+                "HINDSIGHT_DRAIN_BUDGET_S": "4",
+            },
+            clear=False,
+        )
+        self._env.start()
+
+        _seed_entry(self._pending)
+        import drain_pending
+
+        seen = {"timeout": None}
+
+        def capture(*a, **kw):
+            seen["timeout"] = kw.get("timeout")
+            raise urllib.error.URLError("down")
+
+        with patch("urllib.request.urlopen", side_effect=capture):
+            drain_pending.drain({})
+
+        self.assertIsNotNone(seen["timeout"])
+        # Clamped to the ~4s budget remaining, not the raw 5s timeout.
+        self.assertLessEqual(seen["timeout"], 4)
+        self.assertGreaterEqual(seen["timeout"], 1)
+
+    def test_slow_entry_cannot_overshoot_budget_beyond_clamp_floor(self):
+        # A single slow entry must not blow the wall-clock budget by
+        # HINDSIGHT_DRAIN_TIMEOUT - budget. With the clamp, overshoot is
+        # bounded by the 1s clamp floor (+ scheduling epsilon).
+        self._env.stop()
+        self._env = patch.dict(
+            os.environ,
+            {
+                "HINDSIGHT_PENDING_DIR": self._pending,
+                "HINDSIGHT_DRAIN_TIMEOUT": "5",
+                "HINDSIGHT_DRAIN_BUDGET_S": "2",
+            },
+            clear=False,
+        )
+        self._env.start()
+
+        _seed_entry(self._pending)
+        import drain_pending
+
+        def slow_then_fail(*a, **kw):
+            # Sleep exactly as long as the timeout the drainer granted —
+            # emulates a request that runs to its (clamped) timeout.
+            time.sleep(kw.get("timeout", 5))
+            raise urllib.error.URLError("timed out")
+
+        started = time.monotonic()
+        with patch("urllib.request.urlopen", side_effect=slow_then_fail):
+            drain_pending.drain({})
+        elapsed = time.monotonic() - started
+
+        # Budget is 2s; clamped timeout is min(5, ~2)=2, so ~2s of sleep.
+        # Without the clamp the entry would sleep the full 5s. Assert we
+        # stay near the budget, well under the raw 5s timeout.
+        self.assertLess(elapsed, 4.0)
+
     def test_drain_mixed_success_failure(self):
         # Three entries: server returns alternating ok/fail/ok.
         _seed_entry(self._pending, document_id="doc-a")

package/vendor/hindsight-memory/tests/test_pending.py

@@ -144,6 +144,50 @@ class PendingQueueTest(unittest.TestCase):
         # iter_entries no longer surfaces .dead files
         self.assertEqual(pending_mod.iter_entries(), [])
 
+    def test_mark_dead_never_leaves_live_entry_with_dead_at(self):
+        # Crash-window invariant (#1094 item 3): the dead_at stamp must
+        # only ever land on <path>.dead, never on the live <path>.json.
+        # The old two-step form wrote dead_at to the live path first;
+        # simulate a crash *between* the two visible transitions by
+        # stubbing the SECOND os.replace/os.rename so it raises, then
+        # assert no live .json entry carries dead_at.
+        path = pending_mod.enqueue(self._sample_payload(), RuntimeError("boom"))
+        _, entry = pending_mod.iter_entries()[0]
+
+        real_replace = os.replace
+        calls = {"n": 0}
+
+        def replace_fail_after_first(src, dst):
+            # First replace = tmp -> dead_path (the marker). Let it run.
+            # Any later mutation would be the pre-fix live-path write —
+            # there is none in the new single-transition form, but if a
+            # regression reintroduces it, blow up here.
+            calls["n"] += 1
+            if calls["n"] == 1:
+                return real_replace(src, dst)
+            raise OSError("simulated crash mid-mark_dead")
+
+        with patch("os.replace", side_effect=replace_fail_after_first):
+            pending_mod.mark_dead(path, entry)
+
+        # No live .json entry may carry dead_at.
+        for _p, e in pending_mod.iter_entries():
+            self.assertNotIn(
+                "dead_at", e, "a live queue entry must never carry dead_at"
+            )
+        # The .dead marker exists and carries dead_at.
+        dead = path + ".dead"
+        self.assertTrue(os.path.isfile(dead))
+        with open(dead) as f:
+            self.assertIn("dead_at", json.load(f))
+
+    def test_mark_dead_no_tmp_left_behind(self):
+        path = pending_mod.enqueue(self._sample_payload(), RuntimeError("boom"))
+        _, entry = pending_mod.iter_entries()[0]
+        pending_mod.mark_dead(path, entry)
+        leftovers = [n for n in os.listdir(self._dir) if n.endswith(".tmp")]
+        self.assertEqual(leftovers, [])
+
     def test_count_safe_when_dir_missing(self):
         self.assertEqual(pending_mod.count(), 0)