zeitlich 0.2.44 → 0.2.46

package/src/lib/thread/snapshot.test.ts

@@ -0,0 +1,443 @@
+import { describe, expect, it, beforeEach } from "vitest";
+import type Redis from "ioredis";
+import {
+  applySnapshot,
+  clearHotTier,
+  encodeSnapshot,
+} from "./snapshot";
+import { createThreadManager } from "./manager";
+import {
+  getThreadDedupKey,
+  getThreadListKey,
+  getThreadMetaKey,
+  getThreadStateKey,
+} from "./keys";
+import type { PersistedThreadState } from "../state/types";
+import type { ThreadSnapshot } from "./cold-store";
+import { createFakeRedis } from "./test-utils";
+
+const sampleState: PersistedThreadState = {
+  tasks: [
+    [
+      "t1",
+      {
+        id: "t1",
+        subject: "do thing",
+        description: "do it",
+        activeForm: "doing",
+        status: "pending",
+        metadata: {},
+        blockedBy: [],
+        blocks: [],
+      },
+    ],
+  ],
+  custom: { counter: 3 },
+};
+
+describe("encodeSnapshot", () => {
+  let redis: Redis & { _store: Map<string, unknown> };
+
+  beforeEach(() => {
+    redis = createFakeRedis();
+  });
+
+  it("returns null when the thread has not been initialized", async () => {
+    expect(
+      await encodeSnapshot({ redis, threadKey: "messages", threadId: "t-1" })
+    ).toBeNull();
+  });
+
+  it("captures messages, state, and dedup ids when idOf is provided", async () => {
+    const tm = createThreadManager<{ id: string; text: string }>({
+      redis,
+      threadId: "t-1",
+      idOf: (m) => m.id,
+    });
+    await tm.initialize();
+    await tm.append([{ id: "m1", text: "hello" }]);
+    await tm.append([{ id: "m2", text: "world" }]);
+    await tm.saveState(sampleState);
+
+    const snap = await encodeSnapshot({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      idOf: (raw) => (JSON.parse(raw) as { id: string }).id,
+    });
+
+    expect(snap).not.toBeNull();
+    if (!snap) throw new Error("expected snapshot");
+    expect(snap.v).toBe(1);
+    expect(snap.messages.map((raw) => JSON.parse(raw) as { id: string })).toEqual([
+      { id: "m1", text: "hello" },
+      { id: "m2", text: "world" },
+    ]);
+    expect(snap.state).toEqual(sampleState);
+    expect(snap.dedupIds).toEqual(["m1", "m2"]);
+  });
+
+  it("returns an empty dedupIds list when idOf is omitted", async () => {
+    const tm = createThreadManager<{ id: string }>({
+      redis,
+      threadId: "t-1",
+      idOf: (m) => m.id,
+    });
+    await tm.initialize();
+    await tm.append([{ id: "m1" }]);
+
+    const snap = await encodeSnapshot({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+    });
+
+    expect(snap?.dedupIds).toEqual([]);
+  });
+});
+
+describe("applySnapshot", () => {
+  let redis: Redis & { _store: Map<string, unknown> };
+
+  beforeEach(() => {
+    redis = createFakeRedis();
+  });
+
+  it("restores messages + state + dedup keys into Redis", async () => {
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [
+        JSON.stringify({ id: "m1", text: "hello" }),
+        JSON.stringify({ id: "m2", text: "world" }),
+      ],
+      state: sampleState,
+      dedupIds: ["m1", "m2"],
+    };
+    await applySnapshot({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      snapshot: snap,
+    });
+
+    const tm = createThreadManager<{ id: string; text: string }>({
+      redis,
+      threadId: "t-1",
+      idOf: (m) => m.id,
+    });
+    expect(await tm.load()).toEqual([
+      { id: "m1", text: "hello" },
+      { id: "m2", text: "world" },
+    ]);
+    expect(await tm.loadState()).toEqual(sampleState);
+
+    // Re-appending m1 should be deduped because dedup key was re-primed.
+    await tm.append([{ id: "m1", text: "hello" }]);
+    expect(await tm.length()).toBe(2);
+  });
+
+  it("is idempotent when the thread is already hot", async () => {
+    const tm = createThreadManager<{ id: string }>({
+      redis,
+      threadId: "t-1",
+      idOf: (m) => m.id,
+    });
+    await tm.initialize();
+    await tm.append([{ id: "existing" }]);
+
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [JSON.stringify({ id: "from-snapshot" })],
+      state: null,
+      dedupIds: ["from-snapshot"],
+    };
+    await applySnapshot({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      snapshot: snap,
+    });
+
+    expect(await tm.load()).toEqual([{ id: "existing" }]);
+  });
+
+  it("handles an empty snapshot (just sets the meta marker)", async () => {
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [],
+      state: null,
+      dedupIds: [],
+    };
+    await applySnapshot({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      snapshot: snap,
+    });
+
+    const metaKey = getThreadMetaKey("messages", "t-1");
+    expect(await redis.exists(metaKey)).toBe(1);
+    expect(
+      await redis.lrange(getThreadListKey("messages", "t-1"), 0, -1)
+    ).toEqual([]);
+  });
+
+  it("throws and writes nothing when the residue-cleanup DEL fails", async () => {
+    // Stub `redis.del` to reject (mimicking ACL deny, CROSSSLOT, etc.).
+    // The fix guarantees no writes hit the wire when this happens —
+    // the queued data writes never run because DEL is awaited
+    // outside the pipeline.
+    const wrapped = new Proxy(redis, {
+      get(target, prop, receiver): unknown {
+        if (prop === "del") {
+          return async (..._keys: string[]): Promise<number> => {
+            throw new Error("NOPERM: DEL denied by ACL");
+          };
+        }
+        return Reflect.get(target, prop, receiver) as unknown;
+      },
+    });
+
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [JSON.stringify({ id: "m1" })],
+      state: sampleState,
+      dedupIds: ["m1"],
+    };
+    await expect(
+      applySnapshot({
+        redis: wrapped as unknown as Redis,
+        threadKey: "messages",
+        threadId: "t-del-fail",
+        snapshot: snap,
+      })
+    ).rejects.toThrow("NOPERM: DEL denied by ACL");
+
+    // No data writes happened — list, state, meta, and dedup are all
+    // still untouched on the underlying store.
+    expect(
+      await redis.exists(getThreadListKey("messages", "t-del-fail"))
+    ).toBe(0);
+    expect(
+      await redis.exists(getThreadStateKey("messages", "t-del-fail"))
+    ).toBe(0);
+    expect(
+      await redis.exists(getThreadMetaKey("messages", "t-del-fail"))
+    ).toBe(0);
+    expect(
+      await redis.exists(getThreadDedupKey("t-del-fail", "m1"))
+    ).toBe(0);
+  });
+
+  it("clears list/state/dedup residue when a pipelined write fails partway", async () => {
+    // Pipeline stub that *applies* non-failing commands to the
+    // underlying fake (so residue is observable) and errors on rpush
+    // — mimicking a partial OOM where the list write fails but the
+    // dedup SETs queued after it still land. Without compensating
+    // cleanup, those stale dedup keys could silently skip a future
+    // append with the same id.
+    const wrapped = new Proxy(redis, {
+      get(target, prop, receiver): unknown {
+        if (prop === "pipeline") {
+          return (): Record<string, unknown> => {
+            type Op = { method: string; args: unknown[] };
+            const ops: Op[] = [];
+            const chain: Record<string, unknown> = {};
+            for (const m of ["set", "del", "rpush", "expire"]) {
+              chain[m] = (...args: unknown[]): Record<string, unknown> => {
+                ops.push({ method: m, args });
+                return chain;
+              };
+            }
+            chain.exec = async (): Promise<Array<[Error | null, unknown]>> => {
+              const out: Array<[Error | null, unknown]> = [];
+              const callable = target as unknown as Record<
+                string,
+                (...a: unknown[]) => Promise<unknown>
+              >;
+              for (const op of ops) {
+                if (op.method === "rpush") {
+                  out.push([new Error("OOM"), null]);
+                  continue;
+                }
+                const fn = callable[op.method];
+                if (!fn) throw new Error(`stub: unknown ${op.method}`);
+                const result = await fn(...op.args);
+                out.push([null, result]);
+              }
+              return out;
+            };
+            return chain;
+          };
+        }
+        return Reflect.get(target, prop, receiver) as unknown;
+      },
+    });
+
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [JSON.stringify({ id: "m1" })],
+      state: sampleState,
+      dedupIds: ["m1", "m2"],
+    };
+    await expect(
+      applySnapshot({
+        redis: wrapped as unknown as Redis,
+        threadKey: "messages",
+        threadId: "t-residue",
+        snapshot: snap,
+      })
+    ).rejects.toThrow("OOM");
+
+    // Every key the failed pipeline touched is cleared on the throw
+    // path — no list, state, meta, or dedup residue survives.
+    expect(
+      await redis.exists(getThreadListKey("messages", "t-residue"))
+    ).toBe(0);
+    expect(
+      await redis.exists(getThreadStateKey("messages", "t-residue"))
+    ).toBe(0);
+    expect(
+      await redis.exists(getThreadMetaKey("messages", "t-residue"))
+    ).toBe(0);
+    expect(await redis.exists(getThreadDedupKey("t-residue", "m1"))).toBe(0);
+    expect(await redis.exists(getThreadDedupKey("t-residue", "m2"))).toBe(0);
+  });
+
+  it("throws and leaves the meta key unset when a pipelined command fails", async () => {
+    // Wrap the fake's `pipeline()` so `exec()` returns a tuple list
+    // containing a per-command error — mimicking ioredis's behaviour
+    // when Redis runtime errors (OOM, ACL, WRONGTYPE) occur inside a
+    // pipeline. The top-level promise resolves; the error lives in the
+    // result tuple, and applySnapshot must surface it.
+    const wrapped = new Proxy(redis, {
+      get(target, prop, receiver): unknown {
+        if (prop === "pipeline") {
+          return (): Record<string, unknown> => {
+            type Op = { method: string; args: unknown[] };
+            const ops: Op[] = [];
+            const chain: Record<string, unknown> = {};
+            for (const m of ["set", "del", "rpush", "expire"]) {
+              chain[m] = (...args: unknown[]): Record<string, unknown> => {
+                ops.push({ method: m, args });
+                return chain;
+              };
+            }
+            chain.exec = async (): Promise<Array<[Error | null, unknown]>> =>
+              ops.map((op, i) =>
+                op.method === "rpush"
+                  ? [new Error("OOM command not allowed"), null]
+                  : [null, i]
+              );
+            return chain;
+          };
+        }
+        return Reflect.get(target, prop, receiver) as unknown;
+      },
+    });
+
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [JSON.stringify({ id: "m1" })],
+      state: sampleState,
+      dedupIds: ["m1"],
+    };
+    await expect(
+      applySnapshot({
+        redis: wrapped as unknown as Redis,
+        threadKey: "messages",
+        threadId: "t-fail",
+        snapshot: snap,
+      })
+    ).rejects.toThrow("OOM command not allowed");
+
+    expect(await redis.exists(getThreadMetaKey("messages", "t-fail"))).toBe(0);
+  });
+
+  it("issues a single pipeline.exec() rather than per-key writes", async () => {
+    let pipelineCalls = 0;
+    const wrapped = new Proxy(redis, {
+      get(target, prop, receiver): unknown {
+        if (prop === "pipeline") {
+          return (): unknown => {
+            pipelineCalls++;
+            return (target as unknown as { pipeline: () => unknown }).pipeline();
+          };
+        }
+        return Reflect.get(target, prop, receiver) as unknown;
+      },
+    });
+
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: Array.from({ length: 10 }, (_, i) =>
+        JSON.stringify({ id: `m${i}` })
+      ),
+      state: sampleState,
+      dedupIds: Array.from({ length: 10 }, (_, i) => `m${i}`),
+    };
+    await applySnapshot({
+      redis: wrapped as unknown as Redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      snapshot: snap,
+    });
+
+    expect(pipelineCalls).toBe(1);
+  });
+
+  it("clears any partial residue from a previous failed restore", async () => {
+    const listKey = getThreadListKey("messages", "t-1");
+    const stateKey = getThreadStateKey("messages", "t-1");
+    await redis.rpush(listKey, "stale-message");
+    await redis.set(stateKey, JSON.stringify({ stale: true }));
+    // Note: meta is intentionally absent — simulates a half-written restore.
+
+    const snap: ThreadSnapshot = {
+      v: 1,
+      messages: [JSON.stringify({ id: "fresh" })],
+      state: sampleState,
+      dedupIds: ["fresh"],
+    };
+    await applySnapshot({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      snapshot: snap,
+    });
+
+    const tm = createThreadManager<{ id: string }>({
+      redis,
+      threadId: "t-1",
+      idOf: (m) => m.id,
+    });
+    expect(await tm.load()).toEqual([{ id: "fresh" }]);
+    expect(await tm.loadState()).toEqual(sampleState);
+  });
+});
+
+describe("clearHotTier", () => {
+  it("removes list, meta, state, and dedup keys", async () => {
+    const redis = createFakeRedis();
+    const tm = createThreadManager<{ id: string }>({
+      redis,
+      threadId: "t-1",
+      idOf: (m) => m.id,
+    });
+    await tm.initialize();
+    await tm.append([{ id: "m1" }]);
+    await tm.saveState(sampleState);
+
+    await clearHotTier({
+      redis,
+      threadKey: "messages",
+      threadId: "t-1",
+      dedupIds: ["m1"],
+    });
+
+    expect(await redis.exists(getThreadListKey("messages", "t-1"))).toBe(0);
+    expect(await redis.exists(getThreadMetaKey("messages", "t-1"))).toBe(0);
+    expect(await redis.exists(getThreadStateKey("messages", "t-1"))).toBe(0);
+    expect(await redis.exists(getThreadDedupKey("t-1", "m1"))).toBe(0);
+  });
+});

package/src/lib/thread/snapshot.ts

@@ -0,0 +1,163 @@
+/**
+ * Pure Redis I/O helpers for moving a thread between the hot tier
+ * (Redis lists + meta + state + dedup markers) and the cold tier
+ * (a single {@link ThreadSnapshot} blob in a {@link ColdThreadStore}).
+ *
+ * These helpers know nothing about S3 or the adapter-specific message
+ * envelope — they operate on the raw Redis representation. The
+ * tiered thread manager in `tiered.ts` is the only consumer.
+ */
+
+import type Redis from "ioredis";
+import type { PersistedThreadState } from "../state/types";
+import type { ThreadSnapshot } from "./cold-store";
+import {
+  THREAD_TTL_SECONDS,
+  getThreadDedupKey,
+  getThreadListKey,
+  getThreadMetaKey,
+  getThreadStateKey,
+} from "./keys";
+
+/** Inputs shared by every snapshot operation. */
+interface SnapshotCommon {
+  redis: Redis;
+  threadKey: string;
+  threadId: string;
+}
+
+/** Configuration for {@link encodeSnapshot}. */
+export interface EncodeSnapshotConfig extends SnapshotCommon {
+  /**
+   * Extract a dedup id from each raw-serialized message currently in
+   * the thread's Redis list. When omitted, the resulting snapshot has
+   * an empty `dedupIds` array — idempotency guarantees are best-effort
+   * once a thread crosses the hot/cold boundary.
+   */
+  idOf?: (raw: string) => string;
+}
+
+/**
+ * Build a {@link ThreadSnapshot} from the current hot-tier state.
+ *
+ * Returns `null` when no thread exists in the hot tier (the meta key
+ * is absent) — callers should treat that as "nothing to flush".
+ */
+export async function encodeSnapshot(
+  config: EncodeSnapshotConfig
+): Promise<ThreadSnapshot | null> {
+  const { redis, threadKey, threadId, idOf } = config;
+  const metaKey = getThreadMetaKey(threadKey, threadId);
+  if ((await redis.exists(metaKey)) === 0) {
+    return null;
+  }
+  const listKey = getThreadListKey(threadKey, threadId);
+  const stateKey = getThreadStateKey(threadKey, threadId);
+  const messages = await redis.lrange(listKey, 0, -1);
+  const stateRaw = await redis.get(stateKey);
+  const state =
+    stateRaw == null ? null : (JSON.parse(stateRaw) as PersistedThreadState);
+  const dedupIds = idOf ? messages.map(idOf) : [];
+  return { v: 1, messages, state, dedupIds };
+}
+
+/** Configuration for {@link applySnapshot}. */
+export interface ApplySnapshotConfig extends SnapshotCommon {
+  snapshot: ThreadSnapshot;
+  /** TTL applied to every Redis key. Defaults to {@link THREAD_TTL_SECONDS}. */
+  ttlSeconds?: number;
+}
+
+/**
+ * Restore a {@link ThreadSnapshot} into the hot tier.
+ *
+ * Idempotent — if the meta key already exists the thread is already
+ * hot and this is a no-op. The meta key is written **last** so a
+ * crash mid-restore leaves the thread cold (`load` / `append` will
+ * see "thread does not exist") and the next session's hydrate retries
+ * cleanly.
+ */
+export async function applySnapshot(
+  config: ApplySnapshotConfig
+): Promise<void> {
+  const {
+    redis,
+    threadKey,
+    threadId,
+    snapshot,
+    ttlSeconds = THREAD_TTL_SECONDS,
+  } = config;
+  const metaKey = getThreadMetaKey(threadKey, threadId);
+  if ((await redis.exists(metaKey)) === 1) {
+    return;
+  }
+  const listKey = getThreadListKey(threadKey, threadId);
+  const stateKey = getThreadStateKey(threadKey, threadId);
+
+  // Clear partial residue from any prior half-restored attempt.
+  // Awaited *outside* the pipeline so a DEL failure (ACL deny,
+  // CROSSSLOT, …) short-circuits before any writes hit the wire —
+  // pipelines are non-atomic, so a queued DEL wouldn't stop later
+  // commands from accumulating data behind a missing meta marker.
+  await redis.del(listKey, stateKey);
+
+  // Pipeline the data writes (list/state/dedup) in one round-trip.
+  // Meta is written separately, only after every queued command
+  // succeeded, preserving the "meta-last" crash-safety invariant —
+  // a partial restore must leave meta absent so the next hydrate
+  // retries cleanly.
+  const pipeline = redis.pipeline();
+  if (snapshot.messages.length > 0) {
+    pipeline.rpush(listKey, ...snapshot.messages);
+    pipeline.expire(listKey, ttlSeconds);
+  }
+  if (snapshot.state != null) {
+    pipeline.set(stateKey, JSON.stringify(snapshot.state), "EX", ttlSeconds);
+  }
+  for (const id of snapshot.dedupIds) {
+    pipeline.set(getThreadDedupKey(threadId, id), "1", "EX", ttlSeconds);
+  }
+  const results = await pipeline.exec();
+  if (results) {
+    const firstErr = results.find(([err]) => err)?.[0] ?? null;
+    if (firstErr) {
+      // Compensate: pipelines are non-atomic, so writes queued after
+      // a failing command (notably dedup SETs) may have landed. Best-
+      // effort clear every key we touched so a leftover dedup marker
+      // can't silently skip a future append with the same id.
+      await redis
+        .del(
+          listKey,
+          stateKey,
+          ...snapshot.dedupIds.map((id) => getThreadDedupKey(threadId, id))
+        )
+        .catch(() => undefined);
+      throw firstErr;
+    }
+  }
+  await redis.set(metaKey, "1", "EX", ttlSeconds);
+}
+
+/** Configuration for {@link clearHotTier}. */
+export interface ClearHotTierConfig extends SnapshotCommon {
+  /** Dedup ids to delete alongside the list / meta / state keys. */
+  dedupIds?: string[];
+}
+
+/**
+ * Delete every Redis key the thread manager wrote for `(threadKey,
+ * threadId)`. Used by the tiered manager's `flush({ deleteHot: true })`
+ * to drop hot-tier memory after a successful archive write.
+ */
+export async function clearHotTier(
+  config: ClearHotTierConfig
+): Promise<void> {
+  const { redis, threadKey, threadId, dedupIds = [] } = config;
+  const keys = [
+    getThreadListKey(threadKey, threadId),
+    getThreadMetaKey(threadKey, threadId),
+    getThreadStateKey(threadKey, threadId),
+    ...dedupIds.map((id) => getThreadDedupKey(threadId, id)),
+  ];
+  await redis.del(...keys);
+}