wadis 1.0.0

package/package.json

@@ -0,0 +1,47 @@
+{
+   "name": "wadis",
+   "version": "1.0.0",
+   "description": "Wadis is a WebAssembly (Wasm) build of the Redis® in-memory datastore, purpose-built for automated testing 🧪",
+   "repository": "https://github.com/cah4a/wadis",
+   "type": "module",
+   "license": "Redis Source Available License v2 (RSALv2)",
+   "author": "Sancha",
+   "main": "dist/index.cjs",
+   "module": "dist/index.js",
+   "types": "dist/index.d.ts",
+   "engines": {
+      "node": ">=20.0.0",
+      "bun": ">=1.1.0"
+   },
+   "files": [
+      "dist",
+      "src",
+      "README.md",
+      "LICENSE"
+   ],
+   "packageManager": "pnpm@10.19.0",
+   "scripts": {
+      "build": "rm -rf dist && tsup",
+      "build:wasm": "./compile && pnpm run build",
+      "test": "vitest run",
+      "lint": "eslint .",
+      "prepare": "pnpm run build",
+      "patch:apply": "patch -p1 --forward -d vendor/redis < patches/redis-8.2.1.patch",
+      "patch:create": "cd vendor/redis && git diff > ../../patches/redis-8.2.1.patch && cd -",
+      "fmt": "biome format . --write",
+      "fix": "biome check . --write"
+   },
+   "peerDependencies": {
+      "ioredis": "^5.4.1"
+   },
+   "devDependencies": {
+      "@biomejs/biome": "^2.2.4",
+      "@types/node": "^22.7.4",
+      "bullmq": "^5.61.2",
+      "esbuild-plugin-copy": "^2.1.1",
+      "tsup": "^8.5.0",
+      "typescript": "^5.6.3",
+      "vite-tsconfig-paths": "^5.1.4",
+      "vitest": "^2.1.3"
+   }
+}

package/src/adapters/ioredis.ts

@@ -0,0 +1,111 @@
+import type { WadisServer } from "wadisServer";
+import { Socket } from "node:net";
+
+export class WadisServerConnector extends Socket {
+   private conn: {
+      write: (data: Uint8Array | Buffer) => void;
+      read: () => Uint8Array | null;
+      wantsClose: () => boolean;
+      close: () => void;
+   };
+   private poll?: NodeJS.Timeout;
+   private _closed = false;
+   private queue: Promise<void> = Promise.resolve();
+
+   constructor(server: WadisServer) {
+      super();
+      this.conn = server.createConnection();
+      this.startPolling();
+      setImmediate(() => this.emit("connect"));
+   }
+
+   override setTimeout(_msecs: number, _callback?: () => void): this {
+      return this;
+   }
+   override setNoDelay(_noDelay?: boolean): this {
+      return this;
+   }
+   override setKeepAlive(_enable?: boolean, _initialDelay?: number): this {
+      return this;
+   }
+
+   override write(
+      chunk: Uint8Array | string,
+      encoding?: BufferEncoding | ((err?: Error) => void),
+      cb?: (err?: Error) => void,
+   ): boolean {
+      let callback: ((err?: Error) => void) | undefined;
+      let enc: BufferEncoding | undefined;
+      if (typeof encoding === "function") {
+         callback = encoding;
+      } else {
+         enc = encoding as BufferEncoding | undefined;
+         callback = cb;
+      }
+      const buf =
+         typeof chunk === "string"
+            ? Buffer.from(chunk, enc ?? "utf8")
+            : Buffer.from(chunk);
+      // Ensure replies are emitted asynchronously after write returns,
+      // and preserve write ordering with a micro-queue.
+      this.queue = this.queue.then(
+         () =>
+            new Promise<void>((resolve) => {
+               setImmediate(() => {
+                  try {
+                     this.conn.write(buf);
+                  } catch (err) {
+                     this.emit("error", err as Error);
+                  } finally {
+                     resolve();
+                  }
+               });
+            }),
+      );
+      if (callback) setImmediate(() => callback());
+      return true;
+   }
+
+   override end(..._args: unknown[]): this {
+      if (this._closed) return this;
+      this._closed = true;
+      if (this.poll) {
+         clearInterval(this.poll);
+         this.poll = undefined;
+      }
+      this.conn.close();
+      setImmediate(() => this.emit("end"));
+      setImmediate(() => this.emit("close", false));
+      return this;
+   }
+   override destroy(_error?: Error): this {
+      if (this._closed) return this;
+      this._closed = true;
+      if (this.poll) {
+         clearInterval(this.poll);
+         this.poll = undefined;
+      }
+      this.conn.close();
+      setImmediate(() => this.emit("close", true));
+      return this;
+   }
+   private startPolling() {
+      this.poll = setInterval(() => this.drain(), 10);
+      // Do not keep the process alive solely due to this timer
+      // (important for tests that don't explicitly quit the base connection)
+      this.poll.unref?.();
+   }
+
+   private drain() {
+      while (true) {
+         const out = this.conn.read();
+         if (!out || out.length === 0) break;
+         this.emit("data", Buffer.from(out));
+      }
+      // After draining all pending output, if Redis marked this client
+      // to close after reply (QUIT), close the socket gracefully.
+      if (!this._closed && this.conn.wantsClose()) {
+         this.end();
+      }
+   }
+}

package/src/bullmq.test.ts

@@ -0,0 +1,62 @@
+import { type Job, Queue, QueueEvents, Worker } from "bullmq";
+import { Wadis, WadisServer } from "index";
+import { describe, expect, test } from "vitest";
+
+describe("bullmq", () => {
+   test("adding jobs", async () => {
+      const redis = new Wadis();
+      const myQueue = new Queue("my-queue", { connection: redis });
+      const job = await myQueue.add("my-job", { name: "Test Job" });
+
+      expect(await job.getState()).toEqual("waiting");
+      await myQueue.close();
+      await redis.quit();
+   });
+
+   test(
+      "processing jobs",
+      async () => {
+         const redis = new Wadis();
+         const myQueue = new Queue("my-queue", { connection: redis });
+         const job = await myQueue.add("my-job", { name: "Test Job" });
+         const { promise, resolve } = Promise.withResolvers<void>();
+
+         new Worker(
+            "my-queue",
+            async (job: Job) => {
+               setTimeout(resolve, 10);
+               return `Processed ${job.data.name}`;
+            },
+            { connection: redis },
+         );
+
+         await promise;
+         expect(await job.getState()).toBe("completed");
+         expect((await myQueue.getJob(job.id || ""))?.returnvalue).toBe(
+            "Processed Test Job",
+         );
+      },
+      { timeout: 1000 },
+   );
+
+   test("queue events", async () => {
+      const server = await WadisServer.new();
+      const redis = new Wadis({
+         server,
+         maxRetriesPerRequest: null,
+      });
+
+      const myQueue = new Queue("my-queue", { connection: redis });
+      const myQueueEvents = new QueueEvents("my-queue", { connection: redis });
+      await myQueueEvents.waitUntilReady();
+
+      const job = await myQueue.add("my-job", { name: "Test Job" });
+
+      new Worker("my-queue", async (job: Job) => `Processed ${job.data.name}`, {
+         connection: redis,
+      });
+
+      await job.waitUntilFinished(myQueueEvents, 1000);
+      expect(await job.getState()).toBe("completed");
+   });
+});

package/src/index.test.ts

@@ -0,0 +1,98 @@
+import { Wadis, WadisServer } from "index";
+import { describe, expect, test } from "vitest";
+
+describe("ioredis adapter", () => {
+   test("works", async () => {
+      const redis = new Wadis();
+
+      await redis.set("foo", "bar");
+      const duplicate = redis.duplicate();
+      expect(await duplicate.get("foo")).toBe("bar");
+      await duplicate.set("foo", "baz");
+      expect(await redis.get("foo")).toBe("baz");
+      await duplicate.quit();
+
+      expect(await redis.get("foo")).toBe("baz");
+   });
+
+   test("pub/sub works", async () => {
+      const redis = new Wadis();
+
+      const subscriber = redis.duplicate();
+      await subscriber.subscribe("my-channel");
+
+      const messages: string[] = [];
+      subscriber.on("message", (_channel, message) => {
+         messages.push(message);
+      });
+      const publisher = redis.duplicate();
+      await publisher.publish("my-channel", "hello");
+      await publisher.publish("my-channel", "world");
+
+      // wait a tick for messages to be received
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      expect(messages).toEqual(["hello", "world"]);
+
+      await subscriber.quit();
+      await publisher.quit();
+   });
+
+   test("lua scripting works", async () => {
+      const redis = new Wadis();
+
+      const script = `return ARGV[1] .. ARGV[2]`;
+      const result = await redis.eval(script, 0, "Hello, ", "world!");
+
+      expect(result).toBe("Hello, world!");
+   });
+
+   test("cjson.encode works", async () => {
+      const r = new Wadis();
+      const res = await r.eval("return cjson.encode({a=1,b='x'})", 0);
+      expect(res).toBe('{"a":1,"b":"x"}');
+   });
+
+   test("cjson.decode works", async () => {
+      const r = new Wadis();
+      const res = await r.eval(
+         "local t=cjson.decode(ARGV[1]); return t.a+t.b",
+         0,
+         '{"a":2,"b":3}',
+      );
+      expect(res).toBe(5);
+   });
+
+   test("streams blocking unblocks", async () => {
+      const server = await WadisServer.new();
+      const c1 = new Wadis({ server });
+      const c2 = new Wadis({ server });
+
+      await c1.xgroup("CREATE", "s2", "g2", "0", "MKSTREAM");
+
+      const p = c1.xreadgroup(
+         "GROUP",
+         "g2",
+         "w1",
+         "BLOCK" as never,
+         2000,
+         "COUNT",
+         1,
+         "STREAMS",
+         "s2",
+         ">",
+      );
+
+      // wait a tick then add
+      setTimeout(async () => {
+         await c2.xadd("s2", "*", "f", "v");
+      }, 100);
+
+      const res = await p;
+      expect(Array.isArray(res)).toBe(true);
+
+      await c1.quit();
+      await c2.quit();
+      await server.terminate();
+   });
+});

package/src/index.ts

@@ -0,0 +1,32 @@
+import { WadisServer } from "wadisServer";
+import { WadisServerConnector } from "adapters/ioredis";
+import Redis, { AbstractConnector, type RedisOptions } from "ioredis";
+import type { ErrorEmitter } from "ioredis/built/connectors/AbstractConnector";
+
+export * from "wadisServer";
+
+export type WadisOptions = Omit<
+   RedisOptions,
+   "Connector" | "host" | "port" | "username" | "password" | "db"
+> & {
+   server?: WadisServer | Promise<WadisServer>;
+};
+
+export class Wadis extends Redis {
+   constructor({ server, ...options }: WadisOptions = {}) {
+      const wasmServer = server ?? WadisServer.new();
+
+      class Connector extends AbstractConnector {
+         constructor(something: unknown) {
+            // shitty ioredis types here
+            super(something as never);
+         }
+
+         async connect(_: ErrorEmitter) {
+            return new WadisServerConnector(await wasmServer);
+         }
+      }
+
+      super({ Connector, maxRetriesPerRequest: null, ...options });
+   }
+}

package/src/wadisServer.test.ts

@@ -0,0 +1,65 @@
+import { WadisServer } from "wadisServer";
+import { TextDecoder } from "node:util";
+import { beforeEach, describe, expect, test } from "vitest";
+
+describe("wadis server", () => {
+   let server: WadisServer;
+
+   beforeEach(async () => {
+      server = await WadisServer.new();
+   });
+
+   describe("basic commands", () => {
+      test("PING works", async () => {
+         const resp = await server.call("PING");
+         expect(new TextDecoder().decode(resp)).toBe("+PONG\r\n");
+      });
+
+      test("SET/GET works", async () => {
+         const server = await WadisServer.new({ maxmemory: "50mb" });
+         const ok = await server.call("SET", "foo", "bar");
+         expect(new TextDecoder().decode(ok)).toBe("+OK\r\n");
+
+         const val = await server.call("GET", "foo");
+         expect(new TextDecoder().decode(val)).toBe("$3\r\nbar\r\n");
+      });
+
+      test("INCR increments integer keys", async () => {
+         const v1 = await server.call("INCR", "counter");
+         expect(new TextDecoder().decode(v1)).toBe(":1\r\n");
+         const v2 = await server.call("INCR", "counter");
+         expect(new TextDecoder().decode(v2)).toBe(":2\r\n");
+      });
+
+      test("returns error replies for wrong types", async () => {
+         await server.call("SET", "notint", "x");
+         const err = await server.call("INCR", "notint");
+         const s = new TextDecoder().decode(err);
+         expect(s.startsWith("-ERR")).toBe(true);
+      });
+   });
+
+   describe("lua scripts", () => {
+      test("EVAL returns array of values", async () => {
+         const script = "return {KEYS[1], ARGV[1]}";
+         const resp = await server.call("EVAL", script, "1", "key1", "val1");
+         const s = new TextDecoder().decode(resp);
+         expect(s.startsWith("*2\r\n")).toBe(true);
+         expect(s.includes("$4\r\nkey1\r\n")).toBe(true);
+         expect(s.includes("$4\r\nval1\r\n")).toBe(true);
+      });
+
+      test("EVAL can SET/GET", async () => {
+         const script =
+            "redis.call('SET', KEYS[1], ARGV[1]); return redis.call('GET', KEYS[1])";
+         const resp = await server.call("EVAL", script, "1", "foo", "bar");
+         expect(new TextDecoder().decode(resp)).toBe("$3\r\nbar\r\n");
+      });
+
+      test("EVAL returns integer", async () => {
+         const script = "return 7";
+         const resp = await server.call("EVAL", script, "0");
+         expect(new TextDecoder().decode(resp)).toBe(":7\r\n");
+      });
+   });
+});

package/src/wadisServer.ts

@@ -0,0 +1,222 @@
+// @ts-expect-error
+import redis from "wasm/redis.js";
+
+export type WadisOptions = {
+   // Configurable Redis memory cap. Accepts a number of bytes or a Redis-size string like '128mb'.
+   maxmemory?: number | string;
+   // Eviction policy, matches Redis config: 'noeviction', 'allkeys-lru', etc.
+   maxmemoryPolicy?:
+      | "noeviction"
+      | "allkeys-lru"
+      | "allkeys-lfu"
+      | "allkeys-random"
+      | "volatile-lru"
+      | "volatile-lfu"
+      | "volatile-random"
+      | "volatile-ttl"
+      | string;
+   // Redis loglevel: 'debug' | 'verbose' | 'notice' | 'warning'
+   loglevel?: "debug" | "verbose" | "notice" | "warning" | string;
+};
+
+type EmscriptenModule = {
+   cwrap: (
+      ident: string,
+      returnType: string,
+      argTypes: string[],
+   ) => (...args: unknown[]) => unknown;
+   _malloc: (n: number) => number;
+   _free: (ptr: number) => void;
+   writeArrayToMemory?: (arr: Uint8Array, ptr: number) => void;
+   setValue?: (ptr: number, value: number, type: string) => void;
+   getValue: (ptr: number, type: string) => number;
+};
+
+export class WadisServer {
+   private Module: EmscriptenModule | null = null;
+   private _clientHandle: number | null = null;
+
+   private constructor(private _opts: WadisOptions = {}) {}
+
+   static async new(
+      opts: WadisOptions = { loglevel: "warning" },
+   ): Promise<WadisServer> {
+      const server = new WadisServer(opts);
+      await server.start();
+      return server;
+   }
+
+   async start(): Promise<void> {
+      this.Module = (await redis({ noInitialRun: true })) as EmscriptenModule;
+
+      const init = this.Module.cwrap("redis_init", "number", ["number"]) as (
+         level: number,
+      ) => number;
+      const rc = init(this.loglevelToInt(this._opts.loglevel ?? "warning"));
+      if (rc !== 0) throw new Error(`redis_init failed: ${rc}`);
+
+      // Create a persistent client used by call(). This unifies command execution
+      // with the same mechanism the adapter uses, enabling push semantics when needed.
+      this._clientHandle = this._createHandle();
+
+      // Apply runtime configuration overrides if provided.
+      if (this._opts.maxmemory !== undefined) {
+         const val =
+            typeof this._opts.maxmemory === "number"
+               ? String(this._opts.maxmemory)
+               : this._opts.maxmemory;
+         await this.call("CONFIG", "SET", "maxmemory", val);
+      }
+      if (this._opts.maxmemoryPolicy) {
+         await this.call(
+            "CONFIG",
+            "SET",
+            "maxmemory-policy",
+            this._opts.maxmemoryPolicy,
+         );
+      }
+      if (this._opts.loglevel) {
+         await this.call("CONFIG", "SET", "loglevel", this._opts.loglevel);
+      }
+   }
+
+   // Minimal RESP encoder for array of bulk strings: *N\r\n$len\r\nfoo\r\n...
+   private encodeCommand(parts: (string | Buffer)[]): Uint8Array {
+      const chunks: Buffer[] = [];
+      chunks.push(Buffer.from(`*${parts.length}\r\n`));
+      for (const p of parts) {
+         const b = Buffer.isBuffer(p) ? p : Buffer.from(p);
+         chunks.push(Buffer.from(`$${b.length}\r\n`));
+         chunks.push(b);
+         chunks.push(Buffer.from(`\r\n`));
+      }
+      return Buffer.concat(chunks);
+   }
+
+   async call(...parts: (string | Buffer)[]): Promise<Uint8Array> {
+      if (!this.Module) throw new Error("WASM not started");
+      if (!this._clientHandle) this._clientHandle = this._createHandle();
+      const input = this.encodeCommand(parts);
+      this._clientFeed(this._clientHandle, input);
+      const out = this._clientRead(this._clientHandle);
+      return out ?? new Uint8Array(0);
+   }
+
+   private _createHandle(): number {
+      if (!this.Module) throw new Error("WASM not started");
+      const mod = this.Module as EmscriptenModule;
+      const fn = mod.cwrap("redis_create_handle", "number", []) as () => number;
+      const h = fn();
+      if (!h) throw new Error("redis_create_handle failed");
+      return h;
+   }
+
+   private _clientFeed(handle: number, data: Uint8Array | Buffer): void {
+      if (!this.Module) throw new Error("WASM not started");
+      const mod = this.Module as EmscriptenModule;
+      const feed = mod.cwrap("redis_client_feed", "number", [
+         "number",
+         "number",
+         "number",
+      ]) as (h: number, ptr: number, len: number) => number;
+      const malloc = mod._malloc as (n: number) => number;
+      const free = mod._free as (ptr: number) => void;
+      const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+      const inPtr = malloc(input.length);
+      if (typeof mod.writeArrayToMemory === "function") {
+         mod.writeArrayToMemory(input, inPtr);
+      } else if (typeof mod.setValue === "function") {
+         for (let i = 0; i < input.length; i++)
+            mod.setValue(inPtr + i, input[i], "i8");
+      } else {
+         throw new Error(
+            "Emscripten runtime missing writeArrayToMemory/setValue",
+         );
+      }
+      const rc = feed(handle, inPtr, input.length);
+      free(inPtr);
+      if (rc !== 0) throw new Error(`redis_client_feed failed: ${rc}`);
+   }
+
+   private _clientRead(handle: number): Uint8Array | null {
+      if (!this.Module) throw new Error("WASM not started");
+      const mod = this.Module as EmscriptenModule;
+      const malloc = mod._malloc as (n: number) => number;
+      const free = mod._free as (ptr: number) => void;
+      const read = mod.cwrap("redis_client_read", "number", [
+         "number",
+         "number",
+         "number",
+      ]) as (h: number, outPtrPtr: number, outLenPtr: number) => number;
+      const freeOut = mod.cwrap("redis_free", "void", ["number", "number"]) as (
+         ptr: number,
+         len: number,
+      ) => void;
+
+      const outPtrPtr = malloc(4);
+      const outLenPtr = malloc(4);
+      const rc = read(handle, outPtrPtr, outLenPtr);
+      if (rc !== 0) {
+         free(outPtrPtr);
+         free(outLenPtr);
+         throw new Error(`redis_client_read failed: ${rc}`);
+      }
+      const outPtr = mod.getValue(outPtrPtr, "i32") as number;
+      const outLen = mod.getValue(outLenPtr, "i32") as number;
+      free(outPtrPtr);
+      free(outLenPtr);
+      if (!outPtr || outLen === 0) return null;
+
+      const out = new Uint8Array(outLen);
+      for (let i = 0; i < outLen; i++)
+         out[i] = mod.getValue(outPtr + i, "i8") as number;
+      freeOut(outPtr, outLen);
+      return out;
+   }
+
+   private _clientFree(handle: number): void {
+      if (!this.Module) return;
+      const mod = this.Module as EmscriptenModule;
+      const fn = mod.cwrap("redis_client_free", "void", ["number"]) as (
+         h: number,
+      ) => void;
+      fn(handle);
+   }
+
+   private _clientWantsClose(handle: number): boolean {
+      if (!this.Module) return false;
+      const mod = this.Module as EmscriptenModule;
+      const fn = mod.cwrap("redis_client_wants_close", "number", [
+         "number",
+      ]) as (h: number) => number;
+      return fn(handle) !== 0;
+   }
+
+   createConnection() {
+      if (!this.Module) throw new Error("WASM not started");
+      const handle = this._createHandle();
+      return {
+         write: (data: Uint8Array | Buffer) => this._clientFeed(handle, data),
+         read: () => this._clientRead(handle),
+         wantsClose: () => this._clientWantsClose(handle),
+         close: () => this._clientFree(handle),
+      } as const;
+   }
+
+   async terminate() {
+      if (this._clientHandle !== null) {
+         this._clientFree(this._clientHandle);
+      }
+
+      this.Module = null;
+      this._clientHandle = null;
+   }
+
+   private loglevelToInt(level: string): number {
+      const s = level.toLowerCase();
+      if (s === "debug") return 0;
+      if (s === "verbose") return 1;
+      if (s === "notice") return 2;
+      return 3;
+   }
+}